add metrics to track the managed resource count

Author: oliviassss

Diff for: main.go

@@ -17,26 +17,25 @@ limitations under the License.
 package main
 
 import (
-	"k8s.io/client-go/util/workqueue"
-	"os"
-
-	elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
-
+	"context"
 	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
 	zapraw "go.uber.org/zap"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
+	"os"
 	elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
 	elbv2controller "sigs.k8s.io/aws-load-balancer-controller/controllers/elbv2"
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/ingress"
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/service"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
+	elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/inject"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	awsmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws"
@@ -52,6 +51,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	"time"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -84,8 +84,6 @@ func main() {
 	klog.SetLoggerWithOptions(appLogger, klog.ContextualLogger(true))
 
 	var awsMetricsCollector *awsmetrics.Collector
-	lbcMetricsCollector := lbcmetrics.NewCollector(metrics.Registry)
-
 	if metrics.Registry != nil {
 		awsMetricsCollector = awsmetrics.NewCollector(metrics.Registry)
 	}
@@ -107,6 +105,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	// for open-source LBC, we track the resources with finalizers contains "k8s.aws"
+	lbcMetricsCollector := lbcmetrics.NewCollector(metrics.Registry, mgr.GetClient(), "k8s.aws")
+
 	clientSet, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to obtain clientSet")
@@ -163,6 +164,23 @@ func main() {
 		os.Exit(1)
 	}
 
+	// update the managed resource every 30s
+	go func() {
+		ticker := time.NewTicker(30 * time.Second) // Call every 30 seconds
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				// Update metrics
+				err := lbcMetricsCollector.UpdateManagedK8sResourceMetrics(context.Background())
+				if err != nil {
+					setupLog.Error(err, "failed to update managed Kubernetes resource metrics")
+				}
+			}
+		}
+	}()
+
 	// Add liveness probe
 	err = mgr.AddHealthzCheck("health-ping", healthz.Ping)
 	setupLog.Info("adding health check for controller")
@@ -210,6 +228,7 @@ func main() {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
+
 }
 
 // loadControllerConfig loads the controller configuration.

Diff for: pkg/metrics/aws/instruments.go

@@ -59,8 +59,13 @@ func newInstruments(registerer prometheus.Registerer) *instruments {
 		Name:      metricAPIRequestDurationSeconds,
 		Help:      "Latency of an individual HTTP request to the service endpoint",
 	}, []string{labelService, labelOperation})
-
-	registerer.MustRegister(apiCallsTotal, apiCallDurationSeconds, apiCallRetries, apiRequestsTotal, apiRequestDurationSecond)
+	registerer.MustRegister(
+		apiCallsTotal,
+		apiCallDurationSeconds,
+		apiCallRetries,
+		apiRequestsTotal,
+		apiRequestDurationSecond,
+	)
 	return &instruments{
 		apiCallsTotal:            apiCallsTotal,
 		apiCallDurationSeconds:   apiCallDurationSeconds,

Diff for: pkg/metrics/lbc/collector.go

@@ -1,33 +1,53 @@
 package lbc
 
 import (
+	"context"
 	"github.com/prometheus/client_golang/prometheus"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
+	"strings"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"time"
 )
 
+const networkLoadBalancerStr = "nlb"
+
 type MetricCollector interface {
 	// ObservePodReadinessGateReady this metric is useful to determine how fast pods are becoming ready in the load balancer.
 	// Due to some architectural constraints, we can only emit this metric for pods that are using readiness gates.
 	ObservePodReadinessGateReady(namespace string, tgbName string, duration time.Duration)
+
+	// UpdateMetrics fetches and updates all relevant metrics.
+	UpdateManagedK8sResourceMetrics(ctx context.Context) error
 }
 
 type collector struct {
-	instruments *instruments
+	instruments      *instruments
+	runtimeClient    client.Client
+	finalizerKeyWord string
 }
 
 type noOpCollector struct{}
 
 func (n *noOpCollector) ObservePodReadinessGateReady(_ string, _ string, _ time.Duration) {
 }
 
-func NewCollector(registerer prometheus.Registerer) MetricCollector {
-	if registerer == nil {
+func (n *noOpCollector) UpdateManagedK8sResourceMetrics(_ context.Context) error {
+	return nil
+}
+
+func NewCollector(registerer prometheus.Registerer, runtimeClient client.Client, finalizerKeyWord string) MetricCollector {
+	if registerer == nil || runtimeClient == nil {
 		return &noOpCollector{}
 	}
 
 	instruments := newInstruments(registerer)
 	return &collector{
-		instruments: instruments,
+		instruments:      instruments,
+		runtimeClient:    runtimeClient,
+		finalizerKeyWord: finalizerKeyWord,
 	}
 }
 
@@ -37,3 +57,64 @@ func (c *collector) ObservePodReadinessGateReady(namespace string, tgbName strin
 		labelName:      tgbName,
 	}).Observe(duration.Seconds())
 }
+
+func (c *collector) UpdateManagedK8sResourceMetrics(ctx context.Context) error {
+	listOpts := &client.ListOptions{
+		Namespace: "",
+	}
+	ingressCount, serviceCount, tgbCount := 0, 0, 0
+	// Fetch ingress count
+	ingressList := &networkingv1.IngressList{}
+	err := c.runtimeClient.List(ctx, ingressList, listOpts)
+	if err != nil {
+		return err
+	}
+	for _, ingress := range ingressList.Items {
+		for _, finalizer := range ingress.Finalizers {
+			if strings.Contains(finalizer, c.finalizerKeyWord) {
+				ingressCount++
+				break
+			}
+		}
+	}
+	c.instruments.ingressCount.Set(float64(ingressCount))
+
+	// Fetch service count
+	serviceList := &corev1.ServiceList{}
+	err = c.runtimeClient.List(ctx, serviceList, listOpts)
+	if err != nil {
+		return err
+	}
+	for _, service := range serviceList.Items {
+		hasMatchingFinalizer := false
+		for _, finalizer := range service.Finalizers {
+			if strings.Contains(finalizer, c.finalizerKeyWord) {
+				hasMatchingFinalizer = true
+				break
+			}
+		}
+
+		if hasMatchingFinalizer && service.Spec.LoadBalancerClass != nil && strings.Contains(*service.Spec.LoadBalancerClass, networkLoadBalancerStr) {
+			serviceCount++
+		}
+	}
+	c.instruments.serviceCount.Set(float64(serviceCount))
+
+	// Fetch TargetGroupBinding count
+	tgbList := &elbv2api.TargetGroupBindingList{}
+	err = c.runtimeClient.List(ctx, tgbList, listOpts)
+	if err != nil {
+		return err
+	}
+	for _, tgb := range tgbList.Items {
+		for _, finalizer := range tgb.Finalizers {
+			if strings.Contains(finalizer, c.finalizerKeyWord) {
+				tgbCount++
+				break
+			}
+		}
+	}
+	c.instruments.tgbCount.Set(float64(tgbCount))
+
+	return nil
+}

Diff for: pkg/metrics/lbc/instruments.go

@@ -21,6 +21,9 @@ const (
 
 type instruments struct {
 	podReadinessFlipSeconds *prometheus.HistogramVec
+	ingressCount            prometheus.Gauge
+	serviceCount            prometheus.Gauge
+	tgbCount                prometheus.Gauge
 }
 
 // newInstruments allocates and register new metrics to registerer
@@ -31,9 +34,28 @@ func newInstruments(registerer prometheus.Registerer) *instruments {
 		Help:      "Latency from pod getting added to the load balancer until the readiness gate is flipped to healthy.",
 		Buckets:   []float64{10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600},
 	}, []string{labelNamespace, labelName})
-
-	registerer.MustRegister(podReadinessFlipSeconds)
+	ingressCount := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "lb_controller_managed_ingress_count",
+		Help: "Number of ingresses managed by the AWS Load Balancer Controller.",
+	})
+	serviceCount := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "lb_controller_managed_service_count",
+		Help: "Number of service type Load Balancers (NLBs) managed by the AWS Load Balancer Controller.",
+	})
+	tgbCount := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "lb_controller_managed_targetgroupbinding_count",
+		Help: "Number of targetgroupbindings managed by the AWS Load Balancer Controller.",
+	})
+	registerer.MustRegister(
+		podReadinessFlipSeconds,
+		ingressCount,
+		serviceCount,
+		tgbCount,
+	)
 	return &instruments{
 		podReadinessFlipSeconds: podReadinessFlipSeconds,
+		ingressCount:            ingressCount,
+		serviceCount:            serviceCount,
+		tgbCount:                tgbCount,
 	}
 }

Diff for: pkg/metrics/lbc/mockcollector.go

@@ -1,6 +1,7 @@
 package lbc
 
 import (
+	"context"
 	"time"
 )
 
@@ -14,22 +15,41 @@ type MockHistogramMetric struct {
 	duration  time.Duration
 }
 
+// ObservePodReadinessGateReady mocks observing the readiness gate latency.
 func (m *MockCollector) ObservePodReadinessGateReady(namespace string, tgbName string, d time.Duration) {
 	m.recordHistogram(MetricPodReadinessGateReady, namespace, tgbName, d)
 }
 
+// UpdateManagedK8sResourceMetrics mocks updating managed Kubernetes resource metrics.
+func (m *MockCollector) UpdateManagedK8sResourceMetrics(ctx context.Context) error {
+	m.recordInvocation("UpdateManagedK8sResourceMetrics", ctx)
+	return nil // No-op for the mock
+}
+
+// recordHistogram adds a histogram metric invocation.
 func (m *MockCollector) recordHistogram(metricName string, namespace string, name string, d time.Duration) {
-	m.Invocations[metricName] = append(m.Invocations[MetricPodReadinessGateReady], MockHistogramMetric{
+	if _, exists := m.Invocations[metricName]; !exists {
+		m.Invocations[metricName] = []interface{}{}
+	}
+	m.Invocations[metricName] = append(m.Invocations[metricName], MockHistogramMetric{
 		namespace: namespace,
 		name:      name,
 		duration:  d,
 	})
 }
 
-func NewMockCollector() MetricCollector {
+// recordInvocation tracks a method invocation with arguments.
+func (m *MockCollector) recordInvocation(methodName string, args ...interface{}) {
+	if _, exists := m.Invocations[methodName]; !exists {
+		m.Invocations[methodName] = []interface{}{}
+	}
+	m.Invocations[methodName] = append(m.Invocations[methodName], args)
+}
 
+// NewMockCollector creates and returns a new MockCollector.
+func NewMockCollector() MetricCollector {
 	mockInvocations := make(map[string][]interface{})
-	mockInvocations[MetricPodReadinessGateReady] = make([]interface{}, 0)
+	mockInvocations[MetricPodReadinessGateReady] = []interface{}{}
 
 	return &MockCollector{
 		Invocations: mockInvocations,