feat: adds various OCI metrics

reposerver/metrics/metrics.go

@@ -11,15 +11,22 @@ import (
 )
 
 type MetricsServer struct {
-	handler                  http.Handler
-	gitFetchFailCounter      *prometheus.CounterVec
-	gitLsRemoteFailCounter   *prometheus.CounterVec
-	gitRequestCounter        *prometheus.CounterVec
-	gitRequestHistogram      *prometheus.HistogramVec
-	repoPendingRequestsGauge *prometheus.GaugeVec
-	redisRequestCounter      *prometheus.CounterVec
-	redisRequestHistogram    *prometheus.HistogramVec
-	PrometheusRegistry       *prometheus.Registry
+	handler                       http.Handler
+	gitFetchFailCounter           *prometheus.CounterVec
+	gitLsRemoteFailCounter        *prometheus.CounterVec
+	gitRequestCounter             *prometheus.CounterVec
+	gitRequestHistogram           *prometheus.HistogramVec
+	repoPendingRequestsGauge      *prometheus.GaugeVec
+	redisRequestCounter           *prometheus.CounterVec
+	redisRequestHistogram         *prometheus.HistogramVec
+	ociExtractFailCounter         *prometheus.CounterVec
+	ociResolveRevisionFailCounter *prometheus.CounterVec
+	ociDigestMetadataCounter      *prometheus.CounterVec
+	ociGetTagsFailCounter         *prometheus.CounterVec
+	ociTestRepoFailCounter        *prometheus.CounterVec
+	ociRequestCounter             *prometheus.CounterVec
+	ociRequestHistogram           *prometheus.HistogramVec
+	PrometheusRegistry            *prometheus.Registry
 }
 
 type GitRequestType string
@@ -100,6 +107,70 @@ func NewMetricsServer() *MetricsServer {
 	)
 	registry.MustRegister(redisRequestHistogram)
 
+	ociExtractFailCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_extract_fail_total",
+			Help: "Number of OCI extract requests failures by repo server",
+		},
+		[]string{"repo", "revision"},
+	)
+	registry.MustRegister(ociExtractFailCounter)
+
+	ociResolveRevisionFailCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_resolve_revision_fail_total",
+			Help: "Number of OCI resolve revision requests failures by repo server",
+		},
+		[]string{"repo", "revision"},
+	)
+	registry.MustRegister(ociResolveRevisionFailCounter)
+
+	ociDigestMetadataCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_digest_metadata_fail_total",
+			Help: "Number of OCI digest metadata requests failures by repo server",
+		},
+		[]string{"repo", "revision"},
+	)
+	registry.MustRegister(ociDigestMetadataCounter)
+
+	ociGetTagsFailCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_get_tags_fail_total",
+			Help: "Number of OCI get tags failures by repo server",
+		},
+		[]string{"repo"},
+	)
+	registry.MustRegister(ociGetTagsFailCounter)
+
+	ociTestRepoFailCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_test_repo_fail_total",
+			Help: "Number of OCI test repo requests failures by repo server",
+		},
+		[]string{"repo"},
+	)
+	registry.MustRegister(ociTestRepoFailCounter)
+
+	ociRequestCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_oci_request_total",
+			Help: "Number of OCI requests performed by repo server",
+		},
+		[]string{"repo", "request_type"},
+	)
+	registry.MustRegister(ociRequestCounter)
+
+	ociRequestHistogram := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "argocd_oci_request_duration_seconds",
+			Help:    "OCI requests duration seconds.",
+			Buckets: []float64{0.1, 0.25, .5, 1, 2, 4, 10, 20},
+		},
+		[]string{"repo", "request_type"},
+	)
+	registry.MustRegister(ociRequestHistogram)
+
 	return &MetricsServer{
 		handler:                  promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
 		gitFetchFailCounter:      gitFetchFailCounter,
@@ -109,6 +180,8 @@ func NewMetricsServer() *MetricsServer {
 		repoPendingRequestsGauge: repoPendingRequestsGauge,
 		redisRequestCounter:      redisRequestCounter,
 		redisRequestHistogram:    redisRequestHistogram,
+		ociRequestCounter:        ociRequestCounter,
+		ociRequestHistogram:      ociRequestHistogram,
 		PrometheusRegistry:       registry,
 	}
 }
@@ -149,3 +222,37 @@ func (m *MetricsServer) IncRedisRequest(failed bool) {
 func (m *MetricsServer) ObserveRedisRequestDuration(duration time.Duration) {
 	m.redisRequestHistogram.WithLabelValues("argocd-repo-server").Observe(duration.Seconds())
 }
+
+// IncOCIRequest increments the OCI requests counter
+func (m *MetricsServer) IncOCIRequest(repo string, requestType OCIRequestType) {
+	m.ociRequestCounter.WithLabelValues(repo, string(requestType)).Inc()
+}
+
+func (m *MetricsServer) ObserveOCIRequestDuration(repo string, requestType OCIRequestType, duration time.Duration) {
+	m.ociRequestHistogram.WithLabelValues(repo, string(requestType)).Observe(duration.Seconds())
+}
+
+// IncOCIExtractFail increments the OCI failed extract requests counter
+func (m *MetricsServer) IncOCIExtractFail(repo string, revision string) {
+	m.ociExtractFailCounter.WithLabelValues(repo, revision).Inc()
+}
+
+// IncOCIResolveRevisionFailCounter increments the OCI failed resolve revision requests counter
+func (m *MetricsServer) IncOCIResolveRevisionFailCounter(repo string, revision string) {
+	m.ociResolveRevisionFailCounter.WithLabelValues(repo, revision).Inc()
+}
+
+// IncOCIDigestMetadataCounter increments the OCI failed digest metadata requests counter
+func (m *MetricsServer) IncOCIDigestMetadataCounter(repo string, revision string) {
+	m.ociDigestMetadataCounter.WithLabelValues(repo, revision).Inc()
+}
+
+// IncOCIGetTagsFailCounter increments the OCI failed get tags requests counter
+func (m *MetricsServer) IncOCIGetTagsFailCounter(repo string) {
+	m.ociGetTagsFailCounter.WithLabelValues(repo).Inc()
+}
+
+// IncOCITestRepoFailCounter increments the OCI failed test repo requests counter
+func (m *MetricsServer) IncOCITestRepoFailCounter(repo string) {
+	m.ociTestRepoFailCounter.WithLabelValues(repo).Inc()
+}

reposerver/metrics/ocihandlers.go

@@ -0,0 +1,49 @@
+package metrics
+
+import (
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/argoproj/argo-cd/v3/util/oci"
+)
+
+type OCIRequestType string
+
+const (
+	OCIRequestTypeExtract         = "extract"
+	OCIRequestTypeResolveRevision = "resolve-revision"
+	OCIRequestTypeDigestMetadata  = "digest-metadata"
+	OCIRequestTypeGetTags         = "get-tags"
+	OCIRequestTypeTestRepo        = "test-repo"
+)
+
+// NewOCIClientEventHandlers creates event handlers to update OCI repo, related metrics
+func NewOCIClientEventHandlers(metricsServer *MetricsServer) oci.EventHandlers {
+	return oci.EventHandlers{
+		OnExtract: func(repo string) func() {
+			return processMetricFunc(metricsServer, repo, OCIRequestTypeExtract)
+		},
+		OnResolveRevision: func(repo string) func() {
+			return processMetricFunc(metricsServer, repo, OCIRequestTypeResolveRevision)
+		},
+		OnDigestMetadata: func(repo string) func() {
+			return processMetricFunc(metricsServer, repo, OCIRequestTypeDigestMetadata)
+		},
+		OnGetTags: func(repo string) func() {
+			return processMetricFunc(metricsServer, repo, OCIRequestTypeGetTags)
+		},
+		OnTestRepo: func(repo string) func() {
+			return processMetricFunc(metricsServer, repo, OCIRequestTypeTestRepo)
+		},
+	}
+}
+
+func processMetricFunc(metricsServer *MetricsServer, repo string, requestType OCIRequestType) func() {
+	startTime := time.Now()
+	metricsServer.IncOCIRequest(repo, requestType)
+	log.Warnf("processing metric %s for repo %s", requestType, repo)
+	return func() {
+		metricsServer.ObserveOCIRequestDuration(repo, requestType, time.Since(startTime))
+	}
+}

reposerver/metrics/ocihandlers_test.go

@@ -0,0 +1,51 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOCIClientEventHandlers(t *testing.T) {
+	tests := []struct {
+		name     string
+		setup    func()
+		teardown func()
+		testFunc func(t *testing.T)
+	}{
+		{
+			name: "test event handlers",
+			testFunc: func(t *testing.T) {
+				t.Helper()
+				assert.NotPanics(t, func() {
+					metricsServer := NewMetricsServer()
+					eventHandlers := NewOCIClientEventHandlers(metricsServer)
+					eventHandlers.OnExtract("test")
+					eventHandlers.OnTestRepo("test")
+					eventHandlers.OnGetTags("test")
+					eventHandlers.OnResolveRevision("test")
+					eventHandlers.OnDigestMetadata("test")
+					c := metricsServer.ociRequestCounter
+					assert.Equal(t, 5, testutil.CollectAndCount(c))
+					assert.InDelta(t, float64(1), testutil.ToFloat64(c.WithLabelValues("test", OCIRequestTypeExtract)), 0.01)
+					assert.InDelta(t, float64(1), testutil.ToFloat64(c.WithLabelValues("test", OCIRequestTypeResolveRevision)), 0.01)
+					assert.InDelta(t, float64(1), testutil.ToFloat64(c.WithLabelValues("test", OCIRequestTypeDigestMetadata)), 0.01)
+					assert.InDelta(t, float64(1), testutil.ToFloat64(c.WithLabelValues("test", OCIRequestTypeTestRepo)), 0.01)
+					assert.InDelta(t, float64(1), testutil.ToFloat64(c.WithLabelValues("test", OCIRequestTypeTestRepo)), 0.01)
+				})
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.setup != nil {
+				tt.setup()
+			}
+			if tt.teardown != nil {
+				defer tt.teardown()
+			}
+			tt.testFunc(t)
+		})
+	}
+}

reposerver/repository/repository.go

@@ -190,7 +190,7 @@ func (s *Service) Init() error {
 
 // ListOCITags List a subset of the refs (currently, branches and tags) of a git repo
 func (s *Service) ListOCITags(ctx context.Context, q *apiclient.ListRefsRequest) (*apiclient.Refs, error) {
-	ociClient, err := s.newOCIClient(q.Repo.Repo, q.Repo.GetOCICreds(), q.Repo.Proxy, q.Repo.NoProxy, s.initConstants.OCIMediaTypes, oci.WithIndexCache(s.cache), oci.WithImagePaths(s.ociPaths), oci.WithManifestMaxExtractedSize(s.initConstants.OCIManifestMaxExtractedSize), oci.WithDisableManifestMaxExtractedSize(s.initConstants.DisableOCIManifestMaxExtractedSize))
+	ociClient, err := s.newOCIClient(q.Repo.Repo, q.Repo.GetOCICreds(), q.Repo.Proxy, q.Repo.NoProxy, s.initConstants.OCIMediaTypes, s.ociClientStandardOpts()...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating oci client: %w", err)
 	}
@@ -200,6 +200,7 @@ func (s *Service) ListOCITags(ctx context.Context, q *apiclient.ListRefsRequest)
 
 	tags, err := ociClient.GetTags(ctx, false)
 	if err != nil {
+		s.metricsServer.IncOCIGetTagsFailCounter(q.Repo.Repo)
 		return nil, err
 	}
 
@@ -384,6 +385,7 @@ func (s *Service) runRepoOperation(
 
 		ociPath, closer, err := ociClient.Extract(ctx, revision)
 		if err != nil {
+			s.metricsServer.IncOCIExtractFail(repo.Repo, revision)
 			return err
 		}
 		defer utilio.Close(closer)
@@ -2493,13 +2495,14 @@ func (s *Service) GetRevisionMetadata(_ context.Context, q *apiclient.RepoServer
 }
 
 func (s *Service) GetOCIMetadata(ctx context.Context, q *apiclient.RepoServerRevisionChartDetailsRequest) (*v1alpha1.OCIMetadata, error) {
-	client, err := s.newOCIClient(q.Repo.Repo, q.Repo.GetOCICreds(), q.Repo.Proxy, q.Repo.NoProxy, s.initConstants.OCIMediaTypes, oci.WithIndexCache(s.cache), oci.WithImagePaths(s.ociPaths), oci.WithManifestMaxExtractedSize(s.initConstants.OCIManifestMaxExtractedSize), oci.WithDisableManifestMaxExtractedSize(s.initConstants.DisableOCIManifestMaxExtractedSize))
+	client, err := s.newOCIClient(q.Repo.Repo, q.Repo.GetOCICreds(), q.Repo.Proxy, q.Repo.NoProxy, s.initConstants.OCIMediaTypes, s.ociClientStandardOpts()...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize oci client: %w", err)
 	}
 
 	metadata, err := client.DigestMetadata(ctx, q.Revision)
 	if err != nil {
+		s.metricsServer.IncOCIDigestMetadataCounter(q.Repo.Repo, q.Revision)
 		return nil, fmt.Errorf("failed to extract digest metadata for revision %q: %w", q.Revision, err)
 	}
 
@@ -2589,13 +2592,14 @@ func (s *Service) newClientResolveRevision(repo *v1alpha1.Repository, revision s
 }
 
 func (s *Service) newOCIClientResolveRevision(ctx context.Context, repo *v1alpha1.Repository, revision string, noRevisionCache bool) (oci.Client, string, error) {
-	ociClient, err := s.newOCIClient(repo.Repo, repo.GetOCICreds(), repo.Proxy, repo.NoProxy, s.initConstants.OCIMediaTypes, oci.WithIndexCache(s.cache), oci.WithImagePaths(s.ociPaths), oci.WithManifestMaxExtractedSize(s.initConstants.OCIManifestMaxExtractedSize), oci.WithDisableManifestMaxExtractedSize(s.initConstants.DisableOCIManifestMaxExtractedSize))
+	ociClient, err := s.newOCIClient(repo.Repo, repo.GetOCICreds(), repo.Proxy, repo.NoProxy, s.initConstants.OCIMediaTypes, s.ociClientStandardOpts()...)
 	if err != nil {
 		return nil, "", fmt.Errorf("failed to initialize oci client: %w", err)
 	}
 
 	digest, err := ociClient.ResolveRevision(ctx, revision, noRevisionCache)
 	if err != nil {
+		s.metricsServer.IncOCIResolveRevisionFailCounter(repo.Repo, revision)
 		return nil, "", fmt.Errorf("failed to resolve revision %q: %w", revision, err)
 	}
 
@@ -2786,11 +2790,16 @@ func (s *Service) TestRepository(ctx context.Context, q *apiclient.TestRepositor
 			return git.TestRepo(repo.Repo, repo.GetGitCreds(s.gitCredsStore), repo.IsInsecure(), repo.IsLFSEnabled(), repo.Proxy, repo.NoProxy)
 		},
 		"oci": func() error {
-			client, err := oci.NewClient(repo.Repo, repo.GetOCICreds(), repo.Proxy, repo.NoProxy, s.initConstants.OCIMediaTypes)
+			client, err := oci.NewClient(repo.Repo, repo.GetOCICreds(), repo.Proxy, repo.NoProxy,
+				s.initConstants.OCIMediaTypes, oci.WithEventHandlers(metrics.NewOCIClientEventHandlers(s.metricsServer)))
 			if err != nil {
 				return err
 			}
 			_, err = client.TestRepo(ctx)
+			if err != nil {
+				s.metricsServer.IncOCITestRepoFailCounter(q.Repo.Repo)
+			}
+
 			return err
 		},
 		"helm": func() error {
@@ -3139,3 +3148,13 @@ func (s *Service) updateCachedRevision(logCtx *log.Entry, oldRev string, newRev
 	logCtx.Debugf("manifest cache updated for application %s in repo %s from revision %s to revision %s", request.AppName, request.GetRepo().Repo, oldRev, newRev)
 	return nil
 }
+
+func (s *Service) ociClientStandardOpts() []oci.ClientOpts {
+	return []oci.ClientOpts{
+		oci.WithIndexCache(s.cache),
+		oci.WithImagePaths(s.ociPaths),
+		oci.WithManifestMaxExtractedSize(s.initConstants.OCIManifestMaxExtractedSize),
+		oci.WithDisableManifestMaxExtractedSize(s.initConstants.DisableOCIManifestMaxExtractedSize),
+		oci.WithEventHandlers(metrics.NewOCIClientEventHandlers(s.metricsServer)),
+	}
+}