feat(hydrator): hydrateTo and syncSource as a different repository (#22719)

Author: Elyytscha

USERS.md

@@ -420,6 +420,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [WeMaintain](https://www.wemaintain.com/)
 1. [WeMo Scooter](https://www.wemoscooter.com/)
 1. [Whitehat Berlin](https://whitehat.berlin) by Guido Maria Serra +Fenaroli
+1. [WhizUs](https://whizus.com)
 1. [Witick](https://witick.io/)
 1. [Wolffun Game](https://www.wolffungame.com/)
 1. [WooliesX](https://wooliesx.com.au/)

assets/swagger.json

@@ -98,7 +98,10 @@ type AppOptions struct {
 	drySourcePath                   string
 	syncSourceBranch                string
 	syncSourcePath                  string
+	syncSourceRepo                  string
 	hydrateToBranch                 string
+	hydrateToRepo                   string
+	hydrateToPath                   string
 }
 
 func AddAppFlags(command *cobra.Command, opts *AppOptions) {
@@ -112,7 +115,10 @@ func AddAppFlags(command *cobra.Command, opts *AppOptions) {
 	command.Flags().StringVar(&opts.drySourcePath, "dry-source-path", "", "Path in repository to the app directory for the dry source")
 	command.Flags().StringVar(&opts.syncSourceBranch, "sync-source-branch", "", "The branch from which the app will sync")
 	command.Flags().StringVar(&opts.syncSourcePath, "sync-source-path", "", "The path in the repository from which the app will sync")
+	command.Flags().StringVar(&opts.syncSourceRepo, "sync-source-repo", "", "The repository URL from which the app will sync (defaults to dry-source-repo if not set)")
 	command.Flags().StringVar(&opts.hydrateToBranch, "hydrate-to-branch", "", "The branch to hydrate the app to")
+	command.Flags().StringVar(&opts.hydrateToRepo, "hydrate-to-repo", "", "The repository URL to hydrate the app to (defaults to sync-source-repo or dry-source-repo if not set)")
+	command.Flags().StringVar(&opts.hydrateToPath, "hydrate-to-path", "", "The path in the repository to hydrate the app to (defaults to sync-source-path if not set)")
 	command.Flags().IntVar(&opts.revisionHistoryLimit, "revision-history-limit", argoappv1.RevisionHistoryLimit, "How many items to keep in revision history")
 	command.Flags().StringVar(&opts.destServer, "dest-server", "", "K8s cluster URL (e.g. https://kubernetes.default.svc)")
 	command.Flags().StringVar(&opts.destName, "dest-name", "", "K8s cluster Name (e.g. minikube)")
@@ -829,13 +835,31 @@ func constructSourceHydrator(h *argoappv1.SourceHydrator, appOpts AppOptions, fl
 		case "sync-source-path":
 			ensureNotNil(appOpts.syncSourcePath != "")
 			h.SyncSource.Path = appOpts.syncSourcePath
+		case "sync-source-repo":
+			ensureNotNil(appOpts.syncSourceRepo != "")
+			h.SyncSource.RepoURL = appOpts.syncSourceRepo
 		case "hydrate-to-branch":
 			ensureNotNil(appOpts.hydrateToBranch != "")
 			if appOpts.hydrateToBranch == "" {
 				h.HydrateTo = nil
 			} else {
-				h.HydrateTo = &argoappv1.HydrateTo{TargetBranch: appOpts.hydrateToBranch}
+				if h.HydrateTo == nil {
+					h.HydrateTo = &argoappv1.HydrateTo{}
+				}
+				h.HydrateTo.TargetBranch = appOpts.hydrateToBranch
+			}
+		case "hydrate-to-repo":
+			ensureNotNil(appOpts.hydrateToRepo != "")
+			if h.HydrateTo == nil {
+				h.HydrateTo = &argoappv1.HydrateTo{}
+			}
+			h.HydrateTo.RepoURL = appOpts.hydrateToRepo
+		case "hydrate-to-path":
+			ensureNotNil(appOpts.hydrateToPath != "")
+			if h.HydrateTo == nil {
+				h.HydrateTo = &argoappv1.HydrateTo{}
 			}
+			h.HydrateTo.Path = appOpts.hydrateToPath
 		}
 	})
 	return h, hasHydratorFlag

cmd/util/app.go

@@ -132,10 +132,12 @@ func (h *Hydrator) ProcessAppHydrateQueueItem(origApp *appv1.Application) {
 }
 
 func getHydrationQueueKey(app *appv1.Application) types.HydrationQueueKey {
+	hydrateToSource := app.Spec.GetHydrateToSource()
 	key := types.HydrationQueueKey{
 		SourceRepoURL:        git.NormalizeGitURLAllowInvalid(app.Spec.SourceHydrator.DrySource.RepoURL),
 		SourceTargetRevision: app.Spec.SourceHydrator.DrySource.TargetRevision,
-		DestinationBranch:    app.Spec.GetHydrateToSource().TargetRevision,
+		DestinationRepoURL:   git.NormalizeGitURLAllowInvalid(hydrateToSource.RepoURL),
+		DestinationBranch:    hydrateToSource.TargetRevision,
 	}
 	return key
 }
@@ -148,6 +150,7 @@ func (h *Hydrator) ProcessHydrationQueueItem(hydrationKey types.HydrationQueueKe
 	logCtx := log.WithFields(log.Fields{
 		"sourceRepoURL":        hydrationKey.SourceRepoURL,
 		"sourceTargetRevision": hydrationKey.SourceTargetRevision,
+		"destinationRepoURL":   hydrationKey.DestinationRepoURL,
 		"destinationBranch":    hydrationKey.DestinationBranch,
 	})
 
@@ -319,20 +322,30 @@ func (h *Hydrator) validateApplications(apps []*appv1.Application) (map[string]*
 		// Hydrating to root would overwrite or delete files at the top level of the repo,
 		// which can break other applications or shared configuration.
 		// Every hydrated app must write into a subdirectory instead.
-		destPath := app.Spec.SourceHydrator.SyncSource.Path
+		hydrateToSource := app.Spec.GetHydrateToSource()
+		destPath := hydrateToSource.Path
 		if IsRootPath(destPath) {
-			errors[app.QualifiedName()] = fmt.Errorf("app is configured to hydrate to the repository root (branch %q, path %q) which is not allowed", app.Spec.GetHydrateToSource().TargetRevision, destPath)
+			errors[app.QualifiedName()] = fmt.Errorf("app is configured to hydrate to the repository root (branch %q, path %q) which is not allowed", hydrateToSource.TargetRevision, destPath)
+			continue
+		}
+
+		// Validate that the destination repo is permitted in the project
+		destRepoPermitted := proj.IsSourcePermitted(hydrateToSource)
+		if !destRepoPermitted {
+			errors[app.QualifiedName()] = fmt.Errorf("destination repo %s is not permitted in project '%s'", hydrateToSource.RepoURL, proj.Name)
 			continue
 		}
 
 		// TODO: test the dupe detection
 		// TODO: normalize the path to avoid "path/.." from being treated as different from "."
-		if appName, ok := uniquePaths[destPath]; ok {
-			errors[app.QualifiedName()] = fmt.Errorf("app %s hydrator use the same destination: %v", appName, app.Spec.SourceHydrator.SyncSource.Path)
-			errors[appName] = fmt.Errorf("app %s hydrator use the same destination: %v", app.QualifiedName(), app.Spec.SourceHydrator.SyncSource.Path)
+		// Use a combination of repo URL and path for uniqueness since apps can hydrate to different repos
+		destKey := fmt.Sprintf("%s:%s", hydrateToSource.RepoURL, destPath)
+		if appName, ok := uniquePaths[destKey]; ok {
+			errors[app.QualifiedName()] = fmt.Errorf("app %s hydrator use the same destination: %v", appName, destKey)
+			errors[appName] = fmt.Errorf("app %s hydrator use the same destination: %v", app.QualifiedName(), destKey)
 			continue
 		}
-		uniquePaths[destPath] = app.QualifiedName()
+		uniquePaths[destKey] = app.QualifiedName()
 	}
 
 	// If there are any errors, return nil for projects to avoid possible partial processing.
@@ -350,14 +363,15 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 	}
 
 	// These values are the same for all apps being hydrated together, so just get them from the first app.
-	repoURL := apps[0].Spec.GetHydrateToSource().RepoURL
+	destinationRepoURL := apps[0].Spec.GetHydrateToSource().RepoURL
 	targetBranch := apps[0].Spec.GetHydrateToSource().TargetRevision
 	// FIXME: As a convenience, the commit server will create the syncBranch if it does not exist. If the
 	// targetBranch does not exist, it will create it based on the syncBranch. On the next line, we take
 	// the `syncBranch` from the first app and assume that they're all configured the same. Instead, if any
 	// app has a different syncBranch, we should send the commit server an empty string and allow it to
 	// create the targetBranch as an orphan since we can't reliable determine a reasonable base.
 	syncBranch := apps[0].Spec.SourceHydrator.SyncSource.TargetBranch
+	drySourceRepoURL := apps[0].Spec.SourceHydrator.DrySource.RepoURL
 
 	// Get a static SHA revision from the first app so that all apps are hydrated from the same revision.
 	targetRevision, pathDetails, err := h.getManifests(context.Background(), apps[0], "", projects[apps[0].Spec.Project])
@@ -398,20 +412,20 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 		}
 	}
 
-	// Get the commit metadata for the target revision.
-	revisionMetadata, err := h.getRevisionMetadata(context.Background(), repoURL, project, targetRevision)
+	// Get the commit metadata for the target revision from the dry source repo.
+	revisionMetadata, err := h.getRevisionMetadata(context.Background(), drySourceRepoURL, project, targetRevision)
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get revision metadata for %q: %w", targetRevision, err)
 	}
 
-	repo, err := h.dependencies.GetWriteCredentials(context.Background(), repoURL, project)
+	repo, err := h.dependencies.GetWriteCredentials(context.Background(), destinationRepoURL, project)
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrator credentials: %w", err)
 	}
 	if repo == nil {
 		// Try without credentials.
 		repo = &appv1.Repository{
-			Repo: repoURL,
+			Repo: destinationRepoURL,
 		}
 		logCtx.Warn("no credentials found for repo, continuing without credentials")
 	}
@@ -420,7 +434,7 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrated commit message template: %w", err)
 	}
-	commitMessage, errMsg := getTemplatedCommitMessage(repoURL, targetRevision, commitMessageTemplate, revisionMetadata)
+	commitMessage, errMsg := getTemplatedCommitMessage(drySourceRepoURL, targetRevision, commitMessageTemplate, revisionMetadata)
 	if errMsg != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrator commit templated message: %w", errMsg)
 	}
@@ -477,8 +491,14 @@ func (h *Hydrator) getManifests(ctx context.Context, app *appv1.Application, tar
 		manifestDetails[i] = &commitclient.HydratedManifestDetails{ManifestJSON: string(objJSON)}
 	}
 
+	// Use the hydrateTo path if set, otherwise use the syncSource path
+	destPath := app.Spec.SourceHydrator.SyncSource.Path
+	if app.Spec.SourceHydrator.HydrateTo != nil && app.Spec.SourceHydrator.HydrateTo.Path != "" {
+		destPath = app.Spec.SourceHydrator.HydrateTo.Path
+	}
+
 	return resp.Revision, &commitclient.PathDetails{
-		Path:      app.Spec.SourceHydrator.SyncSource.Path,
+		Path:      destPath,
 		Manifests: manifestDetails,
 		Commands:  resp.Commands,
 	}, nil

controller/hydrator/hydrator.go

@@ -167,6 +167,7 @@ func Test_getAppsForHydrationKey_RepoURLNormalization(t *testing.T) {
 	hydrationKey := types.HydrationQueueKey{
 		SourceRepoURL:        "https://example.com/repo",
 		SourceTargetRevision: "main",
+		DestinationRepoURL:   "https://example.com/repo",
 		DestinationBranch:    "main",
 	}
 
@@ -1094,3 +1095,93 @@ func TestHydrator_getManifests_GetRepoObjsError(t *testing.T) {
 	assert.Empty(t, rev)
 	assert.Nil(t, pathDetails)
 }
+
+func TestHydrator_getManifests_UsesHydrateToPath(t *testing.T) {
+	t.Parallel()
+	d := mocks.NewDependencies(t)
+	h := &Hydrator{dependencies: d}
+	app := newTestApp("test-app")
+	app.Spec.SourceHydrator.HydrateTo.Path = "hydrated-path"
+	proj := newTestProject()
+
+	cm := kube.MustToUnstructured(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+	})
+
+	d.EXPECT().GetRepoObjs(mock.Anything, app, app.Spec.SourceHydrator.GetDrySource(), "sha123", proj).Return([]*unstructured.Unstructured{cm}, &repoclient.ManifestResponse{
+		Revision: "sha123",
+		Commands: []string{"cmd1"},
+	}, nil)
+
+	rev, pathDetails, err := h.getManifests(t.Context(), app, "sha123", proj)
+	require.NoError(t, err)
+	assert.Equal(t, "sha123", rev)
+	assert.Equal(t, "hydrated-path", pathDetails.Path)
+}
+
+func TestGetHydrateToSource_DifferentRepo(t *testing.T) {
+	spec := &v1alpha1.ApplicationSpec{
+		SourceHydrator: &v1alpha1.SourceHydrator{
+			DrySource: v1alpha1.DrySource{
+				RepoURL:        "https://example.com/dry-repo",
+				TargetRevision: "main",
+				Path:           "base",
+			},
+			SyncSource: v1alpha1.SyncSource{
+				TargetBranch: "hydrated",
+				Path:         "app",
+			},
+			HydrateTo: &v1alpha1.HydrateTo{
+				RepoURL:      "https://example.com/hydrated-repo",
+				TargetBranch: "staging",
+				Path:         "hydrated-app",
+			},
+		},
+	}
+
+	source := spec.GetHydrateToSource()
+	assert.Equal(t, "https://example.com/hydrated-repo", source.RepoURL)
+	assert.Equal(t, "hydrated-app", source.Path)
+	assert.Equal(t, "staging", source.TargetRevision)
+}
+
+func TestGetSyncSource_DifferentRepo(t *testing.T) {
+	hydrator := &v1alpha1.SourceHydrator{
+		DrySource: v1alpha1.DrySource{
+			RepoURL:        "https://example.com/dry-repo",
+			TargetRevision: "main",
+			Path:           "base",
+		},
+		SyncSource: v1alpha1.SyncSource{
+			RepoURL:      "https://example.com/sync-repo",
+			TargetBranch: "hydrated",
+			Path:         "app",
+		},
+	}
+
+	source := hydrator.GetSyncSource()
+	assert.Equal(t, "https://example.com/sync-repo", source.RepoURL)
+	assert.Equal(t, "app", source.Path)
+	assert.Equal(t, "hydrated", source.TargetRevision)
+}
+
+func TestGetSyncSource_FallsBackToDrySourceRepo(t *testing.T) {
+	hydrator := &v1alpha1.SourceHydrator{
+		DrySource: v1alpha1.DrySource{
+			RepoURL:        "https://example.com/dry-repo",
+			TargetRevision: "main",
+			Path:           "base",
+		},
+		SyncSource: v1alpha1.SyncSource{
+			TargetBranch: "hydrated",
+			Path:         "app",
+		},
+	}
+
+	source := hydrator.GetSyncSource()
+	assert.Equal(t, "https://example.com/dry-repo", source.RepoURL)
+	assert.Equal(t, "app", source.Path)
+	assert.Equal(t, "hydrated", source.TargetRevision)
+}

controller/hydrator/hydrator_test.go

@@ -8,4 +8,7 @@ type HydrationQueueKey struct {
 	SourceRepoURL        string
 	SourceTargetRevision string
 	DestinationBranch    string
+	// DestinationRepoURL must be normalized with git.NormalizeGitURL to ensure proper deduplication when apps hydrate
+	// to different repositories.
+	DestinationRepoURL string
 }