feat(hydrator): syncSource as a different repository (#22719)

Makefile

@@ -482,7 +482,7 @@ start-e2e-local: mod-vendor-local dep-ui-local cli-local
 	kubectl create ns argocd-e2e-external || true
 	kubectl create ns argocd-e2e-external-2 || true
 	kubectl config set-context --current --namespace=argocd-e2e
-	kustomize build test/manifests/base | kubectl apply -f -
+	kustomize build test/manifests/base | kubectl apply --server-side=true -f -
 	kubectl apply -f https://raw.githubusercontent.com/open-cluster-management/api/a6845f2ebcb186ec26b832f60c988537a58f3859/cluster/v1alpha1/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml
 	# Create GPG keys and source directories
 	if test -d /tmp/argo-e2e/app/config/gpg; then rm -rf /tmp/argo-e2e/app/config/gpg/*; fi

USERS.md

@@ -420,6 +420,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [WeMaintain](https://www.wemaintain.com/)
 1. [WeMo Scooter](https://www.wemoscooter.com/)
 1. [Whitehat Berlin](https://whitehat.berlin) by Guido Maria Serra +Fenaroli
+1. [WhizUs](https://whizus.com)
 1. [Witick](https://witick.io/)
 1. [Wolffun Game](https://www.wolffungame.com/)
 1. [WooliesX](https://wooliesx.com.au/)

cmd/util/app.go

@@ -98,6 +98,7 @@ type AppOptions struct {
 	drySourcePath                   string
 	syncSourceBranch                string
 	syncSourcePath                  string
+	syncSourceRepo                  string
 	hydrateToBranch                 string
 }
 
@@ -112,6 +113,7 @@ func AddAppFlags(command *cobra.Command, opts *AppOptions) {
 	command.Flags().StringVar(&opts.drySourcePath, "dry-source-path", "", "Path in repository to the app directory for the dry source")
 	command.Flags().StringVar(&opts.syncSourceBranch, "sync-source-branch", "", "The branch from which the app will sync")
 	command.Flags().StringVar(&opts.syncSourcePath, "sync-source-path", "", "The path in the repository from which the app will sync")
+	command.Flags().StringVar(&opts.syncSourceRepo, "sync-source-repo", "", "The repository URL from which the app will sync (defaults to dry-source-repo if not set)")
 	command.Flags().StringVar(&opts.hydrateToBranch, "hydrate-to-branch", "", "The branch to hydrate the app to")
 	command.Flags().IntVar(&opts.revisionHistoryLimit, "revision-history-limit", argoappv1.RevisionHistoryLimit, "How many items to keep in revision history")
 	command.Flags().StringVar(&opts.destServer, "dest-server", "", "K8s cluster URL (e.g. https://kubernetes.default.svc)")
@@ -829,12 +831,18 @@ func constructSourceHydrator(h *argoappv1.SourceHydrator, appOpts AppOptions, fl
 		case "sync-source-path":
 			ensureNotNil(appOpts.syncSourcePath != "")
 			h.SyncSource.Path = appOpts.syncSourcePath
+		case "sync-source-repo":
+			ensureNotNil(appOpts.syncSourceRepo != "")
+			h.SyncSource.RepoURL = appOpts.syncSourceRepo
 		case "hydrate-to-branch":
 			ensureNotNil(appOpts.hydrateToBranch != "")
 			if appOpts.hydrateToBranch == "" {
 				h.HydrateTo = nil
 			} else {
-				h.HydrateTo = &argoappv1.HydrateTo{TargetBranch: appOpts.hydrateToBranch}
+				if h.HydrateTo == nil {
+					h.HydrateTo = &argoappv1.HydrateTo{}
+				}
+				h.HydrateTo.TargetBranch = appOpts.hydrateToBranch
 			}
 		}
 	})

cmd/util/app_test.go

@@ -369,6 +369,10 @@ func Test_setAppSpecOptions(t *testing.T) {
 		require.NoError(t, f.SetFlag("sync-source-path", "apps"))
 		assert.Equal(t, "apps", f.spec.SourceHydrator.SyncSource.Path)
 
+		// Test sync-source-repo flag - this is the new field for different repository support
+		require.NoError(t, f.SetFlag("sync-source-repo", "https://github.com/argoproj/gitops-manifests"))
+		assert.Equal(t, "https://github.com/argoproj/gitops-manifests", f.spec.SourceHydrator.SyncSource.RepoURL)
+
 		require.NoError(t, f.SetFlag("hydrate-to-branch", "env/test-next"))
 		assert.Equal(t, "env/test-next", f.spec.SourceHydrator.HydrateTo.TargetBranch)
 

controller/hydrator/hydrator.go

@@ -132,10 +132,12 @@ func (h *Hydrator) ProcessAppHydrateQueueItem(origApp *appv1.Application) {
 }
 
 func getHydrationQueueKey(app *appv1.Application) types.HydrationQueueKey {
+	hydrateToSource := app.Spec.GetHydrateToSource()
 	key := types.HydrationQueueKey{
 		SourceRepoURL:        git.NormalizeGitURLAllowInvalid(app.Spec.SourceHydrator.DrySource.RepoURL),
 		SourceTargetRevision: app.Spec.SourceHydrator.DrySource.TargetRevision,
-		DestinationBranch:    app.Spec.GetHydrateToSource().TargetRevision,
+		DestinationRepoURL:   git.NormalizeGitURLAllowInvalid(hydrateToSource.RepoURL),
+		DestinationBranch:    hydrateToSource.TargetRevision,
 	}
 	return key
 }
@@ -148,6 +150,7 @@ func (h *Hydrator) ProcessHydrationQueueItem(hydrationKey types.HydrationQueueKe
 	logCtx := log.WithFields(log.Fields{
 		"sourceRepoURL":        hydrationKey.SourceRepoURL,
 		"sourceTargetRevision": hydrationKey.SourceTargetRevision,
+		"destinationRepoURL":   hydrationKey.DestinationRepoURL,
 		"destinationBranch":    hydrationKey.DestinationBranch,
 	})
 
@@ -319,20 +322,30 @@ func (h *Hydrator) validateApplications(apps []*appv1.Application) (map[string]*
 		// Hydrating to root would overwrite or delete files at the top level of the repo,
 		// which can break other applications or shared configuration.
 		// Every hydrated app must write into a subdirectory instead.
-		destPath := app.Spec.SourceHydrator.SyncSource.Path
+		hydrateToSource := app.Spec.GetHydrateToSource()
+		destPath := hydrateToSource.Path
 		if IsRootPath(destPath) {
-			errors[app.QualifiedName()] = fmt.Errorf("app is configured to hydrate to the repository root (branch %q, path %q) which is not allowed", app.Spec.GetHydrateToSource().TargetRevision, destPath)
+			errors[app.QualifiedName()] = fmt.Errorf("app is configured to hydrate to the repository root (branch %q, path %q) which is not allowed", hydrateToSource.TargetRevision, destPath)
+			continue
+		}
+
+		// Validate that the destination repo is permitted in the project
+		destRepoPermitted := proj.IsSourcePermitted(hydrateToSource)
+		if !destRepoPermitted {
+			errors[app.QualifiedName()] = fmt.Errorf("destination repo %s is not permitted in project '%s'", hydrateToSource.RepoURL, proj.Name)
 			continue
 		}
 
 		// TODO: test the dupe detection
 		// TODO: normalize the path to avoid "path/.." from being treated as different from "."
-		if appName, ok := uniquePaths[destPath]; ok {
-			errors[app.QualifiedName()] = fmt.Errorf("app %s hydrator use the same destination: %v", appName, app.Spec.SourceHydrator.SyncSource.Path)
-			errors[appName] = fmt.Errorf("app %s hydrator use the same destination: %v", app.QualifiedName(), app.Spec.SourceHydrator.SyncSource.Path)
+		// Use a combination of repo URL and path for uniqueness since apps can hydrate to different repos
+		destKey := fmt.Sprintf("%s:%s", hydrateToSource.RepoURL, destPath)
+		if appName, ok := uniquePaths[destKey]; ok {
+			errors[app.QualifiedName()] = fmt.Errorf("app %s hydrator use the same destination: %v", appName, destKey)
+			errors[appName] = fmt.Errorf("app %s hydrator use the same destination: %v", app.QualifiedName(), destKey)
 			continue
 		}
-		uniquePaths[destPath] = app.QualifiedName()
+		uniquePaths[destKey] = app.QualifiedName()
 	}
 
 	// If there are any errors, return nil for projects to avoid possible partial processing.
@@ -350,14 +363,15 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 	}
 
 	// These values are the same for all apps being hydrated together, so just get them from the first app.
-	repoURL := apps[0].Spec.GetHydrateToSource().RepoURL
+	destinationRepoURL := apps[0].Spec.GetHydrateToSource().RepoURL
 	targetBranch := apps[0].Spec.GetHydrateToSource().TargetRevision
 	// FIXME: As a convenience, the commit server will create the syncBranch if it does not exist. If the
 	// targetBranch does not exist, it will create it based on the syncBranch. On the next line, we take
 	// the `syncBranch` from the first app and assume that they're all configured the same. Instead, if any
 	// app has a different syncBranch, we should send the commit server an empty string and allow it to
 	// create the targetBranch as an orphan since we can't reliable determine a reasonable base.
 	syncBranch := apps[0].Spec.SourceHydrator.SyncSource.TargetBranch
+	drySourceRepoURL := apps[0].Spec.SourceHydrator.DrySource.RepoURL
 
 	// Get a static SHA revision from the first app so that all apps are hydrated from the same revision.
 	targetRevision, pathDetails, err := h.getManifests(context.Background(), apps[0], "", projects[apps[0].Spec.Project])
@@ -398,20 +412,20 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 		}
 	}
 
-	// Get the commit metadata for the target revision.
-	revisionMetadata, err := h.getRevisionMetadata(context.Background(), repoURL, project, targetRevision)
+	// Get the commit metadata for the target revision from the dry source repo.
+	revisionMetadata, err := h.getRevisionMetadata(context.Background(), drySourceRepoURL, project, targetRevision)
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get revision metadata for %q: %w", targetRevision, err)
 	}
 
-	repo, err := h.dependencies.GetWriteCredentials(context.Background(), repoURL, project)
+	repo, err := h.dependencies.GetWriteCredentials(context.Background(), destinationRepoURL, project)
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrator credentials: %w", err)
 	}
 	if repo == nil {
 		// Try without credentials.
 		repo = &appv1.Repository{
-			Repo: repoURL,
+			Repo: destinationRepoURL,
 		}
 		logCtx.Warn("no credentials found for repo, continuing without credentials")
 	}
@@ -420,7 +434,7 @@ func (h *Hydrator) hydrate(logCtx *log.Entry, apps []*appv1.Application, project
 	if err != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrated commit message template: %w", err)
 	}
-	commitMessage, errMsg := getTemplatedCommitMessage(repoURL, targetRevision, commitMessageTemplate, revisionMetadata)
+	commitMessage, errMsg := getTemplatedCommitMessage(drySourceRepoURL, targetRevision, commitMessageTemplate, revisionMetadata)
 	if errMsg != nil {
 		return targetRevision, "", errors, fmt.Errorf("failed to get hydrator commit templated message: %w", errMsg)
 	}

controller/hydrator/hydrator_test.go

@@ -167,6 +167,7 @@ func Test_getAppsForHydrationKey_RepoURLNormalization(t *testing.T) {
 	hydrationKey := types.HydrationQueueKey{
 		SourceRepoURL:        "https://example.com/repo",
 		SourceTargetRevision: "main",
+		DestinationRepoURL:   "https://example.com/repo",
 		DestinationBranch:    "main",
 	}
 
@@ -708,6 +709,63 @@ func TestValidateApplications_RootPath(t *testing.T) {
 	require.ErrorContains(t, errs[app.QualifiedName()], "app is configured to hydrate to the repository root")
 }
 
+// TestValidateApplications_SyncSourceRepoNotPermitted tests that the sync source repo
+// must be permitted in the project. When SyncSource.RepoURL is set to a different repo,
+// that repo must be permitted. This tests PR #25464 code.
+func TestValidateApplications_SyncSourceRepoNotPermitted(t *testing.T) {
+	t.Parallel()
+	d := mocks.NewDependencies(t)
+	app := newTestApp("test-app")
+	// Set a different sync source repo URL that is not permitted in the project
+	// The dry source repo (https://example.com/repo) is permitted, but the sync source repo is not
+	app.Spec.SourceHydrator.SyncSource.RepoURL = "https://example.com/not-permitted-repo"
+	// Project permits https://example.com/repo (dry source) but not the sync source repo
+	proj := newTestProject()
+	d.EXPECT().GetProcessableAppProj(app).Return(proj, nil).Once()
+	h := &Hydrator{dependencies: d}
+
+	projects, errs := h.validateApplications([]*v1alpha1.Application{app})
+	require.Nil(t, projects)
+	require.Len(t, errs, 1)
+	// When SyncSource.RepoURL is set, GetSource() returns that repo URL, so the first validation check fails
+	require.ErrorContains(t, errs[app.QualifiedName()], "application repo https://example.com/not-permitted-repo is not permitted in project")
+}
+
+// TestValidateApplications_DestinationRepoPermitted tests that validation passes when
+// the destination repo (sync source) is permitted in the project
+func TestValidateApplications_DestinationRepoPermitted(t *testing.T) {
+	t.Parallel()
+	d := mocks.NewDependencies(t)
+	app := newTestApp("test-app")
+	// Set sync source repo URL that IS permitted in the project
+	app.Spec.SourceHydrator.SyncSource.RepoURL = "https://example.com/repo"
+	// Project permits https://example.com/repo
+	proj := newTestProject()
+	d.EXPECT().GetProcessableAppProj(app).Return(proj, nil).Once()
+	h := &Hydrator{dependencies: d}
+
+	projects, errs := h.validateApplications([]*v1alpha1.Application{app})
+	require.NotNil(t, projects)
+	require.Empty(t, errs)
+}
+
+// TestValidateApplications_DestinationRepoSameAsDrySource tests that validation passes when
+// the sync source repo URL is empty (defaults to dry source repo)
+func TestValidateApplications_DestinationRepoSameAsDrySource(t *testing.T) {
+	t.Parallel()
+	d := mocks.NewDependencies(t)
+	app := newTestApp("test-app")
+	// Empty sync source repo URL - should use dry source repo URL which is permitted
+	app.Spec.SourceHydrator.SyncSource.RepoURL = ""
+	proj := newTestProject()
+	d.EXPECT().GetProcessableAppProj(app).Return(proj, nil).Once()
+	h := &Hydrator{dependencies: d}
+
+	projects, errs := h.validateApplications([]*v1alpha1.Application{app})
+	require.NotNil(t, projects)
+	require.Empty(t, errs)
+}
+
 func TestValidateApplications_DuplicateDestination(t *testing.T) {
 	t.Parallel()
 	d := mocks.NewDependencies(t)
@@ -1094,3 +1152,42 @@ func TestHydrator_getManifests_GetRepoObjsError(t *testing.T) {
 	assert.Empty(t, rev)
 	assert.Nil(t, pathDetails)
 }
+
+func TestGetSyncSource_DifferentRepo(t *testing.T) {
+	hydrator := &v1alpha1.SourceHydrator{
+		DrySource: v1alpha1.DrySource{
+			RepoURL:        "https://example.com/dry-repo",
+			TargetRevision: "main",
+			Path:           "base",
+		},
+		SyncSource: v1alpha1.SyncSource{
+			RepoURL:      "https://example.com/sync-repo",
+			TargetBranch: "hydrated",
+			Path:         "app",
+		},
+	}
+
+	source := hydrator.GetSyncSource()
+	assert.Equal(t, "https://example.com/sync-repo", source.RepoURL)
+	assert.Equal(t, "app", source.Path)
+	assert.Equal(t, "hydrated", source.TargetRevision)
+}
+
+func TestGetSyncSource_FallsBackToDrySourceRepo(t *testing.T) {
+	hydrator := &v1alpha1.SourceHydrator{
+		DrySource: v1alpha1.DrySource{
+			RepoURL:        "https://example.com/dry-repo",
+			TargetRevision: "main",
+			Path:           "base",
+		},
+		SyncSource: v1alpha1.SyncSource{
+			TargetBranch: "hydrated",
+			Path:         "app",
+		},
+	}
+
+	source := hydrator.GetSyncSource()
+	assert.Equal(t, "https://example.com/dry-repo", source.RepoURL)
+	assert.Equal(t, "app", source.Path)
+	assert.Equal(t, "hydrated", source.TargetRevision)
+}

controller/hydrator/types/types.go

@@ -8,4 +8,7 @@ type HydrationQueueKey struct {
 	SourceRepoURL        string
 	SourceTargetRevision string
 	DestinationBranch    string
+	// DestinationRepoURL must be normalized with git.NormalizeGitURL to ensure proper deduplication when apps hydrate
+	// to different repositories.
+	DestinationRepoURL string
 }