refactor(project): Add basic handling for resource tags

* Add new class 'ProjectResources'
* Add new ProjectBuildCache lifecycle hook "buildFinished"
    * This is used to clear build-tags which must not be visible to
      dependant projects (maybe we can find a better solution here)
    * ProjectBuilder must read the tags before triggering the hook

Still missing: Integrate resource tags into hash tree for correct
invalidation

Author: RandomByte

packages/project/lib/build/ProjectBuilder.js

@@ -208,15 +208,15 @@ class ProjectBuilder {
 			});
 		}
 		const pWrites = [];
-		await this.#build(requestedProjects, (projectName, project, projectBuildContext) => {
+		await this.#build(requestedProjects, async (projectName, project, projectBuildContext) => {
 			if (!fsTarget) {
 				// Nothing to write to
 				return;
 			}
 			// Only write requested projects to target
 			// (excluding dependencies that were required to be built, but not requested)
 			this.#log.verbose(`Writing out files for project ${projectName}...`);
-			pWrites.push(this._writeResults(projectBuildContext, fsTarget));
+			await this._writeResults(projectBuildContext, fsTarget, pWrites);
 		});
 		await Promise.all(pWrites);
 	}
@@ -329,13 +329,15 @@ class ProjectBuilder {
 				signal?.throwIfAborted();
 
 				if (projectBuiltCallback && requestedProjects.includes(projectName)) {
-					projectBuiltCallback(projectName, project, projectBuildContext);
+					await projectBuiltCallback(projectName, project, projectBuildContext);
 				}
 
 				if (!alreadyBuilt.includes(projectName) && !process.env.UI5_BUILD_NO_WRITE_CACHE) {
 					this.#log.verbose(`Triggering cache update for project ${projectName}...`);
 					pCacheWrites.push(projectBuildContext.writeBuildCache());
 				}
+
+				projectBuildContext.buildFinished();
 			}
 			this.#log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`);
 		} catch (err) {
@@ -434,9 +436,10 @@ class ProjectBuilder {
 	 *
 	 * @param {object} projectBuildContext Build context for the project
 	 * @param {@ui5/fs/adapters/FileSystem} target Target adapter to write to
+	 * @param {Array<Promise>} deferredWork
 	 * @returns {Promise<void>} Promise resolving when write is complete
 	 */
-	async _writeResults(projectBuildContext, target) {
+	async _writeResults(projectBuildContext, target, deferredWork) {
 		const project = projectBuildContext.getProject();
 		const taskUtil = projectBuildContext.getTaskUtil();
 		const buildConfig = this._buildContext.getBuildConfig();
@@ -472,7 +475,21 @@ class ProjectBuilder {
 			}));
 		}
 
-		await Promise.all(resources.map((resource) => {
+		const resourcesToWrite = resources.filter((resource) => {
+			if (taskUtil.getTag(resource, taskUtil.STANDARD_TAGS.OmitFromBuildResult)) {
+				this.#log.silly(`Skipping resource tagged as "OmitFromBuildResult": ` +
+					resource.getPath());
+				return false; // Skip this resource
+			}
+			return true;
+		});
+
+		deferredWork.push(
+			this._writeToDisk(resourcesToWrite, target, resources, taskUtil, project, isRootProject, outputStyle));
+	}
+
+	async _writeToDisk(resourcesToWrite, target, resources, taskUtil, project, isRootProject, outputStyle) {
+		await Promise.all(resourcesToWrite.map((resource) => {
 			if (taskUtil.getTag(resource, taskUtil.STANDARD_TAGS.OmitFromBuildResult)) {
 				this.#log.silly(`Skipping write of resource tagged as "OmitFromBuildResult": ` +
 					resource.getPath());
@@ -483,12 +500,14 @@ class ProjectBuilder {
 
 		if (isRootProject &&
 			outputStyle === OutputStyleEnum.Flat &&
-			project.getType() !== "application" /* application type is with a default flat build output structure */) {
+			/* application type is with a default flat build output structure */
+			project.getType() !== "application") {
 			const namespace = project.getNamespace();
 			const libraryResourcesPrefix = `/resources/${namespace}/`;
 			const testResourcesPrefix = "/test-resources/";
 			const namespacedRegex = new RegExp(`/(resources|test-resources)/${namespace}`);
-			const processedResourcesSet = resources.reduce((acc, resource) => acc.add(resource.getPath()), new Set());
+			const processedResourcesSet = resources.reduce(
+				(acc, resource) => acc.add(resource.getPath()), new Set());
 
 			// If outputStyle === "Flat", then the FlatReader would have filtered
 			// some resources. We now need to get all of the available resources and
@@ -507,7 +526,8 @@ class ProjectBuilder {
 			skippedResources.forEach((resource) => {
 				if (resource.originalPath.startsWith(testResourcesPrefix)) {
 					this.#log.verbose(
-						`Omitting ${resource.originalPath} from build result. File is part of ${testResourcesPrefix}.`
+						`Omitting ${resource.originalPath} from build result. ` +
+					`File is part of ${testResourcesPrefix}.`
 					);
 				} else if (!resource.originalPath.startsWith(libraryResourcesPrefix)) {
 					this.#log.warn(

packages/project/lib/build/cache/CacheManager.js

@@ -20,7 +20,7 @@ const chacheManagerInstances = new Map();
 const CACACHE_OPTIONS = {algorithms: ["sha256"]};
 
 // Cache version for compatibility management
-const CACHE_VERSION = "v0_1";
+const CACHE_VERSION = "v0_2";
 
 /**
  * Manages persistence for the build cache using file-based storage and cacache

packages/project/lib/build/cache/ProjectBuildCache.js

@@ -36,6 +36,10 @@ export const RESULT_CACHE_STATES = Object.freeze({
  * @property {string} signature Signature of the cached stage
  * @property {@ui5/fs/AbstractReader} stage Reader for the cached stage
  * @property {string[]} writtenResourcePaths Array of resource paths written by the task
+ * @property {Map<string, Map<string, {string|number|boolean|undefined}>>} projectTagOperations
+ * Map of resource paths to their tags that were set or cleared during this stage's execution, for project tags
+ * @property {Map<string, Map<string, {string|number|boolean|undefined}>>} buildTagOperations
+ * Map of resource paths to their tags that were set or cleared during this stage's execution, for build tags
  */
 
 export default class ProjectBuildCache {
@@ -116,6 +120,7 @@ export default class ProjectBuildCache {
 	 */
 	async prepareProjectBuildAndValidateCache(dependencyReader) {
 		this.#currentProjectReader = this.#project.getReader();
+
 		this.#currentDependencyReader = dependencyReader;
 
 		if (this.#combinedIndexState === INDEX_STATES.INITIAL) {
@@ -295,9 +300,9 @@ export default class ProjectBuildCache {
 	 */
 	async #importStages(stageSignatures) {
 		const stageNames = Object.keys(stageSignatures);
-		if (this.#project.getStage()?.getId() === "initial") {
+		if (this.#project.getProjectResources().getStage()?.getId() === "initial") {
 			// Only initialize stages once
-			this.#project.initStages(stageNames);
+			this.#project.getProjectResources().initStages(stageNames);
 		}
 		const importedStages = await Promise.all(stageNames.map(async (stageName) => {
 			const stageSignature = stageSignatures[stageName];
@@ -308,13 +313,14 @@ export default class ProjectBuildCache {
 			}
 			return [stageName, stageCache];
 		}));
-		this.#project.useResultStage();
+		this.#project.getProjectResources().useResultStage();
 		const writtenResourcePaths = new Set();
 		for (const [stageName, stageCache] of importedStages) {
 			// Check whether the stage differs form the one currently in use
 			if (this.#currentStageSignatures.get(stageName)?.join("-") !== stageCache.signature) {
 				// Set stage
-				this.#project.setStage(stageName, stageCache.stage);
+				this.#project.getProjectResources().setStage(stageName, stageCache.stage,
+					stageCache.projectTagOperations, stageCache.buildTagOperations);
 
 				// Store signature for later use in result stage signature calculation
 				this.#currentStageSignatures.set(stageName, stageCache.signature.split("-"));
@@ -387,7 +393,7 @@ export default class ProjectBuildCache {
 		// Store current project reader (= state of the previous stage) for later use (e.g. in recordTaskResult)
 		this.#currentProjectReader = this.#project.getReader();
 		// Switch project to new stage
-		this.#project.useStage(stageName);
+		this.#project.getProjectResources().useStage(stageName);
 		log.verbose(`Preparing task execution for task ${taskName} in project ${this.#project.getName()}...`);
 		if (!taskCache) {
 			log.verbose(`No task cache found`);
@@ -420,7 +426,8 @@ export default class ProjectBuildCache {
 		const stageCache = await this.#findStageCache(stageName, stageSignatures);
 		const oldStageSig = this.#currentStageSignatures.get(stageName)?.join("-");
 		if (stageCache) {
-			this.#project.setStage(stageName, stageCache.stage);
+			this.#project.getProjectResources().setStage(stageName, stageCache.stage,
+				stageCache.projectTagOperations, stageCache.buildTagOperations);
 
 			// Check whether the stage actually changed
 			if (stageCache.signature !== oldStageSig) {
@@ -541,7 +548,7 @@ export default class ProjectBuildCache {
 				return;
 			}
 			log.verbose(`Found cached stage with signature ${stageSignature}`);
-			const {resourceMapping, resourceMetadata} = stageMetadata;
+			const {resourceMapping, resourceMetadata, projectTagOperations, buildTagOperations} = stageMetadata;
 			let writtenResourcePaths;
 			let stageReader;
 			if (resourceMapping) {
@@ -570,10 +577,13 @@ export default class ProjectBuildCache {
 				writtenResourcePaths = Object.keys(resourceMetadata);
 				stageReader = this.#createReaderForStageCache(stageName, stageSignature, resourceMetadata);
 			}
+
 			return {
 				signature: stageSignature,
 				stage: stageReader,
 				writtenResourcePaths,
+				projectTagOperations: tagOpsToMap(projectTagOperations),
+				buildTagOperations: tagOpsToMap(buildTagOperations),
 			};
 		}));
 		return stageCache;
@@ -610,10 +620,12 @@ export default class ProjectBuildCache {
 		const taskCache = this.#taskCache.get(taskName);
 
 		// Identify resources written by task
-		const stage = this.#project.getStage();
+		const stage = this.#project.getProjectResources().getStage();
 		const stageWriter = stage.getWriter();
 		const writtenResources = await stageWriter.byGlob("/**/*");
 		const writtenResourcePaths = writtenResources.map((res) => res.getOriginalPath());
+		const {projectTagOperations, buildTagOperations} =
+			this.#project.getProjectResources().getResourceTagOperations();
 
 		let stageSignature;
 		if (cacheInfo) {
@@ -654,8 +666,8 @@ export default class ProjectBuildCache {
 
 		// Store resulting stage in stage cache
 		this.#stageCache.addSignature(
-			this.#getStageNameForTask(taskName), stageSignature, this.#project.getStage(),
-			writtenResourcePaths);
+			this.#getStageNameForTask(taskName), stageSignature, this.#project.getProjectResources().getStage(),
+			writtenResourcePaths, projectTagOperations, buildTagOperations);
 
 		// Update task cache with new metadata
 		log.verbose(`Task ${taskName} produced ${writtenResourcePaths.length} resources`);
@@ -733,7 +745,7 @@ export default class ProjectBuildCache {
 	 */
 	async setTasks(taskNames) {
 		const stageNames = taskNames.map((taskName) => this.#getStageNameForTask(taskName));
-		this.#project.initStages(stageNames);
+		this.#project.getProjectResources().initStages(stageNames);
 
 		// TODO: Rename function? We simply use it to have a point in time right before the project is built
 	}
@@ -749,7 +761,7 @@ export default class ProjectBuildCache {
 	 * @returns {Promise<string[]>} Array of changed resource paths since the last build
 	 */
 	async allTasksCompleted() {
-		this.#project.useResultStage();
+		this.#project.getProjectResources().useResultStage();
 		if (this.#combinedIndexState === INDEX_STATES.INITIAL) {
 			this.#combinedIndexState = INDEX_STATES.FRESH;
 		}
@@ -763,6 +775,10 @@ export default class ProjectBuildCache {
 		return changedPaths;
 	}
 
+	buildFinished() {
+		this.#project.getProjectResources().buildFinished();
+	}
+
 	/**
 	 * Generates the stage name for a given task
 	 *
@@ -947,7 +963,8 @@ export default class ProjectBuildCache {
 			`with build signature ${this.#buildSignature}`);
 		const stageQueue = this.#stageCache.flushCacheQueue();
 		await Promise.all(stageQueue.map(async ([stageId, stageSignature]) => {
-			const {stage} = this.#stageCache.getCacheForSignature(stageId, stageSignature);
+			const {stage, projectTagOperations, buildTagOperations} =
+				this.#stageCache.getCacheForSignature(stageId, stageSignature);
 			const writer = stage.getWriter();
 
 			let metadata;
@@ -974,7 +991,8 @@ export default class ProjectBuildCache {
 				const resourceMetadata = await this.#writeStageResources(resources, stageId, stageSignature);
 				metadata = {resourceMetadata};
 			}
-
+			metadata.projectTagOperations = tagOpsToObject(projectTagOperations);
+			metadata.buildTagOperations = tagOpsToObject(buildTagOperations);
 			await this.#cacheManager.writeStageCache(
 				this.#project.getId(), this.#buildSignature, stageId, stageSignature, metadata);
 		}));
@@ -1183,3 +1201,23 @@ function createStageSignature(projectSignature, dependencySignature) {
 function createDependencySignature(stageDependencySignatures) {
 	return crypto.createHash("sha256").update(stageDependencySignatures.join("")).digest("hex");
 }
+
+function tagOpsToMap(tagOps) {
+	const map = new Map();
+	for (const [resourcePath, tags] of Object.entries(tagOps)) {
+		map.set(resourcePath, new Map(Object.entries(tags)));
+	}
+	return map;
+}
+
+/**
+ * @param {Map<string, Map<string, {string|number|boolean|undefined}>>} tagOps
+ * Map of resource paths to their tag operations
+ */
+function tagOpsToObject(tagOps) {
+	const obj = Object.create(null);
+	for (const [resourcePath, tags] of tagOps.entries()) {
+		obj[resourcePath] = Object.fromEntries(tags.entries());
+	}
+	return obj;
+}

packages/project/lib/build/cache/StageCache.js

@@ -2,6 +2,8 @@
  * @typedef {object} StageCacheEntry
  * @property {object} stage The cached stage instance (typically a reader or writer)
  * @property {string[]} writtenResourcePaths Array of resource paths written during stage execution
+ * @property {Map<string, Map<string, {string|number|boolean|undefined}>>} resourceTagOperations
+ *  Map of resource paths to their tags that were set or cleared during this stage's execution
  */
 
 /**
@@ -40,8 +42,11 @@ export default class StageCache {
 	 * @param {string} signature Content hash signature of the stage's input resources
 	 * @param {object} stageInstance The stage instance to cache (typically a reader or writer)
 	 * @param {string[]} writtenResourcePaths Array of resource paths written during this stage
+	 * @param {Map<string, Map<string, {string|number|boolean|undefined}>>} projectTagOperations
+	 * @param {Map<string, Map<string, {string|number|boolean|undefined}>>} buildTagOperations
+	 *  Map of resource paths to their tags that were set or cleared during this stage's execution
 	 */
-	addSignature(stageId, signature, stageInstance, writtenResourcePaths) {
+	addSignature(stageId, signature, stageInstance, writtenResourcePaths, projectTagOperations, buildTagOperations) {
 		if (!this.#stageIdToSignatures.has(stageId)) {
 			this.#stageIdToSignatures.set(stageId, new Map());
 		}
@@ -50,6 +55,8 @@ export default class StageCache {
 			signature,
 			stage: stageInstance,
 			writtenResourcePaths,
+			projectTagOperations,
+			buildTagOperations,
 		});
 		this.#cacheQueue.push([stageId, signature]);
 	}

packages/project/lib/build/helpers/ProjectBuildContext.js

@@ -1,4 +1,3 @@
-import ResourceTagCollection from "@ui5/fs/internal/ResourceTagCollection";
 import ProjectBuildLogger from "@ui5/logger/internal/loggers/ProjectBuild";
 import TaskUtil from "./TaskUtil.js";
 import TaskRunner from "../TaskRunner.js";
@@ -40,11 +39,6 @@ class ProjectBuildContext {
 		this._queues = {
 			cleanup: []
 		};
-
-		this._resourceTagCollection = new ResourceTagCollection({
-			allowedTags: ["ui5:OmitFromBuildResult", "ui5:IsBundle"],
-			allowedNamespaces: ["build"]
-		});
 	}
 
 	/**
@@ -178,18 +172,8 @@ class ProjectBuildContext {
 		if (!resource.hasProject()) {
 			this._log.silly(`Associating resource ${resource.getPath()} with project ${this._project.getName()}`);
 			resource.setProject(this._project);
-			// throw new Error(
-			// 	`Unable to get tag collection for resource ${resource.getPath()}: ` +
-			// 	`Resource must be associated to a project`);
-		}
-		const projectCollection = resource.getProject().getResourceTagCollection();
-		if (projectCollection.acceptsTag(tag)) {
-			return projectCollection;
-		}
-		if (this._resourceTagCollection.acceptsTag(tag)) {
-			return this._resourceTagCollection;
 		}
-		throw new Error(`Could not find collection for resource ${resource.getPath()} and tag ${tag}`);
+		return resource.getProject().getResourceTagCollection(resource, tag);
 	}
 
 	/**
@@ -288,6 +272,11 @@ class ProjectBuildContext {
 		// Propagate changed paths to dependents
 		this.propagateResourceChanges(changedPaths);
 	}
+
+	buildFinished() {
+		this.getBuildCache().buildFinished();
+	}
+
 	/**
 	 * Informs the build cache about changed project source resources
 	 *

packages/project/lib/build/helpers/TaskUtil.js

@@ -35,10 +35,10 @@ class TaskUtil {
 	 * 		This tag identifies resources that contain (i.e. bundle) multiple other resources
 	 * @property {string} IsDebugVariant
 	 * 		This tag identifies resources that are a debug variant (typically named with a "-dbg" suffix)
-	 * 		of another resource. This tag is part of the build manifest.
+	 * 		of another resource. This tag is visible to other projects
 	 * @property {string} HasDebugVariant
 	 * 		This tag identifies resources for which a debug variant has been created.
-	 * 		This tag is part of the build manifest.
+	 * 		This tag is visible to other projects
 	 */
 
 	/**