Skip to content

Cleanup and fix the GeneratePackageVersions #8478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bouwkast wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Cleanup and fix the GeneratePackageVersions #8478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 44 additions & 35 deletions tracer/build/_build/Build.Utilities.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ partial class Build
[Parameter("Only update package versions for packages with the following names")]
readonly string[] IncludePackages;

[Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2 days, or 0 when --IncludePackages is set")]
[Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2. Ignored for packages named in --IncludePackages, which always bypass the cooldown.")]
readonly int? PackageVersionCooldownDays;

[LazyLocalExecutable(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\gacutil.exe")]
Expand Down Expand Up @@ -235,53 +235,64 @@ partial class Build
var definitionsFile = BuildDirectory / FileNames.DefinitionsJson;
var supportedVersionsPath = BuildDirectory / "supported_versions.json";

// Build the shouldQueryNuGet predicate from include/exclude filters
Func<string, bool> shouldUpdatePackage = (IncludePackages, ExcludePackages) switch
// Decides the cooldown treatment for each package by name. See CooldownMode for what each value does.
var getCooldownMode = BuildCooldownModeSelector(IncludePackages, ExcludePackages);

// Dependabot re-uses the previous entry verbatim for Freeze; Skip and Normal both refresh.
Func<string, bool> shouldUpdatePackage = name => getCooldownMode(name) is not CooldownMode.Freeze;

static Func<string, CooldownMode> BuildCooldownModeSelector(string[] includePackages, string[] excludePackages)
{
({ } include, _) => name => include.Contains(name, StringComparer.OrdinalIgnoreCase),
(_, { } exclude) => name => !exclude.Contains(name, StringComparer.OrdinalIgnoreCase),
_ => _ => true
};

// Load caches for both pipelines
var cacheFilePath = BuildDirectory / "nuget_version_cache.json";
var previousVersionCache = await NuGetVersionCache.Load(cacheFilePath);
Logger.Information("Loaded NuGet version cache with {Count} entries", previousVersionCache.Count);
// No filter: every package goes through the normal cooldown filter.
if (includePackages is null && excludePackages is null)
{
return _ => CooldownMode.Normal;
}

// --IncludePackages Foo Bar: update only the listed packages (bypassing cooldown);
// freeze every other package so it re-emits its previous output unchanged.
if (includePackages is not null)
{
var targeted = new HashSet<string>(includePackages, StringComparer.OrdinalIgnoreCase);
return name => targeted.Contains(name) ? CooldownMode.BypassCooldown : CooldownMode.Freeze;
}

// --ExcludePackages Foo Bar: freeze the listed packages; everything else updates normally.
var blocked = new HashSet<string>(excludePackages, StringComparer.OrdinalIgnoreCase);
return name => blocked.Contains(name) ? CooldownMode.Freeze : CooldownMode.Normal;
}

var previousSupportedVersions = await GenerateSupportMatrix.LoadPreviousVersions(supportedVersionsPath);
Logger.Information("Loaded previous supported versions with {Count} entries", previousSupportedVersions.Count);

// Derive baseline from supported_versions.json: the max tested version per package
// acts as a floor to prevent cooldown filtering from downgrading previously accepted versions.
// We collect all max tested versions per package (not just the global max) so that
// split-range packages (e.g., GraphQL 4.x-6.x and 7.x-9.x) get a per-range baseline.
var baseline = previousSupportedVersions
// Pull the max tested version per package from supported_versions.json. The cooldown
// filter keeps anything at or below this value so we never downgrade a version we
// already shipped against. Collected as a list per package (not a single global max)
// so split-range packages (e.g. GraphQL 4.x-6.x and 7.x-9.x) get a per-range value.
var previousMaxVersions = previousSupportedVersions
.Where(kvp => kvp.Value.MaxVersionTestedInclusive is not null)
.GroupBy(kvp => kvp.Key.PackageName)
.ToDictionary(
g => g.Key,
g => g.Select(kvp => new Version(kvp.Value.MaxVersionTestedInclusive!)).ToList());
Logger.Information("Derived version baseline with {Count} entries from supported_versions.json", baseline.Count);
Logger.Information("Loaded previous max tested versions for {Count} packages from supported_versions.json", previousMaxVersions.Count);

// Resolve effective cooldown:
// - Explicit --PackageVersionCooldownDays wins
// - --IncludePackages without explicit cooldown defaults to 0
var effectiveCooldownDays = PackageVersionCooldownDays ?? (IncludePackages is not null ? 0 : 2);
var effectiveCooldownDays = PackageVersionCooldownDays ?? 2;

// Pipeline A: generate .g.props/.g.cs files
Logger.Information("Using package version cooldown of {Days} days", effectiveCooldownDays);
var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, shouldUpdatePackage, previousVersionCache, effectiveCooldownDays, baseline);
var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, getCooldownMode, effectiveCooldownDays, previousMaxVersions);
var testedVersions = await versionGenerator.GenerateVersions(Solution);
await NuGetVersionCache.Save(cacheFilePath, versionGenerator.VersionCache);

// Log version changes: bumps, unchanged, and overridden
var versionCache = versionGenerator.VersionCache;
var queriedVersions = versionGenerator.QueriedVersions;
var bumped = 0;
var unchanged = 0;
foreach (var tested in testedVersions)
{
var packageName = tested.NugetPackageSearchName;
baseline.TryGetValue(packageName, out var previousMaxVersions);
var previousMax = previousMaxVersions?
previousMaxVersions.TryGetValue(packageName, out var previousMaxCandidates);
var previousMax = previousMaxCandidates?
.Where(v => v >= tested.MinVersion && v <= tested.MaxVersion)
.OrderByDescending(v => v)
.FirstOrDefault();
Expand All @@ -290,12 +301,12 @@ partial class Build
{
bumped++;
var publishedDate = "(unknown)";
if (versionCache.TryGetValue(packageName, out var cachedVersions))
if (queriedVersions.TryGetValue(packageName, out var versionsForPackage))
{
var match = cachedVersions.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
var match = versionsForPackage.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
if (match?.Published is not null)
{
publishedDate = match.Published.Value.ToString("yyyy-MM-dd");
publishedDate = match.Published.Value.UtcDateTime.ToString("yyyy-MM-dd");
}
}

Expand Down Expand Up @@ -325,13 +336,11 @@ partial class Build

foreach (var entry in versionGenerator.CooldownReport.Entries)
{
var resolvedText = entry.ResolvedVersion is not null ? $"using: {entry.ResolvedVersion}" : "skipped";
Logger.Warning(
" {Package} {Version} overridden (published {Date}, {Resolved})",
" {Package} {Version} overridden (published {Date})",
entry.PackageName,
entry.OverriddenVersion,
entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown",
resolvedText);
entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown");
}

var reportPath = TemporaryDirectory / "cooldown_report.md";
Expand All @@ -347,7 +356,7 @@ partial class Build
var integrations = GenerateIntegrationDefinitions.GetAllIntegrations(assemblies, definitionsFile);

// Pipeline B: generate dependabot files + supported_versions.json
// TestedVersions are cooldown-filtered but the baseline prevents downgrades,
// TestedVersions are cooldown-filtered but the previous max pins prevent downgrades,
// so they accurately reflect what we're testing.
var distinctIntegrations = await DependabotFileManager.BuildDistinctIntegrationMaps(
integrations, testedVersions, shouldUpdatePackage, previousSupportedVersions);
Expand Down
33 changes: 33 additions & 0 deletions tracer/build/_build/GeneratePackageVersions/CooldownMode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// <copyright file="CooldownMode.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

namespace GeneratePackageVersions;

/// <summary>
/// Controls how cooldown filtering is applied to a package.
/// </summary>
public enum CooldownMode
{
/// <summary>
/// Normal cooldown: versions published within the cooldown window are dropped, unless they
/// are at or below the highest version we already tested in a prior run for this entry's range
/// (so cooldown never downgrades a version we already shipped against).
/// </summary>
Normal,

/// <summary>
/// Bypass the cooldown filter: every NuGet version in this entry's range is accepted,
/// including those published within the cooldown window. Used for packages explicitly
/// targeted via --IncludePackages.
/// </summary>
BypassCooldown,

/// <summary>
/// Re-emit the previous run's versions verbatim. The generators are not re-run against NuGet;
/// the previous .g.cs file is parsed and its versions are fed back through Write(). Used for
/// non-targeted packages when --IncludePackages or --ExcludePackages filters are active.
/// </summary>
Freeze,
}
20 changes: 6 additions & 14 deletions tracer/build/_build/GeneratePackageVersions/CooldownReport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ public class CooldownReport
{
private readonly int _cooldownDays;
private readonly List<CooldownEntry> _entries = new();
private readonly HashSet<(string PackageName, string Version)> _seen = new();

public CooldownReport(int cooldownDays)
{
Expand All @@ -32,12 +31,7 @@ public CooldownReport(int cooldownDays)

public void Add(CooldownEntry entry)
{
// Deduplicate: the same version gets flagged once per framework and per selection group,
// but we only need to report it once.
if (_seen.Add((entry.PackageName, entry.OverriddenVersion)))
{
_entries.Add(entry);
}
_entries.Add(entry);
}

public string ToMarkdown()
Expand All @@ -53,18 +47,17 @@ public string ToMarkdown()
sb.AppendLine($"The following versions were published less than **{_cooldownDays} days** ago and have been overridden.");
sb.AppendLine("These require manual review before inclusion.");
sb.AppendLine();
sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) | Using Instead |");
sb.AppendLine("|---------|-------------|--------------------|-----------|------------|---------------|");
sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) |");
sb.AppendLine("|---------|-------------|--------------------|-----------|------------|");

foreach (var entry in _entries)
{
var published = entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown";
var published = entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown";
var age = entry.PublishedDate.HasValue
? ((int)(DateTimeOffset.UtcNow - entry.PublishedDate.Value).TotalDays).ToString()
: "?";
var usingInstead = entry.ResolvedVersion ?? "(skipped)";

sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} | {usingInstead} |");
sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} |");
}

return sb.ToString();
Expand All @@ -84,6 +77,5 @@ public record CooldownEntry(
string PackageName,
string IntegrationName,
string OverriddenVersion,
DateTimeOffset? PublishedDate,
string ResolvedVersion);
DateTimeOffset? PublishedDate);
}
60 changes: 0 additions & 60 deletions tracer/build/_build/GeneratePackageVersions/NuGetVersionCache.cs
View file
Open in desktop

This file was deleted.

Loading
Loading