Skip to content

Commit 66e8d2a

Browse files
lilithlilith
committed
Overhaul logging, blob instrumentation, fix blob leaks in BoundedTaskCollection, MemoryCache, CacheEngine.
1 parent 1c5da38 commit 66e8d2a

File tree

34 files changed

+662
-199
lines changed

34 files changed

+662
-199
lines changed

‎examples/Imageflow.Server.Example/CustomBlobService.cs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using Imazen.Abstractions.Resulting;
1212
using Microsoft.Extensions.DependencyInjection;
1313
using Microsoft.Extensions.Logging;
14+
using Imazen.Abstractions.Logging;
1415

1516
namespace Imageflow.Server.Example
1617
{
@@ -22,8 +23,8 @@ public static IServiceCollection AddImageflowCustomBlobService(this IServiceColl
2223
{
2324
services.AddSingleton<IBlobWrapperProvider>((container) =>
2425
{
25-
var logger = container.GetRequiredService<ILogger<CustomBlobService>>();
26-
return new CustomBlobService(options, logger);
26+
var loggerFactory = container.GetRequiredService<IReLoggerFactory>();
27+
return new CustomBlobService(options, loggerFactory);
2728
});
2829

2930
return services;
@@ -67,10 +68,15 @@ public class CustomBlobService : IBlobWrapperProvider
6768
{
6869
private readonly BlobServiceClient client;
6970

70-
private CustomBlobServiceOptions options;
71-
public CustomBlobService(CustomBlobServiceOptions options, ILogger<CustomBlobService> logger)
71+
private readonly CustomBlobServiceOptions options;
72+
private readonly IReLoggerFactory loggerFactory;
73+
private readonly IReLogger logger;
74+
75+
public CustomBlobService(CustomBlobServiceOptions options, IReLoggerFactory loggerFactory)
7276
{
7377
this.options = options;
78+
this.loggerFactory = loggerFactory;
79+
this.logger = loggerFactory.CreateReLogger("CustomBlobService");
7480
client = new BlobServiceClient(options.ConnectionString, options.BlobClientOptions);
7581
}
7682

@@ -113,7 +119,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
113119
var blobClient = client.GetBlobContainerClient(container).GetBlobClient(blobKey);
114120
var latencyZone = new LatencyTrackingZone($"azure::blob/{container}", 100);
115121
var s = await blobClient.DownloadAsync();
116-
return CodeResult<IBlobWrapper>.Ok(new BlobWrapper(latencyZone,CustomAzureBlobHelpers.CreateAzureBlob(s)));
122+
return CodeResult<IBlobWrapper>.Ok(new BlobWrapper(latencyZone,CustomAzureBlobHelpers.CreateAzureBlob(s, logger.WithReScopeData("virtualPath", virtualPath))));
117123

118124
}
119125
catch (RequestFailedException e)
@@ -132,7 +138,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
132138
}
133139
internal static class CustomAzureBlobHelpers
134140
{
135-
public static StreamBlob CreateAzureBlob(Response<BlobDownloadInfo> response)
141+
public static StreamBlob CreateAzureBlob(Response<BlobDownloadInfo> response, IReLogger logger)
136142
{
137143
var a = new BlobAttributes()
138144
{
@@ -142,7 +148,7 @@ public static StreamBlob CreateAzureBlob(Response<BlobDownloadInfo> response)
142148

143149
};
144150
var stream = response.Value.Content;
145-
return new StreamBlob(a, stream);
151+
return new StreamBlob(a, stream, logger, null);
146152
}
147153
}
148154
}

‎src/Imageflow.Server.Storage.AzureBlob/AzureBlobHelper.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33
using System.Linq;
44
using Azure.Storage.Blobs.Models;
55
using Imazen.Abstractions.Blobs;
6+
using Imazen.Abstractions.Logging;
67

78
namespace Imageflow.Server.Storage.AzureBlob
89
{
910
internal static class AzureBlobHelper
1011
{
11-
internal static StreamBlob CreateConsumableBlob(AzureBlobStorageReference reference, BlobDownloadStreamingResult r)
12+
internal static StreamBlob CreateConsumableBlob(AzureBlobStorageReference reference, BlobDownloadStreamingResult r, IReLogger? logger)
1213
{
1314
// metadata starting with t_ is a tag
1415

@@ -22,7 +23,7 @@ internal static StreamBlob CreateConsumableBlob(AzureBlobStorageReference refere
2223
StorageTags = r.Details.Metadata.Where(kvp => kvp.Key.StartsWith("t_"))
2324
.Select(kvp => SearchableBlobTag.CreateUnvalidated(kvp.Key.Substring(2), kvp.Value)).ToList()
2425
};
25-
return new StreamBlob(attributes, r.Content, r);
26+
return new StreamBlob(attributes, r.Content, logger?.WithReScopeData("azure",reference.GetFullyQualifiedRepresentation()),r );
2627
}
2728
}
2829
}

‎src/Imageflow.Server.Storage.AzureBlob/AzureBlobService.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class AzureBlobService : IBlobWrapperProvider, IBlobCacheProvider, IBlobW
2121

2222
private readonly IAzureClientFactory<BlobServiceClient> clientFactory;
2323

24+
private readonly IReLogger logger;
2425

2526
public string UniqueName { get; }
2627
public IEnumerable<BlobWrapperPrefixZone> GetPrefixesAndZones()
@@ -40,6 +41,7 @@ public LatencyTrackingZone GetLatencyZone(string virtualPath)
4041
public AzureBlobService(AzureBlobServiceOptions options, IReLoggerFactory loggerFactory, BlobServiceClient defaultClient, IAzureClientFactory<BlobServiceClient> clientFactory)
4142
{
4243
UniqueName = options.UniqueName ?? "azure-blob";
44+
logger = loggerFactory.CreateReLogger(UniqueName);
4345
var nameOrInstance = options.GetOrCreateClient();
4446
if (nameOrInstance.HasValue)
4547
{
@@ -110,7 +112,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
110112
var reference = new AzureBlobStorageReference(containerClient.Uri.AbsoluteUri, key);
111113
var s = await blobClient.DownloadStreamingAsync();
112114
var latencyZone = new LatencyTrackingZone($"azure::blob/{mapping.Container}", 100);
113-
return CodeResult<IBlobWrapper>.Ok(new BlobWrapper(latencyZone,AzureBlobHelper.CreateConsumableBlob(reference, s.Value)));
115+
return CodeResult<IBlobWrapper>.Ok(new BlobWrapper(latencyZone,AzureBlobHelper.CreateConsumableBlob(reference, s.Value, logger)));
114116

115117
}
116118
catch (RequestFailedException e)

‎src/Imageflow.Server.Storage.AzureBlob/Caching/AzureBlobCache.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using Azure;
1212
using Imazen.Abstractions.Blobs;
1313
using Imazen.Abstractions.Resulting;
14+
using Imazen.Abstractions.Logging;
1415

1516
namespace Imageflow.Server.Storage.AzureBlob.Caching
1617
{
@@ -22,15 +23,15 @@ internal class AzureBlobCache : IBlobCache
2223
{
2324
private NamedCacheConfiguration config;
2425
private BlobServiceClient defaultBlobServiceClient;
25-
private ILogger logger;
26+
private IReLogger logger;
2627
private ContainerExistenceCache containerExists;
2728
private Dictionary<BlobGroup, BlobServiceClient> serviceClients;
2829

2930

3031
// https://devblogs.microsoft.com/azure-sdk/best-practices-for-using-azure-sdk-with-asp-net-core/
3132

3233
public AzureBlobCache(NamedCacheConfiguration config, Func<string?, BlobServiceClient> blobServiceFactory,
33-
ILoggerFactory loggerFactory)
34+
IReLoggerFactory loggerFactory)
3435
{
3536
this.config = config;
3637
// Map BlobGroupConfigurations dict, replacing those keys with the .Location.BlobClient value
@@ -40,7 +41,7 @@ public AzureBlobCache(NamedCacheConfiguration config, Func<string?, BlobServiceC
4041
p.Value.Location.AzureClient.Resolve(blobServiceFactory))).ToDictionary(x => x.Key, x => x.Value);
4142

4243
this.defaultBlobServiceClient = blobServiceFactory(null);
43-
this.logger = loggerFactory.CreateLogger("AzureBlobCache");
44+
this.logger = loggerFactory.CreateReLogger("AzureBlobCache");
4445

4546
this.containerExists =
4647
new ContainerExistenceCache(config.BlobGroupConfigurations.Values.Select(x => x.Location.ContainerName));
@@ -167,7 +168,7 @@ public async Task<IResult<IBlobWrapper, IBlobCacheFetchFailure>> CacheFetch(IBlo
167168
{
168169
var response = await blob.DownloadStreamingAsync(new BlobDownloadOptions(), cancellationToken);
169170
containerExists.Set(groupConfig.Location.ContainerName, true);
170-
return BlobCacheFetchFailure.OkResult(new BlobWrapper(null,AzureBlobHelper.CreateConsumableBlob(storage, response)));
171+
return BlobCacheFetchFailure.OkResult(new BlobWrapper(null,AzureBlobHelper.CreateConsumableBlob(storage, response, logger)));
171172

172173
}
173174
catch (Azure.RequestFailedException ex)

‎src/Imageflow.Server.Storage.RemoteReader/DomainProxyService.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
9090
if (!resp.IsSuccessStatusCode)
9191
{
9292
logger?.LogWarning(
93-
"RemoteReader blob {VirtualPath} not found. The remote {Url} responded with status: {StatusCode}",
93+
"DomainProxy blob {VirtualPath} not found. The remote {Url} responded with status: {StatusCode}",
9494
virtualPath, url, resp.StatusCode);
95-
return CodeResult<IBlobWrapper>.Err(HttpStatus.NotFound.WithMessage("RemoteReader blob \"{virtualPath}\" not found. The remote \"{url}\" responded with status: {resp.StatusCode}"));
95+
return CodeResult<IBlobWrapper>.Err(HttpStatus.NotFound.WithMessage("DomainProxy blob \"{virtualPath}\" not found. The remote \"{url}\" responded with status: {resp.StatusCode}"));
9696
}
9797

9898
var attrs = new BlobAttributes()
@@ -106,18 +106,18 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
106106
var stream = await resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
107107

108108
return CodeResult<IBlobWrapper>.Ok(new BlobWrapper(prefix.Zone,
109-
new StreamBlob(attrs, stream, resp)));
109+
new StreamBlob(attrs, stream,logger?.WithReScopeData("virtualPath", virtualPath), resp)));
110110
}
111111
catch (BlobMissingException)
112112
{
113113
throw;
114114
}
115115
catch (Exception ex)
116116
{
117-
logger?.LogWarning(ex, "RemoteReader blob error retrieving {Url} for {VirtualPath}", url,
117+
logger?.LogWarning(ex, "DomainProxy blob error retrieving {Url} for {VirtualPath}", url,
118118
virtualPath);
119119
return CodeResult<IBlobWrapper>.Err(HttpStatus.ServerError
120-
.WithMessage("RemoteReader blob error retrieving \"{url}\" for \"{virtualPath}\".")
120+
.WithMessage("DomainProxy blob error retrieving \"{url}\" for \"{virtualPath}\".")
121121
.WithAppend(ex.Message));
122122
}
123123

‎src/Imageflow.Server.Storage.RemoteReader/RemoteReaderService.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,18 @@ public class RemoteReaderService : IBlobWrapperProviderZoned, IBlobWrapperProvid
2121
private readonly List<string> prefixes = new List<string>();
2222
private readonly IHttpClientFactory httpFactory;
2323
private readonly RemoteReaderServiceOptions options;
24-
private readonly IReLogger<RemoteReaderService> logger;
24+
private readonly IReLogger logger;
25+
private readonly IReLoggerFactory loggerFactory;
2526
private readonly Func<Uri, string> httpClientSelector;
2627

2728
public RemoteReaderService(RemoteReaderServiceOptions options
28-
, IReLogger<RemoteReaderService> logger
29+
, IReLoggerFactory loggerFactory
2930
, IHttpClientFactory httpFactory
3031
)
3132
{
3233
this.options = options;
33-
this.logger = logger;
34+
this.loggerFactory = loggerFactory;
35+
this.logger = loggerFactory.CreateReLogger("RemoteReader");
3436
this.httpFactory = httpFactory;
3537
httpClientSelector = options.HttpClientSelector ?? (_ => "");
3638

@@ -148,7 +150,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
148150
var disposeAfter = resp;
149151
var stream = await resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
150152

151-
return new BlobWrapper(GetLatencyZone(virtualPath), new StreamBlob(attributes, stream, disposeAfter));
153+
return new BlobWrapper(GetLatencyZone(virtualPath), new StreamBlob(attributes, stream, logger?.WithReScopeData("virtualPath", virtualPath), disposeAfter));
152154
}
153155
catch (BlobMissingException)
154156
{

‎src/Imageflow.Server.Storage.S3/Caching/S3BlobCache.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@ internal class S3BlobCache : IBlobCache
2424
private readonly IReLoggerFactory loggerFactory;
2525

2626
private readonly S3LifecycleUpdater lifecycleUpdater;
27+
private readonly IReLogger logger;
2728

2829
public S3BlobCache(NamedCacheConfiguration config, IAmazonS3 s3Client, IReLoggerFactory loggerFactory)
2930
{
31+
this.logger = loggerFactory.CreateReLogger($"S3BlobCache('{config.CacheName}')");
3032
this.config = config;
3133
this.s3Client = s3Client;
3234
this.loggerFactory = loggerFactory;
@@ -184,7 +186,7 @@ public async Task<BlobFetchResult> CacheFetch(IBlobCacheRequest request, Cancell
184186
{
185187
var latencyZone = new LatencyTrackingZone($"s3::bucket/{bucket}", 100);
186188
return BlobCacheFetchFailure.OkResult(
187-
new BlobWrapper(latencyZone,S3BlobHelpers.CreateS3Blob(result)));
189+
new BlobWrapper(latencyZone,S3BlobHelpers.CreateS3Blob(result, logger)));
188190
}
189191

190192
// 404/403 are cache misses and return these

‎src/Imageflow.Server.Storage.S3/S3Blob.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33
using System.Linq;
44
using Amazon.S3.Model;
55
using Imazen.Abstractions.Blobs;
6+
using Imazen.Abstractions.Logging;
67

78
namespace Imageflow.Server.Storage.S3
89
{
910
internal static class S3BlobHelpers
1011
{
11-
public static StreamBlob CreateS3Blob(GetObjectResponse r)
12+
public static StreamBlob CreateS3Blob(GetObjectResponse r, IReLogger logger)
1213
{
1314
if (r.HttpStatusCode != System.Net.HttpStatusCode.OK)
1415
{
@@ -28,7 +29,7 @@ public static StreamBlob CreateS3Blob(GetObjectResponse r)
2829
EstimatedExpiry = r.Expiration?.ExpiryDateUtc,
2930
BlobStorageReference = new S3BlobStorageReference(r.BucketName, r.Key)
3031
};
31-
return new StreamBlob(a, r.ResponseStream, r);
32+
return new StreamBlob(a, r.ResponseStream, logger?.WithReScopeData("s3",a.BlobStorageReference.GetFullyQualifiedRepresentation()), r);
3233
}
3334
}
3435
}

‎src/Imageflow.Server.Storage.S3/S3Service.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Imazen.Abstractions.Blobs.LegacyProviders;
1010
using Imazen.Abstractions.Logging;
1111
using Imazen.Abstractions.Resulting;
12+
using Microsoft.Extensions.Logging;
1213

1314
namespace Imageflow.Server.Storage.S3
1415
{
@@ -18,9 +19,11 @@ public class S3Service : IBlobWrapperProvider, IDisposable, IBlobCacheProvider,
1819
private readonly List<S3BlobCache> namedCaches = [];
1920

2021
private readonly IAmazonS3 s3Client;
22+
private readonly IReLogger logger;
2123

2224
public S3Service(S3ServiceOptions options, IAmazonS3 s3Client, IReLoggerFactory loggerFactory)
2325
{
26+
this.logger = loggerFactory.CreateReLogger("s3");
2427
this.s3Client = s3Client;
2528
UniqueName = options.UniqueName;
2629
foreach (var m in options.Mappings)
@@ -70,6 +73,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
7073
m.IgnorePrefixCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal));
7174
if (mapping.Prefix == null)
7275
{
76+
logger?.LogInformation("No S3 mapping found for virtual path \"{virtualPath}\"", virtualPath);
7377
return CodeResult<IBlobWrapper>.Err((HttpStatus.NotFound, $"No S3 mapping found for virtual path \"{virtualPath}\""));
7478

7579
}
@@ -91,7 +95,7 @@ public async Task<CodeResult<IBlobWrapper>> Fetch(string virtualPath)
9195

9296
var latencyZone = new LatencyTrackingZone($"s3::bucket/{mapping.Bucket}", 100);
9397
var s = await client.GetObjectAsync(req);
94-
return new BlobWrapper(latencyZone,S3BlobHelpers.CreateS3Blob(s));
98+
return new BlobWrapper(latencyZone,S3BlobHelpers.CreateS3Blob(s, logger));
9599

96100
} catch (AmazonS3Exception se) {
97101
if (se.StatusCode == System.Net.HttpStatusCode.NotFound || "NoSuchKey".Equals(se.ErrorCode, StringComparison.OrdinalIgnoreCase))

‎src/Imageflow.Server/Internal/MiddlewareOptionsServerBuilder.cs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ internal class LegacyRoutingEngine : IRoutingEngine
116116

117117
public ValueTask<CodeResult<ICacheableBlobPromise>?> RouteToPromiseAsync(MutableRequest request, CancellationToken cancellationToken = default) => routingEngine.RouteToPromiseAsync(request, cancellationToken);
118118

119+
private readonly IReLogger logger;
119120
public LegacyRoutingEngine(
120121
ImageflowMiddlewareOptions options,
121122
IWebHostEnvironment env,
@@ -129,7 +130,7 @@ IEnumerable<IBlobProvider> blobProviders
129130
#pragma warning restore CS0618
130131
)
131132
{
132-
133+
this.logger = loggerFactory.CreateReLogger("LegacyRoutingEngine");
133134
var mappedPaths = options.MappedPaths.Cast<IPathMapping>().ToList();
134135
if (options.MapWebRoot)
135136
{
@@ -216,11 +217,11 @@ IEnumerable<IBlobProvider> blobProviders
216217
if (mappedPaths.Count > 0)
217218
{
218219
builder.AddMediaLayer(new LocalFilesLayer(mappedPaths.Select(a =>
219-
(IPathMapping)new PathMapping(a.VirtualPath, a.PhysicalPath, a.IgnorePrefixCase)).ToList()));
220+
(IPathMapping)new PathMapping(a.VirtualPath, a.PhysicalPath, a.IgnorePrefixCase)).ToList(), logger));
220221
}
221222

222223

223-
builder.AddMediaLayer(new BlobProvidersLayer(blobProviders, blobWrapperProviders));
224+
builder.AddMediaLayer(new BlobProvidersLayer(blobProviders, blobWrapperProviders, logger));
224225

225226

226227
builder.AddEndpointLayer(diagnosticsPage);
@@ -273,8 +274,6 @@ IEnumerable<IBlobProvider> blobProviders
273274
}
274275
}
275276

276-
var logger = loggerFactory.CreateReLogger("Imageflow.Routing");
277-
278277
routingEngine = builder.Build(logger);
279278
}
280279

0 commit comments

Comments
 (0)