Skip to content

Commit 5c0be42

Browse files
sfc-gh-mhofmansfc-gh-mhofman
authored
SNOW-2882054 fix for uploading azure metadata (#1288)
1 parent bf370b0 commit 5c0be42

File tree

4 files changed

+107
-13
lines changed

4 files changed

+107
-13
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#### For the official .NET Release Notes please refer to https://docs.snowflake.com/en/release-notes/clients-drivers/dotnet
22

33
# Changelog
4+
- v5.2.1
5+
- Bug fix: Fix the extremely rare case where intermittent network issues during uploads to Azure Blob Storage prevent metadata updates
46
- v5.2.0
57
- Added multi-targeting support. The appropriate build is selected by NuGet based on target framework and OS.
68
- Fixed CRL validation to reject newly downloaded CRLs if their NextUpdate has already expired.

‎Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ public void TestExtractBucketNameAndPath()
100100
[TestCase(HttpStatusCode.BadRequest, ResultStatus.RENEW_TOKEN)]
101101
[TestCase(HttpStatusCode.NotFound, ResultStatus.NOT_FOUND_FILE)]
102102
[TestCase(HttpStatusCode.Forbidden, ResultStatus.ERROR)] // Any error that isn't the above will return ResultStatus.ERROR
103+
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
104+
[TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)]
105+
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
106+
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.ERROR)]
103107
public void TestGetFileHeader(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
104108
{
105109
// Arrange
@@ -190,6 +194,8 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File
190194
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
191195
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
192196
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
197+
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
198+
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
193199
public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
194200
{
195201
// Arrange
@@ -202,7 +208,7 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
202208
.Returns<string>((blobName) =>
203209
{
204210
var mockBlobClient = new Mock<BlobClient>();
205-
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
211+
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<BlobUploadOptions>(), It.IsAny<CancellationToken>()))
206212
.Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key));
207213

208214
return mockBlobClient.Object;
@@ -234,6 +240,9 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
234240
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
235241
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
236242
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
243+
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
244+
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
245+
[TestCase(HttpStatusCode.TemporaryRedirect, ResultStatus.ERROR)]
237246
public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
238247
{
239248
// Arrange
@@ -246,7 +255,7 @@ public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatu
246255
.Returns<string>((blobName) =>
247256
{
248257
var mockBlobClient = new Mock<BlobClient>();
249-
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
258+
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<BlobUploadOptions>(), It.IsAny<CancellationToken>()))
250259
.Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false));
251260

252261
return mockBlobClient.Object;
@@ -287,6 +296,9 @@ private void AssertForUploadFileTests(ResultStatus expectedResultStatus)
287296
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
288297
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
289298
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
299+
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
300+
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
301+
[TestCase(HttpStatusCode.NotFound, ResultStatus.ERROR)]
290302
public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
291303
{
292304
// Arrange
@@ -334,6 +346,9 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte
334346
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
335347
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
336348
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
349+
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
350+
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
351+
[TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)]
337352
public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
338353
{
339354
// Arrange

‎Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs‎

Lines changed: 87 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,13 @@ public FileHeader GetFileHeader(SFFileMetadata fileMetadata)
110110
}
111111
catch (RequestFailedException ex)
112112
{
113-
fileMetadata = HandleFileHeaderErr(ex, fileMetadata);
113+
HandleFileHeaderErr(ex, fileMetadata);
114+
return null;
115+
}
116+
catch (Exception ex)
117+
{
118+
Logger.Error("Blob client unknown get file header error: " + ex.Message);
119+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
114120
return null;
115121
}
116122

@@ -139,7 +145,13 @@ public async Task<FileHeader> GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca
139145
}
140146
catch (RequestFailedException ex)
141147
{
142-
fileMetadata = HandleFileHeaderErr(ex, fileMetadata);
148+
HandleFileHeaderErr(ex, fileMetadata);
149+
return null;
150+
}
151+
catch (Exception ex)
152+
{
153+
Logger.Error("Blob client unknown get file header error: " + ex.Message);
154+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
143155
return null;
144156
}
145157

@@ -168,6 +180,11 @@ internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, Bl
168180
};
169181
}
170182

183+
if (fileMetadata.stageInfo.isClientSideEncrypted && encryptionMetadata == null)
184+
{
185+
Logger.Error("File is expected to be client-side encrypted but no encryption metadata found.");
186+
}
187+
171188
return new FileHeader
172189
{
173190
digest = GetMetadataValueCaseInsensitive(response, "sfcdigest", false),
@@ -215,12 +232,22 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
215232
{
216233
// Issue the POST/PUT request
217234
fileBytesStream.Position = 0;
218-
blobClient.Upload(fileBytesStream, overwrite: true);
219-
blobClient.SetMetadata(metadata);
235+
var uploadOptions = new BlobUploadOptions
236+
{
237+
Metadata = metadata
238+
};
239+
blobClient.Upload(fileBytesStream, uploadOptions);
220240
}
221241
catch (RequestFailedException ex)
222242
{
223-
fileMetadata = HandleUploadFileErr(ex, fileMetadata);
243+
Logger.Error("Blob client request upload error: " + ex.Message);
244+
HandleUploadFileErr(ex, fileMetadata);
245+
return;
246+
}
247+
catch (Exception ex)
248+
{
249+
Logger.Error("Blob client unknown upload error: " + ex.Message);
250+
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
224251
return;
225252
}
226253

@@ -245,12 +272,22 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS
245272
{
246273
// Issue the POST/PUT request
247274
fileBytesStream.Position = 0;
248-
await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false);
249-
blobClient.SetMetadata(metadata);
275+
var uploadOptions = new BlobUploadOptions
276+
{
277+
Metadata = metadata
278+
};
279+
await blobClient.UploadAsync(fileBytesStream, uploadOptions, cancellationToken).ConfigureAwait(false);
250280
}
251281
catch (RequestFailedException ex)
252282
{
253-
fileMetadata = HandleUploadFileErr(ex, fileMetadata);
283+
Logger.Error("Blob client request upload error: " + ex.Message);
284+
HandleUploadFileErr(ex, fileMetadata);
285+
return;
286+
}
287+
catch (Exception ex)
288+
{
289+
Logger.Error("Blob client unknown upload error: " + ex.Message);
290+
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
254291
return;
255292
}
256293

@@ -331,7 +368,14 @@ public void DownloadFile(SFFileMetadata fileMetadata, string fullDstPath, int ma
331368
catch (RequestFailedException ex)
332369
{
333370
File.Delete(fullDstPath);
334-
fileMetadata = HandleDownloadFileErr(ex, fileMetadata);
371+
HandleDownloadFileErr(ex, fileMetadata);
372+
return;
373+
}
374+
catch (Exception ex)
375+
{
376+
File.Delete(fullDstPath);
377+
Logger.Error("Blob client unknown download error: " + ex.Message);
378+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
335379
return;
336380
}
337381

@@ -364,7 +408,14 @@ public async Task DownloadFileAsync(SFFileMetadata fileMetadata, string fullDstP
364408
catch (RequestFailedException ex)
365409
{
366410
File.Delete(fullDstPath);
367-
fileMetadata = HandleDownloadFileErr(ex, fileMetadata);
411+
HandleDownloadFileErr(ex, fileMetadata);
412+
return;
413+
}
414+
catch (Exception ex)
415+
{
416+
File.Delete(fullDstPath);
417+
Logger.Error("Blob client unknown download error: " + ex.Message);
418+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
368419
return;
369420
}
370421

@@ -383,27 +434,47 @@ private SFFileMetadata HandleFileHeaderErr(RequestFailedException ex, SFFileMeta
383434
}
384435
else
385436
{
437+
Logger.Error($"Unexpected HTTP status for file header operation: {ex.Status} {ex.ErrorCode}");
386438
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
387439
}
388440
return fileMetadata;
389441
}
390442

391443
private SFFileMetadata HandleUploadFileErr(RequestFailedException ex, SFFileMetadata fileMetadata)
392444
{
445+
// 400
393446
if (ex.Status == (int)HttpStatusCode.BadRequest)
394447
{
395448
fileMetadata.resultStatus = ResultStatus.RENEW_PRESIGNED_URL.ToString();
396449
}
450+
// 401
397451
else if (ex.Status == (int)HttpStatusCode.Unauthorized)
398452
{
399453
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
400454
}
455+
// 403, 500, 503
401456
else if (ex.Status == (int)HttpStatusCode.Forbidden ||
402457
ex.Status == (int)HttpStatusCode.InternalServerError ||
403458
ex.Status == (int)HttpStatusCode.ServiceUnavailable)
404459
{
405460
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
406461
}
462+
// other possible Azure blob service error codes: 404, 409, 412, 416
463+
else if (ex.Status == (int)HttpStatusCode.NotFound ||
464+
ex.Status == 409 || // Conflict
465+
ex.Status == (int)HttpStatusCode.PreconditionFailed ||
466+
ex.Status == (int)HttpStatusCode.RequestedRangeNotSatisfiable)
467+
{
468+
String error = $"Unrecoverable HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}";
469+
Logger.Error(error);
470+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
471+
}
472+
else
473+
{
474+
String error = $"Unexpected HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}";
475+
Logger.Error(error);
476+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
477+
}
407478
return fileMetadata;
408479
}
409480

@@ -419,6 +490,12 @@ private SFFileMetadata HandleDownloadFileErr(RequestFailedException ex, SFFileMe
419490
{
420491
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
421492
}
493+
else
494+
{
495+
String error = $"Unexpected HTTP status for file download operation: {ex.Status} {ex.ErrorCode}";
496+
Logger.Error(error);
497+
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
498+
}
422499
return fileMetadata;
423500
}
424501
}

‎Snowflake.Data/Snowflake.Data.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
<Company>Snowflake Computing, Inc</Company>
1414
<Product>Snowflake Connector for .NET</Product>
1515
<Authors>Snowflake</Authors>
16-
<Version>5.2.0</Version>
16+
<Version>5.2.1</Version>
1717
<DebugType>Full</DebugType>
1818
<LangVersion>8</LangVersion>
1919
</PropertyGroup>

0 commit comments

Comments
 (0)