Skip to content

Commit 3b6fdc0

Browse files
committed
Implement provider for azure blob storage
1 parent aae81c9 commit 3b6fdc0

30 files changed

+1921
-1
lines changed

setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
'googledrive = waterbutler.providers.googledrive:GoogleDriveProvider',
5050
'onedrive = waterbutler.providers.onedrive:OneDriveProvider',
5151
'googlecloud = waterbutler.providers.googlecloud:GoogleCloudProvider',
52+
'azureblobstorage = waterbutler.providers.azureblobstorage:AzureBlobStorageProvider',
5253
],
5354
},
5455
)

tests/providers/azureblobstorage/__init__.py

Whitespace-only changes.
Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
"""
2+
Azure Blob Storage test fixtures that load from JSON and XML files
3+
"""
4+
5+
import pytest
6+
import io
7+
import json
8+
import os
9+
from pathlib import Path
10+
from waterbutler.core import streams
11+
from waterbutler.providers.azureblobstorage.provider import AzureBlobStorageProvider
12+
13+
14+
# Get the fixtures directory path
15+
FIXTURES_DIR = Path(__file__).parent / 'fixtures'
16+
17+
18+
def load_fixture(filename):
19+
"""Load a fixture file from the fixtures directory"""
20+
filepath = FIXTURES_DIR / filename
21+
if filename.endswith('.xml'):
22+
with open(filepath, 'r', encoding='utf-8') as f:
23+
return f.read()
24+
else:
25+
with open(filepath, 'r', encoding='utf-8') as f:
26+
return json.load(f)
27+
28+
29+
# ============== Authentication Fixtures (JSON) ==============
30+
31+
@pytest.fixture
32+
def auth():
33+
"""Auth information from the user"""
34+
return load_fixture('auth.json')
35+
36+
37+
@pytest.fixture
38+
def credentials():
39+
"""OAuth credentials from Azure Entra ID"""
40+
return load_fixture('credentials.json')
41+
42+
43+
@pytest.fixture
44+
def settings():
45+
"""Provider settings for Azure Blob Storage"""
46+
return load_fixture('settings.json')
47+
48+
49+
# ============== Response Fixtures (Mixed JSON/XML) ==============
50+
51+
@pytest.fixture
52+
def blob_properties_headers():
53+
"""Standard blob properties headers (JSON format for headers)"""
54+
return load_fixture('blob_properties_headers.json')
55+
56+
57+
# XML Response Fixtures - Direct from Azure API format
58+
@pytest.fixture
59+
def blob_list_xml():
60+
"""Standard blob list XML response"""
61+
return load_fixture('blob_list_response.xml')
62+
63+
64+
@pytest.fixture
65+
def blob_list_with_versions_response():
66+
"""Blob list with version information XML response"""
67+
return load_fixture('blob_list_with_versions.xml')
68+
69+
70+
@pytest.fixture
71+
def empty_list_xml():
72+
"""Empty folder list XML response"""
73+
return load_fixture('empty_list_response.xml')
74+
75+
76+
@pytest.fixture
77+
def root_list_xml():
78+
"""Root container list XML response"""
79+
return load_fixture('root_list_response.xml')
80+
81+
82+
@pytest.fixture
83+
def special_characters_blobs():
84+
"""Blobs with special characters in names XML response"""
85+
return load_fixture('special_characters_blobs.xml')
86+
87+
88+
# Error Response Fixtures
89+
@pytest.fixture
90+
def error_authentication_failed_xml():
91+
"""Authentication failed error XML response"""
92+
return load_fixture('error_authentication_failed.xml')
93+
94+
95+
@pytest.fixture
96+
def error_authorization_failure_xml():
97+
"""Authorization failure error XML response"""
98+
return load_fixture('error_authorization_failure.xml')
99+
100+
101+
@pytest.fixture
102+
def error_not_found_xml():
103+
"""Blob not found error XML response"""
104+
return load_fixture('error_not_found.xml')
105+
106+
107+
@pytest.fixture
108+
def error_internal_error_xml():
109+
"""Internal server error XML response"""
110+
return load_fixture('error_internal_error.xml')
111+
112+
113+
@pytest.fixture
114+
def error_response_xml():
115+
"""Error response XML generator"""
116+
def _error_xml(error_type):
117+
try:
118+
return load_fixture(f'error_{error_type}.xml')
119+
except FileNotFoundError:
120+
# Default error XML
121+
return '''<?xml version="1.0" encoding="utf-8"?>
122+
<Error>
123+
<Code>UnknownError</Code>
124+
<Message>Unknown error occurred</Message>
125+
</Error>'''
126+
127+
return _error_xml
128+
129+
130+
# ============== Provider Fixture ==============
131+
132+
@pytest.fixture
133+
def provider(auth, credentials, settings):
134+
return AzureBlobStorageProvider(auth, credentials, settings)
135+
136+
137+
# ============== Stream Fixtures ==============
138+
139+
@pytest.fixture
140+
def file_content():
141+
"""Basic file content for testing"""
142+
return b'SLEEP IS FOR THE WEAK GO SERVE STREAMS'
143+
144+
145+
@pytest.fixture
146+
def large_file_content():
147+
"""Large file content for multipart upload testing"""
148+
# 10 MB of data
149+
return b'x' * (10 * 1024 * 1024)
150+
151+
152+
@pytest.fixture
153+
def file_like(file_content):
154+
"""File-like object"""
155+
return io.BytesIO(file_content)
156+
157+
158+
@pytest.fixture
159+
def large_file_like(large_file_content):
160+
"""Large file-like object"""
161+
return io.BytesIO(large_file_content)
162+
163+
164+
@pytest.fixture
165+
def file_stream(file_like):
166+
"""File stream for upload testing"""
167+
return streams.FileStreamReader(file_like)
168+
169+
170+
@pytest.fixture
171+
def large_file_stream(large_file_like):
172+
"""Large file stream for multipart upload testing"""
173+
return streams.FileStreamReader(large_file_like)
174+
175+
176+
# ============== Folder Creation Fixtures ==============
177+
178+
@pytest.fixture
179+
def empty_folder_list_xml():
180+
"""Empty folder list response for checking if folder exists"""
181+
return load_fixture('empty_folder_check.xml')
182+
183+
184+
@pytest.fixture
185+
def folder_validation_response_xml():
186+
"""XML response for folder validation (folder exists)"""
187+
return load_fixture('folder_validation_response.xml')
188+
189+
190+
@pytest.fixture
191+
def folder_not_found_response_xml():
192+
"""XML response for folder validation (folder does not exist)"""
193+
return load_fixture('folder_not_found_response.xml')
194+
195+
196+
@pytest.fixture
197+
def folder_exists_xml():
198+
"""XML response indicating folder already has content"""
199+
return load_fixture('folder_exists.xml')
200+
201+
202+
@pytest.fixture
203+
def folder_placeholder_headers():
204+
"""Standard headers for folder placeholder creation"""
205+
return load_fixture('folder_placeholder_headers.json')
206+
207+
208+
@pytest.fixture
209+
def create_folder_test_data():
210+
"""Test data for various folder creation scenarios"""
211+
return {
212+
'simple_folder': {
213+
'path': '/newfolder/',
214+
'name': 'newfolder',
215+
'placeholder': 'newfolder/.osfkeep'
216+
},
217+
'nested_folder': {
218+
'path': '/parent/child/',
219+
'name': 'child',
220+
'placeholder': 'parent/child/.osfkeep'
221+
},
222+
'special_chars_folder': {
223+
'path': '/folder with spaces/',
224+
'name': 'folder with spaces',
225+
'placeholder': 'folder%20with%20spaces/.osfkeep'
226+
}
227+
}
228+
229+
230+
# ============== Helper Functions ==============
231+
232+
@pytest.fixture
233+
def build_error_response():
234+
"""Build a custom error response for testing"""
235+
def _builder(code, message, status=400, auth_detail=None):
236+
xml_body = f'''<?xml version="1.0" encoding="utf-8"?>
237+
<Error>
238+
<Code>{code}</Code>
239+
<Message>{message}</Message>
240+
{f"<AuthenticationErrorDetail>{auth_detail}</AuthenticationErrorDetail>" if auth_detail else ""}
241+
</Error>'''
242+
243+
return {
244+
'status': status,
245+
'body': xml_body,
246+
'headers': {'Content-Type': 'application/xml'}
247+
}
248+
249+
return _builder
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"name": "Test User",
3+
"email": "test@example.com",
4+
"id": "12345",
5+
"provider": "azureblobstorage"
6+
}
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<EnumerationResults ServiceEndpoint="https://teststorageaccount.blob.core.windows.net/" ContainerName="test-container">
3+
<Prefix>test/folder/</Prefix>
4+
<Marker></Marker>
5+
<MaxResults>5000</MaxResults>
6+
<Delimiter>/</Delimiter>
7+
<Blobs>
8+
<Blob>
9+
<Name>test/folder/file1.txt</Name>
10+
<Properties>
11+
<Last-Modified>Mon, 15 Jul 2025 07:28:00 GMT</Last-Modified>
12+
<Etag>"0x8D1A2B3C4D5E6F7"</Etag>
13+
<Content-Length>1024</Content-Length>
14+
<Content-Type>text/plain</Content-Type>
15+
<Content-Encoding></Content-Encoding>
16+
<Content-Language></Content-Language>
17+
<Content-MD5>sQqNsWTgdUEFt6mb5y4/5Q==</Content-MD5>
18+
<Cache-Control></Cache-Control>
19+
<BlobType>BlockBlob</BlobType>
20+
<LeaseStatus>unlocked</LeaseStatus>
21+
<LeaseState>available</LeaseState>
22+
</Properties>
23+
</Blob>
24+
<BlobPrefix>
25+
<Name>test/folder/subfolder/</Name>
26+
</BlobPrefix>
27+
</Blobs>
28+
<NextMarker></NextMarker>
29+
</EnumerationResults>
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<EnumerationResults ServiceEndpoint="https://teststorageaccount.blob.core.windows.net/" ContainerName="test-container">
3+
<Blobs>
4+
<Blob>
5+
<Name>test/report.pdf</Name>
6+
<VersionId>2025-07-15T09:45:00.5555555Z</VersionId>
7+
<IsCurrentVersion>true</IsCurrentVersion>
8+
<Properties>
9+
<Last-Modified>Mon, 15 Jul 2025 09:45:00 GMT</Last-Modified>
10+
<Etag>"0x8D1A2B3C4D5E702"</Etag>
11+
<Content-Length>524288</Content-Length>
12+
<Content-Type>application/pdf</Content-Type>
13+
<BlobType>BlockBlob</BlobType>
14+
<AccessTier>Hot</AccessTier>
15+
</Properties>
16+
</Blob>
17+
<Blob>
18+
<Name>test/report.pdf</Name>
19+
<VersionId>2025-07-15T08:30:00.4444444Z</VersionId>
20+
<IsCurrentVersion>false</IsCurrentVersion>
21+
<Properties>
22+
<Last-Modified>Mon, 15 Jul 2025 08:30:00 GMT</Last-Modified>
23+
<Etag>"0x8D1A2B3C4D5E701"</Etag>
24+
<Content-Length>520192</Content-Length>
25+
<Content-Type>application/pdf</Content-Type>
26+
<BlobType>BlockBlob</BlobType>
27+
<AccessTier>Hot</AccessTier>
28+
</Properties>
29+
</Blob>
30+
</Blobs>
31+
</EnumerationResults>
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"Accept-Ranges": "bytes",
3+
"Content-Length": "1048576",
4+
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
5+
"Content-MD5": "Q2h1Y2sgTm9ycmlzIGFwcHJvdmVz",
6+
"Last-Modified": "Mon, 15 Jul 2025 09:45:00 GMT",
7+
"ETag": "\"0x8D1A2B3C4D5E702\"",
8+
"x-ms-blob-type": "BlockBlob",
9+
"x-ms-lease-status": "unlocked",
10+
"x-ms-lease-state": "available",
11+
"x-ms-access-tier": "Hot",
12+
"x-ms-access-tier-inferred": "true",
13+
"x-ms-server-encrypted": "true",
14+
"x-ms-version-id": "2025-07-15T09:45:00.5555555Z",
15+
"x-ms-is-current-version": "true",
16+
"x-ms-creation-time": "Mon, 15 Jul 2025 09:45:00 GMT",
17+
"x-ms-meta-author": "Data Science Team",
18+
"x-ms-meta-project": "Q3-Analysis",
19+
"x-ms-meta-tags": "quarterly,financial,2025"
20+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ik1uQ19WWmNBVGZNNXBP..."
3+
}
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<EnumerationResults ServiceEndpoint="https://teststorageaccount.blob.core.windows.net/" ContainerName="test-container">
3+
<Prefix>test/newfolder/</Prefix>
4+
<Marker></Marker>
5+
<MaxResults>5000</MaxResults>
6+
<Delimiter>/</Delimiter>
7+
<Blobs>
8+
</Blobs>
9+
<NextMarker></NextMarker>
10+
</EnumerationResults>
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<EnumerationResults ServiceEndpoint="https://teststorageaccount.blob.core.windows.net/" ContainerName="test-container">
3+
<Prefix></Prefix>
4+
<Marker></Marker>
5+
<MaxResults>5000</MaxResults>
6+
<Delimiter>/</Delimiter>
7+
<Blobs>
8+
</Blobs>
9+
<NextMarker></NextMarker>
10+
</EnumerationResults>

0 commit comments

Comments
 (0)