Skip to content
This repository was archived by the owner on Feb 7, 2026. It is now read-only.

Commit ff0b9f4

Browse files
ToastersonToasterson
committed
Add support for file URL without algorithm and refactor JSON serialization for Python-style compatibility
- Added `/file/1/{digest}` route to support file retrieval without specifying the algorithm. - Implemented a Python-compatible JSON serializer to ensure consistent formatting for catalog artifacts. - Replaced `HashMap` with `BTreeMap` for deterministic ordering in catalog serialization and updates. - Updated integration tests to validate the new route functionality and ensure response correctness. - Refactored `format_iso8601_basic` to improve timestamp formatting consistency.
1 parent a921c99 commit ff0b9f4

File tree

6 files changed

+157
-79
lines changed

6 files changed

+157
-79
lines changed

‎libips/src/repository/catalog.rs‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
use miette::Diagnostic;
77
use serde::{Deserialize, Serialize};
8-
use std::collections::HashMap;
8+
use std::collections::BTreeMap;
99
use std::fs;
1010
use std::io;
1111
use std::path::{Path, PathBuf};
@@ -64,7 +64,7 @@ pub enum CatalogError {
6464
pub type Result<T> = std::result::Result<T, CatalogError>;
6565

6666
/// Format a SystemTime as an ISO-8601 'basic format' date in UTC
67-
fn format_iso8601_basic(time: &SystemTime) -> String {
67+
pub fn format_iso8601_basic(time: &SystemTime) -> String {
6868
let datetime = convert_system_time_to_datetime(time);
6969
format!("{}Z", datetime.format("%Y%m%dT%H%M%S.%f"))
7070
}
@@ -141,18 +141,18 @@ pub struct CatalogAttrs {
141141
pub package_version_count: usize,
142142

143143
/// Available catalog parts
144-
pub parts: HashMap<String, CatalogPartInfo>,
144+
pub parts: BTreeMap<String, CatalogPartInfo>,
145145

146146
/// Available update logs
147-
#[serde(skip_serializing_if = "HashMap::is_empty")]
148-
pub updates: HashMap<String, UpdateLogInfo>,
147+
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
148+
pub updates: BTreeMap<String, UpdateLogInfo>,
149149

150150
/// Catalog version
151151
pub version: u32,
152152

153153
/// Optional signature information
154154
#[serde(rename = "_SIGNATURE", skip_serializing_if = "Option::is_none")]
155-
pub signature: Option<HashMap<String, String>>,
155+
pub signature: Option<BTreeMap<String, String>>,
156156
}
157157

158158
impl CatalogAttrs {
@@ -167,8 +167,8 @@ impl CatalogAttrs {
167167
last_modified: timestamp,
168168
package_count: 0,
169169
package_version_count: 0,
170-
parts: HashMap::new(),
171-
updates: HashMap::new(),
170+
parts: BTreeMap::new(),
171+
updates: BTreeMap::new(),
172172
version: CatalogVersion::V1 as u32,
173173
}
174174
}
@@ -208,19 +208,19 @@ pub struct PackageVersionEntry {
208208
pub struct CatalogPart {
209209
/// Packages by publisher and stem
210210
#[serde(flatten)]
211-
pub packages: HashMap<String, HashMap<String, Vec<PackageVersionEntry>>>,
211+
pub packages: BTreeMap<String, BTreeMap<String, Vec<PackageVersionEntry>>>,
212212

213213
/// Optional signature information
214214
#[serde(rename = "_SIGNATURE", skip_serializing_if = "Option::is_none")]
215-
pub signature: Option<HashMap<String, String>>,
215+
pub signature: Option<BTreeMap<String, String>>,
216216
}
217217

218218
impl CatalogPart {
219219
/// Create a new catalog part
220220
pub fn new() -> Self {
221221
CatalogPart {
222222
signature: None,
223-
packages: HashMap::new(),
223+
packages: BTreeMap::new(),
224224
}
225225
}
226226

@@ -235,7 +235,7 @@ impl CatalogPart {
235235
let publisher_packages = self
236236
.packages
237237
.entry(publisher.to_string())
238-
.or_insert_with(HashMap::new);
238+
.or_insert_with(BTreeMap::new);
239239
let stem_versions = publisher_packages
240240
.entry(fmri.stem().to_string())
241241
.or_insert_with(Vec::new);
@@ -310,7 +310,7 @@ pub struct PackageUpdateEntry {
310310

311311
/// Catalog part entries
312312
#[serde(flatten)]
313-
pub catalog_parts: HashMap<String, HashMap<String, Vec<String>>>,
313+
pub catalog_parts: BTreeMap<String, BTreeMap<String, Vec<String>>>,
314314

315315
/// Optional SHA-1 signature of the package manifest
316316
#[serde(rename = "signature-sha-1", skip_serializing_if = "Option::is_none")]
@@ -321,19 +321,19 @@ pub struct PackageUpdateEntry {
321321
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
322322
pub struct UpdateLog {
323323
/// Updates by publisher and stem
324-
pub updates: HashMap<String, HashMap<String, Vec<PackageUpdateEntry>>>,
324+
pub updates: BTreeMap<String, BTreeMap<String, Vec<PackageUpdateEntry>>>,
325325

326326
/// Optional signature information
327327
#[serde(rename = "_SIGNATURE", skip_serializing_if = "Option::is_none")]
328-
pub signature: Option<HashMap<String, String>>,
328+
pub signature: Option<BTreeMap<String, String>>,
329329
}
330330

331331
impl UpdateLog {
332332
/// Create a new update log
333333
pub fn new() -> Self {
334334
UpdateLog {
335335
signature: None,
336-
updates: HashMap::new(),
336+
updates: BTreeMap::new(),
337337
}
338338
}
339339

@@ -343,13 +343,13 @@ impl UpdateLog {
343343
publisher: &str,
344344
fmri: &Fmri,
345345
op_type: CatalogOperationType,
346-
catalog_parts: HashMap<String, HashMap<String, Vec<String>>>,
346+
catalog_parts: BTreeMap<String, BTreeMap<String, Vec<String>>>,
347347
signature: Option<String>,
348348
) {
349349
let publisher_updates = self
350350
.updates
351351
.entry(publisher.to_string())
352-
.or_insert_with(HashMap::new);
352+
.or_insert_with(BTreeMap::new);
353353
let stem_updates = publisher_updates
354354
.entry(fmri.stem().to_string())
355355
.or_insert_with(Vec::new);
@@ -393,10 +393,10 @@ pub struct CatalogManager {
393393
attrs: CatalogAttrs,
394394

395395
/// Catalog parts
396-
parts: HashMap<String, CatalogPart>,
396+
parts: BTreeMap<String, CatalogPart>,
397397

398398
/// Update logs
399-
update_logs: HashMap<String, UpdateLog>,
399+
update_logs: BTreeMap<String, UpdateLog>,
400400
}
401401

402402
impl CatalogManager {
@@ -421,8 +421,8 @@ impl CatalogManager {
421421
catalog_dir: publisher_catalog_dir,
422422
publisher: publisher.to_string(),
423423
attrs,
424-
parts: HashMap::new(),
425-
update_logs: HashMap::new(),
424+
parts: BTreeMap::new(),
425+
update_logs: BTreeMap::new(),
426426
})
427427
}
428428

@@ -572,7 +572,7 @@ impl CatalogManager {
572572
log_name: &str,
573573
fmri: &Fmri,
574574
op_type: CatalogOperationType,
575-
catalog_parts: HashMap<String, HashMap<String, Vec<String>>>,
575+
catalog_parts: BTreeMap<String, BTreeMap<String, Vec<String>>>,
576576
signature: Option<String>,
577577
) -> Result<()> {
578578
if let Some(log) = self.update_logs.get_mut(log_name) {

‎libips/src/repository/catalog_writer.rs‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,47 @@ use std::fs;
77
use std::io::Write;
88
use std::path::{Path, PathBuf};
99

10+
use serde::Serialize;
11+
use serde_json::ser::{Formatter, Serializer};
1012
use tracing::{debug, instrument};
1113

1214
use super::catalog::{CatalogAttrs, CatalogPart, UpdateLog};
1315
use super::{RepositoryError, Result};
1416

17+
// Python-compatible JSON formatter to ensure (', ', ': ') separators
18+
struct PythonFormatter;
19+
20+
impl Formatter for PythonFormatter {
21+
fn begin_object_key<W: ?Sized + Write>(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> {
22+
if !first {
23+
writer.write_all(b", ")?;
24+
}
25+
Ok(())
26+
}
27+
28+
fn begin_object_value<W: ?Sized + Write>(&mut self, writer: &mut W) -> std::io::Result<()> {
29+
writer.write_all(b": ")
30+
}
31+
32+
fn begin_array_value<W: ?Sized + Write>(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> {
33+
if !first {
34+
writer.write_all(b", ")?;
35+
}
36+
Ok(())
37+
}
38+
}
39+
40+
fn serialize_python_style<T: Serialize>(value: &T) -> Result<Vec<u8>> {
41+
let mut bytes = Vec::new();
42+
let formatter = PythonFormatter;
43+
let mut ser = Serializer::with_formatter(&mut bytes, formatter);
44+
value.serialize(&mut ser).map_err(|e| {
45+
RepositoryError::JsonSerializeError(format!("Python-style serialize error: {}", e))
46+
})?;
47+
bytes.push(b'\n');
48+
Ok(bytes)
49+
}
50+
1551
fn sha1_hex(bytes: &[u8]) -> String {
1652
use sha1::Digest as _;
1753
let mut hasher = sha1::Sha1::new();
@@ -53,17 +89,13 @@ fn atomic_write_bytes(path: &Path, bytes: &[u8]) -> Result<()> {
5389
pub(crate) fn write_catalog_attrs(path: &Path, attrs: &mut CatalogAttrs) -> Result<String> {
5490
// Compute signature over content without _SIGNATURE
5591
attrs.signature = None;
56-
let bytes_without_sig = serde_json::to_vec(&attrs).map_err(|e| {
57-
RepositoryError::JsonSerializeError(format!("Catalog attrs serialize error: {}", e))
58-
})?;
92+
let bytes_without_sig = serialize_python_style(&attrs)?;
5993
let sig = sha1_hex(&bytes_without_sig);
60-
let mut sig_map = std::collections::HashMap::new();
94+
let mut sig_map = std::collections::BTreeMap::new();
6195
sig_map.insert("sha-1".to_string(), sig);
6296
attrs.signature = Some(sig_map);
6397

64-
let final_bytes = serde_json::to_vec(&attrs).map_err(|e| {
65-
RepositoryError::JsonSerializeError(format!("Catalog attrs serialize error: {}", e))
66-
})?;
98+
let final_bytes = serialize_python_style(&attrs)?;
6799
debug!(path = %path.display(), bytes = final_bytes.len(), "writing catalog.attrs");
68100
atomic_write_bytes(path, &final_bytes)?;
69101
// safe to unwrap as signature was just inserted
@@ -78,17 +110,13 @@ pub(crate) fn write_catalog_attrs(path: &Path, attrs: &mut CatalogAttrs) -> Resu
78110
pub(crate) fn write_catalog_part(path: &Path, part: &mut CatalogPart) -> Result<String> {
79111
// Compute signature over content without _SIGNATURE
80112
part.signature = None;
81-
let bytes_without_sig = serde_json::to_vec(&part).map_err(|e| {
82-
RepositoryError::JsonSerializeError(format!("Catalog part serialize error: {}", e))
83-
})?;
113+
let bytes_without_sig = serialize_python_style(&part)?;
84114
let sig = sha1_hex(&bytes_without_sig);
85-
let mut sig_map = std::collections::HashMap::new();
115+
let mut sig_map = std::collections::BTreeMap::new();
86116
sig_map.insert("sha-1".to_string(), sig);
87117
part.signature = Some(sig_map);
88118

89-
let final_bytes = serde_json::to_vec(&part).map_err(|e| {
90-
RepositoryError::JsonSerializeError(format!("Catalog part serialize error: {}", e))
91-
})?;
119+
let final_bytes = serialize_python_style(&part)?;
92120
debug!(path = %path.display(), bytes = final_bytes.len(), "writing catalog part");
93121
atomic_write_bytes(path, &final_bytes)?;
94122
Ok(part
@@ -102,17 +130,13 @@ pub(crate) fn write_catalog_part(path: &Path, part: &mut CatalogPart) -> Result<
102130
pub(crate) fn write_update_log(path: &Path, log: &mut UpdateLog) -> Result<String> {
103131
// Compute signature over content without _SIGNATURE
104132
log.signature = None;
105-
let bytes_without_sig = serde_json::to_vec(&log).map_err(|e| {
106-
RepositoryError::JsonSerializeError(format!("Update log serialize error: {}", e))
107-
})?;
133+
let bytes_without_sig = serialize_python_style(&log)?;
108134
let sig = sha1_hex(&bytes_without_sig);
109-
let mut sig_map = std::collections::HashMap::new();
135+
let mut sig_map = std::collections::BTreeMap::new();
110136
sig_map.insert("sha-1".to_string(), sig);
111137
log.signature = Some(sig_map);
112138

113-
let final_bytes = serde_json::to_vec(&log).map_err(|e| {
114-
RepositoryError::JsonSerializeError(format!("Update log serialize error: {}", e))
115-
})?;
139+
let final_bytes = serialize_python_style(&log)?;
116140
debug!(path = %path.display(), bytes = final_bytes.len(), "writing update log");
117141
atomic_write_bytes(path, &final_bytes)?;
118142
Ok(log

‎libips/src/repository/file_backend.rs‎

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use lz4::EncoderBuilder;
1010
use regex::Regex;
1111
use serde::{Deserialize, Serialize};
1212
use sha2::{Digest as Sha2Digest, Sha256};
13-
use std::collections::{HashMap, HashSet};
13+
use std::collections::{HashMap, HashSet, BTreeMap};
1414
use std::fs;
1515
use std::fs::File;
1616
use std::io::{Read, Write};
@@ -228,28 +228,6 @@ pub struct FileBackend {
228228
Option<std::cell::RefCell<crate::repository::obsoleted::ObsoletedPackageManager>>,
229229
}
230230

231-
/// Format a SystemTime as an ISO 8601 timestamp string
232-
fn format_iso8601_timestamp(time: &SystemTime) -> String {
233-
let duration = time
234-
.duration_since(SystemTime::UNIX_EPOCH)
235-
.unwrap_or_else(|_| std::time::Duration::from_secs(0));
236-
237-
let secs = duration.as_secs();
238-
let micros = duration.subsec_micros();
239-
240-
// Format as ISO 8601 with microsecond precision
241-
format!(
242-
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z",
243-
// Convert seconds to date and time components
244-
1970 + secs / 31536000, // year (approximate)
245-
(secs % 31536000) / 2592000 + 1, // month (approximate)
246-
(secs % 2592000) / 86400 + 1, // day (approximate)
247-
(secs % 86400) / 3600, // hour
248-
(secs % 3600) / 60, // minute
249-
secs % 60, // second
250-
micros // microseconds
251-
)
252-
}
253231

254232
/// Transaction for publishing packages
255233
pub struct Transaction {
@@ -759,9 +737,9 @@ impl ReadableRepository for FileBackend {
759737
let updated = if latest_timestamp == SystemTime::UNIX_EPOCH {
760738
// If no files were found, use the current time
761739
let now = SystemTime::now();
762-
format_iso8601_timestamp(&now)
740+
crate::repository::catalog::format_iso8601_basic(&now)
763741
} else {
764-
format_iso8601_timestamp(&latest_timestamp)
742+
crate::repository::catalog::format_iso8601_basic(&latest_timestamp)
765743
};
766744

767745
// Create a PublisherInfo struct and add it to the list
@@ -1801,9 +1779,9 @@ impl FileBackend {
18011779
locale: &str,
18021780
fmri: &crate::fmri::Fmri,
18031781
op_type: crate::repository::catalog::CatalogOperationType,
1804-
catalog_parts: std::collections::HashMap<
1782+
catalog_parts: std::collections::BTreeMap<
18051783
String,
1806-
std::collections::HashMap<String, Vec<String>>,
1784+
std::collections::BTreeMap<String, Vec<String>>,
18071785
>,
18081786
signature_sha1: Option<String>,
18091787
) -> Result<()> {
@@ -1830,7 +1808,7 @@ impl FileBackend {
18301808
Some(p) => p,
18311809
None => {
18321810
let now = std::time::SystemTime::now();
1833-
let ts = format_iso8601_timestamp(&now); // e.g., 20090508T161025.686485Z
1811+
let ts = crate::repository::catalog::format_iso8601_basic(&now); // e.g., 20090508T161025.686485Z
18341812
let stem = ts.split('.').next().unwrap_or(&ts); // take up to seconds
18351813
catalog_dir.join(format!("update.{}.{}", stem, locale))
18361814
}
@@ -1863,7 +1841,7 @@ impl FileBackend {
18631841
Some(s) => s,
18641842
None => {
18651843
let now = std::time::SystemTime::now();
1866-
let ts = format_iso8601_timestamp(&now);
1844+
let ts = crate::repository::catalog::format_iso8601_basic(&now);
18671845
ts.split('.').next().unwrap_or(&ts).to_string()
18681846
}
18691847
};
@@ -2386,18 +2364,18 @@ impl FileBackend {
23862364

23872365
// Prepare update entry if needed
23882366
if create_update_log {
2389-
let mut catalog_parts = HashMap::new();
2367+
let mut catalog_parts = BTreeMap::new();
23902368

23912369
// Add dependency actions to update entry
23922370
if !dependency_actions.is_empty() {
2393-
let mut actions = HashMap::new();
2371+
let mut actions = BTreeMap::new();
23942372
actions.insert("actions".to_string(), dependency_actions);
23952373
catalog_parts.insert("catalog.dependency.C".to_string(), actions);
23962374
}
23972375

23982376
// Add summary actions to update entry
23992377
if !summary_actions.is_empty() {
2400-
let mut actions = HashMap::new();
2378+
let mut actions = BTreeMap::new();
24012379
actions.insert("actions".to_string(), summary_actions);
24022380
catalog_parts.insert("catalog.summary.C".to_string(), actions);
24032381
}
@@ -2427,18 +2405,18 @@ impl FileBackend {
24272405

24282406
// Create a catalog.attrs file
24292407
let now = SystemTime::now();
2430-
let timestamp = format_iso8601_timestamp(&now);
2408+
let timestamp = crate::repository::catalog::format_iso8601_basic(&now);
24312409

24322410
// Get the CatalogAttrs struct definition to see what fields it has
24332411
let mut attrs = crate::repository::catalog::CatalogAttrs {
24342412
created: timestamp.clone(),
24352413
last_modified: timestamp.clone(),
24362414
package_count,
24372415
package_version_count,
2438-
parts: HashMap::new(),
2416+
parts: BTreeMap::new(),
24392417
version: 1, // CatalogVersion::V1 is 1
24402418
signature: None,
2441-
updates: HashMap::new(),
2419+
updates: BTreeMap::new(),
24422420
};
24432421

24442422
// Add part information

0 commit comments

Comments
 (0)