persist: Add Criterion benchmarks for envelope encryption overhead

Cargo.toml

@@ -72,6 +72,7 @@ members = [
     "src/persist",
     "src/persist-cli",
     "src/persist-client",
+    "src/persist-consensus-svc",
     "src/persist-proc",
     "src/persist-types",
     "src/pgcopy",
@@ -197,6 +198,7 @@ default-members = [
     "src/persist",
     "src/persist-cli",
     "src/persist-client",
+    "src/persist-consensus-svc",
     "src/persist-proc",
     "src/persist-types",
     "src/pgcopy",

bin/pyactivate

@@ -90,7 +90,7 @@ def activate_venv(py_dir: Path) -> Path:
             logger.info("python exec error: %s", e)
         print(f"==> Initializing virtualenv in {venv_dir}", file=sys.stderr)
         try:
-            subprocess.check_call(["uv", "venv", venv_dir])
+            subprocess.check_call(["uv", "venv", "--python", "3.13", "--clear", venv_dir])
         except FileNotFoundError:
             # Install Python into the virtualenv via a symlink rather than copying,
             # except on Windows. This matches the behavior of the `python -m venv`

misc/monitoring/grafana/dashboards/consensus-svc.json

@@ -0,0 +1,324 @@
+{
+  "uid": "consensus-svc-demo",
+  "title": "Consensus Service — Group Commit Demo",
+  "editable": true,
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "refresh": "5s",
+  "panels": [
+    {
+      "id": 1,
+      "title": "S3 WAL Writes/s",
+      "description": "Should stay FLAT regardless of shard count. Each tick = one S3 PUT for all shards combined.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 8, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "green" },
+          "custom": {
+            "lineWidth": 3,
+            "fillOpacity": 20,
+            "spanNulls": true,
+            "axisSoftMin": 0
+          },
+          "unit": "ops",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "expr": "rate(mz_consensus_svc_s3_wal_writes_total[30s])",
+          "legendFormat": "S3 PUTs/s",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Active Shards",
+      "description": "Number of persist shards tracked. Grows as objects are created.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 8, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "blue" },
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 15,
+            "spanNulls": true
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "expr": "mz_consensus_svc_active_shards",
+          "legendFormat": "shards",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Operations/s by Type",
+      "description": "CAS committed/rejected, head, scan, truncate rates.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 8, "x": 16, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 30,
+            "stacking": { "mode": "normal" },
+            "spanNulls": true
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "rate(mz_consensus_svc_cas_committed_total[30s])",
+          "legendFormat": "CAS committed",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(mz_consensus_svc_cas_rejected_total[30s])",
+          "legendFormat": "CAS rejected",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(mz_consensus_svc_head_ops_total[30s])",
+          "legendFormat": "head",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(mz_consensus_svc_scan_ops_total[30s])",
+          "legendFormat": "scan",
+          "refId": "D"
+        },
+        {
+          "expr": "rate(mz_consensus_svc_truncate_ops_total[30s])",
+          "legendFormat": "truncate",
+          "refId": "E"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "S3 WAL PUT Latency",
+      "description": "S3 Express One Zone PUT latency for WAL batch writes.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 6, "x": 0, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 2,
+            "spanNulls": true
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, rate(mz_consensus_svc_s3_wal_write_latency_seconds_bucket[1m]))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.99, rate(mz_consensus_svc_s3_wal_write_latency_seconds_bucket[1m]))",
+          "legendFormat": "p99",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.9999, rate(mz_consensus_svc_s3_wal_write_latency_seconds_bucket[1m]))",
+          "legendFormat": "p99.99",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "S3 Snapshot PUT Latency",
+      "description": "S3 Express One Zone PUT latency for snapshot writes.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 6, "x": 6, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 2,
+            "spanNulls": true
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, rate(mz_consensus_svc_s3_snapshot_write_latency_seconds_bucket[1m]))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.99, rate(mz_consensus_svc_s3_snapshot_write_latency_seconds_bucket[1m]))",
+          "legendFormat": "p99",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "Ops per Flush Batch (p50)",
+      "description": "How many WAL ops are batched per flush. Should grow with shard count — this is the batching efficiency.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 6, "x": 12, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "orange" },
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 15,
+            "spanNulls": true
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, rate(mz_consensus_svc_flush_ops_per_batch_bucket[1m]))",
+          "legendFormat": "ops/batch (p50)",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.5, rate(mz_consensus_svc_flush_shards_per_batch_bucket[1m]))",
+          "legendFormat": "shards/batch (p50)",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "In-Memory State",
+      "description": "Total entries and approximate bytes held across all shards.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 6, "x": 18, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 2,
+            "spanNulls": true
+          }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "entries" },
+            "properties": [
+              { "id": "unit", "value": "short" },
+              { "id": "custom.axisPlacement", "value": "left" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "bytes" },
+            "properties": [
+              { "id": "unit", "value": "bytes" },
+              { "id": "custom.axisPlacement", "value": "right" }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "mz_consensus_svc_total_entries",
+          "legendFormat": "entries",
+          "refId": "A"
+        },
+        {
+          "expr": "mz_consensus_svc_approx_bytes",
+          "legendFormat": "bytes",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "S3 Bytes Written/s",
+      "description": "Rate of bytes written to S3, split by WAL and snapshot.",
+      "type": "timeseries",
+      "gridPos": { "h": 10, "w": 8, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 20,
+            "stacking": { "mode": "normal" },
+            "spanNulls": true
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "rate(mz_consensus_svc_s3_wal_write_bytes_total[30s])",
+          "legendFormat": "WAL",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(mz_consensus_svc_s3_snapshot_write_bytes_total[30s])",
+          "legendFormat": "Snapshot",
+          "refId": "B"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 38,
+  "version": 1,
+  "templating": { "list": [] },
+  "annotations": { "list": [] }
+}