feat(http-client)\!: complete hishel 1.x migration with breaking config changes

.gitignore

@@ -1,3 +1,5 @@
+!/harp_apps/dashboard/web/.gitkeep
+!/misc/cache-tests/cache-tests/results/index.mjs
 *$py.class
 *.cover
 *.egg
@@ -45,9 +47,11 @@
 /harp.db
 /harp_apps/dashboard/frontend/stats.html
 /harp_apps/dashboard/web/*
-!/harp_apps/dashboard/web/.gitkeep
+/misc/cache-tests/cache-tests/node_modules/
+/misc/cache-tests/cache-tests/results/harp.json
 /misc/kitchen-sink/harp.db
-/misc/secrets
+/misc/secrets/
+/misc/worktrees/
 /requirements*.txt
 /site
 ENV/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "misc/cache-tests/cache-tests"]
+	path = misc/cache-tests/cache-tests
+	url = https://github.com/http-tests/cache-tests.git

CLAUDE.md

@@ -108,10 +108,10 @@ the code - don't change things just for the sake of change.
 
 #### Refactoring Guidelines
 
-##### 1. Commit Before Refactoring
+##### 1. Stop Before Refactoring
 
-Always commit your working code before starting any refactoring. This gives a
-safe point to return to.
+Always stop ask the user before any refactoring. Suggest he may want to commit the work.
+This gives a safe point to return to.
 
 ##### 2. Look for Useful Abstractions Based on Semantic Meaning
 
@@ -281,6 +281,7 @@ pnpm ui:serve
 For release managers: see the complete release documentation in `docs/contribute/release/python.rst`.
 
 **Quick reference:**
+
 - Update changelog: `docs/contribute/release/changelog.rst`
 - Python package releases: `docs/contribute/release/python.rst`
 - Pre-release tasks: `docs/contribute/release/chores.rst`
@@ -292,4 +293,6 @@ small, safe increments. Every change should be driven by a test that describes
 the desired behavior, and the implementation should be the simplest thing that
 makes that test pass. When in doubt, favor simplicity and readability over
 cleverness.
+
 - no commit unless explicitely asked
+- Always run "uv run ruff check --fix harp harp_apps tests" before considering anything done.

Makefile

@@ -92,6 +92,7 @@ install: install-frontend install-backend  ## Installs harp dependencies (backen
 install-dev: install-backend-dev  ## Installs harp dependencies (backend, dashboard) with development tools.
 
 install-frontend:  ## Installs harp dashboard dependencies (frontend).
+	@mkdir -p harp_apps/dashboard/web
 	$(call execute,cd $(FRONTEND_DIR); $(PNPM) install $(if $(DEBUG),,--silent))
 
 install-backend:  ## Installs harp dependencides (backend).
@@ -112,7 +113,7 @@ reference: harp  ## Generates API reference documentation as ReST files (docs).
 	rm -rf docs/reference/core docs/reference/apps
 	mkdir -p docs/reference/core docs/reference/apps
 	$(UV_RUN) bin/generate_apidoc
-	git add docs/reference/
+	-git add docs/reference/
 
 docs:  ## Build html documentation
 	$(call execute,$(UV_RUN) $(MAKE) -C docs html)
@@ -136,7 +137,7 @@ build-frontend: install-frontend  ## Builds the harp dashboard frontend (compile
 ########################################################################################################################
 
 .PHONY: preqa qa qa-full types format format-backend format-frontend optimize-images
-.PHONY: test test-backend test-frontend test-frontend-update test-frontend-ui-update
+.PHONY: test test-backend test-frontend test-frontend-update test-frontend-ui-update test-e2e-cache
 .PHONY: lint-frontend coverage cloc
 
 preqa: types format reference  ## Runs pre-qa checks (types generation, formatting, api reference).
@@ -190,6 +191,11 @@ test-frontend-update: install-frontend lint-frontend  ## Runs frontend tests whi
 test-frontend-ui-update: install-frontend lint-frontend  ## Update user interface visual snapshots.
 	$(call execute,bin/runc_visualtests pnpm test:ui:update)
 
+test-e2e-cache: install-backend-dev  ## Runs E2E cache tests using cache-tests suite.
+	@echo "Initializing cache-tests submodule..."
+	@git submodule update --init --recursive misc/cache-tests/cache-tests 2>/dev/null || true
+	$(call execute,cd misc/cache-tests && ./run-tests.sh)
+
 lint-frontend: install-frontend  ## Lints the frontend codebase.
 	$(call execute,cd $(FRONTEND_DIR); $(PNPM) build)
 
@@ -243,7 +249,7 @@ help:   ## Shows available commands.
 	@grep -E '^(install|install-dev|install-frontend|install-backend|install-backend-dev|start-dev|start-dev-frontend|build-frontend|wheel):.*?##' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf "    make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 	@echo
 	@echo "\033[1mQuality\033[0m"
-	@grep -E '^(preqa|qa|qa-full|qa-nofront|test|test-backend|test-frontend|test-backend-update|test-frontend-update|test-frontend-ui-update|lint-frontend|format|format-backend|format-frontend|types|coverage|cloc|optimize-images):.*?##' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf "    make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^(preqa|qa|qa-full|qa-nofront|test|test-backend|test-frontend|test-backend-update|test-frontend-update|test-frontend-ui-update|test-e2e-cache|lint-frontend|format|format-backend|format-frontend|types|coverage|cloc|optimize-images):.*?##' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf "    make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 	@echo
 	@echo "\033[1mDocumentation\033[0m"
 	@grep -E '^(reference|docs|docs-dev):.*?##' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf "    make \033[36m%-30s\033[0m %s\n", $$1, $$2}'

bin/runc_visualtests

@@ -10,12 +10,14 @@ entrypoint_script=$(mktemp)
 cat > $entrypoint_script <<EOF
 #! /bin/bash
 
-npm install -g npm@11.3.0 corepack@0.32.0
+set -e
+
+npm install -g npm@11.6.2  corepack@0.34.4
 
 cd /app
 cp /app/package.json /tmp/package.json
 corepack enable pnpm
-corepack use pnpm
+corepack use pnpm@10.22.0
 
 
 echo

docs/apps/dashboard/examples/reference.yml

@@ -3,6 +3,6 @@ dashboard:
   devserver:
     enabled: true
     port: null
-  enable_ui: true
+  enabled: true
   port: 4080
   public_url: null

docs/apps/dashboard/schema.json

@@ -76,6 +76,11 @@
       "additionalProperties": false,
       "description": "Root settings for the dashboard application.",
       "properties": {
+        "enabled": {
+          "default": true,
+          "description": "Whether the application is enabled",
+          "type": "boolean"
+        },
         "port": {
           "default": 4080,
           "description": "Port on which the dashboard application will be served.",
@@ -102,11 +107,6 @@
           ],
           "description": "Development server settings, only useful for internal frontend development."
         },
-        "enable_ui": {
-          "default": true,
-          "description": "DEPRECATED – Whether to enable the dashboard UI.",
-          "type": "boolean"
-        },
         "public_url": {
           "anyOf": [{ "type": "string" }, { "type": "null" }],
           "default": null,

docs/apps/http_cache/examples/cache.yml

@@ -0,0 +1,12 @@
+http_cache:
+  # Cache configuration (optional)
+  enabled: true # Set to false to disable cache entirely (optional)
+
+  policy: # Override the cache policy definition (optional)
+    # Hishel uses SpecificationPolicy with CacheOptions
+    type: hishel.SpecificationPolicy
+    arguments:
+      cache_options:
+        supported_methods: [GET, HEAD] # Which HTTP methods should be cacheable? (default: [GET, HEAD])
+        allow_stale: false # Should stale cache entries be returned? (default: false)
+        shared: true # Is this a shared cache? (default: true)

docs/apps/http_cache/examples/reference.yml

@@ -0,0 +1,24 @@
+http_cache:
+  # Global cache flag (default: true)
+  enabled: true
+
+  # Cache transport (default: harp_apps.http_cache.transports.AsyncCacheTransport)
+  transport:
+    type: harp_apps.http_cache.transports.AsyncCacheTransport
+
+  # Cache policy (default: hishel.SpecificationPolicy with RFC 9111-compliant options)
+  policy:
+    type: hishel.SpecificationPolicy
+    arguments:
+      cache_options:
+        shared: true  # Shared cache mode
+        supported_methods: [GET, HEAD]  # Cacheable methods
+        allow_stale: false  # Don't serve stale responses
+
+  # Cache storage (default: harp_apps.http_cache.storages.AsyncStorage)
+  storage:
+    base: hishel.AsyncBaseStorage
+    type: harp_apps.http_cache.storages.AsyncStorage
+    arguments:
+      ttl: null  # No automatic expiration
+      check_ttl_every: 60.0  # Check for expired entries every 60 seconds

docs/apps/http_cache/index.rst

@@ -0,0 +1,143 @@
+HTTP Cache
+==========
+
+.. tags:: applications
+
+.. versionadded:: 0.10
+
+The ``harp_apps.http_cache`` application provides RFC 9111-compliant HTTP caching for the HTTP client. It was
+extracted from ``harp_apps.http_client`` as a separate application to enable better modularity and reuse.
+
+The caching mechanism is implemented using `Hishel <https://hishel.com/>`_, a powerful HTTP caching library
+that follows the HTTP caching standards defined in RFC 9111.
+
+.. toctree::
+    :hidden:
+    :maxdepth: 1
+
+    Services <services>
+    Settings <settings>
+    Internals </reference/apps/harp_apps.http_cache>
+
+
+Overview
+::::::::
+
+The HTTP cache application provides efficient and standards-compliant HTTP response caching. It integrates
+seamlessly with the ``http_client`` application to reduce redundant network calls by storing and reusing
+responses according to RFC 9111 caching rules.
+
+The cache automatically handles:
+
+- **Freshness lifetime calculation** using max-age, s-maxage, and Expires headers
+- **Cache revalidation** with ETags and Last-Modified headers
+- **Content negotiation** via Vary header support
+- **Cache directives** including no-store, no-cache, private, public, and must-revalidate
+
+
+Features
+::::::::
+
+- **RFC 9111 Compliance:** Implements HTTP caching standards for freshness, validation, and cache control
+- **Shared Cache Mode:** Designed for shared cache scenarios (e.g., proxy caching for multiple clients)
+- **Normalized Cache Keys:** Handles load-balanced backends by normalizing URLs for consistent cache keys
+- **Flexible Storage:** Supports multiple storage backends via the storage application
+- **Configurable Policy:** Customize caching behavior through policy configuration
+
+
+Loading
+:::::::
+
+The HTTP cache application is loaded automatically when present and not disabled. It depends on the
+``http_client`` application, which will be loaded automatically if not already present.
+
+To manually control loading:
+
+.. code-block:: yaml
+
+    applications:
+      - http_cache  # Will auto-load http_client as dependency
+
+    http_cache:
+      enabled: true  # Set to false to disable
+
+
+Configuration
+:::::::::::::
+
+Basic configuration
+-------------------
+
+Here's a basic cache configuration:
+
+.. literalinclude:: ./examples/cache.yml
+    :language: yaml
+
+The cache uses sensible RFC 9111-compliant defaults. For detailed configuration options and
+advanced customization, see :doc:`settings`.
+
+
+How it works
+::::::::::::
+
+Request flow
+------------
+
+1. **Request arrives** at the HTTP client
+2. **Cache lookup** checks if a fresh cached response exists
+3. **Vary header matching** ensures the correct variant is selected
+4. **Freshness check** determines if the cached response is still fresh
+5. **Return cached response** if fresh, or forward to origin server if stale/missing
+6. **Store response** from origin server for future requests
+
+
+Cache key normalization
+-----------------------
+
+The cache includes special handling for load-balanced backends. When the proxy forwards requests to
+different backend instances of the same logical endpoint, the cache normalizes URLs by using the
+endpoint name instead of the actual backend host.
+
+This ensures that requests to different backend instances (e.g., ``backend-1``, ``backend-2``) that
+represent the same logical endpoint share the same cache key. The endpoint name is extracted from the
+request extensions set by the proxy application.
+
+
+RFC 9111 Compliance
+:::::::::::::::::::
+
+The cache implementation is tested against RFC 9111 requirements. The test suite includes:
+
+- **Freshness lifetime** (§4.2): max-age, s-maxage, Expires headers
+- **Validation** (§4.3): ETag, Last-Modified, conditional requests
+- **Cache-Control directives** (§5.2.2): no-store, no-cache, private, public
+- **Vary header** (§4.1): content negotiation and cache key selection
+- **HTTP methods** (§3): GET, HEAD, POST, PUT, DELETE, PATCH cacheability
+- **Status codes** (§3): cacheable responses (200, 301, 404, etc.)
+
+See the test suite in ``harp_apps/http_cache/tests/rfc9111/`` for detailed compliance verification.
+
+
+Configuration reference
+:::::::::::::::::::::::
+
+For complete configuration options and examples, see :doc:`settings`.
+
+
+Internal implementation
+:::::::::::::::::::::::
+
+The internal implementation leverages the following classes:
+
+- :class:`HttpCacheSettings <harp_apps.http_cache.settings.HttpCacheSettings>` - Cache configuration
+- :class:`AsyncCacheTransport <harp_apps.http_cache.transports.AsyncCacheTransport>` - Transport wrapper
+- :class:`AsyncStorage <harp_apps.http_cache.storages.AsyncStorage>` - Storage adapter
+- :class:`WrappedRequest <harp_apps.http_cache.models.WrappedRequest>` - Request normalization
+
+
+Further reading
+:::::::::::::::
+
+- `RFC 9111: HTTP Caching <https://www.rfc-editor.org/rfc/rfc9111.html>`_
+- `Hishel Documentation <https://hishel.com/>`_
+- `HTTP Caching Best Practices <https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching>`_