feat/remove random provider selection

.changeset/khaki-needles-grab.md

@@ -0,0 +1,11 @@
+---
+"@prosopo/procaptcha-frictionless": minor
+"@prosopo/procaptcha-common": minor
+"@prosopo/types-database": minor
+"@prosopo/detector": minor
+"@prosopo/provider": minor
+"@prosopo/types": minor
+---
+
+Remove random provider selection fn in favour of DNS routing
+

docker/docker-compose.provider.yml

@@ -124,6 +124,8 @@ services:
       - "[::]:443:443"
     volumes:
       - ./provider.Caddyfile:/etc/caddy/Caddyfile
+      - ./certs/pronode.crt /etc/caddy/certs/pronode.crt
+      - ./certs/pronode.key /etc/caddy/certs/pronode.key
       - caddy_data:/data
       - caddy_config:/config
     networks:

docker/images/caddy/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@prosopo/caddy-docker",
-	"version": "2.5.6",
+	"version": "2.5.7",
 	"engines": {
 		"node": "^24",
 		"npm": "^11"
@@ -22,6 +22,6 @@
 		}
 	},
 	"devDependencies": {
-		"@prosopo/config": "3.1.20"
+		"@prosopo/config": "3.3.0"
 	}
 }

docker/images/caddy/src/Dockerfile

@@ -3,12 +3,14 @@ RUN apk update && apk add gcc g++ make libpcap-dev libpcap
 RUN CGO_ENABLED=1 xcaddy build \
     --with github.com/mholt/caddy-ratelimit \
     --with github.com/prosopo/chaddy \
-    --with github.com/lolPants/caddy-requestid
+    --with github.com/lolPants/caddy-requestid \
+    --with github.com/caddy-dns/bunny \
+    --with github.com/pberkel/caddy-storage-redis
+
 FROM caddy:2
 RUN apk update && apk add libpcap
 RUN apk add --no-cache curl
 # Create a static directory for Caddy and write robots.txt inside the image
 RUN mkdir -p /srv/static && \
     echo -e "User-agent: *\nDisallow: /" > /srv/static/robots.txt
 COPY --from=builder /usr/bin/caddy /usr/bin/caddy
-

docker/provider.Caddyfile

@@ -1,15 +1,17 @@
 # usage: `caddy run --config ./docker/provider.Caddyfile --envfile docker/env.development`
 {
-	# debug
 	http_port {$CADDY_HTTP_PORT:80}
+	https_port 443
 	auto_https {$CADDY_AUTO_HTTPS:disable_redirects}
 	admin {$CADDY_ADMIN_API::2020} # set the admin api to run on localhost:2020 (default is 2019 which can conflict with caddy daemon)
 
-	# Caddy must be told custom rate_limit module its order
+	acme_ca https://acme-v02.api.letsencrypt.org/directory
+	email {$CADDY_ACME_EMAIL:admin@prosopo.io}
+	admin {$CADDY_ADMIN_API::2020}
+
 	order rate_limit before basicauth
 
 	client_hello {
-		# Configure the maximum allowed ClientHello packet size in bytes (1-16384)
 		max_client_hello_size 16384
 	}
 
@@ -28,60 +30,36 @@
 	}
 }
 
-# HTTP metrics endpoint on all interfaces
-http://:9090 {
-	metrics /metrics
-}
-
-{$CADDY_DOMAIN} {
-
-    @httpOnly {
-        protocol http
-    }
-    redir @httpOnly https://{host}{uri}
-
+# Reusable robots.txt logic
+(robots_logic) {
 	handle /robots.txt {
-		uri strip_prefix / # removes leading /, so it looks directly in root
 		root * /srv/static
 		file_server
 		header Content-Type "text/plain"
 		header X-Robots-Tag "noindex, nofollow"
 	}
+}
 
+# --- REUSABLE PROXY SNIPPET ---
+# This contains all your headers, rate limits, and proxy logic
+(my_proxy_logic) {
 	route {
-		client_hello
 		request_id
 
 		header X-Request-ID "{http.request_id}"
+		header X-Node-Id "{vars.node_id}" # Injected from the handle block
 
 		header / {
-			# Enable HTTP Strict Transport Security (HSTS)
 			Strict-Transport-Security "max-age=31536000;"
-			# Enable cross-site filter (XSS) and tell browser to block detected attacks
 			X-XSS-Protection "1; mode=block"
-			# Disallow the site to be rendered within a frame (clickjacking protection)
 			X-Frame-Options "DENY"
-			# Prevent search engines from indexing
 			X-Robots-Tag "none"
 		}
 
-		# enable prometheus metrics
 		metrics /metrics
 
 		rate_limit {
 			distributed
-
-			# Means that the rate limit is applied to all GET requests, with a limit of 100 requests per minute.
-			# zone get_rate_limit {
-			# 	match {
-			# 		method GET
-			# 	}
-			# 	key static
-			# 	events 100
-			# 	window 1m
-			# }
-
-			# The rate limit is applied to `remote_host` with a limit of 6 requests per 6 seconds (60 requests per minute).
 			zone dynamic_example {
 				key {remote_host}
 				events {$CADDY_RATE_LIMIT_EVENTS}
@@ -90,54 +68,21 @@ http://:9090 {
 			log_key
 		}
 
-		# reverse proxy to the provider container
 		reverse_proxy {$CADDY_PROVIDER_CONTAINER_NAME:provider1}:{$CADDY_PROVIDER_PORT:9229} {$CADDY_PROVIDER_CONTAINER_NAME:provider2}:{$CADDY_PROVIDER_PORT2:9339} {
-			# https://caddyserver.com/docs/modules/http.handlers.reverse_proxy
-
-			# try A, then B, then C, etc.
 			lb_policy first
-
-			# how many times a backend can fail before it is considered unhealthy
 			max_fails 1
-
-			# how long a backend is marked as unhealthy after it has failed (this is a non-zero duration to enable passive health checks). Passive health checks decide a backend's health based on the response code (and whether it responded at all) from normal traffic.
 			fail_duration 1ns
-
-			# 5XX status codes are considered unhealthy, in addition to no response
 			unhealthy_status 5xx
-
-			# long latency on response marks the backend as unhealthy
 			unhealthy_latency 10s
 
 			transport http {
-				# how long to wait for a connection to be established to backend
 				dial_timeout 1s
 			}
 
-			# how long to keep trying backends before giving up
 			lb_try_duration 5s
-
-			# how long to wait between retries of backends (0 doesn't work, set to 1ns for almost immediate retry)
 			lb_try_interval 1ns
 
-			# example failover sequence with failing backends:
-			# - request comes in
-			# - lb_policy first means provider1 is tried first
-			# - request is sent to provider1
-			# - provider1 does not respond within 1s (dial_timeout)
-			# - provider1 is marked as unhealthy
-			# - request is sent to provider2
-			# - provider2 does not respond within 1s (dial_timeout)
-			# - in this time, provider1 is marked as healthy again (fail duration expired)
-			# - provider2 is marked as unhealthy
-			# - request is sent to provider1 again
-			# - provider1 responds within 1s
-			# - request is completed
-			# the request is retried over all backends in turn until either it succeeds or the try_duration is reached
-
-			# https://caddyserver.com/docs/caddyfile/concepts#placeholders
-			# https://caddyserver.com/docs/json/apps/http/#docs
-
+			# All your existing header_up metadata...
 			header_up X-Forwarded-For {http.request.remote.host}
 			header_up X-Forwarded-Port {http.request.remote.port}
 			header_up X-Forwarded-Proto {http.request.scheme}
@@ -214,8 +159,39 @@ http://:9090 {
 			header_up x-tls-client-san-uris "^{http.request.tls.client.san.uris}$" ""
 		}
 	}
+}
+
+# --- 1. MAIN DOMAIN (Mounted Certificate) ---
+pronode.prosopo.io {
+	import robots_logic
+
+	tls /etc/caddy/certs/pronode.crt /etc/caddy/certs/pronode.key {
+		# This tells Caddy: "Don't manage this, just use these files"
+	}
+
+	vars node_id "main-gateway"
+	import my_proxy_logic
 
 	log {
 		format json
 	}
 }
+
+# --- 2. NODE-SPECIFIC DOMAIN (ACME Challenge) ---
+{$CADDY_DOMAIN} {
+	import robots_logic
+	# Caddy will automatically perform the HTTP-01 challenge here
+	# No tls block needed unless you want to specify an email
+
+	vars node_id "I-am-node-{$NODE_ID}"
+	import my_proxy_logic
+
+	log {
+		format json
+	}
+}
+
+# --- METRICS ---
+:9090 {
+	metrics /metrics
+}

packages/api/src/api/ProviderApi.ts

@@ -74,7 +74,6 @@ export default class ProviderApi
 		const body: CaptchaRequestBodyType = {
 			[ApiParams.dapp]: dappAccount,
 			[ApiParams.user]: userAccount,
-			[ApiParams.datasetId]: provider.datasetId,
 		};
 		if (sessionId) {
 			body[ApiParams.sessionId] = sessionId;

packages/database/src/databases/provider.ts

@@ -617,6 +617,20 @@ export class ProviderDatabase
 		await this.tables?.captcha.deleteMany(filter);
 	}
 
+	/**
+	 * @description Get the most recently uploaded dataset ID
+	 */
+	async getMostRecentDatasetId(): Promise<string | undefined> {
+		const dataset = await this.tables?.dataset
+			.findOne()
+			.sort({ _id: -1 }) // Sort by _id descending to get most recent
+			.lean<DatasetBase>();
+
+		const datasetId = dataset?.datasetId;
+		// Ensure we return string | undefined, not Hash (which can be string | number[])
+		return typeof datasetId === "string" ? datasetId : undefined;
+	}
+
 	/**
 	 * @description Get a dataset by Id
 	 */

packages/detector/src/index.d.ts

@@ -11,18 +11,17 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 import { Account } from '@prosopo/types';
 import { BehavioralData } from '@prosopo/types';
 import { ClickEventPoint } from '@prosopo/types';
 import { EnvironmentTypes } from '@prosopo/types';
 import { MouseMovementPoint } from '@prosopo/types';
 import { PackedBehavioralData } from '@prosopo/types';
-import { RandomProvider } from '@prosopo/types';
 import { TouchEventPoint } from '@prosopo/types';
 
-declare const detect: (env: EnvironmentTypes, randomProviderSelectorFn: RandomProviderSelectorFn, container: HTMLElement | undefined, restart: () => void, accountGenerator: () => Promise<Account>) => Promise<{
+declare const detect: (env: EnvironmentTypes, container: HTMLElement | undefined, restart: () => void, accountGenerator: () => Promise<Account>) => Promise<{
     token: string;
-    provider?: RandomProvider;
     shadowDomCleanup: () => void;
     encryptHeadHash: string;
     mouseTracker?: {

packages/env/src/env.ts

@@ -39,6 +39,7 @@ export class Environment implements ProsopoEnvironment {
 	envId: string | undefined;
 	geolocationService: GeolocationService;
 	ready = false;
+	datasetId: string | undefined;
 
 	constructor(
 		config: ProsopoConfigOutput,
@@ -142,6 +143,27 @@ export class Environment implements ProsopoEnvironment {
 				await this.db.connect();
 				this.logger.info(() => ({ msg: "Connected to db" }));
 			}
+			// Set the default datasetId to the most recently uploaded dataset
+			if (this.db && !this.datasetId) {
+				try {
+					this.datasetId = await this.db.getMostRecentDatasetId();
+					if (this.datasetId) {
+						this.logger.info(() => ({
+							msg: "Default dataset ID set",
+							data: { datasetId: this.datasetId },
+						}));
+					} else {
+						this.logger.warn(() => ({
+							msg: "No datasets found in database. Image captchas will not work until a dataset is uploaded.",
+						}));
+					}
+				} catch (err) {
+					this.logger.warn(() => ({
+						msg: "Failed to get most recent dataset ID",
+						data: { error: err },
+					}));
+				}
+			}
 			// Initialize MaxMind geolocation database
 			await this.geolocationService.initialize();
 			this.ready = true;