From 6de46183bbd0456e1ef8491a9b5408d4c44e82fc Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 22:34:50 +0100 Subject: [PATCH 1/3] Remove all SurrealDB references from the codebase SurrealDB was replaced by PostgreSQL and the migration is complete. This removes the SurrealDB client, persistence service, migration tooling, config files, and all fallback logic in PostgresPersistentService. Co-Authored-By: Claude Opus 4.6 --- docs/db-migration.md | 123 --- .../persistence/PersistenceService.groovy | 4 +- .../persistence/WaveContainerRecord.groovy | 3 +- .../service/persistence/WaveScanRecord.groovy | 7 +- .../impl/LocalPersistenceService.groovy | 2 +- .../persistence/impl/SurrealClient.groovy | 89 -- .../impl/SurrealPersistenceService.groovy | 465 ---------- .../persistence/impl/SurrealResult.groovy | 32 - .../migrate/DataMigrationService.groovy | 312 ------- .../persistence/migrate/MigrationOnly.groovy | 35 - .../migrate/cache/DataMigrateCache.groovy | 50 -- .../migrate/cache/DataMigrateEntry.groovy | 41 - .../postgres/PostgresPersistentService.groovy | 66 +- .../service/scan/ScanVulnerability.groovy | 5 +- .../FusionVersionStringDeserializer.groovy | 2 - .../io/seqera/wave/util/StringUtils.groovy | 11 - src/main/resources/application-local.yml | 2 - .../application-surrealdb-legacy.yml | 9 - src/main/resources/application-surrealdb.yml | 9 - .../impl/SurrealPersistenceServiceTest.groovy | 795 ------------------ .../migrate/DataMigrationLockTest.groovy | 69 -- .../migrate/DataMigrationServiceTest.groovy | 188 ----- .../wave/test/SurrealDBTestContainer.groovy | 58 -- .../seqera/wave/util/StringUtilsTest.groovy | 17 - 24 files changed, 22 insertions(+), 2372 deletions(-) delete mode 100644 docs/db-migration.md delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy delete mode 100644 src/main/resources/application-surrealdb-legacy.yml delete mode 100644 src/main/resources/application-surrealdb.yml delete mode 100644 src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy diff --git a/docs/db-migration.md b/docs/db-migration.md deleted file mode 100644 index 380dfb7092..0000000000 --- a/docs/db-migration.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: Wave database migration -description: Migrate your Wave installation from SurrealDB to PostgreSQL -tags: [wave, surrealdb, postgresql, migration] ---- - -[Wave 1.21.0](https://docs.seqera.io/changelog/wave/v1.21.0) introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. - -This guide outlines the steps to migrate your existing Wave installation from SurrealDB to PostgreSQL. - -:::info[**Prerequisites**] -You will need the following to get started: - -- [Wave CLI](./cli/index.md) -- A PostgreSQL database accessible to Wave -::: - -## Database migration - -1. Generate build ID, scan ID, token, and mirror ID data to verify your migration in step 3. - - 1. Run a Wave build operation and capture the `buildId`: - - ```bash - wave --conda-package bwa --wave-endpoint - ``` - - 1. Verify the build and record the build ID: - - ```bash - curl /view/builds/ - ``` - - 1. Verify the scan and record the scan ID: - - ```bash - curl /view/scans/ - ``` - - 1. Create a container augmentation and record the token: - - ```bash - wave -i ubuntu --config-file --wave-endpoint - ``` - - 1. Verify the container and record the token: - - ```bash - curl /view/containers/ - ``` - - 1. Create a mirror operation and note the mirror ID: - - ```bash - wave --mirror -i ubuntu --build-repo --wave-endpoint - ``` - - 1. Verify the mirror and record the mirror ID: - - ```bash - curl /view/mirrors/ - ``` - -1. Migrate your database: - - 1. Add the following to your `MICRONAUT_ENVIRONMENTS`: - - `postgres` - - `surrealdb` - - `migrate` - - `redis` - - 1. Start the Wave application: - - ```console - INFO i.s.w.s.p.m.DataMigrationService - Data migration service initialized - ``` - - 1. Check the logs for the migration status: - - ```console - INFO i.s.w.s.p.m.DataMigrationService - All wave_request records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_scan records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_build records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_mirror records migrated. - ``` - - 1. When all records are migrated, remove `migrate` and `surrealdb` from your `MICRONAUT_ENVIRONMENTS`, then restart Wave. - -1. Use the build ID, scan ID, token, and mirror ID you generated in step 1 to verify the migration: - - 1. Verify the build data: - - ```bash - curl /view/builds/ - ``` - - 1. Verify the scan data: - - ```bash - curl /view/scans/ - ``` - - 1. Verify the container data: - - ```bash - curl /view/containers/ - ``` - - 1. Verify the mirror data: - - ```bash - curl /view/mirrors/ - ``` - -## Wave configuration - -Add the following properties to your Wave configuration file: - -`wave.build.logs.path` -: Sets the full path where build logs will be stored. Can be an S3 URI (e.g., `s3://my-bucket/wave/logs`) or a local filesystem path. - -`wave.build.locks.path` -: Sets the full path where Conda lock files will be stored. Can be an S3 URI (e.g., `s3://my-bucket/wave/locks`) or a local filesystem path. diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 1aee6bc3ad..54935f04b8 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -78,7 +78,7 @@ interface PersistenceService { List allBuilds(String containerId) /** - * Store a {@link WaveContainerRecord} object in the Surreal wave_request table. + * Store a {@link WaveContainerRecord} object in the persistence layer. * * @param data A {@link WaveContainerRecord} object representing a Wave request record */ @@ -101,7 +101,7 @@ interface PersistenceService { WaveContainerRecord loadContainerRequest(String token) /** - * Store a {@link WaveScanRecord} object in the Surreal wave_scan table. + * Store a {@link WaveScanRecord} object in the persistence layer. * * @param data A {@link WaveScanRecord} object representing the a container scan request */ diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index e0ee15ddfa..7261020adc 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -44,8 +44,7 @@ import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime class WaveContainerRecord { /** - * wave request id, this will be the token - * This is container token and it is named as id for surrealdb requirement + * Wave request id, this will be the token */ @PostgresIgnore final String id diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index d1d5fab88b..cbdff2e9ec 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -29,7 +29,6 @@ import groovy.util.logging.Slf4j import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.util.StringUtils /** * Model a Wave container scan result * @@ -75,7 +74,7 @@ class WaveScanRecord implements Cloneable { Path workDir ) { - this.id = StringUtils.surrealId(id) + this.id = id this.buildId = buildId this.mirrorId = mirrorId this.requestId = requestId @@ -93,7 +92,7 @@ class WaveScanRecord implements Cloneable { } WaveScanRecord(ScanEntry scan) { - this.id = StringUtils.surrealId(scan.scanId) + this.id = scan.scanId this.buildId = scan.buildId this.mirrorId = scan.mirrorId this.requestId = scan.requestId @@ -121,7 +120,7 @@ class WaveScanRecord implements Cloneable { } void setId(String id) { - this.id = StringUtils.surrealId(id) + this.id = id } Boolean succeeded() { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index 73869d2a57..aede84bc92 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -39,7 +39,7 @@ import jakarta.inject.Singleton @Singleton @CompileStatic @Secondary -@Requires(notEnv = "surrealdb") +@Requires(notEnv = "postgres") @TraceElapsedTime(thresholdMillis = '${wave.trace.local-persistence.threshold:100}') class LocalPersistenceService implements PersistenceService { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy deleted file mode 100644 index 2901ec22fc..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import groovy.transform.CompileStatic -import io.micronaut.context.annotation.Requires -import io.micronaut.http.annotation.Body -import io.micronaut.http.annotation.Get -import io.micronaut.http.annotation.Header -import io.micronaut.http.annotation.Post -import io.micronaut.http.client.annotation.Client -import io.micronaut.retry.annotation.Retryable -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.scan.ScanVulnerability -import reactor.core.publisher.Flux -/** - * Declarative http client for SurrealDB - * - * @author : jorge - * - */ -@Requires(env='surrealdb') -@CompileStatic -@Header(name = "Content-type", value = "application/json") -@Header(name = "ns", value = '${surreal.default.ns}') -@Header(name = "db", value = '${surreal.default.db}') -@Client(value = '${surreal.default.url}') -@Retryable( - delay = '${wave.surreal.retry.delay:1s}', - maxDelay = '${wave.surreal.retry.maxDelay:10s}', - attempts = '${wave.surreal.retry.attempts:3}', - multiplier = '${wave.surreal.retry.multiplier:1.5}', - predicate = RetryOnIOException ) -interface SurrealClient { - - @Post("/sql") - Flux> sqlAsync(@Header String authorization, @Body String body) - - @Post("/sql") - Flux>> sqlAsyncMany(@Header String authorization, @Body String body) - - @Post("/sql") - Map sqlAsMap(@Header String authorization, @Body String body) - - @Post("/sql") - List> sqlAsList(@Header String authorization, @Body String body) - - @Post("/sql") - String sqlAsString(@Header String authorization, @Body String body) - - @Post('/key/wave_build') - Flux> insertBuildAsync(@Header String authorization, @Body WaveBuildRecord body) - - @Get('/key/wave_build') - String loadBuild(@Header String authorization, String buildId) - - @Post('/key/wave_build') - Map insertBuild(@Header String authorization, @Body WaveBuildRecord body) - - @Get('/key/wave_request/{token}') - String getContainerRequest(@Header String authorization, String token) - - @Post('/key/wave_scan') - Map insertScanRecord(@Header String authorization, @Body WaveScanRecord body) - - @Post('/key/wave_scan_vuln') - Map insertScanVulnerability(@Header String authorization, @Body ScanVulnerability scanVulnerability) - - @Post('/key/wave_mirror') - Flux> insertMirrorAsync(@Header String authorization, @Body MirrorResult body) -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy deleted file mode 100644 index d1f36c8a21..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import java.util.concurrent.CompletableFuture - -import com.fasterxml.jackson.core.type.TypeReference -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Secondary -import io.micronaut.context.annotation.Value -import io.micronaut.core.annotation.Nullable -import io.micronaut.http.client.exceptions.HttpClientResponseException -import io.micronaut.runtime.event.ApplicationStartupEvent -import io.micronaut.runtime.event.annotation.EventListener -import io.seqera.util.trace.TraceElapsedTime -import io.seqera.wave.core.ContainerDigestPair -import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.mirror.MirrorEntry -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.PersistenceService -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.migrate.MigrationOnly -import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.util.JacksonHelper -import jakarta.inject.Inject -import jakarta.inject.Singleton -/** - * Implements a persistence service based based on SurrealDB - * - * @author : jorge - * @author : Paolo Di Tommaso - */ -@Requires(env='surrealdb') -@Slf4j -@Secondary -@Singleton -@CompileStatic -@TraceElapsedTime(thresholdMillis = '${wave.trace.surreal-persistence.threshold:500}') -class SurrealPersistenceService implements PersistenceService { - - @Inject - private SurrealClient surrealDb - - @Value('${surreal.default.user}') - private String user - - @Value('${surreal.default.password}') - private String password - - @Nullable - @Value('${surreal.default.init-db}') - private Boolean initDb - - @EventListener - void onApplicationStartup(ApplicationStartupEvent event) { - if (initDb) - initializeDb() - } - - void initializeDb(){ - // create wave_build table - final ret1 = surrealDb.sqlAsMap(authorization, "define table wave_build SCHEMALESS") - if( ret1.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_build - cause: $ret1") - // create wave_request table - final ret2 = surrealDb.sqlAsMap(authorization, "define table wave_request SCHEMALESS") - if( ret2.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_request - cause: $ret2") - // create wave_scan table - final ret3 = surrealDb.sqlAsMap(authorization, "define table wave_scan SCHEMALESS") - if( ret3.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_scan - cause: $ret3") - // create wave_scan table - final ret4 = surrealDb.sqlAsMap(authorization, "define table wave_scan_vuln SCHEMALESS") - if( ret4.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_scan_vuln - cause: $ret4") - // create wave_mirror table - final ret5 = surrealDb.sqlAsMap(authorization, "define table wave_mirror SCHEMALESS") - if( ret5.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_mirror - cause: $ret5") - } - - protected String getAuthorization() { - "Basic "+"$user:$password".bytes.encodeBase64() - } - - @Override - CompletableFuture saveBuildAsync(WaveBuildRecord build) { - // note: use surreal sql in order to by-pass issue with large payload - // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 - final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}" - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Build request with id '$build.buildId' saved record: ${result}" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving Build request record ${msg}\n${build}", error) - future.complete(null) - }) - return future - } - - @Override - WaveBuildRecord loadBuild(String buildId) { - if( !buildId ) - throw new IllegalArgumentException("Missing 'buildId' argument") - def result = loadBuild0(buildId) - if( result ) - return result - // try to lookup legacy record - final legacyId = BuildRequest.legacyBuildId(buildId) - return legacyId ? loadBuild1(legacyId) : null - } - - private WaveBuildRecord loadBuild0(String buildId) { - final query = "select * from wave_build where buildId = '$buildId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - private WaveBuildRecord loadBuild1(String buildId) { - final query = "select * from wave_build where buildId = '$buildId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - WaveBuildRecord loadBuildSucceed(String targetImage, String digest) { - final query = """\ - select * from wave_build - where - targetImage = '$targetImage' - and digest = '$digest' - and exitStatus = 0 - and duration is not null - order by - startTime desc limit 1 - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - WaveBuildRecord latestBuild(String containerId) { - final query = """ - select * - from wave_build - where buildId ~ '${containerId}${BuildRequest.SEP}' - order by startTime desc limit 1 - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - List allBuilds(String containerId) { - final query = """ - select * - from wave_build - where string::matches(buildId, '^(bd-)?${containerId}_[0-9]+') - order by startTime desc - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @Override - CompletableFuture saveContainerRequestAsync(WaveContainerRecord data) { - // note: use surreal sql in order to by-pass issue with large payload - // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 - final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}" - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$data.id' saved record: ${result}" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving container request record ${msg}\n${data}", error) - future.complete(null) - }) - return future - } - - CompletableFuture updateContainerRequestAsync(String token, ContainerDigestPair digest) { - final query = """\ - UPDATE wave_request:$token SET - sourceDigest = '$digest.source', - waveDigest = '${digest.target}' - """.stripIndent() - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$token' updated record: ${result}" - return future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error update container record=$token => ${msg}\ndigest=${digest}\n", error) - return future.complete(null) - }) - return future - } - - @Override - WaveContainerRecord loadContainerRequest(String token) { - if( !token ) - throw new IllegalArgumentException("Missing 'token' argument") - final json = surrealDb.getContainerRequest(getAuthorization(), token) - log.trace "Container request with token '$token' loaded: ${json}" - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(patchSurrealId(json,"wave_request"), type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - static protected String patchSurrealId(String json, String table) { - return json.replaceFirst(/"id":\s*"${table}:(\w*)"/) { List it-> /"id":"${it[1]}"/ } - } - - @Override - CompletableFuture saveScanRecordAsync(WaveScanRecord scanRecord) { - final vulnerabilities = scanRecord.vulnerabilities ?: List.of() - - // create a multi-command surreal sql statement to insert all vulnerabilities - // and the scan record in a single operation - List ids = new ArrayList<>(101) - String statement = '' - // save all vulnerabilities - for( ScanVulnerability it : vulnerabilities ) { - statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n" - ids << "wave_scan_vuln:⟨$it.id⟩".toString() - } - - // scan object - final copy = scanRecord.clone() - copy.vulnerabilities = List.of() - final json = JacksonHelper.toJson(copy) - - // add the wave_scan record - statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() - final future = new CompletableFuture() - // store the statement using an async operation - surrealDb - .sqlAsyncMany(getAuthorization(), statement) - .subscribe({result -> - log.trace "Scan record save result=$result" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving scan record => ${msg}\n", error) - future.complete(null) - }) - return future - } - - protected String patchScanVulnerabilities(String json, List ids) { - final value = "\"vulnerabilities\":${ids.collect(it-> "\"$it\"").toString()}" - json.replaceFirst(/"vulnerabilities":\s*\[]/, value) - } - - @Override - boolean existsScanRecord(String scanId) { - final statement = "SELECT count() FROM wave_scan where id == 'wave_scan:⟨$scanId⟩'" - final json = surrealDb.sqlAsString(getAuthorization(), statement) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0].count==1 : false - return result - } - - @Override - WaveScanRecord loadScanRecord(String scanId) { - if( !scanId ) - throw new IllegalArgumentException("Missing 'scanId' argument") - final statement = "SELECT * FROM wave_scan where id == 'wave_scan:⟨$scanId⟩' FETCH vulnerabilities" - final json = surrealDb.sqlAsString(getAuthorization(), statement) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - List allScans(String scanId) { - final query = """ - select * - from wave_scan - where string::matches(type::string(id), '^wave_scan:⟨(sc-)?${scanId}_[0-9]+') - order by startTime desc - FETCH vulnerabilities - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - // === mirror operations - - /** - * Load a mirror state record - * - * @param mirrorId The ID of the mirror record - * @return The corresponding {@link MirrorEntry} object or null if it cannot be found - */ - MirrorResult loadMirrorResult(String mirrorId) { - final query = "select * from wave_mirror where mirrorId = '$mirrorId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - /** - * Load a mirror state record given the target image name and the image digest. - * It returns the latest succeed mirror result. - * - * @param targetImage The target mirrored image name - * @param digest The image content SHA256 digest - * @return The corresponding {@link MirrorEntry} object or null if it cannot be found - */ - MirrorResult loadMirrorSucceed(String targetImage, String digest) { - final query = """ - select * from wave_mirror - where - targetImage = '$targetImage' - and digest = '$digest' - and exitCode = 0 - and status = '${MirrorResult.Status.COMPLETED}' - order by - creationTime desc limit 1 - """ - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - /** - * Persists a {@link MirrorEntry} object - * - * @param mirror {@link MirrorEntry} object - */ - @Override - CompletableFuture saveMirrorResultAsync(MirrorResult mirror) { - final future = new CompletableFuture() - surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> - log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" - return future.complete(null) - }, {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving Mirror request record ${msg}\n${mirror}", error) - return future.complete(null) - }) - return future - } - - @MigrationOnly - List getBuildsPaginated(int limit, int offset) { - final query = "select * from wave_build limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getRequestsPaginated(int limit, int offset){ - final query = "select * from wave_request limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getScansPaginated(int limit, int offset){ - final query = """ - select * - from wave_scan - limit $limit start $offset - FETCH vulnerabilities - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getMirrorsPaginated(int limit, int offset){ - final query = "select * from wave_mirror limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy deleted file mode 100644 index fa130928ca..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -/** - * Model a Surreal Resultset - * - * @author Paolo Di Tommaso - */ -class SurrealResult { - - String time - String status - T[] result - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy deleted file mode 100644 index a9b046c18b..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import static io.seqera.wave.util.DurationUtils.* - -import java.time.Duration -import java.util.concurrent.ScheduledFuture -import java.util.function.Consumer -import java.util.function.Function - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.ApplicationContext -import io.micronaut.context.annotation.Context -import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Value -import io.micronaut.context.env.Environment -import io.micronaut.data.exceptions.DataAccessException -import io.micronaut.runtime.event.annotation.EventListener -import io.micronaut.runtime.server.event.ServerShutdownEvent -import io.micronaut.runtime.server.event.ServerStartupEvent -import io.micronaut.scheduling.TaskScheduler -import io.seqera.util.redis.JedisLock -import io.seqera.util.redis.JedisLockManager -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealClient -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateCache -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateEntry -import io.seqera.wave.service.persistence.postgres.PostgresPersistentService -import jakarta.inject.Inject -import redis.clients.jedis.Jedis -import redis.clients.jedis.JedisPool -import static io.seqera.wave.util.DurationUtils.randomDuration - -/** - * Service to migrate data from SurrealDB to Postgres - * - * @author Munish Chouhan - */ -@Requires(env='migrate') -@Slf4j -@Context -@CompileStatic -@MigrationOnly -class DataMigrationService { - - public static final String TABLE_NAME_BUILD = 'wave_build' - public static final String TABLE_NAME_REQUEST = 'wave_request' - public static final String TABLE_NAME_SCAN = 'wave_scan' - public static final String TABLE_NAME_MIRROR = 'wave_mirror' - - @Value('${wave.db.migrate.page-size:200}') - private int pageSize - - @Value('${wave.db.migrate.delay:5s}') - private Duration delay - - @Value('${wave.db.migrate.initial-delay:70s}') - private Duration launchDelay - - @Value('${wave.db.migrate.initial-delay:10s}') - private Duration initialDelay - - @Value('${wave.db.migrate.iteration-delay:100ms}') - private Duration iterationDelay - - @Inject - private SurrealPersistenceService surrealService - - @Inject - private PostgresPersistentService postgresService - - @Inject - private SurrealClient surrealDb - - @Inject - private DataMigrateCache dataMigrateCache - - @Inject - private TaskScheduler taskScheduler - - @Inject - private Environment environment - - @Inject - private ApplicationContext applicationContext - - @Inject - private JedisPool pool - - private volatile Jedis conn - - private volatile JedisLock lock - - private static final String LOCK_KEY = "migrate-lock/v2" - - private volatile ScheduledFuture mirrorTask - private volatile ScheduledFuture buildTask - private volatile ScheduledFuture scanTask - private volatile ScheduledFuture requestTask - - - @EventListener - void start(ServerStartupEvent event) { - if (!environment.activeNames.contains("surrealdb") || !environment.activeNames.contains("postgres")) { - throw new IllegalStateException("Both 'surrealdb' and 'postgres' environments must be active.") - } - // launch async to not block bootstrap - taskScheduler.schedule(launchDelay, ()->{ - try { - launchMigration() - } - catch (InterruptedException e) { - log.info "Migration launch has been interrupted (1)" - } - catch (Throwable e) { - log.info("Unexpected exception during Migration launch", e) - } - }) - } - - @EventListener - void stop(ServerShutdownEvent event) { - log.info "Releasing lock and closing connection" - // remove the lock & close the connection - lock?.release() - conn?.close() - } - - void launchMigration() { - log.info("Data migration service initialized with page size: $pageSize, delay: $delay, initial delay: $initialDelay") - // acquire the lock to only run one instance at time - conn = pool.getResource() - lock = acquireLock(conn, LOCK_KEY) - if( !lock ) { - log.debug "Skipping migration since lock cannot be acquired" - conn.close() - conn = null - return - } - log.info("Data migration initiated") - - dataMigrateCache.putIfAbsent(TABLE_NAME_BUILD, new DataMigrateEntry(TABLE_NAME_BUILD, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_REQUEST, new DataMigrateEntry(TABLE_NAME_REQUEST, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_SCAN, new DataMigrateEntry(TABLE_NAME_SCAN, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_MIRROR, new DataMigrateEntry(TABLE_NAME_MIRROR, 0)) - - buildTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateBuildRecords) - requestTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateRequests) - scanTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateScanRecords) - mirrorTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateMirrorRecords) - } - - /** - * Migrate data from SurrealDB to Postgres - */ - void migrateBuildRecords() { - migrateRecords(TABLE_NAME_BUILD, - (Integer offset)-> surrealService.getBuildsPaginated(pageSize, offset), - (WaveBuildRecord it)-> postgresService.saveBuild(it), - buildTask ) - } - - /** - * Migrate container requests from SurrealDB to Postgres - */ - void migrateRequests() { - migrateRecords(TABLE_NAME_REQUEST, - (Integer offset)-> surrealService.getRequestsPaginated(pageSize, offset), - (WaveContainerRecord request)-> postgresService.saveContainerRequest(fixRequestId(request.id), request), - requestTask ) - } - - protected static String fixRequestId(String id){ - if (id?.contains("wave_request:")){ - if(id.contains("wave_request:⟨")){ - return id.takeAfter("wave_request:⟨").takeBefore("⟩") - } - return id.takeAfter("wave_request:") - } - return id - } - - /** - * Migrate scan records from SurrealDB to Postgres - */ - void migrateScanRecords() { - migrateRecords(TABLE_NAME_SCAN, - (Integer offset)-> surrealService.getScansPaginated(pageSize, offset), - (WaveScanRecord it)-> postgresService.saveScanRecord(it), - scanTask ) - } - - /** - * Migrate mirror records from SurrealDB to Postgres - */ - void migrateMirrorRecords() { - migrateRecords(TABLE_NAME_MIRROR, - (Integer offset)-> surrealService.getMirrorsPaginated(pageSize, offset), - (MirrorResult it)-> postgresService.saveMirrorResult(it), - mirrorTask ) - } - - void migrateRecords(String tableName, Function> fetch, Consumer saver, ScheduledFuture task) { - try { - migrateRecords0(tableName, fetch, saver, task) - } - catch (InterruptedException e) { - log.info "Migration $tableName has been interrupted (2)" - Thread.currentThread().interrupt() - } - catch (Throwable t) { - log.error("Unexpected migration error - ${t.message}", t) - } - } - - void migrateRecords0(String tableName, Function> fetch, Consumer saver, ScheduledFuture task) { - log.info "Initiating $tableName migration" - int offset = dataMigrateCache.get(tableName).offset - def records = fetch.apply(offset) - - if (!records) { - log.info("All $tableName records migrated.") - task.cancel(false) - return - } - - int count=0 - for (def it : records) { - try { - if( Thread.currentThread().isInterrupted() ) { - log.info "Thread is interrupted - exiting $tableName method" - break - } - saver.accept(it) - dataMigrateCache.put(tableName, new DataMigrateEntry(tableName, ++offset)) - if( ++count % 50 == 0 ) - log.info "Migration ${tableName}; processed ${count} records" - Thread.sleep(iterationDelay.toMillis()) - } - catch (InterruptedException e) { - log.info "Migration $tableName has been interrupted (3)" - Thread.currentThread().interrupt() - } - catch (DataAccessException dataAccessException) { - if (dataAccessException.message.contains("duplicate key value violates unique constraint")) { - log.warn("Duplicate key error for $tableName record: ${dataAccessException.message}") - dataMigrateCache.put(tableName, new DataMigrateEntry(tableName, ++offset)) - } else { - log.error("Error saving=> $tableName record: ${dataAccessException.message}") - } - } - catch (Exception e) { - log.error("Error saving $tableName record: ${e.message}", e) - } - } - - log.info("Migrated ${records.size()} $tableName records (offset $offset)") - } - - // == --- jedis lock handling - static JedisLock acquireLock(Jedis conn, String key, Duration timeout=Duration.ofMinutes(10)) { - try { - final max = timeout.toMillis() - final begin = System.currentTimeMillis() - while( !Thread.currentThread().isInterrupted() ) { - if( System.currentTimeMillis()-begin > max ) { - log.info "Lock acquire timeout reached" - return null - } - final lock = new JedisLockManager(conn) - .withLockAutoExpireDuration(Duration.ofDays(30)) - .tryAcquire(key) - if( lock ) - return lock - log.info "Unable to acquire lock - await 1s before retrying" - sleep(10_000) - } - } - catch (InterruptedException e) { - log.info "Migration acquire lock has been interrupted (4)" - Thread.currentThread().interrupt() - return null - } - catch (Throwable t) { - log.error("Unexpected error while trying to acquire the lock - ${t.message}", t) - return null - } - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy deleted file mode 100644 index c8efc7a7a3..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import java.lang.annotation.ElementType -import java.lang.annotation.Retention -import java.lang.annotation.RetentionPolicy -import java.lang.annotation.Target - -/** - * Marker annotation to annotate logic required for Surreal to Postgres migration - * - * @author Paolo Di Tommaso - */ -@Retention(RetentionPolicy.RUNTIME) -@Target([ElementType.TYPE, ElementType.METHOD]) -@interface MigrationOnly { - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy deleted file mode 100644 index 31910dc2d7..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate.cache - -import java.time.Duration - -import groovy.transform.CompileStatic -import io.seqera.serde.moshi.MoshiEncodeStrategy -import io.seqera.data.store.state.AbstractStateStore -import io.seqera.data.store.state.impl.StateProvider -import jakarta.inject.Singleton -/** - * Cache for data migration entries - * - * @author Munish Chouhan - */ -@Singleton -@CompileStatic -class DataMigrateCache extends AbstractStateStore { - - DataMigrateCache(StateProvider provider) { - super(provider, new MoshiEncodeStrategy() {}) - } - - @Override - protected String getPrefix() { - return 'migrate-surreal/v1' - } - - @Override - protected Duration getDuration() { - return Duration.ofDays(30) - } -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy deleted file mode 100644 index 938e3db88b..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate.cache - -import groovy.transform.CompileStatic -import groovy.transform.EqualsAndHashCode -import groovy.transform.ToString - -/** - * Data migration entry - * - * @author Munish Chouhan - */ -@ToString(includeNames = true, includePackage = false) -@EqualsAndHashCode -@CompileStatic -class DataMigrateEntry { - String tableName - int offset - - DataMigrateEntry(String tableName, int offset) { - this.tableName = tableName - this.offset = offset - } -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy index 01a01ec0df..3508cee637 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires -import io.micronaut.core.annotation.Nullable import io.micronaut.runtime.event.ApplicationStartupEvent import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors @@ -36,8 +35,6 @@ import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.MigrationOnly import io.seqera.wave.service.persistence.postgres.data.BuildRepository import io.seqera.wave.service.persistence.postgres.data.BuildRow import io.seqera.wave.service.persistence.postgres.data.MirrorRepository @@ -75,10 +72,6 @@ class PostgresPersistentService implements PersistenceService { @Inject private ScanRepository scanRepository - @Inject - @Nullable - private SurrealPersistenceService surrealPersistenceService - @Inject @Named(TaskExecutors.BLOCKING) private ExecutorService ioExecutor @@ -116,21 +109,12 @@ class PostgresPersistentService implements PersistenceService { "Unable to save build record=$data")) } - @MigrationOnly - void saveBuild(WaveBuildRecord data) { - log.trace "Saving build record=$data" - final json = Mapper.toJson(data) - final row = new BuildRow(id:data.buildId, data:json, createdAt: Instant.now()) - buildRepository.save(row) - } - @Override WaveBuildRecord loadBuild(String buildId) { log.trace "Loading build record id=${buildId}" final row = buildRepository.findById(buildId).orElse(null) - if( !row ) { - return surrealPersistenceService?.loadBuild(buildId) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: buildId] ) } @@ -138,9 +122,8 @@ class PostgresPersistentService implements PersistenceService { WaveBuildRecord loadBuildSucceed(String targetImage, String digest) { log.trace "Loading build record with image=${targetImage}; digest=${digest}" final row = buildRepository.findByTargetAndDigest(targetImage, digest) - if( !row ){ - return surrealPersistenceService?.loadBuildSucceed(targetImage, digest) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: row.id] ) } @@ -148,9 +131,8 @@ class PostgresPersistentService implements PersistenceService { WaveBuildRecord latestBuild(String containerId) { log.trace "Loading latest build with containerId=${containerId}" final row = buildRepository.findLatestByBuildId(containerId) - if( !row ){ - return surrealPersistenceService?.latestBuild(containerId) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: row.id] ) } @@ -160,17 +142,10 @@ class PostgresPersistentService implements PersistenceService { final result = buildRepository.findAllByBuildId(containerId) return result ? result.collect((it)-> Mapper.fromJson(WaveBuildRecord, it.data, [buildId: it.id])) - : surrealPersistenceService?.allBuilds(containerId) + : List.of() } // ===== --- container records ---- ===== - @MigrationOnly - void saveContainerRequest(String id, WaveContainerRecord data) { - final json = Mapper.toJson(data) - final entity = new RequestRow(id: id, data:json, createdAt: Instant.now()) - requestRepository.save(entity) - } - @Override CompletableFuture saveContainerRequestAsync(WaveContainerRecord data) { log.trace "Saving container request data=${data}" @@ -194,7 +169,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading container request token=${token}" final row = requestRepository.findById(token).orElse(null) if( !row || !row.data ) - return surrealPersistenceService?.loadContainerRequest(token) + return null return Mapper.fromJson(WaveContainerRecord, row.data, [id: token]) } @@ -210,27 +185,19 @@ class PostgresPersistentService implements PersistenceService { "Unable to save scan record data=${data}")) } - @MigrationOnly - void saveScanRecord(WaveScanRecord data) { - log.trace "Saving scan record data=${data}" - final json = Mapper.toJson(data) - final entity = new ScanRow(id: data.id, data:json, createdAt: Instant.now()) - scanRepository.save(entity) - } - @Override WaveScanRecord loadScanRecord(String scanId) { log.trace "Loading scan record id=${scanId}" final row = scanRepository.findById(scanId).orElse(null) if( !row || !row.data ) - return surrealPersistenceService?.loadScanRecord(scanId) + return null return Mapper.fromJson(WaveScanRecord, row.data, [id: scanId]) } @Override boolean existsScanRecord(String scanId) { log.trace "Exist scan record id=${scanId}" - return scanRepository.existsById(scanId) ?: surrealPersistenceService?.existsScanRecord(scanId) + return scanRepository.existsById(scanId) } @Override @@ -239,7 +206,7 @@ class PostgresPersistentService implements PersistenceService { final result = scanRepository.findAllByScanId(scanId) return result ? result.collect((it)-> Mapper.fromJson(WaveScanRecord, it.data, [id: it.id])) - : surrealPersistenceService?.allScans(scanId) + : List.of() } // ===== --- mirror records ---- ===== @@ -249,7 +216,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading mirror result with id=${mirrorId}" final row = mirrorRepository.findById(mirrorId).orElse(null) if( !row ) - return surrealPersistenceService?.loadMirrorResult(mirrorId) + return null return Mapper.fromJson(MirrorResult, row.data, [mirrorId: mirrorId]) } @@ -258,7 +225,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading mirror succeed with image=${targetImage}; digest=${digest}" final row = mirrorRepository.findByTargetAndDigest(targetImage, digest, MirrorResult.Status.COMPLETED) if( !row ) - return surrealPersistenceService?.loadMirrorSucceed(targetImage, digest) + return null return Mapper.fromJson(MirrorResult, row.data, [mirrorId: row.id]) } @@ -272,11 +239,4 @@ class PostgresPersistentService implements PersistenceService { "Unable to save mirror result data=${data}")) } - @MigrationOnly - void saveMirrorResult(MirrorResult data) { - log.trace "Saving mirror result data=${data}" - final json = Mapper.toJson(data) - final entity = new MirrorRow(id: data.mirrorId, data:json, createdAt: Instant.now()) - mirrorRepository.save(entity) - } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy index bd73f9e1d1..5ed4dc6541 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy @@ -23,7 +23,6 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.seqera.wave.util.StringUtils import org.jetbrains.annotations.NotNull /** * Model for Scan Vulnerability @@ -54,7 +53,7 @@ class ScanVulnerability implements Comparable { ScanVulnerability(){} ScanVulnerability(String id, String severity, String title, String pkgName, String installedVersion, String fixedVersion, String primaryUrl) { - this.id = StringUtils.surrealId(id) + this.id = id this.severity = severity this.title = title this.pkgName = pkgName @@ -64,7 +63,7 @@ class ScanVulnerability implements Comparable { } void setId(String id) { - this.id = StringUtils.surrealId(id) + this.id = id } @Override diff --git a/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy b/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy index 0d8bed77dc..fd54775524 100644 --- a/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy +++ b/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy @@ -23,14 +23,12 @@ import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.JsonDeserializer import com.fasterxml.jackson.databind.JsonNode import groovy.transform.CompileStatic -import io.seqera.wave.service.persistence.migrate.MigrationOnly /** * Custom deserializer for Fusion version strings. * * @author Munish Chouhan */ -@MigrationOnly @CompileStatic class FusionVersionStringDeserializer extends JsonDeserializer { @Override diff --git a/src/main/groovy/io/seqera/wave/util/StringUtils.groovy b/src/main/groovy/io/seqera/wave/util/StringUtils.groovy index a610b64928..36208b56e1 100644 --- a/src/main/groovy/io/seqera/wave/util/StringUtils.groovy +++ b/src/main/groovy/io/seqera/wave/util/StringUtils.groovy @@ -68,17 +68,6 @@ class StringUtils { return m.matches() ? m.group(1) : null } - static String surrealId(String id) { - if( !id ) - return null - final p = id.indexOf(':') - if( p!=-1 ) - id = id.substring(p+1) - if( id.startsWith('⟨') && id.endsWith('⟩')) - id = id.substring(1,id.length()-1) - return id - } - static String pathConcat(String base, String path) { assert base, "Missing 'base' argument" // strip ending slash diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index e6a8b4cc9e..44480ae3e2 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -32,8 +32,6 @@ wave: trace: local-persistence: threshold: 100 - surreal-persistence: - threshold: 100 proxy-service: threshold: 100 --- diff --git a/src/main/resources/application-surrealdb-legacy.yml b/src/main/resources/application-surrealdb-legacy.yml deleted file mode 100644 index dcfd8b90b0..0000000000 --- a/src/main/resources/application-surrealdb-legacy.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -surreal: - legacy: - ns: "seqera" - db: "wave" - url: "http://surrealdb:8000" - user: "root" - password: "root" -... diff --git a/src/main/resources/application-surrealdb.yml b/src/main/resources/application-surrealdb.yml deleted file mode 100644 index d60e8da74d..0000000000 --- a/src/main/resources/application-surrealdb.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -surreal: - default: - ns : "seqera" - db : "wave" - url: "http://surrealdb:8000" - user: "root" - password: "root" -... diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy deleted file mode 100644 index 2147795690..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import spock.lang.Specification - -import java.nio.file.Path -import java.time.Duration -import java.time.Instant -import java.util.concurrent.CompletableFuture - -import io.micronaut.context.ApplicationContext -import io.micronaut.http.HttpRequest -import io.micronaut.http.client.HttpClient -import io.micronaut.http.client.exceptions.HttpClientResponseException -import io.seqera.wave.api.BuildCompression -import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.api.ContainerLayer -import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.core.ContainerDigestPair -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.request.ContainerRequest -import io.seqera.wave.service.builder.BuildEvent -import io.seqera.wave.service.builder.BuildFormat -import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.service.mirror.MirrorEntry -import io.seqera.wave.service.mirror.MirrorRequest -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.test.SurrealDBTestContainer -import io.seqera.wave.tower.PlatformId -import io.seqera.wave.tower.User -import io.seqera.wave.util.JacksonHelper -import org.apache.commons.lang3.RandomStringUtils - -/** - * @author : jorge - * - */ -class SurrealPersistenceServiceTest extends Specification implements SurrealDBTestContainer { - - ApplicationContext applicationContext - - String getSurrealDbURL() { - "http://$surrealHostName:$surrealPort" - } - - def setup() { - restartDb() - applicationContext = ApplicationContext.run([ - surreal:['default': [ - user : 'root', - password : 'root', - ns : 'test', - db : 'test', - url : surrealDbURL, - 'init-db': false - ]]] - , 'test', 'surrealdb') - } - - def cleanup() { - applicationContext.close() - } - - void "can connect"() { - given: - def httpClient = HttpClient.create(new URL(surrealDbURL)) - - when: - def str = httpClient.toBlocking() - .retrieve( - HttpRequest.POST("/sql", "SELECT * FROM count()") - .headers(['ns': 'test', 'db': 'test', 'accept':'application/json']) - .basicAuth('root', 'root'), Map) - - then: - str.result.first() == 1 - } - - void "can insert an async build"() { - given: - final String dockerFile = """\ - FROM quay.io/nextflow/bash - RUN echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt - ENV NOW=${System.currentTimeMillis()} - """ - final String condaFile = """ - echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt - """ - def storage = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: dockerFile, - condaFile: condaFile, - workspace: Path.of("."), - targetImage: 'docker.io/my/repo:container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1', - ) - def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) - def event = new BuildEvent(request, result) - def build = WaveBuildRecord.fromEvent(event) - - when: - storage.initializeDb() - and: - storage.saveBuildAsync(build) - then: - sleep 100 - def stored = storage.loadBuild(request.buildId) - stored.buildId == request.buildId - stored.requestIp == '127.0.0.1' - } - - def 'should load a build record' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: 'FROM foo:latest', - condaFile: 'conda::recipe', - workspace: Path.of("."), - targetImage: 'docker.io/my/repo:container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1', - ) - def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) - def event = new BuildEvent(request, result) - def record = WaveBuildRecord.fromEvent(event) - - and: - persistence.saveBuildAsync(record) - - when: - sleep 100 - def loaded = persistence.loadBuild(record.buildId) - - then: - loaded == record - } - - def 'should find latest succeed' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def target = 'docker.io/my/target' - def digest = 'sha256:12345' - and: - def request1 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_1', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_2', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_3', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - and: - def result1 = new BuildResult(request1.buildId, 1, "err", request1.startTime, Duration.ofSeconds(2), digest) - def rec1 = WaveBuildRecord.fromEvent(new BuildEvent(request1, result1)) - surreal.insertBuild(auth, rec1) - and: - def result2 = new BuildResult(request2.buildId, 0, "ok", request2.startTime, Duration.ofSeconds(2), digest) - def rec2 = WaveBuildRecord.fromEvent(new BuildEvent(request2, result2)) - surreal.insertBuild(auth, rec2) - and: - def result3 = new BuildResult(request3.buildId, 0, "ok", request3.startTime, Duration.ofSeconds(2), digest) - def rec3 = WaveBuildRecord.fromEvent(new BuildEvent(request3, result3)) - surreal.insertBuild(auth, rec3) - - expect: - persistence.loadBuildSucceed(target, digest) == rec3 - } - - def 'should find latest build' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def request1 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_1' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_2' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - - def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request1, result1))) - and: - def result2 = new BuildResult(request2.buildId, -1, "ok", request2.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request2, result2))) - and: - def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request3, result3))) - - expect: - persistence.latestBuild('abc').buildId == 'bd-abc_3' - persistence.latestBuild('bd-abc').buildId == 'bd-abc_3' - persistence.latestBuild('xyz') == null - } - - def 'should save and update a build' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: 'FROM foo:latest', - condaFile: 'conda::recipe', - workspace: Path.of("/some/path"), - targetImage: 'buildrepo:recipe-container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1' - ) - and: - def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') - - and: - def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) - - when: - persistence.saveBuildAsync(build1) - sleep 100 - then: - persistence.loadBuild(request.buildId) == build1 - - } - - def 'should load a request record' () { - given: - def largeContainerFile = RandomStringUtils.random(25600, true, true) - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def TOKEN = '123abc' - def cfg = new ContainerConfig(entrypoint: ['/opt/fusion'], - layers: [ new ContainerLayer(location: 'https://fusionfs.seqera.io/releases/v2.2.8-amd64.json')]) - def req = new SubmitContainerTokenRequest( - towerEndpoint: 'https://tower.nf', - towerWorkspaceId: 100, - containerConfig: cfg, - containerPlatform: ContainerPlatform.of('amd64'), - buildRepository: 'build.docker.io', - cacheRepository: 'cache.docker.io', - fingerprint: 'xyz', - timestamp: Instant.now().toString() - ) - def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = ContainerRequest.of(requestId: TOKEN, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: largeContainerFile ) - def wave = "wave.io/wt/$TOKEN/hello-world" - def addr = "100.200.300.400" - def exp = Instant.now().plusSeconds(3600) - and: - def request = new WaveContainerRecord(req, data, wave, addr, exp) - and: - persistence.saveContainerRequestAsync(request) - and: - sleep 200 // <-- the above request is async, give time to save it - - when: - def loaded = persistence.loadContainerRequest(TOKEN) - then: - loaded == request - loaded.containerFile == largeContainerFile - - // should update the record - when: - persistence.updateContainerRequestAsync(TOKEN, new ContainerDigestPair('111', '222')) - and: - sleep 200 - then: - def updated = persistence.loadContainerRequest(TOKEN) - and: - updated.sourceDigest == '111' - updated.sourceImage == request.sourceImage - and: - updated.waveDigest == '222' - updated.waveImage == request.waveImage - - } - - def 'should save a scan and load a result' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def surrealDb = applicationContext.getBean(SurrealClient) - def NOW = Instant.now() - def SCAN_ID = 'a1' - def BUILD_ID = '100' - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') - def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') - def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null, null) - when: - persistence.saveScanRecordAsync(scan) - sleep 200 - then: - def result = persistence.loadScanRecord(SCAN_ID) - and: - result == scan - and: - surrealDb - .sqlAsMap(auth, "select * from wave_scan_vuln") - .result - .size() == 3 - and: - persistence.existsScanRecord(SCAN_ID) - - when: - def SCAN_ID2 = 'b2' - def BUILD_ID2 = '102' - def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1, CVE4], 1, "Error 'quote'", null) - and: - // should save the same CVE into another build - persistence.saveScanRecordAsync(scanRecord2) - sleep 200 - then: - def result2 = persistence.loadScanRecord(SCAN_ID2) - and: - result2 == scanRecord2 - and: - surrealDb - .sqlAsMap(auth, "select * from wave_scan_vuln") - .result - .size() == 4 - } - - def 'should save a scan and check it exists' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def surrealDb = applicationContext.getBean(SurrealClient) - def NOW = Instant.now() - def SCAN_ID = 'a1' - def BUILD_ID = '100' - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) - - expect: - !persistence.existsScanRecord(SCAN_ID) - - when: - persistence.saveScanRecordAsync(scan) - sleep 200 - then: - persistence.existsScanRecord(SCAN_ID) - } - - //== mirror records tests - - void "should save and load a mirror record by id"() { - given: - def storage = applicationContext.getBean(SurrealPersistenceService) - and: - def request = MirrorRequest.create( - 'source.io/foo', - 'target.io/foo', - 'sha256:12345', - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-123', - Instant.now(), - "GMT", - Mock(PlatformId) - ) - and: - storage.initializeDb() - and: - def result = MirrorEntry.of(request).getResult() - storage.saveMirrorResultAsync(result) - sleep 100 - - when: - def stored = storage.loadMirrorResult(request.mirrorId) - then: - stored == result - } - - void "should save and load a mirror record by target and digest"() { - given: - def digest = 'sha256:12345' - def timestamp = Instant.now() - def source = 'source.io/foo' - def target = 'target.io/foo' - and: - def storage = applicationContext.getBean(SurrealPersistenceService) - and: - def request1 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-1', - timestamp.minusSeconds(180), - "GMT", - Mock(PlatformId) ) - and: - def request2 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-2', - timestamp.minusSeconds(120), - "GMT", - Mock(PlatformId) ) - and: - def request3 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-3', - timestamp.minusSeconds(60), - "GMT", - Mock(PlatformId) ) - - and: - storage.initializeDb() - and: - def result1 = MirrorResult.of(request1).complete(1, 'err') - def result2 = MirrorResult.of(request2).complete(0, 'ok') - def result3 = MirrorResult.of(request3).complete(0, 'ok') - storage.saveMirrorResultAsync(result1) - storage.saveMirrorResultAsync(result2) - storage.saveMirrorResultAsync(result3) - sleep 100 - - when: - def stored = storage.loadMirrorSucceed(target, digest) - then: - stored == result3 - } - - def 'should remove surreal table from json' () { - given: - def json = /{"id":"wave_request:1234abc", "this":"one", "that":123 }/ - expect: - SurrealPersistenceService.patchSurrealId(json, "wave_request") - == /{"id":"1234abc", "this":"one", "that":123 }/ - } - - def 'should save 50KB container and conda file' (){ - given: - def data = RandomStringUtils.random(25600, true, true) - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: data, - condaFile: data, - workspace: Path.of("/some/path"), - targetImage: 'buildrepo:recipe-container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - compression: BuildCompression.gzip - ) - and: - def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') - - and: - def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) - - when: - persistence.saveBuildAsync(build1) - sleep 100 - then: - persistence.loadBuild(request.buildId) == build1 - } - - def 'should find all builds' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def request1 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_1' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_2' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - def request4 = BuildRequest.of( containerId: 'abc', buildId: 'bd-xyz_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(0), identity: PlatformId.NULL) - - def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) - def record1 = WaveBuildRecord.fromEvent(new BuildEvent(request1, result1)) - surreal.insertBuild(auth, record1) - and: - def result2 = new BuildResult(request2.buildId, -1, "ok", request2.startTime, Duration.ofSeconds(2), null) - def record2 = WaveBuildRecord.fromEvent(new BuildEvent(request2, result2)) - surreal.insertBuild(auth, record2) - and: - def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) - def record3 = WaveBuildRecord.fromEvent(new BuildEvent(request3, result3)) - surreal.insertBuild(auth, record3) - and: - def result4 = new BuildResult(request4.buildId, -1, "ok", request4.startTime, Duration.ofSeconds(2), null) - def record4 = WaveBuildRecord.fromEvent(new BuildEvent(request4, result4)) - surreal.insertBuild(auth, record4) - - expect: - persistence.allBuilds('abc') == [record3, record2, record1] - and: - persistence.allBuilds('bd-abc') == [record3, record2, record1] - and: - persistence.allBuilds('ab') == null - } - - def 'should find all scans' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') - def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') - def scan1 = new WaveScanRecord('sc-1234567890abcdef_1', '100', null, null, CONTAINER_IMAGE, PLATFORM, Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3, CVE4], null, null, null) - def scan2 = new WaveScanRecord('sc-1234567890abcdef_2', '101', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null, null) - def scan3 = new WaveScanRecord('sc-1234567890abcdef_3', '102', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2], null, null, null) - def scan4 = new WaveScanRecord('sc-01234567890abcdef_4', '103', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) - - when: - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - persistence.saveScanRecordAsync(scan3) - persistence.saveScanRecordAsync(scan4) - and: - sleep 200 - then: - persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1] - and: - persistence.allScans("1234567890") == null - } - - void 'should get paginated mirror results'() { - given: - def digest = 'sha256:12345' - def timestamp = Instant.now() - def source = 'source.io/foo' - def target = 'target.io/foo' - and: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def request1 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-1', - timestamp.minusSeconds(180), - "GMT", - Mock(PlatformId) ) - and: - def request2 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-2', - timestamp.minusSeconds(120), - "GMT", - Mock(PlatformId) ) - and: - def request3 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-3', - timestamp.minusSeconds(60), - "GMT", - Mock(PlatformId) ) - and: - def result1 = MirrorResult.of(request1).complete(1, 'err') - def result2 = MirrorResult.of(request2).complete(0, 'ok') - def result3 = MirrorResult.of(request3).complete(0, 'ok') - and: - persistence.saveMirrorResultAsync(result1) - persistence.saveMirrorResultAsync(result2) - persistence.saveMirrorResultAsync(result3) - sleep(300) - - when: - def mirrors = persistence.getMirrorsPaginated(3,0) - - then: - mirrors.size() == 3 - and: - mirrors.contains(result1) - mirrors.contains(result2) - mirrors.contains(result3) - } - - void 'should retrieve paginated container requests'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def TOKEN = '123abc' - def cfg = new ContainerConfig(entrypoint: ['/opt/fusion'], - layers: [ new ContainerLayer(location: 'https://fusionfs.seqera.io/releases/v2.2.8-amd64.json')]) - def req = new SubmitContainerTokenRequest( - towerEndpoint: 'https://tower.nf', - towerWorkspaceId: 100, - containerConfig: cfg, - containerPlatform: ContainerPlatform.of('amd64'), - buildRepository: 'build.docker.io', - cacheRepository: 'cache.docker.io', - fingerprint: 'xyz', - timestamp: Instant.now().toString() - ) - def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = ContainerRequest.of(requestId: TOKEN, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: "from ubuntu" ) - def wave = "wave.io/wt/$TOKEN/hello-world" - def addr = "100.200.300.400" - def exp = Instant.now().plusSeconds(3600) - and: - def request = new WaveContainerRecord(req, data, wave, addr, exp) - and: - persistence.saveContainerRequestAsync(request) - sleep(100) - - when: - def requests = persistence.getRequestsPaginated(1,0) - - then: - requests.size() == 1 - and: - requests[0].waveImage == 'wave.io/wt/123abc/hello-world' - } - - void 'should retrieve paginated scan records when data exists'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scan1 = new WaveScanRecord(id: 'scan1', containerImage: 'image1', status: 'SUCCEEDED', vulnerabilities:[]) - def scan2 = new WaveScanRecord(id: 'scan2', containerImage: 'image2', status: 'FAILED', vulnerabilities:[]) - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - sleep(200) - - when: - def scans = persistence.getScansPaginated(2, 0) - - then: - scans.size() == 2 - and: - scans.contains(scan1) - scans.contains(scan2) - } - - void 'should return null when no scan records exist for pagination'() { - when: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scans = persistence.getScansPaginated(2, 0) - - then: - scans == null - } - - void 'should handle pagination offset correctly for scan records'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scan1 = new WaveScanRecord(id: 'scan1', containerImage: 'image1', status: 'SUCCEEDED', vulnerabilities:[]) - def scan2 = new WaveScanRecord(id: 'scan2', containerImage: 'image2', status: 'FAILED', vulnerabilities:[]) - def scan3 = new WaveScanRecord(id: 'scan3', containerImage: 'image3', status: 'PENDING', vulnerabilities:[]) - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - persistence.saveScanRecordAsync(scan3) - sleep(200) - - when: - def scans = persistence.getScansPaginated(2, 1) - - then: - scans.size() == 2 - scans.contains(scan2) - scans.contains(scan3) - } - - def 'getRequestsPaginated should return list of WaveContainerRecord'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def surreal = applicationContext.getBean(SurrealClient) - and: - def records = [ - [ - id : "req-123", - fusionVersion: [number: "3.0.0", arch: "x86"], - workspaceId : 100, - containerImage: "ubuntu:20.04" - ], - [ - id : "req-456", - fusionVersion: "2.4.0", - workspaceId : 101, - containerImage: "alpine:latest" - ], - [ - id : "req-789", - workspaceId : 101, - containerImage: "alpine:latest" - ] - ] - and: - records.each { record -> - def json = JacksonHelper.toJson(record) - def query = "INSERT INTO wave_request $json" - final future = new CompletableFuture() - surreal - .sqlAsync(persistence.getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$data.id' saved record: ${result}" - future.complete(null) - }, - {error-> - log.error "Failed to save container request with token '$data.id': ${error.message}" - future.completeExceptionally(error) - }) - } - - sleep 300 - when: - def result = persistence.getRequestsPaginated(10, 0) - - then: - result.size() == 3 - and: - result[0].id == 'wave_request:⟨req-123⟩' - result[0].fusionVersion == '3.0.0' - result[0].containerImage == 'ubuntu:20.04' - and: - result[1].id == 'wave_request:⟨req-456⟩' - result[1].fusionVersion == '2.4.0' - result[1].containerImage == 'alpine:latest' - and: - result[2].id == 'wave_request:⟨req-789⟩' - result[2].fusionVersion == null - result[2].containerImage == 'alpine:latest' - } -} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy deleted file mode 100644 index 0aae3be3a1..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import spock.lang.Shared -import spock.lang.Specification -import spock.lang.Timeout - -import java.time.Duration - -import io.micronaut.context.ApplicationContext -import io.seqera.fixtures.redis.RedisTestContainer -import redis.clients.jedis.JedisPool - -/** - * - * @author Paolo Di Tommaso - */ -@Timeout(30) -class DataMigrationLockTest extends Specification implements RedisTestContainer { - - @Shared - ApplicationContext context - - def setup() { - context = ApplicationContext.run('test', 'redis') - } - - def 'should get lock or timeout' () { - given: - def key = '/foo/v1' - def connection = context.getBean(JedisPool).getResource() - - when: - def lock1 = DataMigrationService.acquireLock(connection, key) - then: - lock1 != null - - when: - def lock2 = DataMigrationService.acquireLock(connection, key, Duration.ofMillis(100)) - then: - lock2 == null - - when: - lock1.release() - lock2 = DataMigrationService.acquireLock(connection, key) - then: - lock2 != null - - cleanup: - connection?.close() - } -} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy deleted file mode 100644 index b57827e54b..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import spock.lang.Specification - -import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateCache -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateEntry -import io.seqera.wave.service.persistence.postgres.PostgresPersistentService -import io.seqera.wave.service.persistence.impl.SurrealClient -import io.seqera.wave.service.request.ContainerRequest -import io.seqera.wave.tower.PlatformId -import io.seqera.wave.tower.User - -/** - * - * @author Munish Chouhan - */ -class DataMigrationServiceTest extends Specification { - - def surrealService = Mock(SurrealPersistenceService) - def postgresService = Mock(PostgresPersistentService) - def surrealClient = Mock(SurrealClient) - def dataMigrateCache = Mock(DataMigrateCache) - - def service - int pageSize = 1000 - - void setup() { - service = new DataMigrationService() - service.surrealService = surrealService - service.postgresService = postgresService - service.surrealDb = surrealClient - service.pageSize = pageSize - service.dataMigrateCache = dataMigrateCache - } - - def "should not migrate if build records are empty"() { - given: - surrealService.getBuildsPaginated(pageSize,0) >> [] - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - when: - service.migrateBuildRecords() - - then: - 0 * postgresService.saveBuildAsync(_) - and: - 0 * dataMigrateCache.put(_, _) - } - - def "should migrate build records in batches of 100"() { - given: - def builds = (1..100).collect { new WaveBuildRecord(buildId: it) } - surrealService.getBuildsPaginated(pageSize,0) >> builds - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - - when: - service.migrateBuildRecords() - - then: - 100 * postgresService.saveBuild(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_BUILD, new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 100)) - } - - def "should catch and log exception during build migration"() { - given: - surrealService.getBuildsPaginated(pageSize,0) >> [[id: 1]] - postgresService.saveBuildAsync(_) >> { throw new RuntimeException("DB error") } - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - - when: - service.migrateBuildRecords() - - then: - noExceptionThrown() - } - - def "should not migrate if request records are empty"() { - given: - surrealService.getRequestsPaginated(pageSize,0) >> [] - dataMigrateCache.get(DataMigrationService.TABLE_NAME_REQUEST) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 0) - - when: - service.migrateRequests() - - then: - 0 * postgresService.saveContainerRequest(_) - and: - 0 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_REQUEST, pageSize) - } - - def "should migrate request records in batches"() { - given: - def requests = (1..101).collect { new WaveContainerRecord( - new SubmitContainerTokenRequest(towerWorkspaceId: it), - new ContainerRequest(requestId: it, identity: PlatformId.of(new User(id: it), Mock(SubmitContainerTokenRequest))), - null, null, null) } - surrealService.getRequestsPaginated(pageSize,0) >> requests - dataMigrateCache.get(DataMigrationService.TABLE_NAME_REQUEST) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 0) - - when: - service.migrateRequests() - - then: - 101 * postgresService.saveContainerRequest(_,_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_REQUEST, new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 101)) - } - - def "should migrate scan records in batches"() { - given: - def scans = (1..99).collect { new WaveScanRecord(id: it) } - surrealService.getScansPaginated(pageSize, 0) >> scans - dataMigrateCache.get(DataMigrationService.TABLE_NAME_SCAN) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_SCAN, 0) - - when: - service.migrateScanRecords() - - then: - 99 * postgresService.saveScanRecord(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_SCAN, new DataMigrateEntry(DataMigrationService.TABLE_NAME_SCAN, 99)) - } - - def "should migrate mirror records in batches"() { - given: - def mirrors = (1..15).collect { new MirrorResult() } - surrealService.getMirrorsPaginated(pageSize, 0) >> mirrors - dataMigrateCache.get(DataMigrationService.TABLE_NAME_MIRROR) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_MIRROR, 0) - - when: - service.migrateMirrorRecords() - - then: - 15 * postgresService.saveMirrorResult(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_MIRROR, new DataMigrateEntry(DataMigrationService.TABLE_NAME_MIRROR, 15)) - } - - def "fixRequestId should extract or return correct id for various input formats"() { - expect: - DataMigrationService.fixRequestId(input) == expected - - where: - input | expected - "wave_request:12345" | "12345" - "wave_request:⟨12345⟩" | "12345" - "wave_request:⟨abc-987⟩" | "abc-987" - "wave_request:" | "" - "wave_request:⟨⟩" | "" - "wave_request:⟨" | "" - "wave_request:⟨abc⟩extra" | "abc" - "12345" | "12345" - "some_other_prefix:12345" | "some_other_prefix:12345" - "" | "" - null | null - } -} diff --git a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy deleted file mode 100644 index c13cf2ebbe..0000000000 --- a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.test - -import spock.lang.Shared - -import org.slf4j.LoggerFactory -import org.testcontainers.containers.GenericContainer -import org.testcontainers.containers.output.Slf4jLogConsumer -import org.testcontainers.containers.wait.strategy.Wait -import org.testcontainers.utility.DockerImageName - -/** - * - * @author Jorge Aguilera - */ -trait SurrealDBTestContainer { - private static final def LOGGER = LoggerFactory.getLogger(SurrealDBTestContainer.class); - - @Shared - static GenericContainer surrealContainer = new GenericContainer(DockerImageName.parse("surrealdb/surrealdb:v1.5.4")) - .withExposedPorts(8000) - .withCommand("start","--user", "root", "--pass", "root", '--log', 'debug') - .waitingFor( - Wait.forLogMessage(".*Started web server on .*\\n", 1) - ) - .withLogConsumer(new Slf4jLogConsumer(LOGGER)) - - void restartDb(){ - if( surrealContainer.running) - surrealContainer.stop() - surrealContainer.start() - } - - String getSurrealHostName(){ - surrealContainer.getHost() - } - - String getSurrealPort(){ - surrealContainer.getMappedPort(8000) - } -} diff --git a/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy b/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy index b93ac24336..4fd0659f7c 100644 --- a/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy @@ -54,23 +54,6 @@ class StringUtilsTest extends Specification { null | '/a/bc/' } - @Unroll - def 'should strip surreal prefix' () { - expect: - StringUtils.surrealId(ID) == EXPECTED - - where: - ID | EXPECTED - null | null - 'foo' | 'foo' - and: - 'foo:100' | '100' - 'foo-bar:1-2-3' | '1-2-3' - and: - 'foo:⟨100⟩' | '100' - 'foo-bar:⟨1-2-3⟩' | '1-2-3' - } - @Unroll def 'should concat paths' () { expect: From 01599d7d9ca2de169269cb54f128a946c43c970a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 22:43:23 +0100 Subject: [PATCH 2/3] Fix test expectations for allBuilds/allScans returning empty list After removing SurrealDB fallback, these methods return an empty list instead of null when no records are found. Co-Authored-By: Claude Opus 4.6 --- .../persistence/postgres/PostgresPersistentServiceTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy index e09d0e74c5..d9eea73bef 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy @@ -197,7 +197,7 @@ class PostgresPersistentServiceTest extends Specification { and: persistentService.allBuilds('bd-abc') == [record3, record2, record1] and: - persistentService.allBuilds('ab') == null + persistentService.allBuilds('ab') == [] } // ===== --- container request---- ===== @@ -409,6 +409,6 @@ class PostgresPersistentServiceTest extends Specification { then: persistentService.allScans("1234567890abcdef") == [scan3, scan2, scan1] and: - persistentService.allScans("1234567890") == null + persistentService.allScans("1234567890") == [] } } From 11bc1efee831d71c369638c6e979858c84754137 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 4 Mar 2026 07:55:13 +0100 Subject: [PATCH 3/3] updated docs and removed dead code Signed-off-by: munishchouhan --- docs/migrations/1-21-0.md | 11 ++--- .../persistence/PersistenceService.groovy | 4 +- .../impl/RetryOnIOException.groovy | 40 ------------------- 3 files changed, 6 insertions(+), 49 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy diff --git a/docs/migrations/1-21-0.md b/docs/migrations/1-21-0.md index 290379d203..f5da57da55 100644 --- a/docs/migrations/1-21-0.md +++ b/docs/migrations/1-21-0.md @@ -7,12 +7,9 @@ Wave 1.21.0 was released on May 29, 2025. ## Mandatory steps -Wave 1.21.0 introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. +Wave 1.21.0 introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. SurrealDB support has been fully removed as of Wave 1.32.5. -To upgrade your existing data from SurrealDB to PostgreSQL: +Add the following properties to your Wave configuration file: -1. Follow the steps in the [Wave database migration](../db-migration.md) guide. -2. Add the following properties to your Wave configuration file: - - - `wave.build.logs.path`: Sets the full path where build logs will be stored (e.g., `s3://my-bucket/wave/logs` or a local path). - - `wave.build.locks.path`: Sets the full path where Conda lock files will be stored (e.g., `s3://my-bucket/wave/locks` or a local path). +- `wave.build.logs.path`: Sets the full path where build logs will be stored (e.g., `s3://my-bucket/wave/logs` or a local path). +- `wave.build.locks.path`: Sets the full path where Conda lock files will be stored (e.g., `s3://my-bucket/wave/locks` or a local path). diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 54935f04b8..c51d622d22 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -73,7 +73,7 @@ interface PersistenceService { * Retrieve all {@link WaveBuildRecord} object for the given container id * * @param containerId The container id for which all the builds record should be retrieved - * @return The corresponding {@link WaveBuildRecord} object or {@code null} if no record is found + * @return The corresponding list of {@link WaveBuildRecord} objects or an empty list if no record is found */ List allBuilds(String containerId) @@ -152,7 +152,7 @@ interface PersistenceService { * Retrieve all {@link WaveScanRecord} object for the given partial scan id * * @param scanId The scan id for which all the scan records should be retrieved - * @return The corresponding {@link WaveScanRecord} object or {@code null} if no record is found + * @return The corresponding list of {@link WaveScanRecord} objects or an empty list if no record is found */ List allScans(String scanId) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy deleted file mode 100644 index bedc1ec3d6..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.http.client.exceptions.ReadTimeoutException -import io.micronaut.retry.annotation.RetryPredicate - -/** - * Policy that retries when the exception is a {@link IOException} or was caused by a {@link IOException} - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -class RetryOnIOException implements RetryPredicate { - @Override - boolean test(Throwable t) { - final result = t instanceof IOException || t.cause instanceof IOException || t instanceof ReadTimeoutException - log.debug "Checking error retry for exception [retry=$result]: $t" - return result - } -}