diff --git a/docs/db-migration.md b/docs/db-migration.md deleted file mode 100644 index 380dfb7092..0000000000 --- a/docs/db-migration.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: Wave database migration -description: Migrate your Wave installation from SurrealDB to PostgreSQL -tags: [wave, surrealdb, postgresql, migration] ---- - -[Wave 1.21.0](https://docs.seqera.io/changelog/wave/v1.21.0) introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. - -This guide outlines the steps to migrate your existing Wave installation from SurrealDB to PostgreSQL. - -:::info[**Prerequisites**] -You will need the following to get started: - -- [Wave CLI](./cli/index.md) -- A PostgreSQL database accessible to Wave -::: - -## Database migration - -1. Generate build ID, scan ID, token, and mirror ID data to verify your migration in step 3. - - 1. Run a Wave build operation and capture the `buildId`: - - ```bash - wave --conda-package bwa --wave-endpoint - ``` - - 1. Verify the build and record the build ID: - - ```bash - curl /view/builds/ - ``` - - 1. Verify the scan and record the scan ID: - - ```bash - curl /view/scans/ - ``` - - 1. Create a container augmentation and record the token: - - ```bash - wave -i ubuntu --config-file --wave-endpoint - ``` - - 1. Verify the container and record the token: - - ```bash - curl /view/containers/ - ``` - - 1. Create a mirror operation and note the mirror ID: - - ```bash - wave --mirror -i ubuntu --build-repo --wave-endpoint - ``` - - 1. Verify the mirror and record the mirror ID: - - ```bash - curl /view/mirrors/ - ``` - -1. Migrate your database: - - 1. Add the following to your `MICRONAUT_ENVIRONMENTS`: - - `postgres` - - `surrealdb` - - `migrate` - - `redis` - - 1. Start the Wave application: - - ```console - INFO i.s.w.s.p.m.DataMigrationService - Data migration service initialized - ``` - - 1. Check the logs for the migration status: - - ```console - INFO i.s.w.s.p.m.DataMigrationService - All wave_request records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_scan records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_build records migrated. - INFO i.s.w.s.p.m.DataMigrationService - All wave_mirror records migrated. - ``` - - 1. When all records are migrated, remove `migrate` and `surrealdb` from your `MICRONAUT_ENVIRONMENTS`, then restart Wave. - -1. Use the build ID, scan ID, token, and mirror ID you generated in step 1 to verify the migration: - - 1. Verify the build data: - - ```bash - curl /view/builds/ - ``` - - 1. Verify the scan data: - - ```bash - curl /view/scans/ - ``` - - 1. Verify the container data: - - ```bash - curl /view/containers/ - ``` - - 1. Verify the mirror data: - - ```bash - curl /view/mirrors/ - ``` - -## Wave configuration - -Add the following properties to your Wave configuration file: - -`wave.build.logs.path` -: Sets the full path where build logs will be stored. Can be an S3 URI (e.g., `s3://my-bucket/wave/logs`) or a local filesystem path. - -`wave.build.locks.path` -: Sets the full path where Conda lock files will be stored. Can be an S3 URI (e.g., `s3://my-bucket/wave/locks`) or a local filesystem path. diff --git a/docs/migrations/1-21-0.md b/docs/migrations/1-21-0.md index 290379d203..f5da57da55 100644 --- a/docs/migrations/1-21-0.md +++ b/docs/migrations/1-21-0.md @@ -7,12 +7,9 @@ Wave 1.21.0 was released on May 29, 2025. ## Mandatory steps -Wave 1.21.0 introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. +Wave 1.21.0 introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. SurrealDB support has been fully removed as of Wave 1.32.5. -To upgrade your existing data from SurrealDB to PostgreSQL: +Add the following properties to your Wave configuration file: -1. Follow the steps in the [Wave database migration](../db-migration.md) guide. -2. Add the following properties to your Wave configuration file: - - - `wave.build.logs.path`: Sets the full path where build logs will be stored (e.g., `s3://my-bucket/wave/logs` or a local path). - - `wave.build.locks.path`: Sets the full path where Conda lock files will be stored (e.g., `s3://my-bucket/wave/locks` or a local path). +- `wave.build.logs.path`: Sets the full path where build logs will be stored (e.g., `s3://my-bucket/wave/logs` or a local path). +- `wave.build.locks.path`: Sets the full path where Conda lock files will be stored (e.g., `s3://my-bucket/wave/locks` or a local path). diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 1aee6bc3ad..c51d622d22 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -73,12 +73,12 @@ interface PersistenceService { * Retrieve all {@link WaveBuildRecord} object for the given container id * * @param containerId The container id for which all the builds record should be retrieved - * @return The corresponding {@link WaveBuildRecord} object or {@code null} if no record is found + * @return The corresponding list of {@link WaveBuildRecord} objects or an empty list if no record is found */ List allBuilds(String containerId) /** - * Store a {@link WaveContainerRecord} object in the Surreal wave_request table. + * Store a {@link WaveContainerRecord} object in the persistence layer. * * @param data A {@link WaveContainerRecord} object representing a Wave request record */ @@ -101,7 +101,7 @@ interface PersistenceService { WaveContainerRecord loadContainerRequest(String token) /** - * Store a {@link WaveScanRecord} object in the Surreal wave_scan table. + * Store a {@link WaveScanRecord} object in the persistence layer. * * @param data A {@link WaveScanRecord} object representing the a container scan request */ @@ -152,7 +152,7 @@ interface PersistenceService { * Retrieve all {@link WaveScanRecord} object for the given partial scan id * * @param scanId The scan id for which all the scan records should be retrieved - * @return The corresponding {@link WaveScanRecord} object or {@code null} if no record is found + * @return The corresponding list of {@link WaveScanRecord} objects or an empty list if no record is found */ List allScans(String scanId) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index e0ee15ddfa..7261020adc 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -44,8 +44,7 @@ import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime class WaveContainerRecord { /** - * wave request id, this will be the token - * This is container token and it is named as id for surrealdb requirement + * Wave request id, this will be the token */ @PostgresIgnore final String id diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index d1d5fab88b..cbdff2e9ec 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -29,7 +29,6 @@ import groovy.util.logging.Slf4j import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.util.StringUtils /** * Model a Wave container scan result * @@ -75,7 +74,7 @@ class WaveScanRecord implements Cloneable { Path workDir ) { - this.id = StringUtils.surrealId(id) + this.id = id this.buildId = buildId this.mirrorId = mirrorId this.requestId = requestId @@ -93,7 +92,7 @@ class WaveScanRecord implements Cloneable { } WaveScanRecord(ScanEntry scan) { - this.id = StringUtils.surrealId(scan.scanId) + this.id = scan.scanId this.buildId = scan.buildId this.mirrorId = scan.mirrorId this.requestId = scan.requestId @@ -121,7 +120,7 @@ class WaveScanRecord implements Cloneable { } void setId(String id) { - this.id = StringUtils.surrealId(id) + this.id = id } Boolean succeeded() { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index 73869d2a57..aede84bc92 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -39,7 +39,7 @@ import jakarta.inject.Singleton @Singleton @CompileStatic @Secondary -@Requires(notEnv = "surrealdb") +@Requires(notEnv = "postgres") @TraceElapsedTime(thresholdMillis = '${wave.trace.local-persistence.threshold:100}') class LocalPersistenceService implements PersistenceService { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy deleted file mode 100644 index bedc1ec3d6..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/RetryOnIOException.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.http.client.exceptions.ReadTimeoutException -import io.micronaut.retry.annotation.RetryPredicate - -/** - * Policy that retries when the exception is a {@link IOException} or was caused by a {@link IOException} - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -class RetryOnIOException implements RetryPredicate { - @Override - boolean test(Throwable t) { - final result = t instanceof IOException || t.cause instanceof IOException || t instanceof ReadTimeoutException - log.debug "Checking error retry for exception [retry=$result]: $t" - return result - } -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy deleted file mode 100644 index 2901ec22fc..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import groovy.transform.CompileStatic -import io.micronaut.context.annotation.Requires -import io.micronaut.http.annotation.Body -import io.micronaut.http.annotation.Get -import io.micronaut.http.annotation.Header -import io.micronaut.http.annotation.Post -import io.micronaut.http.client.annotation.Client -import io.micronaut.retry.annotation.Retryable -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.scan.ScanVulnerability -import reactor.core.publisher.Flux -/** - * Declarative http client for SurrealDB - * - * @author : jorge - * - */ -@Requires(env='surrealdb') -@CompileStatic -@Header(name = "Content-type", value = "application/json") -@Header(name = "ns", value = '${surreal.default.ns}') -@Header(name = "db", value = '${surreal.default.db}') -@Client(value = '${surreal.default.url}') -@Retryable( - delay = '${wave.surreal.retry.delay:1s}', - maxDelay = '${wave.surreal.retry.maxDelay:10s}', - attempts = '${wave.surreal.retry.attempts:3}', - multiplier = '${wave.surreal.retry.multiplier:1.5}', - predicate = RetryOnIOException ) -interface SurrealClient { - - @Post("/sql") - Flux> sqlAsync(@Header String authorization, @Body String body) - - @Post("/sql") - Flux>> sqlAsyncMany(@Header String authorization, @Body String body) - - @Post("/sql") - Map sqlAsMap(@Header String authorization, @Body String body) - - @Post("/sql") - List> sqlAsList(@Header String authorization, @Body String body) - - @Post("/sql") - String sqlAsString(@Header String authorization, @Body String body) - - @Post('/key/wave_build') - Flux> insertBuildAsync(@Header String authorization, @Body WaveBuildRecord body) - - @Get('/key/wave_build') - String loadBuild(@Header String authorization, String buildId) - - @Post('/key/wave_build') - Map insertBuild(@Header String authorization, @Body WaveBuildRecord body) - - @Get('/key/wave_request/{token}') - String getContainerRequest(@Header String authorization, String token) - - @Post('/key/wave_scan') - Map insertScanRecord(@Header String authorization, @Body WaveScanRecord body) - - @Post('/key/wave_scan_vuln') - Map insertScanVulnerability(@Header String authorization, @Body ScanVulnerability scanVulnerability) - - @Post('/key/wave_mirror') - Flux> insertMirrorAsync(@Header String authorization, @Body MirrorResult body) -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy deleted file mode 100644 index d1f36c8a21..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import java.util.concurrent.CompletableFuture - -import com.fasterxml.jackson.core.type.TypeReference -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Secondary -import io.micronaut.context.annotation.Value -import io.micronaut.core.annotation.Nullable -import io.micronaut.http.client.exceptions.HttpClientResponseException -import io.micronaut.runtime.event.ApplicationStartupEvent -import io.micronaut.runtime.event.annotation.EventListener -import io.seqera.util.trace.TraceElapsedTime -import io.seqera.wave.core.ContainerDigestPair -import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.mirror.MirrorEntry -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.PersistenceService -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.migrate.MigrationOnly -import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.util.JacksonHelper -import jakarta.inject.Inject -import jakarta.inject.Singleton -/** - * Implements a persistence service based based on SurrealDB - * - * @author : jorge - * @author : Paolo Di Tommaso - */ -@Requires(env='surrealdb') -@Slf4j -@Secondary -@Singleton -@CompileStatic -@TraceElapsedTime(thresholdMillis = '${wave.trace.surreal-persistence.threshold:500}') -class SurrealPersistenceService implements PersistenceService { - - @Inject - private SurrealClient surrealDb - - @Value('${surreal.default.user}') - private String user - - @Value('${surreal.default.password}') - private String password - - @Nullable - @Value('${surreal.default.init-db}') - private Boolean initDb - - @EventListener - void onApplicationStartup(ApplicationStartupEvent event) { - if (initDb) - initializeDb() - } - - void initializeDb(){ - // create wave_build table - final ret1 = surrealDb.sqlAsMap(authorization, "define table wave_build SCHEMALESS") - if( ret1.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_build - cause: $ret1") - // create wave_request table - final ret2 = surrealDb.sqlAsMap(authorization, "define table wave_request SCHEMALESS") - if( ret2.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_request - cause: $ret2") - // create wave_scan table - final ret3 = surrealDb.sqlAsMap(authorization, "define table wave_scan SCHEMALESS") - if( ret3.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_scan - cause: $ret3") - // create wave_scan table - final ret4 = surrealDb.sqlAsMap(authorization, "define table wave_scan_vuln SCHEMALESS") - if( ret4.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_scan_vuln - cause: $ret4") - // create wave_mirror table - final ret5 = surrealDb.sqlAsMap(authorization, "define table wave_mirror SCHEMALESS") - if( ret5.status != "OK") - throw new IllegalStateException("Unable to define SurrealDB table wave_mirror - cause: $ret5") - } - - protected String getAuthorization() { - "Basic "+"$user:$password".bytes.encodeBase64() - } - - @Override - CompletableFuture saveBuildAsync(WaveBuildRecord build) { - // note: use surreal sql in order to by-pass issue with large payload - // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 - final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}" - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Build request with id '$build.buildId' saved record: ${result}" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving Build request record ${msg}\n${build}", error) - future.complete(null) - }) - return future - } - - @Override - WaveBuildRecord loadBuild(String buildId) { - if( !buildId ) - throw new IllegalArgumentException("Missing 'buildId' argument") - def result = loadBuild0(buildId) - if( result ) - return result - // try to lookup legacy record - final legacyId = BuildRequest.legacyBuildId(buildId) - return legacyId ? loadBuild1(legacyId) : null - } - - private WaveBuildRecord loadBuild0(String buildId) { - final query = "select * from wave_build where buildId = '$buildId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - private WaveBuildRecord loadBuild1(String buildId) { - final query = "select * from wave_build where buildId = '$buildId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - WaveBuildRecord loadBuildSucceed(String targetImage, String digest) { - final query = """\ - select * from wave_build - where - targetImage = '$targetImage' - and digest = '$digest' - and exitStatus = 0 - and duration is not null - order by - startTime desc limit 1 - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - WaveBuildRecord latestBuild(String containerId) { - final query = """ - select * - from wave_build - where buildId ~ '${containerId}${BuildRequest.SEP}' - order by startTime desc limit 1 - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - List allBuilds(String containerId) { - final query = """ - select * - from wave_build - where string::matches(buildId, '^(bd-)?${containerId}_[0-9]+') - order by startTime desc - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @Override - CompletableFuture saveContainerRequestAsync(WaveContainerRecord data) { - // note: use surreal sql in order to by-pass issue with large payload - // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 - final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}" - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$data.id' saved record: ${result}" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving container request record ${msg}\n${data}", error) - future.complete(null) - }) - return future - } - - CompletableFuture updateContainerRequestAsync(String token, ContainerDigestPair digest) { - final query = """\ - UPDATE wave_request:$token SET - sourceDigest = '$digest.source', - waveDigest = '${digest.target}' - """.stripIndent() - final future = new CompletableFuture() - surrealDb - .sqlAsync(getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$token' updated record: ${result}" - return future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error update container record=$token => ${msg}\ndigest=${digest}\n", error) - return future.complete(null) - }) - return future - } - - @Override - WaveContainerRecord loadContainerRequest(String token) { - if( !token ) - throw new IllegalArgumentException("Missing 'token' argument") - final json = surrealDb.getContainerRequest(getAuthorization(), token) - log.trace "Container request with token '$token' loaded: ${json}" - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(patchSurrealId(json,"wave_request"), type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - static protected String patchSurrealId(String json, String table) { - return json.replaceFirst(/"id":\s*"${table}:(\w*)"/) { List it-> /"id":"${it[1]}"/ } - } - - @Override - CompletableFuture saveScanRecordAsync(WaveScanRecord scanRecord) { - final vulnerabilities = scanRecord.vulnerabilities ?: List.of() - - // create a multi-command surreal sql statement to insert all vulnerabilities - // and the scan record in a single operation - List ids = new ArrayList<>(101) - String statement = '' - // save all vulnerabilities - for( ScanVulnerability it : vulnerabilities ) { - statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n" - ids << "wave_scan_vuln:⟨$it.id⟩".toString() - } - - // scan object - final copy = scanRecord.clone() - copy.vulnerabilities = List.of() - final json = JacksonHelper.toJson(copy) - - // add the wave_scan record - statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() - final future = new CompletableFuture() - // store the statement using an async operation - surrealDb - .sqlAsyncMany(getAuthorization(), statement) - .subscribe({result -> - log.trace "Scan record save result=$result" - future.complete(null) - }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving scan record => ${msg}\n", error) - future.complete(null) - }) - return future - } - - protected String patchScanVulnerabilities(String json, List ids) { - final value = "\"vulnerabilities\":${ids.collect(it-> "\"$it\"").toString()}" - json.replaceFirst(/"vulnerabilities":\s*\[]/, value) - } - - @Override - boolean existsScanRecord(String scanId) { - final statement = "SELECT count() FROM wave_scan where id == 'wave_scan:⟨$scanId⟩'" - final json = surrealDb.sqlAsString(getAuthorization(), statement) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0].count==1 : false - return result - } - - @Override - WaveScanRecord loadScanRecord(String scanId) { - if( !scanId ) - throw new IllegalArgumentException("Missing 'scanId' argument") - final statement = "SELECT * FROM wave_scan where id == 'wave_scan:⟨$scanId⟩' FETCH vulnerabilities" - final json = surrealDb.sqlAsString(getAuthorization(), statement) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - @Override - List allScans(String scanId) { - final query = """ - select * - from wave_scan - where string::matches(type::string(id), '^wave_scan:⟨(sc-)?${scanId}_[0-9]+') - order by startTime desc - FETCH vulnerabilities - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - // === mirror operations - - /** - * Load a mirror state record - * - * @param mirrorId The ID of the mirror record - * @return The corresponding {@link MirrorEntry} object or null if it cannot be found - */ - MirrorResult loadMirrorResult(String mirrorId) { - final query = "select * from wave_mirror where mirrorId = '$mirrorId'" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - /** - * Load a mirror state record given the target image name and the image digest. - * It returns the latest succeed mirror result. - * - * @param targetImage The target mirrored image name - * @param digest The image content SHA256 digest - * @return The corresponding {@link MirrorEntry} object or null if it cannot be found - */ - MirrorResult loadMirrorSucceed(String targetImage, String digest) { - final query = """ - select * from wave_mirror - where - targetImage = '$targetImage' - and digest = '$digest' - and exitCode = 0 - and status = '${MirrorResult.Status.COMPLETED}' - order by - creationTime desc limit 1 - """ - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result[0] : null - return result - } - - /** - * Persists a {@link MirrorEntry} object - * - * @param mirror {@link MirrorEntry} object - */ - @Override - CompletableFuture saveMirrorResultAsync(MirrorResult mirror) { - final future = new CompletableFuture() - surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> - log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" - return future.complete(null) - }, {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error saving Mirror request record ${msg}\n${mirror}", error) - return future.complete(null) - }) - return future - } - - @MigrationOnly - List getBuildsPaginated(int limit, int offset) { - final query = "select * from wave_build limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getRequestsPaginated(int limit, int offset){ - final query = "select * from wave_request limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getScansPaginated(int limit, int offset){ - final query = """ - select * - from wave_scan - limit $limit start $offset - FETCH vulnerabilities - """.stripIndent() - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - - @MigrationOnly - List getMirrorsPaginated(int limit, int offset){ - final query = "select * from wave_mirror limit $limit start $offset" - final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} - final data= json ? JacksonHelper.fromJson(json, type) : null - final result = data && data[0].result ? data[0].result : null - return result ? Arrays.asList(result) : null - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy deleted file mode 100644 index fa130928ca..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealResult.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -/** - * Model a Surreal Resultset - * - * @author Paolo Di Tommaso - */ -class SurrealResult { - - String time - String status - T[] result - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy deleted file mode 100644 index a9b046c18b..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationService.groovy +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import static io.seqera.wave.util.DurationUtils.* - -import java.time.Duration -import java.util.concurrent.ScheduledFuture -import java.util.function.Consumer -import java.util.function.Function - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.ApplicationContext -import io.micronaut.context.annotation.Context -import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Value -import io.micronaut.context.env.Environment -import io.micronaut.data.exceptions.DataAccessException -import io.micronaut.runtime.event.annotation.EventListener -import io.micronaut.runtime.server.event.ServerShutdownEvent -import io.micronaut.runtime.server.event.ServerStartupEvent -import io.micronaut.scheduling.TaskScheduler -import io.seqera.util.redis.JedisLock -import io.seqera.util.redis.JedisLockManager -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealClient -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateCache -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateEntry -import io.seqera.wave.service.persistence.postgres.PostgresPersistentService -import jakarta.inject.Inject -import redis.clients.jedis.Jedis -import redis.clients.jedis.JedisPool -import static io.seqera.wave.util.DurationUtils.randomDuration - -/** - * Service to migrate data from SurrealDB to Postgres - * - * @author Munish Chouhan - */ -@Requires(env='migrate') -@Slf4j -@Context -@CompileStatic -@MigrationOnly -class DataMigrationService { - - public static final String TABLE_NAME_BUILD = 'wave_build' - public static final String TABLE_NAME_REQUEST = 'wave_request' - public static final String TABLE_NAME_SCAN = 'wave_scan' - public static final String TABLE_NAME_MIRROR = 'wave_mirror' - - @Value('${wave.db.migrate.page-size:200}') - private int pageSize - - @Value('${wave.db.migrate.delay:5s}') - private Duration delay - - @Value('${wave.db.migrate.initial-delay:70s}') - private Duration launchDelay - - @Value('${wave.db.migrate.initial-delay:10s}') - private Duration initialDelay - - @Value('${wave.db.migrate.iteration-delay:100ms}') - private Duration iterationDelay - - @Inject - private SurrealPersistenceService surrealService - - @Inject - private PostgresPersistentService postgresService - - @Inject - private SurrealClient surrealDb - - @Inject - private DataMigrateCache dataMigrateCache - - @Inject - private TaskScheduler taskScheduler - - @Inject - private Environment environment - - @Inject - private ApplicationContext applicationContext - - @Inject - private JedisPool pool - - private volatile Jedis conn - - private volatile JedisLock lock - - private static final String LOCK_KEY = "migrate-lock/v2" - - private volatile ScheduledFuture mirrorTask - private volatile ScheduledFuture buildTask - private volatile ScheduledFuture scanTask - private volatile ScheduledFuture requestTask - - - @EventListener - void start(ServerStartupEvent event) { - if (!environment.activeNames.contains("surrealdb") || !environment.activeNames.contains("postgres")) { - throw new IllegalStateException("Both 'surrealdb' and 'postgres' environments must be active.") - } - // launch async to not block bootstrap - taskScheduler.schedule(launchDelay, ()->{ - try { - launchMigration() - } - catch (InterruptedException e) { - log.info "Migration launch has been interrupted (1)" - } - catch (Throwable e) { - log.info("Unexpected exception during Migration launch", e) - } - }) - } - - @EventListener - void stop(ServerShutdownEvent event) { - log.info "Releasing lock and closing connection" - // remove the lock & close the connection - lock?.release() - conn?.close() - } - - void launchMigration() { - log.info("Data migration service initialized with page size: $pageSize, delay: $delay, initial delay: $initialDelay") - // acquire the lock to only run one instance at time - conn = pool.getResource() - lock = acquireLock(conn, LOCK_KEY) - if( !lock ) { - log.debug "Skipping migration since lock cannot be acquired" - conn.close() - conn = null - return - } - log.info("Data migration initiated") - - dataMigrateCache.putIfAbsent(TABLE_NAME_BUILD, new DataMigrateEntry(TABLE_NAME_BUILD, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_REQUEST, new DataMigrateEntry(TABLE_NAME_REQUEST, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_SCAN, new DataMigrateEntry(TABLE_NAME_SCAN, 0)) - dataMigrateCache.putIfAbsent(TABLE_NAME_MIRROR, new DataMigrateEntry(TABLE_NAME_MIRROR, 0)) - - buildTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateBuildRecords) - requestTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateRequests) - scanTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateScanRecords) - mirrorTask = taskScheduler.scheduleWithFixedDelay(randomDuration(initialDelay, 0.5f), delay, this::migrateMirrorRecords) - } - - /** - * Migrate data from SurrealDB to Postgres - */ - void migrateBuildRecords() { - migrateRecords(TABLE_NAME_BUILD, - (Integer offset)-> surrealService.getBuildsPaginated(pageSize, offset), - (WaveBuildRecord it)-> postgresService.saveBuild(it), - buildTask ) - } - - /** - * Migrate container requests from SurrealDB to Postgres - */ - void migrateRequests() { - migrateRecords(TABLE_NAME_REQUEST, - (Integer offset)-> surrealService.getRequestsPaginated(pageSize, offset), - (WaveContainerRecord request)-> postgresService.saveContainerRequest(fixRequestId(request.id), request), - requestTask ) - } - - protected static String fixRequestId(String id){ - if (id?.contains("wave_request:")){ - if(id.contains("wave_request:⟨")){ - return id.takeAfter("wave_request:⟨").takeBefore("⟩") - } - return id.takeAfter("wave_request:") - } - return id - } - - /** - * Migrate scan records from SurrealDB to Postgres - */ - void migrateScanRecords() { - migrateRecords(TABLE_NAME_SCAN, - (Integer offset)-> surrealService.getScansPaginated(pageSize, offset), - (WaveScanRecord it)-> postgresService.saveScanRecord(it), - scanTask ) - } - - /** - * Migrate mirror records from SurrealDB to Postgres - */ - void migrateMirrorRecords() { - migrateRecords(TABLE_NAME_MIRROR, - (Integer offset)-> surrealService.getMirrorsPaginated(pageSize, offset), - (MirrorResult it)-> postgresService.saveMirrorResult(it), - mirrorTask ) - } - - void migrateRecords(String tableName, Function> fetch, Consumer saver, ScheduledFuture task) { - try { - migrateRecords0(tableName, fetch, saver, task) - } - catch (InterruptedException e) { - log.info "Migration $tableName has been interrupted (2)" - Thread.currentThread().interrupt() - } - catch (Throwable t) { - log.error("Unexpected migration error - ${t.message}", t) - } - } - - void migrateRecords0(String tableName, Function> fetch, Consumer saver, ScheduledFuture task) { - log.info "Initiating $tableName migration" - int offset = dataMigrateCache.get(tableName).offset - def records = fetch.apply(offset) - - if (!records) { - log.info("All $tableName records migrated.") - task.cancel(false) - return - } - - int count=0 - for (def it : records) { - try { - if( Thread.currentThread().isInterrupted() ) { - log.info "Thread is interrupted - exiting $tableName method" - break - } - saver.accept(it) - dataMigrateCache.put(tableName, new DataMigrateEntry(tableName, ++offset)) - if( ++count % 50 == 0 ) - log.info "Migration ${tableName}; processed ${count} records" - Thread.sleep(iterationDelay.toMillis()) - } - catch (InterruptedException e) { - log.info "Migration $tableName has been interrupted (3)" - Thread.currentThread().interrupt() - } - catch (DataAccessException dataAccessException) { - if (dataAccessException.message.contains("duplicate key value violates unique constraint")) { - log.warn("Duplicate key error for $tableName record: ${dataAccessException.message}") - dataMigrateCache.put(tableName, new DataMigrateEntry(tableName, ++offset)) - } else { - log.error("Error saving=> $tableName record: ${dataAccessException.message}") - } - } - catch (Exception e) { - log.error("Error saving $tableName record: ${e.message}", e) - } - } - - log.info("Migrated ${records.size()} $tableName records (offset $offset)") - } - - // == --- jedis lock handling - static JedisLock acquireLock(Jedis conn, String key, Duration timeout=Duration.ofMinutes(10)) { - try { - final max = timeout.toMillis() - final begin = System.currentTimeMillis() - while( !Thread.currentThread().isInterrupted() ) { - if( System.currentTimeMillis()-begin > max ) { - log.info "Lock acquire timeout reached" - return null - } - final lock = new JedisLockManager(conn) - .withLockAutoExpireDuration(Duration.ofDays(30)) - .tryAcquire(key) - if( lock ) - return lock - log.info "Unable to acquire lock - await 1s before retrying" - sleep(10_000) - } - } - catch (InterruptedException e) { - log.info "Migration acquire lock has been interrupted (4)" - Thread.currentThread().interrupt() - return null - } - catch (Throwable t) { - log.error("Unexpected error while trying to acquire the lock - ${t.message}", t) - return null - } - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy deleted file mode 100644 index c8efc7a7a3..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/MigrationOnly.groovy +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import java.lang.annotation.ElementType -import java.lang.annotation.Retention -import java.lang.annotation.RetentionPolicy -import java.lang.annotation.Target - -/** - * Marker annotation to annotate logic required for Surreal to Postgres migration - * - * @author Paolo Di Tommaso - */ -@Retention(RetentionPolicy.RUNTIME) -@Target([ElementType.TYPE, ElementType.METHOD]) -@interface MigrationOnly { - -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy deleted file mode 100644 index 31910dc2d7..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateCache.groovy +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate.cache - -import java.time.Duration - -import groovy.transform.CompileStatic -import io.seqera.serde.moshi.MoshiEncodeStrategy -import io.seqera.data.store.state.AbstractStateStore -import io.seqera.data.store.state.impl.StateProvider -import jakarta.inject.Singleton -/** - * Cache for data migration entries - * - * @author Munish Chouhan - */ -@Singleton -@CompileStatic -class DataMigrateCache extends AbstractStateStore { - - DataMigrateCache(StateProvider provider) { - super(provider, new MoshiEncodeStrategy() {}) - } - - @Override - protected String getPrefix() { - return 'migrate-surreal/v1' - } - - @Override - protected Duration getDuration() { - return Duration.ofDays(30) - } -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy b/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy deleted file mode 100644 index 938e3db88b..0000000000 --- a/src/main/groovy/io/seqera/wave/service/persistence/migrate/cache/DataMigrateEntry.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate.cache - -import groovy.transform.CompileStatic -import groovy.transform.EqualsAndHashCode -import groovy.transform.ToString - -/** - * Data migration entry - * - * @author Munish Chouhan - */ -@ToString(includeNames = true, includePackage = false) -@EqualsAndHashCode -@CompileStatic -class DataMigrateEntry { - String tableName - int offset - - DataMigrateEntry(String tableName, int offset) { - this.tableName = tableName - this.offset = offset - } -} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy index 01a01ec0df..3508cee637 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentService.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires -import io.micronaut.core.annotation.Nullable import io.micronaut.runtime.event.ApplicationStartupEvent import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors @@ -36,8 +35,6 @@ import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.MigrationOnly import io.seqera.wave.service.persistence.postgres.data.BuildRepository import io.seqera.wave.service.persistence.postgres.data.BuildRow import io.seqera.wave.service.persistence.postgres.data.MirrorRepository @@ -75,10 +72,6 @@ class PostgresPersistentService implements PersistenceService { @Inject private ScanRepository scanRepository - @Inject - @Nullable - private SurrealPersistenceService surrealPersistenceService - @Inject @Named(TaskExecutors.BLOCKING) private ExecutorService ioExecutor @@ -116,21 +109,12 @@ class PostgresPersistentService implements PersistenceService { "Unable to save build record=$data")) } - @MigrationOnly - void saveBuild(WaveBuildRecord data) { - log.trace "Saving build record=$data" - final json = Mapper.toJson(data) - final row = new BuildRow(id:data.buildId, data:json, createdAt: Instant.now()) - buildRepository.save(row) - } - @Override WaveBuildRecord loadBuild(String buildId) { log.trace "Loading build record id=${buildId}" final row = buildRepository.findById(buildId).orElse(null) - if( !row ) { - return surrealPersistenceService?.loadBuild(buildId) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: buildId] ) } @@ -138,9 +122,8 @@ class PostgresPersistentService implements PersistenceService { WaveBuildRecord loadBuildSucceed(String targetImage, String digest) { log.trace "Loading build record with image=${targetImage}; digest=${digest}" final row = buildRepository.findByTargetAndDigest(targetImage, digest) - if( !row ){ - return surrealPersistenceService?.loadBuildSucceed(targetImage, digest) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: row.id] ) } @@ -148,9 +131,8 @@ class PostgresPersistentService implements PersistenceService { WaveBuildRecord latestBuild(String containerId) { log.trace "Loading latest build with containerId=${containerId}" final row = buildRepository.findLatestByBuildId(containerId) - if( !row ){ - return surrealPersistenceService?.latestBuild(containerId) - } + if( !row ) + return null return Mapper.fromJson( WaveBuildRecord, row.data, [buildId: row.id] ) } @@ -160,17 +142,10 @@ class PostgresPersistentService implements PersistenceService { final result = buildRepository.findAllByBuildId(containerId) return result ? result.collect((it)-> Mapper.fromJson(WaveBuildRecord, it.data, [buildId: it.id])) - : surrealPersistenceService?.allBuilds(containerId) + : List.of() } // ===== --- container records ---- ===== - @MigrationOnly - void saveContainerRequest(String id, WaveContainerRecord data) { - final json = Mapper.toJson(data) - final entity = new RequestRow(id: id, data:json, createdAt: Instant.now()) - requestRepository.save(entity) - } - @Override CompletableFuture saveContainerRequestAsync(WaveContainerRecord data) { log.trace "Saving container request data=${data}" @@ -194,7 +169,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading container request token=${token}" final row = requestRepository.findById(token).orElse(null) if( !row || !row.data ) - return surrealPersistenceService?.loadContainerRequest(token) + return null return Mapper.fromJson(WaveContainerRecord, row.data, [id: token]) } @@ -210,27 +185,19 @@ class PostgresPersistentService implements PersistenceService { "Unable to save scan record data=${data}")) } - @MigrationOnly - void saveScanRecord(WaveScanRecord data) { - log.trace "Saving scan record data=${data}" - final json = Mapper.toJson(data) - final entity = new ScanRow(id: data.id, data:json, createdAt: Instant.now()) - scanRepository.save(entity) - } - @Override WaveScanRecord loadScanRecord(String scanId) { log.trace "Loading scan record id=${scanId}" final row = scanRepository.findById(scanId).orElse(null) if( !row || !row.data ) - return surrealPersistenceService?.loadScanRecord(scanId) + return null return Mapper.fromJson(WaveScanRecord, row.data, [id: scanId]) } @Override boolean existsScanRecord(String scanId) { log.trace "Exist scan record id=${scanId}" - return scanRepository.existsById(scanId) ?: surrealPersistenceService?.existsScanRecord(scanId) + return scanRepository.existsById(scanId) } @Override @@ -239,7 +206,7 @@ class PostgresPersistentService implements PersistenceService { final result = scanRepository.findAllByScanId(scanId) return result ? result.collect((it)-> Mapper.fromJson(WaveScanRecord, it.data, [id: it.id])) - : surrealPersistenceService?.allScans(scanId) + : List.of() } // ===== --- mirror records ---- ===== @@ -249,7 +216,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading mirror result with id=${mirrorId}" final row = mirrorRepository.findById(mirrorId).orElse(null) if( !row ) - return surrealPersistenceService?.loadMirrorResult(mirrorId) + return null return Mapper.fromJson(MirrorResult, row.data, [mirrorId: mirrorId]) } @@ -258,7 +225,7 @@ class PostgresPersistentService implements PersistenceService { log.trace "Loading mirror succeed with image=${targetImage}; digest=${digest}" final row = mirrorRepository.findByTargetAndDigest(targetImage, digest, MirrorResult.Status.COMPLETED) if( !row ) - return surrealPersistenceService?.loadMirrorSucceed(targetImage, digest) + return null return Mapper.fromJson(MirrorResult, row.data, [mirrorId: row.id]) } @@ -272,11 +239,4 @@ class PostgresPersistentService implements PersistenceService { "Unable to save mirror result data=${data}")) } - @MigrationOnly - void saveMirrorResult(MirrorResult data) { - log.trace "Saving mirror result data=${data}" - final json = Mapper.toJson(data) - final entity = new MirrorRow(id: data.mirrorId, data:json, createdAt: Instant.now()) - mirrorRepository.save(entity) - } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy index bd73f9e1d1..5ed4dc6541 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy @@ -23,7 +23,6 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.seqera.wave.util.StringUtils import org.jetbrains.annotations.NotNull /** * Model for Scan Vulnerability @@ -54,7 +53,7 @@ class ScanVulnerability implements Comparable { ScanVulnerability(){} ScanVulnerability(String id, String severity, String title, String pkgName, String installedVersion, String fixedVersion, String primaryUrl) { - this.id = StringUtils.surrealId(id) + this.id = id this.severity = severity this.title = title this.pkgName = pkgName @@ -64,7 +63,7 @@ class ScanVulnerability implements Comparable { } void setId(String id) { - this.id = StringUtils.surrealId(id) + this.id = id } @Override diff --git a/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy b/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy index 0d8bed77dc..fd54775524 100644 --- a/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy +++ b/src/main/groovy/io/seqera/wave/util/FusionVersionStringDeserializer.groovy @@ -23,14 +23,12 @@ import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.JsonDeserializer import com.fasterxml.jackson.databind.JsonNode import groovy.transform.CompileStatic -import io.seqera.wave.service.persistence.migrate.MigrationOnly /** * Custom deserializer for Fusion version strings. * * @author Munish Chouhan */ -@MigrationOnly @CompileStatic class FusionVersionStringDeserializer extends JsonDeserializer { @Override diff --git a/src/main/groovy/io/seqera/wave/util/StringUtils.groovy b/src/main/groovy/io/seqera/wave/util/StringUtils.groovy index a610b64928..36208b56e1 100644 --- a/src/main/groovy/io/seqera/wave/util/StringUtils.groovy +++ b/src/main/groovy/io/seqera/wave/util/StringUtils.groovy @@ -68,17 +68,6 @@ class StringUtils { return m.matches() ? m.group(1) : null } - static String surrealId(String id) { - if( !id ) - return null - final p = id.indexOf(':') - if( p!=-1 ) - id = id.substring(p+1) - if( id.startsWith('⟨') && id.endsWith('⟩')) - id = id.substring(1,id.length()-1) - return id - } - static String pathConcat(String base, String path) { assert base, "Missing 'base' argument" // strip ending slash diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index e6a8b4cc9e..44480ae3e2 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -32,8 +32,6 @@ wave: trace: local-persistence: threshold: 100 - surreal-persistence: - threshold: 100 proxy-service: threshold: 100 --- diff --git a/src/main/resources/application-surrealdb-legacy.yml b/src/main/resources/application-surrealdb-legacy.yml deleted file mode 100644 index dcfd8b90b0..0000000000 --- a/src/main/resources/application-surrealdb-legacy.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -surreal: - legacy: - ns: "seqera" - db: "wave" - url: "http://surrealdb:8000" - user: "root" - password: "root" -... diff --git a/src/main/resources/application-surrealdb.yml b/src/main/resources/application-surrealdb.yml deleted file mode 100644 index d60e8da74d..0000000000 --- a/src/main/resources/application-surrealdb.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -surreal: - default: - ns : "seqera" - db : "wave" - url: "http://surrealdb:8000" - user: "root" - password: "root" -... diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy deleted file mode 100644 index 2147795690..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.impl - -import spock.lang.Specification - -import java.nio.file.Path -import java.time.Duration -import java.time.Instant -import java.util.concurrent.CompletableFuture - -import io.micronaut.context.ApplicationContext -import io.micronaut.http.HttpRequest -import io.micronaut.http.client.HttpClient -import io.micronaut.http.client.exceptions.HttpClientResponseException -import io.seqera.wave.api.BuildCompression -import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.api.ContainerLayer -import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.core.ContainerDigestPair -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.request.ContainerRequest -import io.seqera.wave.service.builder.BuildEvent -import io.seqera.wave.service.builder.BuildFormat -import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.service.mirror.MirrorEntry -import io.seqera.wave.service.mirror.MirrorRequest -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.scan.ScanVulnerability -import io.seqera.wave.test.SurrealDBTestContainer -import io.seqera.wave.tower.PlatformId -import io.seqera.wave.tower.User -import io.seqera.wave.util.JacksonHelper -import org.apache.commons.lang3.RandomStringUtils - -/** - * @author : jorge - * - */ -class SurrealPersistenceServiceTest extends Specification implements SurrealDBTestContainer { - - ApplicationContext applicationContext - - String getSurrealDbURL() { - "http://$surrealHostName:$surrealPort" - } - - def setup() { - restartDb() - applicationContext = ApplicationContext.run([ - surreal:['default': [ - user : 'root', - password : 'root', - ns : 'test', - db : 'test', - url : surrealDbURL, - 'init-db': false - ]]] - , 'test', 'surrealdb') - } - - def cleanup() { - applicationContext.close() - } - - void "can connect"() { - given: - def httpClient = HttpClient.create(new URL(surrealDbURL)) - - when: - def str = httpClient.toBlocking() - .retrieve( - HttpRequest.POST("/sql", "SELECT * FROM count()") - .headers(['ns': 'test', 'db': 'test', 'accept':'application/json']) - .basicAuth('root', 'root'), Map) - - then: - str.result.first() == 1 - } - - void "can insert an async build"() { - given: - final String dockerFile = """\ - FROM quay.io/nextflow/bash - RUN echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt - ENV NOW=${System.currentTimeMillis()} - """ - final String condaFile = """ - echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt - """ - def storage = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: dockerFile, - condaFile: condaFile, - workspace: Path.of("."), - targetImage: 'docker.io/my/repo:container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1', - ) - def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) - def event = new BuildEvent(request, result) - def build = WaveBuildRecord.fromEvent(event) - - when: - storage.initializeDb() - and: - storage.saveBuildAsync(build) - then: - sleep 100 - def stored = storage.loadBuild(request.buildId) - stored.buildId == request.buildId - stored.requestIp == '127.0.0.1' - } - - def 'should load a build record' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: 'FROM foo:latest', - condaFile: 'conda::recipe', - workspace: Path.of("."), - targetImage: 'docker.io/my/repo:container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1', - ) - def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) - def event = new BuildEvent(request, result) - def record = WaveBuildRecord.fromEvent(event) - - and: - persistence.saveBuildAsync(record) - - when: - sleep 100 - def loaded = persistence.loadBuild(record.buildId) - - then: - loaded == record - } - - def 'should find latest succeed' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def target = 'docker.io/my/target' - def digest = 'sha256:12345' - and: - def request1 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_1', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_2', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( targetImage: target, containerId: 'abc', buildId: 'bd-abc_3', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - and: - def result1 = new BuildResult(request1.buildId, 1, "err", request1.startTime, Duration.ofSeconds(2), digest) - def rec1 = WaveBuildRecord.fromEvent(new BuildEvent(request1, result1)) - surreal.insertBuild(auth, rec1) - and: - def result2 = new BuildResult(request2.buildId, 0, "ok", request2.startTime, Duration.ofSeconds(2), digest) - def rec2 = WaveBuildRecord.fromEvent(new BuildEvent(request2, result2)) - surreal.insertBuild(auth, rec2) - and: - def result3 = new BuildResult(request3.buildId, 0, "ok", request3.startTime, Duration.ofSeconds(2), digest) - def rec3 = WaveBuildRecord.fromEvent(new BuildEvent(request3, result3)) - surreal.insertBuild(auth, rec3) - - expect: - persistence.loadBuildSucceed(target, digest) == rec3 - } - - def 'should find latest build' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def request1 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_1' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_2' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - - def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request1, result1))) - and: - def result2 = new BuildResult(request2.buildId, -1, "ok", request2.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request2, result2))) - and: - def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) - surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request3, result3))) - - expect: - persistence.latestBuild('abc').buildId == 'bd-abc_3' - persistence.latestBuild('bd-abc').buildId == 'bd-abc_3' - persistence.latestBuild('xyz') == null - } - - def 'should save and update a build' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: 'FROM foo:latest', - condaFile: 'conda::recipe', - workspace: Path.of("/some/path"), - targetImage: 'buildrepo:recipe-container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - buildId: '12345_1' - ) - and: - def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') - - and: - def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) - - when: - persistence.saveBuildAsync(build1) - sleep 100 - then: - persistence.loadBuild(request.buildId) == build1 - - } - - def 'should load a request record' () { - given: - def largeContainerFile = RandomStringUtils.random(25600, true, true) - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def TOKEN = '123abc' - def cfg = new ContainerConfig(entrypoint: ['/opt/fusion'], - layers: [ new ContainerLayer(location: 'https://fusionfs.seqera.io/releases/v2.2.8-amd64.json')]) - def req = new SubmitContainerTokenRequest( - towerEndpoint: 'https://tower.nf', - towerWorkspaceId: 100, - containerConfig: cfg, - containerPlatform: ContainerPlatform.of('amd64'), - buildRepository: 'build.docker.io', - cacheRepository: 'cache.docker.io', - fingerprint: 'xyz', - timestamp: Instant.now().toString() - ) - def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = ContainerRequest.of(requestId: TOKEN, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: largeContainerFile ) - def wave = "wave.io/wt/$TOKEN/hello-world" - def addr = "100.200.300.400" - def exp = Instant.now().plusSeconds(3600) - and: - def request = new WaveContainerRecord(req, data, wave, addr, exp) - and: - persistence.saveContainerRequestAsync(request) - and: - sleep 200 // <-- the above request is async, give time to save it - - when: - def loaded = persistence.loadContainerRequest(TOKEN) - then: - loaded == request - loaded.containerFile == largeContainerFile - - // should update the record - when: - persistence.updateContainerRequestAsync(TOKEN, new ContainerDigestPair('111', '222')) - and: - sleep 200 - then: - def updated = persistence.loadContainerRequest(TOKEN) - and: - updated.sourceDigest == '111' - updated.sourceImage == request.sourceImage - and: - updated.waveDigest == '222' - updated.waveImage == request.waveImage - - } - - def 'should save a scan and load a result' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def surrealDb = applicationContext.getBean(SurrealClient) - def NOW = Instant.now() - def SCAN_ID = 'a1' - def BUILD_ID = '100' - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') - def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') - def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null, null) - when: - persistence.saveScanRecordAsync(scan) - sleep 200 - then: - def result = persistence.loadScanRecord(SCAN_ID) - and: - result == scan - and: - surrealDb - .sqlAsMap(auth, "select * from wave_scan_vuln") - .result - .size() == 3 - and: - persistence.existsScanRecord(SCAN_ID) - - when: - def SCAN_ID2 = 'b2' - def BUILD_ID2 = '102' - def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1, CVE4], 1, "Error 'quote'", null) - and: - // should save the same CVE into another build - persistence.saveScanRecordAsync(scanRecord2) - sleep 200 - then: - def result2 = persistence.loadScanRecord(SCAN_ID2) - and: - result2 == scanRecord2 - and: - surrealDb - .sqlAsMap(auth, "select * from wave_scan_vuln") - .result - .size() == 4 - } - - def 'should save a scan and check it exists' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def surrealDb = applicationContext.getBean(SurrealClient) - def NOW = Instant.now() - def SCAN_ID = 'a1' - def BUILD_ID = '100' - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) - - expect: - !persistence.existsScanRecord(SCAN_ID) - - when: - persistence.saveScanRecordAsync(scan) - sleep 200 - then: - persistence.existsScanRecord(SCAN_ID) - } - - //== mirror records tests - - void "should save and load a mirror record by id"() { - given: - def storage = applicationContext.getBean(SurrealPersistenceService) - and: - def request = MirrorRequest.create( - 'source.io/foo', - 'target.io/foo', - 'sha256:12345', - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-123', - Instant.now(), - "GMT", - Mock(PlatformId) - ) - and: - storage.initializeDb() - and: - def result = MirrorEntry.of(request).getResult() - storage.saveMirrorResultAsync(result) - sleep 100 - - when: - def stored = storage.loadMirrorResult(request.mirrorId) - then: - stored == result - } - - void "should save and load a mirror record by target and digest"() { - given: - def digest = 'sha256:12345' - def timestamp = Instant.now() - def source = 'source.io/foo' - def target = 'target.io/foo' - and: - def storage = applicationContext.getBean(SurrealPersistenceService) - and: - def request1 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-1', - timestamp.minusSeconds(180), - "GMT", - Mock(PlatformId) ) - and: - def request2 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-2', - timestamp.minusSeconds(120), - "GMT", - Mock(PlatformId) ) - and: - def request3 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-3', - timestamp.minusSeconds(60), - "GMT", - Mock(PlatformId) ) - - and: - storage.initializeDb() - and: - def result1 = MirrorResult.of(request1).complete(1, 'err') - def result2 = MirrorResult.of(request2).complete(0, 'ok') - def result3 = MirrorResult.of(request3).complete(0, 'ok') - storage.saveMirrorResultAsync(result1) - storage.saveMirrorResultAsync(result2) - storage.saveMirrorResultAsync(result3) - sleep 100 - - when: - def stored = storage.loadMirrorSucceed(target, digest) - then: - stored == result3 - } - - def 'should remove surreal table from json' () { - given: - def json = /{"id":"wave_request:1234abc", "this":"one", "that":123 }/ - expect: - SurrealPersistenceService.patchSurrealId(json, "wave_request") - == /{"id":"1234abc", "this":"one", "that":123 }/ - } - - def 'should save 50KB container and conda file' (){ - given: - def data = RandomStringUtils.random(25600, true, true) - def persistence = applicationContext.getBean(SurrealPersistenceService) - final request = BuildRequest.of( - containerId: 'container1234', - containerFile: data, - condaFile: data, - workspace: Path.of("/some/path"), - targetImage: 'buildrepo:recipe-container1234', - identity: PlatformId.NULL, - platform: ContainerPlatform.of('amd64'), - cacheRepository: 'docker.io/my/cache', - ip: '127.0.0.1', - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER, - maxDuration: Duration.ofMinutes(1), - compression: BuildCompression.gzip - ) - and: - def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') - - and: - def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) - - when: - persistence.saveBuildAsync(build1) - sleep 100 - then: - persistence.loadBuild(request.buildId) == build1 - } - - def 'should find all builds' () { - given: - def surreal = applicationContext.getBean(SurrealClient) - def persistence = applicationContext.getBean(SurrealPersistenceService) - def auth = persistence.getAuthorization() - def request1 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_1' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) - def request2 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_2' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) - def request3 = BuildRequest.of( containerId: 'abc', buildId: 'bd-abc_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) - def request4 = BuildRequest.of( containerId: 'abc', buildId: 'bd-xyz_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(0), identity: PlatformId.NULL) - - def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) - def record1 = WaveBuildRecord.fromEvent(new BuildEvent(request1, result1)) - surreal.insertBuild(auth, record1) - and: - def result2 = new BuildResult(request2.buildId, -1, "ok", request2.startTime, Duration.ofSeconds(2), null) - def record2 = WaveBuildRecord.fromEvent(new BuildEvent(request2, result2)) - surreal.insertBuild(auth, record2) - and: - def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) - def record3 = WaveBuildRecord.fromEvent(new BuildEvent(request3, result3)) - surreal.insertBuild(auth, record3) - and: - def result4 = new BuildResult(request4.buildId, -1, "ok", request4.startTime, Duration.ofSeconds(2), null) - def record4 = WaveBuildRecord.fromEvent(new BuildEvent(request4, result4)) - surreal.insertBuild(auth, record4) - - expect: - persistence.allBuilds('abc') == [record3, record2, record1] - and: - persistence.allBuilds('bd-abc') == [record3, record2, record1] - and: - persistence.allBuilds('ab') == null - } - - def 'should find all scans' () { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') - def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') - def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') - def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') - def scan1 = new WaveScanRecord('sc-1234567890abcdef_1', '100', null, null, CONTAINER_IMAGE, PLATFORM, Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3, CVE4], null, null, null) - def scan2 = new WaveScanRecord('sc-1234567890abcdef_2', '101', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null, null) - def scan3 = new WaveScanRecord('sc-1234567890abcdef_3', '102', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2], null, null, null) - def scan4 = new WaveScanRecord('sc-01234567890abcdef_4', '103', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) - - when: - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - persistence.saveScanRecordAsync(scan3) - persistence.saveScanRecordAsync(scan4) - and: - sleep 200 - then: - persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1] - and: - persistence.allScans("1234567890") == null - } - - void 'should get paginated mirror results'() { - given: - def digest = 'sha256:12345' - def timestamp = Instant.now() - def source = 'source.io/foo' - def target = 'target.io/foo' - and: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def request1 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-1', - timestamp.minusSeconds(180), - "GMT", - Mock(PlatformId) ) - and: - def request2 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-2', - timestamp.minusSeconds(120), - "GMT", - Mock(PlatformId) ) - and: - def request3 = MirrorRequest.create( - source, - target, - digest, - ContainerPlatform.DEFAULT, - Path.of('/workspace'), - '{auth json}', - 'scan-3', - timestamp.minusSeconds(60), - "GMT", - Mock(PlatformId) ) - and: - def result1 = MirrorResult.of(request1).complete(1, 'err') - def result2 = MirrorResult.of(request2).complete(0, 'ok') - def result3 = MirrorResult.of(request3).complete(0, 'ok') - and: - persistence.saveMirrorResultAsync(result1) - persistence.saveMirrorResultAsync(result2) - persistence.saveMirrorResultAsync(result3) - sleep(300) - - when: - def mirrors = persistence.getMirrorsPaginated(3,0) - - then: - mirrors.size() == 3 - and: - mirrors.contains(result1) - mirrors.contains(result2) - mirrors.contains(result3) - } - - void 'should retrieve paginated container requests'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def TOKEN = '123abc' - def cfg = new ContainerConfig(entrypoint: ['/opt/fusion'], - layers: [ new ContainerLayer(location: 'https://fusionfs.seqera.io/releases/v2.2.8-amd64.json')]) - def req = new SubmitContainerTokenRequest( - towerEndpoint: 'https://tower.nf', - towerWorkspaceId: 100, - containerConfig: cfg, - containerPlatform: ContainerPlatform.of('amd64'), - buildRepository: 'build.docker.io', - cacheRepository: 'cache.docker.io', - fingerprint: 'xyz', - timestamp: Instant.now().toString() - ) - def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = ContainerRequest.of(requestId: TOKEN, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: "from ubuntu" ) - def wave = "wave.io/wt/$TOKEN/hello-world" - def addr = "100.200.300.400" - def exp = Instant.now().plusSeconds(3600) - and: - def request = new WaveContainerRecord(req, data, wave, addr, exp) - and: - persistence.saveContainerRequestAsync(request) - sleep(100) - - when: - def requests = persistence.getRequestsPaginated(1,0) - - then: - requests.size() == 1 - and: - requests[0].waveImage == 'wave.io/wt/123abc/hello-world' - } - - void 'should retrieve paginated scan records when data exists'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scan1 = new WaveScanRecord(id: 'scan1', containerImage: 'image1', status: 'SUCCEEDED', vulnerabilities:[]) - def scan2 = new WaveScanRecord(id: 'scan2', containerImage: 'image2', status: 'FAILED', vulnerabilities:[]) - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - sleep(200) - - when: - def scans = persistence.getScansPaginated(2, 0) - - then: - scans.size() == 2 - and: - scans.contains(scan1) - scans.contains(scan2) - } - - void 'should return null when no scan records exist for pagination'() { - when: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scans = persistence.getScansPaginated(2, 0) - - then: - scans == null - } - - void 'should handle pagination offset correctly for scan records'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - and: - def scan1 = new WaveScanRecord(id: 'scan1', containerImage: 'image1', status: 'SUCCEEDED', vulnerabilities:[]) - def scan2 = new WaveScanRecord(id: 'scan2', containerImage: 'image2', status: 'FAILED', vulnerabilities:[]) - def scan3 = new WaveScanRecord(id: 'scan3', containerImage: 'image3', status: 'PENDING', vulnerabilities:[]) - persistence.saveScanRecordAsync(scan1) - persistence.saveScanRecordAsync(scan2) - persistence.saveScanRecordAsync(scan3) - sleep(200) - - when: - def scans = persistence.getScansPaginated(2, 1) - - then: - scans.size() == 2 - scans.contains(scan2) - scans.contains(scan3) - } - - def 'getRequestsPaginated should return list of WaveContainerRecord'() { - given: - def persistence = applicationContext.getBean(SurrealPersistenceService) - def surreal = applicationContext.getBean(SurrealClient) - and: - def records = [ - [ - id : "req-123", - fusionVersion: [number: "3.0.0", arch: "x86"], - workspaceId : 100, - containerImage: "ubuntu:20.04" - ], - [ - id : "req-456", - fusionVersion: "2.4.0", - workspaceId : 101, - containerImage: "alpine:latest" - ], - [ - id : "req-789", - workspaceId : 101, - containerImage: "alpine:latest" - ] - ] - and: - records.each { record -> - def json = JacksonHelper.toJson(record) - def query = "INSERT INTO wave_request $json" - final future = new CompletableFuture() - surreal - .sqlAsync(persistence.getAuthorization(), query) - .subscribe({result -> - log.trace "Container request with token '$data.id' saved record: ${result}" - future.complete(null) - }, - {error-> - log.error "Failed to save container request with token '$data.id': ${error.message}" - future.completeExceptionally(error) - }) - } - - sleep 300 - when: - def result = persistence.getRequestsPaginated(10, 0) - - then: - result.size() == 3 - and: - result[0].id == 'wave_request:⟨req-123⟩' - result[0].fusionVersion == '3.0.0' - result[0].containerImage == 'ubuntu:20.04' - and: - result[1].id == 'wave_request:⟨req-456⟩' - result[1].fusionVersion == '2.4.0' - result[1].containerImage == 'alpine:latest' - and: - result[2].id == 'wave_request:⟨req-789⟩' - result[2].fusionVersion == null - result[2].containerImage == 'alpine:latest' - } -} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy deleted file mode 100644 index 0aae3be3a1..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationLockTest.groovy +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import spock.lang.Shared -import spock.lang.Specification -import spock.lang.Timeout - -import java.time.Duration - -import io.micronaut.context.ApplicationContext -import io.seqera.fixtures.redis.RedisTestContainer -import redis.clients.jedis.JedisPool - -/** - * - * @author Paolo Di Tommaso - */ -@Timeout(30) -class DataMigrationLockTest extends Specification implements RedisTestContainer { - - @Shared - ApplicationContext context - - def setup() { - context = ApplicationContext.run('test', 'redis') - } - - def 'should get lock or timeout' () { - given: - def key = '/foo/v1' - def connection = context.getBean(JedisPool).getResource() - - when: - def lock1 = DataMigrationService.acquireLock(connection, key) - then: - lock1 != null - - when: - def lock2 = DataMigrationService.acquireLock(connection, key, Duration.ofMillis(100)) - then: - lock2 == null - - when: - lock1.release() - lock2 = DataMigrationService.acquireLock(connection, key) - then: - lock2 != null - - cleanup: - connection?.close() - } -} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy deleted file mode 100644 index b57827e54b..0000000000 --- a/src/test/groovy/io/seqera/wave/service/persistence/migrate/DataMigrationServiceTest.groovy +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2025, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.persistence.migrate - -import spock.lang.Specification - -import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.service.mirror.MirrorResult -import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.persistence.impl.SurrealPersistenceService -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateCache -import io.seqera.wave.service.persistence.migrate.cache.DataMigrateEntry -import io.seqera.wave.service.persistence.postgres.PostgresPersistentService -import io.seqera.wave.service.persistence.impl.SurrealClient -import io.seqera.wave.service.request.ContainerRequest -import io.seqera.wave.tower.PlatformId -import io.seqera.wave.tower.User - -/** - * - * @author Munish Chouhan - */ -class DataMigrationServiceTest extends Specification { - - def surrealService = Mock(SurrealPersistenceService) - def postgresService = Mock(PostgresPersistentService) - def surrealClient = Mock(SurrealClient) - def dataMigrateCache = Mock(DataMigrateCache) - - def service - int pageSize = 1000 - - void setup() { - service = new DataMigrationService() - service.surrealService = surrealService - service.postgresService = postgresService - service.surrealDb = surrealClient - service.pageSize = pageSize - service.dataMigrateCache = dataMigrateCache - } - - def "should not migrate if build records are empty"() { - given: - surrealService.getBuildsPaginated(pageSize,0) >> [] - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - when: - service.migrateBuildRecords() - - then: - 0 * postgresService.saveBuildAsync(_) - and: - 0 * dataMigrateCache.put(_, _) - } - - def "should migrate build records in batches of 100"() { - given: - def builds = (1..100).collect { new WaveBuildRecord(buildId: it) } - surrealService.getBuildsPaginated(pageSize,0) >> builds - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - - when: - service.migrateBuildRecords() - - then: - 100 * postgresService.saveBuild(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_BUILD, new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 100)) - } - - def "should catch and log exception during build migration"() { - given: - surrealService.getBuildsPaginated(pageSize,0) >> [[id: 1]] - postgresService.saveBuildAsync(_) >> { throw new RuntimeException("DB error") } - dataMigrateCache.get(DataMigrationService.TABLE_NAME_BUILD) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_BUILD, 0) - - when: - service.migrateBuildRecords() - - then: - noExceptionThrown() - } - - def "should not migrate if request records are empty"() { - given: - surrealService.getRequestsPaginated(pageSize,0) >> [] - dataMigrateCache.get(DataMigrationService.TABLE_NAME_REQUEST) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 0) - - when: - service.migrateRequests() - - then: - 0 * postgresService.saveContainerRequest(_) - and: - 0 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_REQUEST, pageSize) - } - - def "should migrate request records in batches"() { - given: - def requests = (1..101).collect { new WaveContainerRecord( - new SubmitContainerTokenRequest(towerWorkspaceId: it), - new ContainerRequest(requestId: it, identity: PlatformId.of(new User(id: it), Mock(SubmitContainerTokenRequest))), - null, null, null) } - surrealService.getRequestsPaginated(pageSize,0) >> requests - dataMigrateCache.get(DataMigrationService.TABLE_NAME_REQUEST) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 0) - - when: - service.migrateRequests() - - then: - 101 * postgresService.saveContainerRequest(_,_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_REQUEST, new DataMigrateEntry(DataMigrationService.TABLE_NAME_REQUEST, 101)) - } - - def "should migrate scan records in batches"() { - given: - def scans = (1..99).collect { new WaveScanRecord(id: it) } - surrealService.getScansPaginated(pageSize, 0) >> scans - dataMigrateCache.get(DataMigrationService.TABLE_NAME_SCAN) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_SCAN, 0) - - when: - service.migrateScanRecords() - - then: - 99 * postgresService.saveScanRecord(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_SCAN, new DataMigrateEntry(DataMigrationService.TABLE_NAME_SCAN, 99)) - } - - def "should migrate mirror records in batches"() { - given: - def mirrors = (1..15).collect { new MirrorResult() } - surrealService.getMirrorsPaginated(pageSize, 0) >> mirrors - dataMigrateCache.get(DataMigrationService.TABLE_NAME_MIRROR) >> - new DataMigrateEntry(DataMigrationService.TABLE_NAME_MIRROR, 0) - - when: - service.migrateMirrorRecords() - - then: - 15 * postgresService.saveMirrorResult(_) - and: - 1 * dataMigrateCache.put(DataMigrationService.TABLE_NAME_MIRROR, new DataMigrateEntry(DataMigrationService.TABLE_NAME_MIRROR, 15)) - } - - def "fixRequestId should extract or return correct id for various input formats"() { - expect: - DataMigrationService.fixRequestId(input) == expected - - where: - input | expected - "wave_request:12345" | "12345" - "wave_request:⟨12345⟩" | "12345" - "wave_request:⟨abc-987⟩" | "abc-987" - "wave_request:" | "" - "wave_request:⟨⟩" | "" - "wave_request:⟨" | "" - "wave_request:⟨abc⟩extra" | "abc" - "12345" | "12345" - "some_other_prefix:12345" | "some_other_prefix:12345" - "" | "" - null | null - } -} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy index e09d0e74c5..d9eea73bef 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy @@ -197,7 +197,7 @@ class PostgresPersistentServiceTest extends Specification { and: persistentService.allBuilds('bd-abc') == [record3, record2, record1] and: - persistentService.allBuilds('ab') == null + persistentService.allBuilds('ab') == [] } // ===== --- container request---- ===== @@ -409,6 +409,6 @@ class PostgresPersistentServiceTest extends Specification { then: persistentService.allScans("1234567890abcdef") == [scan3, scan2, scan1] and: - persistentService.allScans("1234567890") == null + persistentService.allScans("1234567890") == [] } } diff --git a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy deleted file mode 100644 index c13cf2ebbe..0000000000 --- a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.test - -import spock.lang.Shared - -import org.slf4j.LoggerFactory -import org.testcontainers.containers.GenericContainer -import org.testcontainers.containers.output.Slf4jLogConsumer -import org.testcontainers.containers.wait.strategy.Wait -import org.testcontainers.utility.DockerImageName - -/** - * - * @author Jorge Aguilera - */ -trait SurrealDBTestContainer { - private static final def LOGGER = LoggerFactory.getLogger(SurrealDBTestContainer.class); - - @Shared - static GenericContainer surrealContainer = new GenericContainer(DockerImageName.parse("surrealdb/surrealdb:v1.5.4")) - .withExposedPorts(8000) - .withCommand("start","--user", "root", "--pass", "root", '--log', 'debug') - .waitingFor( - Wait.forLogMessage(".*Started web server on .*\\n", 1) - ) - .withLogConsumer(new Slf4jLogConsumer(LOGGER)) - - void restartDb(){ - if( surrealContainer.running) - surrealContainer.stop() - surrealContainer.start() - } - - String getSurrealHostName(){ - surrealContainer.getHost() - } - - String getSurrealPort(){ - surrealContainer.getMappedPort(8000) - } -} diff --git a/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy b/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy index b93ac24336..4fd0659f7c 100644 --- a/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/StringUtilsTest.groovy @@ -54,23 +54,6 @@ class StringUtilsTest extends Specification { null | '/a/bc/' } - @Unroll - def 'should strip surreal prefix' () { - expect: - StringUtils.surrealId(ID) == EXPECTED - - where: - ID | EXPECTED - null | null - 'foo' | 'foo' - and: - 'foo:100' | '100' - 'foo-bar:1-2-3' | '1-2-3' - and: - 'foo:⟨100⟩' | '100' - 'foo-bar:⟨1-2-3⟩' | '1-2-3' - } - @Unroll def 'should concat paths' () { expect: