olake/.github/workflows/integration-tests.yml at master · datazip-inc/olake · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
name: Integration Tests
on:
  push:
    branches:
      - "master"
    paths:
      - '**/*.go'
      - '**/*.java'
  pull_request:
    branches:
      - "*"
    paths:
      - '**/*.go'
      - '**/*.java'

env:
  MYSQL_ROOT_PASSWORD: root1234

jobs:
  integration-tests:
    environment: integration_tests
    runs-on: 32gb-runner
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version: '1.24.x'

      - name: Set up Java for Maven
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'

      - name: Install DB2 CLI Driver
        run: |
          go run github.com/ibmdb/go_ibm_db/[email protected]
          echo "IBM_DB_HOME=$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver" >> $GITHUB_ENV
          IBM_DB_HOME=$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver
          echo "CGO_CFLAGS=-I$IBM_DB_HOME/include" >> $GITHUB_ENV
          echo "CGO_LDFLAGS=-L$IBM_DB_HOME/lib -Wl,-rpath,$IBM_DB_HOME/lib" >> $GITHUB_ENV
          echo "LD_LIBRARY_PATH=$IBM_DB_HOME/lib" >> $GITHUB_ENV

      - name: Start Test Infrastructure
        run: |
          docker compose -f ./drivers/mysql/docker-compose.yml up -d
          docker compose -f ./drivers/postgres/docker-compose.yml up -d
          docker compose -f ./drivers/mongodb/docker-compose.yml up -d
          docker compose -f ./drivers/oracle/docker-compose.yml up -d
          docker compose -f ./drivers/db2/docker-compose.yml up -d
          docker compose -f ./drivers/mssql/docker-compose.yml up -d
          docker compose -f ./destination/iceberg/local-test/docker-compose.yml up minio mc postgres spark-iceberg -d

      - name: Wait for MySQL
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 30
          retry_wait_seconds: 5
          command: |
            docker exec olake_mysql-test mysql -h localhost -u root -p${{ env.MYSQL_ROOT_PASSWORD }} -e "SELECT 1"

      - name: Wait for PostgreSQL
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 30
          retry_wait_seconds: 5
          command: |
            docker exec olake_postgres-test psql -h localhost -U postgres -d postgres -c "SELECT 1"

      - name: Wait for MongoDB
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 30
          retry_wait_seconds: 5
          command: |
            docker exec primary_mongo mongosh --host localhost --port 27017 -u mongodb -p secure_password123 --authenticationDatabase admin --eval "db.adminCommand('ping')"

      - name: Wait for Oracle
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 30
          retry_wait_seconds: 5
          command: |
            docker exec oracle-23c bash -c "echo 'SELECT 1 FROM dual;' | sqlplus -s system/secret1234@//localhost:1521/ORCL"

      - name: Wait for DB2
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 10
          max_attempts: 30
          retry_wait_seconds: 25
          command: |
            docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'"

      - name: Wait for MSSQL
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 30
          retry_wait_seconds: 5
          command: |
            docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -Q "SELECT 1"

      - name: Initialize MSSQL Database
        run: |
          docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -d master -i /docker-entrypoint-initdb.d/01-init.sql


      - name: Set up Data Directories
        run: |
            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/postgres-data
            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/minio-data
            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/ivy-cache
            sudo chown -R 999:999 /home/runner/work/olake/olake/destination/iceberg/local-test/data
            sudo chmod -R 777 /home/runner/work/olake/olake/destination/iceberg/local-test/data

      - name: Install Go Dependencies
        run: go mod download

      - name: Build Project
        run: go build -v ./...

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Build Iceberg Sink
        working-directory: ./destination/iceberg/olake-iceberg-java-writer
        run: mvn clean package -DskipTests

      - name: Run Integration Tests
        run: |
          go test -v -p 6 ./drivers/mysql/internal/... ./drivers/postgres/internal/... ./drivers/mongodb/internal/... ./drivers/oracle/internal/... ./drivers/db2/internal/... ./drivers/mssql/internal/... -timeout 0 -run 'Integration'

      - name: Cleanup
        if: always()
        run: |
          docker compose -f ./destination/iceberg/local-test/docker-compose.yml down
          docker compose -f ./drivers/mysql/docker-compose.yml down
          docker compose -f ./drivers/postgres/docker-compose.yml down
          docker compose -f ./drivers/oracle/docker-compose.yml down
          docker compose -f ./drivers/mongodb/docker-compose.yml down
          docker compose -f ./drivers/db2/docker-compose.yml down
          docker compose -f ./drivers/mssql/docker-compose.yml down