opengrants-platform/docker-compose.yml at master · metagov/opengrants-platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
services:
  dagster:
    container_name: dagster
    build:
      context: .
      dockerfile: og_dagster/Dockerfile
    restart: unless-stopped
    ports:
      - '3000:3000'
    volumes:
      - ./og_dagster:/app
      - ./dbt_project:/app/dbt_project
      - ./duckdb:/data/duckdb
      - ./raw_data:/app/raw_data
    environment:
      - DAGSTER_HOME=/app
      - DATABASE_URL="${DATABASE_URL}"
      - DUCKDB_CACHE_PATH=/data/duckdb/local.duckdb

    env_file:
      - .env
    command:
      - /bin/sh
      - -c
      - |
        dbt parse --project-dir /app/dbt_project --profiles-dir /app/dbt_project 2>/dev/null || echo "dbt parse skipped (non-fatal)"
        exec dagster dev -h 0.0.0.0 -p 3000 -w workspace.yaml

  dagster-readonly:
    container_name: dagster-readonly
    build:
      context: .
      dockerfile: og_dagster/Dockerfile
    restart: unless-stopped
    ports:
      - '3001:3000'
    volumes:
      - ./og_dagster:/app
      - ./dbt_project:/app/dbt_project
      - ./duckdb:/data/duckdb
      - ./raw_data:/app/raw_data
    environment:
      - DAGSTER_HOME=/app
      - DATABASE_URL="${DATABASE_URL}"
      - DUCKDB_CACHE_PATH=/data/duckdb/local.duckdb
    env_file:
      - .env
    command:
      - /bin/sh
      - -c
      - |
        echo "=================================================="
        echo "🚀 Dagster webserver starting...in 2mins. Note: Port mapped 3001:3000"
        echo "📊 Access it at: http://localhost:3001"
        echo "=================================================="
        exec dagster-webserver --read-only -h 0.0.0.0 -p 3000 -w workspace.yaml

  dlt:
    container_name: dlt
    build:
      context: ./dlt_pipelines
      dockerfile: Dockerfile
    restart: on-failure
    volumes:
      - ./dlt_pipelines:/app
      - ./duckdb:/data/duckdb
    environment:
      - DUCKDB_CACHE_PATH=/data/duckdb/local.duckdb
      - DATABASE_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=require"


    env_file:
      - .env
    command:
      - python
      - loaders/privote_loader.py

  dbt:
    container_name: dbt
    build:
      context: ./dbt_project
      dockerfile: Dockerfile
    restart: on-failure
    volumes:
      - ./dbt_project:/app
      - ./duckdb:/data/duckdb
    env_file:
      - .env
    environment:
      - DBT_TARGET=prod_postgres
    entrypoint:
      - dbt
    working_dir: /app

volumes:
  duckdb: null