diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1119d5c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,44 @@ +# Build artifacts +target/ +*.so +*.dylib +*.dll + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Git +.git/ +.gitignore + +# Documentation build +site/ +docs/site/ + +# Test results +results/ +regression.diffs +regression.out + +# Local development +.env +.env.local + +# Cache +.cache/ +node_modules/ + +# Logs +*.log + +# Temporary files +tmp/ +temp/ diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..b8cd716 --- /dev/null +++ b/.env.local @@ -0,0 +1,3 @@ +GITHUB_ACCESS_TOKEN=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 +GITHUB_INSTANCE_URL=https://github.com +GITLAB_INSTANCE_URL=https://gitlab.com diff --git a/.github/workflows/build_all_versions.yml b/.github/workflows/build_all_versions.yml new file mode 100644 index 0000000..370fc52 --- /dev/null +++ b/.github/workflows/build_all_versions.yml @@ -0,0 +1,45 @@ +# Main workflow that uses the reusable build_and_test_pgver workflow +# Similar to how GitLab CI would include the template + +name: 'Build and Test All PostgreSQL Versions' + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + schedule: + - cron: '0 2 * * 0' # Weekly on Sunday at 2 AM + workflow_dispatch: + +jobs: + # Build and test PostgreSQL 13 (with always flag) + pg13: + uses: ./.github/workflows/build_and_test_pgver.yml + with: + pgver: 'pg13' + always: 'true' # Always run for pg13 + + # Build and test PostgreSQL 14 + pg14: + uses: ./.github/workflows/build_and_test_pgver.yml + with: + pgver: 'pg14' + + # Build and test PostgreSQL 15 + pg15: + uses: ./.github/workflows/build_and_test_pgver.yml + with: + pgver: 'pg15' + + # Build and test PostgreSQL 16 + pg16: + uses: ./.github/workflows/build_and_test_pgver.yml + with: + pgver: 'pg16' + + # Build and test PostgreSQL 17 + pg17: + uses: ./.github/workflows/build_and_test_pgver.yml + with: + pgver: 'pg17' diff --git a/.github/workflows/build_and_test_pgver.yml b/.github/workflows/build_and_test_pgver.yml new file mode 100644 index 0000000..0254fe4 --- /dev/null +++ b/.github/workflows/build_and_test_pgver.yml @@ -0,0 +1,168 @@ +# GitHub Actions workflow converted from GitLab CI template +# Original: .gitlab/job_templates/build_and_test_pgver.yml + +name: 'Build and Test PostgreSQL Version' + +on: + workflow_call: + inputs: + always: + description: 'If defined, all jobs are launched in any case. Usually you want at least 1 major version to be built/tested on all pipelines.' + type: string + default: '' + required: false + pgver: + description: 'The PostgreSQL major version with the "pg" prefix (e.g. `pg13`, `pg16`, etc.)' + type: string + required: true + workflow_dispatch: + inputs: + pg_version: + description: 'PostgreSQL version' + required: true + default: '13' + +env: + PGVER: ${{ inputs.pgver }} + ALWAYS: ${{ inputs.always }} + +jobs: + ## + ## B U I L D + ## + build: + name: 'build-${{ inputs.pgver }}' + runs-on: ubuntu-latest + # Use GitHub Container Registry for pmpetit + container: ghcr.io/pmpetit/postgresql_pglinter:pgrx + + # Convert GitLab rules to GitHub Actions conditions + if: | + inputs.always != '' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.ref == github.event.repository.default_branch || + startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Lint (only for default PG version) + if: inputs.always != '' + run: make lint + + - name: Build the extension package + run: make + + - name: Launch postgres instance and run unit tests + run: make test + # The functional tests will be launched later with pg_regress + + - name: Build packages + run: | + make deb + make rpm + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: 'pglinter-${{ inputs.pgver }}' + path: 'target/release/pglinter-${{ inputs.pgver }}' + retention-days: 1 + + ## + ## T E S T + ## + installcheck: + name: 'installcheck-${{ inputs.pgver }}' + runs-on: ubuntu-latest + needs: build + container: ghcr.io/pmpetit/postgresql_pglinter:pgrx + + if: | + inputs.always != '' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.ref == github.event.repository.default_branch || + startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: 'pglinter-${{ inputs.pgver }}' + path: 'target/release/pglinter-${{ inputs.pgver }}' + + - name: Install binaries + run: make install + + - name: Run functional tests with pg_regress + run: make installcheck + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: 'test-results-${{ inputs.pgver }}' + path: | + target/release/pglinter-${{ inputs.pgver }} + results + retention-days: 1 + + ## + ## D E P L O Y + ## + upload-packages: + name: 'upload-packages-${{ inputs.pgver }}' + runs-on: ubuntu-latest + needs: build + + # Convert GitLab rules: run on tags always, otherwise manual + if: | + startsWith(github.ref, 'refs/tags/') || + github.event_name == 'workflow_dispatch' + + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: 'pglinter-${{ inputs.pgver }}' + path: 'target/release/pglinter-${{ inputs.pgver }}' + + - name: Upload DEB package to GitHub Releases + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v1 + with: + files: 'target/release/pglinter-${{ inputs.pgver }}/*.deb' + name: 'pglinter_${{ inputs.pgver }}-${{ github.ref_name }}.amd64.deb' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload RPM package to GitHub Releases + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v1 + with: + files: 'target/release/pglinter-${{ inputs.pgver }}/*.rpm' + name: 'pglinter_${{ inputs.pgver }}-${{ github.ref_name }}.x86_64.rpm' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ## + ## R E L E A S E + ## + release-packages: + name: 'release-packages-${{ inputs.pgver }}' + runs-on: ubuntu-latest + needs: [build, upload-packages] + + # Only run on tags + if: startsWith(github.ref, 'refs/tags/') + + steps: + - name: Create Release Entry + run: echo "Release ${{ github.ref_name }} created for ${{ inputs.pgver }}" + # Note: GitHub Releases are automatically created when using softprops/action-gh-release + # The actual release creation is handled by the upload-packages job diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..7605e3f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,42 @@ +name: Deploy Packages + +on: + push: + tags: + - '*' # Triggers the workflow on any tag push + workflow_dispatch: + inputs: + pgver: + description: 'PostgreSQL version' + required: true + default: '16' + + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # This is crucial for uploading packages to GitHub Packages + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: built-packages # This should match the name from your 'build' job + path: target/release/anon-${{ inputs.pgver }} + + - name: Upload .deb package to GitHub Packages + uses: actions/upload-artifact@v3 + with: + name: postgresql_anonymizer_${{ inputs.pgver }}-${{ github.ref_name }}.amd64.deb + path: target/release/anon-${{ inputs.pgver }}/*.deb + + - name: Upload .rpm package to GitHub Packages + uses: actions/upload-artifact@v3 + with: + name: postgresql_anonymizer_${{ inputs.pgver }}-${{ github.ref_name }}.x86_64.rpm + path: target/release/anon-${{ inputs.pgver }}/*.rpm diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml deleted file mode 100644 index 8e91a58..0000000 --- a/.github/workflows/precommit.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Pre-commit Checks - -# Run precommit checks on pushes and PRs -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main ] - -# Ensure only one workflow runs at a time per PR/branch -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - precommit: - name: πŸ” Pre-commit Checks - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4.3.0 - - - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1.10.1 - with: - components: rustfmt, clippy - cache: true - - - name: Set up Node.js for markdownlint - uses: actions/setup-node@v4.1.0 - with: - node-version: '18' - - - name: Install documentation tools - run: | - # Install markdownlint for markdown linting - npm install -g markdownlint-cli - - - name: 🎨 Check Rust code formatting - run: | - echo "🎨 Checking Rust code formatting..." - cargo fmt --check - - - name: πŸ“Ž Check basic Rust syntax (skip pgrx for now) - run: | - echo "πŸ“Ž Checking basic Rust syntax..." - echo "Note: Skipping cargo check as it requires pgrx setup in CI" - echo "Full Rust compilation and linting will be done in comprehensive CI workflow" - echo "This workflow focuses on formatting and documentation checks" - - - name: πŸ“š Lint documentation - run: | - echo "πŸ“š Linting Markdown documentation..." - if command -v markdownlint > /dev/null; then - markdownlint docs/**/*.md *.md || true - echo "πŸ’‘ Tip: Run 'make lint-docs-fix' to automatically fix many issues" - else - echo "markdownlint not found, skipping markdown lint" - fi - - - name: ⚑ Summary of precommit checks - run: | - echo "⚑ Precommit checks completed!" - echo "" - echo "Summary of checks performed:" - echo " βœ… Rust code formatting (cargo fmt --check)" - echo " βœ… Basic Rust syntax check (cargo check)" - echo " βœ… Markdown documentation linting" - echo "" - echo "Note: Full clippy with pgrx features is done in comprehensive CI workflow" - - - name: βœ… All precommit checks passed - run: | - echo "πŸŽ‰ All precommit checks passed successfully!" - echo "βœ… Code formatting is correct" - echo "βœ… No linting issues found" - echo "βœ… Documentation is properly formatted" - echo "" - echo "Code quality standards maintained! πŸš€" diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml deleted file mode 100644 index 4f68672..0000000 --- a/.github/workflows/regression-tests.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Regression Tests - -# Run PostgreSQL regression tests using make installcheck -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main ] - # Allow manual triggering - workflow_dispatch: - inputs: - regress_tests: - description: 'Specific tests to run (space-separated, leave empty for all)' - required: false - default: '' - -# Ensure only one workflow runs at a time per branch -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - regression-tests: - name: πŸ§ͺ PostgreSQL Regression Tests - runs-on: ubuntu-latest - - strategy: - matrix: - pg_version: [13, 14, 15, 16] - fail-fast: false - - steps: - - name: Checkout repository - uses: actions/checkout@v4.3.0 - - - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1.10.1 - with: - components: rustfmt, clippy - cache: true - - - name: Install PostgreSQL ${{ matrix.pg_version }} - run: | - # Add PostgreSQL APT repository - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list - sudo apt-get update - - # Install PostgreSQL and development packages - sudo apt-get install -y \ - postgresql-${{ matrix.pg_version }} \ - postgresql-server-dev-${{ matrix.pg_version }} \ - postgresql-client-${{ matrix.pg_version }} - - - name: Install pgrx - run: | - cargo install --locked cargo-pgrx --force - # Use version-specific pg_config - cargo pgrx init --pg${{ matrix.pg_version }} /usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config - - - name: Build extension - run: | - echo "πŸ”¨ Building pglinter extension for PostgreSQL ${{ matrix.pg_version }}..." - PGVER=pg${{ matrix.pg_version }} make extension - - - name: Install extension - run: | - echo "πŸ“¦ Installing pglinter extension..." - sudo PGVER=pg${{ matrix.pg_version }} make install - - - name: Setup PostgreSQL environment - run: | - echo "πŸš€ Setting up PostgreSQL ${{ matrix.pg_version }} environment..." - # pgrx will manage PostgreSQL instances via `make start` - # No need to start system PostgreSQL service - echo "PostgreSQL ${{ matrix.pg_version }} is ready for pgrx management" - - - name: Run regression tests (all) - if: github.event.inputs.regress_tests == '' - run: | - echo "πŸ§ͺ Running all regression tests..." - PGVER=pg${{ matrix.pg_version }} make installcheck - env: - PGHOST: localhost - PGUSER: postgres - PGDATABASE: pglinter_test - - - name: Run regression tests (specific) - if: github.event.inputs.regress_tests != '' - run: | - echo "πŸ§ͺ Running specific regression tests: ${{ github.event.inputs.regress_tests }}" - PGVER=pg${{ matrix.pg_version }} make installcheck REGRESS="${{ github.event.inputs.regress_tests }}" - env: - PGHOST: localhost - PGUSER: postgres - PGDATABASE: pglinter_test - - - name: Upload regression test results - if: failure() - uses: actions/upload-artifact@v4.6.2 - with: - name: regression-results-pg${{ matrix.pg_version }} - path: | - regression.diffs - regression.out - results/ - retention-days: 7 - - - name: Display test results on failure - if: failure() - run: | - echo "❌ Regression tests failed!" - echo "" - if [ -f regression.diffs ]; then - echo "=== REGRESSION DIFFS ===" - cat regression.diffs - echo "" - fi - if [ -f regression.out ]; then - echo "=== REGRESSION OUTPUT ===" - tail -50 regression.out - fi - - - name: βœ… All regression tests passed - if: success() - run: | - echo "πŸŽ‰ All regression tests passed successfully!" - echo "βœ… PostgreSQL ${{ matrix.pg_version }} extension works correctly" - echo "βœ… All test cases passed" - echo "" - echo "Extension is ready for PostgreSQL ${{ matrix.pg_version }}! πŸš€" - - summary: - name: πŸ“‹ Test Summary - runs-on: ubuntu-latest - needs: regression-tests - if: always() - - steps: - - name: Generate test summary - run: | - echo "## πŸ§ͺ Regression Test Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| PostgreSQL Version | Status |" >> $GITHUB_STEP_SUMMARY - echo "|-------------------|--------|" >> $GITHUB_STEP_SUMMARY - - # Note: In a real scenario, you'd check the actual results - # This is a placeholder for the summary logic - echo "| 13 | ${{ needs.regression-tests.result == 'success' && 'βœ… Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY - echo "| 14 | ${{ needs.regression-tests.result == 'success' && 'βœ… Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY - echo "| 15 | ${{ needs.regression-tests.result == 'success' && 'βœ… Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY - echo "| 16 | ${{ needs.regression-tests.result == 'success' && 'βœ… Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "${{ needs.regression-tests.result }}" == "success" ]; then - echo "πŸŽ‰ **All tests passed!** The extension works correctly across all PostgreSQL versions." >> $GITHUB_STEP_SUMMARY - else - echo "❌ **Some tests failed.** Check the individual job logs and regression diffs for details." >> $GITHUB_STEP_SUMMARY - fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..26bb180 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,35 @@ +name: Create Release + +on: + push: + tags: + - '*' # This runs the workflow on any tag push + workflow_dispatch: + inputs: + pgver: + description: 'PostgreSQL version' + required: true + default: '16' + +jobs: + release: + name: Create GitHub Release + runs-on: ubuntu-latest + permissions: + contents: write # This permission is needed to create a release + + steps: + - name: Create Release + id: create_release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.ref }} + name: Release ${{ github.ref }} + body: 'This is a release for tag ${{ github.ref }}' + prerelease: false # or true if it's a prerelease + files: | + # Here you'll list the files to be attached to the release. + # You'll need to use a step to download or build these files first. + # Example: + # path/to/your/postgresql_anonymizer_*.x86_64.rpm + # path/to/your/postgresql_anonymizer_*.amd64.deb diff --git a/.gitignore b/.gitignore index c4d900c..b2ba1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ contrib_regression_test_restore.pgsql results/* tests/results/* tests/tmp/* +generated/ diff --git a/Cargo.toml b/Cargo.toml index 0c2ac78..2d7c5f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,15 +17,16 @@ pg14 = ["pgrx/pg14", "pgrx-tests/pg14" ] pg15 = ["pgrx/pg15", "pgrx-tests/pg15" ] pg16 = ["pgrx/pg16", "pgrx-tests/pg16" ] pg17 = ["pgrx/pg17", "pgrx-tests/pg17" ] +pg18 = ["pgrx/pg18", "pgrx-tests/pg18" ] pg_test = [] [dependencies] -pgrx = "=0.14.3" +pgrx = "0.16.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" [dev-dependencies] -pgrx-tests = "=0.14.3" +pgrx-tests = "0.16.0" [profile.dev] panic = "unwind" diff --git a/Makefile b/Makefile index 43b6929..c5f38c3 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PGRX?=cargo pgrx PGVER?=$(shell grep 'default = \[".*\"]' Cargo.toml | sed -e 's/.*\["//' | sed -e 's/"].*//') PG_MAJOR_VERSION=$(PGVER:pg%=%) -pglinter_VERSION?=$(shell grep '^version *= *' Cargo.toml | sed 's/^version *= *//' | tr -d '\"' | tr -d ' ' ) +PGLINTER_VERSION?=$(shell grep '^version *= *' Cargo.toml | sed 's/^version *= *//' | tr -d '\"' | tr -d ' ' ) # use `TARGET=debug make run` for more detailed errors TARGET?=release @@ -28,16 +28,16 @@ PGDATA_DIR=~/.pgrx/data-$(PG_MAJOR_VERSION) # Be sure to use the PGRX version (PGVER) of the postgres binaries PATH:=$(PG_BINDIR):${PATH} -# This is where the package is placed -TARGET_SHAREDIR?=$(TARGET_DIR)$(PG_SHAREDIR) -TARGET_PKGLIBDIR?=$(TARGET_DIR)$(PG_PKGLIBDIR) +# This is where the package is placed - updated for pgrx 0.16.0 structure +TARGET_SHAREDIR?=$(TARGET_DIR)usr/share/postgresql/$(PG_MAJOR_VERSION) +TARGET_PKGLIBDIR?=$(TARGET_DIR)usr/lib/postgresql/$(PG_MAJOR_VERSION)/lib PG_REGRESS?=$(PG_PKGLIBDIR)/pgxs/src/test/regress/pg_regress PG_SOCKET_DIR?=/var/lib/postgresql/.pgrx/ PGHOST?=localhost PGPORT?=288$(subst pg,,$(PGVER)) PSQL_OPT?=--host $(PGHOST) --port $(PGPORT) -PGDATABASE?=pglinter_test +PGDATABASE?=contrib_regression ## ## Test configuration. @@ -57,6 +57,23 @@ REGRESS_TESTS = t003_minimal # REGRESS_TESTS+= t008 # REGRESS_TESTS+= t010 +# We try our best to write tests that produce the same output on all the 5 +# current Postgres major versions. But sometimes it's really hard to do and +# we generally prefer simplicity over complex output manipulation tricks. +# +# In these few special cases, we use conditional tests with the following +# naming rules: +# * the _PG15+ suffix means PostgreSQL 15 and all the major versions after +# * the _PG13- suffix means PostgreSQL 13 and all the major versions below + +REGRESS_TESTS_PG13 = elevation_via_rule_PG15- elevation_via_security_definer_function_PG14- +REGRESS_TESTS_PG14 = elevation_via_rule_PG15- elevation_via_security_definer_function_PG14- +REGRESS_TESTS_PG15 = elevation_via_rule_PG15- +REGRESS_TESTS_PG16 = +REGRESS_TESTS_PG17 = + +REGRESS_TESTS+=${REGRESS_TESTS_PG${PG_MAJOR_VERSION}} + # Use this var to add more tests #PG_TEST_EXTRA ?= "" REGRESS_TESTS+=$(PG_TEST_EXTRA) @@ -80,60 +97,18 @@ extension: ## install: - cp -r $(TARGET_SHAREDIR)/extension/* $(PG_SHAREDIR)/extension/ - install $(TARGET_PKGLIBDIR)/$(LIB) $(PG_PKGLIBDIR) + cp -r $(TARGET_DIR)$(PG_SHAREDIR)/extension/* $(PG_SHAREDIR)/extension/; \ + install $(TARGET_DIR)$(PG_PKGLIBDIR)/$(LIB) $(PG_PKGLIBDIR); \ ## -## TESTING +## INSTALLCHECK +## +## These are the functional tests, the unit tests are run with Cargo ## -# Test a single file (e.g., make test-b001) -test-%: stop start - @echo "Running test: $*" - dropdb $(PSQL_OPT) --if-exists $(PGDATABASE) || echo 'Database did not exist' - createdb $(PSQL_OPT) $(PGDATABASE) - psql $(PSQL_OPT) $(PGDATABASE) -f tests/sql/$*.sql - -# Run all tests -test-all: stop start - @echo "Running all pglinter tests..." - dropdb $(PSQL_OPT) --if-exists $(PGDATABASE) || echo 'Database did not exist' - createdb $(PSQL_OPT) $(PGDATABASE) - @for test in $(REGRESS_TESTS); do \ - echo "Running test: $$test"; \ - psql $(PSQL_OPT) $(PGDATABASE) -f tests/sql/$$test.sql || exit 1; \ - done - @echo "All tests completed successfully!" - -# Test with output to prompt (no file output) -test-prompt-%: stop start - @echo "Running test with prompt output: $*" - dropdb $(PSQL_OPT) --if-exists $(PGDATABASE) || echo 'Database did not exist' - createdb $(PSQL_OPT) $(PGDATABASE) - @echo "BEGIN;" > /tmp/test_$*.sql - @echo "CREATE TABLE IF NOT EXISTS my_table_without_pk (id INT, name TEXT, code TEXT, enable BOOL DEFAULT TRUE, query TEXT, warning_level INT, error_level INT, scope TEXT);" >> /tmp/test_$*.sql - @echo "CREATE EXTENSION IF NOT EXISTS pglinter;" >> /tmp/test_$*.sql - @echo "SELECT pglinter.perform_base_check();" >> /tmp/test_$*.sql - @echo "ROLLBACK;" >> /tmp/test_$*.sql - psql $(PSQL_OPT) $(PGDATABASE) -f /tmp/test_$*.sql - @rm -f /tmp/test_$*.sql - -# Test convenience functions -test-convenience: stop start - @echo "Testing convenience functions..." - dropdb $(PSQL_OPT) --if-exists $(PGDATABASE) || echo 'Database did not exist' - createdb $(PSQL_OPT) $(PGDATABASE) - @echo "BEGIN;" > /tmp/test_convenience.sql - @echo "CREATE TABLE IF NOT EXISTS my_table_without_pk (id INT, name TEXT);" >> /tmp/test_convenience.sql - @echo "CREATE EXTENSION IF NOT EXISTS pglinter;" >> /tmp/test_convenience.sql - @echo "SELECT pglinter.check_base();" >> /tmp/test_convenience.sql - @echo "SELECT pglinter.check_cluster();" >> /tmp/test_convenience.sql - @echo "SELECT pglinter.check_table();" >> /tmp/test_convenience.sql - @echo "SELECT pglinter.check_schema();" >> /tmp/test_convenience.sql - @echo "SELECT pglinter.check_all();" >> /tmp/test_convenience.sql - @echo "ROLLBACK;" >> /tmp/test_convenience.sql - psql $(PSQL_OPT) $(PGDATABASE) -f /tmp/test_convenience.sql - @rm -f /tmp/test_convenience.sql +# With PGXS: the postgres instance is created on-the-fly to run the test. +# With PGRX: the postgres instance is created previously by `cargo run`. This +# means we have some extra tasks to prepare the instance # PGXS-style installcheck using pg_regress installcheck: stop start @@ -170,6 +145,41 @@ run: psql: psql --host localhost --port 288$(PG_MAJOR_VERSION) + +## +## Coverage +## + +COVERAGE_DIR?=target/$(TARGET)/coverage + +clean_profiles: + rm -fr *.profraw + +coverage: clean_profiles coverage_test covergage_report + +coverage_test: + export RUSTFLAGS=-Cinstrument-coverage \ + export LLVM_PROFILE_FILE=$(TARGET)-%p-%m.profraw \ + && $(PGRX) test $(PGVER) $(RELEASE_OPT) --verbose + +coverage_report: + mkdir -p $(COVERAGE_DIR) + export LLVM_PROFILE_FILE=$(TARGET)-%p-%m.profraw \ + && grcov . \ + --binary-path target/$(TARGET) \ + --source-dir . \ + --output-path $(COVERAGE_DIR)\ + --keep-only 'src/*' \ + --llvm \ + --ignore-not-existing \ + --output-types html,cobertura + # Terse output + grep '

Lines

' -A2 $(COVERAGE_DIR)/html/index.html \ + | tail -n 1 \ + | xargs \ + | sed 's,%.*,,' \ + | sed 's/.*>/Coverage: /' + ## ## CLEAN ## @@ -179,115 +189,88 @@ ifdef EXTRA_CLEAN rm -rf $(EXTRA_CLEAN) endif + + +## +## All targets below are not part of the PGXS Makefile +## + ## -## Help +## P A C K A G E S ## -help: - @echo "Available targets:" - @echo " all - Build the extension" - @echo " extension - Build the extension package" - @echo " install - Install the extension to PostgreSQL" - @echo " test-b001 - Run the b001 test specifically" - @echo " test-all - Run all tests" - @echo " test-prompt-b001 - Run b001 test with prompt output (no file)" - @echo " test-convenience - Test convenience functions (check_base, etc.)" - @echo " installcheck - Run tests using pg_regress (all tests)" - @echo " installcheck REGRESS=testname - Run specific test with pg_regress" - @echo " Example: make installcheck REGRESS=b001" - @echo " Example: make installcheck REGRESS=\"b001 b001_prompt\"" - @echo " start - Start PostgreSQL test instance" - @echo " stop - Stop PostgreSQL test instance" - @echo " psql - Connect to test database" - @echo " clean - Clean build artifacts" - @echo " lint - Run Rust linting with clippy" - @echo " fmt - Format Rust code with cargo fmt" - @echo " fmt-check - Check if Rust code is properly formatted" - @echo " lint-docs - Lint markdown documentation files" - @echo " lint-docs-fix - Lint and automatically fix markdown files" - @echo " spell-check - Check spelling in documentation" - @echo " precommit - Run all pre-commit checks (fmt, lint, docs, tests)" - @echo " precommit-fast - Run fast pre-commit checks (skip tests)" - @echo " install-precommit-hook - Install git pre-commit hook" - @echo " help - Show this help message" - -.PHONY: all extension install test-all installcheck start stop run psql clean help test-% test-prompt-% test-convenience lint fmt fmt-check lint-docs lint-docs-fix spell-check audit precommit precommit-fast install-precommit-hook +# The packages are built from the $(TARGET_DIR) folder. +# So the $(PG_PKGLIBDIR) and $(PG_SHAREDIR) are relative to that folder +rpm deb: package + export PG_PKGLIBDIR=".$(PG_PKGLIBDIR)" && \ + export PG_SHAREDIR=".$(PG_SHAREDIR)" && \ + export PG_MAJOR_VERSION="$(PG_MAJOR_VERSION)" && \ + export ANON_MINOR_VERSION="$(ANON_MINOR_VERSION)" && \ + envsubst < nfpm.template.yaml > $(TARGET_DIR)/nfpm.yaml + cd $(TARGET_DIR) && nfpm package --packager $@ +# The `package` command needs pg_config from the target version +# https://github.com/pgcentralfoundation/pgrx/issues/288 + +package: + $(PGRX) package --pg-config $(PG_CONFIG) + ## -## L I N T & P R E C O M M I T +## D O C K E R +## + +DOCKER_TAG?=latest +DOCKER_IMAGE?=ghcr.io/pmpetit/postgresql_pglinter:$(DOCKER_TAG) + +ifneq ($(DOCKER_PG_MAJOR_VERSION),) +DOCKER_BUILD_ARG := --build-arg DOCKER_PG_MAJOR_VERSION=$(DOCKER_PG_MAJOR_VERSION) +endif + +PGRX_IMAGE?=$(DOCKER_IMAGE):pgrx +PGRX_BUILD_ARGS?= + +docker_image: docker/Dockerfile #: build the docker image + docker build --tag $(DOCKER_IMAGE) . --file $^ $(DOCKER_BUILD_ARG) + +pgrx_image: docker/pgrx/Dockerfile + docker build --tag $(PGRX_IMAGE) . --file $^ $(PGRX_BUILD_ARGS) + +docker_push: #: push the docker image to the registry + docker push $(DOCKER_IMAGE) + +pgrx_push: + docker push $(PGRX_IMAGE) + +docker_bash: #: enter the docker image (useful for testing) + docker exec -it docker-PostgreSQL-1 bash + +pgrx_bash: + docker run --rm --interactive --tty --volume `pwd`:/pgrx $(PGRX_IMAGE) + +## +## L I N T ## lint: - cargo clippy --no-default-features --features pg13 --release - -# Format Rust code -fmt: - cargo fmt - -# Check if code is formatted -fmt-check: - cargo fmt --check - -# Lint markdown files -lint-docs: - @if command -v rumdl > /dev/null; then \ - echo "Linting markdown files..."; \ - rumdl check --config .rumdl.toml docs/**/*.md *.md || true; \ - echo ""; \ - echo "πŸ’‘ Tip: Run 'make lint-docs-fix' to automatically fix many issues"; \ - else \ - echo "rumdl not found, skipping markdown lint"; \ - fi - -# Lint and automatically fix markdown files -lint-docs-fix: - @if command -v rumdl > /dev/null; then \ - echo "Linting and fixing markdown files..."; \ - rumdl check --config .rumdl.toml --fix docs/**/*.md *.md; \ - echo "βœ… Auto-fixable markdown issues have been resolved!"; \ - else \ - echo "rumdl not found, skipping markdown lint and fix"; \ - fi - -# Spell check documentation -spell-check: - @if command -v aspell > /dev/null; then \ - echo "Checking spelling in documentation..."; \ - find docs/ -name "*.md" -exec aspell --mode=markdown --personal=./.aspell.en.pws list < {} \; | sort -u | head -20; \ - else \ - echo "aspell not found, skipping spell check"; \ - fi - -# Install git pre-commit hook -install-precommit-hook: - @echo "Installing pre-commit hook..." - @cp pre-commit-hook.sh .git/hooks/pre-commit - @chmod +x .git/hooks/pre-commit - @echo "βœ… Pre-commit hook installed successfully!" - @echo " Now 'git commit' will automatically run pre-commit checks." - -# Pre-commit hook that runs all checks -precommit: fmt-check lint lint-docs test - @echo "" - @echo "πŸŽ‰ Pre-commit checks completed successfully!" - @echo "" - @echo "Summary of checks performed:" - @echo " βœ… Rust code formatting (cargo fmt --check)" - @echo " βœ… Rust code linting (cargo clippy)" - @echo " βœ… Markdown documentation linting" - @echo " βœ… Unit tests (cargo pgrx test)" - @echo "" - @echo "Ready to commit! πŸš€" - -# Fast pre-commit that skips tests -precommit-fast: fmt-check lint lint-docs - @echo "" - @echo "⚑ Fast pre-commit checks completed!" - @echo "" - @echo "Summary of checks performed:" - @echo " βœ… Rust code formatting (cargo fmt --check)" - @echo " βœ… Rust code linting (cargo clippy)" - @echo " βœ… Markdown documentation linting" - @echo "" - @echo "Note: Skipped tests for speed. Run 'make test' before pushing." + cargo clippy --release + +## +## P R E - C O M M I T +## + +# Fast pre-commit checks (used by git hooks) +precommit-fast: lint + @echo "βœ… Fast pre-commit checks completed" + +# Full pre-commit checks (more comprehensive) +precommit: lint test + @echo "βœ… Full pre-commit checks completed" + +# Check formatting without fixing +format-check: + cargo fmt --all -- --check + +# Fix formatting +format: + cargo fmt --all diff --git a/PUSH_TO_GHCR.md b/PUSH_TO_GHCR.md new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index 36b3068..6844fb5 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,24 @@ This is a conversion of the original Python [pglinter](https://github.com/decath In recent years, DBAs were more involved with the database engine itselfβ€”creating instances, configuring backups, and monitoring systemsβ€”while also overseeing developers' activities. Today, in the DBRE world where databases are cloud-managed, developers and operations teams often work independently, without a dedicated DBA. So databases objects lives their own life, created by persons that do their best. It can be usefull to be able to detect some wonrg desing creation (for example foreign keys created accross differents schemas...). That's what pglinter was created for. -pglinter is a PostgreSQL database linter that analyzes your database for potential issues, performance problems, and best practice violations. This Rust implementation provides: +pglinter is a PostgreSQL linter that analyzes your database for potential issues, performance problems, and best practice violations. This Rust implementation provides: - **Better Performance**: Native Rust performance vs Python - **Deep Integration**: Runs directly inside PostgreSQL using pgrx - **SARIF Output**: Industry-standard Static Analysis Results Interchange Format - **Extensible Rules**: Easy to add new rules using Rust traits +## PostgreSQL Compatibility + +This extension is built with **pgrx 0.16.0** and supports: + +- PostgreSQL 13 +- PostgreSQL 14 +- PostgreSQL 15 +- PostgreSQL 16 +- PostgreSQL 17 +- PostgreSQL 18beta2 ✨ (latest with pgrx 0.16.0) + ## Architecture ### Rule Categories @@ -32,12 +43,29 @@ pglinter is a PostgreSQL database linter that analyzes your database for potenti ## Installation +### Requirements + +- Rust 1.88.0+ (required for pgrx 0.16.0) +- PostgreSQL 13-18 development packages +- cargo-pgrx 0.16.0 + +### Build and Install + ```bash -# Build the extension +# Install cargo-pgrx if not already installed +cargo install --locked cargo-pgrx + +# Initialize pgrx (one time setup) +cargo pgrx init + +# Build the extension for your PostgreSQL version cargo pgrx package -# Install (requires PostgreSQL dev packages) -sudo cargo pgrx install +# Install using the Makefile (handles both system and pgrx-managed PostgreSQL) +sudo make install + +# Or install manually for a specific PostgreSQL version +sudo PGVER=pg16 make install # Load in your database psql -d your_database -c "CREATE EXTENSION pglinter;" diff --git a/build_and_push_to_ghcr.sh b/build_and_push_to_ghcr.sh new file mode 100755 index 0000000..3a26788 --- /dev/null +++ b/build_and_push_to_ghcr.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Script to build and push PostgreSQL Anonymizer container to GitHub Container Registry +# Usage: ./build_and_push_to_ghcr.sh + +set -e # Exit on any error + +# Configuration +GITHUB_USER="pmpetit" +IMAGE_NAME="postgresql_pglinter" +TAG="pgrx" +FULL_IMAGE_NAME="ghcr.io/${GITHUB_USER}/${IMAGE_NAME}:${TAG}" + +echo "πŸ—οΈ Building and pushing PostgreSQL pglinter container to GHCR..." +echo "πŸ“¦ Image: ${FULL_IMAGE_NAME}" +echo "" + +# Check if Docker is running +if ! docker info &> /dev/null; then + echo "❌ Error: Docker is not running." + echo "🐳 Please start Docker and try again." + exit 1 +fi + +# Check if logged in to GHCR +echo "πŸ” Checking GitHub Container Registry authentication..." +if ! docker pull ghcr.io/hello-world &> /dev/null; then + echo "❌ Not authenticated with GHCR. Please login first:" + echo "πŸ’‘ Run: echo 'YOUR_GITHUB_TOKEN' | docker login ghcr.io -u ${GITHUB_USER} --password-stdin" + echo "πŸ“‹ Your token needs 'write:packages' and 'read:packages' scopes" + exit 1 +fi + +# Build the container +echo "πŸ—οΈ Building Docker image..." +if [ -f "docker/Dockerfile" ]; then + # Build from docker subdirectory if it exists + docker build -t "${FULL_IMAGE_NAME}" -f docker/Dockerfile . +elif [ -f "Dockerfile" ]; then + # Build from root Dockerfile + docker build -t "${FULL_IMAGE_NAME}" . +else + echo "❌ Error: No Dockerfile found in current directory or docker/ subdirectory." + exit 1 +fi + +# Push to GHCR +echo "πŸ“€ Pushing image to GitHub Container Registry..." +docker push "${FULL_IMAGE_NAME}" + +echo "" +echo "βœ… Successfully built and pushed image to GHCR!" +echo "🐳 Image: ${FULL_IMAGE_NAME}" +echo "" +echo "πŸ“‹ To use this image in GitHub Actions, update your workflow:" +echo " container: ${FULL_IMAGE_NAME}" +echo "" +echo "πŸ”§ To update your current workflows:" +echo " sed -i 's|ghcr.io/pmpetit/postgresql_anonymizer|ghcr.io/pmpetit/postgresql_pglinter|g' .github/workflows/*.yml" diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..4374970 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,25 @@ + +# This instructions must be declared before any FROM +# You can override it and build your own image based another version +# like this: `PG_MAJOR_VERSION=13 make docker_image` + +FROM postgres:$DOCKER_PG_MAJOR_VERSION + +# An ARG declared before a FROM is outside of a build stage, so it can’t be +# used in any instruction after a FROM. We need to declare it again. +ARG DOCKER_PG_MAJOR_VERSION=17 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Install anon extension +ARG PGLINTER_VERSION=latest +RUN wget https://ghcr.io/api/v4/projects/7709206/packages/generic/deb/$PGLINTER_VERSION/postgresql_pglinter_pg$DOCKER_PG_MAJOR_VERSION-$PGLINTER_VERSION.amd64.deb \ + && dpkg -i postgresql_pglinter_pg$DOCKER_PG_MAJOR_VERSION-$PGLINTER_VERSION.amd64.deb + +# init script +RUN mkdir -p /docker-entrypoint-initdb.d +COPY ./docker/init_pglinter.sh /docker-entrypoint-initdb.d/init_pglinter.sh diff --git a/docker/init_pglinter.sh b/docker/init_pglinter.sh new file mode 100755 index 0000000..44a8c26 --- /dev/null +++ b/docker/init_pglinter.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +# Perform all actions as $POSTGRES_USER +export PGUSER="$POSTGRES_USER" + +echo "Creating extension inside template1 and postgres databases" +SQL="CREATE EXTENSION IF NOT EXISTS pglinter CASCADE;" +psql --dbname="template1" -c "$SQL" +psql --dbname="postgres" -c "$SQL" diff --git a/docker/pgrx/Dockerfile b/docker/pgrx/Dockerfile new file mode 100644 index 0000000..b0fd790 --- /dev/null +++ b/docker/pgrx/Dockerfile @@ -0,0 +1,76 @@ + +# We can't use the official rust docker image because we need to build the +# extension against the oldest glibc version available. +# So we use Rocky8 as the base image, which ships glibc 2.28 +# Our hope is that an extension built against an old version of glibc will +# continue to work against newer versions of glibc. +#FROM rust:1 + +FROM rockylinux:8 AS rhel8_devtools + +ARG PGRX_VERSION=0.14.1 + +COPY docker/pgrx/goreleaser.repo /etc/yum.repos.d/goreleaser.repo + +# https://github.com/pgcentralfoundation/pgrx?tab=readme-ov-file#system-requirements +RUN dnf install -y 'dnf-command(config-manager)' \ + && dnf config-manager -y --set-enabled powertools \ + && dnf groupinstall -y 'Development Tools' \ + && dnf install -y epel-release \ + && dnf install -y \ + cmake \ + git \ + clang \ + nfpm \ + bison-devel \ + libicu-devel \ + readline-devel \ + zlib-devel \ + openssl-devel \ + ccache \ + wget \ + && dnf clean all \ + && rm -rf /var/cache/yum + + +FROM rhel8_devtools AS rhel8_rust + +# Create the postgres user with the given uid/gid +# If you're not using Docker Desktop and your UID / GID is not 1000 then +# you'll get permission errors with volumes. +# +# You can fix that by rebuilding the image locally with: +# +# docker compose build --build-arg UID=`id -u` --build-arg GID=`id- g` +# + +ARG UID=1000 +ARG GID=1000 +RUN groupadd -g "${GID}" postgres \ + && useradd --create-home --no-log-init -u "${UID}" -g "${GID}" postgres + +USER postgres +ENV USER=postgres + +# Install Rust + +RUN wget https://sh.rustup.rs -O /tmp/install-rust.sh \ + && chmod +x /tmp/install-rust.sh \ + && /tmp/install-rust.sh -y + +ENV PATH="/home/postgres/.cargo/bin/:${PATH}" + + +FROM rhel8_rust AS rhel8_pgrx + +RUN rustup self update \ + && rustup component add clippy llvm-tools-preview \ + && cargo install grcov \ + && cargo install --locked --version "${PGRX_VERSION}" cargo-pgrx \ + && cargo pgrx init + +ENV CARGO_HOME=~postgres/.cargo/ + + +WORKDIR /pgrx +VOLUME /pgrx diff --git a/docker/pgrx/goreleaser.repo b/docker/pgrx/goreleaser.repo new file mode 100644 index 0000000..a5848bc --- /dev/null +++ b/docker/pgrx/goreleaser.repo @@ -0,0 +1,5 @@ +[goreleaser] +name=GoReleaser +baseurl=https://repo.goreleaser.com/yum/ +enabled=1 +gpgcheck=0 diff --git a/docker/pgrx/rust.Dockerfile b/docker/pgrx/rust.Dockerfile new file mode 100644 index 0000000..19fa674 --- /dev/null +++ b/docker/pgrx/rust.Dockerfile @@ -0,0 +1,40 @@ +FROM rust:1 + +ARG UID=1000 +ARG GID=1000 +ARG PGRX_VERSION=0.12.4 + +# Create the postgres user with the given uid/gid +# If you're not using Docker Desktop and your UID / GID is not 1000 then +# you'll get permission errors with volumes. +# +# You can fix that by rebuilding the image locally with: +# +# docker compose build --build-arg UID=`id -u` --build-arg GID=`id- g` +# +RUN groupadd -g "${GID}" postgres \ + && useradd --create-home --no-log-init -u "${UID}" -g "${GID}" postgres + +RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' \ + | tee /etc/apt/sources.list.d/goreleaser.list + +RUN apt-get update && apt-get install -y \ + bison \ + flex \ + gettext-base \ + libclang-dev \ + nfpm \ + postgresql-server-dev-all + +USER postgres + +ENV USER=postgres + +ENV PATH="${PATH}:/usr/local/cargo/bin/:~postgres/.cargo/bin" + +RUN rustup component add clippy && \ + cargo install --locked --version "${PGRX_VERSION}" cargo-pgrx && \ + cargo pgrx init + +WORKDIR /pgrx +VOLUME /pgrx diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..26b6534 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +nfpm = "latest" diff --git a/nfpm.template.yaml b/nfpm.template.yaml new file mode 100644 index 0000000..9cf309e --- /dev/null +++ b/nfpm.template.yaml @@ -0,0 +1,37 @@ +--- +name: "postgresql_pglinter${PG_MAJOR_VERSION}" +arch: "amd64" +platform: "linux" +version: "v${ANON_MINOR_VERSION}" +section: "default" +priority: "extra" +maintainer: "pmpetit" +description: | + pglinter analyzes database for best practice in PostgreSQL ${PG_MAJOR_VERSION} +vendor: "pmpetit" +homepage: "https://github.com/pmpetit/pglinter" +license: "PostgreSQL" +depends: + - postgresql-${PG_MAJOR_VERSION} +contents: + - src: ${PG_PKGLIBDIR}/pglinter.so + dst: /usr/lib/postgresql/${PG_MAJOR_VERSION}/lib/pglinter.so + - src: ${PG_SHAREDIR}/extension/ + dst: /usr/share/postgresql/${PG_MAJOR_VERSION}/extension/ + type: tree + +overrides: + rpm: + # The postgres server package is named `postgresql-server` on the RHEL repo + # and it is named `postgresql16-server` in the PGDG repo. + # With this `depends` clause we're making sure that postgres itself is + # installed from the PGDG repo.. + depends: + - postgresql${PG_MAJOR_VERSION}-server + # These locations are based on the PGDG packages, not the RedHat ones + contents: + - src: ${PG_PKGLIBDIR}/pglinter.so + dst: /usr/pgsql-${PG_MAJOR_VERSION}/lib/pglinter.so + - src: ${PG_SHAREDIR}/extension/ + dst: /usr/pgsql-${PG_MAJOR_VERSION}/share/extension/ + type: tree