diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..9a72ab806 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,114 @@ +name: CI + +on: + push: + branches: [ main, develop, "release/*", "feature/*" ] + pull_request: + branches: [ main, develop ] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x, 20.x, 22.x] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Lint check + run: | + npm run lint --workspace=packages/core + npm run lint --workspace=packages/prisma-integration + + - name: TypeScript compilation check + run: | + npm run build --workspace=packages/core + npm run build --workspace=packages/prisma-integration + + - name: Run unit tests (excluding Docker-dependent tests) + run: | + npm run test --workspace=packages/core + npm run test --workspace=packages/prisma-integration + + - name: Run coverage tests + run: | + npm run coverage --workspace=packages/core + npm run coverage --workspace=packages/prisma-integration + if: matrix.node-version == '20.x' + continue-on-error: true + + - name: Upload coverage reports + uses: codecov/codecov-action@v4 + if: matrix.node-version == '20.x' + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./packages/*/coverage/lcov.info + fail_ci_if_error: false + verbose: true + continue-on-error: true + + package-tests: + runs-on: ubuntu-latest + + strategy: + matrix: + package: [core, prisma-integration] + node-version: [20.x] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build ${{ matrix.package }} package + run: npm run build --workspace=packages/${{ matrix.package }} + + - name: Test ${{ matrix.package }} package + run: npm run test --workspace=packages/${{ matrix.package }} + + - name: Lint ${{ matrix.package }} package + run: npm run lint --workspace=packages/${{ matrix.package }} + + benchmark: + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build core package + run: npm run build:core + + - name: Run benchmarks + run: npm run benchmark --workspace=packages/core + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 000000000..ab6aca253 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,69 @@ +name: Integration Tests + +on: + workflow_dispatch: # Manual trigger only for now + # Temporarily disabled for PR testing + # push: + # branches: [ main, develop ] + # pull_request: + # branches: [ main, develop ] + +jobs: + integration-tests: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: demo_user + POSTGRES_PASSWORD: demo_password + POSTGRES_DB: prisma_comparison_demo + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build packages + run: npm run build + + - name: Setup integration test environment + working-directory: ./examples/prisma-comparison-demo + env: + DATABASE_URL: postgresql://demo_user:demo_password@localhost:5432/prisma_comparison_demo + run: | + npm install + npx prisma generate + npx prisma db push + npx prisma db seed + + - name: Run integration tests + working-directory: ./examples/prisma-comparison-demo + env: + DATABASE_URL: postgresql://demo_user:demo_password@localhost:5432/prisma_comparison_demo + run: | + npm test + + - name: Upload test reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: integration-test-reports + path: ./examples/prisma-comparison-demo/reports/ + retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml new file mode 100644 index 000000000..8c41fb666 --- /dev/null +++ b/.github/workflows/pr-check.yml @@ -0,0 +1,60 @@ +name: PR Check + +on: + pull_request: + branches: [ main, develop ] + +jobs: + quick-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Lint check + run: npm run lint + + - name: TypeScript compilation check + run: npm run build + + - name: Run unit tests (core) + run: npm run test --workspace=packages/core + + - name: Run unit tests (prisma-integration) + run: npm run test --workspace=packages/prisma-integration + + validate-packages: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Check package.json validity + run: | + npm run build:core + cd packages/core && npm pack --dry-run + + - name: Check prisma-integration package + run: | + npm run build --workspace=packages/prisma-integration + cd packages/prisma-integration && npm pack --dry-run \ No newline at end of file diff --git a/docs/usage-guides/cte-management-api-usage-guide.md b/docs/usage-guides/cte-management-api-usage-guide.md new file mode 100644 index 000000000..6a27bc803 --- /dev/null +++ b/docs/usage-guides/cte-management-api-usage-guide.md @@ -0,0 +1,671 @@ +# CTE Management API Usage Guide + +This guide provides comprehensive instructions for using the CTE Management API in rawsql-ts. The API enables programmatic manipulation of Common Table Expressions (CTEs), providing powerful capabilities for building complex SQL queries dynamically. + +## Overview + +The CTE Management API allows you to: +- Add, remove, and replace CTEs programmatically +- Query CTE existence and retrieve CTE names +- Control PostgreSQL-specific optimization with MATERIALIZED hints +- Build complex multi-step data transformation pipelines +- Handle errors gracefully with custom error types + +## API Reference + +### Core Methods + +#### `addCTE(name: string, query: SelectQuery, options?: CTEOptions): this` + +Adds a new CTE to the query. + +```typescript +const query = SelectQueryParser.parse('SELECT * FROM users').toSimpleQuery(); +const summaryQuery = SelectQueryParser.parse('SELECT COUNT(*) as total FROM orders'); + +// Add a basic CTE +query.addCTE('order_summary', summaryQuery.toSimpleQuery()); + +// Add with PostgreSQL materialization hint +query.addCTE('expensive_calc', complexQuery.toSimpleQuery(), { materialized: true }); +``` + +**Parameters:** +- `name`: CTE name (must be non-empty) +- `query`: The SELECT query for the CTE +- `options`: Optional configuration +- `materialized`: `true` (MATERIALIZED), `false` (NOT MATERIALIZED), or `null` (default) + +**Throws:** +- `InvalidCTENameError`: If name is empty or whitespace +- `DuplicateCTEError`: If CTE with same name already exists + +#### `removeCTE(name: string): this` + +Removes an existing CTE from the query. + +```typescript +query.removeCTE('order_summary'); +``` + +**Throws:** +- `CTENotFoundError`: If CTE doesn't exist + +#### `hasCTE(name: string): boolean` + +Checks if a CTE with the given name exists. + +```typescript +if (query.hasCTE('order_summary')) { + console.log('CTE exists'); +} +``` + +#### `getCTENames(): string[]` + +Returns an array of all CTE names in the query. + +```typescript +const cteNames = query.getCTENames(); +// Returns: ['order_summary', 'user_stats', 'product_data'] +``` + +#### `replaceCTE(name: string, query: SelectQuery, options?: CTEOptions): this` + +Replaces an existing CTE with a new query. + +```typescript +const updatedQuery = SelectQueryParser.parse('SELECT COUNT(*) as total, AVG(amount) as avg FROM orders'); +query.replaceCTE('order_summary', updatedQuery.toSimpleQuery()); +``` + +**Throws:** +- `CTENotFoundError`: If CTE doesn't exist + +### Converting Query Types + +The CTE Management API is available on `SimpleSelectQuery`. Use `toSimpleQuery()` to convert other query types: + +```typescript +// From parser result +const parsed = SelectQueryParser.parse('SELECT * FROM users'); +const query = parsed.toSimpleQuery(); + +// From BinarySelectQuery (UNION, INTERSECT, EXCEPT) +const unionQuery = SelectQueryParser.parse('SELECT id FROM users UNION SELECT id FROM customers'); +const simpleQuery = unionQuery.toSimpleQuery(); + +// From ValuesQuery +const valuesQuery = SelectQueryParser.parse('VALUES (1, 2), (3, 4)'); +const simpleQuery = valuesQuery.toSimpleQuery(); +``` + +## PostgreSQL MATERIALIZED Optimization + +PostgreSQL 12+ supports optimization hints for CTEs. Use the `materialized` option to control query execution: + +### When to Use MATERIALIZED + +**Force materialization (`materialized: true`):** +- Expensive computations used multiple times +- Queries with side effects +- When you need consistent results across references + +```typescript +const expensiveCalc = SelectQueryParser.parse(` + SELECT user_id, + complex_aggregation(data) as result + FROM large_table + WHERE expensive_condition(data) +`); + +query.addCTE('cached_results', expensiveCalc.toSimpleQuery(), { + materialized: true +}); +``` + +**Prevent materialization (`materialized: false`):** +- Simple filters or projections +- Queries used only once +- When you want the optimizer to inline the CTE + +```typescript +const simpleFilter = SelectQueryParser.parse(` + SELECT * FROM products WHERE category = 'electronics' +`); + +query.addCTE('electronics', simpleFilter.toSimpleQuery(), { + materialized: false +}); +``` + +**Let PostgreSQL decide (default):** +- Most common case +- Allows query planner to optimize + +```typescript +query.addCTE('standard_cte', someQuery.toSimpleQuery()); +// or explicitly: { materialized: null } +``` + +## Common Patterns + +### Building Data Pipelines + +Create multi-step data transformation pipelines: + +```typescript +import { SelectQueryParser, SqlFormatter } from 'rawsql-ts'; + +function buildSalesAnalysisPipeline(startDate: string) { + const pipeline = SelectQueryParser.parse('SELECT * FROM final_analysis').toSimpleQuery(); + + // Step 1: Extract raw data + const rawDataQuery = SelectQueryParser.parse(` + SELECT + sale_id, + customer_id, + product_id, + sale_date, + quantity, + unit_price, + quantity * unit_price as total_amount + FROM sales + WHERE sale_date >= '${startDate}' + `); + + pipeline.addCTE('raw_sales', rawDataQuery.toSimpleQuery(), { materialized: true }); + + // Step 2: Join with customer data + const enrichedQuery = SelectQueryParser.parse(` + SELECT + rs.*, + c.customer_name, + c.customer_segment, + c.region + FROM raw_sales rs + JOIN customers c ON rs.customer_id = c.customer_id + `); + + pipeline.addCTE('enriched_sales', enrichedQuery.toSimpleQuery()); + + // Step 3: Aggregate by segment and region + const aggregateQuery = SelectQueryParser.parse(` + SELECT + customer_segment, + region, + COUNT(DISTINCT customer_id) as customer_count, + COUNT(*) as transaction_count, + SUM(total_amount) as revenue, + AVG(total_amount) as avg_transaction_value + FROM enriched_sales + GROUP BY customer_segment, region + `); + + pipeline.addCTE('segment_analysis', aggregateQuery.toSimpleQuery()); + + // Final query uses all CTEs + const finalQuery = SelectQueryParser.parse(` + SELECT + sa.*, + sa.revenue / sa.customer_count as revenue_per_customer, + RANK() OVER (PARTITION BY customer_segment ORDER BY revenue DESC) as region_rank + FROM segment_analysis sa + WHERE customer_count >= 10 + ORDER BY customer_segment, revenue DESC + `); + + // Replace the placeholder with actual final query + pipeline.replaceCTE('final_analysis', finalQuery.toSimpleQuery()); + + return pipeline; +} + +// Use the pipeline +const analysisQuery = buildSalesAnalysisPipeline('2024-01-01'); +const formatter = new SqlFormatter(); +const { formattedSql, params } = formatter.format(analysisQuery); +``` + +### Recursive CTEs + +Build recursive queries for hierarchical data: + +```typescript +const hierarchyQuery = SelectQueryParser.parse('SELECT * FROM category_tree').toSimpleQuery(); + +// Anchor: top-level categories +const anchorQuery = SelectQueryParser.parse(` + SELECT + category_id, + category_name, + parent_id, + 0 as level, + ARRAY[category_id] as path + FROM categories + WHERE parent_id IS NULL +`); + +// Recursive part +const recursiveQuery = SelectQueryParser.parse(` + SELECT + c.category_id, + c.category_name, + c.parent_id, + ct.level + 1, + ct.path || c.category_id + FROM categories c + JOIN category_tree ct ON c.parent_id = ct.category_id +`); + +// Combine with UNION ALL for recursion +const recursiveCTE = anchorQuery.toUnion(recursiveQuery); +hierarchyQuery.addCTE('category_tree', recursiveCTE.toSimpleQuery()); + +// Use the recursive CTE +const treeQuery = SelectQueryParser.parse(` + SELECT + category_id, + REPEAT(' ', level) || category_name as indented_name, + path + FROM category_tree + ORDER BY path +`); + +hierarchyQuery.replaceCTE('category_tree', treeQuery.toSimpleQuery()); +``` + +### Dynamic CTE Generation + +Generate CTEs based on runtime conditions: + +```typescript +interface FilterConfig { + includeInactive?: boolean; + regions?: string[]; + minRevenue?: number; +} + +function buildDynamicReport(config: FilterConfig) { + const report = SelectQueryParser.parse('SELECT * FROM report_data').toSimpleQuery(); + + // Base customer CTE + let customerFilter = 'WHERE 1=1'; + if (!config.includeInactive) { + customerFilter += ' AND status = \'active\''; + } + if (config.regions && config.regions.length > 0) { + const regionList = config.regions.map(r => `'${r}'`).join(','); + customerFilter += ` AND region IN (${regionList})`; + } + + const customerQuery = SelectQueryParser.parse(` + SELECT customer_id, customer_name, region, status + FROM customers + ${customerFilter} + `); + + report.addCTE('filtered_customers', customerQuery.toSimpleQuery()); + + // Add revenue CTE if threshold specified + if (config.minRevenue) { + const revenueQuery = SelectQueryParser.parse(` + SELECT + c.customer_id, + c.customer_name, + SUM(o.total_amount) as total_revenue + FROM filtered_customers c + JOIN orders o ON c.customer_id = o.customer_id + GROUP BY c.customer_id, c.customer_name + HAVING SUM(o.total_amount) >= ${config.minRevenue} + `); + + report.addCTE('high_value_customers', revenueQuery.toSimpleQuery()); + + // Update final query to use revenue filter + const finalQuery = SelectQueryParser.parse(` + SELECT * FROM high_value_customers + ORDER BY total_revenue DESC + `); + report.replaceCTE('report_data', finalQuery.toSimpleQuery()); + } else { + // Use basic customer list + const finalQuery = SelectQueryParser.parse(` + SELECT * FROM filtered_customers + ORDER BY customer_name + `); + report.replaceCTE('report_data', finalQuery.toSimpleQuery()); + } + + return report; +} +``` + +## Error Handling + +The CTE Management API provides specific error types for different scenarios: + +### Error Types + +```typescript +import { + DuplicateCTEError, + CTENotFoundError, + InvalidCTENameError +} from 'rawsql-ts'; +``` + +### Handling Duplicate CTEs + +```typescript +function safeCTEAdd(query: SimpleSelectQuery, name: string, cteQuery: SelectQuery) { + try { + query.addCTE(name, cteQuery.toSimpleQuery()); + console.log(`CTE '${name}' added successfully`); + } catch (error) { + if (error instanceof DuplicateCTEError) { + console.log(`CTE '${error.cteName}' already exists, replacing...`); + query.replaceCTE(name, cteQuery.toSimpleQuery()); + } else { + throw error; + } + } +} +``` + +### Handling Missing CTEs + +```typescript +function safeCTERemove(query: SimpleSelectQuery, name: string) { + try { + query.removeCTE(name); + console.log(`CTE '${name}' removed`); + } catch (error) { + if (error instanceof CTENotFoundError) { + console.log(`CTE '${error.cteName}' not found, skipping removal`); + } else { + throw error; + } + } +} +``` + +### Validation Helper + +```typescript +function validateCTEName(name: string): boolean { + try { + if (!name || name.trim() === '') { + throw new InvalidCTENameError(name, 'CTE name cannot be empty'); + } + // Additional validation rules + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + throw new InvalidCTENameError(name, 'CTE name must be a valid identifier'); + } + return true; + } catch (error) { + if (error instanceof InvalidCTENameError) { + console.error(`Invalid CTE name: ${error.message}`); + return false; + } + throw error; + } +} +``` + +## Performance Considerations + +### O(1) Name Lookups + +The CTE Management API uses an internal Set for name tracking, providing O(1) performance for: +- `hasCTE()` checks +- Duplicate detection in `addCTE()` +- Existence validation in `removeCTE()` and `replaceCTE()` + +This makes it efficient even with many CTEs: + +```typescript +const query = SelectQueryParser.parse('SELECT 1').toSimpleQuery(); + +// Add 100 CTEs +for (let i = 0; i < 100; i++) { + const cte = SelectQueryParser.parse(`SELECT ${i} as num`); + query.addCTE(`cte_${i}`, cte.toSimpleQuery()); +} + +// O(1) lookup performance +console.log(query.hasCTE('cte_50')); // Fast lookup +console.log(query.getCTENames().length); // Returns 100 +``` + +### Memory Efficiency + +CTEs are stored as references to SelectQuery objects. Consider memory usage when building large pipelines: + +```typescript +// Good: Reuse query objects when possible +const baseQuery = SelectQueryParser.parse('SELECT * FROM large_table'); +const filtered1 = baseQuery.toSimpleQuery(); +const filtered2 = baseQuery.toSimpleQuery(); + +// Less efficient: Parse same query multiple times +const query1 = SelectQueryParser.parse('SELECT * FROM large_table').toSimpleQuery(); +const query2 = SelectQueryParser.parse('SELECT * FROM large_table').toSimpleQuery(); +``` + +## Integration with Other rawsql-ts Features + +### With SqlFormatter + +Format queries with CTEs using SqlFormatter: + +```typescript +const formatter = new SqlFormatter({ + keywordCase: 'upper', + indentSize: 2 +}); + +const { formattedSql } = formatter.format(queryWithCTEs); +``` + +### With DynamicQueryBuilder + +Combine CTE management with dynamic query building: + +```typescript +// Start with CTEs +const baseQuery = SelectQueryParser.parse('SELECT * FROM enriched_data').toSimpleQuery(); +baseQuery.addCTE('user_segments', segmentQuery.toSimpleQuery()); +baseQuery.addCTE('product_categories', categoryQuery.toSimpleQuery()); + +// Apply dynamic filters +const builder = new DynamicQueryBuilder(); +const finalQuery = builder.buildQuery(baseQuery, { + filter: { segment: 'premium', category: 'electronics' }, + sort: { revenue: { desc: true } }, + paging: { page: 1, pageSize: 20 } +}); +``` + +### With PostgresJsonQueryBuilder + +Use CTEs to prepare data for JSON transformation: + +```typescript +const query = SelectQueryParser.parse('SELECT * FROM user_data').toSimpleQuery(); + +// Add CTE to denormalize data +const denormalizedQuery = SelectQueryParser.parse(` + SELECT + u.user_id, + u.user_name, + o.order_id, + o.order_date, + oi.product_name, + oi.quantity + FROM users u + LEFT JOIN orders o ON u.user_id = o.user_id + LEFT JOIN order_items oi ON o.order_id = oi.order_id +`); + +query.addCTE('denormalized', denormalizedQuery.toSimpleQuery()); + +// Transform to JSON +const jsonBuilder = new PostgresJsonQueryBuilder(); +const jsonQuery = jsonBuilder.buildJson(query, { + rootName: 'user', + rootEntity: { + id: 'user', + name: 'User', + columns: { id: 'user_id', name: 'user_name' } + }, + nestedEntities: [ + { + id: 'orders', + parentId: 'user', + propertyName: 'orders', + relationshipType: 'array', + columns: { orderId: 'order_id', orderDate: 'order_date' } + } + ] +}); +``` + +## Best Practices + +### 1. Use Meaningful CTE Names + +```typescript +// Good: Descriptive names +query.addCTE('active_users_with_recent_orders', ...); +query.addCTE('monthly_revenue_by_segment', ...); + +// Bad: Generic names +query.addCTE('cte1', ...); +query.addCTE('temp', ...); +``` + +### 2. Order CTEs Logically + +Build CTEs in dependency order for clarity: + +```typescript +// 1. Base data extraction +query.addCTE('raw_transactions', ...); + +// 2. Data enrichment +query.addCTE('enriched_transactions', ...); + +// 3. Aggregations +query.addCTE('daily_summaries', ...); + +// 4. Final calculations +query.addCTE('trend_analysis', ...); +``` + +### 3. Use Materialization Hints Wisely + +```typescript +// Expensive computation used multiple times: MATERIALIZED +const expensiveAggregation = parseExpensiveQuery(); +query.addCTE('cached_aggregates', expensiveAggregation, { materialized: true }); + +// Simple filter used once: NOT MATERIALIZED +const simpleFilter = parseSimpleFilter(); +query.addCTE('filtered_data', simpleFilter, { materialized: false }); + +// Let optimizer decide for moderate complexity +const standardQuery = parseStandardQuery(); +query.addCTE('processed_data', standardQuery); +``` + +### 4. Handle Errors Gracefully + +```typescript +function buildQuerySafely(ctes: Array<{ name: string; sql: string }>) { + const query = SelectQueryParser.parse('SELECT 1').toSimpleQuery(); + const added: string[] = []; + + for (const cte of ctes) { + try { + const cteQuery = SelectQueryParser.parse(cte.sql); + query.addCTE(cte.name, cteQuery.toSimpleQuery()); + added.push(cte.name); + } catch (error) { + console.error(`Failed to add CTE '${cte.name}':`, error.message); + // Rollback on error + for (const name of added) { + query.removeCTE(name); + } + throw error; + } + } + + return query; +} +``` + +### 5. Document Complex Pipelines + +```typescript +/** + * Builds a customer lifetime value (CLV) analysis pipeline + * + * CTEs created: + * - customer_transactions: All transactions with customer data + * - customer_metrics: Per-customer aggregations + * - clv_segments: CLV calculation and segmentation + * + * @param startDate Analysis start date + * @param endDate Analysis end date + * @returns Query with CLV analysis CTEs + */ +function buildCLVPipeline(startDate: string, endDate: string): SimpleSelectQuery { + // Implementation... +} +``` + +## Troubleshooting + +### Common Issues + +1. **"Cannot read property 'toSimpleQuery' of undefined"** + - Ensure SelectQueryParser.parse() succeeded + - Check SQL syntax is valid + +2. **"CTE 'x' already exists"** + - Use `hasCTE()` to check before adding + - Consider using `replaceCTE()` instead + +3. **"CTE 'x' not found"** + - Verify CTE was added successfully + - Check for typos in CTE names + +4. **Materialization hints not appearing in output** + - Ensure using PostgreSQL SQL formatter preset + - Verify formatter supports MATERIALIZED syntax + +### Debugging Tips + +```typescript +// Log CTE state during pipeline building +function debugCTEPipeline(query: SimpleSelectQuery, stage: string) { + console.log(`=== ${stage} ===`); + console.log('CTEs:', query.getCTENames()); + query.getCTENames().forEach(name => { + console.log(`- ${name}: exists = ${query.hasCTE(name)}`); + }); +} + +// Use during pipeline construction +const pipeline = SelectQueryParser.parse('SELECT 1').toSimpleQuery(); +debugCTEPipeline(pipeline, 'Initial state'); + +pipeline.addCTE('stage1', stage1Query.toSimpleQuery()); +debugCTEPipeline(pipeline, 'After stage 1'); + +pipeline.addCTE('stage2', stage2Query.toSimpleQuery()); +debugCTEPipeline(pipeline, 'After stage 2'); +``` + +This guide provides comprehensive coverage of the CTE Management API, enabling you to build complex, dynamic SQL queries with confidence. \ No newline at end of file diff --git a/docs/usage-guides/model-driven-json-mapping-usage-guide.md b/docs/usage-guides/model-driven-json-mapping-usage-guide.md index 644ce1ad3..2c043ccc9 100644 --- a/docs/usage-guides/model-driven-json-mapping-usage-guide.md +++ b/docs/usage-guides/model-driven-json-mapping-usage-guide.md @@ -24,11 +24,11 @@ Model-Driven JSON Mapping enables: "structure": { "todoId": "todo_id", "title": { - "from": "title", + "column": "title", "type": "string" }, "description": { - "from": "description", + "column": "description", "type": "string" }, "completed": "completed", @@ -40,11 +40,11 @@ Model-Driven JSON Mapping enables: "structure": { "userId": "user_id", "userName": { - "from": "user_name", + "column": "user_name", "type": "string" }, "email": { - "from": "email", + "column": "email", "type": "string" } } @@ -55,11 +55,11 @@ Model-Driven JSON Mapping enables: "structure": { "categoryId": "category_id", "categoryName": { - "from": "category_name", + "column": "category_name", "type": "string" }, "color": { - "from": "color", + "column": "color", "type": "string" } } @@ -70,17 +70,13 @@ Model-Driven JSON Mapping enables: "structure": { "commentId": "comment_id", "text": { - "from": "comment_text", + "column": "comment_text", "type": "string" }, "createdAt": "comment_created_at" } } - }, - "protectedStringFields": [ - "title", "description", "user_name", "email", - "category_name", "color", "comment_text" - ] + } } ``` @@ -92,7 +88,8 @@ Model-Driven JSON Mapping enables: |-------|------|----------|-------------| | `typeInfo` | object | Yes | TypeScript interface information | | `structure` | object | Yes | The main data structure mapping | -| `protectedStringFields` | array | No | List of database columns that should be protected with type: "string" | + +**Note:** The `protectedStringFields` array from earlier documentation is not used in the actual implementation. Type protection is automatically extracted from fields marked with `type: "string"`. ### TypeInfo Object @@ -103,7 +100,7 @@ Model-Driven JSON Mapping enables: ### Structure Mapping -The `structure` object maps TypeScript properties to database columns using two formats: +The `structure` object maps TypeScript properties to database columns using three formats: #### Simple String Mapping ```json @@ -112,7 +109,7 @@ The `structure` object maps TypeScript properties to database columns using two } ``` -#### Advanced Object Mapping +#### Advanced Object Mapping (Legacy) ```json { "fieldName": { @@ -122,6 +119,18 @@ The `structure` object maps TypeScript properties to database columns using two } ``` +#### Advanced Object Mapping (New) +```json +{ + "fieldName": { + "column": "database_column_name", + "type": "string" + } +} +``` + +**Note:** Both `from` and `column` formats are supported for backward compatibility. + ### Nested Object Mapping ```json { @@ -152,10 +161,13 @@ The `structure` object maps TypeScript properties to database columns using two | Field | Type | Required | Description | |-------|------|----------|-------------| -| `from` | string | Yes (for complex mappings) | Source database column or alias | -| `type` | string | No | Data type: "string", "number", "boolean", "object", "array" | +| `from` | string | Yes (for complex mappings, legacy) | Source database column or alias | +| `column` | string | Yes (for complex mappings, new) | Source database column | +| `type` | string | No | Data type: "string", "number", "boolean", "object", "array", "auto" | | `structure` | object | Yes (for objects/arrays) | Nested structure definition | +**Note:** Use either `from` OR `column` (not both). The `column` format is preferred for new mappings. + ## Advantages Over Legacy Formats ### 1. Intuitive Structure @@ -225,10 +237,7 @@ interface User { "type": "string" }, "createdAt": "created_at" - }, - "protectedStringFields": [ - "name", "email" - ] + } } ``` @@ -250,11 +259,11 @@ interface User { "from": "p", "structure": { "title": { - "from": "profile_title", + "column": "profile_title", "type": "string" }, "bio": { - "from": "profile_bio", + "column": "profile_bio", "type": "string" } } @@ -350,11 +359,11 @@ Use `type: "string"` for: { "structure": { "title": { - "from": "title", + "column": "title", "type": "string" }, "email": { - "from": "email", + "column": "email", "type": "string" }, "id": "user_id", // Numbers don't need type protection @@ -363,6 +372,10 @@ Use `type: "string"` for: } ``` +### Automatic Type Protection Extraction + +The implementation automatically extracts type protection information from fields marked with `type: "string"`. You don't need to specify a separate `protectedStringFields` array. + ## AI Generation Template Use this template to prompt AI tools for generating Model-Driven JSON Mapping files: @@ -451,13 +464,52 @@ Please generate the complete JSON mapping file following the Model-Driven JSON M ### 3. Type Safety - Always include `typeInfo` with correct interface name and import path - Use `type: "string"` for user-generated content -- Add `protectedStringFields` for security-sensitive columns +- Type protection is automatically extracted from field definitions ### 4. SQL Query Alignment - Ensure SQL column aliases match the mapping exactly - Use table aliases in SQL and reference them in `from` fields - Structure JOINs to support the nested object hierarchy +## API Usage + +The Model-Driven JSON Mapping is processed using the `convertModelDrivenMapping` function: + +```typescript +import { convertModelDrivenMapping, ModelDrivenJsonMapping } from 'rawsql-ts'; + +const modelMapping: ModelDrivenJsonMapping = { + typeInfo: { + interface: "UserProfile", + importPath: "src/contracts/user-profile.ts" + }, + structure: { + userId: "user_id", + name: { column: "name", type: "string" }, + email: { column: "email", type: "string" } + } +}; + +// Convert to legacy format for PostgresJsonQueryBuilder +const { jsonMapping, typeProtection } = convertModelDrivenMapping(modelMapping); + +// Use with PostgresJsonQueryBuilder +const jsonQuery = postgresBuilder.buildJson(query, jsonMapping); +``` + +### Validation + +The implementation includes validation to ensure mapping correctness: + +```typescript +import { validateModelDrivenMapping } from 'rawsql-ts'; + +const errors = validateModelDrivenMapping(modelMapping); +if (errors.length > 0) { + console.error('Validation errors:', errors); +} +``` + ## Migration from Legacy Formats The Model-Driven format is automatically detected and converted by the MappingFileProcessor. Legacy formats are still supported but deprecated. @@ -473,6 +525,7 @@ The system automatically detects format based on structure: - **Simplified Maintenance**: Easier to understand and modify - **Better Type Safety**: Enhanced type information and validation - **Future-Proof**: New features will prioritize model-driven format +- **Automatic Entity ID Generation**: No need to manually manage entity IDs ## Integration with Static Analysis diff --git a/package.json b/package.json index eba225ecd..0218653aa 100644 --- a/package.json +++ b/package.json @@ -1,60 +1,58 @@ -{ - "name": "rawsql-ts-workspace", - "description": "Monorepo workspace for rawsql-ts packages", - "private": true, - "workspaces": [ - "packages/*" - ], - "scripts": { - "test": "vitest run", - "test:watch": "vitest", - "test:ui": "vitest --ui", - "test:coverage": "vitest run --coverage", - "test:workspaces": "npm run test --workspaces", - "test:watch:workspaces": "npm run test:watch --workspaces", - "coverage:workspaces": "npm run coverage --workspaces", - "build": "npm run build --workspaces", - "build:core": "npm run build --workspace=packages/core", - "clean": "npm run clean --workspaces", - "lint": "npm run lint --workspaces", - "benchmark": "npm run benchmark --workspace=packages/core" - }, - "keywords": [ - "sql", - "sql-parser", - "sql-transformer", - "workspace", - "monorepo" - ], - "author": "msugiura", - "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/mk3008/rawsql-ts.git" - }, - "devDependencies": { - "@types/benchmark": "^2.1.5", - "@types/node": "^22.13.10", - "@typescript-eslint/eslint-plugin": "^8.26.1", - "@typescript-eslint/parser": "^8.26.1", - "benchmark": "^2.1.4", - "esbuild": "^0.25.5", - "eslint": "^9.22.0", - "eslint-config-prettier": "^10.1.1", - "eslint-plugin-filenames": "^1.3.2", - "eslint-plugin-prettier": "^5.2.3", - "microtime": "^3.1.1", - "node-sql-parser": "^5.3.8", - "sql-formatter": "^15.5.2", - "sql-parser-cst": "^0.33.1", - "ts-node": "^10.9.2", - "typescript": "^5.8.2", - "vitest": "^1.5.2" - }, - "optionalDependencies": { - "@rollup/rollup-linux-x64-gnu": "^4.44.0" - }, - "dependencies": { - "@rollup/rollup-win32-x64-msvc": "^4.44.0" - } -} +{ + "name": "rawsql-ts-workspace", + "description": "Monorepo workspace for rawsql-ts packages", + "private": true, + "workspaces": [ + "packages/*" + ], + "scripts": { + "test": "vitest run", + "test:watch": "vitest", + "test:ui": "vitest --ui", + "test:coverage": "vitest run --coverage", + "test:workspaces": "npm run test --workspaces", + "test:watch:workspaces": "npm run test:watch --workspaces", + "coverage:workspaces": "npm run coverage --workspaces", + "build": "npm run build --workspaces", + "build:core": "npm run build --workspace=packages/core", + "clean": "npm run clean --workspaces", + "lint": "npm run lint --workspaces", + "benchmark": "npm run benchmark --workspace=packages/core" + }, + "keywords": [ + "sql", + "sql-parser", + "sql-transformer", + "workspace", + "monorepo" + ], + "author": "msugiura", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/mk3008/rawsql-ts.git" + }, + "devDependencies": { + "@types/benchmark": "^2.1.5", + "@types/node": "^22.13.10", + "@typescript-eslint/eslint-plugin": "^8.26.1", + "@typescript-eslint/parser": "^8.26.1", + "benchmark": "^2.1.4", + "esbuild": "^0.25.5", + "eslint": "^9.22.0", + "eslint-config-prettier": "^10.1.1", + "eslint-plugin-filenames": "^1.3.2", + "eslint-plugin-prettier": "^5.2.3", + "microtime": "^3.1.1", + "node-sql-parser": "^5.3.8", + "sql-formatter": "^15.5.2", + "sql-parser-cst": "^0.33.1", + "ts-node": "^10.9.2", + "typescript": "^5.8.2", + "vitest": "^1.5.2" + }, + "optionalDependencies": { + "@rollup/rollup-linux-x64-gnu": "^4.44.0", + "@rollup/rollup-win32-x64-msvc": "^4.44.0" + } +} diff --git a/packages/core/README.md b/packages/core/README.md index 5c5cb0c2d..a392416ea 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -23,6 +23,7 @@ It is designed for extensibility and advanced SQL analysis, with initial focus o - High-speed SQL parsing and AST analysis (over 3x faster than major libraries) - Rich utilities for SQL structure transformation and analysis - Advanced SQL formatting capabilities, including multi-line formatting and customizable styles +- **Programmatic CTE Management** - Add, remove, and manipulate Common Table Expressions (CTEs) programmatically with support for PostgreSQL MATERIALIZED/NOT MATERIALIZED hints - **JSON-to-TypeScript type transformation** - Automatically convert JSON-ified SQL results (dates as strings, BigInts) back to proper TypeScript types with configurable transformation rules - **All-in-one dynamic query building with `DynamicQueryBuilder`** - combines filtering, sorting, pagination, and JSON serialization in a single, type-safe interface - Dynamic SQL parameter injection for building flexible search queries with `SqlParamInjector` (supports like, ilike, in, any, range queries, OR/AND conditions and more) @@ -121,6 +122,58 @@ console.log(params); --- +## CTE Management API + +The CTE Management API provides programmatic control over Common Table Expressions (CTEs), allowing you to build and manipulate WITH clauses dynamically. This is particularly useful for building complex analytical queries, data transformation pipelines, and hierarchical data structures. + +```typescript +import { SelectQueryParser, SqlFormatter } from 'rawsql-ts'; + +// Build a multi-step data pipeline with CTEs +const pipeline = SelectQueryParser.parse('SELECT * FROM final_results').toSimpleQuery(); + +// Step 1: Add raw data CTE with PostgreSQL MATERIALIZED hint +const salesData = SelectQueryParser.parse(` + SELECT customer_id, order_date, amount, product_category + FROM sales WHERE order_date >= '2024-01-01' +`); +pipeline.addCTE('raw_sales', salesData.toSimpleQuery(), { materialized: true }); + +// Step 2: Add aggregation CTE +const monthlyStats = SelectQueryParser.parse(` + SELECT customer_id, DATE_TRUNC('month', order_date) as month, + SUM(amount) as total, COUNT(*) as orders + FROM raw_sales + GROUP BY customer_id, DATE_TRUNC('month', order_date) +`); +pipeline.addCTE('monthly_stats', monthlyStats.toSimpleQuery()); + +// Manage CTEs programmatically +console.log(pipeline.getCTENames()); // ['raw_sales', 'monthly_stats'] +console.log(pipeline.hasCTE('raw_sales')); // true + +// Replace the final query to use the CTEs +const finalQuery = SelectQueryParser.parse(` + SELECT * FROM monthly_stats WHERE total > 10000 +`); +pipeline.replaceCTE('final_results', finalQuery.toSimpleQuery()); + +// Format and execute +const formatter = new SqlFormatter(); +const { formattedSql } = formatter.format(pipeline); +// Output: WITH "raw_sales" AS MATERIALIZED (...), "monthly_stats" AS (...) SELECT * FROM monthly_stats WHERE total > 10000 +``` + +Key features include: +- **Dynamic CTE Management**: Add, remove, replace, and query CTEs programmatically +- **PostgreSQL MATERIALIZED Support**: Control query optimization with MATERIALIZED/NOT MATERIALIZED hints +- **Type Safety**: Full TypeScript support with error handling for duplicate names and invalid operations +- **Performance Optimized**: O(1) CTE name lookups for efficient operations + +For comprehensive examples and advanced usage patterns, see the [CTE Management API Usage Guide](../../docs/usage-guides/cte-management-api-usage-guide.md). + +--- + ## SelectQueryParser & Query Types rawsql-ts provides robust parsers for `SELECT`, `INSERT`, and `UPDATE` statements, automatically handling SQL comments and providing detailed error messages. By converting SQL into a generic Abstract Syntax Tree (AST), it enables a wide variety of transformation processes. diff --git a/packages/core/package.json b/packages/core/package.json index 70a3d7557..4cd50930f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,61 +1,61 @@ -{ - "name": "rawsql-ts", - "version": "0.11.2-beta", - "description": "[beta]High-performance SQL parser and AST analyzer written in TypeScript. Provides fast parsing and advanced transformation capabilities.", - "main": "dist/src/index.js", - "module": "dist/esm/index.js", - "types": "dist/src/index.d.ts", - "browser": "dist/esm/index.min.js", - "homepage": "https://github.com/mk3008/rawsql-ts/tree/main/packages/core", - "scripts": { - "test": "vitest run", - "test:watch": "vitest", - "coverage": "vitest run --coverage", - "build": "tsc --build", - "build:browser": "tsc --project tsconfig.browser.json && npm run build:copy", - "build:copy": "node -e \"const fs = require('fs'); fs.copyFileSync('dist/esm/src/index.js', 'dist/esm/index.js'); fs.copyFileSync('dist/esm/src/index.js.map', 'dist/esm/index.js.map');\"", - "build:minify": "esbuild src/index.ts --bundle --minify --outfile=dist/index.min.js --format=cjs --sourcemap && esbuild src/index.ts --bundle --minify --outfile=dist/esm/index.min.js --format=esm --sourcemap", - "build:all": "npm run clean && npm run build && npm run build:browser && npm run build:minify", - "release": "npm run build:all && npm pack --dry-run && npm publish", - "clean": "tsc --build --clean && node -e \"const fs = require('fs'); if (fs.existsSync('dist')) fs.rmSync('dist', {recursive: true, force: true});\"", - "lint": "eslint . --ext .ts,.tsx --fix", - "benchmark": "ts-node benchmarks/parse-benchmark.ts" - }, - "keywords": [ - "sql", - "sql-parser", - "sql-transformer", - "ast", - "sql-ast", - "sql-formatter" - ], - "author": "msugiura", - "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/mk3008/rawsql-ts.git", - "directory": "packages/core" - }, - "devDependencies": { - "@types/benchmark": "^2.1.5", - "@types/node": "^22.13.10", - "@typescript-eslint/eslint-plugin": "^8.26.1", - "@typescript-eslint/parser": "^8.26.1", - "benchmark": "^2.1.4", - "esbuild": "^0.25.5", - "eslint": "^9.22.0", - "eslint-config-prettier": "^10.1.1", - "eslint-plugin-filenames": "^1.3.2", - "eslint-plugin-prettier": "^5.2.3", - "microtime": "^3.1.1", - "node-sql-parser": "^5.3.8", - "sql-formatter": "^15.5.2", - "sql-parser-cst": "^0.33.1", - "ts-node": "^10.9.2", - "typescript": "^5.8.2", - "vitest": "^1.5.2" - }, - "files": [ - "dist" - ] +{ + "name": "rawsql-ts", + "version": "0.11.3-beta", + "description": "[beta]High-performance SQL parser and AST analyzer written in TypeScript. Provides fast parsing and advanced transformation capabilities.", + "main": "dist/src/index.js", + "module": "dist/esm/index.js", + "types": "dist/src/index.d.ts", + "browser": "dist/esm/index.min.js", + "homepage": "https://github.com/mk3008/rawsql-ts/tree/main/packages/core", + "scripts": { + "test": "vitest run", + "test:watch": "vitest", + "coverage": "vitest run --coverage", + "build": "tsc --build", + "build:browser": "tsc --project tsconfig.browser.json && npm run build:copy", + "build:copy": "node -e \"const fs = require('fs'); fs.copyFileSync('dist/esm/src/index.js', 'dist/esm/index.js'); fs.copyFileSync('dist/esm/src/index.js.map', 'dist/esm/index.js.map');\"", + "build:minify": "esbuild src/index.ts --bundle --minify --outfile=dist/index.min.js --format=cjs --sourcemap && esbuild src/index.ts --bundle --minify --outfile=dist/esm/index.min.js --format=esm --sourcemap", + "build:all": "npm run clean && npm run build && npm run build:browser && npm run build:minify", + "release": "npm run build:all && npm pack --dry-run && npm publish", + "clean": "tsc --build --clean && node -e \"const fs = require('fs'); if (fs.existsSync('dist')) fs.rmSync('dist', {recursive: true, force: true});\"", + "lint": "eslint . --ext .ts,.tsx --fix", + "benchmark": "ts-node ../../benchmarks/parse-benchmark.ts" + }, + "keywords": [ + "sql", + "sql-parser", + "sql-transformer", + "ast", + "sql-ast", + "sql-formatter" + ], + "author": "msugiura", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/mk3008/rawsql-ts.git", + "directory": "packages/core" + }, + "devDependencies": { + "@types/benchmark": "^2.1.5", + "@types/node": "^22.13.10", + "@typescript-eslint/eslint-plugin": "^8.26.1", + "@typescript-eslint/parser": "^8.26.1", + "benchmark": "^2.1.4", + "esbuild": "^0.25.5", + "eslint": "^9.22.0", + "eslint-config-prettier": "^10.1.1", + "eslint-plugin-filenames": "^1.3.2", + "eslint-plugin-prettier": "^5.2.3", + "microtime": "^3.1.1", + "node-sql-parser": "^5.3.8", + "sql-formatter": "^15.5.2", + "sql-parser-cst": "^0.33.1", + "ts-node": "^10.9.2", + "typescript": "^5.8.2", + "vitest": "^1.5.2" + }, + "files": [ + "dist" + ] } \ No newline at end of file