diff --git a/application-quality-reference-deployment/Chart.yaml b/application-quality-reference-deployment/Chart.yaml index fa19b94..7f4a2d8 100644 --- a/application-quality-reference-deployment/Chart.yaml +++ b/application-quality-reference-deployment/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: application-quality-reference-deployment -description: A reference deployment of the Application Quality BB (including OpenSearch and SonarQube) +description: A reference deployment of the Application Quality BB (including Grafana) icon: https://upload.wikimedia.org/wikipedia/commons/3/3b/PlaceholderRoss.png @@ -26,12 +26,10 @@ version: 0.1.0 appVersion: "0.1.0" dependencies: -- name: opensearch - version: "2.27.0" - repository: "https://opensearch-project.github.io/helm-charts/" -- name: opensearch-dashboards - version: "2.25.0" - repository: "https://opensearch-project.github.io/helm-charts/" +- name: grafana + version: "9.2.4" + repository: "https://grafana.github.io/helm-charts" + condition: grafana.enabled - name: application-quality version: "0.1.0" repository: "file://../helm" \ No newline at end of file diff --git a/application-quality-reference-deployment/README.md b/application-quality-reference-deployment/README.md index 8ed45f6..53d20b9 100644 --- a/application-quality-reference-deployment/README.md +++ b/application-quality-reference-deployment/README.md @@ -1,4 +1,4 @@ # Helm chart for reference deployment -This is a deployment of the Application Quality BB, including an OpenSearch deployment. +This is a deployment of the Application Quality BB, including Grafana. This leads to a fully functional setup. diff --git a/application-quality-reference-deployment/grafana/dashboards/pipeline-execution.json b/application-quality-reference-deployment/grafana/dashboards/pipeline-execution.json new file mode 100644 index 0000000..0bb36bf --- /dev/null +++ b/application-quality-reference-deployment/grafana/dashboards/pipeline-execution.json @@ -0,0 +1,184 @@ +{ + "folderUid": "application-quality-service", + "overwrite": false, + "dashboard": { + "editable": true, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": true, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Pipeline Executions", + "tooltip": "Open the pipeline executions dashboard", + "type": "link", + "url": "/d/pipeline-executions/pipeline-executions" + } + ], + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Pipeline Execution\n\n* Pipeline **${pipeline_id}** (**${pipeline_name}**)\n* Execution ID: **${execution_id}**", + "mode": "markdown" + }, + "pluginVersion": "12.0.0", + "title": "", + "transparent": true, + "type": "text" + }, + { + "gridPos": { + "h": 6, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 3, + "libraryPanel": { + "name": "Pipeline Inputs", + "uid": "pipeline-inputs" + }, + "title": "Pipeline Inputs", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 6, + "libraryPanel": { + "name": "Pipeline Usage Report", + "uid": "pipeline-usage-report" + }, + "title": "Pipeline Usage Report", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "libraryPanel": { + "name": "Pipeline Execution Reports List", + "uid": "pipeline-executions-report-list" + }, + "title": "Pipeline Execution Reports", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 4, + "libraryPanel": { + "name": "Usage Report per Metric", + "uid": "usage-report-per-metric" + }, + "title": "Usage Report per Metric", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 5, + "libraryPanel": { + "name": "Usage Report per Tool", + "uid": "usage-report-per-tool" + }, + "title": "Usage Report per Tool", + "type": "library-panel-ref" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "hide": 2, + "label": "Pipeline ID", + "name": "pipeline_name", + "options": [], + "query": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "description": "", + "hide": 2, + "label": "Pipeline ID", + "name": "pipeline_id", + "options": [], + "query": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT id FROM executions_admin WHERE id IS NOT NULL ", + "hide": 2, + "label": "Execution ID", + "name": "execution_id", + "options": [], + "query": "SELECT id FROM executions_admin WHERE id IS NOT NULL ", + "refresh": 1, + "regex": "", + "sort": 4, + "type": "query" + } + ] + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "Pipeline Execution", + "uid": "pipeline-execution" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/dashboards/pipeline-executions.json b/application-quality-reference-deployment/grafana/dashboards/pipeline-executions.json new file mode 100644 index 0000000..1881eb5 --- /dev/null +++ b/application-quality-reference-deployment/grafana/dashboards/pipeline-executions.json @@ -0,0 +1,86 @@ +{ + "folderUid": "application-quality-service", + "overwrite": false, + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "links": [], + "panels": [ + { + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "libraryPanel": { + "name": "Pipeline Executions Daily Statistics", + "uid": "pipeline-executions-daily-statistics" + }, + "title": "Pipeline Executions Daily Statistics", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "libraryPanel": { + "name": "Pipeline Executions Timeline", + "uid": "pipeline-executions-timeline" + }, + "title": "Pipeline Executions Timeline", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "libraryPanel": { + "name": "Pipeline Executions List", + "uid": "pipeline-executions-list" + }, + "title": "Pipeline Executions", + "type": "library-panel-ref" + } + ], + "preload": false, + "refresh": "10s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Pipeline Executions", + "uid": "pipeline-executions" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/dashboards/pylint-dashboard.json b/application-quality-reference-deployment/grafana/dashboards/pylint-dashboard.json new file mode 100644 index 0000000..235dadd --- /dev/null +++ b/application-quality-reference-deployment/grafana/dashboards/pylint-dashboard.json @@ -0,0 +1,127 @@ +{ + "folderUid": "application-quality-service-reports", + "overwrite": false, + "dashboard": { + "editable": true, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": true, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Pipeline Execution ${execution_id}", + "tooltip": "Go to pipeline run ${execution_id} dashboard", + "type": "link", + "url": "/d/pipeline-execution/pipeline-execution?${__all_values}" + } + ], + "panels": [ + { + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "libraryPanel": { + "name": "Pipeline Execution Report Header", + "uid": "pipeline-execution-report-header" + }, + "title": "", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 1, + "libraryPanel": { + "name": "Pylint Report", + "uid": "pylint-report" + }, + "title": "Pylint Report", + "type": "library-panel-ref" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT id FROM reports_admin WHERE name = 'pylint'", + "hide": 2, + "description": "", + "label": "Report ID", + "name": "report_id", + "options": [], + "query": "SELECT id FROM reports_admin WHERE name = 'pylint'", + "refresh": 1, + "regex": "", + "sort": 4, + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT run_id FROM reports_admin WHERE id = $report_id", + "hide": 2, + "label": "Execution ID", + "name": "execution_id", + "options": [], + "query": "SELECT run_id FROM reports_admin WHERE id = $report_id", + "refresh": 1, + "regex": "", + "sort": 4, + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "hide": 2, + "name": "pipeline_id", + "options": [], + "query": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "hide": 2, + "name": "pipeline_name", + "options": [], + "query": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "Pylint Report", + "uid": "default-pylint-report" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/dashboards/ruff-dashboard.json b/application-quality-reference-deployment/grafana/dashboards/ruff-dashboard.json new file mode 100644 index 0000000..0a418da --- /dev/null +++ b/application-quality-reference-deployment/grafana/dashboards/ruff-dashboard.json @@ -0,0 +1,127 @@ +{ + "folderUid": "application-quality-service-reports", + "overwrite": false, + "dashboard": { + "editable": true, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": true, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Pipeline Execution ${execution_id}", + "tooltip": "Go to pipeline run ${execution_id} dashboard", + "type": "link", + "url": "/d/pipeline-execution/pipeline-execution?${__all_values}" + } + ], + "panels": [ + { + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "libraryPanel": { + "name": "Pipeline Execution Report Header", + "uid": "pipeline-execution-report-header" + }, + "title": "", + "type": "library-panel-ref" + }, + { + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "libraryPanel": { + "name": "Ruff Report", + "uid": "ruff-report" + }, + "title": "Ruff Report", + "type": "library-panel-ref" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT id FROM reports_admin WHERE name = 'ruff'", + "hide": 2, + "description": "", + "label": "Report ID", + "name": "report_id", + "options": [], + "query": "SELECT id FROM reports_admin WHERE name = 'ruff'", + "refresh": 1, + "regex": "", + "sort": 4, + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT run_id FROM reports_admin WHERE id = $report_id", + "hide": 2, + "label": "Execution ID", + "name": "execution_id", + "options": [], + "query": "SELECT run_id FROM reports_admin WHERE id = $report_id", + "refresh": 1, + "regex": "", + "sort": 4, + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "hide": 2, + "name": "pipeline_name", + "options": [], + "query": "SELECT pipeline_name FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "definition": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "hide": 2, + "name": "pipeline_id", + "options": [], + "query": "SELECT pipeline_id FROM executions_admin WHERE id = $execution_id", + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "Ruff Report", + "uid": "default-ruff-report" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/folders/10-application-quality-service.json b/application-quality-reference-deployment/grafana/folders/10-application-quality-service.json new file mode 100644 index 0000000..9c3a1d7 --- /dev/null +++ b/application-quality-reference-deployment/grafana/folders/10-application-quality-service.json @@ -0,0 +1,4 @@ +{ + "uid": "application-quality-service", + "title": "Application Quality Service" +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/folders/11-application-quality-service-reports.json b/application-quality-reference-deployment/grafana/folders/11-application-quality-service-reports.json new file mode 100644 index 0000000..ed5620d --- /dev/null +++ b/application-quality-reference-deployment/grafana/folders/11-application-quality-service-reports.json @@ -0,0 +1,4 @@ +{ + "uid": "application-quality-service-reports", + "title": "Application Quality Service Reports" +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-report-header.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-report-header.json new file mode 100644 index 0000000..5a568f3 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-report-header.json @@ -0,0 +1,36 @@ +{ + "folderUid": "application-quality-service-reports", + "uid": "pipeline-execution-report-header", + "name": "Pipeline Execution Report Header", + "kind": 1, + "type": "text", + "description": "", + "model": { + "description": "", + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Pipeline Execution Report\n\nSee below the details of the execution report ...\n\n* Pipeline **${pipeline_id}** (${pipeline_name})\n* Pipeline execution ID: [**${execution_id}**](/d/pipeline-execution/pipeline-execution?)\n* Execution report ID: **${report_id}**", + "mode": "markdown" + }, + "pluginVersion": "12.0.1", + "transparent": true, + "type": "text", + "targets": [ + { + "refId": "A", + "datasource": { + "uid": "application-quality-db", + "type": "grafana-postgresql-datasource" + } + } + ], + "datasource": { + "uid": "application-quality-db", + "type": "grafana-postgresql-datasource" + } + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-reports-list.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-reports-list.json new file mode 100644 index 0000000..250dba9 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-execution-reports-list.json @@ -0,0 +1,173 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-executions-report-list", + "name": "Pipeline Execution Reports List", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Report" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Go to report", + "url": "/d/default-${__data.fields.Tool}-report/${__data.fields.Tool}-report?${__all_variables}&var-report_id=${__data.fields.Report}" + } + ] + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "data-links" + } + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT name AS \"Tool\", created_at AS \"Created\", id AS \"Report\" FROM reports_admin WHERE run_id = ${execution_id:int}", + "refId": "A", + "sql": { + "columns": [ + { + "alias": "\"Name\"", + "parameters": [ + { + "name": "name", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Output creation\"", + "parameters": [ + { + "name": "created_at", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Link\"", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "999b8aa9-cdef-4012-b456-7197a2227454", + "properties": { + "field": "run_id", + "fieldSrc": "field", + "operator": "equal", + "value": [ + 2 + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "number" + ] + }, + "type": "rule" + } + ], + "id": "8abb8b8a-0123-4456-b89a-b197a20c2547", + "type": "group" + }, + "whereString": "run_id = 2" + }, + "table": "reports_admin" + } + ], + "title": "Pipeline Execution Reports", + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-daily-statistics.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-daily-statistics.json new file mode 100644 index 0000000..6f2aa36 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-daily-statistics.json @@ -0,0 +1,758 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-executions-daily-statistics", + "name": "Pipeline Executions Daily Statistics", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "No pipeline executions found in the selected date range", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Starting" + }, + "properties": [ + { + "id": "noValue" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "noValue" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "noValue" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time :: date AS \"Date\", COUNT(*) AS \"Total executions\" FROM executions_admin WHERE $__timeFilter(start_time) GROUP BY start_time :: date", + "refId": "Total runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Starting\"\nFROM executions_admin\nWHERE (status = 'starting' AND $__timeFilter(start_time))\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Starting runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Running\" FROM executions_admin WHERE (status = 'running' AND $__timeFilter(start_time)) GROUP BY start_time::date ORDER BY start_time::date ASC", + "refId": "Running runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Active Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "active" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "98a88b88-89ab-4cde-b012-319704dd9c20", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'active' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Succeeded\"\nFROM executions_admin \nWHERE (status = 'succeeded' AND $__timeFilter(start_time))\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Successful runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Successful Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "succeeded" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "989baa88-89ab-4cde-b012-319704de0ce9", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'succeeded' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT\n start_time :: date AS \"Date\",\n COUNT(*) AS \"Failed\"\nFROM\n executions_admin \nWHERE\n (\n status = 'failed'\n AND $__timeFilter(start_time)\n )\nGROUP BY\n start_time :: date\nORDER BY\n start_time :: date ASC", + "refId": "Failed runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT\n start_time :: date AS \"Date\",\n SUM(total_tasks) AS \"Total tasks\",\n SUM(total_cpu_hours) AS \"Total CPU [h]\",\n SUM(elapsed_hours) AS \"Total elapsed [h]\",\n SUM(total_disk_megabytes) AS \"Total disk [MB]\",\n SUM(total_ram_megabyte_hours) AS \"Total RAM [MB/h]\"\n -- SUM((usage_report->>'total_tasks')::int) AS \"Total tasks\",\n -- SUM((usage_report->>'total_cpu_hours')::float) AS \"Total CPU [h]\",\n -- SUM((usage_report->>'elapsed_hours')::float) AS \"Total elapsed [h]\",\n -- SUM((usage_report->>'total_disk_megabytes')::float) AS \"Total disk [MB]\",\n -- SUM((usage_report->>'total_ram_megabyte_hours')::float) AS \"Total RAM [MB/h]\"\nFROM\n executions_admin \nWHERE\n $__timeFilter(start_time)\nGROUP BY\n start_time :: date\nORDER BY\n start_time :: date ASC", + "refId": "Total reports", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + } + ], + "title": "Pipeline Executions Daily Statistics", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Date", + "mode": "outer" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-MM-DD", + "destinationType": "string", + "targetField": "Date", + "timezone": "browser" + } + ], + "fields": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Date" + } + ] + } + } + ], + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-list.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-list.json new file mode 100644 index 0000000..9dca196 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-list.json @@ -0,0 +1,243 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-executions-list", + "name": "Pipeline Executions List", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "noValue": "No pipeline executions found in the selected date range", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Links" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "Details", + "url": "/d/pipeline-execution/pipeline-execution?var-execution_id=${__data.fields.Links}" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "data-links" + } + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "End time" + }, + "properties": [ + { + "id": "noValue" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "builder", + "format": "table", + "rawSql": "SELECT pipeline_name, start_time, completion_time, status, id FROM executions_admin WHERE $__timeFilter(start_time) ORDER BY start_time LIMIT null ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "pipeline_name", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "completion_time", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "status", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": null, + "orderBy": { + "property": { + "name": [ + "start_time" + ], + "type": "string" + }, + "type": "property" + }, + "whereJsonTree": { + "children1": [ + { + "id": "bb8aaa89-cdef-4012-b456-719712903b33", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "9b98988b-0123-4456-b89a-b197128fb9b8", + "type": "group" + }, + "whereString": "$__timeFilter(start_time)" + }, + "table": "executions_admin" + } + ], + "title": "Pipeline Executions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": false + }, + "includeByName": {}, + "indexByName": { + "completion_time": 2, + "id": 4, + "pipeline_name": 0, + "start_time": 1, + "status": 3 + }, + "renameByName": { + "completion_time": "End time", + "id": "Links", + "pipeline_name": "Pipeline name", + "start_time": "Start time", + "status": "Status" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Start time" + } + ] + } + } + ], + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-timeline.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-timeline.json new file mode 100644 index 0000000..76aa21a --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-executions-timeline.json @@ -0,0 +1,810 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-executions-timeline", + "name": "Pipeline Executions Timeline", + "kind": 1, + "type": "timeseries", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [ + { + "options": { + "Succeeded": { + "color": "dark-green", + "index": 0, + "text": "SUCCEEDED" + } + }, + "type": "value" + } + ], + "noValue": "No pipeline executions found in the selected date range", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Starting" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Succeeded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Starting\"\nFROM executions_admin\nWHERE (status = 'starting' AND $__timeFilter(start_time))\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Starting runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Running\" FROM executions_admin WHERE (status = 'running' AND $__timeFilter(start_time)) GROUP BY start_time::date ORDER BY start_time::date ASC", + "refId": "Running runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Active Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "active" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "98a88b88-89ab-4cde-b012-319704dd9c20", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'active' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Succeeded\"\nFROM executions_admin\nWHERE (status = 'succeeded' AND $__timeFilter(start_time))\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Successful runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Successful Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "succeeded" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "989baa88-89ab-4cde-b012-319704de0ce9", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'succeeded' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Failed\"\nFROM executions_admin\nWHERE (status = 'failed' AND $__timeFilter(start_time))\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Failed runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT start_time::date AS \"Date\", COUNT(*) AS \"Total\"\nFROM executions_admin\nWHERE $__timeFilter(start_time)\nGROUP BY start_time::date\nORDER BY start_time::date ASC", + "refId": "Total runs", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT\n start_time :: date AS \"Date\",\n SUM((usage_report->>'total_tasks')::int) AS \"Total tasks\",\n SUM((usage_report->>'total_cpu_hours')::float) AS \"Total CPU [h]\",\n SUM((usage_report->>'elapsed_hours')::float) AS \"Total elapsed [h]\",\n SUM((usage_report->>'total_disk_megabytes')::float) AS \"Total disk [MB]\",\n SUM((usage_report->>'total_ram_megabyte_hours')::float) AS \"Total RAM [MB/h]\"\nFROM\n executions_admin\nWHERE\n $__timeFilter(start_time)\nGROUP BY\n start_time :: date\nORDER BY\n start_time :: date ASC", + "refId": "Total reports", + "sql": { + "columns": [ + { + "alias": "\"Pipeline ID\"", + "parameters": [ + { + "name": "pipeline_id", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "alias": "\"Starting Runs\"", + "name": "COUNT", + "parameters": [ + { + "name": "start_time", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "pipeline_id", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "orderBy": { + "property": { + "name": [ + "pipeline_id" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "ASC", + "whereJsonTree": { + "children1": [ + { + "id": "8aaa8999-cdef-4012-b456-719704cfdf08", + "properties": { + "field": "status", + "fieldSrc": "field", + "operator": "equal", + "value": [ + "starting" + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "text" + ] + }, + "type": "rule" + }, + { + "id": "9bb9aaab-4567-489a-bcde-f19704d92340", + "properties": { + "field": "start_time", + "fieldSrc": "field", + "operator": "macros", + "value": [ + "timeFilter" + ], + "valueError": [ + null, + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "8a999a98-0123-4456-b89a-b19704c7ab1b", + "type": "group" + }, + "whereString": "(status = 'starting' AND $__timeFilter(start_time))" + }, + "table": "backend_pipelinerun" + } + ], + "title": "Pipeline Executions Timeline", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Date", + "mode": "outer" + } + }, + { + "disabled": true, + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-MM-DD", + "destinationType": "string", + "targetField": "Date", + "timezone": "browser" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-inputs-list.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-inputs-list.json new file mode 100644 index 0000000..68db163 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-inputs-list.json @@ -0,0 +1,206 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-inputs", + "name": "Pipeline Inputs", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Value" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Field" + } + ] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT e.inputs :: jsonb FROM executions_admin e WHERE id = $execution_id ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "*", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": null, + "whereJsonTree": { + "children1": [ + { + "id": "aaab98a9-cdef-4012-b456-7197cc5405ba", + "properties": { + "field": "id", + "fieldSrc": "field", + "operator": "equal", + "value": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "number" + ] + }, + "type": "rule" + } + ], + "id": "b988bbaa-0123-4456-b89a-b197cc4ebfe1", + "type": "group" + } + }, + "table": "executions_admin" + } + ], + "title": "Pipeline Inputs", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "auto", + "keepTime": false, + "replace": true, + "source": "inputs" + } + }, + { + "id": "transpose", + "options": { + "firstFieldName": "", + "restFieldsName": "" + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "substring", + "options": { + "value": "." + } + }, + "fieldName": "Field" + } + ], + "match": "any", + "type": "include" + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Field" + } + ] + } + } + ], + "transparent": true, + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pipeline-usage-report.json b/application-quality-reference-deployment/grafana/library-panels/pipeline-usage-report.json new file mode 100644 index 0000000..f1138da --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pipeline-usage-report.json @@ -0,0 +1,299 @@ +{ + "folderUid": "application-quality-service", + "uid": "pipeline-usage-report", + "name": "Pipeline Usage Report", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "elapsed_seconds" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_disk_megabytes" + }, + "properties": [ + { + "id": "unit", + "value": "decmbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_ram_megabyte_hours" + }, + "properties": [ + { + "id": "unit", + "value": "h" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max_parallel_ram_megabytes" + }, + "properties": [ + { + "id": "unit", + "value": "decmbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ram_mb_allowed" + }, + "properties": [ + { + "id": "unit", + "value": "decmbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_cpu_hours" + }, + "properties": [ + { + "id": "unit", + "value": "h" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "start_time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIsoNoDateIfToday" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "finish_time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIsoNoDateIfToday" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Max parallel RAM [MB]" + }, + "properties": [ + { + "id": "custom.width", + "value": 182 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT status, usage_report FROM executions_admin WHERE id = $execution_id", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "usage_report", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50, + "whereJsonTree": { + "children1": [ + { + "id": "aab8baa9-89ab-4cde-b012-3197a65a737f", + "properties": { + "field": "id", + "fieldSrc": "field", + "operator": "equal", + "value": [ + 1 + ], + "valueError": [ + null + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "number" + ] + }, + "type": "rule" + } + ], + "id": "9b9b89ba-0123-4456-b89a-b197a63a721f", + "type": "group" + }, + "whereString": "id = 1" + }, + "table": "executions_admin" + } + ], + "title": "Pipeline Usage Report", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "auto", + "keepTime": false, + "replace": false, + "source": "usage_report" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "children": true, + "cores_allowed": false, + "finish_time": false, + "max_parallel_tasks": false, + "start_time": false, + "total_disk_megabytes": false, + "total_tasks": false, + "usage_report": true + }, + "includeByName": {}, + "indexByName": { + "children": 1, + "cores_allowed": 6, + "elapsed_hours": 7, + "elapsed_seconds": 9, + "finish_time": 4, + "max_parallel_cpus": 10, + "max_parallel_ram_megabytes": 12, + "max_parallel_tasks": 11, + "ram_mb_allowed": 8, + "start_time": 3, + "status": 2, + "total_cpu_hours": 13, + "total_disk_megabytes": 14, + "total_ram_megabyte_hours": 15, + "total_tasks": 5, + "usage_report": 0 + }, + "renameByName": { + "cores_allowed": "Cores", + "elapsed_hours": "Elapsed [h]", + "elapsed_seconds": "Duration [s]", + "finish_time": "Finish time", + "max_parallel_cpus": "Max parallel CPUs", + "max_parallel_ram_megabytes": "Max parallel RAM [MB]", + "max_parallel_tasks": "Max parallel tasks", + "ram_mb_allowed": "RAM allowed [MB]", + "start_time": "Start time", + "status": "Status", + "total_cpu_hours": "CPU [h]ours", + "total_disk_megabytes": "Total disk [MB]", + "total_ram_megabyte_hours": "Total RAM [MB/h]", + "total_tasks": "Tasks" + } + } + } + ], + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/pylint-report.json b/application-quality-reference-deployment/grafana/library-panels/pylint-report.json new file mode 100644 index 0000000..938d994 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/pylint-report.json @@ -0,0 +1,173 @@ +{ + "folderUid": "application-quality-service-reports", + "uid": "pylint-report", + "name": "Pylint Report", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "This report is empty", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Code" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Location" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Object" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + }, + { + "id": "noValue" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Code" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open documentation for ${__data.fields.Code}", + "url": "https://pylint.readthedocs.io/en/stable/user_guide/messages/${__data.fields.type}/${__data.fields.symbol}.html" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "symbol" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT elem ->> 'path' AS \"Path to File\", elem ->> 'obj' AS \"Object\", elem ->> 'message' AS \"Message\", elem ->> 'type' AS type, elem ->> 'symbol' AS symbol, elem ->> 'message-id' AS \"Code\", 'L' || (elem ->> 'line') || 'C' || (elem ->> 'column') || CASE WHEN elem ->> 'endLine' IS NOT NULL AND elem ->> 'endColumn' IS NOT NULL THEN '-L' || (elem ->> 'endLine') || 'C' || (elem ->> 'endColumn') ELSE '' END AS \"Location\" FROM (SELECT jsonb_array_elements(output :: jsonb) AS elem FROM reports_admin WHERE id = ${report_id}) AS sub;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Pylint Report", + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/ruff-report.json b/application-quality-reference-deployment/grafana/library-panels/ruff-report.json new file mode 100644 index 0000000..8cf1a88 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/ruff-report.json @@ -0,0 +1,176 @@ +{ + "folderUid": "application-quality-service-reports", + "uid": "ruff-report", + "name": "Ruff Report", + "kind": 1, + "type": "table", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "This report is empty", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Code" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cell #" + }, + "properties": [ + { + "id": "custom.width", + "value": 60 + }, + { + "id": "noValue" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Location in Cell" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Safe?" + }, + "properties": [ + { + "id": "custom.width", + "value": 60 + }, + { + "id": "noValue" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Code" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open documentation for ${__data.fields.Code}", + "url": "https://docs.astral.sh/ruff/rules/${__data.fields.Code}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Fix" + }, + "properties": [ + { + "id": "noValue" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "file" + } + ] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT split_part(elem ->> 'filename', '/', array_length(string_to_array(elem ->> 'filename', '/'), 1)) AS \"Notebook File\", elem ->> 'message' AS \"Message\", elem ->> 'code' AS \"Code\", (elem ->> 'cell') :: int AS \"Cell #\", 'L' || (elem -> 'location' ->> 'row') || 'C' || (elem -> 'location' ->> 'column') || '-L' || (elem -> 'end_location' ->> 'row') || 'C' || (elem -> 'end_location' ->> 'column') AS \"Location in Cell\", elem -> 'fix' ->> 'message' AS \"Fix\", (elem -> 'fix' ->> 'applicability') = 'safe' AS \"Safe?\" FROM (SELECT jsonb_array_elements(output :: jsonb) AS elem FROM reports_admin WHERE id = $report_id) AS sub;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Ruff Report", + "type": "table" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/usage-report-per-metric.json b/application-quality-reference-deployment/grafana/library-panels/usage-report-per-metric.json new file mode 100644 index 0000000..7d3d254 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/usage-report-per-metric.json @@ -0,0 +1,156 @@ +{ + "folderUid": "application-quality-service", + "uid": "usage-report-per-metric", + "name": "Usage Report per Metric", + "kind": 1, + "type": "barchart", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xField": "Field", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n-- e.id AS execution_id,\n child ->> 'name' AS step_name,\n (child ->> 'cpu_hours') :: float AS cpu_hours,\n (child ->> 'elapsed_seconds') :: float AS elapsed_seconds,\n (child ->> 'ram_megabyte_hours') :: float AS ram_megabyte_hours\nFROM\n executions_admin e,\n jsonb_array_elements(e.usage_report :: jsonb -> 'children') AS child\nWHERE\n e.usage_report IS NOT NULL\n AND id = $execution_id \n AND (child ->> 'name') NOT LIKE 'clone_%'\n AND (child ->> 'name') NOT LIKE 'filter_%'\n AND (child ->> 'name') NOT LIKE 'save_%'\nLIMIT\n 50;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "*", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 10, + "whereJsonTree": { + "children1": [ + { + "id": "b89a8b9a-cdef-4012-b456-719768bf3b6e", + "properties": { + "field": "usage_report", + "fieldSrc": "field", + "operator": "is_not_null", + "value": [], + "valueSrc": [], + "valueType": [] + }, + "type": "rule" + } + ], + "id": "8b9998a8-0123-4456-b89a-b19768778aa4", + "type": "group" + }, + "whereString": "usage_report IS NOT NULL" + }, + "table": "executions" + } + ], + "title": "Usage Report per Metric", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "step_name" + } + ] + } + }, + { + "id": "transpose", + "options": {} + } + ], + "type": "barchart" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/grafana/library-panels/usage-report-per-tool.json b/application-quality-reference-deployment/grafana/library-panels/usage-report-per-tool.json new file mode 100644 index 0000000..d7fa5c6 --- /dev/null +++ b/application-quality-reference-deployment/grafana/library-panels/usage-report-per-tool.json @@ -0,0 +1,207 @@ +{ + "folderUid": "application-quality-service", + "uid": "usage-report-per-tool", + "name": "Usage Report per Tool", + "kind": 1, + "type": "barchart", + "description": "", + "model": { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#73BF69", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "elapsed_seconds" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ram_megabyte_hours" + }, + "properties": [ + { + "id": "unit", + "value": "decmbytes" + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cpu_seconds" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xField": "step_name", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "application-quality-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n-- e.id AS execution_id,\n child ->> 'name' AS step_name,\n (child ->> 'cpu_hours') :: float * 3600 AS cpu_seconds,\n (child ->> 'elapsed_seconds') :: float AS elapsed_seconds,\n (child ->> 'ram_megabyte_hours') :: float AS ram_megabyte_hours\nFROM\n executions_admin e,\n jsonb_array_elements(e.usage_report :: jsonb -> 'children') AS child\nWHERE\n e.usage_report IS NOT NULL\n AND id = $execution_id \n AND (child ->> 'name') NOT LIKE 'clone_%'\n AND (child ->> 'name') NOT LIKE 'filter_%'\n AND (child ->> 'name') NOT LIKE 'save_%'\nLIMIT\n 50;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "*", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 10, + "whereJsonTree": { + "children1": [ + { + "id": "b89a8b9a-cdef-4012-b456-719768bf3b6e", + "properties": { + "field": "usage_report", + "fieldSrc": "field", + "operator": "is_not_null", + "value": [], + "valueSrc": [], + "valueType": [] + }, + "type": "rule" + } + ], + "id": "8b9998a8-0123-4456-b89a-b19768778aa4", + "type": "group" + }, + "whereString": "usage_report IS NOT NULL" + }, + "table": "executions" + } + ], + "title": "Usage Report per Tool", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "step_name" + } + ] + } + } + ], + "type": "barchart" + } +} \ No newline at end of file diff --git a/application-quality-reference-deployment/templates/opensearch-admin-certificate.yaml b/application-quality-reference-deployment/templates/opensearch-admin-certificate.yaml deleted file mode 100644 index 354b0aa..0000000 --- a/application-quality-reference-deployment/templates/opensearch-admin-certificate.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: opensearch-admin-certificate - labels: - app.kubernetes.io/component: "opensearch" - {{- include "application-quality-reference-deployment.labels" . | nindent 4 }} -spec: - secretName: opensearch-admin-certificate - isCA: false - usages: - - digital signature - - key encipherment - - server auth - - client auth - commonName: opensearch-admin - issuerRef: - {{- toYaml .Values.global.defaultInternalIssuerRef | nindent 4 }} - privateKey: - algorithm: RSA - encoding: PKCS8 - size: 2048 \ No newline at end of file diff --git a/application-quality-reference-deployment/templates/opensearch-dashboards-client-certificate.yaml b/application-quality-reference-deployment/templates/opensearch-dashboards-client-certificate.yaml deleted file mode 100644 index 05d8664..0000000 --- a/application-quality-reference-deployment/templates/opensearch-dashboards-client-certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: opensearch-dashboards-client-certificate - labels: - app.kubernetes.io/component: "opensearch-dashboards" - {{- include "application-quality-reference-deployment.labels" . | nindent 4 }} -spec: - secretName: opensearch-dashboards-client-certificate - isCA: false - usages: - - digital signature - - key encipherment - - server auth - - client auth - commonName: dashboards-client - #dnsNames: - # - "application-quality-dashboards.{{ .Release.Namespace }}.{{ .Values.global.internalDomain }}" - # - "application-quality-dashboards" - issuerRef: - {{- toYaml .Values.global.defaultInternalIssuerRef | nindent 4 }} - privateKey: - algorithm: RSA - encoding: PKCS8 - size: 2048 \ No newline at end of file diff --git a/application-quality-reference-deployment/templates/opensearch-dashboards-tls-certificate.yaml b/application-quality-reference-deployment/templates/opensearch-dashboards-tls-certificate.yaml deleted file mode 100644 index 9b5c266..0000000 --- a/application-quality-reference-deployment/templates/opensearch-dashboards-tls-certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: opensearch-dashboards-tls-certificate - labels: - app.kubernetes.io/component: "opensearch-dashboards" - {{- include "application-quality-reference-deployment.labels" . | nindent 4 }} -spec: - secretName: opensearch-dashboards-tls-certificate - isCA: false - usages: - - digital signature - - key encipherment - - server auth - - client auth - commonName: application-quality-opensearch-dashboards - dnsNames: - - "application-quality-opensearch-dashboards.{{ .Release.Namespace }}.{{ .Values.global.internalDomain }}" - - "application-quality-opensearch-dashboards" - issuerRef: - {{- toYaml .Values.global.defaultInternalIssuerRef | nindent 4 }} - privateKey: - algorithm: RSA - encoding: PKCS8 - size: 2048 \ No newline at end of file diff --git a/application-quality-reference-deployment/templates/opensearch-tls-certificate.yaml b/application-quality-reference-deployment/templates/opensearch-tls-certificate.yaml deleted file mode 100644 index 093c6a5..0000000 --- a/application-quality-reference-deployment/templates/opensearch-tls-certificate.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: opensearch-tls-certificate - labels: - app.kubernetes.io/component: "opensearch" - {{- include "application-quality-reference-deployment.labels" . | nindent 4 }} -spec: - secretName: opensearch-tls-certificate - isCA: false - usages: - - digital signature - - key encipherment - - server auth - - client auth - commonName: application-quality-opensearch-node - dnsNames: - - "opensearch-cluster-master-headless.{{ .Release.Namespace }}.{{ .Values.global.internalDomain }}" - - "opensearch-cluster-master.{{ .Release.Namespace }}.{{ .Values.global.internalDomain }}" - - "opensearch-cluster-master-headless" - - "opensearch-cluster-master" - issuerRef: - {{- toYaml .Values.global.defaultInternalIssuerRef | nindent 4 }} - privateKey: - algorithm: RSA - encoding: PKCS8 - size: 2048 \ No newline at end of file diff --git a/application-quality-reference-deployment/values.yaml b/application-quality-reference-deployment/values.yaml index f150457..ecdce47 100644 --- a/application-quality-reference-deployment/values.yaml +++ b/application-quality-reference-deployment/values.yaml @@ -26,12 +26,14 @@ application-quality: # Application Quality backend api: image: - tag: 2025-04-07.1 + repository: eoepca/eoepca-appquality-backend + tag: 2.0.0-rc2 # Application Quality frontend web: image: - tag: 2025-04-07.2 + repository: eoepca/eoepca-appquality-frontend + tag: 2.0.0-rc2 publicUrl: "https://application-quality.develop.eoepca.org" persistence: @@ -52,13 +54,8 @@ application-quality: vclusters: enabled: false - opensearch: - enabled: false - url: "http://opensearch-cluster-master.application-quality.svc.cluster.local:9200" - #indexRuns: - #indexReports: - username: appquality-backend-rw - password: "" + grafana: + enabled: true oidc: enabled: true @@ -91,440 +88,190 @@ application-quality: type: ClusterIP port: 80 - -opensearch: - # Needed for predictable DNS and resource names - fullnameOverride: "application-quality-opensearch" - - replicas: "2" - - persistence: - enabled: true - storageClass: managed-nfs-storage-retain - size: 12Gi - - sysctl: - enabled: false - sysctlInit: - enabled: true - +# https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +grafana: rbac: create: false - serviceAccountAnnotations: {} - serviceAccountName: "application-quality-opensearch-dashboards-dashboards" - # Controls whether or not the Service Account token is automatically mounted to /var/run/secrets/kubernetes.io/serviceaccount - automountServiceAccountToken: false - - extraEnvs: - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - - secretMounts: - - name: certs - secretName: opensearch-tls-certificate - path: /usr/share/opensearch/config/esnode - - name: admin-certs - secretName: opensearch-admin-certificate - path: /usr/share/opensearch/config/admin - - opensearchJavaOpts: "-Xmx2G -Xms2G" - - # https://github.com/opensearch-project/helm-charts/blob/main/charts/opensearch/values.yaml#L148-L151 - resources: - requests: - cpu: "1000m" - memory: "1G" - limits: - cpu: "4000m" - memory: "4G" - - config: - # https://github.com/opensearch-project/helm-charts/blob/main/charts/opensearch/values.yaml#L36-L48 - log4j2.properties: | - status = error + serviceAccount: + create: false + autoscaling: + enabled: false + testFramework: + enabled: false + # Enable creating the grafana configmap + createConfigmap: true + + # Extra configmaps to mount in grafana pods + # Values are templated. + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + # optional: false - appender.console.type = Console - appender.console.name = console - appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - - rootLogger.level = info - rootLogger.appenderRef.console.ref = console - # c.a.d.a.h.j.AbstractHTTPJwtAuthenticator - logger.securityjwt.name = com.amazon.dlic.auth.http.jwt - logger.securityjwt.level = trace - - opensearch.yml: > - cluster.name: opensearch-cluster + extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + extraContainers: | + - name: init-application-quality-data + image: curlimages/curl:8.14.1 + command: ["/bin/sh", "-c"] + args: + - | + echo "Starting initialisation script in Grafana pod..." + # Wait for the Grafana API to repond + GRAFANA_API_URL="http://${APPLICATION_QUALITY_CORE_GRAFANA_SERVICE_HOST}:${APPLICATION_QUALITY_CORE_GRAFANA_SERVICE_PORT}" + echo "Waiting for Grafana API at ${GRAFANA_API_URL}/api/health to respond..." + until curl --fail --silent "${GRAFANA_API_URL}/api/health"; do + echo "Grafana API not yet available, retrying in 5 seconds..." + sleep 5 + done + echo -e "\\nGrafana API is now available. Proceeding with the initialisation." + cd /tmp + curl -LO https://github.com/EOEPCA/application-quality/archive/refs/heads/reference-deployment.zip + unzip -o reference-deployment.zip + cd application-quality-reference-deployment/application-quality-reference-deployment/grafana + for i in folders/*.json; do + echo -e "\\nCreating folder defined in: $i" + curl -L "${GRAFANA_API_URL}/api/folders" \ + -H "Content-Type: application/json" \ + -u $GRAFANA_SECURITY_ADMIN_USER:$GRAFANA_SECURITY_ADMIN_PASSWORD \ + -d @$i + done + for i in library-panels/*.json; do + echo -e "\\nCreating library panel defined in: $i" + curl -L "${GRAFANA_API_URL}/api/library-elements" \ + -H "Content-Type: application/json" \ + -u $GRAFANA_SECURITY_ADMIN_USER:$GRAFANA_SECURITY_ADMIN_PASSWORD \ + -d @$i + done + for i in dashboards/*.json; do + echo -e "\\nCreating dashboard defined in: $i" + curl -L "${GRAFANA_API_URL}/api/dashboards/db" \ + -H "Content-Type: application/json" \ + -u $GRAFANA_SECURITY_ADMIN_USER:$GRAFANA_SECURITY_ADMIN_PASSWORD \ + -d @$i + done + echo -e "\\nInitialisation complete - Keeping the container alive ..." + while true; do sleep 5; done + envFrom: + - secretRef: + name: application-quality-grafana-dashboards-admin-creds + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + runAsNonRoot: true + + podPortName: grafana + gossipPortName: gossip + ## Deployment annotations + # annotations: {} - network.host: 0.0.0.0 - - plugins: - security: - nodes_dn: - - 'CN=application-quality-opensearch-node' - ssl: - transport: - pemcert_filepath: esnode/tls.crt - pemkey_filepath: esnode/tls.key - pemtrustedcas_filepath: esnode/ca.crt - enforce_hostname_verification: false - resolve_hostname: false - http: - enabled: true - pemcert_filepath: esnode/tls.crt - pemkey_filepath: esnode/tls.key - pemtrustedcas_filepath: esnode/ca.crt - clientauth_mode: OPTIONAL - allow_unsafe_democertificates: false - allow_default_init_securityindex: true - authcz: - admin_dn: - # Use the "commonName" from the Certificate object (see in templates) - # - CN=opensearch-admin,OU=client,O=client,L=test,C=de - - CN=opensearch-admin - audit.type: internal_opensearch - enable_snapshot_restore_privilege: true - check_snapshot_restore_write_privileges: true - restapi: - roles_enabled: ["all_access", "security_rest_api_access"] - system_indices: - enabled: true - indices: - [ - ".opendistro-alerting-config", - ".opendistro-alerting-alert*", - ".opendistro-anomaly-results*", - ".opendistro-anomaly-detector*", - ".opendistro-anomaly-checkpoints", - ".opendistro-anomaly-detection-state", - ".opendistro-reports-*", - ".opendistro-notifications-*", - ".opendistro-notebooks", - ".opendistro-asynchronous-search-response*", - ] - securityConfig: + service: + # https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml#L231 enabled: true - path: "/usr/share/opensearch/config/opensearch-security" - config: - dataComplete: false - data: - config.yml: |- - _meta: - type: "config" - config_version: 2 - - config: - dynamic: - kibana: - multitenancy_enabled: true - # https://opensearch.org/docs/latest/security/configuration/best-practices/#7-consider-disabling-the-private-tenant - #private_tenant_enabled: false - server_username: 'dashboards-client' - index: '.opensearch-dashboards' - - do_not_fail_on_forbidden: false - - http: - anonymous_auth_enabled: false - - authc: - openid_auth_domain: - http_enabled: true - transport_enabled: true - order: 0 - http_authenticator: - type: openid - #challenge: false - challenge: true - config: - ## How to determine the username - subject_key: preferred_username - ## How to determine which backend_roles a user belongs to - roles_key: opensearch-roles - openid_connect_url: https://auth.eoepca-testing.spaceapplications.com/realms/eoepca/.well-known/openid-configuration - authentication_backend: - type: noop - - basic_internal_auth_domain: - description: "Authenticate via HTTP Basic against internal users database" - http_enabled: true - transport_enabled: true - order: 1 - http_authenticator: - type: basic - challenge: true - authentication_backend: - type: intern - - clientcert_auth_domain: - description: "Authenticate via SSL client certificates" - http_enabled: true - transport_enabled: false - order: 2 - http_authenticator: - type: clientcert - config: - username_attribute: cn - challenge: false - authentication_backend: - type: noop - - internal_users.yml: |- - _meta: - type: "internalusers" - config_version: 2 - - spaceapps: - hash: "$2y$12$Fz7OW72EtL1KPMdPgtL.YuASfcjHCMm9NSGUaa6yxo9mtdVQcQNcS" - reserved: true - backend_roles: - - "admin" - description: "Temporary admin user (Delete me)" - - appquality-backend-ro: - hash: "$2y$12$EzrBjng/196TSFJl15LwMOpvY2mDDs.pqPBcM3nQmB02YFQ.yU4GC" - description: "User for accessing Application Quality data in read-only mode" - - appquality-backend-rw: - hash: "$2y$12$w8obM9zTRNAZB/R66objZedabMGPKz8IN3vifs5hjVSJwUgV3BnqC" - description: "User for accessing Application Quality data in read-write mode" - - - roles.yml: |- - _meta: - type: "roles" - config_version: 2 - - application-quality-ro: - cluster_permissions: - - cluster_all - index_permissions: - - index_patterns: - - "application-quality-*" - - "appquality-*" - allowed_actions: - - read - - application-quality-rw: - cluster_permissions: - - cluster_all - index_permissions: - - index_patterns: - - "application-quality-*" - - "appquality-*" - allowed_actions: - - crud - - application-quality-own-data-access: - reserved: false - hidden: false - cluster_permissions: [] - index_permissions: - - index_patterns: - - application-quality-* - ## Document Level Security filter - dls: "{ \"bool\": { \"must\" : { \"match\": { \"attributes.authentication_subject\" : \"${user.name}\" } } } }" - allowed_actions: - - crud - - - roles_mapping.yml: |- - _meta: - type: "rolesmapping" - config_version: 2 - - ## all_access is a built-in role - all_access: - reserved: false - backend_roles: [] - #- "admin" - users: - - "spaceapps" - description: "Maps admin to all_access" - - dashboard_server: - reserved: true - hidden: false - backend_roles: [] - hosts: [] - users: - - "dashboards-client" - - "spaceapps" - and_backend_roles: [] - - kibana_server: - reserved: true - hidden: false - # backend_roles: [] - backend_roles: - # This is a group membership defined in KeyCloak - - "[opensearch_all_access]" - - "alice" - - "bob" - - "eric" - hosts: [] - users: - - "dashboards-client" - - "spaceapps" - and_backend_roles: [] - - kibana_user: - reserved: true - hidden: false - backend_roles: [] - hosts: [] - users: - - "appquality-backend-ro" - - "appquality-backend-rw" - and_backend_roles: [] - - application-quality-ro: - users: - - "appquality-backend-ro" - - application-quality-rw: - reserved: false - users: - - "appquality-backend-rw" - backend_roles: - # This is a group membership defined in KeyCloak - - "[opensearch_all_access]" - - "alice" - - "bob" - - "eric" - - # action_groups.yml: |- - # tenants.yml: |- - - -opensearch-dashboards: - - # Needed for predictable DNS and resource names - fullnameOverride: "application-quality-opensearch-dashboards" + type: ClusterIP + port: 80 + targetPort: 3000 + + hostAliases: [] ingress: enabled: true ingressClassName: nginx - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - ingress.kubernetes.io/ssl-redirect: 'true' - nginx.ingress.kubernetes.io/ssl-redirect: 'true' - nginx.ingress.kubernetes.io/backend-protocol: 'HTTPS' - labels: {} hosts: - - host: application-quality.develop.eoepca.org - paths: - - path: /dashboards - pathType: Prefix - backend: - serviceName: application-quality-opensearch-dashboards - servicePort: 5601 + - application-quality.develop.eoepca.org + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + path: /dashboards/?(.*) + pathType: Prefix tls: - secretName: application-quality-tls hosts: - application-quality.develop.eoepca.org - config: - opensearch_dashboards.yml: - server: - name: "Application Quality Dashboards" - host: 0.0.0.0 - # Change the entry point/base URL of opensearch-dashboards - basePath: /dashboards - rewriteBasePath: true - maxPayloadBytes: 1048576 - ssl: - # Enables SSL communication between the OpenSearch Dashboards server and the user’s web browser. - enabled: true - certificateAuthorities: [ "/usr/share/opensearch-dashboards/certs/ca.crt" ] - certificate: /usr/share/opensearch-dashboards/certs/tls.crt - key: /usr/share/opensearch-dashboards/certs/tls.key - - opensearch: - logQueries: true - - ssl: - # Establishes communication between OpenSearch and OpenSearch Dashboards. Valid values are full, certificate, or none. - # certificate = verify the certificate but not the hostname - verificationMode: full - alwaysPresentCertificate: true - certificateAuthorities: [ "/usr/share/opensearch-dashboards/admincerts/ca.crt" ] - certificate: /usr/share/opensearch-dashboards/admincerts/tls.crt - key: /usr/share/opensearch-dashboards/admincerts/tls.key - - logging: - verbose: true - - opensearch_security: - # https://opensearch.org/docs/latest/security/authentication-backends/openid-connect/#configuration-settings - auth.multiple_auth_enabled: true - auth.type: ["basicauth", "openid"] - - openid: - # OIDC parameters provided using environment variables - connect_url: ${OPENSEARCH_SECURITY_OPENID_CONNECT_URL} - client_id: ${OPENSEARCH_SECURITY_OPENID_CLIENT_ID} - client_secret: ${OPENSEARCH_SECURITY_OPENID_CLIENT_SECRET} - base_redirect_url: ${OPENSEARCH_SECURITY_OPENID_BASE_REDIRECT_URL} - # scope: openid profile email - scope: openid roles profile user_name email - header: Authorization - verify_hostnames: false - trust_dynamic_headers: "true" - - opensearchDashboards: - # The default application to load. - # defaultAppId: "home" - - branding: - # "https://eoepca.readthedocs.io/projects/deploy/en/stable/img/favicon.ico" - logo: - defaultUrl: "https://eoepca.org/media_portal/images/eoepca_logo_no_bg.original.png" - #darkModeUrl: "" - mark: - defaultUrl: "https://eoepca.org/media_portal/images/eoepca_logo_no_bg.original.png" - #darkModeUrl: "" - loadingLogo: - defaultUrl: "https://eoepca.org/media_portal/images/eoepca_logo_no_bg.original.png" - #darkModeUrl: "" - faviconUrl: "https://eoepca.org/media_portal/images/eoepca_logo_no_bg.original.png" - applicationTitle: "Application Quality Dashboards" - useExpandedHeader: false - - # Set the backend roles in groups or users, whoever has the backend roles or exactly match the user ids defined in this config will be regard as dashboard admin. - # Dashboard admin will have the access to all the workspaces(workspace.enabled: true) and objects inside OpenSearch Dashboards. - # The default config is [], and no one will be dashboard admin. - # If the user config is set to wildcard ["*"], anyone will be dashboard admin. - # dashboardAdmin.groups: ["dashboard_admin"] - # dashboardAdmin.users: ["dashboard_admin"] - # dashboardAdmin.users: ["dashboards_client", "spaceapps"] - - # Set the value of this setting to true to enable plugin application config. By default it is disabled. - application_config.enabled: true - - # Set the value of this setting to true to enable plugin CSP handler. By default it is disabled. - # It requires the application config plugin as its dependency. - # csp_handler.enabled: false - - home: - disableWelcomeScreen: false - disableNewThemeModal: false - - # Load environment variables from kubernetes secret or config map - envFrom: - - secretRef: - name: application-quality-opensearch-dashboards-openid-config - # - configMapRef: - # name: application-quality-opensearch-dashboards-openid-config - - secretMounts: - - name: certs - secretName: opensearch-dashboards-tls-certificate - path: /usr/share/opensearch-dashboards/certs - - name: clientcerts - secretName: opensearch-dashboards-client-certificate - path: /usr/share/opensearch-dashboards/clientcerts - - name: admincerts - secretName: opensearch-admin-certificate - path: /usr/share/opensearch-dashboards/admincerts \ No newline at end of file + persistence: + type: pvc + enabled: true + storageClassName: managed-nfs-storage-retain + volumeName: application-quality-grafana-pvc + accessModes: + - ReadWriteOnce + size: 1Gi + + admin: + ## Name of the secret. Can be templated. + existingSecret: application-quality-grafana-dashboards-admin-creds + userKey: GRAFANA_SECURITY_ADMIN_USER + passwordKey: GRAFANA_SECURITY_ADMIN_PASSWORD + + ## Pass the plugins you want installed as a list. + ## + plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: "Application Quality DB (Admin)" + uid: application-quality-db + type: postgres + url: application-quality-core-db.application-quality.svc.cluster.local:5432 + user: admin + secureJsonData: + password: changeme + jsonData: + database: appquality + sslmode: "disable" # disable/require/verify-ca/verify-full + maxOpenConns: 100 + maxIdleConns: 100 + maxIdleConnsAuto: true + connMaxLifetime: 14400 + postgresVersion: 1300 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + + alerting: {} + notifiers: {} + dashboardProviders: {} + dashboards: {} + dashboardsConfigMaps: {} + + envFromSecret: application-quality-grafana-dashboards-oauth-config + + grafana.ini: + + # https://grafana.com/docs/grafana/next/setup-grafana/configure-security/configure-authentication/generic-oauth/ + auth.generic_oauth: + enabled: true + allow_sign_up: true + auto_login: false + #team_ids: + #allowed_organizations: + name: "OAuth / OIDC" + #client_id: -- Found in GF_AUTH_GENERIC_OAUTH_CLIENT_ID + #client_secret: -- Found in GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET + scopes: openid roles profile user_name email + auth_url: https://auth.eoepca-testing.spaceapplications.com/realms/eoepca/protocol/openid-connect/auth + token_url: https://auth.eoepca-testing.spaceapplications.com/realms/eoepca/protocol/openid-connect/token + api_url: https://auth.eoepca-testing.spaceapplications.com/realms/eoepca/protocol/openid-connect/userinfo + use_pkce: true + use_refresh_token: true + signout_redirect_url: https://auth.eoepca-testing.spaceapplications.com/realms/eoepca/protocol/openid-connect/logout?client_id=application-quality-develop-dashboards&response_type=code&post_logout_redirect_uri=https://application-quality.develop.eoepca.org/dashboards/ + + server: + root_url: https://application-quality.develop.eoepca.org/dashboards diff --git a/helm/eoepca-develop-values.yaml b/helm/eoepca-develop-values.yaml index 4d6eb07..d5b41d5 100644 --- a/helm/eoepca-develop-values.yaml +++ b/helm/eoepca-develop-values.yaml @@ -32,13 +32,8 @@ calrissian: #db: -opensearch: +grafana: enabled: false - url: "http://opensearch-cluster-master.application-quality.svc.cluster.local:9200" - #indexRuns: - #indexReports: - username: appquality-backend-rw - password: "" oidc: enabled: true diff --git a/helm/templates/api-configmap.yaml b/helm/templates/api-configmap.yaml index c96e60b..504e196 100644 --- a/helm/templates/api-configmap.yaml +++ b/helm/templates/api-configmap.yaml @@ -24,14 +24,8 @@ data: DB_USER: {{ .Values.db.user }} DB_PASSWORD: {{ .Values.db.password | quote }} - # Opensearch - OPENSEARCH_ENABLED: {{ .Values.opensearch.enabled | default false | quote }} - OPENSEARCH_URL: {{ .Values.opensearch.url }} - OPENSEARCH_INDEX_RUNS: {{ .Values.opensearch.indexRuns | default "application-quality-pipeline-runs" }} - OPENSEARCH_INDEX_REPORTS: {{ .Values.opensearch.indexReports | default "application-quality-pipeline-run-reports" }} - # TODO: From secret - OPENSEARCH_USERNAME: {{ .Values.opensearch.username }} - OPENSEARCH_PASSWORD: {{ .Values.opensearch.password | quote }} + # Grafana + GRAFANA_ENABLED: {{ .Values.grafana.enabled | default false | quote }} # TBD AQBB_STORAGECLASS: {{ .Values.calrissian.storageClassName }} diff --git a/helm/templates/api-deployment.yaml b/helm/templates/api-deployment.yaml index 56b6756..d4ec99e 100644 --- a/helm/templates/api-deployment.yaml +++ b/helm/templates/api-deployment.yaml @@ -45,9 +45,9 @@ spec: # name: application-quality-db-creds - secretRef: name: application-quality-auth-client - {{- if .Values.opensearch.enabled }} + {{- if .Values.grafana.enabled }} - secretRef: - name: {{ include "application-quality.fullname" . }}-opensearch + name: application-quality-grafana-dashboards-admin-creds {{- end }} resources: requests: diff --git a/helm/values.yaml b/helm/values.yaml index 6ed25f1..08a579b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -62,18 +62,8 @@ calrissian: maxRam: "2Gi" storageClassName: nfs-client -opensearch: - # Enable or disable saving runs and reports in Opensearch (default: false) +grafana: enabled: false - # Internal URL of the Opensearch servier (e.g. https://opensearch-cluster-master.application-quality-opensearch.svc.cluster.local:9200) - #url: - # Name of the Opensearch index where the pipeline runs must be stored (default: application-quality-pipeline-runs) - #indexRuns: - # Name of the Opensearch index where the pipeline run reports must be stored (default: application-quality-pipeline-run-reports) - #indexReports: - # Username and password of an Opensearch user who has access to the two indices above - #username: - #password: web_server: access_scheme: http