Merge pull request #38 from DataKitchen/release/4.0.9

Release/4.0.9

Author: datakitchen-devops

README.md

@@ -135,6 +135,9 @@ export TG_DECRYPT_SALT=<encryption_salt>
 export TESTGEN_USERNAME=<username>
 export TESTGEN_PASSWORD=<password>
 
+# Set an arbitrary base64-encoded string to be used for signing authentication tokens
+export TG_JWT_HASHING_KEY=<base64_key>
+
 # Set an accessible path for storing application logs
 export TESTGEN_LOG_FILE_PATH=<path_for_logs>
 ```
@@ -149,12 +152,12 @@ Make sure the PostgreSQL database server is up and running. Initialize the appli
 testgen setup-system-db --yes
 ```
 
-### Run the TestGen UI
+### Run the application modules
 
-Run the following command to start the TestGen UI. It will open the browser at [http://localhost:8501](http://localhost:8501).
+Run the following command to start TestGen. It will open the browser at [http://localhost:8501](http://localhost:8501).
 
 ```shell
-testgen ui run
+testgen run-app
 ```
 
 Verify that you can login to the UI with the `TESTGEN_USERNAME` and `TESTGEN_PASSWORD` values that you configured in the environment variables.

deploy/charts/testgen-app/templates/_environment.yaml

@@ -9,6 +9,11 @@
     secretKeyRef:
       name: {{ .Values.testgen.authSecrets.name | quote }}
       key: "decrypt-password"
+- name: TG_JWT_HASHING_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.testgen.authSecrets.name | quote }}
+      key: "jwt-hashing-key"
 - name: TG_METADATA_DB_HOST
   value: {{ .Values.testgen.databaseHost | quote }}
 - name: TG_METADATA_DB_NAME

deploy/charts/testgen-app/templates/secrets.yaml

@@ -12,4 +12,5 @@ type: Opaque
 data:
   decrypt-salt: {{ randAlphaNum 32 | b64enc | quote }}
   decrypt-password: {{ randAlphaNum 32 | b64enc | quote }}
+  jwt-hashing-key: {{ randBytes 32 | b64enc | quote }}
 {{- end }}

deploy/testgen.dockerfile

@@ -25,11 +25,10 @@ RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/s
 ENV TESTGEN_VERSION=${TESTGEN_VERSION}
 ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO}
 ENV TG_RELEASE_CHECK=docker
-ENV STREAMLIT_SERVER_MAX_UPLOAD_SIZE=200
 
 USER testgen
 
 WORKDIR /dk
 
 ENTRYPOINT ["testgen"]
-CMD [ "ui", "run" ]
+CMD [ "run-app" ]

docs/local_development.md

@@ -63,6 +63,8 @@ Create a `local.env` file with the following environment variables, replacing th
 ```shell
 export TESTGEN_DEBUG=yes
 export TESTGEN_LOG_TO_FILE=no
+export TG_ANALYTICS=no
+export TG_JWT_HASHING_KEY=<base64_key>
 export TESTGEN_USERNAME=<username>
 export TESTGEN_PASSWORD=<password>
 export TG_DECRYPT_SALT=<decrypt_salt>
@@ -98,8 +100,24 @@ testgen run-tests --project-key DEFAULT --test-suite-key default-suite-1
 testgen quick-start --simulate-fast-forward
 ```
 
-### Run Streamlit
-Run the local Streamlit-based TestGen application. It will open the browser at [http://localhost:8501](http://localhost:8501).
+### Run the Application
+
+TestGen has two modules that have to be running: The web user interface (UI) and the Scheduler.
+The scheduler starts jobs (profiling, test execution, ...) at their scheduled times.
+
+The following command starts both modules, each in their own process:
+
+```shell
+testgen run-app
+```
+
+Alternatively, you can run each individually:
+
+
+```shell
+testgen run-app ui
+```
+
 ```shell
-testgen ui run
+testgen run-app scheduler
 ```

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "3.7.9"
+version = "4.0.9"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "[email protected]" },
@@ -38,17 +38,12 @@ dependencies = [
     "pycryptodome==3.21",
     "prettytable==3.7.0",
     "requests_extensions==1.1.3",
-    "bz2file==0.98",
-    "trogon==0.4.0",
     "numpy==1.26.4",
     "pandas==2.1.4",
-    "streamlit==1.38.0",
+    "streamlit==1.44.1",
     "streamlit-extras==0.3.0",
     "streamlit-aggrid==0.3.4.post3",
-    "streamlit-antd-components==0.2.2",
-    "streamlit-plotly-events==0.0.6",
     "plotly_express==0.4.1",
-    "streamlit-option-menu==0.3.6",
     "streamlit-authenticator==0.2.3",
     "streamlit-javascript==0.1.5",
     "progress==1.6",
@@ -62,6 +57,7 @@ dependencies = [
     "reportlab==4.2.2",
     "pydantic==1.10.13",
     "streamlit-pydantic==0.6.0",
+    "cron-converter==1.2.1",
 
     # Pinned to match the manually compiled libs or for security
     "pyarrow==18.1.0",
@@ -245,7 +241,7 @@ ignore = ["TRY003", "S608", "S404", "F841", "B023"]
 "__init__.py" = ["F403"]
 "testgen/__main__.py" = ["ARG001", "S603"]
 "tasks.py" = ["F403"]
-"tests*" = ["S101", "T201"]
+"tests*" = ["S101", "T201", "ARG001"]
 "invocations/**" = ["ARG001", "T201"]
 "testgen/common/encrypt.py" = ["S413"]
 "testgen/ui/pdf/dk_logo.py" = ["T201"]

testgen/__main__.py

@@ -1,14 +1,13 @@
 import logging
 import os
+import signal
 import subprocess
 import sys
-import typing
 from dataclasses import dataclass, field
 
 import click
 from click.core import Context
 from progress.spinner import MoonSpinner
-from trogon import tui
 
 from testgen import settings
 from testgen.commands.run_execute_tests import run_execution_steps
@@ -40,14 +39,16 @@
     get_tg_db,
     get_tg_host,
     get_tg_schema,
-    logs,
     version_service,
 )
+from testgen.scheduler import register_scheduler_job, run_scheduler
 from testgen.ui.queries import profiling_run_queries, test_run_queries
 from testgen.utils import plugins
 
 LOG = logging.getLogger("testgen")
 
+APP_MODULES = ["ui", "scheduler"]
+
 
 @dataclass
 class Configuration:
@@ -58,8 +59,16 @@ class Configuration:
 pass_configuration = click.make_pass_decorator(Configuration)
 
 
-@tui()
+class CliGroup(click.Group):
+    def invoke(self, ctx: Context):
+        try:
+            super().invoke(ctx)
+        except Exception:
+            LOG.exception("There was an unexpected error")
+
+
 @click.group(
+    cls=CliGroup,
     help=f"This version: {settings.VERSION} \n\nLatest version: {version_service.get_latest_version()} \n\nSchema revision: {get_schema_revision()}"
 )
 @click.option(
@@ -83,7 +92,7 @@ def cli(ctx: Context, verbose: bool):
         sys.exit(1)
 
     if (
-        ctx.invoked_subcommand not in ["ui", "tui", "setup-system-db", "upgrade-system-version", "quick-start"]
+        ctx.invoked_subcommand not in ["run-app", "ui", "setup-system-db", "upgrade-system-version", "quick-start"]
         and not is_db_revision_up_to_date()
     ):
         click.secho("The system database schema is outdated. Automatically running the following command:", fg="red")
@@ -93,6 +102,7 @@ def cli(ctx: Context, verbose: bool):
     LOG.debug("Current Step: Main Program")
 
 
+@register_scheduler_job
 @cli.command("run-profile", help="Generates a new profile of the table group.")
 @pass_configuration
 @click.option(
@@ -143,6 +153,7 @@ def run_test_generation(configuration: Configuration, table_group_id: str, test_
     click.echo("\n" + message)
 
 
+@register_scheduler_job
 @cli.command("run-tests", help="Performs tests defined for a test suite.")
 @click.option(
     "-pk",
@@ -590,19 +601,19 @@ def list_table_groups(configuration: Configuration, project_key: str, display: b
 def ui(): ...
 
 
-@ui.command("run", help="Run the browser application with default settings")
-@click.option("-d", "--debug", is_flag=True, default=False)
-def run(debug: bool):
+@ui.command("plugins", help="List installed application plugins")
+def list_ui_plugins():
+    installed_plugins = list(plugins.discover())
+
+    click.echo(click.style(len(installed_plugins), fg="bright_magenta") + click.style(" plugins installed", bold=True))
+    for plugin in installed_plugins:
+        click.echo(click.style(" + ", fg="bright_green") + f"{plugin.package: <30}" + f"\tversion: {plugin.version}")
+
+
+def run_ui():
     from testgen.ui.scripts import patch_streamlit
-    configure_logging(
-        level=logging.INFO,
-        log_format="%(message)s",
-    )
 
     status_code: int = -1
-    logger = logging.getLogger("testgen")
-    stderr: typing.TextIO = typing.cast(typing.TextIO, logs.LogPipe(logger, logging.INFO))
-    stdout: typing.TextIO = typing.cast(typing.TextIO, logs.LogPipe(logger, logging.INFO))
 
     use_ssl = os.path.isfile(settings.SSL_CERT_FILE) and os.path.isfile(settings.SSL_KEY_FILE)
 
@@ -621,31 +632,49 @@ def run(debug: bool):
                 "run",
                 app_file,
                 "--browser.gatherUsageStats=false",
+                "--client.showErrorDetails=none",
+                "--client.toolbarMode=minimal",
                 f"--server.sslCertFile={settings.SSL_CERT_FILE}" if use_ssl else "",
                 f"--server.sslKeyFile={settings.SSL_KEY_FILE}" if use_ssl else "",
                 "--",
-                f"{'--debug' if debug else ''}",
+                f"{'--debug' if settings.IS_DEBUG else ''}",
             ],
-            stdout=stdout,
-            stderr=stderr,
+            env={**os.environ, "TG_JOB_SOURCE": "UI"}
         )
     except Exception:
         LOG.exception(f"Testgen UI exited with status code {status_code}")
-        raise
-    finally:
-        if stderr:
-            stderr.close()
-        if stdout:
-            stdout.close()
 
 
-@ui.command("plugins", help="List installed application plugins")
-def list_ui_plugins():
-    installed_plugins = list(plugins.discover())
+@cli.command("run-app", help="Runs TestGen's application modules")
+@click.argument(
+    "module",
+    type=click.Choice(["all", *APP_MODULES]),
+    default="all",
+)
+def run_app(module):
 
-    click.echo(click.style(len(installed_plugins), fg="bright_magenta") + click.style(" plugins installed", bold=True))
-    for plugin in installed_plugins:
-        click.echo(click.style(" + ", fg="bright_green") + f"{plugin.package: <30}" + f"\tversion: {plugin.version}")
+    match module:
+        case "ui":
+            run_ui()
+
+        case "scheduler":
+            run_scheduler()
+
+        case "all":
+            children = [
+                subprocess.Popen([sys.executable, sys.argv[0], "run-app", m], start_new_session=True)
+                for m in APP_MODULES
+            ]
+
+            def term_children(signum, _):
+                for child in children:
+                    child.send_signal(signum)
+
+            signal.signal(signal.SIGINT, term_children)
+            signal.signal(signal.SIGTERM, term_children)
+
+            for child in children:
+                child.wait()
 
 
 if __name__ == "__main__":

testgen/commands/run_execute_cat_tests.py

@@ -1,5 +1,7 @@
 import logging
+from datetime import UTC, datetime
 
+from testgen import settings
 from testgen.commands.queries.execute_cat_tests_query import CCATExecutionSQL
 from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results
 from testgen.common import (
@@ -9,6 +11,7 @@
     WriteListToDB,
     date_service,
 )
+from testgen.common.mixpanel_service import MixpanelService
 
 LOG = logging.getLogger("testgen")
 
@@ -61,17 +64,35 @@ def ParseCATResults(clsCATExecute):
 
 
 def FinalizeTestRun(clsCATExecute: CCATExecutionSQL):
-    lstQueries = [clsCATExecute.FinalizeTestResultsSQL(),
-                  clsCATExecute.PushTestRunStatusUpdateSQL(),
-                  clsCATExecute.FinalizeTestSuiteUpdateSQL(),
-                  clsCATExecute.CalcPrevalenceTestResultsSQL(),
-                  clsCATExecute.TestScoringRollupRunSQL(),
-                  clsCATExecute.TestScoringRollupTableGroupSQL()]
-    RunActionQueryList(("DKTG"), lstQueries)
-    run_refresh_score_cards_results(
-        project_code=clsCATExecute.project_code,
-        add_history_entry=True,
-        refresh_date=date_service.parse_now(clsCATExecute.run_date),
+    _, row_counts = RunActionQueryList(("DKTG"), [
+        clsCATExecute.FinalizeTestResultsSQL(),
+        clsCATExecute.PushTestRunStatusUpdateSQL(),
+        clsCATExecute.FinalizeTestSuiteUpdateSQL(),
+    ])
+    end_time = datetime.now(UTC)
+    
+    try:
+        RunActionQueryList(("DKTG"), [
+            clsCATExecute.CalcPrevalenceTestResultsSQL(),
+            clsCATExecute.TestScoringRollupRunSQL(),
+            clsCATExecute.TestScoringRollupTableGroupSQL(),
+        ])
+        run_refresh_score_cards_results(
+            project_code=clsCATExecute.project_code,
+            add_history_entry=True,
+            refresh_date=date_service.parse_now(clsCATExecute.run_date),
+        )
+    except Exception:
+        LOG.exception("Error refreshing scores after test run")
+        pass
+
+    MixpanelService().send_event(
+        "run-tests",
+        source=settings.ANALYTICS_JOB_SOURCE,
+        sql_flavor=clsCATExecute.flavor,
+        test_count=row_counts[0],
+        run_duration=(end_time - date_service.parse_now(clsCATExecute.run_date)).total_seconds(),
+        scoring_duration=(datetime.now(UTC) - end_time).total_seconds(),
     )
 
 

testgen/commands/run_execute_tests.py

@@ -103,10 +103,7 @@ def run_execution_steps_in_background(project_code, test_suite):
         empty_cache()
         background_thread = threading.Thread(
             target=run_execution_steps,
-            args=(
-                project_code,
-                test_suite
-            ),
+            args=(project_code, test_suite),
         )
         background_thread.start()
     else:
@@ -115,7 +112,12 @@ def run_execution_steps_in_background(project_code, test_suite):
         subprocess.Popen(script)  # NOQA S603
 
 
-def run_execution_steps(project_code: str, test_suite: str, minutes_offset: int=0, spinner: Spinner=None) -> str:
+def run_execution_steps(
+    project_code: str,
+    test_suite: str,
+    minutes_offset: int=0,
+    spinner: Spinner=None,
+) -> str:
     # Initialize required parms for all steps
     has_errors = False
     error_msg = ""