Merge pull request #144 from geoadmin/develop

New Release v0.20.0 - #minor

Author: ltshb

.pylintrc

@@ -181,6 +181,9 @@ missing-member-max-choices=1
 # List of decorators that change the signature of a decorated function.
 signature-mutators=
 
+# enable certain c-extensions
+extension-pkg-allow-list=lxml
+
 
 [FORMAT]
 

Makefile

@@ -60,10 +60,16 @@ ci:
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
+env:
+ifeq ("$(wildcard ./.env)","")
+	@echo ".env file not found, copying it from .env.default..."
+else
+	@echo ".env file found, leaving untouched..."
+endif
+
 .PHONY: setup
-setup: $(SETTINGS_TIMESTAMP) ## Create virtualenv with all packages for development
+setup: $(SETTINGS_TIMESTAMP) env ## Create virtualenv with all packages for development
 	pipenv install --dev
-	cp .env.default .env
 	pipenv shell
 
 .PHONY: format

Pipfile

@@ -27,6 +27,9 @@ opentelemetry-instrumentation-django = "0.60b1"
 opentelemetry-instrumentation-logging = "0.60b1"
 opentelemetry-instrumentation-psycopg = "0.60b1"
 opentelemetry-instrumentation-system-metrics = "0.60b1"
+dyntastic = "~=0.18.0"
+lxml = "~=6.0"
+rdflib = "~=7.6"
 
 [dev-packages]
 yapf = "*"
@@ -45,13 +48,14 @@ mypy = "*"
 debugpy = "*"
 bandit = "*"
 django-stubs = "~=5.2"
-boto3-stubs = {extras = ["cognito-idp"], version = "~=1.37"}
+boto3-stubs = {extras = ["cognito-idp","dynamodb"], version = "~=1.37"}
 pytest-xdist = "*"
 pytest-cov = "*"
 types-nanoid = "~=2.0"
 types-gevent = "~=25.4"
 types-requests = "~=2.32"
 types-gunicorn = "~=23.0"
+lxml-stubs = "~=0.5"
 
 [requires]
 python_version = "3.12"

Pipfile.lock

@@ -24,6 +24,7 @@ class DatasetAdmin(admin.ModelAdmin):  # type:ignore[type-arg]
 
     list_display = ('dataset_id', 'title_en', 'provider')
     list_filter = (('provider', admin.RelatedOnlyFieldListFilter),)
+    search_fields = ('dataset_id',)
     readonly_fields = ('created', 'updated')
 
     def get_form(

app/bod/management/commands/bod_sync.py

@@ -0,0 +1,149 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class BaseModelWithDynamoDBSerialization(BaseModel):
+    """BaseModel with custom serialization for DynamoDB"""
+
+    def as_dynamodb_item(self) -> dict[str, Any]:
+        """Convert the dataset to a DynamoDB item format
+
+        Returns:
+            dict[str, Any]: The dataset represented as a DynamoDB item.
+        """
+
+        def serialize(value: Any) -> dict[str, Any]:
+            if value is None:
+                return {"NULL": True}
+            if isinstance(value, int):
+                return {"N": str(value)}
+            if isinstance(value, str):
+                return {"S": value}
+            if isinstance(value, list):
+                return {"L": [serialize(i) for i in value]}
+            if isinstance(value, dict):
+                return {"M": {k: serialize(v) for k, v in value.items()}}
+            raise ValueError(f"Unexpected type {type(value)}")
+
+        item = self.model_dump(mode="json")
+        return {key: serialize(value) for key, value in item.items()}
+
+
+class ExportDataset(BaseModelWithDynamoDBSerialization):
+    dataset_id: str
+    title_de: str
+    title_fr: str
+    title_en: str
+    title_it: str | None
+    title_rm: str | None
+    description_de: str
+    description_fr: str
+    description_en: str
+    description_it: str | None
+    description_rm: str | None
+    attribution: list[str]
+    provider: list[str]
+    created: str
+    updated: str
+    geocat_id: str
+    _legacy_id: int
+
+
+class ExportProvider(BaseModelWithDynamoDBSerialization):
+    provider_id: str
+    created: str
+    updated: str
+    name_de: str
+    name_fr: str
+    name_en: str
+    name_it: str | None
+    name_rm: str | None
+    acronym_de: str
+    acronym_fr: str
+    acronym_en: str
+    acronym_it: str | None
+    acronym_rm: str | None
+    _legacy_id: int
+
+
+class Keyword(BaseModel):
+    type: str | None
+    thesaurus: str | None
+    concept: str | None
+    translation_de: str | None
+    translation_fr: str | None
+    translation_en: str | None
+    translation_it: str | None
+    translation_rm: str | None
+
+
+class KeywordList(BaseModelWithDynamoDBSerialization):
+    dataset_id: str
+    geocat_id: str
+    keywords: list[Keyword]
+
+
+class OnlineResource(BaseModel):
+    url: str | None
+    url_de: str | None
+    url_fr: str | None
+    url_en: str | None
+    url_it: str | None
+    url_rm: str | None
+    protocol: str | None
+    name_de: str | None
+    name_fr: str | None
+    name_en: str | None
+    name_it: str | None
+    name_rm: str | None
+    description_de: str | None
+    description_fr: str | None
+    description_en: str | None
+    description_it: str | None
+    description_rm: str | None
+
+
+class Contact(BaseModel):
+    role: str | None
+    org_name: str | None
+    org_name_de: str | None
+    org_name_fr: str | None
+    org_name_en: str | None
+    org_name_it: str | None
+    org_name_rm: str | None
+    org_acronym: str | None
+    org_acronym_de: str | None
+    org_acronym_fr: str | None
+    org_acronym_en: str | None
+    org_acronym_it: str | None
+    org_acronym_rm: str | None
+    org_email: str | None
+    position_name_de: str | None
+    position_name_fr: str | None
+    position_name_en: str | None
+    position_name_it: str | None
+    position_name_rm: str | None
+    individual_name: str | None
+    individual_first_name: str | None
+    individual_last_name: str | None
+    contact_direct_number: str | None
+    contact_voice: str | None
+    contact_facsimile: str | None
+    contact_city: str | None
+    contact_administrative_area: str | None
+    contact_postal_code: str | None
+    contact_country: str | None
+    contact_electronic_mail_address: str | None
+    contact_street_name: str | None
+    contact_street_number: str | None
+    contact_post_box: str | None
+    hours_of_service: str | None
+    contact_instructions: str | None
+    online_resources: list[OnlineResource]
+
+
+class ContactList(BaseModelWithDynamoDBSerialization):
+    dataset_id: str
+    geocat_id: str
+    contacts: list[Contact]

app/distributions/admin.py

@@ -0,0 +1,133 @@
+import json
+from typing import Any
+
+import boto3
+import environ
+from distributions.export_models import ExportDataset
+from distributions.export_models import ExportProvider
+from distributions.models import Dataset
+from provider.models import Provider
+from utils.command import CustomBaseCommand
+
+from django.core import serializers
+from django.core.management.base import CommandParser
+
+env = environ.Env()
+boto3.setup_default_session(profile_name="swisstopo-swissgeo-dev")
+
+SAMPLE_IDS = [
+    "ch.bafu.schutzgebiete-luftfahrt",
+    "ch.swisstopo.lubis-luftbilder-dritte-kantone",
+    "ch.bav.sachplan-infrastruktur-schiene_anhorung",
+    "ch.agroscope.korridore-feuchtgebietsarten_qualitaet",
+    "ch.meteoschweiz.messwerte-pollen-buche-1h",
+]
+
+
+class Command(CustomBaseCommand):
+    help = "Exports datasets from the system to DynamoDB"
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--clear",
+            action="store_true",
+            help="Delete existing objects before importing",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Dry run, abort transaction in the end",
+        )
+        parser.add_argument(
+            "--sample",
+            action="store_true",
+            help="Export a sample of datasets (10) instead of all datasets",
+        )
+        parser.add_argument(
+            "--providers",
+            action="store_true",
+            help="Import providers",
+        )
+        parser.add_argument(
+            "--attributions",
+            action="store_true",
+            help="Import attributions",
+        )
+        parser.add_argument(
+            "--datasets",
+            action="store_true",
+            help="Import datasets",
+        )
+        parser.add_argument(
+            "--target-env",
+            type=str,
+            choices=["dev", "int", "prod"],
+            default="dev",
+            help="Specify the target environment",
+        )
+        parser.add_argument(
+            "--profile-name",
+            type=str,
+            nargs="?",
+            default=None,
+            help="Specify the profile name (only needed locally)",
+        )
+        parser.add_argument(
+            "--debug",
+            action="store_true",
+            help="Enable debug logging",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        """Main entry point of command."""
+        if options["profile_name"]:
+            self.session = boto3.Session(profile_name=options["profile_name"])  # pylint: disable=attribute-defined-outside-init
+        else:
+            self.session = boto3.Session()  # pylint: disable=attribute-defined-outside-init
+
+        if options["datasets"]:
+            self.export_datasets(*args, **options)
+        if options["providers"]:
+            self.export_providers(*args, **options)
+
+    def export_datasets(self, *args: Any, **options: Any) -> None:
+
+        dynamodb_client = self.session.client("dynamodb", region_name="eu-central-1")
+
+        if options["sample"]:
+            self.print("Exporting only a sample of datasets (10)...")
+            qs = Dataset.objects.filter(dataset_id__in=SAMPLE_IDS)
+        else:
+            qs = Dataset.objects.all()
+        datasets = json.loads(
+            serializers.serialize(
+                "json", qs, use_natural_foreign_keys=True, use_natural_primary_keys=True
+            )
+        )
+
+        for dataset in datasets:
+            exp_item = ExportDataset(**dataset["fields"])
+            item = exp_item.as_dynamodb_item()
+            self.print(f"Exporting dataset {exp_item.dataset_id} to DynamoDB")
+            dynamodb_client.put_item(
+                TableName=f"harvest-datasets-{options['target_env']}", Item=item
+            )
+
+    def export_providers(self, *args: Any, **options: Any) -> None:
+        dynamodb_client = self.session.client("dynamodb", region_name="eu-central-1")
+
+        qs = Provider.objects.all()
+        providers = json.loads(
+            serializers.serialize(
+                "json", qs, use_natural_foreign_keys=True, use_natural_primary_keys=True
+            )
+        )
+
+        for provider in providers:
+            exp_item = ExportProvider(**provider["fields"])
+            item = exp_item.as_dynamodb_item()
+            self.print(f"Exporting provider {exp_item.provider_id} to DynamoDB")
+            dynamodb_client.put_item(
+                TableName=f"harvest-providers-{options['target_env']}", Item=item
+            )