feat: Cleanup of stale sensor references (#2106)

* feat: implement cleanup of stale sensor references in assets on sensor deletion

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* feat: streamline cleanup of sensor references in assets

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* feat: replace cleanup of sensor references with delete_sensor function for improved data integrity

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* feat: enhance sensor reference cleanup with detailed audit logging for improved traceability

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* Update flexmeasures/data/services/sensors.py

Co-authored-by: Nicolas Höning <nicolas@seita.nl>
Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com>

* Update flexmeasures/api/v3_0/tests/test_sensors_api.py

Co-authored-by: Nicolas Höning <nicolas@seita.nl>
Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com>

* feat: enhance sensor reference pruning functions for improved JSON handling and clarity

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* feat: clarify audit log messages for deleted sensor references

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* docs(sensors): fix doctest examples for prune helpers

Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/0ffebf72-a172-44fc-8399-423b45337462

Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com>

* feat: update variable names for clarity in sensor reference cleanup

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

* chore: add changelog entry

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>

---------

Signed-off-by: joshuaunity <oghenerobojosh01@gmail.com>
Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com>
Co-authored-by: Nicolas Höning <nicolas@seita.nl>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: 3 people

documentation/changelog.rst

@@ -44,6 +44,7 @@ New features
 * Separate the ``StorageScheduler``'s tie-breaking preference for a full :abbr:`SoC (state of charge)` from its reported energy costs [see `PR #2023 <https://www.github.com/FlexMeasures/flexmeasures/pull/2023>`_ and `PR #2108 <https://www.github.com/FlexMeasures/flexmeasures/pull/2108>`_]
 * Improve asset graph hover interaction with a vertical ruler across subcharts, while keeping hover dots for easier visual tracking [see `PR #2079 <https://www.github.com/FlexMeasures/flexmeasures/pull/2079>`_]
 * Improve asset audit log messages for JSON field edits (especially ``sensors_to_show`` and nested flex-config values) [see `PR #2055 <https://www.github.com/FlexMeasures/flexmeasures/pull/2055>`_]
+* Clean up stale sensor references from ``flex-config`` and ``sensors_to_show`` when deleting a sensor, using JSONB queries to find affected assets before pruning those references [see `PR #2106 <https://www.github.com/FlexMeasures/flexmeasures/pull/2106>`_]
 
 Infrastructure / Support
 ----------------------

flexmeasures/api/v3_0/sensors.py

@@ -65,6 +65,7 @@
 from flexmeasures.data.schemas.scheduling import GetScheduleSchema
 from flexmeasures.data.schemas.units import UnitField
 from flexmeasures.data.services.sensors import get_sensor_stats
+from flexmeasures.data.services.sensors import delete_sensor as delete_sensor_and_data
 from flexmeasures.data.services.scheduling import (
     create_scheduling_job,
     get_data_source_for_job,
@@ -1350,19 +1351,8 @@ def delete(self, id: int, sensor: Sensor):
             - Sensors
         """
 
-        """Delete time series data."""
-        db.session.execute(delete(TimedBelief).filter_by(sensor_id=sensor.id))
-
-        AssetAuditLog.add_record(
-            sensor.generic_asset, f"Deleted sensor '{sensor.name}': {sensor.id}"
-        )
-
         sensor_name = sensor.name
-        AssetAuditLog.add_record(
-            sensor.generic_asset,
-            f"Deleted sensor '{sensor_name}': {id}",
-        )
-        db.session.execute(delete(Sensor).filter_by(id=sensor.id))
+        delete_sensor_and_data(sensor)
         db.session.commit()
         current_app.logger.info("Deleted sensor '%s'." % sensor_name)
         return {}, 204

flexmeasures/api/v3_0/tests/test_sensors_api.py

@@ -3,6 +3,7 @@
 import pytest
 import math
 import io
+import json
 
 from flask import url_for
 from sqlalchemy import select, func
@@ -536,6 +537,29 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
     existing_sensor_id = existing_sensor.id
     sensor_count = db.session.scalar(select(func.count()).select_from(Sensor))
 
+    asset = existing_sensor.generic_asset
+    asset.flex_model = {
+        "soc-max": {"sensor": existing_sensor_id},
+        "static-limit": "10 kW",
+    }
+    asset.flex_context = {
+        "consumption-price": {"sensor": existing_sensor_id},
+        "inflexible-device-sensors": [existing_sensor_id],
+    }
+    asset.sensors_to_show = [
+        {"title": "Power", "plots": [{"sensor": existing_sensor_id}]},
+        existing_sensor_id,
+    ]
+    asset.sensors_to_show_as_kpis = [
+        {
+            "sensor": existing_sensor_id,
+            "title": "Temperature KPI",
+            "function": "sum",
+        }
+    ]
+    db.session.add(asset)
+    db.session.commit()
+
     delete_sensor_response = client.delete(
         url_for("SensorAPI:delete", id=existing_sensor_id),
     )
@@ -552,12 +576,46 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
         db.session.scalar(select(func.count()).select_from(Sensor)) == sensor_count - 1
     )
 
+    asset_after = db.session.get(GenericAsset, asset.id)
+    assert asset_after.flex_model.get("soc-max") is None
+    assert asset_after.flex_model.get("static-limit") == "10 kW"
+    assert asset_after.flex_context.get("consumption-price") is None
+    assert asset_after.flex_context.get("inflexible-device-sensors") == []
+    assert str(existing_sensor_id) not in json.dumps(asset_after.sensors_to_show)
+    assert str(existing_sensor_id) not in json.dumps(
+        asset_after.sensors_to_show_as_kpis
+    )
+
     check_audit_log_event(
         db=db,
         event=f"Deleted sensor '{existing_sensor.name}': {existing_sensor.id}",
         user=requesting_user,
         asset=existing_sensor.generic_asset,
     )
+    check_audit_log_event(
+        db=db,
+        event=f"Removed sensor reference '{existing_sensor.name}': {existing_sensor.id} from flex-model (because sensor has been deleted).",
+        user=requesting_user,
+        asset=existing_sensor.generic_asset,
+    )
+    check_audit_log_event(
+        db=db,
+        event=f"Removed sensor reference '{existing_sensor.name}': {existing_sensor.id} from flex-context (because sensor has been deleted).",
+        user=requesting_user,
+        asset=existing_sensor.generic_asset,
+    )
+    check_audit_log_event(
+        db=db,
+        event=f"Removed sensor reference '{existing_sensor.name}': {existing_sensor.id} from sensors-to-show (because sensor has been deleted).",
+        user=requesting_user,
+        asset=existing_sensor.generic_asset,
+    )
+    check_audit_log_event(
+        db=db,
+        event=f"Removed sensor reference '{existing_sensor.name}': {existing_sensor.id} from sensors-to-show-as-kpis (because sensor has been deleted).",
+        user=requesting_user,
+        asset=existing_sensor.generic_asset,
+    )
 
 
 @pytest.mark.parametrize(

flexmeasures/cli/data_delete.py

@@ -27,6 +27,7 @@
     SourceIdField,
 )
 from flexmeasures.data.services.users import find_user_by_email, delete_user
+from flexmeasures.data.services.sensors import delete_sensor as delete_sensor_and_data
 from flexmeasures.cli.utils import (
     abort,
     done,
@@ -539,18 +540,17 @@ def delete_sensor(
     sensors: list[Sensor],
 ):
     """Delete sensors and their (time series) data."""
-    n = delete(TimedBelief).where(
-        TimedBelief.sensor_id.in_(sensor.id for sensor in sensors)
-    )
-    statements = []
-    for sensor in sensors:
-        statements.append(delete(Sensor).filter_by(id=sensor.id))
+    n_beliefs = db.session.execute(
+        select(func.count())
+        .select_from(TimedBelief)
+        .where(TimedBelief.sensor_id.in_([sensor.id for sensor in sensors]))
+    ).scalar_one()
     click.confirm(
-        f"Delete {', '.join(sensor.__repr__() for sensor in sensors)}, along with {n} beliefs?",
+        f"Delete {', '.join(sensor.__repr__() for sensor in sensors)}, along with {n_beliefs} beliefs?",
         abort=True,
     )
-    for statement in statements:
-        db.session.execute(statement)
+    for sensor in sensors:
+        delete_sensor_and_data(sensor)
     db.session.commit()