Error logging for analytics tasks

uvdat/core/tasks/analytics/create_road_network.py

@@ -62,17 +62,25 @@ def metadata_for_row(row):
 
 @shared_task
 def create_road_network(result_id):
+    import logging
+
     import osmnx
     from xdg_base_dirs import xdg_cache_home
 
+    logger = logging.getLogger(__name__)
     osmnx.settings.cache_folder = xdg_cache_home() / "osmnx"
-
     result = TaskResult.objects.get(id=result_id)
-    try:
-        location = result.inputs.get("location")
-        if location is None:
-            raise ValueError("location not provided")
 
+    # Input validation
+    location = result.inputs.get("location")
+    if location is None:
+        result.write_error("Location not provided")
+
+    if result.error:
+        result.complete()
+        return
+
+    try:
         result.write_status("Fetching road data via OSMnx...")
         roads = osmnx.graph_from_place(location, network_type="drive")
         road_nodes, road_edges = osmnx.graph_to_gdfs(roads)
@@ -149,6 +157,8 @@ def create_road_network(result_id):
         vector_data.get_summary()
 
         result.outputs = {"roads": dataset.id}
-    except Exception as e:
-        result.error = str(e)
+
+    except Exception:
+        logger.exception()
+        result.error = "An error occurred during this task. See logs for details."
     result.complete()

uvdat/core/tasks/analytics/flood_network_failure.py

@@ -57,111 +57,115 @@ def run_task(self, *, project, **inputs):
 
 @shared_task
 def flood_network_failure(result_id):
+    import logging
+
+    logger = logging.getLogger(__name__)
     result = TaskResult.objects.get(id=result_id)
 
+    # Input validation
+    network = None
+    network_id = result.inputs.get("network")
+    if network_id is None:
+        result.write_error("Network not provided")
+    else:
+        try:
+            network = Network.objects.get(id=network_id)
+        except Network.DoesNotExist:
+            result.write_error("Network not found")
+
+    flood_sim = None
+    flood_sim_id = result.inputs.get("flood_simulation")
+    if flood_sim_id is None:
+        result.write_error("Flood simulation not provided")
+    else:
+        try:
+            flood_sim = TaskResult.objects.get(id=flood_sim_id)
+        except TaskResult.DoesNotExist:
+            result.write_error("Flood simulation not found")
+
+    tolerance = result.inputs.get("depth_tolerance_meters")
+    if tolerance is None:
+        result.write_error("Depth tolerance not provided")
+    else:
+        try:
+            tolerance = float(tolerance)
+        except ValueError:
+            result.write_error("Depth tolerance not valid")
+        if tolerance <= 0:
+            result.write_error("Depth tolerance must be greater than 0")
+
+    radius_meters = result.inputs.get("station_radius_meters")
+    if radius_meters is None:
+        result.write_error("Station radius not provided")
+    else:
+        try:
+            radius_meters = float(radius_meters)
+        except ValueError:
+            result.write_error("Station radius not valid")
+        if radius_meters < 10:
+            # data is at 10 meter resolution
+            result.write_error("Station radius must be greater than 10")
+
+    if result.error:
+        result.complete()
+        return
+
     try:
-        # Verify inputs
-        network = None
-        network_id = result.inputs.get("network")
-        if network_id is None:
-            result.write_error("Network not provided")
-        else:
-            try:
-                network = Network.objects.get(id=network_id)
-            except Network.DoesNotExist:
-                result.write_error("Network not found")
-
-        flood_sim = None
-        flood_sim_id = result.inputs.get("flood_simulation")
-        if flood_sim_id is None:
-            result.write_error("Flood simulation not provided")
-        else:
-            try:
-                flood_sim = TaskResult.objects.get(id=flood_sim_id)
-            except TaskResult.DoesNotExist:
-                result.write_error("Flood simulation not found")
-
-        tolerance = result.inputs.get("depth_tolerance_meters")
-        if tolerance is None:
-            result.write_error("Depth tolerance not provided")
-        else:
-            try:
-                tolerance = float(tolerance)
-            except ValueError:
-                result.write_error("Depth tolerance not valid")
-            if tolerance <= 0:
-                result.write_error("Depth tolerance must be greater than 0")
-
-        radius_meters = result.inputs.get("station_radius_meters")
-        if radius_meters is None:
-            result.write_error("Station radius not provided")
-        else:
-            try:
-                radius_meters = float(radius_meters)
-            except ValueError:
-                result.write_error("Station radius not valid")
-            if radius_meters < 10:
-                # data is at 10 meter resolution
-                result.write_error("Station radius must be greater than 10")
-
-        # Run task
-        if not result.error:
-            # Update name
-            result.name = (
-                f"Failures for Network {network.id} with Flood Result {flood_sim.id}, "
-                f"{tolerance} Tolerance, {radius_meters} Radius"
+        # Update name
+        result.name = (
+            f"Failures for Network {network.id} with Flood Result {flood_sim.id}, "
+            f"{tolerance} Tolerance, {radius_meters} Radius"
+        )
+        result.save()
+
+        n_nodes = network.nodes.count()
+        flood_dataset_id = flood_sim.outputs.get("flood")
+        flood_layer = Layer.objects.get(dataset__id=flood_dataset_id)
+
+        # this uses radius_meters to get a rectangular region, not a circular one
+        def get_station_region(point):
+            earth_radius_meters = 6378000
+            lat_delta = (radius_meters / earth_radius_meters) * (180 / math.pi)
+            lon_delta = (
+                (radius_meters / earth_radius_meters)
+                * (180 / math.pi)
+                / math.cos(point.y * math.pi / 180)
             )
-            result.save()
-
-            n_nodes = network.nodes.count()
-            flood_dataset_id = flood_sim.outputs.get("flood")
-            flood_layer = Layer.objects.get(dataset__id=flood_dataset_id)
-
-            # this uses radius_meters to get a rectangular region, not a circular one
-            def get_station_region(point):
-                earth_radius_meters = 6378000
-                lat_delta = (radius_meters / earth_radius_meters) * (180 / math.pi)
-                lon_delta = (
-                    (radius_meters / earth_radius_meters)
-                    * (180 / math.pi)
-                    / math.cos(point.y * math.pi / 180)
-                )
-                return {
-                    "top": point.y + lat_delta,
-                    "bottom": point.y - lat_delta,
-                    "left": point.x - lon_delta,
-                    "right": point.x + lon_delta,
-                    "units": "EPSG:4326",
-                }
-
-            # Precompute node regions
-            node_regions = {
-                node.id: get_station_region(node.location) for node in network.nodes.all()
+            return {
+                "top": point.y + lat_delta,
+                "bottom": point.y - lat_delta,
+                "left": point.x - lon_delta,
+                "right": point.x + lon_delta,
+                "units": "EPSG:4326",
             }
 
-            # Assume that all frames in flood_layer refer to frames of the same RasterData
-            raster = flood_layer.frames.first().raster
-            raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff)
-            source = tilesource.get_tilesource_from_path(raster_path)
-            metadata = source.getMetadata()
-
-            animation_results = {}
-            node_failures = []
-            for frame in metadata.get("frames", []):
-                frame_index = frame.get("Index")
-                result.write_status(
-                    f"Evaluating flood levels at {n_nodes} nodes for frame {frame_index}..."
+        # Precompute node regions
+        node_regions = {node.id: get_station_region(node.location) for node in network.nodes.all()}
+
+        # Assume that all frames in flood_layer refer to frames of the same RasterData
+        raster = flood_layer.frames.first().raster
+        raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff)
+        source = tilesource.get_tilesource_from_path(raster_path)
+        metadata = source.getMetadata()
+
+        animation_results = {}
+        node_failures = []
+        for frame in metadata.get("frames", []):
+            frame_index = frame.get("Index")
+            result.write_status(
+                f"Evaluating flood levels at {n_nodes} nodes for frame {frame_index}..."
+            )
+            for node_id, node_region in node_regions.items():
+                region_data, _ = source.getRegion(
+                    region=node_region,
+                    frame=frame_index,
+                    format="numpy",
                 )
-                for node_id, node_region in node_regions.items():
-                    region_data, _ = source.getRegion(
-                        region=node_region,
-                        frame=frame_index,
-                        format="numpy",
-                    )
-                    if node_id not in node_failures and np.any(np.where(region_data > tolerance)):
-                        node_failures.append(node_id)
-                animation_results[frame_index] = node_failures.copy()
-            result.outputs = {"failures": animation_results}
-    except Exception as e:
-        result.error = str(e)
+                if node_id not in node_failures and np.any(np.where(region_data > tolerance)):
+                    node_failures.append(node_id)
+            animation_results[frame_index] = node_failures.copy()
+        result.outputs = {"failures": animation_results}
+    except Exception:
+        logger.exception()
+        result.error = "An error occurred during this task. See logs for details."
     result.complete()

uvdat/core/tasks/analytics/flood_simulation.py

@@ -60,25 +60,31 @@ def run_task(self, *, project, **inputs):
 
 @shared_task
 def flood_simulation(result_id):
+    import logging
+
     from uvdat_flood_sim import run_sim, write_multiframe_geotiff
 
+    logger = logging.getLogger(__name__)
     result = TaskResult.objects.get(id=result_id)
 
-    try:
-        for input_key in [
-            "initial_conditions_id",
-            "time_period",
-            "hydrograph",
-            "potential_evapotranspiration_percentile",
-            "soil_moisture_percentile",
-            "ground_water_percentile",
-            "annual_probability",
-        ]:
-            if result.inputs.get(input_key) is None:
-                result.write_error(f"{input_key} not provided")
-                result.complete()
-                return
+    # Input validation
+    for input_key in [
+        "initial_conditions_id",
+        "time_period",
+        "hydrograph",
+        "potential_evapotranspiration_percentile",
+        "soil_moisture_percentile",
+        "ground_water_percentile",
+        "annual_probability",
+    ]:
+        if result.inputs.get(input_key) is None:
+            result.write_error(f"{input_key} not provided")
+
+    if result.error:
+        result.complete()
+        return
 
+    try:
         result.write_status("Interpreting input values")
         initial_conditions_id = result.inputs.get("initial_conditions_id")
         time_period = result.inputs.get("time_period")
@@ -209,6 +215,8 @@ def flood_simulation(result_id):
             )
 
             result.outputs = {"flood": dataset.id}
-    except Exception as e:
-        result.error = str(e)
+
+    except Exception:
+        logger.exception()
+        result.error = "An error occurred during this task. See logs for details."
     result.complete()