Refine DE/ZA fusion converters per review feedback

- Use crop extension for crop fields
- Remove unnecessary reprojection and area conversion
- Rely on framework handling for extra columns

Author: avis-giri

fiboa_cli/datasets/de_fusion.py

@@ -1,6 +1,9 @@
 from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter
 
 
+CROP_EXTENSION = "https://fiboa.org/crop-extension/v0.2.0/schema.yaml"
+
+
 class DeFusionConverter(FiboaBaseConverter):
     sources = {
         "https://data.source.coop/esa/fusion-competition/br-17E-243N-crop-labels-test-2019.geojson": "de_test_2019.geojson",
@@ -17,6 +20,8 @@ class DeFusionConverter(FiboaBaseConverter):
     provider = "ESA Fusion Competition via Source Cooperative <https://source.coop/esa/fusion-competition>"
     attribution = "https://data.source.coop/esa/fusion-competition"
     license = "CC-BY-4.0"
+    extensions = {CROP_EXTENSION}
+    area_is_in_ha = False
     columns = {
         "geometry": "geometry",
         "id": "id",
@@ -27,12 +32,6 @@ class DeFusionConverter(FiboaBaseConverter):
     column_additions = {
         "determination:datetime": "2019-01-01T00:00:00Z",
     }
-    missing_schemas = {
-        "properties": {
-            "crop:code": {"type": "uint16"},
-            "crop:name": {"type": "string"},
-        }
-    }
 
     def _normalize_geojson_properties(self, feature):
         # These GeoJSON files have no top-level feature id, only fid inside properties.
@@ -42,12 +41,4 @@ def _normalize_geojson_properties(self, feature):
         return feature
 
     def migrate(self, gdf):
-        # Reproject from EPSG:25833 to WGS84 as required by fiboa spec
-        if gdf.crs is not None and gdf.crs.to_epsg() != 4326:
-            gdf = gdf.to_crs("EPSG:4326")
-
-        # Convert area from m² to hectares
-        gdf["SHAPE_AREA"] = gdf["SHAPE_AREA"] / 10000
-
         return super().migrate(gdf)
-

fiboa_cli/datasets/de_fusion_ml.py

@@ -16,11 +16,6 @@ def file_migration(self, gdf, path, uri, layer=None):
     def migrate(self, gdf):
         # Assign split from temp marker
         gdf["split"] = gdf["_source_split"].astype(object)
-
-        # Build unique IDs from split + fid before dropping the temp marker.
-        # Using split as prefix avoids collisions between train/test files
-        # since both files contain overlapping fid values.
+        # Build unique IDs from split + fid to avoid collisions between files
         gdf["id"] = gdf["_source_split"] + "_" + gdf["fid"].astype(str)
-
-        gdf = gdf.drop(columns=["_source_split"])
         return super().migrate(gdf)

fiboa_cli/datasets/za_fusion.py

@@ -1,6 +1,9 @@
 from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter
 
 
+CROP_EXTENSION = "https://fiboa.org/crop-extension/v0.2.0/schema.yaml"
+
+
 class ZaFusionConverter(FiboaBaseConverter):
     sources = {
         "https://data.source.coop/esa/fusion-competition/sa-19E-258N-crop-labels-train-2017.geojson": "za_train_258N.geojson",
@@ -18,6 +21,8 @@ class ZaFusionConverter(FiboaBaseConverter):
     provider = "ESA Fusion Competition via Source Cooperative <https://source.coop/esa/fusion-competition>"
     attribution = "https://data.source.coop/esa/fusion-competition"
     license = "CC-BY-4.0"
+    extensions = {CROP_EXTENSION}
+    area_is_in_ha = False
     columns = {
         "geometry": "geometry",
         "id": "id",
@@ -28,25 +33,11 @@ class ZaFusionConverter(FiboaBaseConverter):
     column_additions = {
         "determination:datetime": "2017-01-01T00:00:00Z",
     }
-    missing_schemas = {
-        "properties": {
-            "crop:code": {"type": "uint16"},
-            "crop:name": {"type": "string"},
-        }
-    }
 
     def _normalize_geojson_properties(self, feature):
-        # These GeoJSON files have no top-level feature id, only fid inside properties.
         if "id" not in feature["properties"]:
             feature["properties"]["id"] = feature["properties"].get("fid", None)
         return feature
 
     def migrate(self, gdf):
-        # Reproject from EPSG:32734 to WGS84 as required by fiboa spec
-        if gdf.crs is not None and gdf.crs.to_epsg() != 4326:
-            gdf = gdf.to_crs("EPSG:4326")
-
-        # Convert area from m² to hectares
-        gdf["SHAPE_AREA"] = gdf["SHAPE_AREA"] / 10000
-
         return super().migrate(gdf)

fiboa_cli/datasets/za_fusion_ml.py

@@ -5,16 +5,13 @@
 class ZaFusionMlConverter(MlSplitsMixin, ZaFusionConverter):
 
     def file_migration(self, gdf, path, uri, layer=None):
-        # train files contain "train", test file contains "test"
         if "train" in path:
             gdf["_source_split"] = "train"
         else:
             gdf["_source_split"] = "test"
         return super().file_migration(gdf, path, uri, layer)
 
     def migrate(self, gdf):
-        # Assign split from temp marker and build unique IDs
         gdf["split"] = gdf["_source_split"].astype(object)
         gdf["id"] = gdf["_source_split"] + "_" + gdf["fid"].astype(str)
-        gdf = gdf.drop(columns=["_source_split"])
         return super().migrate(gdf)