Merge pull request #321 from OpenGeoscience/file-streams

Stream local file content to File Fields

Author: annehaley

uvdat/core/management/commands/ingest.py

@@ -11,7 +11,7 @@
 
 from django.contrib.auth.models import User
 from django.contrib.gis.geos import Point
-from django.core.files.base import ContentFile
+from django.core.files import File
 from django.db.models import Sum
 from django.db.models.functions import Coalesce
 import djclick as click
@@ -149,7 +149,7 @@ def ingest_file(file_info, *, index=0, dataset=None, chart=None, replace=False,
         )
         click.secho(f"FileItem {new_file_item.name} created.", fg="green")
         with file_location.open("rb") as f:
-            new_file_item.file.save(file_path, ContentFile(f.read()))
+            new_file_item.file.save(file_path, File(f))
 
 
 def ingest_projects(data: list[ProjectItem], *, replace=False) -> None:

uvdat/core/tasks/analytics/flood_simulation.py

@@ -6,7 +6,7 @@
 
 from celery import shared_task
 from django.conf import settings
-from django.core.files.base import ContentFile
+from django.core.files import File
 
 from uvdat.core.models import Chart, Colormap, Dataset, FileItem, LayerStyle, TaskResult
 
@@ -149,7 +149,7 @@ def flood_simulation(result_id):
             metadata=metadata,
         )
         with output_path.open("rb") as f:
-            file_item.file.save(output_path.name, ContentFile(f.read()))
+            file_item.file.save(output_path.name, File(f))
         dataset.spawn_conversion_task(
             layer_options=[
                 {

uvdat/core/tasks/analytics/geoai_segmentation.py

@@ -4,7 +4,7 @@
 
 from celery import shared_task
 from django.conf import settings
-from django.core.files.base import ContentFile
+from django.core.files import File
 from django_large_image import utilities
 import large_image
 from pyproj import CRS, Transformer
@@ -148,7 +148,7 @@ def geoai_segmentation(result_id):  # noqa: PLR0915
         file_size=mask_path.stat().st_size,
     )
     with mask_path.open("rb") as f:
-        raster_file_item.file.save(mask_path, ContentFile(f.read()))
+        raster_file_item.file.save(mask_path, File(f))
 
     dataset.spawn_conversion_task(asynchronous=False)
     result.write_outputs({"result": dataset.id})

uvdat/core/tasks/conversion.py

@@ -6,7 +6,7 @@
 import tempfile
 import zipfile
 
-from django.core.files.base import ContentFile
+from django.core.files import File
 from django_large_image import utilities
 import geopandas
 import numpy as np
@@ -135,7 +135,7 @@ def convert_files(*files, file_item=None, combine=False):  # noqa: C901, PLR0912
             metadata=metadata,
         )
         with cog_path.open("rb") as f:
-            raster_data.cloud_optimized_geotiff.save(cog_path.name, ContentFile(f.read()))
+            raster_data.cloud_optimized_geotiff.save(cog_path.name, File(f))
         logger.info("%s created for %s", raster_data, cog.get("name"))