Add feature for importing CSV file

docs/api_reference/model_view.md

@@ -36,6 +36,9 @@
         - column_export_exclude_list
         - export_types
         - export_max_rows
+        - can_import
+        - column_import_list
+        - column_import_exclude_list
         - form
         - form_args
         - form_columns

docs/configurations.md

@@ -71,6 +71,7 @@ The following options are available:
 - `can_delete`: If the model instances can be deleted via SQLAdmin. Default value is `True`.
 - `can_view_details`: If the model instance details can be viewed via SQLAdmin. Default value is `True`.
 - `can_export`: If the model data can be exported in the list page. Default value is `True`.
+- `can_import`: If the model data can be imported from a CSV file in the list page. Default value is `False`.
 
 !!! example
 
@@ -342,6 +343,47 @@ The export options can be set per model and includes the following options:
 - `export_max_rows`: Maximum number of rows to be exported. Default value is `0` which means unlimited.
 - `export_types`: List of export types to be enabled. Default value is `["csv","json"]`.
 
+## Import options
+
+SQLAdmin supports importing data from a CSV file in the list page.
+If the model has relations, the association will be according to the returned data `__str__` method of the SQLAlchemy model.
+Relation models in a CSV file separated by a comma.
+
+![import_csv](assets/images/import_csv.png)
+
+The import options can be set per model and includes the following options:
+
+* `can_import`: If the model can be imported. Default value is `False`.
+* `column_import_list`: List of columns to include in the import data. Default is all model columns.
+* `column_import_exclude_list`: List of columns to exclude in the import data.
+
+!!! example
+
+    ```python
+    class User(Base):
+        __tablename__ = "users"
+        id = Column(Integer, primary_key=True)
+        name = Column(String)
+        addresses: Mapped[list["Address"] | None] = relationship(
+            "Address", secondary=association_table
+        )
+
+        def __str__(self) -> str:
+            return f"User {self.id}"
+
+
+    class Address(Base):
+        __tablename__ = "addresses"
+        id = Column(Integer, primary_key=True)
+
+        def __str__(self) -> str:
+            return f"Address {self.id}"
+
+    class UserAdmin(ModelView, model=User):
+        can_import = True
+        column_import_list = [User.name, User.addresses]
+    ```
+
 ## Templates
 
 The template files are built using Jinja2 and can be completely overridden in the configurations.

sqladmin/_queries.py

@@ -3,7 +3,7 @@
 from typing import TYPE_CHECKING, Any
 
 import anyio
-from sqlalchemy import select
+from sqlalchemy import Table, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session, selectinload
 from sqlalchemy.sql.expression import Select, and_, or_
@@ -215,6 +215,61 @@ async def _insert_async(self, data: dict[str, Any], request: Request) -> Any:
             await self.model_view.after_model_change(data, obj, True, request)
             return obj
 
+    def _insert_sync_many(
+        self, data: list[dict[str, Any]], request: Request
+    ) -> list[Any]:
+        objs = []
+        with self.model_view.session_maker(expire_on_commit=False) as session:
+            for row in data:
+                obj = self.model_view.model()
+                anyio.from_thread.run(
+                    self.model_view.on_model_change, row, obj, True, request
+                )
+                obj = self._set_attributes_sync(session, obj, row)
+                objs.append(obj)
+            session.add_all(objs)
+            session.commit()
+            for obj, row in zip(objs, data):
+                anyio.from_thread.run(
+                    self.model_view.after_model_change, row, obj, True, request
+                )
+        return objs
+
+    async def _insert_async_many(
+        self, data: list[dict[str, Any]], request: Request
+    ) -> list[Any]:
+        objs = []
+        async with self.model_view.session_maker(expire_on_commit=False) as session:
+            for row in data:
+                obj = self.model_view.model()
+                await self.model_view.on_model_change(row, obj, True, request)
+                obj = await self._set_attributes_async(session, obj, row)
+                objs.append(obj)
+            session.add_all(objs)
+            await session.commit()
+            for obj, row in zip(objs, data):
+                await self.model_view.after_model_change(row, obj, True, request)
+        return objs
+
+    async def _get_relation_objects_async(self, relation: Table) -> Any:
+        stmt = select(relation)
+        async with self.model_view.session_maker(expire_on_commit=False) as session:
+            result = await session.scalars(stmt)
+        return result.all()
+
+    def _get_relation_objects_sync(self, relation: Table) -> Any:
+        stmt = select(relation)
+        with self.model_view.session_maker(expire_on_commit=False) as session:
+            return session.scalars(stmt).all()
+
+    async def get_relation_objects(self, relation: Table) -> Any:
+        if self.model_view.is_async:
+            return await self._get_relation_objects_async(relation)
+        else:
+            return await anyio.to_thread.run_sync(
+                self._get_relation_objects_sync, relation
+            )
+
     async def delete(self, obj: Any, request: Request) -> None:
         if self.model_view.is_async:
             await self._delete_async(obj, request)
@@ -227,6 +282,12 @@ async def insert(self, data: dict, request: Request) -> Any:
         else:
             return await anyio.to_thread.run_sync(self._insert_sync, data, request)
 
+    async def insert_many(self, data: list[dict[str, Any]], request: Request) -> Any:
+        if self.model_view.is_async:
+            return await self._insert_async_many(data, request)
+        else:
+            return await anyio.to_thread.run_sync(self._insert_sync_many, data, request)
+
     async def update(self, pk: Any, data: dict, request: Request) -> Any:
         if self.model_view.is_async:
             return await self._update_async(pk, data, request)

sqladmin/application.py

@@ -36,6 +36,7 @@
 from sqladmin.helpers import (
     get_object_identifier,
     is_async_session_maker,
+    parse_csv,
     slugify_action_name,
 )
 from sqladmin.models import BaseView, ModelView
@@ -321,6 +322,11 @@ async def _export(self, request: Request) -> None:
         if request.path_params["export_type"] not in model_view.export_types:
             raise HTTPException(status_code=404)
 
+    async def _import(self, request: Request) -> None:
+        model_view = self._find_model_view(request.path_params["identity"])
+        if not model_view.can_import or not model_view.is_accessible(request):
+            raise HTTPException(status_code=403)
+
 
 class Admin(BaseAdminView):
     """Main entrypoint to admin interface.
@@ -425,6 +431,12 @@ async def http_exception(
             Route(
                 "/{identity}/export/{export_type}", endpoint=self.export, name="export"
             ),
+            Route(
+                "/{identity}/import",
+                endpoint=self.import_endpoint,
+                name="import",
+                methods=["POST"],
+            ),
             Route(
                 "/{identity}/ajax/lookup", endpoint=self.ajax_lookup, name="ajax_lookup"
             ),
@@ -629,6 +641,58 @@ async def export(self, request: Request) -> Response:
         )
         return await model_view.export_data(rows, export_type=export_type)
 
+    @login_required
+    async def import_endpoint(self, request: Request) -> Response:
+        """Import model endpoint."""
+
+        await self._import(request)
+
+        identity = request.path_params["identity"]
+        model_view = self._find_model_view(identity)
+
+        try:
+            csv_content = await self._handle_form_file(request)
+            if not csv_content:
+                return Response(content="Undefined file.", status_code=400)
+
+            data = parse_csv(csv_content, model_view._import_prop_names)
+
+        except Exception as e:
+            logger.exception(e)
+            return Response(content=f"Failed parse CSV file.\n{e}", status_code=400)
+
+        Form = await model_view.scaffold_form(model_view._form_create_rules)
+        for relation in model_view._mapper.relationships:
+            relation_name = relation.key
+            relation_mapper = relation.mapper.class_
+
+            relation_objs = await model_view.get_relation_objects(relation_mapper)
+            if not relation_objs:
+                continue
+            for relation_obj in relation_objs:
+                for row in data:
+                    n_row = []
+                    for value in row.getlist(relation_name):
+                        if value == str(relation_obj):
+                            n_row.append(str(relation_obj.id))
+                        else:
+                            n_row.append(value)
+                    row.setlist(relation_name, n_row)
+
+        import_models = []
+        for row in data:
+            form = Form(row)
+            if not form.validate():
+                continue
+            form_data_dict = self._denormalize_wtform_data(form.data, model_view.model)
+            import_models.append(form_data_dict)
+
+        await model_view.insert_many_models(request, import_models)
+
+        return RedirectResponse(
+            url=request.url_for("admin:list", identity=identity), status_code=302
+        )
+
     async def login(self, request: Request) -> Response:
         assert self.authentication_backend is not None
 
@@ -720,6 +784,19 @@ async def _handle_form_data(self, request: Request, obj: Any = None) -> FormData
                 form_data.append((key, value))
         return FormData(form_data)
 
+    async def _handle_form_file(self, request: Request) -> bytes | None:
+        async with request.form(max_files=1) as form:
+            csv_file = form.get("csvfile")
+            assert isinstance(csv_file, UploadFile)
+            if (
+                not csv_file
+                or not csv_file.filename
+                or not csv_file.filename.endswith(".csv")
+            ):
+                return None
+            csv_content = await csv_file.read()
+        return csv_content
+
     def _normalize_wtform_data(self, obj: Any) -> dict:
         form_data = {}
         for field_name in WTFORMS_ATTRS:

sqladmin/helpers.py

@@ -18,6 +18,7 @@
 from sqlalchemy import Column, inspect
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import RelationshipProperty, sessionmaker
+from starlette.datastructures import MultiDict
 
 from sqladmin._types import MODEL_PROPERTY
 
@@ -174,6 +175,28 @@ def stream_to_csv(
     return callback(writer)  # type: ignore
 
 
+def parse_csv(
+    csv_content: bytes, columns: list[str], delimiter: str = ";"
+) -> list[MultiDict]:
+    if csv_content[:3] == b"\xef\xbb\xbf":
+        csv_content = csv_content[3:]
+    _csv_content = csv_content.decode("utf-8").splitlines()
+    reader = csv.DictReader(_csv_content, delimiter=delimiter)
+    result = []
+    for row in reader:
+        md = MultiDict()
+        for column, value in row.items():
+            if column not in columns:
+                continue
+            if value and "," in value:
+                for iter_value in value.split(","):
+                    md.append(column, iter_value)
+            else:
+                md.append(column, value)
+        result.append(md)
+    return result
+
+
 def get_primary_keys(model: Any) -> tuple[Column, ...]:
     return tuple(inspect(model).mapper.primary_key)