Merge pull request #8 from martindurant/serde

Make everything reducable to dictionaries

Author: martindurant

src/projspec/artifact/base.py

@@ -12,17 +12,9 @@
 
 
 class BaseArtifact:
-    def __init__(
-        self,
-        proj: Project,
-        requires: list | None = None,
-        cmd: list[str] | None = None,
-        **kw,
-    ):
+    def __init__(self, proj: Project, cmd: list[str] | None = None):
         self.proj = proj
-        self.requires = requires or []
         self.cmd = cmd
-        self.__dict__.update(kw)
         self.proc = None
 
     def _is_clean(self) -> bool:
@@ -58,17 +50,11 @@ def make(self, *args, **kwargs):
     def _make(self, *args, **kwargs):
         subprocess.check_call(self.cmd, cwd=self.proj.url, **kwargs)
 
-    def remake(self, reqs=False):
+    def remake(self):
         """Recreate the artifact and any runtime it depends on"""
-        if reqs:
-            self.clean_req()
         self.clean()
         self.make()
 
-    def clean_req(self):
-        for req in self.requires:
-            req.clean()
-
     def clean(self):
         """Remove artifact"""
         # this default implementation leaves nothing to clean
@@ -91,6 +77,22 @@ def __init_subclass__(cls, **kwargs):
     def snake_name(cls):
         return camel_to_snake(cls.__name__)
 
+    def to_dict(self, compact=True):
+        """Distil the instance to JSON compatible dict
+
+        compact: if True, will produce condensed output, perhaps justa  string.
+        """
+        if compact:
+            return self._repr2()
+        dic = {
+            k: v
+            for k, v in self.__dict__.items()
+            if not k.startswith("_") and k not in ("proj", "proc")
+        }
+        dic["klass"] = ["artifact", self.snake_name()]
+        dic["proc"] = None
+        return dic
+
 
 def get_artifact_cls(name: str) -> type[BaseArtifact]:
     """Find an artifact class by snake-case name."""

src/projspec/content/base.py

@@ -30,6 +30,21 @@ def __init_subclass__(cls, **kwargs):
     def snake_name(cls):
         return camel_to_snake(cls.__name__)
 
+    def to_dict(self, compact=False):
+        if compact:
+            return self._repr2()
+        dic = {
+            k: getattr(self, k)
+            for k in self.__dataclass_fields__
+            if k not in ("proj", "artifacts")
+        }
+        dic["artifacts"] = []
+        dic["klass"] = ["content", self.snake_name()]
+        for k in list(dic):
+            if isinstance(dic[k], Enum):
+                dic[k] = dic[k].value
+        return dic
+
 
 def get_content_cls(name: str) -> type[BaseContent]:
     """Find a content class by snake-case name."""

src/projspec/proj/__init__.py

@@ -1,4 +1,4 @@
-from .base import Project, ProjectSpec
+from .base import Project, ProjectSpec, get_projspec_class
 from .conda_package import CondaRecipe, RattlerRecipe
 from .conda_project import CondaProject
 from .documentation import RTD, MDBook
@@ -8,9 +8,10 @@
 from .pyscript import PyScript
 from .python_code import PythonCode, PythonLibrary
 from .rust import Rust, RustPython
-from .uv import UV
+from .uv import Uv
 
 __all__ = [
+    "get_projspec_class",
     "Project",
     "ProjectSpec",
     "CondaRecipe",
@@ -26,5 +27,5 @@
     "RTD",
     "Rust",
     "RustPython",
-    "UV",
+    "Uv",
 ]

src/projspec/proj/base.py

@@ -6,10 +6,16 @@
 import fsspec.implementations.local
 import toml
 
-from projspec.utils import AttrDict, IndentDumper, camel_to_snake, flatten
+from projspec.utils import (
+    AttrDict,
+    IndentDumper,
+    PickleableTomlDecoder,
+    camel_to_snake,
+    flatten,
+)
 
 logger = logging.getLogger("projspec")
-registry = set()
+registry = {}
 default_excludes = {
     ".venv",  # venv, pipenv, uv
     ".pixi",
@@ -32,6 +38,9 @@ def __init__(
     ):
         if fs is None:
             fs, path = fsspec.url_to_fs(path, **(storage_options or {}))
+        else:
+            storage_options = fs.storage_options
+        self.storage_options = storage_options or {}
         self.fs = fs
         self.url = path
         self.specs = AttrDict()
@@ -60,7 +69,8 @@ def resolve(
         """
         fullpath = "/".join([self.url, subpath]) if subpath else self.url
         # sorting to ensure consistency
-        for cls in sorted(registry, key=str):
+        for name in sorted(registry):
+            cls = registry[name]
             try:
                 logger.debug("resolving %s as %s", fullpath, cls)
                 name = cls.__name__
@@ -142,7 +152,7 @@ def pyproject(self):
         if "pyproject.toml" in self.basenames:
             try:
                 with self.fs.open(self.basenames["pyproject.toml"], "rt") as f:
-                    return toml.load(f)
+                    return toml.load(f, decoder=PickleableTomlDecoder())
             except (OSError, ValueError, TypeError):
                 # debug/warn?
                 pass
@@ -172,9 +182,27 @@ def __contains__(self, item) -> bool:
             item in _ for _ in self.children.values()
         )
 
-    def to_dict(self) -> dict:
-        dic = AttrDict(specs=self.specs, children=self.children)
-        return dic.to_dict()
+    def to_dict(self, compact=True) -> dict:
+        dic = AttrDict(
+            specs=self.specs,
+            children=self.children,
+            url=self.url,
+            storage_options=self.storage_options,
+        )
+        dic["klass"] = "project"
+        return dic.to_dict(compact=compact)
+
+    @staticmethod
+    def from_dict(dic):
+        from projspec.utils import from_dict
+
+        proj = object.__new__(Project)
+        proj.specs = from_dict(dic["specs"], proj)
+        proj.children = from_dict(dic["children"], proj)
+        proj.url = dic["url"]
+        proj.storage_options = dic["storage_options"]
+        proj.fs, _ = fsspec.url_to_fs(proj.url, **proj.storage_options)
+        return proj
 
 
 class ProjectSpec:
@@ -186,8 +214,8 @@ class ProjectSpec:
 
     spec_doc = ""  # URL to prose about this spec
 
-    def __init__(self, root: Project, subpath: str = ""):
-        self.root = root
+    def __init__(self, proj: Project, subpath: str = ""):
+        self.proj = proj
         self.subpath = subpath  # not used yet
         self._contents = AttrDict()
         self._artifacts = AttrDict()
@@ -198,9 +226,9 @@ def __init__(self, root: Project, subpath: str = ""):
     def path(self) -> str:
         """Location of this project spec"""
         return (
-            self.root.url + "/" + self.subpath
+            self.proj.url + "/" + self.subpath
             if self.subpath
-            else self.root.url
+            else self.proj.url
         )
 
     def match(self) -> bool:
@@ -241,7 +269,7 @@ def clean(self) -> None:
 
     @classmethod
     def __init_subclass__(cls, **kwargs):
-        registry.add(cls)
+        registry[camel_to_snake(cls.__name__)] = cls
 
     def __repr__(self):
         import yaml
@@ -253,6 +281,20 @@ def __repr__(self):
             base += f"\nArtifacts:\n{yaml.dump(self.artifacts.to_dict(), Dumper=IndentDumper).rstrip()}\n"
         return base
 
-    def to_dict(self) -> dict:
-        dic = AttrDict(contents=self.contents, artifacts=self.artifacts)
-        return dic.to_dict()
+    def to_dict(self, compact=True) -> dict:
+        dic = AttrDict(
+            _contents=self.contents,
+            _artifacts=self.artifacts,
+            subpath=self.subpath,
+            klass=["projspec", self.snake_name()],
+        )
+        return dic.to_dict(compact=compact)
+
+    @classmethod
+    def snake_name(cls) -> str:
+        """Convert a project name to snake-case"""
+        return camel_to_snake(cls.__name__)
+
+
+def get_projspec_class(name: str) -> type:
+    return registry[name]

src/projspec/proj/briefcase.py

@@ -5,4 +5,4 @@ class Briefcase(ProjectSpec):
     spec_doc = "https://briefcase.readthedocs.io/en/stable/reference/configuration.html"
 
     def match(self) -> bool:
-        return "briefcase" in self.root.pyproject.get("tool", {})
+        return "briefcase" in self.proj.pyproject.get("tool", {})

src/projspec/proj/conda_package.py

@@ -12,7 +12,7 @@ class CondaRecipe(ProjectSpec):
 
     def match(self) -> bool:
         return not {"meta.yaml", "meta.yml", "conda.yaml"}.isdisjoint(
-            self.root.basenames
+            self.proj.basenames
         )
 
     def parse(self) -> None:
@@ -21,9 +21,9 @@ def parse(self) -> None:
 
         meta = None
         for fn in ("meta.yaml", "meta.yml", "conda.yaml"):
-            if fn in self.root.basenames:
+            if fn in self.proj.basenames:
                 try:
-                    with self.root.fs.open(self.root.basenames[fn], "rb") as f:
+                    with self.proj.fs.open(self.proj.basenames[fn], "rb") as f:
                         meta0 = _yaml_no_jinja(f)
                     # TODO: multiple output recipe
                     if "package" in meta0:
@@ -38,14 +38,14 @@ def parse(self) -> None:
                     pass
         if meta is None:
             raise ValueError
-        art = CondaPackage(proj=self.root, cmd=["conda-build", self.root.url])
-        self._artifacts = AttrDict(**{meta["package"]["name"]: art})
+        art = CondaPackage(proj=self.proj, cmd=["conda-build", self.proj.url])
+        self._artifacts = AttrDict(conda_package=art)
         # TODO: read envs from "outputs" like for Rattler, below?
         self._contents = AttrDict(
             environment=AttrDict(
                 {
                     k: Environment(
-                        proj=self.root,
+                        proj=self.proj,
                         artifacts={art},
                         packages=v,
                         stack=Stack.CONDA,
@@ -65,19 +65,19 @@ class RattlerRecipe(CondaRecipe):
     # conda recipes are also valid for rattler if they don't have complex jinja.
 
     def match(self) -> bool:
-        return "recipe.yaml" in self.root.basenames
+        return "recipe.yaml" in self.proj.basenames
 
     def parse(self) -> None:
         from projspec.artifact.installable import CondaPackage
         from projspec.content.environment import Environment, Precision, Stack
 
-        if "recipe.yaml" in self.root.basenames:
-            with self.root.fs.open(
-                self.root.basenames["recipe.yaml"], "rb"
+        if "recipe.yaml" in self.proj.basenames:
+            with self.proj.fs.open(
+                self.proj.basenames["recipe.yaml"], "rb"
             ) as f:
                 meta = _yaml_no_jinja(f)
-        elif "meta.yaml" in self.root.basenames:
-            with self.root.fs.open(self.root.basenames["meta.yaml"], "rb") as f:
+        elif "meta.yaml" in self.proj.basenames:
+            with self.proj.fs.open(self.proj.basenames["meta.yaml"], "rb") as f:
                 meta = _yaml_no_jinja(f)
         else:
             raise ValueError
@@ -86,9 +86,9 @@ def parse(self) -> None:
             "rattler-build",
             "build",
             "-r",
-            self.root.url,
+            self.proj.url,
             "--output-dir",
-            f"{self.root.url}/output",
+            f"{self.proj.url}/output",
         ]
         name = next(
             filter(
@@ -101,10 +101,10 @@ def parse(self) -> None:
         )
 
         path = (
-            f"{self.root.url}/output/{name}" if self.root.is_local() else None
+            f"{self.proj.url}/output/{name}" if self.proj.is_local() else None
         )
         art = CondaPackage(
-            proj=self.root,
+            proj=self.proj,
             cmd=cmd,
             path=path,
             name=name,
@@ -125,7 +125,7 @@ def parse(self) -> None:
                 environment=AttrDict(
                     {
                         k: Environment(
-                            proj=self.root,
+                            proj=self.proj,
                             artifacts={art},
                             packages=v,
                             stack=Stack.CONDA,