WIP: Config for external block storage from JSON

Author: franzpoeschel

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -33,6 +33,7 @@
 
 #include <istream>
 #include <nlohmann/json.hpp>
+#include <variant>
 #if openPMD_HAVE_MPI
 #include <mpi.h>
 #endif
@@ -196,6 +197,15 @@ struct JsonDatatypeHandling
 
 namespace openPMD
 {
+namespace dataset_mode_types
+{
+    struct Dataset_t
+    {};
+    struct Template_t
+    {};
+    using External_t = std::shared_ptr<ExternalBlockStorage>;
+} // namespace dataset_mode_types
+
 class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 {
     using json = nlohmann::json;
@@ -218,8 +228,6 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
         std::string originalExtension);
 #endif
 
-    ExternalBlockStorage externalBlockStorage;
-
     void init(openPMD::json::TracingJSON config);
 
     ~JSONIOHandlerImpl() override;
@@ -286,42 +294,6 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     std::future<void> flush();
 
-private:
-#if openPMD_HAVE_MPI
-    std::optional<MPI_Comm> m_communicator;
-#endif
-
-    using FILEHANDLE = std::fstream;
-
-    // map each Writable to its associated file
-    // contains only the filename, without the OS path
-    std::unordered_map<Writable *, File> m_files;
-
-    std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
-
-    // files that have logically, but not physically been written to
-    std::unordered_set<File> m_dirty;
-
-    /*
-     * Is set by constructor.
-     */
-    FileFormat m_fileFormat{};
-
-    /*
-     * Under which key do we find the backend configuration?
-     * -> "json" for the JSON backend, "toml" for the TOML backend.
-     */
-    std::string backendConfigKey() const;
-
-    /*
-     * First return value: The location of the JSON value (either "json" or
-     * "toml") Second return value: The value that was maybe found at this place
-     */
-    std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
-    getBackendConfig(openPMD::json::TracingJSON &) const;
-
-    std::string m_originalExtension;
-
     /*
      * Was the config value explicitly user-chosen, or are we still working with
      * defaults?
@@ -336,17 +308,36 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // Dataset IO mode //
     /////////////////////
 
-    enum class DatasetMode
+    struct DatasetMode
+        : std::variant<
+              dataset_mode_types::Dataset_t,
+              dataset_mode_types::Template_t,
+              dataset_mode_types::External_t>
     {
-        Dataset,
-        Template
+        using Dataset_t = dataset_mode_types::Dataset_t;
+        using Template_t = dataset_mode_types::Template_t;
+        using External_t = dataset_mode_types::External_t;
+        constexpr static Dataset_t Dataset{};
+        constexpr static Template_t Template{};
+
+        using variant_t = std::variant<
+            dataset_mode_types::Dataset_t,
+            dataset_mode_types::Template_t,
+            External_t>;
+        using variant_t ::operator=;
+
+        // casts needed because of
+        // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943
+        inline auto as_base() const -> variant_t const &
+        {
+            return *this;
+        }
+        inline auto as_base() -> variant_t &
+        {
+            return *this;
+        }
     };
 
-    // IOMode m_mode{};
-    // SpecificationVia m_IOModeSpecificationVia =
-    // SpecificationVia::DefaultValue; bool m_printedSkippedWriteWarningAlready
-    // = false;
-
     struct DatasetMode_s
     {
         // Initialized in init()
@@ -361,8 +352,6 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
                 m_mode, m_specificationVia, m_skipWarnings};
         }
     };
-    DatasetMode_s m_datasetMode;
-    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
 
     ///////////////////////
     // Attribute IO mode //
@@ -381,8 +370,50 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
         AttributeMode m_mode{};
         SpecificationVia m_specificationVia = SpecificationVia::DefaultValue;
     };
-    AttributeMode_s m_attributeMode;
 
+private:
+#if openPMD_HAVE_MPI
+    std::optional<MPI_Comm> m_communicator;
+#endif
+
+    using FILEHANDLE = std::fstream;
+
+    // map each Writable to its associated file
+    // contains only the filename, without the OS path
+    std::unordered_map<Writable *, File> m_files;
+
+    std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
+
+    // files that have logically, but not physically been written to
+    std::unordered_set<File> m_dirty;
+
+    /*
+     * Is set by constructor.
+     */
+    FileFormat m_fileFormat{};
+
+    /*
+     * Under which key do we find the backend configuration?
+     * -> "json" for the JSON backend, "toml" for the TOML backend.
+     */
+    std::string backendConfigKey() const;
+
+    /*
+     * First return value: The location of the JSON value (either "json" or
+     * "toml") Second return value: The value that was maybe found at this place
+     */
+    std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
+    getBackendConfig(openPMD::json::TracingJSON &) const;
+    static std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
+    getBackendConfig(
+        openPMD::json::TracingJSON &, std::string const &configLocation);
+
+    std::string m_originalExtension;
+
+    DatasetMode_s m_datasetMode;
+    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
+
+    AttributeMode_s m_attributeMode;
     AttributeMode_s
     retrieveAttributeMode(openPMD::json::TracingJSON &config) const;
 
@@ -432,7 +463,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // essentially: m_i = \prod_{j=0}^{i-1} extent_j
     static Extent getMultiplicators(Extent const &extent);
 
-    static std::pair<Extent, DatasetMode> getExtent(nlohmann::json &j);
+    static std::pair<Extent, DatasetMode>
+    getExtent(nlohmann::json &j, DatasetMode const &baseMode);
 
     // remove single '/' in the beginning and end of a string
     static std::string removeSlashes(std::string);

include/openPMD/toolkit/ExternalBlockStorage.hpp

@@ -71,6 +71,11 @@ class ExternalBlockStorage
         std::optional<std::string> infix, // e.g. for distinguishing MPI ranks
         T const *data) -> std::string;
 
+    auto externalStorageLocation() const -> nlohmann::json
+    {
+        return "implement me";
+    }
+
     static void sanitizeString(std::string &s);
 };