Feature: storage, database (#31)

* feature: in-memory storage

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* feature: spaced storage

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* feature: database config

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* feature: fd limit

* feature: rocksdb as spaced storage impl

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* feature: literals for tests

* feature: storage tests

* feature: target all_tests

* feature: chain spec

* refactor: code format

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* feature: parse values with suffix

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* fix: CI issue

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* refactor: move parsing functions to separated util-header

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* fix: review issue

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

* clean: remove migration feature

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

---------

Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]>

Author: xDimon

CMakeLists.txt

@@ -13,13 +13,13 @@ endif ()
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_EXTENSIONS ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API ON)
 
 project(cpp-jam
     VERSION 0.0.1
-    LANGUAGES CXX
+    LANGUAGES CXX C
 )
 
 if(DEFINED CMAKE_TOOLCHAIN_FILE AND CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg")
@@ -66,7 +66,7 @@ find_package(Python3 REQUIRED)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(libb2 REQUIRED IMPORTED_TARGET GLOBAL libb2)
 
-find_package(Boost CONFIG REQUIRED COMPONENTS algorithm outcome program_options)
+find_package(Boost CONFIG REQUIRED COMPONENTS algorithm outcome program_options property_tree)
 find_package(fmt CONFIG REQUIRED)
 find_package(yaml-cpp CONFIG REQUIRED)
 find_package(qdrvm-crates CONFIG REQUIRED)
@@ -75,19 +75,29 @@ find_package(soralog CONFIG REQUIRED)
 find_package(Boost.DI CONFIG REQUIRED)
 find_package(qtils CONFIG REQUIRED)
 find_package(prometheus-cpp CONFIG REQUIRED)
+find_package(RocksDB CONFIG REQUIRED)
 
-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  add_compile_options(-fmodules-ts)
-elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
-  add_compile_options(-fmodules)
-endif()
+# TODO Temporarily commented out until gcc is updated (gcc-13 crashes because of this).
+# if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+#   add_compile_options(-fmodules-ts)
+# elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
+#   add_compile_options(-fmodules)
+# endif()
 
 add_library(headers INTERFACE)
 target_include_directories(headers INTERFACE
-    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src_>
+    $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
+    $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/generated>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
+include_directories(${CMAKE_SOURCE_DIR})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+include_directories(${CMAKE_SOURCE_DIR}/src/third_party)
+include_directories(${CMAKE_SOURCE_DIR}/src/_TODO)
+include_directories(${CMAKE_BINARY_DIR}/generated)
+
 add_subdirectory(src)
 
 if (TESTING)

example/config.yaml

@@ -2,6 +2,11 @@ general:
   name: NameFromConfig
   base_path: /tmp/jam_node
   modules_dir: modules
+  spec_file: jamduna-spec.json
+
+database:
+  directory: db
+  cache_size: 1G
 
 metrics:
   enabled: true
@@ -33,4 +38,5 @@ logging:
             - name: application
             - name: rpc
             - name: metrics
-            - name: threads
+            - name: threads
+            - name: storage

example/jamduna-spec.json

@@ -31,3 +31,9 @@ add_subdirectory(se)
 
 # Modules subsystem
 add_subdirectory(modules)
+
+# Storage
+add_subdirectory(storage)
+
+# Utilities
+add_subdirectory(utils)

src/CMakeLists.txt

@@ -26,27 +26,47 @@ add_library(build_version
     ${CMAKE_BINARY_DIR}/generated/app/build_version.cpp
 )
 
-add_library(app_configuration SHARED configuration.cpp)
+add_library(app_configuration SHARED
+    configuration.cpp
+)
 target_link_libraries(app_configuration
     Boost::boost
     fmt::fmt
 )
 
-add_library(app_configurator SHARED configurator.cpp)
+add_library(app_configurator SHARED
+    configurator.cpp
+)
 target_link_libraries(app_configurator
     app_configuration
     yaml-cpp::yaml-cpp
     Boost::program_options
     build_version
 )
 
-add_library(app_state_manager SHARED impl/state_manager_impl.cpp)
+add_library(app_state_manager SHARED
+    impl/state_manager_impl.cpp
+)
 target_link_libraries(app_state_manager
     qtils::qtils
     logger
 )
 
-add_library(application SHARED impl/application_impl.cpp)
+add_library(chain_spec SHARED
+    impl/chain_spec_impl.cpp
+)
+target_link_libraries(chain_spec
+    logger
+    Boost::property_tree
+    app_configuration
+)
+add_dependencies(chain_spec
+    generate_common_types
+)
+
+add_library(application SHARED
+    impl/application_impl.cpp
+)
 target_link_libraries(application
     qtils::qtils
     app_configuration

src/app/CMakeLists.txt

@@ -0,0 +1,38 @@
+/**
+ * Copyright Quadrivium LLC
+ * All Rights Reserved
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <qtils/byte_vec.hpp>
+
+#include "jam_types/types.tmp.hpp"
+
+namespace jam {
+
+  using NodeAddress = Stub;
+
+}
+
+namespace jam::app {
+
+  class ChainSpec {
+   public:
+    virtual ~ChainSpec() = default;
+
+    virtual const std::string &id() const = 0;
+
+    virtual const std::vector<NodeAddress> &bootNodes() const = 0;
+
+    virtual const qtils::ByteVec &genesisHeader() const = 0;
+
+    virtual const std::map<qtils::ByteVec, qtils::ByteVec> &genesisState()
+        const = 0;
+  };
+
+}  // namespace jam::app

src/app/chain_spec.hpp

@@ -10,6 +10,10 @@ namespace jam::app {
   Configuration::Configuration()
       : version_("undefined"),
         name_("unnamed"),
+        database_{
+            .directory = "db",
+            .cache_size = 1 << 30,
+        },
         metrics_{
             .endpoint{},
             .enabled{},
@@ -27,10 +31,18 @@ namespace jam::app {
     return base_path_;
   }
 
+  const std::filesystem::path &Configuration::specFile() const {
+    return spec_file_;
+  }
+
   const std::filesystem::path &Configuration::modulesDir() const {
     return modules_dir_;
   }
 
+  const Configuration::DatabaseConfig &Configuration::database() const {
+    return database_;
+  }
+
   const Configuration::MetricsConfig &Configuration::metrics() const {
     return metrics_;
   }

src/app/configuration.cpp

@@ -8,30 +8,38 @@
 #include <filesystem>
 #include <string>
 
-#include <filesystem>
 #include <boost/asio/ip/tcp.hpp>
 #include <utils/ctor_limiters.hpp>
 
 namespace jam::app {
-  class Configuration final : Singleton<Configuration> {
+  class Configuration : Singleton<Configuration> {
    public:
     using Endpoint = boost::asio::ip::tcp::endpoint;
 
+    struct DatabaseConfig {
+      std::filesystem::path directory = "db";
+      size_t cache_size = 1 << 30;  // 1GiB
+    };
+
     struct MetricsConfig {
       Endpoint endpoint;
       std::optional<bool> enabled;
     };
 
     Configuration();
+    virtual ~Configuration() = default;
 
     // /// Generate yaml-file with actual config
     // virtual void generateConfigFile() const = 0;
 
     [[nodiscard]] virtual const std::string &nodeVersion() const;
     [[nodiscard]] virtual const std::string &nodeName() const;
     [[nodiscard]] virtual const std::filesystem::path &basePath() const;
+    [[nodiscard]] virtual const std::filesystem::path &specFile() const;
     [[nodiscard]] virtual const std::filesystem::path &modulesDir() const;
 
+    [[nodiscard]] virtual const DatabaseConfig &database() const;
+
     [[nodiscard]] virtual const MetricsConfig &metrics() const;
 
    private:
@@ -40,8 +48,10 @@ namespace jam::app {
     std::string version_;
     std::string name_;
     std::filesystem::path base_path_;
+    std::filesystem::path spec_file_;
     std::filesystem::path modules_dir_;
 
+    DatabaseConfig database_;
     MetricsConfig metrics_;
   };