Replace storage database with rocksdb (#455)

* Add rocksdb demo

* Fix version

* Switch to amulet-rocksdb

* Update python versions

* Remove legacy fallback code

* Reformat

* Remove beta from leveldb requirement

* Improve workflows

Add cmake install and refactor

Author: gentlegiantJGC

.github/workflows/python-build.yml

@@ -12,14 +12,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ['3.11', '3.12']
         os: [macos-latest, windows-latest]
+
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Clone
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -32,6 +36,7 @@ jobs:
       run: |
         python -m build
         twine upload dist/* --skip-existing
+
     - name: Build Docs
       shell: bash
       env:

.github/workflows/python-stylecheck.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.10']
+        python-version: ['3.11']
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/python-unittests.yml

@@ -16,18 +16,28 @@ jobs:
   unittests:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ['3.11', '3.12']
         os: [windows-latest, macos-latest, ubuntu-latest]
+
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Clone
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
+    - name: Setup cmake
+      if: matrix.os == 'ubuntu-latest'
+      uses: jwlawson/actions-setup-cmake@v2
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
+
     - name: Test with unittest
       run: python -m unittest discover -v -s tests

.readthedocs.yml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: '3.10'
+    python: '3.11'
 
 python:
   install:

amulet/api/level/base_level/base_level.py

@@ -26,7 +26,7 @@
 )
 from amulet.api.chunk.status import StatusFormats
 from amulet.api.cache import TempDir
-from leveldb import LevelDB
+from rocksdb import RocksDB, Options, WriteOptions, CompressionType
 from amulet.utils.generator import generator_unpacker
 from amulet.utils.world_utils import block_coords_to_chunk_coords
 from .chunk_manager import ChunkManager
@@ -72,9 +72,21 @@ def __init__(self, path: str, format_wrapper: api_wrapper.FormatWrapper):
         self._history_manager = MetaHistoryManager()
 
         self._temp_dir = TempDir()
-        self._history_db = LevelDB(
-            os.path.join(self._temp_dir, "history_db"), create_if_missing=True
+
+        options = Options()
+        options.create_if_missing = True
+        options.compression_type = CompressionType.ZstdCompression
+
+        write_options = WriteOptions()
+        write_options.sync = False
+        write_options.disable_wal = True
+
+        self._history_db = RocksDB(
+            os.path.join(self._temp_dir, "history_db"),
+            options=options,
+            write_options=write_options,
         )
+
         self._chunks: ChunkManager = ChunkManager(self, self._history_db)
         self._players = PlayerManager(self)
 
@@ -522,7 +534,7 @@ def close(self):
         Use changed method to check if there are any changes that should be saved before closing.
         """
         self.level_wrapper.close()
-        self._history_db.close(compact=False)
+        self._history_db.close()
 
     def unload(self, safe_area: Optional[Tuple[Dimension, int, int, int, int]] = None):
         """

amulet/api/level/base_level/chunk_manager.py

@@ -10,7 +10,7 @@
 from amulet.api.errors import ChunkDoesNotExist, ChunkLoadError
 from amulet.api.history.history_manager import DatabaseHistoryManager
 from amulet.api import level as api_level
-from leveldb import LevelDB
+from rocksdb import RocksDB
 
 
 class ChunkDBEntry(DBRevisionManager):
@@ -19,7 +19,7 @@ class ChunkDBEntry(DBRevisionManager):
     def __init__(
         self,
         world: api_level.BaseLevel,
-        history_db: LevelDB,
+        history_db: RocksDB,
         prefix: str,
         initial_state: EntryType,
     ):
@@ -66,7 +66,7 @@ class ChunkManager(DatabaseHistoryManager):
     DoesNotExistError = ChunkDoesNotExist
     LoadError = ChunkLoadError
 
-    def __init__(self, level: api_level.BaseLevel, history_db: LevelDB):
+    def __init__(self, level: api_level.BaseLevel, history_db: RocksDB):
         """
         Construct a :class:`ChunkManager` instance.
 

amulet/level/formats/anvil_world/_sector_manager.py

@@ -2,75 +2,7 @@
 
 from typing import NamedTuple, List
 import threading
-import sys
-
-if sys.version_info >= (3, 10):
-    # bisect only supports key as of 3.10
-    from bisect import bisect_left, bisect_right
-
-else:
-
-    def bisect_left(a, x, lo=0, hi=None, *, key=None):
-        """Return the index where to insert item x in list a, assuming a is sorted.
-        The return value i is such that all e in a[:i] have e < x, and all e in
-        a[i:] have e >= x.  So if x already appears in the list, a.insert(i, x) will
-        insert just before the leftmost x already there.
-        Optional args lo (default 0) and hi (default len(a)) bound the
-        slice of a to be searched.
-        """
-
-        if lo < 0:
-            raise ValueError("lo must be non-negative")
-        if hi is None:
-            hi = len(a)
-        # Note, the comparison uses "<" to match the
-        # __lt__() logic in list.sort() and in heapq.
-        if key is None:
-            while lo < hi:
-                mid = (lo + hi) // 2
-                if a[mid] < x:
-                    lo = mid + 1
-                else:
-                    hi = mid
-        else:
-            while lo < hi:
-                mid = (lo + hi) // 2
-                if key(a[mid]) < x:
-                    lo = mid + 1
-                else:
-                    hi = mid
-        return lo
-
-    def bisect_right(a, x, lo=0, hi=None, *, key=None):
-        """Return the index where to insert item x in list a, assuming a is sorted.
-        The return value i is such that all e in a[:i] have e <= x, and all e in
-        a[i:] have e > x.  So if x already appears in the list, a.insert(i, x) will
-        insert just after the rightmost x already there.
-        Optional args lo (default 0) and hi (default len(a)) bound the
-        slice of a to be searched.
-        """
-
-        if lo < 0:
-            raise ValueError("lo must be non-negative")
-        if hi is None:
-            hi = len(a)
-        # Note, the comparison uses "<" to match the
-        # __lt__() logic in list.sort() and in heapq.
-        if key is None:
-            while lo < hi:
-                mid = (lo + hi) // 2
-                if x < a[mid]:
-                    hi = mid
-                else:
-                    lo = mid + 1
-        else:
-            while lo < hi:
-                mid = (lo + hi) // 2
-                if x < key(a[mid]):
-                    hi = mid
-                else:
-                    lo = mid + 1
-        return lo
+from bisect import bisect_left, bisect_right
 
 
 class NoValidSector(Exception):

setup.cfg

@@ -14,15 +14,16 @@ platforms = any
 
 [options]
 include_package_data = True
-python_requires = >=3.9
+python_requires = >=3.11
 install_requires =
     numpy~=1.17
     amulet-nbt~=2.0
     pymctranslate~=1.2
     portalocker~=2.4
-    amulet-leveldb~=1.0b0
+    amulet-leveldb~=1.0
     platformdirs~=3.1
     lz4~=4.3
+    amulet-rocksdb~=1.0
 
 packages = find: