Some tests pass

Author: johnml1135

machine/corpora/scripture_update_block.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from .scripture_ref import ScriptureRef
+from .scripture_update_element import ScriptureUpdateElement, ScriptureUpdateElementType
+from .usfm_token import UsfmToken, UsfmTokenType
+
+
+class ScriptureUpdateBlock:
+
+    def __init__(self) -> None:
+        self._ref: ScriptureRef = ScriptureRef()
+        self._elements: list[ScriptureUpdateElement] = []
+
+    def add_existing_text(self, token: UsfmToken, marked_for_removal: bool = False) -> None:
+        self._elements.append(
+            ScriptureUpdateElement(ScriptureUpdateElementType.EXISTING_TEXT, [token], marked_for_removal)
+        )
+
+    def add_inserted_text(self, tokens: list[UsfmToken]) -> None:
+        self._elements.append(ScriptureUpdateElement(ScriptureUpdateElementType.INSERTED_TEXT, tokens.copy()))
+
+    def add_token(self, token: UsfmToken, marked_for_removal: bool = False) -> None:
+        if token.type == UsfmTokenType.TEXT:
+            self._elements.append(
+                ScriptureUpdateElement(ScriptureUpdateElementType.EXISTING_TEXT, [token], marked_for_removal)
+            )
+        else:
+            self._elements.append(ScriptureUpdateElement(ScriptureUpdateElementType.OTHER, [token], marked_for_removal))
+
+    def add_tokens(self, tokens: list[UsfmToken], marked_for_removal: bool = False) -> None:
+        if len(tokens) == 0:
+            return
+        self._elements.append(
+            ScriptureUpdateElement(ScriptureUpdateElementType.OTHER, tokens.copy(), marked_for_removal)
+        )
+
+    def update_ref(self, ref: ScriptureRef) -> None:
+        self._ref = ref
+
+    def clear(self) -> None:
+        self._elements.clear()
+        self._ref = ScriptureRef()
+
+    def get_tokens(self) -> list[UsfmToken]:
+        return [token for element in self._elements for token in element.get_tokens()]

machine/corpora/scripture_update_block_handler_base.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .scripture_update_block import ScriptureUpdateBlock
+
+
+class ScriptureUpdateBlockHandlerBase:
+
+    def process_block(self, block: ScriptureUpdateBlock) -> ScriptureUpdateBlock:
+        raise NotImplementedError("Must be implemented in subclass")

machine/corpora/scripture_update_block_handler_first_elements_first.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from .scripture_update_block import ScriptureUpdateBlock
+from .scripture_update_block_handler_base import ScriptureUpdateBlockHandlerBase
+from .scripture_update_element import ScriptureUpdateElementType
+
+
+class ScriptureUpdateBlockHandlerFirstElementsFirst(ScriptureUpdateBlockHandlerBase):
+
+    def process_block(self, block: ScriptureUpdateBlock) -> ScriptureUpdateBlock:
+        # If a paragraph, embed or style element occurs before existing text, move it before inserted text as well.
+        current_insert_index = 0
+        for current_index in range(len(block._elements)):
+            element = block._elements[current_index]
+            if element.type == ScriptureUpdateElementType.EXISTING_TEXT:
+                # we found existing text, so we stop looking for elements to move
+                break
+            if current_index != current_insert_index and element.type != ScriptureUpdateElementType.INSERTED_TEXT:
+                block._elements.remove(element)
+                block._elements.insert(current_insert_index, element)
+                current_insert_index += 1
+
+        return block

machine/corpora/scripture_update_element.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum, auto
+
+from .usfm_token import UsfmToken
+
+
+class ScriptureUpdateElementType(Enum):
+    EXISTING_TEXT = auto()
+    INSERTED_TEXT = auto()
+    OTHER = auto()
+
+
+@dataclass
+class ScriptureUpdateElement:
+    type: ScriptureUpdateElementType
+    tokens: list[UsfmToken]
+    marked_for_removal: bool = False
+
+    def get_tokens(self) -> list[UsfmToken]:
+        if self.marked_for_removal:
+            return []
+        return self.tokens

machine/corpora/update_usfm_parser_handler.py

@@ -1,8 +1,12 @@
 from enum import Enum, auto
 from typing import List, Optional, Sequence, Tuple, Union
 
+from ..scripture.verse_ref import VerseRef
 from .scripture_ref import ScriptureRef
 from .scripture_ref_usfm_parser_handler import ScriptureRefUsfmParserHandler
+from .scripture_update_block import ScriptureUpdateBlock
+from .scripture_update_block_handler_base import ScriptureUpdateBlockHandlerBase
+from .scripture_update_block_handler_first_elements_first import ScriptureUpdateBlockHandlerFirstElementsFirst
 from .usfm_parser_state import UsfmParserState
 from .usfm_stylesheet import UsfmStylesheet
 from .usfm_token import UsfmAttribute, UsfmToken, UsfmTokenType
@@ -31,13 +35,20 @@ def __init__(
         embed_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
         style_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.STRIP,
         preserve_paragraph_styles: Optional[Sequence[str]] = None,
+        update_block_handlers: Optional[list[ScriptureUpdateBlockHandlerBase]] = None,
     ) -> None:
         super().__init__()
         self._rows = rows or []
         self._tokens: List[UsfmToken] = []
-        self._new_tokens: List[UsfmToken] = []
-        self._new_embed_tokens: List[UsfmToken] = []
+        self._updated_text: List[UsfmToken] = []
+        self._updated_embed_text: List[UsfmToken] = []
+        self._update_block: ScriptureUpdateBlock = ScriptureUpdateBlock()
+        self._embed_update_block: ScriptureUpdateBlock = ScriptureUpdateBlock()
         self._id_text = id_text
+        if update_block_handlers is None:
+            self._update_block_handlers = [ScriptureUpdateBlockHandlerFirstElementsFirst()]
+        else:
+            self._update_block_handlers = update_block_handlers
         if preserve_paragraph_styles is None:
             self._preserve_paragraph_styles = set(["r", "rem"])
         elif isinstance(preserve_paragraph_styles, str):
@@ -60,21 +71,20 @@ def tokens(self) -> List[UsfmToken]:
 
     def end_usfm(self, state: UsfmParserState) -> None:
         self._collect_tokens(state)
-
+        self._process_update_block()
         super().end_usfm(state)
 
     def start_book(self, state: UsfmParserState, marker: str, code: str) -> None:
         self._collect_tokens(state)
         start_book_tokens: List[UsfmToken] = []
         if self._id_text is not None:
             start_book_tokens.append(UsfmToken(UsfmTokenType.TEXT, text=self._id_text + " "))
-        self._push_new_tokens(start_book_tokens)
+        self._update_block.add_tokens(start_book_tokens)
 
         super().start_book(state, marker, code)
 
     def end_book(self, state: UsfmParserState, marker: str) -> None:
-        self._pop_new_tokens()
-
+        self._process_update_block()
         super().end_book(state, marker)
 
     def start_para(
@@ -99,6 +109,7 @@ def start_para(
         super().start_para(state, marker, unknown, attributes)
 
     def end_para(self, state: UsfmParserState, marker: str) -> None:
+        self._process_update_block()
         super().end_para(state, marker)
         self._in_preserved_paragraph = False
 
@@ -114,7 +125,7 @@ def start_cell(self, state: UsfmParserState, marker: str, align: str, colspan: i
 
     def end_cell(self, state: UsfmParserState, marker: str) -> None:
         self._collect_tokens(state)
-
+        self._process_update_block()
         super().end_cell(state, marker)
 
     def start_sidebar(self, state: UsfmParserState, marker: str, category: str) -> None:
@@ -125,6 +136,7 @@ def start_sidebar(self, state: UsfmParserState, marker: str, category: str) -> N
     def end_sidebar(self, state: UsfmParserState, marker: str, closed: bool) -> None:
         if closed:
             self._collect_tokens(state)
+            self._process_update_block()
 
         super().end_sidebar(state, marker, closed)
 
@@ -137,6 +149,7 @@ def chapter(
         pub_number: str,
     ) -> None:
         self._collect_tokens(state)
+        self._process_update_block()
 
         super().chapter(state, number, marker, alt_number, pub_number)
 
@@ -148,6 +161,7 @@ def milestone(
         attributes: Sequence[UsfmAttribute],
     ) -> None:
         self._collect_tokens(state)
+        self._process_update_block()
 
         super().milestone(state, marker, start_milestone, attributes)
 
@@ -160,6 +174,7 @@ def verse(
         pub_number: str,
     ) -> None:
         self._collect_tokens(state)
+        self._process_update_block()
 
         super().verse(state, number, marker, alt_number, pub_number)
 
@@ -196,6 +211,7 @@ def _start_embed(
         state: UsfmParserState,
         scripture_ref: ScriptureRef,
     ) -> None:
+        self._embed_update_block.update_ref(scripture_ref)
         self._embed_row_texts = self._advance_rows([scripture_ref])
         self._embed_updated = any(self._embed_row_texts)
 
@@ -212,6 +228,7 @@ def _end_embed(
         else:
             self._collect_tokens(state)
 
+        self._process_embed_update_block()
         self._embed_row_texts.clear()
         self._embed_updated = False
 
@@ -251,20 +268,20 @@ def unmatched(self, state: UsfmParserState, marker: str) -> None:
 
     def _start_verse_text(self, state: UsfmParserState, scripture_refs: Sequence[ScriptureRef]) -> None:
         row_texts: List[str] = self._advance_rows(scripture_refs)
-        self._push_new_tokens([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in row_texts])
+        self._push_updated_text([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in row_texts])
 
     def _end_verse_text(self, state: UsfmParserState, scripture_refs: Sequence[ScriptureRef]) -> None:
         self._pop_new_tokens()
 
     def _start_non_verse_text(self, state: UsfmParserState, scripture_ref: ScriptureRef) -> None:
         row_texts = self._advance_rows([scripture_ref])
-        self._push_new_tokens([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in row_texts])
+        self._push_updated_text([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in row_texts])
 
     def _end_non_verse_text(self, state: UsfmParserState, scripture_ref: ScriptureRef) -> None:
         self._pop_new_tokens()
 
     def _start_note_text(self, state: UsfmParserState) -> None:
-        self._push_new_embed_tokens([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in self._embed_row_texts])
+        self._push_updated_embed_text([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in self._embed_row_texts])
 
     def _end_note_text(self, state: UsfmParserState, scripture_ref: ScriptureRef) -> None:
         self._embed_row_texts.clear()
@@ -301,13 +318,15 @@ def _advance_rows(self, seg_scr_refs: Sequence[ScriptureRef]) -> List[str]:
         return row_texts
 
     def _collect_tokens(self, state: UsfmParserState) -> None:
-        self._tokens.extend(self._new_tokens)
-        self._new_tokens.clear()
+        self._use_updated_text()
         while self._token_index <= state.index + state.special_token_count:
-            self._tokens.append(state.tokens[self._token_index])
+            self._update_block.add_token(state.tokens[self._token_index])
             self._token_index += 1
 
     def _skip_tokens(self, state: UsfmParserState) -> None:
+        while self._token_index <= state.index + state.special_token_count:
+            self._update_block.add_token(state.tokens[self._token_index], marked_for_removal=True)
+            self._token_index += 1
         self._token_index = state.index + 1 + state.special_token_count
 
     def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True) -> bool:
@@ -343,24 +362,24 @@ def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True)
 
         if use_new_tokens:
             if in_embed:
-                self._add_new_embed_tokens()
+                self._use_updated_embed_text()
             else:
-                self._add_new_tokens()
+                self._use_updated_text()
 
         if existing_text and (
             self._text_behavior == UpdateUsfmTextBehavior.PREFER_EXISTING or self._is_in_preserved_paragraph(marker)
         ):
             if in_embed:
-                self._clear_new_embed_tokens()
+                self._clear_updated_embed_text()
             else:
-                self._clear_new_tokens()
+                self._clear_updated_text()
 
         embed_in_new_verse_text = (
             any(self._replace_stack) or self._text_behavior == UpdateUsfmTextBehavior.STRIP_EXISTING
         ) and in_embed
         if embed_in_new_verse_text or self._embed_updated:
             if self._embed_behavior == UpdateUsfmMarkerBehavior.STRIP:
-                self._clear_new_embed_tokens()
+                self._clear_updated_embed_text()
                 return True
             if not self._is_in_note_text() or in_nested_embed:
                 return False
@@ -375,33 +394,56 @@ def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True)
     def _has_new_text(self) -> bool:
         return any(self._replace_stack) and self._replace_stack[-1]
 
-    def _push_new_tokens(self, tokens: List[UsfmToken]) -> None:
+    def _update_verse_ref(self, verse_ref: VerseRef, marker: str) -> None:
+        super()._update_verse_ref(verse_ref, marker)
+        self._update_block.update_ref(ScriptureRef(verse_ref.copy()))
+
+    def _create_non_verse_ref(self) -> ScriptureRef:
+        ref = super()._create_non_verse_ref()
+        self._update_block.update_ref(ref)
+        return ref
+
+    def _process_update_block(self) -> None:
+        self._use_updated_text()
+        for handler in self._update_block_handlers:
+            self._update_block = handler.process_block(self._update_block)
+        self._tokens.extend(self._update_block.get_tokens())
+        self._update_block.clear()
+
+    def _process_embed_update_block(self) -> None:
+        self._use_updated_embed_text()
+        for handler in self._update_block_handlers:
+            self._embed_update_block = handler.process_block(self._embed_update_block)
+        self._update_block.add_tokens(self._embed_update_block.get_tokens())
+        self._embed_update_block.clear()
+
+    def _push_updated_text(self, tokens: List[UsfmToken]) -> None:
         self._replace_stack.append(any(tokens))
         if tokens:
-            self._new_tokens.extend(tokens)
+            self._updated_text.extend(tokens)
 
-    def _add_new_tokens(self) -> None:
-        if self._new_tokens:
-            self._tokens.extend(self._new_tokens)
-        self._new_tokens.clear()
+    def _use_updated_text(self) -> None:
+        if self._updated_text:
+            self._update_block.add_inserted_text(self._updated_text)
+        self._updated_text.clear()
 
-    def _clear_new_tokens(self) -> None:
-        self._new_tokens.clear()
+    def _clear_updated_text(self) -> None:
+        self._updated_text.clear()
 
-    def _push_new_embed_tokens(self, tokens: List[UsfmToken]) -> None:
+    def _push_updated_embed_text(self, tokens: List[UsfmToken]) -> None:
         self._replace_stack.append(any(tokens))
         if tokens:
-            self._new_embed_tokens.extend(tokens)
+            self._updated_embed_text.extend(tokens)
 
-    def _add_new_embed_tokens(self) -> None:
-        if self._new_embed_tokens:
-            self._tokens.extend(self._new_embed_tokens)
-        self._new_embed_tokens.clear()
+    def _use_updated_embed_text(self) -> None:
+        if self._updated_embed_text:
+            self._embed_update_block.add_inserted_text(self._updated_embed_text)
+        self._updated_embed_text.clear()
 
-    def _clear_new_embed_tokens(self) -> None:
-        self._new_embed_tokens.clear()
+    def _clear_updated_embed_text(self) -> None:
+        self._updated_embed_text.clear()
 
-    def _push_token_as_previous(self) -> None:
+    def _push_updated_text_as_previous(self) -> None:
         self._replace_stack.append(self._replace_stack[-1])
 
     def _pop_new_tokens(self) -> None: