Add tests for beginning-of-verse embeds

Fix the embed at beginning issue.
Add paragraph marker control
Correct behavior for stripping text

Author: isaac091

machine/corpora/paratext_project_text_updater_base.py

@@ -22,6 +22,7 @@ def update_usfm(
         rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,
         full_name: Optional[str] = None,
         text_behavior: UpdateUsfmTextBehavior = UpdateUsfmTextBehavior.PREFER_EXISTING,
+        paragraph_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
         embed_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
         style_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.STRIP,
     ) -> Optional[str]:
@@ -31,7 +32,12 @@ def update_usfm(
         with self._open(file_name) as sfm_file:
             usfm: str = sfm_file.read().decode(self._settings.encoding)
         handler = UpdateUsfmParserHandler(
-            rows, None if full_name is None else f"- {full_name}", text_behavior, embed_behavior, style_behavior
+            rows,
+            None if full_name is None else f"- {full_name}",
+            text_behavior,
+            paragraph_behavior,
+            embed_behavior,
+            style_behavior,
         )
         try:
             parse_usfm(usfm, handler, self._settings.stylesheet, self._settings.versification)

machine/corpora/scripture_ref_usfm_parser_handler.py

@@ -19,6 +19,7 @@ class ScriptureTextType(Enum):
     NOTE_TEXT = auto()
 
 
+PRESERVE_PARAGRAPH_STYLES = ("r", "rem")
 EMBED_PART_START_CHAR_STYLES = ("f", "x", "z")
 EMBED_STYLES = ("f", "fe", "fig", "fm", "x")
 
@@ -29,6 +30,7 @@ def __init__(self) -> None:
         self._cur_elements_stack: List[ScriptureElement] = []
         self._cur_text_type_stack: List[ScriptureTextType] = []
         self._duplicate_verse: bool = False
+        self._in_preserved_paragraph: bool = False
         self._in_embed: bool = False
         self._in_note_text: bool = False
         self._in_nested_embed: bool = False
@@ -74,13 +76,16 @@ def start_para(
         unknown: Optional[bool],
         attributes: Optional[Sequence[UsfmAttribute]],
     ) -> None:
+        if self._is_preserve_paragraph_type(marker):
+            self._in_preserved_paragraph = True
         if self._cur_verse_ref.is_default:
             self._update_verse_ref(state.verse_ref, marker)
         if not state.is_verse_text:
             self._start_parent_element(marker)
             self._start_non_verse_text_wrapper(state)
 
     def end_para(self, state: UsfmParserState, marker: str) -> None:
+        self._in_preserved_paragraph = False
         if self._current_text_type == ScriptureTextType.NONVERSE:
             self._end_parent_element()
             self._end_non_verse_text_wrapper(state)
@@ -270,9 +275,12 @@ def _check_convert_verse_para_to_non_verse(self, state: UsfmParserState) -> None
     def _is_in_embed(self, marker: Optional[str]) -> bool:
         return self._in_embed or self._is_embed_style(marker)
 
+    def _is_in_preserved_paragraph(self, marker: Optional[str]) -> bool:
+        return self._in_preserved_paragraph or self._is_preserve_paragraph_type(marker)
+
     def _is_in_nested_embed(self, marker: Optional[str]) -> bool:
         return self._in_nested_embed or (
-            marker is not None and marker[0] == "+" and marker[1] in EMBED_PART_START_CHAR_STYLES
+            marker is not None and marker.startswith("+") and marker[1] in EMBED_PART_START_CHAR_STYLES
         )
 
     def _is_note_text(self, marker: Optional[str]) -> bool:
@@ -282,4 +290,7 @@ def _is_embed_part_style(self, marker: Optional[str]) -> bool:
         return marker is not None and marker.startswith(EMBED_PART_START_CHAR_STYLES)
 
     def _is_embed_style(self, marker: Optional[str]) -> bool:
-        return marker in EMBED_STYLES
+        return marker is not None and marker.strip("*") in EMBED_STYLES
+
+    def _is_preserve_paragraph_type(self, marker: Optional[str]) -> bool:
+        return marker in PRESERVE_PARAGRAPH_STYLES

machine/corpora/update_usfm_parser_handler.py

@@ -27,15 +27,18 @@ def __init__(
         rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,
         id_text: Optional[str] = None,
         text_behavior: UpdateUsfmTextBehavior = UpdateUsfmTextBehavior.PREFER_EXISTING,
+        paragraph_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
         embed_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
         style_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.STRIP,
     ) -> None:
         super().__init__()
         self._rows = rows or []
         self._tokens: List[UsfmToken] = []
         self._new_tokens: List[UsfmToken] = []
+        self._new_embed_tokens: List[UsfmToken] = []
         self._id_text = id_text
         self._text_behavior = text_behavior
+        self._paragraph_behavior = paragraph_behavior
         self._embed_behavior = embed_behavior
         self._style_behavior = style_behavior
         self._replace_stack: List[bool] = []
@@ -74,7 +77,14 @@ def start_para(
         unknown: bool,
         attributes: Optional[Sequence[UsfmAttribute]],
     ) -> None:
-        self._collect_tokens(state)
+        if (
+            state.verse_ref.verse_num != 0
+            and (self._has_new_text() or self._text_behavior == UpdateUsfmTextBehavior.STRIP_EXISTING)
+            and self._paragraph_behavior == UpdateUsfmMarkerBehavior.STRIP
+        ):
+            self._skip_tokens(state)
+        else:
+            self._collect_tokens(state)
 
         super().start_para(state, marker, unknown, attributes)
 
@@ -202,13 +212,13 @@ def ref(self, state: UsfmParserState, marker: str, display: str, target: str) ->
         super().ref(state, marker, display, target)
 
     def text(self, state: UsfmParserState, text: str) -> None:
+        super().text(state, text)
+
         if self._replace_with_new_tokens(state):
             self._skip_tokens(state)
         else:
             self._collect_tokens(state)
 
-        super().text(state, text)
-
     def opt_break(self, state: UsfmParserState) -> None:
         if self._replace_with_new_tokens(state):
             self._skip_tokens(state)
@@ -240,7 +250,7 @@ def _end_non_verse_text(self, state: UsfmParserState, scripture_ref: ScriptureRe
         self._pop_new_tokens()
 
     def _start_note_text(self, state: UsfmParserState) -> None:
-        self._push_new_tokens([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in self._embed_row_texts])
+        self._push_new_embed_tokens([UsfmToken(UsfmTokenType.TEXT, text=t + " ") for t in self._embed_row_texts])
 
     def _end_note_text(self, state: UsfmParserState, scripture_ref: ScriptureRef) -> None:
         self._embed_row_texts.clear()
@@ -287,13 +297,9 @@ def _skip_tokens(self, state: UsfmParserState) -> None:
         self._token_index = state.index + 1 + state.special_token_count
 
     def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True) -> bool:
-        if self._text_behavior == UpdateUsfmTextBehavior.STRIP_EXISTING:
-            self._add_new_tokens()
-            return True
-
-        new_text: bool = bool(self._replace_stack) and self._replace_stack[-1]
         marker: Optional[str] = state.token if state.token is None else state.token.marker
         in_embed: bool = self._is_in_embed(marker)
+
         in_nested_embed: bool = self._is_in_nested_embed(marker)
         is_style_tag: bool = marker is not None and not self._is_embed_part_style(marker)
 
@@ -303,8 +309,14 @@ def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True)
         )
 
         use_new_tokens = (
-            new_text
-            and (not existing_text or self._text_behavior == UpdateUsfmTextBehavior.PREFER_NEW)
+            (
+                (self._text_behavior == UpdateUsfmTextBehavior.STRIP_EXISTING)
+                or (
+                    self._has_new_text()
+                    and (not existing_text or self._text_behavior == UpdateUsfmTextBehavior.PREFER_NEW)
+                )
+            )
+            and not self._is_in_preserved_paragraph(marker)
             and (
                 not in_embed
                 or (
@@ -316,26 +328,37 @@ def _replace_with_new_tokens(self, state: UsfmParserState, closed: bool = True)
         )
 
         if use_new_tokens:
-            self._add_new_tokens()
+            if in_embed:
+                self._add_new_embed_tokens()
+            else:
+                self._add_new_tokens()
 
         if existing_text and self._text_behavior == UpdateUsfmTextBehavior.PREFER_EXISTING:
-            self._clear_new_tokens()
+            if in_embed:
+                self._clear_new_embed_tokens()
+            else:
+                self._clear_new_tokens()
 
-        embed_in_new_verse_text = any(self._replace_stack) and in_embed
+        embed_in_new_verse_text = (
+            any(self._replace_stack) or self._text_behavior == UpdateUsfmTextBehavior.STRIP_EXISTING
+        ) and in_embed
         if embed_in_new_verse_text or self._embed_updated:
             if self._embed_behavior == UpdateUsfmMarkerBehavior.STRIP:
-                self._clear_new_tokens()
+                self._clear_new_embed_tokens()
                 return True
             if not self._is_in_note_text() or in_nested_embed:
                 return False
 
         skip_tokens = use_new_tokens and closed
 
-        if new_text and is_style_tag:
+        if use_new_tokens and is_style_tag:
             skip_tokens = self._style_behavior == UpdateUsfmMarkerBehavior.STRIP
 
         return skip_tokens
 
+    def _has_new_text(self) -> bool:
+        return bool(self._replace_stack) and self._replace_stack[-1]
+
     def _push_new_tokens(self, tokens: List[UsfmToken]) -> None:
         self._replace_stack.append(any(tokens))
         if tokens:
@@ -349,6 +372,19 @@ def _add_new_tokens(self) -> None:
     def _clear_new_tokens(self) -> None:
         self._new_tokens.clear()
 
+    def _push_new_embed_tokens(self, tokens: List[UsfmToken]) -> None:
+        self._replace_stack.append(any(tokens))
+        if tokens:
+            self._new_embed_tokens.extend(tokens)
+
+    def _add_new_embed_tokens(self) -> None:
+        if self._new_embed_tokens:
+            self._tokens.extend(self._new_embed_tokens)
+        self._new_embed_tokens.clear()
+
+    def _clear_new_embed_tokens(self) -> None:
+        self._new_embed_tokens.clear()
+
     def _push_token_as_previous(self) -> None:
         self._replace_stack.append(self._replace_stack[-1])