Remove RTL markers from verse token data; add tests

Author: Enkidu93

src/SIL.Machine/Corpora/UsfmTokenizer.cs

@@ -175,7 +175,7 @@ public IReadOnlyList<UsfmToken> Tokenize(string usfm, bool preserveWhitespace =
                                     marker,
                                     null,
                                     null,
-                                    GetNextWord(usfm, ref index, preserveWhitespace)
+                                    SanitizeVerseData(GetNextWord(usfm, ref index, preserveWhitespace))
                                 )
                                 {
                                     LineNumber = lineNum,
@@ -563,6 +563,11 @@ private static string GetNextWord(string usfm, ref int index, bool preserveWhite
             return data;
         }
 
+        private static string SanitizeVerseData(string verseData)
+        {
+            return verseData.Replace("‏", "");
+        }
+
         /// <summary>
         /// Converts all control characters, carriage returns and tabs into
         /// spaces, and then strips duplicate spaces.

tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs

@@ -906,6 +906,39 @@ public void UpdateBlock_Verse_Range()
         );
     }
 
+    [Test]
+    public void UpdateBlock_Verse_Range_RightToLeftMarker()
+    {
+        var rows = new List<UpdateUsfmRow> { new UpdateUsfmRow(ScrRef("MAT 1:1", "MAT 1:2", "MAT 1:3"), "Update 1-3") };
+        string usfm =
+            @"\id MAT - Test
+\c 1
+\v 1‏-3 verse 1 through 3
+";
+        TestUsfmUpdateBlockHandler usfmUpdateBlockHandler = new TestUsfmUpdateBlockHandler();
+        string updatedUsfm = UpdateUsfm(
+            rows,
+            usfm,
+            embedBehavior: UpdateUsfmMarkerBehavior.Preserve,
+            usfmUpdateBlockHandlers: [usfmUpdateBlockHandler]
+        );
+        string expectedUsfm =
+            @"\id MAT - Test
+\c 1
+\v 1-3 Update 1-3
+";
+        Assert.That(updatedUsfm.Replace("\r\n", "\n"), Is.EqualTo(expectedUsfm));
+        Assert.That(usfmUpdateBlockHandler.Blocks.Count, Is.EqualTo(1));
+
+        UsfmUpdateBlock usfmUpdateBlock = usfmUpdateBlockHandler.Blocks[0];
+        AssertUpdateBlockEquals(
+            usfmUpdateBlock,
+            ["MAT 1:1", "MAT 1:2", "MAT 1:3"],
+            (UsfmUpdateBlockElementType.Text, "Update 1-3 ", false),
+            (UsfmUpdateBlockElementType.Text, "verse 1 through 3 ", true)
+        );
+    }
+
     [Test]
     public void UpdateBlock_Footnote_PreserveEmbeds()
     {

tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs

@@ -425,6 +425,40 @@ public void GetRows_PrivateUseMarker()
         });
     }
 
+    [Test]
+    public void GetRows_VerseRangeWithRightToLeftMarker()
+    {
+        TextRow[] rows = GetRows(
+            @"\id MAT - Test
+\h
+\mt
+\c 1
+\v 1‏-2 Verse one and two.
+"
+        );
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(rows, Has.Length.EqualTo(2));
+
+            Assert.That(
+                rows[0].Ref,
+                Is.EqualTo(ScriptureRef.Parse("MAT 1:1")),
+                string.Join(",", rows.ToList().Select(tr => tr.Ref.ToString()))
+            );
+            Assert.That(
+                rows[0].Text,
+                Is.EqualTo("Verse one and two."),
+                string.Join(",", rows.ToList().Select(tr => tr.Text))
+            );
+            Assert.That(
+                rows[1].Ref,
+                Is.EqualTo(ScriptureRef.Parse("MAT 1:2")),
+                string.Join(",", rows.ToList().Select(tr => tr.Ref.ToString()))
+            );
+        });
+    }
+
     private static TextRow[] GetRows(string usfm, bool includeMarkers = false, bool includeAllText = false)
     {
         UsfmMemoryText text =