Consistently use text elements for indexing and string length; add manual test for running quote convention analysis

Author: Enkidu93

src/SIL.Machine/PunctuationAnalysis/QuotationMarkFinder.cs

@@ -1,6 +1,8 @@
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using PCRE;
+using SIL.Extensions;
 
 namespace SIL.Machine.PunctuationAnalysis
 {
@@ -43,11 +45,18 @@ public List<QuotationMarkStringMatch> FindAllPotentialQuotationMarksInTextSegmen
                     _quoteConventions.IsValidOpeningQuotationMark(match.Groups[0].Value)
                     || _quoteConventions.IsValidClosingQuotationMark(match.Groups[0].Value)
                 )
-                .Select(m => new QuotationMarkStringMatch(
-                    textSegment,
-                    m.Groups[0].Index,
-                    m.Groups[0].Index + m.Groups[0].Length
-                ))
+                .Select(m =>
+                {
+                    int[] textElementBeginnings = StringInfo.ParseCombiningCharacters(textSegment.Text);
+                    int endIndex = textElementBeginnings.IndexOf(m.Groups[0].EndIndex);
+                    if (endIndex == -1)
+                        endIndex = textElementBeginnings.Length;
+                    return new QuotationMarkStringMatch(
+                        textSegment,
+                        textElementBeginnings.IndexOf(m.Groups[0].Index),
+                        endIndex
+                    );
+                })
                 .ToList();
         }
     }

src/SIL.Machine/PunctuationAnalysis/TextSegment.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using SIL.Machine.Corpora;
 
 namespace SIL.Machine.PunctuationAnalysis
@@ -70,7 +71,7 @@ public override int GetHashCode()
             return hashCode * 31 + ImmediatePrecedingMarker.GetHashCode();
         }
 
-        public int Length => Text.Length;
+        public int Length => StringInfo.ParseCombiningCharacters(Text).Length;
 
         public string SubstringBefore(int index)
         {

tests/SIL.Machine.Tests/Corpora/CorporaTestHelpers.cs

@@ -16,8 +16,10 @@ internal static class CorporaTestHelpers
     );
     public static readonly string UsfmTestProjectPath = Path.Combine(TestDataPath, "usfm", "Tes");
     public static readonly string UsfmTargetProjectPath = Path.Combine(TestDataPath, "usfm", "target");
+    public static readonly string UsfmTargetProjectZipPath = Path.Combine(TestDataPath, "project", "target");
     public static readonly string UsfmTargetCustomVrsPath = Path.Combine(TestDataPath, "usfm", "target", "custom.vrs");
     public static readonly string UsfmSourceProjectPath = Path.Combine(TestDataPath, "usfm", "source");
+    public static readonly string UsfmSourceProjectZipPath = Path.Combine(TestDataPath, "project", "source");
     public static readonly string UsxTestProjectPath = Path.Combine(TestDataPath, "usx", "Tes");
     public static readonly string TextTestProjectPath = Path.Combine(TestDataPath, "txt");
     public static readonly string DeuterocanonicalsSourcePath = Path.Combine(

tests/SIL.Machine.Tests/Corpora/UsfmManualTests.cs

@@ -1,6 +1,7 @@
 using System.IO.Compression;
 using System.Text.Json;
 using NUnit.Framework;
+using SIL.Machine.PunctuationAnalysis;
 
 namespace SIL.Machine.Corpora;
 
@@ -170,4 +171,28 @@ async Task GetUsfmAsync(string projectPath)
             await GetUsfmAsync(ParatextProjectPath);
         }
     }
+
+    [Test]
+    [Ignore("This is for manual testing only.  Remove this tag to run the test.")]
+    public void AnalyzeCorporaQuoteConventions()
+    {
+        var sourceHandler = new QuoteConventionDetector();
+        using ZipArchive zipArchive = ZipFile.OpenRead(CorporaTestHelpers.UsfmSourceProjectZipPath);
+        var quoteConventionDetector = new ZipParatextProjectQuoteConventionDetector(zipArchive);
+        quoteConventionDetector.GetQuoteConventionAnalysis(sourceHandler);
+
+        var targetHandler = new QuoteConventionDetector();
+        using ZipArchive zipArchive2 = ZipFile.OpenRead(CorporaTestHelpers.UsfmTargetProjectZipPath);
+        var quoteConventionDetector2 = new ZipParatextProjectQuoteConventionDetector(zipArchive2);
+        quoteConventionDetector2.GetQuoteConventionAnalysis(targetHandler);
+
+        QuoteConventionAnalysis sourceAnalysis = sourceHandler.DetectQuotationConvention();
+        QuoteConventionAnalysis targetAnalysis = targetHandler.DetectQuotationConvention();
+
+        Assert.Multiple(() =>
+        {
+            Assert.NotNull(sourceAnalysis);
+            Assert.NotNull(targetAnalysis);
+        });
+    }
 }