Use PCRE.NET to mirror python regexes

Author: Enkidu93

src/SIL.Machine/PunctuationAnalysis/QuotationMarkFinder.cs

@@ -1,13 +1,12 @@
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
-using System.Text.RegularExpressions;
+using PCRE;
 
 namespace SIL.Machine.PunctuationAnalysis
 {
     public class QuotationMarkFinder
     {
-        private static readonly Regex TypewriterGuillemetsPattern = new Regex(@"(<<|>>|<|>)", RegexOptions.Compiled);
+        private static readonly PcreRegex QuotationMarkPattern = new PcreRegex(@"(\p{Quotation_Mark}|<<|>>|<|>)");
         private readonly QuoteConventionSet _quoteConventions;
 
         public QuotationMarkFinder(QuoteConventionSet quoteConventions)
@@ -37,30 +36,9 @@ IReadOnlyList<TextSegment> textSegments
 
         public List<QuotationMarkStringMatch> FindAllPotentialQuotationMarksInTextSegment(TextSegment textSegment)
         {
-            TextElementEnumerator charactersEnumerator = StringInfo.GetTextElementEnumerator(textSegment.Text);
-            int index = 0;
-            List<QuotationMarkStringMatch> quotationMarkStringMatches = new List<QuotationMarkStringMatch>();
-            while (charactersEnumerator.MoveNext())
-            {
-                string currentCharacterString = charactersEnumerator.Current.ToString();
-                if (
-                    (
-                        QuotationMarkStringMatch.HasUnicodeProperty(currentCharacterString, "QUOTATION MARK")
-                        || QuotationMarkStringMatch.HasUnicodeProperty(currentCharacterString, "APOSTROPHE")
-                    )
-                    && (
-                        _quoteConventions.IsValidOpeningQuotationMark(currentCharacterString)
-                        || _quoteConventions.IsValidClosingQuotationMark(currentCharacterString)
-                    )
-                )
-                {
-                    quotationMarkStringMatches.Add(new QuotationMarkStringMatch(textSegment, index, index + 1));
-                }
-                index++;
-            }
-            List<QuotationMarkStringMatch> typewriterGuillemetMatches = TypewriterGuillemetsPattern
+            return QuotationMarkPattern
                 .Matches(textSegment.Text)
-                .Cast<Match>()
+                .Cast<PcreMatch>()
                 .Where(match =>
                     _quoteConventions.IsValidOpeningQuotationMark(match.Groups[0].Value)
                     || _quoteConventions.IsValidClosingQuotationMark(match.Groups[0].Value)
@@ -71,11 +49,6 @@ public List<QuotationMarkStringMatch> FindAllPotentialQuotationMarksInTextSegmen
                     m.Groups[0].Index + m.Groups[0].Length
                 ))
                 .ToList();
-
-            return quotationMarkStringMatches
-                .Concat(typewriterGuillemetMatches)
-                .OrderBy(match => match.StartIndex)
-                .ToList();
         }
     }
 }

src/SIL.Machine/PunctuationAnalysis/QuotationMarkStringMatch.cs

@@ -1,13 +1,14 @@
 using System;
 using System.Globalization;
 using System.Text.RegularExpressions;
-using System.Unicode;
+using PCRE;
 
 namespace SIL.Machine.PunctuationAnalysis
 {
     public class QuotationMarkStringMatch
     {
-        // No LatinLetterPattern or LetterPattern because C# does not support it in the same way as Python. Using UnicodeInfo to mirror machine.py
+        private static readonly PcreRegex LetterPattern = new PcreRegex(@"[\p{L}\N{U+0001E200}-\N{U+0001E28F}]");
+        private static readonly PcreRegex LatinLetterPattern = new PcreRegex(@"^\p{Script_Extensions=Latin}$");
         private static readonly Regex WhitespacePattern = new Regex(@"[\s~]", RegexOptions.Compiled);
         private static readonly Regex PunctuationPattern = new Regex(@"[\.,;\?!\)\]\-—۔،؛]", RegexOptions.Compiled);
         private static readonly Regex QuoteIntroducerPattern = new Regex(@"[:,]\s*$", RegexOptions.Compiled);
@@ -55,9 +56,15 @@ public bool IsValidClosingQuotationMark(QuoteConventionSet quoteConventions) =>
         public bool NextCharacterMatches(Regex regexPattern) =>
             NextCharacter != null && regexPattern.IsMatch(NextCharacter);
 
+        public bool NextCharacterMatches(PcreRegex regexPattern) =>
+            NextCharacter != null && regexPattern.IsMatch(NextCharacter);
+
         public bool PreviousCharacterMatches(Regex regexPattern) =>
             PreviousCharacter != null && regexPattern.IsMatch(PreviousCharacter);
 
+        public bool PreviousCharacterMatches(PcreRegex regexPattern) =>
+            PreviousCharacter != null && regexPattern.IsMatch(PreviousCharacter);
+
         public string PreviousCharacter
         {
             get
@@ -98,9 +105,15 @@ public string NextCharacter
         public bool LeadingSubstringMatches(Regex regexPattern) =>
             regexPattern.IsMatch(TextSegment.SubstringBefore(StartIndex));
 
+        public bool LeadingSubstringMatches(PcreRegex regexPattern) =>
+            regexPattern.IsMatch(TextSegment.SubstringBefore(StartIndex));
+
         public bool TrailingSubstringMatches(Regex regexPattern) =>
             regexPattern.IsMatch(TextSegment.SubstringAfter(EndIndex));
 
+        public bool TrailingSubstringMatches(PcreRegex regexPattern) =>
+            regexPattern.IsMatch(TextSegment.SubstringAfter(EndIndex));
+
         // This assumes that the two matches occur in the same verse
         public bool Precedes(QuotationMarkStringMatch other)
         {
@@ -151,72 +164,27 @@ public bool HasTrailingPunctuation()
 
         public bool HasLetterInLeadingSubstring()
         {
-            string leadingSubstring = TextSegment.SubstringBefore(StartIndex);
-            if (leadingSubstring.Length == 0)
-                return false;
-
-            TextElementEnumerator charactersEnumerator = StringInfo.GetTextElementEnumerator(leadingSubstring);
-            while (charactersEnumerator.MoveNext())
-            {
-                if (!IsLetter(charactersEnumerator.Current.ToString()))
-                    return false;
-            }
-            return true;
+            return LeadingSubstringMatches(LetterPattern);
         }
 
         public bool HasLetterInTrailingSubstring()
         {
-            string trailingSubstring = TextSegment.SubstringAfter(EndIndex);
-            if (trailingSubstring.Length == 0)
-                return false;
-            TextElementEnumerator charactersEnumerator = StringInfo.GetTextElementEnumerator(trailingSubstring);
-            while (charactersEnumerator.MoveNext())
-            {
-                if (!IsLetter(charactersEnumerator.Current.ToString()))
-                    return false;
-            }
-            return true;
+            return TrailingSubstringMatches(LetterPattern);
         }
 
         public bool HasLeadingLatinLetter()
         {
-            return PreviousCharacter != null && IsLatinScript(PreviousCharacter);
+            return PreviousCharacterMatches(LatinLetterPattern);
         }
 
         public bool HasTrailingLatinLetter()
         {
-            return NextCharacter != null && IsLatinScript(NextCharacter);
+            return NextCharacterMatches(LatinLetterPattern);
         }
 
         public bool HasQuoteIntroducerInLeadingSubstring()
         {
             return LeadingSubstringMatches(QuoteIntroducerPattern);
         }
-
-        public static bool HasUnicodeProperty(string characterString, string attribute)
-        {
-            if (characterString.Length == 1)
-            {
-                return UnicodeInfo.GetName(characterString[0]).Contains(attribute);
-            }
-            else if (char.IsSurrogatePair(characterString[0], characterString[1]))
-            {
-                //Get true unicode value
-                int combinedCharacterValue =
-                    (((int)characterString[0] - 0xD800) * 0x400) + ((int)characterString[1] - 0xDC00) + 0x10000;
-                return UnicodeInfo.GetName(combinedCharacterValue).Contains(attribute);
-            }
-            return false;
-        }
-
-        private bool IsLatinScript(string characterString)
-        {
-            return HasUnicodeProperty(characterString, "LATIN");
-        }
-
-        private bool IsLetter(string characterString)
-        {
-            return HasUnicodeProperty(characterString, "LETTER");
-        }
     }
 }

src/SIL.Machine/SIL.Machine.csproj

@@ -38,12 +38,12 @@
 	<ItemGroup>
 		<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
 		<PackageReference Include="Nito.AsyncEx" Version="5.1.2" />
+		<PackageReference Include="PCRE.NET" Version="1.2.0" />
 		<PackageReference Include="Sandwych.QuickGraph.Core" Version="1.0.0" />
 		<PackageReference Include="SIL.Scripture" Version="12.0.1" />
 		<PackageReference Include="System.Text.Encoding.CodePages" Version="6.0.0" />
 		<PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0.0" />
 		<PackageReference Include="CaseExtensions" Version="1.1.0" />
-		<PackageReference Include="UnicodeInformation" Version="2.7.1" />
 	</ItemGroup>
 
 	<ItemGroup Condition="'$(TargetFramework)' == 'net461'">