Merge pull request #67 from ddn0/add-re2

Use re2 instead of re for match expressions

Author: slott56

README.rst

@@ -37,6 +37,21 @@ Installation
 
 You now have the CEL run-time available to Python-based applications.
 
+
+re2
+---
+
+CEL specifies that regular expressions use re2 syntax,
+https://github.com/google/re2/wiki/Syntax. To keep its dependencies minimal and
+this implementation easily embeddable, cel-python uses the Python standard
+library ``re`` syntax by default. If a ``re2`` package is installed or the
+``re2`` extra is provided, cel-python will use ``re2`` syntax instead.
+
+::
+
+    pip install cel-python[re2]
+
+
 Command Line
 ============
 

docs/source/c7n_functions.rst

@@ -515,13 +515,13 @@ It returns two values: the sentinel and, generally, a converted value that shoul
 -   'age' -- ``parse_date(value), datetime.datetime.now(tz=tzutc()) - timedelta(sentinel)``
     Note that these are reversed to make it easier to compare age against a given value.
     A global ``Now`` variable removes the need for an implicit age computation.
-    The :func:`parse_date` is the :func:`dateutil.parser.parse` function.
+    The :func:`parse_date` is the :func:`pendulum.parse` function.
 
 -   'integer' -- ``sentinel, int(str(value).strip())``
 
 -   'expiration' -- ``datetime.datetime.now(tz=tzutc()) + timedelta(sentinel), parse_date(value)``
     A global ``Now`` variable removes the need for an implicit expiration computation.
-    The :func:`parse_date` is the :func:`dateutil.parser.parse` function.
+    The :func:`parse_date` is the :func:`pendulum.parse` function.
 
 -   'normalize' -- ``sentinel, value.strip().lower()``
 

features/steps/c7n_integration.py

@@ -38,7 +38,8 @@
 from unittest.mock import Mock
 
 from behave import *
-from dateutil.parser import parse as parse_date
+# from dateutil.parser import parse as parse_date
+from pendulum import parse as parse_date
 
 import celpy
 import celpy.c7nlib

poetry.lock

@@ -29,19 +29,20 @@ packages = [{ include = "celpy", from = "src"}]
 [tool.poetry.dependencies]
 python = "^3.9"
 lark = "^0.12"
-python-dateutil = "^2.9.0.post0"
-types-python-dateutil = "^2.9.0.20240316"
 
 # specify as optional for pip install cel-python[c7n] or poetry install -e behavior
 jmespath = { version = "^1.0.1", optional = true }
-pyyaml = { version = "^6.0.1", optional = true }
-types-pyyaml = { version = "^6.0.12.20240311", optional = true}
 
 # specify as a non default group for building automatically in ci without -e
+pendulum = "^3.1.0"
+pyyaml = "^6.0.2"
+types-pyyaml = "^6.0.12.20250516"
 [tool.poetry.group.c7n.dependencies]
 jmespath = "^1.0.1"
-pyyaml = "^6.0.1"
-types-pyyaml = "^6.0.12.20240311"
+google-re2 = { version = "^1.0", optional = true }
+
+[tool.poetry.extras]
+re2 = ["google-re2"]
 
 [tool.poetry.group.dev.dependencies]
 behave = "^1.2.6"
@@ -51,6 +52,8 @@ pytest = "^8.2.1"
 pytest-cov = "^5.0.0"
 sphinx = "^6.0"
 tox = "^4.15.0"
+pre-commit = "^3.5"
+google-re2-stubs = "^0.1.0"
 
 [build-system]
 requires = ["poetry-core"]

pyproject.toml

@@ -288,7 +288,8 @@ def process(self, resources):
 from typing import (Any, Callable, Dict, Iterator, List, Optional, Type, Union,
                     cast)
 
-import dateutil
+# import dateutil
+from pendulum import parse as parse_date
 import jmespath  # type: ignore [import-untyped]
 
 from celpy import InterpretedRunner, celtypes
@@ -677,7 +678,8 @@ def image(resource: celtypes.MapType) -> celtypes.Value:
         image_name = ""
 
     return json_to_cel(
-        {"CreationDate": dateutil.parser.isoparse(creation_date), "Name": image_name}
+        # {"CreationDate": dateutil.parser.isoparse(creation_date), "Name": image_name}
+        {"CreationDate": parse_date(creation_date), "Name": image_name}
     )
 
 

src/celpy/c7nlib.py

@@ -172,13 +172,13 @@
 ===============
 
 An implementation may have additional timezone names that must be injected into
-th dateutil.gettz() processing.
+the ``pendulum`` processing. (Formerly ``dateutil.gettz()``.)
 
 For example, there may be the following sequence:
 
-1. A lowercase match for an alias or an existing dateutil timezone.
+1. A lowercase match for an alias or an existing timezone.
 
-2. A titlecase match for an existing dateutil timezone.
+2. A titlecase match for an existing timezone.
 
 3. The fallback, which is a +/-HH:MM string.
 
@@ -194,8 +194,10 @@
                     Optional, Sequence, Tuple, Type, TypeVar, Union, cast,
                     overload)
 
-import dateutil.parser
-import dateutil.tz
+import pendulum
+from pendulum import timezone
+import pendulum.tz.exceptions
+
 
 logger = logging.getLogger("celtypes")
 
@@ -1035,12 +1037,12 @@ class TimestampType(datetime.datetime):
     The Joda project (https://www.joda.org/joda-time/timezones.html)
     says "Time zone data is provided by the public IANA time zone database."
 
-    The ``dateutil`` project (https://pypi.org/project/python-dateutil/)
-    is used for TZ handling and timestamp parsing.
+    TZ handling and timestamp parsing is doine with
+    the ``pendulum`` (https://pendulum.eustace.io) project.
 
     Additionally, there is a ``TZ_ALIASES`` mapping available in this class to permit additional
     timezone names. By default, the mapping is empty, and the only names
-    available are those recognized by :mod:`dateutil.tz`.
+    available are those recognized by :mod:`pendulum.timezone`.
     """
 
     TZ_ALIASES: Dict[str, str] = {}
@@ -1076,7 +1078,7 @@ def __new__(
 
         elif isinstance(source, str):
             # Use dateutil to try a variety of text formats.
-            parsed_datetime = dateutil.parser.isoparse(source)
+            parsed_datetime = cast(datetime.datetime, pendulum.parse(source))
             return super().__new__(
                 cls,
                 year=parsed_datetime.year,
@@ -1143,10 +1145,15 @@ def tz_name_lookup(cls, tz_name: str) -> Optional[datetime.tzinfo]:
             Tweak ``celpy.celtypes.TimestampType.TZ_ALIASES``.
         """
         tz_lookup = str(tz_name)
+        tz: Optional[datetime.tzinfo]
         if tz_lookup in cls.TZ_ALIASES:
-            tz = dateutil.tz.gettz(cls.TZ_ALIASES[tz_lookup])
+            tz = timezone(cls.TZ_ALIASES[tz_lookup])
         else:
-            tz = dateutil.tz.gettz(tz_lookup)
+            try:
+                tz = cast(datetime.tzinfo, timezone(tz_lookup))
+            except pendulum.tz.exceptions.InvalidTimezone:
+                # ±hh:mm format...
+                tz = cls.tz_offset_parse(tz_name)
         return tz
 
     @classmethod
@@ -1165,11 +1172,9 @@ def tz_offset_parse(cls, tz_name: str) -> Optional[datetime.tzinfo]:
     def tz_parse(tz_name: Optional[str]) -> Optional[datetime.tzinfo]:
         if tz_name:
             tz = TimestampType.tz_name_lookup(tz_name)
-            if tz is None:
-                tz = TimestampType.tz_offset_parse(tz_name)
             return tz
         else:
-            return dateutil.tz.UTC
+            return timezone("UTC")
 
     def getDate(self, tz_name: Optional[StringType] = None) -> IntType:
         new_tz = self.tz_parse(tz_name)

src/celpy/celtypes.py

@@ -51,6 +51,12 @@
 import celpy.celtypes
 from celpy.celparser import tree_dump
 
+_USE_RE2 = True
+try:
+    import re2
+except ImportError:  # pragma: no cover
+    _USE_RE2 = False
+
 # A CEL type annotation. Used in an environment to describe objects as well as functions.
 # This is a list of types, plus Callable for conversion functions.
 Annotation = Union[
@@ -59,10 +65,8 @@
     Type[celpy.celtypes.FunctionType],  # Concrete class for annotations
 ]
 
-
 logger = logging.getLogger("evaluation")
 
-
 class CELSyntaxError(Exception):
     """CEL Syntax error -- the AST did not have the expected structure."""
     def __init__(self, arg: Any, line: Optional[int] = None, column: Optional[int] = None) -> None:
@@ -293,6 +297,27 @@ def operator_in(item: Result, container: Result) -> Result:
     return result
 
 
+def _function_matches_re(text: str, pattern: str) -> Result:
+    try:
+        m = re.search(pattern, text)
+    except re.error as ex:
+        return CELEvalError("match error", ex.__class__, ex.args)
+
+    return celpy.celtypes.BoolType(m is not None)
+
+
+def _function_matches_re2(text: str, pattern: str) -> Result:
+    try:
+        m = re2.search(pattern, text)
+    except re2.error as ex:
+        return CELEvalError("match error", ex.__class__, ex.args)
+
+    return celpy.celtypes.BoolType(m is not None)
+
+
+function_matches = _function_matches_re2 if _USE_RE2 else _function_matches_re
+
+
 def function_size(container: Result) -> Result:
     """
     The size() function applied to a Value. Delegate to Python's :py:func:`len`.
@@ -340,7 +365,7 @@ def function_size(container: Result) -> Result:
     # StringType methods
     "endsWith": lambda s, text: celpy.celtypes.BoolType(s.endswith(text)),
     "startsWith": lambda s, text: celpy.celtypes.BoolType(s.startswith(text)),
-    "matches": lambda s, pattern: celpy.celtypes.BoolType(re.search(pattern, s) is not None),
+    "matches": function_matches,
     "contains": lambda s, text: celpy.celtypes.BoolType(text in s),
     # TimestampType methods. Type details are redundant, but required because of the lambdas
     "getDate": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getDate(tz_name)),

src/celpy/evaluation.py

@@ -37,6 +37,7 @@
 import celpy.evaluation  # For monkeypatching
 from celpy import celparser, celtypes
 from celpy.evaluation import *
+from celpy.evaluation import _function_matches_re, _function_matches_re2
 
 
 def test_exception_syntax_error():
@@ -158,6 +159,25 @@ def test_operator_in():
     assert isinstance(operator_in(celtypes.IntType(-1), container_2), CELEvalError)
 
 
+def test_function_matches_re2():
+    empty_string = celtypes.StringType("")
+    # re2-specific patterns which behave differently than standard re
+    assert _function_matches_re2(empty_string, "^\\z")
+    assert isinstance(_function_matches_re2(empty_string, "^\\Z"), CELEvalError)
+
+
+def test_function_matches_re():
+    empty_string = celtypes.StringType("")
+    # re2-specific patterns which behave differently than standard re
+    assert isinstance(_function_matches_re(empty_string, "^\\z"), CELEvalError)
+    assert _function_matches_re(empty_string, "^\\Z")
+
+
+def test_function_matches():
+    empty_string = celtypes.StringType("")
+    assert function_matches(empty_string, "^$")
+
+
 def test_function_size():
     container_1 = celtypes.ListType([
         celtypes.IntType(42),

tests/test_evaluation.py

@@ -1,11 +1,11 @@
 Name              Lines  Precise  Imprecise  Any  Empty  Unanalyzed
 -------------------------------------------------------------------
 celpy               293       76          0    4    213           0
-celpy.__main__      465      164         15   42    244           0
+celpy.__main__      465      172          7   42    244           0
 celpy.adapter       137       35          3    9     85           5
-celpy.c7nlib       1582      340         15  154   1073           0
+celpy.c7nlib       1584      340         15  154   1075           0
 celpy.celparser     402      208          2   23    169           0
-celpy.celtypes     1495      430         17  221    788          39
-celpy.evaluation   2446      827         33  173   1398          15
+celpy.celtypes     1500      435         14  221    791          39
+celpy.evaluation   2471      839         33  176   1408          15
 xlate                 0        0          0    0      0           0
-xlate.c7n_to_cel   1730      384        105  145   1091           5
+xlate.c7n_to_cel   1730      387        102  145   1091           5