Merge pull request #51 from jg-rp/fake-root

Add non-standard fake root identifier.

Author: jg-rp

.github/workflows/tests.yaml

@@ -9,7 +9,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        exclude:
+          - os: macos-latest
+            python-version: "3.7"
+          - os: windows-latest
+            python-version: "3.7"
     steps:
       - uses: actions/checkout@v3
         with:

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Python JSONPath Change Log
 
+## Version 0.11.0 (unreleased)
+
+**Fixes**
+
+- The lexer now sorts environment-controlled tokens by their length in descending order. This allows one custom token to be a prefix of another.
+
+**Features**
+
+- Added the non-standard "fake root" identifier, which defaults to `^` and can be customized with the `fake_root_token` attribute on a `JSONPathEnvironment` subclass. Using the fake root identifier is equivalent to the standard root identifier (`$`), but wraps the target JSON value in an array, so the root value can be conditionally selected using a filter.
+- Non-standard environment-controlled tokens can now be disabled by setting them to the empty string.
+
 ## Version 0.10.3
 
 **Changes**

docs/syntax.md

@@ -162,6 +162,16 @@ Filter expressions can call predefined [function extensions](functions.md) too.
 $.categories[?count(@.products.*) >= 2]
 ```
 
+### Fake root (`^`)
+
+**_New in version 0.11.0_**
+
+This non-standard "fake root" identifier behaves like the standard root identifier (`$`), but wraps the target JSON document in a single-element array, so as to make it selectable with a filter selector.
+
+```text
+^[?length(categories) > 0]
+```
+
 ### Union (`|`) and intersection (`&`)
 
 Union (`|`) and intersection (`&`) are similar to Python's set operations, but we don't dedupe the matches (matches will often contain unhashable objects).
@@ -209,3 +219,4 @@ And this is a list of features that are uncommon or unique to Python JSONPath.
 - `#` is the current key/property or index identifier when filtering a mapping or sequence.
 - `_` is a filter context selector. With usage similar to `$` and `@`, `_` exposes arbitrary data from the `filter_context` argument to `findall()` and `finditer()`.
 - `~` is a "keys" or "properties" selector.
+- `^` is a "fake root" identifier. It is equivalent to `$`, but wraps the target JSON document in a single-element array, so the root value can be conditionally selected with a filter selector.

jsonpath/env.py

@@ -38,6 +38,7 @@
 from .path import JSONPath
 from .stream import TokenStream
 from .token import TOKEN_EOF
+from .token import TOKEN_FAKE_ROOT
 from .token import TOKEN_INTERSECTION
 from .token import TOKEN_UNION
 from .token import Token
@@ -89,6 +90,8 @@ class attributes `root_token`, `self_token` and `filter_context_token`.
             **New in version 0.10.0**
 
     Attributes:
+        fake_root_token (str): The pattern used to select a "fake" root node, one level
+            above the real root node.
         filter_context_token (str): The pattern used to select extra filter context
             data. Defaults to `"_"`.
         intersection_token (str): The pattern used as the intersection operator.
@@ -112,6 +115,7 @@ class attributes `root_token`, `self_token` and `filter_context_token`.
 
     # These should be unescaped strings. `re.escape` will be called
     # on them automatically when compiling lexer rules.
+    fake_root_token = "^"
     filter_context_token = "_"
     intersection_token = "&"
     key_token = "#"
@@ -174,8 +178,9 @@ def compile(self, path: str) -> Union[JSONPath, CompoundJSONPath]:  # noqa: A003
         """
         tokens = self.lexer.tokenize(path)
         stream = TokenStream(tokens)
+        fake_root = stream.current.kind == TOKEN_FAKE_ROOT
         _path: Union[JSONPath, CompoundJSONPath] = JSONPath(
-            env=self, selectors=self.parser.parse(stream)
+            env=self, selectors=self.parser.parse(stream), fake_root=fake_root
         )
 
         if stream.current.kind != TOKEN_EOF:
@@ -190,18 +195,22 @@ def compile(self, path: str) -> Union[JSONPath, CompoundJSONPath]:  # noqa: A003
 
                 if stream.current.kind == TOKEN_UNION:
                     stream.next_token()
+                    fake_root = stream.current.kind == TOKEN_FAKE_ROOT
                     _path = _path.union(
                         JSONPath(
                             env=self,
                             selectors=self.parser.parse(stream),
+                            fake_root=fake_root,
                         )
                     )
                 elif stream.current.kind == TOKEN_INTERSECTION:
                     stream.next_token()
+                    fake_root = stream.current.kind == TOKEN_FAKE_ROOT
                     _path = _path.intersection(
                         JSONPath(
                             env=self,
                             selectors=self.parser.parse(stream),
+                            fake_root=fake_root,
                         )
                     )
                 else:  # pragma: no cover

jsonpath/lex.py

@@ -16,6 +16,7 @@
 from .token import TOKEN_DOT_PROPERTY
 from .token import TOKEN_DOUBLE_QUOTE_STRING
 from .token import TOKEN_EQ
+from .token import TOKEN_FAKE_ROOT
 from .token import TOKEN_FALSE
 from .token import TOKEN_FILTER
 from .token import TOKEN_FILTER_CONTEXT
@@ -119,6 +120,17 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
 
     def compile_rules(self) -> Pattern[str]:
         """Prepare regular expression rules."""
+        env_tokens = [
+            (TOKEN_ROOT, self.env.root_token),
+            (TOKEN_FAKE_ROOT, self.env.fake_root_token),
+            (TOKEN_SELF, self.env.self_token),
+            (TOKEN_KEY, self.env.key_token),
+            (TOKEN_UNION, self.env.union_token),
+            (TOKEN_INTERSECTION, self.env.intersection_token),
+            (TOKEN_FILTER_CONTEXT, self.env.filter_context_token),
+            (TOKEN_KEYS, self.env.keys_selector_token),
+        ]
+
         rules = [
             (TOKEN_DOUBLE_QUOTE_STRING, self.double_quote_pattern),
             (TOKEN_SINGLE_QUOTE_STRING, self.single_quote_pattern),
@@ -131,13 +143,13 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_DDOT, r"\.\."),
             (TOKEN_AND, self.logical_and_pattern),
             (TOKEN_OR, self.logical_or_pattern),
-            (TOKEN_ROOT, re.escape(self.env.root_token)),
-            (TOKEN_SELF, re.escape(self.env.self_token)),
-            (TOKEN_KEY, re.escape(self.env.key_token)),
-            (TOKEN_UNION, re.escape(self.env.union_token)),
-            (TOKEN_INTERSECTION, re.escape(self.env.intersection_token)),
-            (TOKEN_FILTER_CONTEXT, re.escape(self.env.filter_context_token)),
-            (TOKEN_KEYS, re.escape(self.env.keys_selector_token)),
+            *[
+                (token, re.escape(pattern))
+                for token, pattern in sorted(
+                    env_tokens, key=lambda x: len(x[1]), reverse=True
+                )
+                if pattern
+            ],
             (TOKEN_WILD, r"\*"),
             (TOKEN_FILTER, r"\?"),
             (TOKEN_IN, r"in"),

jsonpath/parse.py

@@ -53,6 +53,7 @@
 from .token import TOKEN_DOUBLE_QUOTE_STRING
 from .token import TOKEN_EOF
 from .token import TOKEN_EQ
+from .token import TOKEN_FAKE_ROOT
 from .token import TOKEN_FALSE
 from .token import TOKEN_FILTER
 from .token import TOKEN_FILTER_CONTEXT
@@ -213,8 +214,12 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
         self.env = env
 
         self.token_map: Dict[str, Callable[[TokenStream], FilterExpression]] = {
+            TOKEN_DOUBLE_QUOTE_STRING: self.parse_string_literal,
+            TOKEN_FAKE_ROOT: self.parse_root_path,
             TOKEN_FALSE: self.parse_boolean,
+            TOKEN_FILTER_CONTEXT: self.parse_filter_context_path,
             TOKEN_FLOAT: self.parse_float_literal,
+            TOKEN_FUNCTION: self.parse_function_extension,
             TOKEN_INT: self.parse_integer_literal,
             TOKEN_KEY: self.parse_current_key,
             TOKEN_LIST_START: self.parse_list_literal,
@@ -227,12 +232,9 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
             TOKEN_RE_PATTERN: self.parse_regex,
             TOKEN_ROOT: self.parse_root_path,
             TOKEN_SELF: self.parse_self_path,
-            TOKEN_FILTER_CONTEXT: self.parse_filter_context_path,
-            TOKEN_DOUBLE_QUOTE_STRING: self.parse_string_literal,
             TOKEN_SINGLE_QUOTE_STRING: self.parse_string_literal,
             TOKEN_TRUE: self.parse_boolean,
             TOKEN_UNDEFINED: self.parse_undefined,
-            TOKEN_FUNCTION: self.parse_function_extension,
         }
 
         self.list_item_map: Dict[str, Callable[[TokenStream], FilterExpression]] = {
@@ -250,25 +252,26 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
         self.function_argument_map: Dict[
             str, Callable[[TokenStream], FilterExpression]
         ] = {
+            TOKEN_DOUBLE_QUOTE_STRING: self.parse_string_literal,
+            TOKEN_FAKE_ROOT: self.parse_root_path,
             TOKEN_FALSE: self.parse_boolean,
+            TOKEN_FILTER_CONTEXT: self.parse_filter_context_path,
             TOKEN_FLOAT: self.parse_float_literal,
+            TOKEN_FUNCTION: self.parse_function_extension,
             TOKEN_INT: self.parse_integer_literal,
             TOKEN_KEY: self.parse_current_key,
             TOKEN_NIL: self.parse_nil,
             TOKEN_NONE: self.parse_nil,
             TOKEN_NULL: self.parse_nil,
-            TOKEN_SINGLE_QUOTE_STRING: self.parse_string_literal,
-            TOKEN_DOUBLE_QUOTE_STRING: self.parse_string_literal,
-            TOKEN_TRUE: self.parse_boolean,
             TOKEN_ROOT: self.parse_root_path,
             TOKEN_SELF: self.parse_self_path,
-            TOKEN_FILTER_CONTEXT: self.parse_filter_context_path,
-            TOKEN_FUNCTION: self.parse_function_extension,
+            TOKEN_SINGLE_QUOTE_STRING: self.parse_string_literal,
+            TOKEN_TRUE: self.parse_boolean,
         }
 
     def parse(self, stream: TokenStream) -> Iterable[JSONPathSelector]:
         """Parse a JSONPath from a stream of tokens."""
-        if stream.current.kind == TOKEN_ROOT:
+        if stream.current.kind in {TOKEN_ROOT, TOKEN_FAKE_ROOT}:
             stream.next_token()
         yield from self.parse_path(stream, in_filter=False)
 
@@ -533,9 +536,14 @@ def parse_grouped_expression(self, stream: TokenStream) -> FilterExpression:
         return expr
 
     def parse_root_path(self, stream: TokenStream) -> FilterExpression:
-        stream.next_token()
+        root = stream.next_token()
+        assert root.kind in {TOKEN_ROOT, TOKEN_FAKE_ROOT}  # XXX:
         return RootPath(
-            JSONPath(env=self.env, selectors=self.parse_path(stream, in_filter=True))
+            JSONPath(
+                env=self.env,
+                selectors=self.parse_path(stream, in_filter=True),
+                fake_root=root.kind == TOKEN_FAKE_ROOT,
+            )
         )
 
     def parse_self_path(self, stream: TokenStream) -> FilterExpression:

jsonpath/path.py

@@ -35,22 +35,27 @@ class JSONPath:
         env: The `JSONPathEnvironment` this path is bound to.
         selectors: An iterable of `JSONPathSelector` objects, as generated by
             a `Parser`.
+        fake_root: Indicates if target JSON values should be wrapped in a single-
+            element array, so as to make the target root value selectable.
+
 
     Attributes:
         env: The `JSONPathEnvironment` this path is bound to.
         selectors: The `JSONPathSelector` instances that make up this path.
     """
 
-    __slots__ = ("env", "selectors")
+    __slots__ = ("env", "fake_root", "selectors")
 
     def __init__(
         self,
         *,
         env: JSONPathEnvironment,
         selectors: Iterable[JSONPathSelector],
+        fake_root: bool = False,
     ) -> None:
         self.env = env
         self.selectors = tuple(selectors)
+        self.fake_root = fake_root
 
     def __str__(self) -> str:
         return self.env.root_token + "".join(
@@ -122,7 +127,7 @@ def finditer(
         matches: Iterable[JSONPathMatch] = [
             JSONPathMatch(
                 filter_context=filter_context or {},
-                obj=_data,
+                obj=[_data] if self.fake_root else _data,
                 parent=None,
                 path=self.env.root_token,
                 parts=(),
@@ -161,7 +166,7 @@ async def finditer_async(
         async def root_iter() -> AsyncIterable[JSONPathMatch]:
             yield self.env.match_class(
                 filter_context=filter_context or {},
-                obj=_data,
+                obj=[_data] if self.fake_root else _data,
                 parent=None,
                 path=self.env.root_token,
                 parts=(),

jsonpath/token.py

@@ -15,6 +15,7 @@
 TOKEN_DOT_INDEX = sys.intern("DINDEX")
 TOKEN_DOT_PROPERTY = sys.intern("DOT_PROPERTY")
 TOKEN_FILTER = sys.intern("FILTER")
+TOKEN_FAKE_ROOT = sys.intern("FAKE_ROOT")
 TOKEN_KEY = sys.intern("KEY")
 TOKEN_KEYS = sys.intern("KEYS")
 TOKEN_RBRACKET = sys.intern("RBRACKET")

tests/test_env.py

@@ -5,6 +5,7 @@
 import pytest
 
 from jsonpath import JSONPathEnvironment
+from jsonpath import JSONPathSyntaxError
 from jsonpath import JSONPathTypeError
 
 
@@ -173,6 +174,41 @@ class MyJSONPathEnvironment(JSONPathEnvironment):
     assert env.findall("$.foo.*", data) == [1, 2, 3]
 
 
+def test_custom_fake_root_identifier_token() -> None:
+    """Test that we can change the non-standard fake root identifier."""
+
+    class MyJSONPathEnvironment(JSONPathEnvironment):
+        fake_root_token = "$$"
+
+    env = MyJSONPathEnvironment()
+    data = {"foo": {"a": 1, "b": 2, "c": 3}}
+    assert env.findall("$$[[email protected] == 1]", data) == [data]
+    assert env.findall("$$[[email protected] == 7]", data) == []
+    assert env.findall("$.*", data) == [{"a": 1, "b": 2, "c": 3}]
+
+
+def test_disable_fake_root_identifier() -> None:
+    """Test that we can disable the non-standard fake root identifier."""
+
+    class MyJSONPathEnvironment(JSONPathEnvironment):
+        fake_root_token = ""
+
+    env = MyJSONPathEnvironment()
+    with pytest.raises(JSONPathSyntaxError):
+        env.compile("^[[email protected] == 42]")
+
+
+def test_disable_keys_selector() -> None:
+    """Test that we can disable the non-standard keys selector."""
+
+    class MyJSONPathEnvironment(JSONPathEnvironment):
+        keys_selector_token = ""
+
+    env = MyJSONPathEnvironment()
+    with pytest.raises(JSONPathSyntaxError):
+        env.compile("*..~")
+
+
 def test_disable_well_typed_checks() -> None:
     """Test that we can disable checks for well-typedness."""
     env = JSONPathEnvironment(well_typed=True)

tests/test_find.py

@@ -57,6 +57,24 @@ class Case:
         },
         want=[{"foo": 1}, {"foo": 2}],
     ),
+    Case(
+        description="select root value using fake root",
+        path="^[[email protected] > 7]",
+        data={"some": {"thing": 42}},
+        want=[{"some": {"thing": 42}}],
+    ),
+    Case(
+        description="fake root in a filter query",
+        path="^[[email protected] > value(^.*.num)]",
+        data={"some": {"thing": 42}, "num": 7},
+        want=[{"some": {"thing": 42}, "num": 7}],
+    ),
+    Case(
+        description="recurse object keys",
+        path="$..~",
+        data={"some": {"thing": "else", "foo": {"bar": "baz"}}},
+        want=["some", "thing", "foo", "bar"],
+    ),
 ]