feat: expose jmespath powertools functions (#736)

Author: heitorlessa

aws_lambda_powertools/logging/logger.py

@@ -446,7 +446,7 @@ def set_package_logger(
     -------
     **Enables debug logging for AWS Lambda Powertools package**
 
-        >>> from aws_lambda_powertools.logging.logger import set_package_logger
+        >>> aws_lambda_powertools.logging.logger import set_package_logger
         >>> set_package_logger()
 
     Parameters

aws_lambda_powertools/utilities/feature_flags/appconfig.py

@@ -4,10 +4,10 @@
 
 from botocore.config import Config
 
+from aws_lambda_powertools.utilities import jmespath_utils
 from aws_lambda_powertools.utilities.parameters import AppConfigProvider, GetParameterError, TransformParameterError
 
 from ... import Logger
-from ...shared import jmespath_utils
 from .base import StoreProvider
 from .exceptions import ConfigurationStoreError, StoreClientError
 

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -16,7 +16,6 @@
 
 from aws_lambda_powertools.shared import constants
 from aws_lambda_powertools.shared.cache_dict import LRUDict
-from aws_lambda_powertools.shared.jmespath_utils import PowertoolsFunctions
 from aws_lambda_powertools.shared.json_encoder import Encoder
 from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
 from aws_lambda_powertools.utilities.idempotency.exceptions import (
@@ -25,6 +24,7 @@
     IdempotencyKeyError,
     IdempotencyValidationError,
 )
+from aws_lambda_powertools.utilities.jmespath_utils import PowertoolsFunctions
 
 logger = logging.getLogger(__name__)
 

aws_lambda_powertools/shared/jmespath_utils.py renamed to aws_lambda_powertools/utilities/jmespath_utils/__init__.py

@@ -30,8 +30,27 @@ def _func_powertools_base64_gzip(self, value):
         return uncompressed.decode()
 
 
-def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_options: Optional[Dict]) -> Any:
-    """Searches data using JMESPath expression
+def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_options: Optional[Dict] = None) -> Any:
+    """Searches and extracts data using JMESPath
+
+    Envelope being the JMESPath expression to extract the data you're after
+
+    Built-in JMESPath functions include: powertools_json, powertools_base64, powertools_base64_gzip
+
+    Examples
+    --------
+
+    **Deserialize JSON string and extracts data from body key**
+
+        from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope
+        from aws_lambda_powertools.utilities.typing import LambdaContext
+
+
+        def handler(event: dict, context: LambdaContext):
+            # event = {"body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}"}  # noqa: E800
+            payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)")
+            customer = payload.get("customerId")  # now deserialized
+            ...
 
     Parameters
     ----------
@@ -42,6 +61,7 @@ def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_o
     jmespath_options : Dict
         Alternative JMESPath options to be included when filtering expr
 
+
     Returns
     -------
     Any

aws_lambda_powertools/utilities/jmespath_utils/envelopes.py

@@ -0,0 +1,8 @@
+API_GATEWAY_REST = "powertools_json(body)"
+API_GATEWAY_HTTP = API_GATEWAY_REST
+SQS = "Records[*].powertools_json(body)"
+SNS = "Records[0].Sns.Message | powertools_json(@)"
+EVENTBRIDGE = "detail"
+CLOUDWATCH_EVENTS_SCHEDULED = EVENTBRIDGE
+KINESIS_DATA_STREAM = "Records[*].kinesis.powertools_json(powertools_base64(data))"
+CLOUDWATCH_LOGS = "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]"

aws_lambda_powertools/utilities/validation/validator.py

@@ -1,8 +1,9 @@
 import logging
 from typing import Any, Callable, Dict, Optional, Union
 
+from aws_lambda_powertools.utilities import jmespath_utils
+
 from ...middleware_factory import lambda_handler_decorator
-from ...shared import jmespath_utils
 from .base import validate_data_against_schema
 
 logger = logging.getLogger(__name__)

docs/utilities/idempotency.md

@@ -209,7 +209,7 @@ Imagine the function executes successfully, but the client never receives the re
 !!! warning "Idempotency for JSON payloads"
     The payload extracted by the `event_key_jmespath` is treated as a string by default, so will be sensitive to differences in whitespace even when the JSON payload itself is identical.
 
-    To alter this behaviour, we can use the [JMESPath built-in function](/utilities/jmespath_functions) *powertools_json()* to treat the payload as a JSON object rather than a string.
+    To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function) `powertools_json()` to treat the payload as a JSON object rather than a string.
 
 === "payment.py"
 

docs/utilities/jmespath_functions.md

@@ -3,9 +3,106 @@ title: JMESPath Functions
 description: Utility
 ---
 
-You might have events or responses that contain non-encoded JSON, where you need to decode so that you can access portions of the object or ensure the Powertools utility receives a JSON object. This is a common use case when using the [validation](/utilities/validation) or [idempotency](/utilities/idempotency) utilities.  
+!!! tip "JMESPath is a query language for JSON used by AWS CLI, AWS Python SDK, and AWS Lambda Powertools for Python."
 
-## Built-in JMESPath functions
+Built-in [JMESPath](https://jmespath.org/){target="_blank"} Functions to easily deserialize common encoded JSON payloads in Lambda functions.
+
+## Key features
+
+* Deserialize JSON from JSON strings, base64, and compressed data
+* Use JMESPath to extract and combine data recursively
+
+## Getting started
+
+You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation.
+
+Lambda Powertools also have utilities like [validation](validation.md), [idempotency](idempotency.md), or [feature flags](feature_flags.md) where you might need to extract a portion of your data before using them.
+
+### Extracting data
+
+You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}.
+
+=== "app.py"
+
+	```python hl_lines="1 7"
+    from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope
+
+    from aws_lambda_powertools.utilities.typing import LambdaContext
+
+
+    def handler(event: dict, context: LambdaContext):
+        payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)")
+        customer = payload.get("customerId")  # now deserialized
+        ...
+	```
+
+=== "event.json"
+
+    ```json
+    {
+        "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}"
+    }
+    ```
+
+### Built-in envelopes
+
+We provide built-in envelopes for popular JMESPath expressions used when looking to decode/deserialize JSON objects within AWS Lambda Event Sources.
+
+=== "app.py"
+
+	```python hl_lines="1 7"
+    from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope, envelopes
+
+    from aws_lambda_powertools.utilities.typing import LambdaContext
+
+
+    def handler(event: dict, context: LambdaContext):
+        payload = extract_data_from_envelope(data=event, envelope=envelopes.SNS)
+        customer = payload.get("customerId")  # now deserialized
+        ...
+	```
+
+=== "event.json"
+
+    ```json hl_lines="6"
+    {
+        "Records": [
+            {
+                "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
+                "receiptHandle": "MessageReceiptHandle",
+                "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\",\"booking\":{\"id\":\"5b2c4803-330b-42b7-811a-c68689425de1\",\"reference\":\"ySz7oA\",\"outboundFlightId\":\"20c0d2f2-56a3-4068-bf20-ff7703db552d\"},\"payment\":{\"receipt\":\"https:\/\/pay.stripe.com\/receipts\/acct_1Dvn7pF4aIiftV70\/ch_3JTC14F4aIiftV700iFq2CHB\/rcpt_K7QsrFln9FgFnzUuBIiNdkkRYGxUL0X\",\"amount\":100}}",
+                "attributes": {
+                    "ApproximateReceiveCount": "1",
+                    "SentTimestamp": "1523232000000",
+                    "SenderId": "123456789012",
+                    "ApproximateFirstReceiveTimestamp": "1523232000001"
+                },
+                "messageAttributes": {},
+                "md5OfBody": "7b270e59b47ff90a553787216d55d91d",
+                "eventSource": "aws:sqs",
+                "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
+                "awsRegion": "us-east-1"
+            }
+        ]
+    }
+    ```
+
+These are all built-in envelopes you can use along with their expression as a reference:
+
+Envelope | JMESPath expression
+------------------------------------------------- | ---------------------------------------------------------------------------------
+**`API_GATEWAY_REST`** | `powertools_json(body)`
+**`API_GATEWAY_HTTP`** | `API_GATEWAY_REST`
+**`SQS`** | `Records[*].powertools_json(body)`
+**`SNS`** | `Records[0].Sns.Message | powertools_json(@)`
+**`EVENTBRIDGE`** | `detail`
+**`CLOUDWATCH_EVENTS_SCHEDULED`** | `EVENTBRIDGE`
+**`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))`
+**`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]`
+
+## Advanced
+
+### Built-in JMESPath functions
 You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data.
 
 !!! info
@@ -134,33 +231,35 @@ This sample will decompress and decode base64 data, then use JMESPath pipeline e
 !!! warning
     This should only be used for advanced use cases where you have special formats not covered by the built-in functions.
 
-    This will **replace all provided built-in functions such as `powertools_json`, so you will no longer be able to use them**.
-
 For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param.
 
-=== "custom_jmespath_function.py"
+In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`:
 
-    ```python hl_lines="2 6-10 14"
-    from aws_lambda_powertools.utilities.validation import validator
-    from jmespath import functions
+=== "custom_jmespath_function.py"
 
-    import schemas
+    ```python hl_lines="2-3 6-9 11 17"
+	from aws_lambda_powertools.utilities.jmespath_utils import (
+	    PowertoolsFunctions, extract_data_from_envelope)
+	from jmespath.functions import signature
 
-    class CustomFunctions(functions.Functions):
 
-        @functions.signature({'types': ['string']})
+    class CustomFunctions(PowertoolsFunctions):
+        @signature({'types': ['string']})  # Only decode if value is a string
         def _func_special_decoder(self, s):
             return my_custom_decoder_logic(s)
 
     custom_jmespath_options = {"custom_functions": CustomFunctions()}
 
-    @validator(schema=schemas.INPUT, jmespath_options=**custom_jmespath_options)
     def handler(event, context):
-        return event
+		# use the custom name after `_func_`
+		extract_data_from_envelope(data=event,
+								  envelope="special_decoder(body)",
+								  jmespath_options=**custom_jmespath_options)
+        ...
     ```
 
-=== "schemas.py"
+=== "event.json"
 
-    ```python hl_lines="7 14 16 23 39 45 47 52"
-    --8<-- "docs/shared/validation_basic_jsonschema.py"
+    ```json
+    	{"body": "custom_encoded_data"}
     ```