From b010ebfc476947e7cf60dbade6353d739b390a07 Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 15:15:31 +0200 Subject: [PATCH 1/7] feat: alter DynamoDB persistence class to set the table attribute lazily, making it easier to test. --- .../idempotency/persistence/dynamodb.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 73f241bd613..f82c23657d7 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -62,11 +62,11 @@ def __init__( >>> return {"StatusCode": 200} """ - boto_config = boto_config or Config() - session = boto3_session or boto3.session.Session() - self._ddb_resource = session.resource("dynamodb", config=boto_config) + self._boto_config = boto_config or Config() + self._boto3_session = boto3_session or boto3.session.Session() + + self._table = None self.table_name = table_name - self.table = self._ddb_resource.Table(self.table_name) self.key_attr = key_attr self.expiry_attr = expiry_attr self.status_attr = status_attr @@ -74,6 +74,25 @@ def __init__( self.validation_key_attr = validation_key_attr super(DynamoDBPersistenceLayer, self).__init__() + @property + def table(self): + """ + Caching property to store boto3 dynamodb Table resource + + """ + if self._table: + return self._table + ddb_resource = self._boto3_session.resource("dynamodb", config=self._boto_config) + self._table = ddb_resource.Table(self.table_name) + return self._table + + @table.setter + def table(self, table): + """ + Allow table instance variable to be set directly, primarily for use in tests + """ + self._table = table + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: """ Translate raw item records from DynamoDB to DataRecord @@ -125,7 +144,7 @@ def _put_record(self, data_record: DataRecord) -> None: ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr}, ExpressionAttributeValues={":now": int(now.timestamp())}, ) - except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException: + except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") raise IdempotencyItemAlreadyExistsError From 62a98f233016c83a50298dc2dccc7dbe473bb9b3 Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 15:19:35 +0200 Subject: [PATCH 2/7] feat: add environment variable to allow disabling idempotency functionality to make testing easier --- aws_lambda_powertools/shared/constants.py | 2 ++ .../utilities/idempotency/idempotency.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/aws_lambda_powertools/shared/constants.py b/aws_lambda_powertools/shared/constants.py index 622ffbce47b..45b46d236f9 100644 --- a/aws_lambda_powertools/shared/constants.py +++ b/aws_lambda_powertools/shared/constants.py @@ -21,3 +21,5 @@ XRAY_SDK_MODULE: str = "aws_xray_sdk" XRAY_SDK_CORE_MODULE: str = "aws_xray_sdk.core" + +IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED" diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 06c9a578aa2..6984cfbbd8e 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -3,9 +3,11 @@ """ import functools import logging +import os from typing import Any, Callable, Dict, Optional, cast from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from aws_lambda_powertools.shared.constants import IDEMPOTENCY_DISABLED_ENV from aws_lambda_powertools.shared.types import AnyCallableT from aws_lambda_powertools.utilities.idempotency.base import IdempotencyHandler from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig @@ -56,6 +58,9 @@ def idempotent( >>> return {"StatusCode": 200} """ + if os.getenv(IDEMPOTENCY_DISABLED_ENV): + return handler(event, context) + config = config or IdempotencyConfig() args = event, context idempotency_handler = IdempotencyHandler( @@ -122,6 +127,9 @@ def process_order(customer_id: str, order: dict, **kwargs): @functools.wraps(function) def decorate(*args, **kwargs): + if os.getenv(IDEMPOTENCY_DISABLED_ENV): + return function(*args, **kwargs) + payload = kwargs.get(data_keyword_argument) if payload is None: From b0d66659375aabd860168a96efa523a6f0dd43f4 Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 15:49:16 +0200 Subject: [PATCH 3/7] docs: add "testing your code" section to the idempotency docs --- docs/utilities/idempotency.md | 117 ++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index a9a5a129e63..4793d93b8c7 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -765,6 +765,123 @@ The idempotency utility can be used with the `validator` decorator. Ensure that !!! tip "JMESPath Powertools functions are also available" Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility. + +## Testing your code + +The idempotency utility provides several routes to test your code. + +### Disabling the idempotency utility +When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` +with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture: + +=== "tests.py" + + ```python hl_lines="2 3" + def test_idempotent_lambda_handler(monkeypatch): + # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions + monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1) + + result = handler() + ... + ``` +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + +### Testing with DynamoDB Local + +To test with DynamoDB local, you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url. + +=== "tests.py" + + ```python hl_lines="6 7 8" + import boto3 + + import app + + # Create our own Table resource using the endpoint for our DynamoDB Local instance + resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000') + table = resource.Table(app.persistence_layer.table_name) + app.persistence_layer.table = table + + def test_idempotent_lambda(): + result = app.handler({'testkey': 'testvalue'}, {}) + assert result['payment_id'] == 12345 + ``` + +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + +### How do I mock all DynamoDB I/O operations + +The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table) which it uses to access DynamoDB. +This means it is possible to pass a mocked Table resource, or stub various methods. + +=== "tests.py" + + ```python hl_lines="6 7 8 9" + from unittest.mock import MagicMock + + import app + + def test_idempotent_lambda(): + table = MagicMock() + app.persistence_layer.table = table + result = app.handler({'testkey': 'testvalue'}, {}) + table.put_item.assert_called() + ... + ``` + +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out From 7f9dade9cf1b5fa9ceec883571c24bca50a11c8e Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 16:25:49 +0200 Subject: [PATCH 4/7] chore: Add link to dynamodb local Co-authored-by: Heitor Lessa --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 4793d93b8c7..f3a296d5b0c 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -805,7 +805,7 @@ with a truthy value. If you prefer setting this for specific tests, and are usin ### Testing with DynamoDB Local -To test with DynamoDB local, you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url. +To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url. === "tests.py" From 7286396e5833defefa260cbae18c05689efc1bae Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 16:26:34 +0200 Subject: [PATCH 5/7] chore: alter example to move code inside test function Co-authored-by: Heitor Lessa --- docs/utilities/idempotency.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index f3a296d5b0c..6f74c07a918 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -814,12 +814,12 @@ To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/ import app - # Create our own Table resource using the endpoint for our DynamoDB Local instance - resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000') - table = resource.Table(app.persistence_layer.table_name) - app.persistence_layer.table = table - def test_idempotent_lambda(): + # Create our own Table resource using the endpoint for our DynamoDB Local instance + resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000') + table = resource.Table(app.persistence_layer.table_name) + app.persistence_layer.table = table + result = app.handler({'testkey': 'testvalue'}, {}) assert result['payment_id'] == 12345 ``` From 7eb2198e759f49e78522f2fc4397a951c64872c0 Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 16:46:00 +0200 Subject: [PATCH 6/7] chore: fix indentation in examples --- docs/utilities/idempotency.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 6f74c07a918..b063ebf5553 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -795,7 +795,7 @@ with a truthy value. If you prefer setting this for specific tests, and are usin @idempotent(persistence_store=persistence_layer) def handler(event, context): - print('expensive operation') + print('expensive operation') return { "payment_id": 12345, "message": "success", @@ -819,7 +819,7 @@ To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/ resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000') table = resource.Table(app.persistence_layer.table_name) app.persistence_layer.table = table - + result = app.handler({'testkey': 'testvalue'}, {}) assert result['payment_id'] == 12345 ``` @@ -835,7 +835,7 @@ To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/ @idempotent(persistence_store=persistence_layer) def handler(event, context): - print('expensive operation') + print('expensive operation') return { "payment_id": 12345, "message": "success", @@ -874,7 +874,7 @@ This means it is possible to pass a mocked Table resource, or stub various metho @idempotent(persistence_store=persistence_layer) def handler(event, context): - print('expensive operation') + print('expensive operation') return { "payment_id": 12345, "message": "success", From 2b8a4f2c0afbe09b7af2724bae0e0f76301a5e81 Mon Sep 17 00:00:00 2001 From: Tom McCarthy Date: Fri, 1 Oct 2021 16:46:31 +0200 Subject: [PATCH 7/7] chore: add test for new env var --- .../idempotency/test_idempotency.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index cb0d43ae6fa..b1d0914d181 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -3,6 +3,7 @@ import json import sys from hashlib import md5 +from unittest.mock import MagicMock import jmespath import pytest @@ -994,3 +995,25 @@ def dummy(payload): # WHEN dummy(payload=data_two) + + +def test_idempotency_disabled_envvar(monkeypatch, lambda_context, persistence_store: DynamoDBPersistenceLayer): + # Scenario to validate no requests sent to dynamodb table when 'POWERTOOLS_IDEMPOTENCY_DISABLED' is set + mock_event = {"data": "value"} + + persistence_store.table = MagicMock() + + monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", "1") + + @idempotent_function(data_keyword_argument="data", persistence_store=persistence_store) + def dummy(data): + return {"message": "hello"} + + @idempotent(persistence_store=persistence_store) + def dummy_handler(event, context): + return {"message": "hi"} + + dummy(data=mock_event) + dummy_handler(mock_event, lambda_context) + + assert len(persistence_store.table.method_calls) == 0