Dynamo is not patched and tries to connect to AWS

[phubbard]

This code fails with this error:

session.py                 292 DEBUG    Loading variable profile from defaults.
session.py                 292 DEBUG    Loading variable profile from defaults.
session.py                 289 DEBUG    Loading variable region from config file with value 'us-west-2'.
session.py                 292 DEBUG    Loading variable profile from defaults.
session.py                 292 DEBUG    Loading variable ca_bundle from defaults.
session.py                 292 DEBUG    Loading variable profile from defaults.
hooks.py                   210 DEBUG    Event choose-service-name: calling handler <function handle_service_name_alias at 0x103c4f8c8>
hooks.py                   210 DEBUG    Event creating-client-class.dynamodb: calling handler <function add_generate_presigned_url at 0x103f43488>
args.py                    167 DEBUG    The s3 config key is not a dictionary type, ignoring its value of: None
endpoint.py                245 DEBUG    Setting dynamodb timeout as (60, 60)
client.py                  126 DEBUG    Registering retry handlers for service: dynamodb
factory.py                  66 DEBUG    Loading dynamodb:dynamodb
hooks.py                   210 DEBUG    Event creating-resource-class.dynamodb.ServiceResource: calling handler <function lazy_call.<locals>._handler at 0x10549e510>
action.py                   81 DEBUG    Calling dynamodb:create_table with {'TableName': 'asasdfudit', 'KeySchema': [{'AttributeName': 'intuit_tid', 'KeyType': 'HASH'}, {'AttributeName': 'utc_timestamp', 'KeyType': 'RANGE'}], 'AttributeDefinitions': [{'AttributeName': 'authid', 'AttributeType': 'S'}, {'AttributeName': 'action', 'AttributeType': 'S'}], 'ProvisionedThroughput': {'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5}}
hooks.py                   210 DEBUG    Event provide-client-params.dynamodb.CreateTable: calling handler <function copy_dynamodb_params at 0x105a450d0>
hooks.py                   210 DEBUG    Event before-parameter-build.dynamodb.CreateTable: calling handler <bound method TransformationInjector.inject_condition_expressions of <boto3.dynamodb.transform.TransformationInjector object at 0x105be93c8>>
hooks.py                   210 DEBUG    Event before-parameter-build.dynamodb.CreateTable: calling handler <bound method TransformationInjector.inject_attribute_value_input of <boto3.dynamodb.transform.TransformationInjector object at 0x105be93c8>>
hooks.py                   210 DEBUG    Event before-parameter-build.dynamodb.CreateTable: calling handler <function generate_idempotent_uuid at 0x103f981e0>
endpoint.py                101 DEBUG    Making request for OperationModel(name=CreateTable) with params: {'url_path': '/', 'query_string': '', 'method': 'POST', 'headers': {'X-Amz-Target': 'DynamoDB_20120810.CreateTable', 'Content-Type': 'application/x-amz-json-1.0', 'User-Agent': 'Boto3/1.9.35 Python/3.7.0 Darwin/17.7.0 Botocore/1.12.35 Resource'}, 'body': b'{"TableName": "asasdfudit", "KeySchema": [{"AttributeName": "intuit_tid", "KeyType": "HASH"}, {"AttributeName": "utc_timestamp", "KeyType": "RANGE"}], "AttributeDefinitions": [{"AttributeName": "authid", "AttributeType": "S"}, {"AttributeName": "action", "AttributeType": "S"}], "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}}', 'url': 'https://dynamodb.us-west-2.amazonaws.com/', 'context': {'client_region': 'us-west-2', 'client_config': <botocore.config.Config object at 0x105bcf3c8>, 'has_streaming_input': False, 'auth_type': None}}
hooks.py                   210 DEBUG    Event request-created.dynamodb.CreateTable: calling handler <bound method RequestSigner.handler of <botocore.signers.RequestSigner object at 0x105bcf390>>
hooks.py                   210 DEBUG    Event choose-signer.dynamodb.CreateTable: calling handler <function set_operation_specific_signer at 0x103f980d0>
auth.py                    364 DEBUG    Calculating signature using v4 auth.
auth.py                    365 DEBUG    CanonicalRequest:
POST
/

content-type:application/x-amz-json-1.0
host:dynamodb.us-west-2.amazonaws.com
x-amz-date:20181101T174455Z
x-amz-security-token:FQoGZXIvYXdzEAcaDFN2lHW7fDbdz4GUNiKUAqcmNGdJmZRBIBXO5lzRKCNc8xB/DCu5pqB1krV2A5wxgHW42Uh3gh5yx17Dlt931fqMeigCnz9e+BYiXJvnj7/FIStofKI+L04Pe0WUeoEml7uUMMMT7yyLlp9dA+bTVKiQLx1KN3fdtWoTewYKZfvitClYYecAs+m79Vh81agkGKtc2cGpbQ/t7HFhPw7IBPyV9GzBHuHPajf+2xvmmL0VJxlaplJREdE0TuHLHIybvH3HrZQqQwYe++DD7I81lo6il0arml7HCipwoORtGSRNODDWAlunFNgVjh3SvkVERWp5LBmHJhq6NnlEhhDMpoLPj0NDPLaxifc7MCwFs2UKVhWBDyzq6gTrQK8sig3IHcS4jSifgfLbBQ==
x-amz-target:DynamoDB_20120810.CreateTable

content-type;host;x-amz-date;x-amz-security-token;x-amz-target
7fb88e2c41aa22321ba77c158df817faf8f20f5a5a9907c54722365d595ea133
auth.py                    367 DEBUG    StringToSign:
AWS4-HMAC-SHA256
20181101T174455Z
20181101/us-west-2/dynamodb/aws4_request
11c7e9990a5a2cacb030c0135f97ed9dfd4d684f03409ce2ea409fdb92bedfa2
auth.py                    369 DEBUG    Signature:
b2d01f9573acd2cb49a168a44268331d0503a1ac77277e16b20e96d5ead5ca46
endpoint.py                166 DEBUG    Sending http request: <AWSPreparedRequest stream_output=False, method=POST, url=https://dynamodb.us-west-2.amazonaws.com/, headers={'X-Amz-Target': b'DynamoDB_20120810.CreateTable', 'Content-Type': b'application/x-amz-json-1.0', 'User-Agent': b'Boto3/1.9.35 Python/3.7.0 Darwin/17.7.0 Botocore/1.12.35 Resource', 'X-Amz-Date': b'20181101T174455Z', 'X-Amz-Security-Token': b'FQoGZXIvYXdzEAcaDFN2lHW7fDbdz4GUNiKUAqcmNGdJmZRBIBXO5lzRKCNc8xB/DCu5pqB1krV2A5wxgHW42Uh3gh5yx17Dlt931fqMeigCnz9e+BYiXJvnj7/FIStofKI+L04Pe0WUeoEml7uUMMMT7yyLlp9dA+bTVKiQLx1KN3fdtWoTewYKZfvitClYYecAs+m79Vh81agkGKtc2cGpbQ/t7HFhPw7IBPyV9GzBHuHPajf+2xvmmL0VJxlaplJREdE0TuHLHIybvH3HrZQqQwYe++DD7I81lo6il0arml7HCipwoORtGSRNODDWAlunFNgVjh3SvkVERWp5LBmHJhq6NnlEhhDMpoLPj0NDPLaxifc7MCwFs2UKVhWBDyzq6gTrQK8sig3IHcS4jSifgfLbBQ==', 'Authorization': b'AWS4-HMAC-SHA256 Credential=ASIAWSQ6RL4X5RUL5I43/20181101/us-west-2/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-security-token;x-amz-target, Signature=b2d01f9573acd2cb49a168a44268331d0503a1ac77277e16b20e96d5ead5ca46', 'Content-Length': '354'}>
retry.py                   210 DEBUG    Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
connectionpool.py          813 DEBUG    Starting new HTTPS connection (1): dynamodb.us-west-2.amazonaws.com:443
connectionpool.py          393 DEBUG    https://dynamodb.us-west-2.amazonaws.com:443 "POST / HTTP/1.1" 400 125
parsers.py                 234 DEBUG    Response headers: {'Server': 'Server', 'Date': 'Thu, 01 Nov 2018 17:44:55 GMT', 'Content-Type': 'application/x-amz-json-1.0', 'Content-Length': '125', 'Connection': 'keep-alive', 'x-amzn-RequestId': 'E589NLT1E0KILIBSNQFI6BLSEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'x-amz-crc32': '2167418451'}
parsers.py                 235 DEBUG    Response body:
b'{"__type":"com.amazon.coral.service#ExpiredTokenException","message":"The security token included in the request is expired"}'
hooks.py                   210 DEBUG    Event needs-retry.dynamodb.CreateTable: calling handler <botocore.retryhandler.RetryHandler object at 0x105bcf7b8>
retryhandler.py            187 DEBUG    No retry needed.
hooks.py                   210 DEBUG    Event after-call.dynamodb.CreateTable: calling handler <bound method TransformationInjector.inject_attribute_value_output of <boto3.dynamodb.transform.TransformationInjector object at 0x105be93c8>>

Here is the test:

#!/usr/bin/env python3

# Integration tests, using localstack.

import os
import boto3
from aws_xray_sdk.core import xray_recorder
import pytest_localstack

xray_recorder.configure(sampling=False, context_missing='LOG_ERROR')

localstack = pytest_localstack.patch_fixture(
    services=['dynamodb'],  
    scope='module',  # Use the same Localstack container for all tests in this module.
    autouse=True,  # Automatically use this fixture in tests.
    region_name = 'us-west-2'
)

class TestIntegration:
    def setup_method(self):
        xray_recorder.begin_segment('integration_test')
        self.ddb = boto3.resource('dynamodb')
        self.table = self.ddb.create_table(TableName='audit',
                              KeySchema=[
                                  {
                                      'AttributeName': 'intuit_tid',
                                      'KeyType': 'HASH'
                                  },
                                  {
                                      'AttributeName': 'utc_timestamp',
                                      'KeyType': 'RANGE'
                                  }
                              ],
                              AttributeDefinitions=[
                                  {
                                      'AttributeName': 'authid',
                                      'AttributeType': 'S'
                                  },
                                  {
                                      'AttributeName': 'action',
                                      'AttributeType': 'S'
                                  },

                              ],
                              ProvisionedThroughput={
                                  'ReadCapacityUnits': 5,
                                  'WriteCapacityUnits': 5
                              })

    def teardown_method(self):
        xray_recorder.end_segment()

    def test_audit_db(self):
        assert self.table.item_count == 0

[jtdoepke]

Hi @phubbard

It seems like pytest fixtures don't get invoked for the setup_method(), so the DynamoDB table is being created before the botocore clients have been patched. Here a way that works by using a fixture instead of setup_method():

#!/usr/bin/env python3

# Integration tests, using localstack.

import os
import boto3
import pytest
from aws_xray_sdk.core import xray_recorder
import pytest_localstack

xray_recorder.configure(sampling=False, context_missing='LOG_ERROR')

localstack = pytest_localstack.patch_fixture(
    services=['dynamodb'],  
    scope='module',  # Use the same Localstack container for all tests in this module.
    autouse=True,  # Automatically use this fixture in tests.
    region_name = 'us-west-2'
)

class TestIntegration:
    @pytest.fixture(autouse=True)
    def setup_and_teardown(self):
        xray_recorder.begin_segment('integration_test')
        self.ddb = boto3.resource('dynamodb')
        self.table = self.ddb.create_table(TableName='audit',
                              KeySchema=[
                                  {
                                      'AttributeName': 'intuit_tid',
                                      'KeyType': 'HASH'
                                  },
                                  {
                                      'AttributeName': 'utc_timestamp',
                                      'KeyType': 'RANGE'
                                  }
                              ],
                              AttributeDefinitions=[
                                  {
                                      'AttributeName': 'authid',
                                      'AttributeType': 'S'
                                  },
                                  {
                                      'AttributeName': 'action',
                                      'AttributeType': 'S'
                                  },

                              ],
                              ProvisionedThroughput={
                                  'ReadCapacityUnits': 5,
                                  'WriteCapacityUnits': 5
                              })
        yield  # wait for test to complete, then teardown
        xray_recorder.end_segment()

    def test_audit_db(self):
        assert self.table.item_count == 0

[phubbard]

Thank you!