Daily Coding - Reading data from AWS DynamoDB tables

Here is a quick post to read options data for my project silkrouteinvestments.

Source: silkroute-dynamodb-analysis.ipynb

Steps are as follows:

• Setup the config file in '.aws.dynamo.env'
• Install libraries (I have a lot of un-necessary ones here .. legacy issues!)
• This is a script generated using jupyter nbconvert --to script silkroute-dynamodb-analysis.ipynb

#!/usr/bin/env python
# coding: utf-8

# In[2]:


import sys, json
import pandas as pd
from pymongo import MongoClient
import numpy as np
import platform
from pprint import pprint
from os.path import expanduser
import datetime
from os.path import join, dirname
from dotenv import load_dotenv
import os


# In[3]:


# credit: https://github.com/theskumar/python-dotenv
# OR, explicitly providing path to '.env'
from pathlib import Path  # python3 only
# 

cwd = os.getcwd()
print(cwd)
env_path = Path(cwd) / '.aws.dynamo.env'
print(env_path)

load_dotenv(dotenv_path=env_path, verbose=True)


# In[4]:


AWSAccessKeyId = os.getenv("AWSAccessKeyId")
AWSSecretAccessKey = os.getenv("AWSSecretAccessKey")
AWSRegion = os.getenv("AWSRegion")
AWSBucket = os.getenv("AWSBucket")

print(AWSAccessKeyId)


# In[9]:


# https://stackoverflow.com/questions/36780856/complete-scan-of-dynamodb-with-boto3
import boto3
import requests
from urllib.parse import urlparse
from io import BytesIO;
import contextlib
import mimetypes
from slugify import slugify
import pathlib


session = boto3.Session(
    aws_access_key_id=AWSAccessKeyId,
    aws_secret_access_key=AWSSecretAccessKey
)

client = boto3.client('dynamodb')
dynamodb = session.resource('dynamodb')


# In[10]:


response = client.list_tables()

print(response['TableNames'])


# In[11]:


table = dynamodb.Table('options')


# In[12]:


# credit: https://stackoverflow.com/a/38619425/644081
response = table.scan()
data = response['Items']


# In[13]:


# credit: https://stackoverflow.com/questions/45636460/use-pythons-pandas-to-treat-aws-dynamodb-data
from dynamodb_json import json_util as json

obj = pd.DataFrame(json.loads(data))
obj