Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade Code to AWS SDK v3 #409

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 15 additions & 12 deletions lib/actions/purgeTable.js
Original file line number Diff line number Diff line change
@@ -1,22 +1,25 @@
const { DescribeTableCommand } = require('@aws-sdk/client-dynamodb')
const { doSearch } = require('../util')

const findPrimaryKeys = context => {
const params = {
TableName: context.tableName
}

return context.dynamodb.describeTable(params).promise()
.then(tableDescription => {
const tableSchema = tableDescription.Table.KeySchema
const DescribeTablePromise = context.dynamodbClient
.send(new DescribeTableCommand(params))

context.primaryKeys = ['HASH', 'RANGE']
.map(keyType => tableSchema.find(element => element.KeyType === keyType))
.filter(attribute => attribute)
.map(attribute => attribute.AttributeName)
return DescribeTablePromise.then(tableDescription => {
const tableSchema = tableDescription.Table.KeySchema

context.primaryKeys = ['HASH', 'RANGE']
.map(keyType => tableSchema.find(element => element.KeyType === keyType))
.filter(attribute => attribute)
.map(attribute => attribute.AttributeName)

return context
})

return context
})
}

const findAllElements = context => {
Expand All @@ -31,7 +34,7 @@ const findAllElements = context => {
ProjectionExpression: Object.keys(ExpressionAttributeNames).join(', '),
}

return doSearch(context.dynamodb, context.tableName, scanParams)
return doSearch(context.dynamodbClient, context.tableName, scanParams)
.then(items => {
context.items = items
return context
Expand Down Expand Up @@ -85,10 +88,10 @@ const deleteAllElements = context => {
* @param dynamodb the AWS dynamodb service that holds the connection
* @returns {Promise<any[] | never>} concatenation of all delete request promises
*/
const purgeTable = (tableName, dynamodb) => {
const purgeTable = (tableName, dynamodbClient) => {
const context = {
tableName,
dynamodb
dynamodbClient
}

return findPrimaryKeys(context)
Expand Down
202 changes: 80 additions & 122 deletions lib/backend.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
const express = require('express')
const path = require('path')
const fs = require('fs')
const os = require('os')
const errorhandler = require('errorhandler')
const { extractKey, extractKeysForItems, parseKey, doSearch } = require('./util')
const { purgeTable } = require('./actions/purgeTable')
Expand All @@ -11,6 +9,11 @@ const pickBy = require('lodash.pickby')
const clc = require('cli-color')
const cookieParser = require('cookie-parser')
const DEFAULT_THEME = process.env.DEFAULT_THEME || 'light'
const { DynamoDBClient, ListTablesCommand } = require('@aws-sdk/client-dynamodb')
const { CreateTableCommand, DescribeTableCommand } = require('@aws-sdk/client-dynamodb')
const { DeleteTableCommand, ScanCommand } = require('@aws-sdk/client-dynamodb')
const { DynamoDBDocumentClient, GetCommand } = require('@aws-sdk/lib-dynamodb')
const { PutCommand, DeleteCommand } = require('@aws-sdk/lib-dynamodb')

function loadDynamoEndpoint(env, dynamoConfig) {
if (typeof env.DYNAMO_ENDPOINT === 'string') {
Expand Down Expand Up @@ -43,43 +46,20 @@ function loadDynamoEndpoint(env, dynamoConfig) {
* @param AWS - the AWS SDK object
* @returns {{endpoint: string, sslEnabled: boolean, region: string, accessKeyId: string}}
*/
function loadDynamoConfig(env, AWS) {
function loadDynamoConfig(env) {
const dynamoConfig = {
endpoint: 'http://localhost:8000',
sslEnabled: false,
region: 'us-east-1',
accessKeyId: 'key',
secretAccessKey: env.AWS_SECRET_ACCESS_KEY || 'secret'
region: 'us-east-1'
}

loadDynamoEndpoint(env, dynamoConfig)

if (AWS.config) {
if (AWS.config.region !== undefined) {
dynamoConfig.region = AWS.config.region
}

if (AWS.config.credentials) {
if (AWS.config.credentials.accessKeyId !== undefined) {
dynamoConfig.accessKeyId = AWS.config.credentials.accessKeyId
}
}
}

if (env.AWS_REGION) {
dynamoConfig.region = env.AWS_REGION
}

if (env.AWS_ACCESS_KEY_ID) {
dynamoConfig.accessKeyId = env.AWS_ACCESS_KEY_ID
}

return dynamoConfig
}

const createAwsConfig = (AWS) => {
const createAwsConfig = () => {
rchl marked this conversation as resolved.
Show resolved Hide resolved
const env = process.env
const dynamoConfig = loadDynamoConfig(env, AWS)
const dynamoConfig = loadDynamoConfig(env)

console.log(clc.blackBright(` database endpoint: \t${dynamoConfig.endpoint}`))
console.log(clc.blackBright(` region: \t\t${dynamoConfig.region}`))
Expand All @@ -88,50 +68,30 @@ const createAwsConfig = (AWS) => {
return dynamoConfig
}

function getHomeDir() {
const env = process.env
const home = env.HOME || env.USERPROFILE
|| (env.HOMEPATH ? ((env.HOMEDRIVE || 'C:/') + env.HOMEPATH) : null)

if (home) {
return home
}

if (typeof os.homedir === 'function') {
return os.homedir()
}

return null
}

exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
exports.createServer = (dynamodbClient, docClient, expressInstance = express()) => {
const app = expressInstance
app.set('json spaces', 2)
app.set('view engine', 'ejs')
app.set('views', path.resolve(__dirname, '..', 'views'))

if (!dynamodb || !docClient) {
const homeDir = getHomeDir()

if (homeDir && fs.existsSync(path.join(homeDir, '.aws', 'credentials')) &&
fs.existsSync(path.join(homeDir, '.aws', 'config'))) {
process.env.AWS_SDK_LOAD_CONFIG = 1
}

const AWS = require('aws-sdk')

if (!dynamodb) {
dynamodb = new AWS.DynamoDB(createAwsConfig(AWS))
if (!dynamodbClient || !docClient) {
if (!dynamodbClient) {
dynamodbClient = new DynamoDBClient(createAwsConfig())
}

docClient = docClient || new AWS.DynamoDB.DocumentClient({ service: dynamodb })
docClient = docClient || DynamoDBDocumentClient.from(dynamodbClient)
}

const listTables = (...args) => dynamodb.listTables(...args).promise()
const describeTable = (...args) => dynamodb.describeTable(...args).promise()
const getItem = (...args) => docClient.get(...args).promise()
const putItem = (...args) => docClient.put(...args).promise()
const deleteItem = (...args) => docClient.delete(...args).promise()
const listTables = (...args) =>
dynamodbClient.send(new ListTablesCommand(...args))
const describeTable = (...args) =>
dynamodbClient.send(new DescribeTableCommand(...args))
const getItem = (...args) =>
docClient.send(new GetCommand(...args))
const putItem = (...args) =>
docClient.send(new PutCommand(...args))
const deleteItem = (...args) =>
docClient.send(new DeleteCommand(...args))

app.use(errorhandler())
app.use('/assets', express.static(path.join(__dirname, '..', 'public')))
Expand Down Expand Up @@ -218,8 +178,8 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
KeyType: 'HASH'
}
]
if (isAttributeNotAlreadyCreated(attributeDefinitions,
secondaryIndex.HashAttributeName)) {
if (
isAttributeNotAlreadyCreated(attributeDefinitions, secondaryIndex.HashAttributeName)) {
attributeDefinitions.push({
AttributeName: secondaryIndex.HashAttributeName,
AttributeType: secondaryIndex.HashAttributeType
Expand Down Expand Up @@ -250,8 +210,8 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {

if (secondaryIndex.IndexType === 'global') {
index.ProvisionedThroughput = {
ReadCapacityUnits: req.body.ReadCapacityUnits,
WriteCapacityUnits: req.body.WriteCapacityUnits
ReadCapacityUnits: Number(req.body.ReadCapacityUnits),
WriteCapacityUnits: Number(req.body.WriteCapacityUnits)
}
globalSecondaryIndexes.push(index)
} else {
Expand All @@ -266,19 +226,18 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
if (globalSecondaryIndexes === undefined || globalSecondaryIndexes.length === 0) {
globalSecondaryIndexes = undefined
}
return dynamodb
.createTable({
TableName: req.body.TableName,
ProvisionedThroughput: {
ReadCapacityUnits: req.body.ReadCapacityUnits,
WriteCapacityUnits: req.body.WriteCapacityUnits
},
GlobalSecondaryIndexes: globalSecondaryIndexes,
LocalSecondaryIndexes: localSecondaryIndexes,
KeySchema: keySchema,
AttributeDefinitions: attributeDefinitions
})
.promise()
const CreateTablePromise = dynamodbClient.send(new CreateTableCommand({
TableName: req.body.TableName,
ProvisionedThroughput: {
ReadCapacityUnits: Number(req.body.ReadCapacityUnits),
WriteCapacityUnits: Number(req.body.WriteCapacityUnits)
},
GlobalSecondaryIndexes: globalSecondaryIndexes,
LocalSecondaryIndexes: localSecondaryIndexes,
KeySchema: keySchema,
AttributeDefinitions: attributeDefinitions
}))
return CreateTablePromise
.then(() => {
res.status(204).end()
}).catch(error => {
Expand All @@ -292,9 +251,8 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
if (tablesList.length === 0) {
return res.send('There are no tables to delete')
}
await Promise.all(tablesList.map(table => dynamodb
.deleteTable({ TableName: table.TableName })
.promise()
await Promise.all(tablesList.map(table =>
dynamodbClient.send(new DeleteTableCommand({ TableName: table.TableName }))
))
return res.send('Tables deleted')
}))
Expand All @@ -304,22 +262,22 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
if (tablesList.length === 0) {
return res.send('There are no tables to purge')
}
await Promise.all(tablesList.map(table => purgeTable(table.TableName, dynamodb)))
await Promise.all(tablesList.map(table => purgeTable(table.TableName, dynamodbClient)))
return res.send('Tables purged')
}))

app.delete('/tables/:TableName', asyncMiddleware((req, res) => {
const TableName = req.params.TableName
return dynamodb
.deleteTable({ TableName })
.promise()

const deleteTablePromise = dynamodbClient.send(new DeleteTableCommand({ TableName }))
return deleteTablePromise
.then(() => {
res.status(204).end()
})
}))

app.delete('/tables/:TableName/all', asyncMiddleware((req, res) => {
return purgeTable(req.params.TableName, dynamodb)
return purgeTable(req.params.TableName, dynamodbClient)
.then(() => {
res.status(200).end()
})
Expand All @@ -330,12 +288,12 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
if (req.query.hash) {
if (req.query.range) {
return res.redirect(
`/tables/${encodeURIComponent(TableName)}/items/${
encodeURIComponent(req.query.hash)},${encodeURIComponent(req.query.range)}`
`/tables/${encodeURIComponent(TableName)}/items/${encodeURIComponent(req.query.hash)},
${encodeURIComponent(req.query.range)}`
)
} else {
return res.redirect(`/tables/${
encodeURIComponent(TableName)}/items/${encodeURIComponent(req.query.hash)}`)
return res.redirect(
`/tables/${encodeURIComponent(TableName)}/items/${encodeURIComponent(req.query.hash)}`)
}
}

Expand Down Expand Up @@ -371,36 +329,36 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
})
}))

const getPage = (docClient, keySchema, TableName, scanParams, pageSize,
startKey, operationType) => {
const pageItems = []
const getPage =
(docClient, keySchema, TableName, scanParams, pageSize, startKey, operationType) => {
const pageItems = []

function onNewItems(items, lastStartKey) {
for (let i = 0; i < items.length && pageItems.length < pageSize + 1; i++) {
pageItems.push(items[i])
}
function onNewItems(items, lastStartKey) {
for (let i = 0; i < items.length && pageItems.length < pageSize + 1; i++) {
pageItems.push(items[i])
}

// If there is more items to query (!lastStartKey) then don't stop until
// we are over pageSize count. Stopping at exactly pageSize count would
// not extract key of last item later and make pagination not work.
return pageItems.length > pageSize || !lastStartKey
}
// If there is more items to query (!lastStartKey) then don't stop until
// we are over pageSize count. Stopping at exactly pageSize count would
// not extract key of last item later and make pagination not work.
return pageItems.length > pageSize || !lastStartKey
}

return doSearch(docClient, TableName, scanParams, 10, startKey, onNewItems, operationType)
.then(items => {
let nextKey = null
return doSearch(docClient, TableName, scanParams, 10, startKey, onNewItems, operationType)
.then(items => {
let nextKey = null

if (items.length > pageSize) {
items = items.slice(0, pageSize)
nextKey = extractKey(items[pageSize - 1], keySchema)
}
if (items.length > pageSize) {
items = items.slice(0, pageSize)
nextKey = extractKey(items[pageSize - 1], keySchema)
}

return {
pageItems: items,
nextKey,
}
})
}
return {
pageItems: items,
nextKey,
}
})
}

app.get('/tables/:TableName', asyncMiddleware((req, res) => {
const TableName = req.params.TableName
Expand Down Expand Up @@ -435,7 +393,6 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
const TableName = req.params.TableName
req.query = pickBy(req.query)
const filters = req.query.filters ? JSON.parse(req.query.filters) : {}

return describeTable({ TableName })
.then(description => {
const ExclusiveStartKey = req.query.startKey
Expand Down Expand Up @@ -523,8 +480,9 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {

const pageSize = req.query.pageSize || 25

return getPage(docClient, description.Table.KeySchema, TableName,
params, pageSize, startKey, req.query.operationType)
return getPage(
docClient, description.Table.KeySchema, TableName,
params, pageSize, startKey, req.query.operationType)
.then(results => {
const { pageItems, nextKey } = results

Expand Down Expand Up @@ -568,7 +526,7 @@ exports.createServer = (dynamodb, docClient, expressInstance = express()) => {
const TableName = req.params.TableName
return Promise.all([
describeTable({ TableName }),
docClient.scan({ TableName }).promise()
() => docClient.send(new ScanCommand({ TableName }))
])
.then(([description, items]) => {
const data = Object.assign({}, description, items)
Expand Down
Loading