I'm new to Dynamo and have been trying to query the DB for a range.
I'm doing a query for an ISO date string in my Lambda like this:
const start = event['queryStringParameters']['startDate'];
const end = event['queryStringParameters']['endDate'];
let params = {
TableName: "Parcel",
KeyConditionExpression: "#trackNo = :trackNo AND #ShipmentDate BETWEEN :start AND :end",
ExpressionAttributeNames: {
"#trackNo": "TrackingNumber",
"#ShipmentDate": "ShipmentDate"
},
ExpressionAttributeValues: {
":trackNo": "TrackingNumber",
":start": start,
":end": end
}
but I am not getting any records in the range...
I've searched this for hours... I cannot figure out how to perform this query.
Example Data:
{
"Carrier": "FedEx",
"ShipmentDate": "2019-09-02",
"TrackingNumber": "TN_2019-09-02"
}
Functioning scan paradigm:
let params = {
TableName: "Parcel",
FilterExpression: "#ShipmentDate BETWEEN :start AND :end",
ExpressionAttributeNames: {
"#ShipmentDate": "ShipmentDate"
},
ExpressionAttributeValues: {
":start": start,
":end": end
}
};
try {
const data = await documentClient.scan(params).promise();
responseBody = JSON.stringify(data);
statusCode = 200;
} catch (err) {
responseBody = `Unable to get shipments: ${err}`
statusCode = 403;
}
Related
This is my handler function.
const updateOrder = (request) => {
const id = request.pathParams.id;
const orderObject = request.body;
if(!id || !orderObject)
throw new Error ('Please provide required id and object in order to update the order')
const params = {
TableName: "pizza-order",
Key: {
"orderId": id
},
UpdateExpression: "SET #n = :n, #a = :a ",
ExpressionAttributeNames: {
"#n": "pizza",
"#a": "address"
},
ConditionExpression: '' ,
ExpressionAttributeValues: {
":n": orderObject.pizza,
":a": orderObject.address,
},
};
return docClient.update(params).promise()
}
And the order Object is:
{
"pizza":"THe outstanding pizza",
"Address": "feni"
}
And it returns error like this. I don't know why it is happening. Very new to serverless development. anyone know the solution?
It was a very silly mistake. First in the order Object there is a typo:
{
"pizza":"THe outstanding pizza",
"address": "feni" //previously it was Address: "feni
}
And the second reason is ConditionExpression Can not be empty.
const updateOrder = (request) => {
const id = request.pathParams.id;
const orderObject = request.body;
if(!id || !orderObject)
throw new Error ('Please provide required id and object in order to update the order')
const params = {
TableName: "pizza-order",
Key: {
"orderId": id
},
UpdateExpression: "SET #n = :n, #a = :a ",
ExpressionAttributeNames: {
"#n": "pizza",
"#a": "address"
},
ConditionExpression: '' , // this can not be empty
ExpressionAttributeValues: {
":n": orderObject.pizza,
":a": orderObject.address,
},
};
return docClient.update(params).promise()
}
I am trying to get the data where the product name is same and implementing the code by using nodejs.`
AWS.config.update({region : "us-east-1"});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "ProductLocation",
FilterExpression: "#pn = :a",
ExpressionAttributeNames: {
"#pn": "Product_Name",
},
ExpressionAttributeValues: {
":a" : "rava"
}
};
console.log("Scanning Product_Name table.");
docClient.scan(params, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
console.log(data);
}
}
Below is my dynamodb data
Your ExpressionAttributeValues should be:
{":a" : {S:"rava"}}
i.e. include the data type
Reference: see the complete example
I have a table that has more than 25 items and wrote a basic script to break them into sub arrays of 25 items each then loops thru that collection of sub arrays to run a batch write item command in the AWS DynamoDB Client. The issue I am getting is a returned validation error. When I run the same seed file via the aws-cli it seeds the table perfectly. This makes me think it has something to do with my script. See anything I am missing? Thanks in advance!
var { DynamoDB } = require('aws-sdk');
var db = new DynamoDB.DocumentClient({
region: 'localhost',
endpoint: 'http://localhost:8000',
});
const allItems = require('./allItems.json');
const tableName = 'some-table-name';
console.log({ tableName, allItems });
var batches = [];
var currentBatch = [];
var count = 0;
for (let i = 0; i < allItems.length; i++) {
//push item to the current batch
count++;
currentBatch.push(allItems[i]);
if (count === 25) {
batches.push(currentBatch);
currentBatch = [];
}
}
//if there are still items left in the curr batch, add to the collection of batches
if (currentBatch.length > 0 && currentBatch.length !== 25) {
batches.push(currentBatch);
}
var completedRequests = 0;
var errors = false;
//request handler for DynamoDB
function requestHandler(err, data) {
console.log('In the request handler...');
return function (err, data) {
completedRequests++;
errors = errors ? true : err;
//log error
if (errors) {
console.error('Request caused a DB error.');
console.error('ERROR: ' + err);
console.error(JSON.stringify(err, null, 2));
} else {
var res = {
statusCode: 200,
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true,
},
body: JSON.stringify(data),
isBase64Encoded: false,
};
console.log(`Success: returned ${data}`);
return res;
}
if (completedRequests == batches.length) {
return errors;
}
};
}
//Make request
var params;
for (let j = 0; j < batches.length; j++) {
//items go in params.RequestedItems.id array
//format for the items is {PutRequest : {Item: ITEM_OBJECT}}
params = '{"RequestItems": {"' + tableName + '": []}}';
params = JSON.parse(params);
params.RequestItems[tableName] = batches[j];
console.log('before db.batchWriteItem: ', params);
try {
//send to db
db.batchWrite(params, requestHandler(params));
} catch{
console.error(err)
}
}
Here is the formatted request object and the error:
before db.batchWriteItem:
{ RequestItems:
{ 'some-table-name': [ [Object], [Object], [Object], [Object] ] }
}
In the request handler...
Request caused a DB error.
ERROR: ValidationException: Invalid attribute value type
{
"message": "Invalid attribute value type",
"code": "ValidationException",
"time": "2020-08-04T10:51:13.751Z",
"requestId": "dd49628c-6ee9-4275-9349-6edca29636fd",
"statusCode": 400,
"retryable": false,
"retryDelay": 47.94198279972915
}
You are using the DocumentClient in the nodejs code. This will automatically convert the data format used by DynamoDB to a more easily consumable format.
e.g.
{
"id": {
"S": "A string value"
}
}
would become
{
"id": "A string value"
}
The CLI does not perform this data conversion.
You can use the regular DynamoDB client to not perform this conversion in Nodejs. e.g. const db = new Dynamodb()
I created an item in dynamodb using Node js, the item has multiple attributes such as brand, category, discount, validity, etc. I am using uuid to generate ids for each item. Now let's say I want to update the validity attribute of the item, in which case I am currently sending the entire json object with the value of validity modified to the new value.
This is definitely not optimal, please help me find an optimal solution.
const params = {
TableName: process.env.PRODUCT_TABLE,
Key: {
id: event.pathParameters.id,
},
ExpressionAttributeNames: {
'#discount': 'discount',
},
ExpressionAttributeValues: {
':brand': data.brand,
':category': data.category,
':discount': data.discount,
':denominations': data.denominations,
":validity": data.validity,
":redemption": data.redemption
},
UpdateExpression: 'SET #discount = :discount, denominations = :denominations, brand = :brand, category = :category, validity = :validity, redemption = :redemption',
ReturnValues: 'ALL_NEW',
};
I want to send just the attribute I want to update with the new value, if I want to change the validity from 6 months to 8 months, I should just send something like:
{
"validity": "8 months"
}
And it should update the validity attribute of the item.
Same should apply to any other attribute of the item.
'use strict';
const AWS = require('aws-sdk');
const dynamoDb = new AWS.DynamoDB.DocumentClient();
module.exports.update = (event, context, callback) => {
const data = JSON.parse(event.body);
let attr = {};
let nameobj = {};
let exp = 'SET #';
let arr = Object.keys(data);
let attrname = {};
arr.map((key) => {attr[`:${key}`]=data[key]});
arr.map((key) => {
exp += `${key} = :${key}, `
});
arr.map((key) => {nameobj[`#${key}`]=data[key]});
attrname = {
[Object.keys(nameobj)[0]] : nameobj[Object.keys(nameobj)[0]]
}
const params = {
TableName: process.env.PRODUCT_TABLE,
Key: {
id: event.pathParameters.id,
},
ExpressionAttributeNames: attrname,
ExpressionAttributeValues: attr,
UpdateExpression: exp,
ReturnValues: 'ALL_NEW',
};
// update the todo in the database
dynamoDb.update(params, (error, result) => {
// handle potential errors
if (error) {
console.error(error);
callback(null, {
statusCode: error.statusCode || 501,
headers: { 'Content-Type': 'text/plain' },
body: 'Couldn\'t update the card',
});
return;
}
// create a response
const response = {
statusCode: 200,
body: JSON.stringify(result.Attributes),
};
callback(null, response);
});
};
Contrary to others comments, this is very possible, use the UpdateItem action.
Language agnostic API docs
JavaScript specific API docs
If you want to dynamically create the query, try something like this:
const generateUpdateQuery = (fields) => {
let exp = {
UpdateExpression: 'set',
ExpressionAttributeNames: {},
ExpressionAttributeValues: {}
}
Object.entries(fields).forEach(([key, item]) => {
exp.UpdateExpression += ` #${key} = :${key},`;
exp.ExpressionAttributeNames[`#${key}`] = key;
exp.ExpressionAttributeValues[`:${key}`] = item
})
exp.UpdateExpression = exp.UpdateExpression.slice(0, -1);
return exp
}
let data = {
'field' : { 'subfield': 123 },
'other': '456'
}
let expression = generateUpdateQuery(data)
let params = {
// Key, Table, etc..
...expression
}
console.log(params)
Output:
{
UpdateExpression: 'set #field = :field, #other = :other',
ExpressionAttributeNames: {
'#field': 'field',
'#other': 'other'
},
ExpressionAttributeValues: {
':field': {
'subfield': 123
},
':other': '456'
}
}
Using Javascript SDK V3:
Import from the right package:
import { DynamoDBClient PutItemCommandInput, UpdateItemCommandInput, UpdateItemCommand } from '#aws-sdk/client-dynamodb';
Function to dynamically do partial updates to the item:
(the code below is typescript can be easily converted to Javascript, just remove the types!)
function updateItem(id: string, item: any) {
const dbClient = new DynamoDBClient({region: 'your-region-here });
let exp = 'set ';
let attNames: any = { };
let attVal: any = { };
for(const attribute in item) {
const valKey = `:${attribute}`;
attNames[`#${attribute}`] = attribute;
exp += `#${attribute} = ${valKey}, `;
const val = item[attribute];
attVal[valKey] = { [getDynamoType(val)]: val };
}
exp = exp.substring(0, exp.length - 2);
const params: UpdateItemCommandInput = {
TableName: 'your-table-name-here',
Key: { id: { S: id } },
UpdateExpression: exp,
ExpressionAttributeValues: attVal,
ExpressionAttributeNames: attNames,
ReturnValues: 'ALL_NEW',
};
try {
console.debug('writing to db: ', params);
const command = new UpdateItemCommand(params);
const res = await dbClient.send(command);
console.debug('db res: ', res);
return true;
} catch (err) {
console.error('error writing to dynamoDB: ', err);
return false;
}
}
And to use it (we can do partial updates as well):
updateItem('some-unique-id', { name: 'some-attributes' });
What i did is create a helper class.
Here is a simple function : Add all the attribute and values that goes into, if the value is null or undefined it won't be in the expression.
I recommande to create a helper class with typescript and add more functions and other stuff like generator of expressionAttributeValues , expressionAttributeNames ... , Hope this help.
function updateExpression(attributes, values) {
const expression = attributes.reduce((res, attribute, index) => {
if (values[index]) {
res += ` #${attribute}=:${attribute},`;
}
return res;
}, "SET ");
return expression.slice(0, expression.length - 1)
}
console.log(
updateExpression(["id", "age", "power"], ["e8a8da9a-fab0-55ba-bae3-6392e1ebf624", 28, undefined])
);
You can use code and generate the params object based on the object you provide. It's just a JavaScript object, you walk through the items so that the update expression only contains the fields you have provided.
This is not really a DynamoDB question in that this is more a general JS coding question.
You can use UpdateItem; to familiarize yourself with DynamoDb queries I would suggest you DynamoDb NoSQL workbench:
https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/workbench.settingup.html
It can generate snippets for you based on your queries.
DynamoDb NoSQL workbench screenshot query
Here is my code that I'm using for making queries:
var scanParams = {
TableName : 'xxxx',
FilterExpression : '( (event = :e0) AND (event = :e1 AND eventTime > :et1 AND eventTime < :et2) )',
ExpressionAttributeValues: {
':e0': { S: 'ME 21' },
':e1': { S: 'ME 21' },
':et1': { N: 1509267218 },
':et2': { N: 1509353618 }
},
ProjectionExpression: "event, customer_id, visitor",
};
In configuration of the respective dynamodb table it seems like I've added Nummber for eventTime column.
Here is the error:
error happened { MultipleValidationErrors: There were 2 validation errors:
* InvalidParameterType: Expected params.ExpressionAttributeValues[':et1'].N to be a string
* InvalidParameterType: Expected params.ExpressionAttributeValues[':et2'].N to be a string
at ParamValidator.validate (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/param_validator.js:40:28)
at Request.VALIDATE_PARAMETERS (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/event_listeners.js:125:42)
at Request.callListeners (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at callNextListener (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/sequential_executor.js:95:12)
at /home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/event_listeners.js:85:9
at finish (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/config.js:315:7)
at /home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/config.js:333:9
at SharedIniFileCredentials.get (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/credentials.js:126:7)
at getAsyncCredentials (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/config.js:327:24)
at Config.getCredentials (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/config.js:347:9)
at Request.VALIDATE_CREDENTIALS (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/event_listeners.js:80:26)
at Request.callListeners (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/sequential_executor.js:101:18)
at Request.emit (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/request.js:683:14)
at Request.transition (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/state_machine.js:14:12)
at Request.runTo (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/request.js:403:15)
at /home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/request.js:791:12
at Request.promise (/home/jahidul/workspace/backstage-opticon/node_modules/aws-sdk/lib/request.js:777:12)
at DynamoDBService.scanItem (/home/jahidul/workspace/backstage-opticon/shared/services/dynamodb/dynamodb.service.ts:52:39)
at /home/jahidul/workspace/backstage-opticon/job-scripts/dyno-test.js:57:12
at sailsReady (/home/jahidul/workspace/backstage-opticon/node_modules/sails/lib/app/lift.js:49:12)
at /home/jahidul/workspace/backstage-opticon/node_modules/async/lib/async.js:251:17
at /home/jahidul/workspace/backstage-opticon/node_modules/async/lib/async.js:154:25
at /home/jahidul/workspace/backstage-opticon/node_modules/async/lib/async.js:248:21
at /home/jahidul/workspace/backstage-opticon/node_modules/async/lib/async.js:612:34
Any idea? Thanks in advance.
Use the parameter below:
var scanParams = {
TableName : 'xxxx',
FilterExpression : '( (event = :e0) AND (event = :e1 AND eventTime > :et1 AND eventTime < :et2) )',
ExpressionAttributeValues: {
':e0': { "S": "ME 21" },
':e1': { "S": "ME 21" },
':et1': { "N": "1509267218" },
':et2': { "N": "1509353618" }
},
ProjectionExpression: "event, customer_id, visitor",
};
In Dynamo DB, the type (Number) is represented by "N" and the value must be in string format "1509353618". Hope this will resolve your problem.
Actually, you don't need to include the data type for et1 and et2 variables. The DynamoDB API should be able to automatically interpret it as NUMBER. Similarly, for variables e0 and e1.
Please try the below code.
var scanParams = {
TableName : 'xxxx',
FilterExpression : '( (event = :e0) AND (event = :e1 AND eventTime > :et1 AND eventTime < :et2) )',
ExpressionAttributeValues: {
':e0': 'ME 21',
':e1': 'ME 21',
':et1': 1509267218,
':et2': 1509353618
},
ProjectionExpression: "event, customer_id, visitor",
};
Full Tested Code:-
You may need to change the table name and key attribute names in the below code.
var AWS = require("aws-sdk");
var creds = new AWS.Credentials('akid', 'secret', 'session');
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000",
credentials: creds
});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "table4",
FilterExpression: "userid = :user_id1 AND (userid = :user_id2 AND ts > :et1 AND ts < :et2)",
ExpressionAttributeValues: { ":user_id1": 'ME 21',
":user_id2": 'ME 21',
":et1" : '1509267216',
":et2" : 1509353618,
}
};
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
data.Items.forEach(function (itemdata) {
console.log("Item :", ++count, JSON.stringify(itemdata));
});
// continue scanning if we have more items
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}