Using aws-lambda to insert data into DynamoDB - amazon-web-services

When trying to place a new entry to the table it misses the execution of the put function. Fuuny, it used to work, and now for whichever reasons it doesn't. allValues is an array, and returns the data fully. Whenever trying to call saveData it does execute everything apart put function.
For some reason, I have tried it a few times, and it worked fine, but suddenly it doesn't place the entry anymore. Tried to play around, but can't seem to find an issue? Might be caused by asynchronous?
//Amazon sdk
let AWS = require("aws-sdk");
let db = require('database');
//Amazon config
AWS.config.update({
region: "us-east-1",
endpoint: "https://dynamodb.us-east-1.amazonaws.com"
});
//DynamoDb client
var docClient = new AWS.DynamoDB.DocumentClient();
//Axios will handle HTTP requests to web service
let axios = require ('axios');
//The ID of the student's data that I will download
let requestID= '*********';
//URL where student data is available
let url = '*************************';
exports.handler = async (event) => {
//Get synthetic data
let allValues = (await axios.get(url + requestID)).data.target;
let allStart = (await axios.get(url + requestID)).data.start;
for(let i = 0; i < allValues.length; i++){
//db.saveData(i, allValues[i], allStart);
}
db.saveData();
};
and my database.js
et AWS = require("aws-sdk");
//Create new DocumentClient
let documentClient = new AWS.DynamoDB.DocumentClient();
//Returns all of the connection IDs
//module.exports.saveData = async (sId, sValue, sTime) => {
module.exports.saveData = async () => {
let params = {
TableName: "SynData",
Item:{
"synId": 66,
"synValue": 1579.21,
"synTime": "2019-01-01"
}
};
//return documentClient.put(params).promise();
documentClient.put(params, function(err, data) {
if (err) {
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Added item:", JSON.stringify(data, null, 2));
}
});
};
The logs do not show any return for put function.

saveData() is async, therefore it returns a promise. You need to await on it. Also, I don't understand why you gave up on the .promise() call. Using callbacks is outdated and much more convoluted.
Your code should look like this:
module.exports.saveData = async () => {
let params = {
TableName: "SynData",
Item:{
"synId": 66,
"synValue": 1579.21,
"synTime": "2019-01-01"
}
};
return await documentClient.put(params).promise();
};
On your client, just await on saveData():
for(let i = 0; i < allValues.length; i++){
await db.saveData(i, allValues[i], allStart);
}

Related

DAX Client JS with Lambda triggers multiple errors such as ECONRESET, Client does not have permission to invoke DefineKeySchema

Hi I have a Dax cluster on top of a DynamoDB. Everything is set in the same VPC and subnets are created correctly. Also the permissions are setup correcty:
Lambda has full access to dynamodb:* and dax:*
Dax cluster policy has full access for DynamoDB - dynamodb:* and also a trust policy sts:assumeRole for dax service.
This is how I connect to Dax client inside my Lambda:
const AWS = require("aws-sdk");
const AmazonDaxClient = require("amazon-dax-client"); //VERSION: 1.2.9
let docClient = new AWS.DynamoDB.DocumentClient({
region: "eu-west-1",
convertEmptyValues: true,
});
let daxService = null;
let daxClient = null;
async function connectToDAX(useDAX = false, force = false) {
try {
if (!useDAX) {
return docClient;
} else {
if (force) {
daxService = null;
daxClient = null;
}
if (daxService == null) {
const endpoint = await amazon.getParam(
`/${lambdaService.getCurrentStage()}/project/aws/daxClusterEndpoint`
);
const region = await amazon.getParam(
`/${lambdaService.getCurrentStage()}/prjoect/region`
);
const daxConfig = {
endpoints: [endpoint],
region: region,
};
daxService = new AmazonDaxClient(daxConfig);
daxClient = new AWS.DynamoDB.DocumentClient({
service: daxService,
region: "eu-west-1",
convertEmptyValues: true,
});
console.log("created new DAX connection");
}
return daxClient;
}
} catch (e) {
console.log("error on connectToDax", e);
return null;
}
}
async function queryTable(params, dax = false) {
try {
let client = dax ? daxClient : docClient;
if (dax && !daxClient) {
client = await connectToDAX(dax);
}
return await new Promise(function (resolve, reject) {
client.query(params, function (err, data) {
if (err) {
console.log("queryTable", params);
console.log("query error", err);
reject(err);
} else {
resolve(data);
}
});
});
} catch (e) {
console.log("error dynamodb queryTable", e);
daxClient = null;
daxService = null;
console.log("retrying queryTable with new dax connection");
return await queryTable(params, dax);
}
}
So, I am sending like 100 requests at a time towards API where my Lambda is getting invoked and it calls queryTable().
I am getting a lot of errors, 30% of all requests are Errors, which I would like to fix.
Errors like:
Unknown application error occurred
{"errorType":"Error","errorMessage":"read ECONNRESET","code":"ECONNRESET","errno":-104,"syscall":"read","stack":["Error: read ECONNRESET"," at TCP.onStreamRead (internal/stream_base_commons.js:209:20)"]}
Client does not have permission to invoke DefineKeySchema
So I have spent a lot of time troubleshooting this. Went trough all permissions and setup, but if it was something wrong there, NONE of the request would be successful.
I am thinking that the problem is with the DAX Client connection when Lambda concurrent invocations are happening. I have tried to force each concurrent execution to create new Dax Client connection, but that resulted even with more errors.
I am retrying as you can see in the queryTable() function.
I don't know what to do anymore. Please give some input.

Flutter aws amplify not returning data when calling graphql api

On button click I have programmed to call a graphql api which is connected to a Lambda function and the function is pulling data from a dynamodb table. The query does not produce any error, but it doesn't give me any results as well. I have also checked the cloudwatch logs and I dont see any traces of the function being called. Not sure on the careless mistake I am making here.
Here is my api
void findUser() async {
try {
String graphQLDocument = '''query getUserById(\$userId: ID!) {
getUserById(userId: \$id) {
id
name
}
}''';
var operation = Amplify.API.query(
request: GraphQLRequest<String>(
document: graphQLDocument,
variables: {'id': 'USER-14160000000'}));
var response = await operation.response;
var data = response.data;
print('Query result: ' + data);
} on ApiException catch (e) {
print('Query failed: $e');
}
}
Here is my lambda function -
const getUserById = require('./user-queries/getUserById');
exports.handler = async (event) => {
var userId = event.arguments.userId;
var name = event.arguments.name;
var avatarUrl = event.arguments.avatarUrl;
//console.log('Received Event - ', JSON.stringify(event,3));
console.log(userId);
switch(event.info.fieldName) {
case "getUserById":
return getUserById(userId);
}
};
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region: 'ca-central-1'});
async function getUserById(userId) {
const params = {
TableName:"Bol-Table",
KeyConditionExpression: 'pk = :hashKey and sk = :sortKey',
ExpressionAttributeValues: {
':hashKey': userId,
':sortKey': 'USER'
}
};
try {
const Item = await docClient.query(params).promise();
console.log(Item);
return {
id: Item.Items[0].pk,
name: Item.Items[0].details.displayName,
avatarUrl: Item.Items[0].details.avatarUrl,
createdAt: Item.Items[0].details.createdAt,
updatedAt: Item.Items[0].details.updatedAt
};
} catch(err) {
console.log("BOL Error: ", err);
}
}
module.exports = getUserById;
Upon button click I get this
Moving my comment to an answer:
Can you try changing your graphQLDocumnet to
String graphQLDocument = '''query getUserById(\$id: ID!) {
getUserById(userId: \$id) {
id
name
}
}''';
Your variable is $userId and then $id. Try calling it $id in both places like in your variables object.
Your flutter code is working fine but in lambda from the aws is returning blank string "" to not to print anything

lambda function returning null for deleting item in DynamoDB

hi Ive been trying to get my lambda function to delete an item in dynamo db but the function is simply returning null and i have no idea how to even start debugging it, hoping someone here has the knowledge to help
my table has guid as its primary partition key and username as its sort key
heres my code in .js
const AWS = require("aws-sdk");
// Initialising the DynamoDB SDK
const documentClient = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event) => {
const { guid, username } = event
const params = {
TableName: "Items", // The name of your DynamoDB table
Key:{
"guid": {"S" : guid},
"username": {"S" : username}
}
};
try {
// Utilising the scan method to get all items in the table
documentClient.delete(params, function(err, data) {
if (err) {
return("Unable to delete item. Error JSON:", JSON.stringify(err, null, 2));
} else {
return("DeleteItem succeeded:", JSON.stringify(data, null, 2));
}
});
}
catch (e) {
return {
statusCode: 500,
body: e
};
}
};
this is the payload for the test event im using in lambda
{
"guid": "34",
"username": "newusername"
}
You are using async function handler. So your function probably just finishes before your code actually has a chance to execute.
You can overcome this issue by wrapping your code around new Promise as shown in the docs

How to point AWS-SDK DynamoDB to a serverless DynamoDB local

I am trying to write a script that will loop thru an array of items for a DynamoDB table and run a batch write command. My functionality is good, but I am having trouble with DynamoDB. Would be great if I could point my AWS.DynamoDB.DocumentClient() to my localhost running DynamoDB. Any tips?
Would also consider a way to just run the commands via the aws cli but I am not sure how to do that. I am running Node.js so it maybe possible?
Here is my code:
var AWS = require('aws-sdk');
AWS.config.update({ region: 'eu-central-1' });
var DynamoDB = new AWS.DynamoDB.DocumentClient()
DynamoDB.endpoint = 'http://localhost:8000';
const allItems = require('./resource.json');
const tableName = 'some-table-name';
console.log({ tableName, allItems });
var batches = [];
var currentBatch = [];
var count = 0;
for (let i = 0; i < allItems.length; i++) {
//push item to the current batch
count++;
currentBatch.push(allItems[i]);
if (count % 25 === 0) {
batches.push(currentBatch);
currentBatch = [];
}
}
//if there are still items left in the curr batch, add to the collection of batches
if (currentBatch.length > 0 && currentBatch.length !== 25) {
batches.push(currentBatch);
}
var completedRequests = 0;
var errors = false;
//request handler for DynamoDB
function requestHandler(err, data) {
console.log('In the request handler...');
return function (err, data) {
completedRequests++;
errors = errors ? true : err;
//log error
if (errors) {
console.error(JSON.stringify(err, null, 2));
console.error('Request caused a DB error.');
console.error('ERROR: ' + err);
console.error(JSON.stringify(err, null, 2));
} else {
var res = {
statusCode: 200,
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true,
},
body: JSON.stringify(data),
isBase64Encoded: false,
};
console.log(`Success: returned ${data}`);
return res;
}
if (completedRequests == batches.length) {
return errors;
}
};
}
//Make request
var params;
for (let j = 0; j < batches.length; j++) {
//items go in params.RequestedItems.id array
//format for the items is {PutRequest : {Item: ITEM_OBJECT}}
params = '{"RequestItems": {"' + tableName + '": []}}';
params = JSON.parse(params);
params.RequestItems[tableName] = batches[j];
console.log('before db.batchWriteItem: ', params);
//send to db
DynamoDB.batchWrite(params, requestHandler(params));
}
I figured it out and will leave this here for anyone that may need it.
var { DynamoDB } = require('aws-sdk');
var db = new DynamoDB.DocumentClient({
region: 'localhost',
endpoint: 'http://localhost:8000',
});

Connect AWS mobile backend to DynamoDB

I am trying to use AWS mobile backend (using lambda function) to insert into dynamoDB (also configured at the mobile backend) but with no success so far.
The relevant code:
'use strict';
console.log("Loading function");
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:process.env.MOBILE_HUB_PROJECT_REGION});
exports.handler = function(event, context, callback) {
var responseCode = 200;
var requestBody, pathParams, queryStringParams, headerParams, stage,
stageVariables, cognitoIdentityId, httpMethod, sourceIp, userAgent,
requestId, resourcePath;
console.log("request: " + JSON.stringify(event));
// Request Body
requestBody = event.body;
if (requestBody !== undefined && requestBody !== null) {
// Set 'test-status' field in the request to test sending a specific response status code (e.g., 503)
responseCode = JSON.parse(requestBody)['test-status'];
}
// Path Parameters
pathParams = event.path;
// Query String Parameters
queryStringParams = event.queryStringParameters;
// Header Parameters
headerParams = event.headers;
if (event.requestContext !== null && event.requestContext !== undefined) {
var requestContext = event.requestContext;
// API Gateway Stage
stage = requestContext.stage;
// Unique Request ID
requestId = requestContext.requestId;
// Resource Path
resourcePath = requestContext.resourcePath;
var identity = requestContext.identity;
// Amazon Cognito User Identity
cognitoIdentityId = identity.cognitoIdentityId;
// Source IP
sourceIp = identity.sourceIp;
// User-Agent
userAgent = identity.userAgent;
}
// API Gateway Stage Variables
stageVariables = event.stageVariables;
// HTTP Method (e.g., POST, GET, HEAD)
httpMethod = event.httpMethod;
// TODO: Put your application logic here...
let params = {
Item:{
"prop1":0,
"prop2":"text"
},
TableName:"testTable"
};
docClient.put(params, function(data, err){
if(err)
responseCode = 500;
else
{
responseCode = 200;
context.succeed(data);
}
});
// For demonstration purposes, we'll just echo these values back to the client
var responseBody = {
requestBody : requestBody,
pathParams : pathParams,
queryStringParams : queryStringParams,
headerParams : headerParams,
stage : stage,
stageVariables : stageVariables,
cognitoIdentityId : cognitoIdentityId,
httpMethod : httpMethod,
sourceIp : sourceIp,
userAgent : userAgent,
requestId : requestId,
resourcePath : resourcePath
};
var response = {
statusCode: responseCode,
headers: {
"x-custom-header" : "custom header value"
},
body: JSON.stringify(responseBody)
};
console.log("response: " + JSON.stringify(response))
context.succeed(response);
};
this doesn't put the item to the table for some reason.
I gave the necessary permissions using the roles part, anything I am missing?
**responseCode is only for testing purposes.
Edit:
tried AWS node.js lambda request dynamodb but no response (no err, no return data) and doesn't work either.
Edit2:
Added the full handler code. (it the default generated code when creating first AWS lambda).
I have refactored some bits of your code to look much simpler and use async/await (make sure to select Node 8.10 as the running environment for your function) instead of callbacks. I also got rid of the context and callback parameters, as they were used for older versions of NodeJS. Once you're using Node 8+, async/await should be the default option.
Also, it is possible to chain a .promise() on docClient.putItem, so you can easily await on it, making your code way simpler. I have left only the DynamoDB part (which is what is relevant to your question)
'use strict';
console.log("Loading function");
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:process.env.MOBILE_HUB_PROJECT_REGION});
exports.handler = async (event) => {
let params = {
Item:{
"prop0":1,
"prop2":"text"
},
TableName:"testTable"
};
try {
await docClient.put(params).promise();
} catch (e) {
console.log(e)
return {
messsage: e.message
}
}
return { message: 'Data inserted successfully' };
};
Things to keep in mind if still it does not work:
Make sure your Lambda function has the right permissions to insert items on DynamoDB (AmazonDynamoDBFullAccess will do it)
You ALWAYS have to provide the partition key when inserting items to DynamoDB. On your example, the JSON only has two properties: prop1 and prop2. If none of them are the partition key, your code will certainly fail.
Make sure you table also exists
If you code fails, just check CloudWatch logs as any exception is now captured and printed out on the console.
The reason why no data is written in the table is because the call to DynamoDB put is asynchronous and will return by calling your callback. But during that time, the rest of the code continues to execute and your function eventually finish before the call to DynamoDB has a chance to complete.
You can use the await / async keywords to make your code sychronous :
async function writeToDynamoDB(params) {
return new Promise((resolve,reject) => {
docClient.put(params, function(data, err){
if(err)
reject(500);
else
resolve(data);
});
});
}
let params = ...
var data = await writeToDynamoDB(params)
You can find sample code I wrote (in Typescript) at https://github.com/sebsto/maxi80-alexa/blob/master/lambda/src/DDBController.ts