Facing issue: scan dynamo DB - amazon-web-services

I've created apis in nodejs to scan and query dynamoDB table(contains 70K records).
I'm stuck with scan api. I've two APIs for scan -
1. Scanning table with limit of 500 - getting proper response
2. Scanning table without any limit - response code 403
And if I'm calling the same query without any limit through cli, getting full response.
Not getting what's the problem.
api.get('/deviceData', function (request) { // GET all users
return dynamoDb.scan({
TableName: 'student',
Limit: 500
}).promise()
.then(response => response.Items);
});
api.get('/deviceData-scan', function (request) { // GET all users
return dynamoDb.scan({
TableName: 'student'
}).promise()
.then(response => response.Items);
});

Please check how to use scan multiple time its help you.
function getData() {
let params = {
TableName: 'student'
};
let finalResult = [];
let queryExecute = function () {
documentClient.scan(params, function (err, result) {
if (err) {
console.log('failure', err);
} else {
finalResult = finalResult.concat(result.Items);
// check if more data exists.
if (result.LastEvaluatedKey) {
params.ExclusiveStartKey = result.LastEvaluatedKey;
queryExecute();
} else {
console.log('all data', finalResult);
}
}
});
};
queryExecute();
},

Related

How to put an item on DynamoDB table using AWS SDK for Node.js?

I have a DynamoDB table named MYTABLE
I'm trying to insert some data into this table via LambdaEdge.
My Lambda function is triggered via a CloudFront Distribution.
I have this function :
var AWS = require("aws-sdk");
AWS.config.update({region: 'us-east-1'});
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = async (event, context) => {
var params = {
TableName: 'MYTABLE',
Item: {
'id' : {'S': 'something'},
'value' : {'S': 'something'}
}
};
// Call DynamoDB to add the item to the table
await ddb.putItem(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
}
});
const response = {
status: '302',
statusDescription: 'Found',
headers: {
location: [{
key: 'Location',
value: 'https://www.google.com',
}],
},
};
return response;
};
The function runs properly (the redirection works) but no data is inserted into the table.
Does anyone know what I'm doing wrong please ?
Thanks.
Cheers,

DynamoDB : Query always returning null

I am quite new on DynamoDB, I just create a table where I inserted some dummy rows just to test the read / write functions.
Here is my data.
I create a Lambda to access these data, via Partition Key ok Sort Key but I ended up with a null return every time :'(
Here is the code of my lambda :
const dynamodb = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10', region: 'eu-west-1'});
exports.handler = async (event,context,callback) => {
var params = {
TableName : "DontMissAPlaceTable",
KeyConditionExpression: "Partition_Key = :id",
ExpressionAttributeValues: {
":id": {"S" : "media_001"}
}
};
dynamodb.query(params, function(err, data) {
if (err) {
console.log(err, err.stack);
callback(null, {
statusCode: '500',
body: err
});
} else {
console.log(data)
callback(null, {
statusCode: '200',
body: data
});
}
})
};
I first thought it was lack of access to the table but it seems I have the right permissions.
I am quite sure it's a dummy problem but i can't figure it out ...
Thanks for help :D
Your Lambda function is async but your code uses callbacks. By the time the callback is reached, your function has already been terminated because it ran asychronously.
Rather than using the outdated, confusing callback approach, just stick with async/await. The AWS SDK for Node.js already provides a .promise() method which you can call on asynchronous methods that use a callback, making it seamless to await on them.
That said, change your code to:
const dynamodb = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10', region: 'eu-west-1'});
exports.handler = async (event) => {
const params = {
TableName : "DontMissAPlaceTable",
KeyConditionExpression: "Partition_Key = :id",
ExpressionAttributeValues: {
":id": "media_001"
}
};
return {
statusCode: 200,
body: JSON.stringify(await dynamodb.query(params).promise())
}
};
Since you are already using the DocumentClient API, you don't need to specify the types ("S", "N", etc) when querying.
Also, make sure that Partition_Key really is your partition key. You use the name you defined in your table, no HashKey nor PartitionKey, meaning if you called your Hash Key id, then id should be used.

Null Response while querying DynamoDB

I am querying dynamodb from lambda function written in node.js env -
Trying to query for table CurrencyPrice where Currency column has value "BLC".
When I test my function in lambda console, - it prints until second console log - "querying DB ConsolePrice" and returns a NULL response. It does not print either of the next two console logs and not sure if it is even connecting to DB.
It seems that the code does not go into ddb.query() function at all - had tried putting all loggers in this function but none gets printed.
I have tried checking all possible aws documentation but not able to understand why this function is not getting executed.
My code looks something like below -
var AWS = require ('aws-sdk');
exports.handler = async (event) =>
{
AWS.config.update({region: 'ap-southeast-2'});
console.log("i am in function");
// Create DynamoDB service object
var ddb = new AWS.DynamoDB.DocumentClient();
var table = 'CurrencyPrice';
var params = {
"Select": "ALL_ATTRIBUTES","TableName": "CurrencyPrice",
};
console.log("querying DB" + table);
ddb.query(params, function(err, data) {
console.log("i am in ddb query");
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null,2));
} else {
console.log(data);
}
});
};
Current result that I am getting in lambda console:
Response:
null
Request ID:"XXXX"
Function Logs:
START RequestId: XXX Version: $LATEST
2019-06-02T13:31:55.189Z XXXX INFO i am in function
2019-06-02T13:31:55.331Z XXXX INFO querying DBCurrencyPrice
2019-06-02T13:31:55.390Z XXXX INFO { Select: 'ALL_ATTRIBUTES', TableName: 'CurrencyPrice' }
END RequestId: XXXX
I expect that at least it prints "Unable to Query" or actual data that it connects to DB and query?
If you're using async/await you would want to return a promise.
var AWS = require("aws-sdk");
AWS.config.update({ region: "ap-southeast-2" });
var ddb = new AWS.DynamoDB.DocumentClient();
exports.handler = async event => {
console.log("i am in function");
// Create DynamoDB service object
var table = "CurrencyPrice";
var params = {
Select: "ALL_ATTRIBUTES",
TableName: "CurrencyPrice"
};
console.log("querying DB" + table);
return ddb
.query(params)
.promise()
.then((err, data) => {
console.log("i am in ddb query");
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log(data);
}
});
};
The AWS example
ddbClient.query(params, callback)
shows a synchronous matter. Execution continues without waiting for the callback finishes. That's why you got Null.
You will have to wrap this .query() call into a Promise, so the execution will wait for completion of this callback.
Make query a promise and wait for its fulfilled state.
const promise = await ddb.query(params, function(err, data) {
console.log("i am in ddb query");
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null,2));
} else {
console.log(data);
// process your data
}
}).promise();
return promise;

AWS Lamda/API Gateway Every Second Call Results in Internal Server Error

I'm need to 'Nodejs' and 'Serveless'. I've created a 'Serverless' API and deployed to AWS. Everything works as expected. The issue i have and i can't seem to find anything about this is, on every second call i get an internal server error. the first call is, returns data as expected.
I've deployed to AWS only in a dev stage. I'm wondering if there is some configuration i'm missing or something?
If you need the 'Serverless' config or code examples i can provide.
Thanks.
ANSWER
I think there was an issue with the DB call not returning data in time for the callback, therefore i was finding inconsistent results.
So basically what i did was create a Database class returning Promises like so...
'use strict';
const mysql = require('mysql');
/**
* Database
*/
class Database {
constructor(config) {
if (!this.dbConnection) {
console.log('connect to DB');
this.dbConnection = mysql.createPool(config);
this.dbConnection.on('connection', (connection) => {
console.info('Connection Made!');
});
}
}
query(sql, args) {
return new Promise((resolve, reject) => {
this.dbConnection.query(sql, args, (err, rows) => {
if (err) {
reject(err);
}
resolve(rows);
})
});
}
close() {
return new Promise((resolve, reject) => {
this.dbConnection.end((error) => {
if (error) {
reject(error);
}
resolve();
});
});
}
}
module.exports = Database;
So when i made my query there was a result ready for the callback.
'use strict';
const Database = require('./lib/Database');
const {successResponse, errorResponse} = require('./lib/response');
const CategoryResource = require('./resource/Category');
module.exports.list = (event, context, callback) => {
let sql = 'SELECT * FROM categories AS c WHERE c.company_id = ? AND c.parent_id IS NULL AND c.status = 1 LIMIT ?, ?;';
const company = parseInt(event.queryStringParameters.company);
let page = 1;
let limit = 20;
if (null != event.queryStringParameters) {
if ('page' in event.queryStringParameters) {
page = parseInt(event.queryStringParameters.page);
}
if ('limit' in event.queryStringParameters) {
limit = parseInt(event.queryStringParameters.limit);
}
}
let start = (page - 1) * limit;
if (isNaN(company)) {
callback(null, errorResponse(400, 'Company ID Required', 'Parameter company_id is required.', []));
return;
}
let Category = new Database();
let categoryResource = [];
Category
.query(sql, [company, start, limit])
.then(response => {
Category.close();
response.forEach((category) => {
categoryResource.push(CategoryResource(category));
});
callback(null, successResponse(200, {
"total": response.length,
"perPage": limit,
"currentPage": page,
"data": categoryResource
}));
})
.catch((error) => {
callback(null, errorResponse(error.code, error.sqlMessage, error.sql, {
code: error.errno,
field: error.sqlMessage,
message: error.sqlMessage
}));
Category.close();
});
};
I hope that helps anyone that may have run into the same issue.
If every other time you get an internal server error, that means your code is syntactically sound but has some sort of logic error. It's impossible to help without example code, but some of the more common errors I've seen that only sometimes occur can be:
race conditions (if you're doing parallel access of the same array, for example)
array access errors (length+1 instead of length-1, less-than-zero, or your iterators are jumping someplace in memory they shouldn't)
simply mentioning the wrong variable (putting an i instead of a j, for example)
Unfortunately, without specific examples, the best we can offer is wild speculation and personal experience. Have you tried looking at AWS's CloudWatch and what it says about your execution? There should be some errors logged in there too.
I think there was an issue with the DB call not returning data in time for the callback, therefore i was finding inconsistent results.
So basically what i did was create a Database class returning Promises like so...
'use strict';
const mysql = require('mysql');
/**
* Database
*/
class Database {
constructor(config) {
if (!this.dbConnection) {
console.log('connect to DB');
this.dbConnection = mysql.createPool(config);
this.dbConnection.on('connection', (connection) => {
console.info('Connection Made!');
});
}
}
query(sql, args) {
return new Promise((resolve, reject) => {
this.dbConnection.query(sql, args, (err, rows) => {
if (err) {
reject(err);
}
resolve(rows);
})
});
}
close() {
return new Promise((resolve, reject) => {
this.dbConnection.end((error) => {
if (error) {
reject(error);
}
resolve();
});
});
}
}
module.exports = Database;
So when i made my query there was a result ready for the callback.
'use strict';
const Database = require('./lib/Database');
const {successResponse, errorResponse} = require('./lib/response');
const CategoryResource = require('./resource/Category');
module.exports.list = (event, context, callback) => {
let sql = 'SELECT * FROM categories AS c WHERE c.company_id = ? AND c.parent_id IS NULL AND c.status = 1 LIMIT ?, ?;';
const company = parseInt(event.queryStringParameters.company);
let page = 1;
let limit = 20;
if (null != event.queryStringParameters) {
if ('page' in event.queryStringParameters) {
page = parseInt(event.queryStringParameters.page);
}
if ('limit' in event.queryStringParameters) {
limit = parseInt(event.queryStringParameters.limit);
}
}
let start = (page - 1) * limit;
if (isNaN(company)) {
callback(null, errorResponse(400, 'Company ID Required', 'Parameter company_id is required.', []));
return;
}
let Category = new Database();
let categoryResource = [];
Category
.query(sql, [company, start, limit])
.then(response => {
Category.close();
response.forEach((category) => {
categoryResource.push(CategoryResource(category));
});
callback(null, successResponse(200, {
"total": response.length,
"perPage": limit,
"currentPage": page,
"data": categoryResource
}));
})
.catch((error) => {
callback(null, errorResponse(error.code, error.sqlMessage, error.sql, {
code: error.errno,
field: error.sqlMessage,
message: error.sqlMessage
}));
Category.close();
});
};
I hope that helps anyone that may have run into the same issue.

How to test AWS integration services flow?

I got the following flow:
Put file to S3
Triggered Lambda function
DynamoDB table
Due to this flow, I created an integration test which first put a file to S3.
After that, I checked the record over the DynamoDB table which been updated by the Lambda function.
The problem occures when I'm searching for my record, actually I couldn't find it because there is some time interval that I need to wait for the table to be updated. In the second run I was able to see the record from the previous test run.
My question is how can I make this integration test? How can I check for the updated table records after which needs to be inserted after some time using the lambda trigger call.
What I tried so far:
Use Promise
Use setTimeout()
These weren't good enough for me, Please advice some better approach.
My code:
it('Writing single record to DynamoDB - when data is valid JSON and updating the DB is Succeed',
function(done) {
var putObjectParams = {
Bucket: BUCKET,
Key: FILE_KEY,
Body: 'some file content'
};
s3.putObject(putObjectParams, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log(JSON.stringify(data));
}
});
var docClient = new AWS.DynamoDB.DocumentClient();
var queryParams = {
TableName: ENV + "-WFMHistoricalUnprocessedFiles",
ProjectionExpression: "filePath, ingressTime, dir, fileName",
KeyConditionExpression: "#column = :fileFullPath",
ExpressionAttributeNames: {
"#column": "filePath"
},
ExpressionAttributeValues: {
":fileFullPath": BUCKET + "/" + FILE_KEY
}
};
docClient.query(queryParams, function(err, data) {
if (err) {
console.log("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("Query succeeded.");
data.Items.forEach(function(item) {
console.log(JSON.stringify(item));
});
}
});
});