I am trying to have lambda trigger a codebuild function when it hits the point within the lambda function, here is the current code im using for lamda:
console.log('Loading function');
const aws = require('aws-sdk');
const s3 = new aws.S3();
exports.handler = async (event, context) => {
const codebuild = new aws.CodeBuild();
let body = JSON.parse(event.body);
let key = body.model;
var getParams = {
Bucket: 'bucketname', // your bucket name,
Key: key + '/config/training_parameters.json' // path to the object you're looking for
}
if (key) {
const objects = await s3.listObjects({
Bucket: 'bucketname',
Prefix: key + "/data"
}).promise();
console.log(objects)
if (objects.Contents.length == 3) {
console.log("Pushing")
await s3.getObject(getParams, function(err, data) {
if (err)
console.log(err);
if (data) {
let objectData = JSON.parse(data.Body.toString('utf-8'));
const build = {
projectName: "projname",
environmentVariablesOverride: [
{
name: 'MODEL_NAME',
value: objectData.title,
type: 'PLAINTEXT',
},
]
};
console.log(objectData.title)
codebuild.startBuild(build,function(err, data){
if (err) {
console.log(err, err.stack);
}
else {
console.log(data);
}
});
console.log("Done with codebuild")
}
}).promise();
const message = {
'message': 'Execution started successfully!',
}
return {
'statusCode': 200,
'headers': {'Content-Type': 'application/json'},
'body': JSON.stringify(message)
};
}
}
};
Specifically this part should trigger it:
codebuild.startBuild(build,function(err, data){
if (err) {
console.log(err, err.stack);
}
else {
console.log(data);
}
});
But its not, it even outputs after the function? Im thinking its something to do with promises/await/async but cant find the right solution? Any help would be appreciated.
As you pointed out the problem is related to promises. Change your code like this:
const result = await codebuild.startBuild(build).promise();
And if you configured your lambda permissions for CodeBuild it should work.
You can change your s3.getObject the same way without the callback function:
const file = await s3.getObject(getParams).promise();
console.log(file.Body);
Related
My Lambda function does not call the S3 upload function where it is supposed to send a URL back that will be assigned to the DynamoDB database. I can't seem to pin-point what's wrong here. I have tried to just call the Lambda upload to S3 function without the rest of the code and it work's fine.
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const BUCKET_NAME = 'BUCKET_NAME';
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let body;
let statusCode = 200;
const uploadFileToS3 = async (fileBinary) => {
try {
const base64File = fileBinary;
const decodedFile = Buffer.from(
base64File.replace(/^data:image\/\w+;base64,/, ''),
'base64'
);
const params = {
Bucket: BUCKET_NAME,
Key: `images/${new Date().toISOString()}.jpeg`,
Body: decodedFile,
ContentType: 'image/jpeg',
};
const uploadResult = await s3.upload(params).promise();
console.log(uploadResult)
return uploadResult;
} catch (e) {
console.error(e);
}
};
try {
switch (event.routeKey) {
case 'PUT /items':
let requestJSON = JSON.parse(event.body);
const fileURL = await uploadFileToS3(requestJSON.itemPicture);
await dynamo
.put({
TableName: 'TABLE_NAME',
Item: {
itemId: requestJSON.itemId,
userId: requestJSON.userId,
itemTitle: requestJSON.itemTitle,
itemDesc: requestJSON.itemDesc,
itemLocation: requestJSON.itemLocation,
itemPrice: requestJSON.itemPrice,
itemPicture: fileURL,
},
})
.promise();
body = `Put item ${requestJSON.itemId}`;
break;
default:
throw new Error(`Unsupported route: ` + `${event.routeKey}`);
}
} catch (err) {
statusCode = 400;
body = err.message;
} finally {
body = JSON.stringify(body);
}
return {
statusCode,
body,
headers,
};
};
This is the code Iam using,
var AWS = require('aws-sdk');
var iotdata = new AWS.IotData({
endpoint: '###########.iot.ap-south-1.amazonaws.com'
});
exports.handler = async (event) => {
var params = {
payload: Buffer.from('...') || 'STRING_VALUE'
encoded on your behalf */, /* required */
thingName: 'ESP32', /* required */
//shadowName: 'STRING_VALUE'
};
iotdata.updateThingShadow(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
};
I referred to this from the given link,
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/IotData.html#updateThingShadow-property
But I cant update my shadow.
I also dont understand in which format I should include the payload.
I am running this on AWS Lambda function in Nodejs 12.x environment. I also dont receive any errors in cloudwatch . Cloudwatch tells me execution result has succeeded? Can you help me?
I have already given permission for Lambda to updateShadow.
This the final correct code .....
var AWS = require('aws-sdk');
var iotdata = new AWS.IotData({
endpoint: '#######.iot.ap-south-1.amazonaws.com'
});
exports.handler = async (event) => {
var POST_DATA = JSON.stringify({"state":{"desired":{"state":10}}});
await new Promise((resolve, reject) => {
var params = {
payload:POST_DATA ,
thingName: 'ESP32', /* required */
//shadowName: 'STRING_VALUE'
};
iotdata.updateThingShadow(params, function(err, data) {
if (err) {
console.log(err, err.stack);
console.log("error................")// an error occurred
reject(err);
} else {
console.log(data); // successful response
resolve(data)
}
});
})
}
The issue here is that you are using the async Lambda handler. So while your updateThingShadow() call is in progress, the Lambda function completes and exits. You need to wait for that call to complete. There are two ways to do that -
Approach 1: (A simpler async-await syntax)
var AWS = require('aws-sdk');
var iotdata = new AWS.IotData({
endpoint: '###########.iot.ap-south-1.amazonaws.com'
});
exports.handler = async (event) => {
var params = {
// Define params
};
try {
const data = await iotdata.updateThingShadow(params).promise();
console.log(data);
} catch (err) {
console.log(err);
}
};
Approach 2: (If you prefer using the callback approach)
var AWS = require('aws-sdk');
var iotdata = new AWS.IotData({
endpoint: '###########.iot.ap-south-1.amazonaws.com'
});
exports.handler = async (event) => {
var params = {
// Define params
};
await new Promise((resolve, reject) => {
iotdata.updateThingShadow(params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
reject(err); // End the promise with an error
} else {
console.log(data); // successful response
resolve(data); // End the promise with success
}
});
})
};
I am trying to scan the Dynamodb table form my following code, can anyone please guide me what is wrong here.
const AWS = require("aws-sdk");
const dynamodb = new AWS.DynamoDB({
region: "eu-west-1",
apiVersion: "2012-08-10"
});
exports.handler = async (event, callback) => {
const params = {
TableName: "job_Status"
};
dynamodb.scan(params, (err, data) => {
if (err) {
console.log(err);
callback(err);
} else {
console.log(data);
callback(null, data);
}
});
};
I have given full dynamodb access role to the function but still it gives me the null response. Any idea what can be wrong here?
Response:
I tried with dynaomClient which not working too.
const AWS = require("aws-sdk");
const db = new AWS.DynamoDB.DocumentClient({
region : 'eu-west-1'
});
exports.handler = async (event, callback) => {
const params = {
TableName: "job_Status"
};
db.scan(params, (err, data) => {
if (err) {
console.log(err);
callback(err);
} else {
console.log(data);
callback(null, data);
}
});
};
Your Lambda function is async but your code uses callbacks. By the time the callback is reached, your function has already been terminated because it ran asynchronously. I'd speculate that the null output you see is the return value from the Lambda function, not your console.log.
Replace your call to scan with the following:
try{
let results = await db.scan(params).promise()
console.log(results);
} catch(err){
console.log(err)
}
For more info, check out the AWS documentation about working with promises.
I have a Lambda which looks like so:
module.exports.handler = (event, context, callback) => {
AWS.config.setPromisesDependency(null);
const uploadPromise = s3.upload(params).promise();
uploadPromise.then((data) => {
const response = {
...
};
return response;
}).catch((error) => {
console.log(error);
});
};
Calling it from Postman results in server error in Postman. CloudWatch logs have no further info.
Doing:
s3.upload(params, (error, data) => {
if (error) {
console.error("error occurred storing to s3: ", error);
return;
}
const response = {
...
};
return response;
});
does not result in a server error.
I am trying to follow the information from AWS that can be found here:
https://aws.amazon.com/blogs/developer/support-for-promises-in-the-sdk/
Postman is able to upload to Lambda by doing the following with async/await and try/catch:
exports.handler = async function(event, context) {
const s3 = new AWS.S3();
const encodedImage = util.inspect(event.body);
const decodedImage = Buffer.from(encodedImage, "base64");
const filePath = "test.png";
const params = {
Body: decodedImage,
Bucket: "my bucket",
Key: filePath,
ACL: "public-read",
ContentType: "mime/png"
};
try {
const result = await s3.upload(params).promise();
const response = {
statusCode: 200,
headers: {
my_header: "my_value"
},
body: JSON.stringify(result),
isBase64Encoded: false
};
return response;
} catch (error) {
console.log('error')
}
};
This code works just fine locally using nodejs. Images download from s3, write to file.
However, in Lambda (using nodejs 8.10) I'm getting "Internal Server Error" when testing the function with this in the Logs:
"Execution failed due to configuration error: Malformed Lambda proxy response"
I am using the lambda proxy response in the callback, but clearly some AWS SDK error with S3 is not getting caught.
I do have a role setup with S3 full access that the Lambda has access to.
What am I missing with my first Lambda function? Docs and tutorials I've followed correctly and it is not working.
const async = require('async')
const aws = require('aws-sdk')
const fs = require('fs')
const exec = require('child_process').exec
const bucket = 'mybucket'
const s3Src = 'bucket_prefix'
const s3Dst = 'new_prefix'
const local = `${__dirname}/local/`
aws.config.region = 'us-west-2'
const s3 = new aws.S3()
exports.handler = async (event, context, callback) => {
const outputImage = 'hello_world.png'
const rack = JSON.parse(event.body)
const images = my.images
async.waterfall([
function download(next) {
let downloaded = 0
let errors = false
let errorMessages = []
for (let i = 0; i < images.length; i++) {
let key = `${s3Src}/${images[i].prefix}/${images[i].image}`,
localImage = `${local}${images[i].image}`
getBucketObject(bucket, key, localImage).then(() => {
++downloaded
if (downloaded === images.length) { // js is non blocking, need to check if all images have been downloaded. If so, then go to next function
if (errors) {
next(errorMessages.join(' '))
} else {
next(null)
}
}
}).catch(error => {
errorMessages.push(`${error} - ${localImage}`)
++downloaded
errors = true
})
}
function getBucketObject(bucket, key, dest) {
return new Promise((resolve, reject) => {
let ws = fs.createWriteStream(dest)
ws.once('error', (err) => {
return reject(err)
})
ws.once('finish', () => {
return resolve(dest)
})
let s3Stream = s3.getObject({
Bucket: bucket,
Key: key
}).createReadStream()
s3Stream.pause() // Under load this will prevent first few bytes from being lost
s3Stream.on('error', (err) => {
return reject(err)
})
s3Stream.pipe(ws)
s3Stream.resume()
})
}
}
], err => {
if (err) {
let response = {
"statusCode": 400,
"headers": {
"my_header": "my_value"
},
"body": JSON.stringify(err),
"isBase64Encoded": false
}
callback(null, response)
} else {
let response = {
"statusCode": 200,
"headers": {
"my_header": "my_value"
},
"body": JSON.stringify(`<img src="${local}${outputImage}" />`),
"isBase64Encoded": false
}
callback(null, response)
}
}
)
}
Response should be always sent to callback function. Your code sends response only on error. That's why Lambda executor thinks your code fails.
BTW - should your functions in async.waterfall be separated with coma, as two tasks?
Locally, I've been running nodejs 10.10 and lambda currently is at 8.10. That is a big part I'm sure. In the end I had to remove the async. I had to move the getBucketObject function out of the waterfall. Once I made those adjustments it started working. And another issue was the downloaded images needed to go into "/tmp" directory.
const aws = require('aws-sdk')
const async = require('async')
const fs = require('fs')
const bucket = 'mybucket'
const s3Src = 'mys3src'
const local = '/tmp/'
aws.config.region = 'us-west-2'
const s3 = new aws.S3()
exports.handler = (event, context, callback) => {
const outputImage = 'hello_world.png'
async.waterfall([
function download(next) {
let downloaded = 0,
errorMessages = []
for (let i = 0; i < event['images'].length; i++) {
let key = `${s3Src}/${event['images'][i]['prefix']}/${event['images'][i]['image']}`,
localImage = `${local}${event['images'][i]['image']}`
getBucketObject(bucket, key, localImage).then(() => {
downloaded++
if (downloaded === event['images'].length) {
if (errorMessages.length > 0) {
next(errorMessages.join(' '))
} else {
console.log('All downloaded')
next(null)
}
}
}).catch(error => {
downloaded++
errorMessages.push(`${error} - ${localImage}`)
if (downloaded === event['images'].length) {
next(errorMessages.join(' '))
}
})
}
}
], err => {
if (err) {
console.error(err)
callback(null, {
"statusCode": 400,
"body": JSON.stringify(err),
"isBase64Encoded": false
})
} else {
console.log('event image created!')
callback(null, {
"statusCode": 200,
"body": JSON.stringify(`<img src="${local}${outputImage}" />`),
"isBase64Encoded": false
})
}
}
)
}
function getBucketObject(bucket, key, dest) {
return new Promise((resolve, reject) => {
let ws = fs.createWriteStream(dest)
ws.once('error', (err) => {
return reject(err)
})
ws.once('finish', () => {
return resolve(dest)
})
let s3Stream = s3.getObject({
Bucket: bucket,
Key: key
}).createReadStream()
s3Stream.pause() // Under load this will prevent first few bytes from being lost
s3Stream.on('error', (err) => {
return reject(err)
})
s3Stream.pipe(ws)
s3Stream.resume()
})
}