I am trying to create a lambda function which will check if a particular repository exist in codecommit.
Lamda service role is having admin priviledge. Below is the code.
The lambda is unable to call getRepository method. It is niether giving any exception nor passing. Any help on this?
console.log("Before calling cc") This is last printed statement. After that I am not getting any success or error log.
const CloudFormation = require('aws-sdk/clients/cloudformation');
const Codecommit = require('aws-sdk/clients/codecommit');
exports.handler = async (event) => {
try{
console.log("event",event);
console.log("event",JSON.stringify(event));
var repositoryName = event.detail.repositoryName;
var cfn = new CloudFormation({
region: "ap-northeast-1"
});
var cc = new Codecommit({
region: "ap-northeast-1"
});
const stackName = repositoryName+"-infra-stack";
var cloneUrl;
console.log("RepositoryName"+repositoryName);
console.log("StackName"+stackName);
var codeCommitParam = {
repositoryName: repositoryName
};
try{
console.log("Before calling cc")
cc.getRepository(codeCommitParam, function(err, data) {
if (err){
console.log(err, err.stack);
}else {
console.log(data.repositoryMetadata.cloneUrlHttp);
cloneUrl=data.repositoryMetadata.cloneUrlHttp;
console.log("Clone url "+cloneUrl);
checkStackDescription();
}
});
}catch(error){
console.log(error);
}
}
I believe this is coming down to the JavaScript in the Lambda being invoked asynchronously so the Lambda is finishing invoking before the callback processes the response.
Try updating to use this synchronously by updating to the below syntax.
console.log("Before calling cc")
let result = await cc.getRepository(codeCommitParam).promise();
console.log(result);
Be aware that result could either be an error or valid response.
Related
Hi I have a Dax cluster on top of a DynamoDB. Everything is set in the same VPC and subnets are created correctly. Also the permissions are setup correcty:
Lambda has full access to dynamodb:* and dax:*
Dax cluster policy has full access for DynamoDB - dynamodb:* and also a trust policy sts:assumeRole for dax service.
This is how I connect to Dax client inside my Lambda:
const AWS = require("aws-sdk");
const AmazonDaxClient = require("amazon-dax-client"); //VERSION: 1.2.9
let docClient = new AWS.DynamoDB.DocumentClient({
region: "eu-west-1",
convertEmptyValues: true,
});
let daxService = null;
let daxClient = null;
async function connectToDAX(useDAX = false, force = false) {
try {
if (!useDAX) {
return docClient;
} else {
if (force) {
daxService = null;
daxClient = null;
}
if (daxService == null) {
const endpoint = await amazon.getParam(
`/${lambdaService.getCurrentStage()}/project/aws/daxClusterEndpoint`
);
const region = await amazon.getParam(
`/${lambdaService.getCurrentStage()}/prjoect/region`
);
const daxConfig = {
endpoints: [endpoint],
region: region,
};
daxService = new AmazonDaxClient(daxConfig);
daxClient = new AWS.DynamoDB.DocumentClient({
service: daxService,
region: "eu-west-1",
convertEmptyValues: true,
});
console.log("created new DAX connection");
}
return daxClient;
}
} catch (e) {
console.log("error on connectToDax", e);
return null;
}
}
async function queryTable(params, dax = false) {
try {
let client = dax ? daxClient : docClient;
if (dax && !daxClient) {
client = await connectToDAX(dax);
}
return await new Promise(function (resolve, reject) {
client.query(params, function (err, data) {
if (err) {
console.log("queryTable", params);
console.log("query error", err);
reject(err);
} else {
resolve(data);
}
});
});
} catch (e) {
console.log("error dynamodb queryTable", e);
daxClient = null;
daxService = null;
console.log("retrying queryTable with new dax connection");
return await queryTable(params, dax);
}
}
So, I am sending like 100 requests at a time towards API where my Lambda is getting invoked and it calls queryTable().
I am getting a lot of errors, 30% of all requests are Errors, which I would like to fix.
Errors like:
Unknown application error occurred
{"errorType":"Error","errorMessage":"read ECONNRESET","code":"ECONNRESET","errno":-104,"syscall":"read","stack":["Error: read ECONNRESET"," at TCP.onStreamRead (internal/stream_base_commons.js:209:20)"]}
Client does not have permission to invoke DefineKeySchema
So I have spent a lot of time troubleshooting this. Went trough all permissions and setup, but if it was something wrong there, NONE of the request would be successful.
I am thinking that the problem is with the DAX Client connection when Lambda concurrent invocations are happening. I have tried to force each concurrent execution to create new Dax Client connection, but that resulted even with more errors.
I am retrying as you can see in the queryTable() function.
I don't know what to do anymore. Please give some input.
I have followed this blog to update the code of a lambda function using a jar file stored in a S3 bucket. the execution was succeded, but it is not updating the code of target lambda function
Code snippet
console.log('Loading function');
var AWS = require('aws-sdk');
var lambda = new AWS.Lambda();
exports.handler = function(event, context) {
var functionName = "runJarFile";
var bucket = "jarfiletest2";
var key = "lambda-java-example-0.0.1-SNAPSHOT.jar.zip";
console.log("uploaded to lambda function: " + functionName);
var params = {
FunctionName: functionName,
S3Key: key,
S3Bucket: bucket,
Publish: true
};
lambda.updateFunctionCode(params, function(err, data) {
if (err) {
console.log(err, err.stack);
context.fail(err);
} else {
console.log(data);
context.succeed(data);
}
});
};
Thanks in advance
It's difficult to comment on this without knowing the details about the destination function. What's the output of the GetFunction API call of that Lambda, before and after calling the UpdateFunctionConfig call?
I am interested to see the SHA-256 hash of the code, and the last modified timestamp off that API call before and after calling UpdateFunctionConfig:
{
...
"CodeSha256": "5tT2qgzYUHoqwR616pZ2dpkn/0J1FrzJmlKidWaaCgk=",
"LastModified": "2019-09-24T18:20:35.054+0000"
...
}
If the values are exactly the same, can you add this check as per the blog post to see if the bucket and the object exists?
if (bucket == "YOUR_BUCKET_NAME" && key == "YOUR_CODE.zip" && version) {
// your code
} else {
context.succeed("skipping zip " + key + " in bucket " + bucket + " with version " + version);
}
Pls try to remove 'Publish: true' to call the latest version not the specified version
We're trying to develop a self-invoking lambda to process S3 files in chunks. The lambda role has the policies needed for the invocation attached.
Here's the code for the self-invoking lambda:
export const processFileHandler: Handler = async (
event: S3CreateEvent,
context: Context,
callback: Callback,
) => {
let bucket = loGet(event, 'Records[0].s3.bucket.name');
let key = loGet(event, 'Records[0].s3.object.key');
let totalFileSize = loGet(event, 'Records[0].s3.object.size');
const lastPosition = loGet(event, 'position', 0);
const nextRange = getNextSizeRange(lastPosition, totalFileSize);
context.callbackWaitsForEmptyEventLoop = false;
let data = await loadDataFromS3ByRange(bucket, key, nextRange);
await database.connect();
log.debug(`Successfully connected to the database`);
const docs = await getParsedDocs(data, lastPosition);
log.debug(`upserting ${docs.length} records to database`);
if (docs.length) {
try {
// upserting logic
log.debug(`total documents added: ${await docs.length}`);
} catch (err) {
await recurse(nextRange.end, event, context);
log.debug(`error inserting docs: ${JSON.stringify(err)}`);
}
}
if (nextRange.end < totalFileSize) {
log.debug(`Last ${context.getRemainingTimeInMillis()} milliseconds left`);
if (context.getRemainingTimeInMillis() < 10 * 10 * 10 * 6) {
log.debug(`Less than 6000 milliseconds left`);
log.debug(`Invoking next iteration`);
await recurse(nextRange.end, event, context);
callback(null, {
message: `Lambda timed out processing file, please continue from LAST_POSITION: ${nextRange.start}`,
});
}
} else {
callback(null, { message: `Successfully completed the chunk processing task` });
}
};
Where recurse is an invocation call to the same lambda. Rest of the things work as expected it just times out whenever the call stack comes on this invocation request:
const recurse = async (position: number, event: S3CreateEvent, context: Context) => {
let newEvent = Object.assign(event, { position });
let request = {
FunctionName: context.invokedFunctionArn,
InvocationType: 'Event',
Payload: JSON.stringify(newEvent),
};
let resp = await lambda.invoke(request).promise();
console.log('Invocation complete', resp);
return resp;
};
This is the stack trace logged to CloudWatch:
{
"errorMessage": "connect ETIMEDOUT 63.32.72.196:443",
"errorType": "NetworkingError",
"stackTrace": [
"Object._errnoException (util.js:1022:11)",
"_exceptionWithHostPort (util.js:1044:20)",
"TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)"
]
}
Not a good idea to create a self-invoking lambda function. In case of an error (could also be a bad handler call on AWS side) a lambda function might re-run several times. Very hard to monitor and debug.
I would suggest using Step Functions. I believe this tutorial can help Iterating a Loop Using Lambda
From the top of my head, if you prefer not dealing with Step Functions, you could create a Lambda trigger for an SQS queue. Then you pass a message to the queue if you want to run the lambda function another time.
I am starting a step function from Lambda and the Lambda function is tied to an API Gateway. For some reason, when I try to test the Lambda function, I see hundreds of executions failed and running in loop. I just triggered the step function once. I am missing something here. Can you please advise.
const AWS = require("aws-sdk");
const uuidv4 = require("uuid/v4");
/*----------------------------------------------------------------------- */
/* Implementation */
/*----------------------------------------------------------------------- */
exports.handler = async event => {
var _dt = await ExecuteStepFunction()
return _dt;
}
function ExecuteStepFunction() {
const stepFunctions = new AWS.StepFunctions();
return new Promise((res, rej) => {
var params = {
stateMachineArn: 'arn:aws:states:us-east-1:xxxxxxxxxxxxx:stateMachine:xxTestSateMachine',
input: JSON.stringify(''),
name: uuidv4()
};
stepFunctions.startExecution(params, function (err, data) {
if (err) {
rej(err);
}
else {
res(data);
}
});
});
}
I tried thIS approach provided in the this link (https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-api-gateway.html) where the API gateway directly triggers the step function but I am receiving the following error. After trying to fix this, I move to the above option of starting the function using the API.
{
"__type": "com.amazon.coral.service#UnrecognizedClientException",
"message": "The security token included in the request is invalid"
}
I am trying to use AWS mobile backend (using lambda function) to insert into dynamoDB (also configured at the mobile backend) but with no success so far.
The relevant code:
'use strict';
console.log("Loading function");
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:process.env.MOBILE_HUB_PROJECT_REGION});
exports.handler = function(event, context, callback) {
var responseCode = 200;
var requestBody, pathParams, queryStringParams, headerParams, stage,
stageVariables, cognitoIdentityId, httpMethod, sourceIp, userAgent,
requestId, resourcePath;
console.log("request: " + JSON.stringify(event));
// Request Body
requestBody = event.body;
if (requestBody !== undefined && requestBody !== null) {
// Set 'test-status' field in the request to test sending a specific response status code (e.g., 503)
responseCode = JSON.parse(requestBody)['test-status'];
}
// Path Parameters
pathParams = event.path;
// Query String Parameters
queryStringParams = event.queryStringParameters;
// Header Parameters
headerParams = event.headers;
if (event.requestContext !== null && event.requestContext !== undefined) {
var requestContext = event.requestContext;
// API Gateway Stage
stage = requestContext.stage;
// Unique Request ID
requestId = requestContext.requestId;
// Resource Path
resourcePath = requestContext.resourcePath;
var identity = requestContext.identity;
// Amazon Cognito User Identity
cognitoIdentityId = identity.cognitoIdentityId;
// Source IP
sourceIp = identity.sourceIp;
// User-Agent
userAgent = identity.userAgent;
}
// API Gateway Stage Variables
stageVariables = event.stageVariables;
// HTTP Method (e.g., POST, GET, HEAD)
httpMethod = event.httpMethod;
// TODO: Put your application logic here...
let params = {
Item:{
"prop1":0,
"prop2":"text"
},
TableName:"testTable"
};
docClient.put(params, function(data, err){
if(err)
responseCode = 500;
else
{
responseCode = 200;
context.succeed(data);
}
});
// For demonstration purposes, we'll just echo these values back to the client
var responseBody = {
requestBody : requestBody,
pathParams : pathParams,
queryStringParams : queryStringParams,
headerParams : headerParams,
stage : stage,
stageVariables : stageVariables,
cognitoIdentityId : cognitoIdentityId,
httpMethod : httpMethod,
sourceIp : sourceIp,
userAgent : userAgent,
requestId : requestId,
resourcePath : resourcePath
};
var response = {
statusCode: responseCode,
headers: {
"x-custom-header" : "custom header value"
},
body: JSON.stringify(responseBody)
};
console.log("response: " + JSON.stringify(response))
context.succeed(response);
};
this doesn't put the item to the table for some reason.
I gave the necessary permissions using the roles part, anything I am missing?
**responseCode is only for testing purposes.
Edit:
tried AWS node.js lambda request dynamodb but no response (no err, no return data) and doesn't work either.
Edit2:
Added the full handler code. (it the default generated code when creating first AWS lambda).
I have refactored some bits of your code to look much simpler and use async/await (make sure to select Node 8.10 as the running environment for your function) instead of callbacks. I also got rid of the context and callback parameters, as they were used for older versions of NodeJS. Once you're using Node 8+, async/await should be the default option.
Also, it is possible to chain a .promise() on docClient.putItem, so you can easily await on it, making your code way simpler. I have left only the DynamoDB part (which is what is relevant to your question)
'use strict';
console.log("Loading function");
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:process.env.MOBILE_HUB_PROJECT_REGION});
exports.handler = async (event) => {
let params = {
Item:{
"prop0":1,
"prop2":"text"
},
TableName:"testTable"
};
try {
await docClient.put(params).promise();
} catch (e) {
console.log(e)
return {
messsage: e.message
}
}
return { message: 'Data inserted successfully' };
};
Things to keep in mind if still it does not work:
Make sure your Lambda function has the right permissions to insert items on DynamoDB (AmazonDynamoDBFullAccess will do it)
You ALWAYS have to provide the partition key when inserting items to DynamoDB. On your example, the JSON only has two properties: prop1 and prop2. If none of them are the partition key, your code will certainly fail.
Make sure you table also exists
If you code fails, just check CloudWatch logs as any exception is now captured and printed out on the console.
The reason why no data is written in the table is because the call to DynamoDB put is asynchronous and will return by calling your callback. But during that time, the rest of the code continues to execute and your function eventually finish before the call to DynamoDB has a chance to complete.
You can use the await / async keywords to make your code sychronous :
async function writeToDynamoDB(params) {
return new Promise((resolve,reject) => {
docClient.put(params, function(data, err){
if(err)
reject(500);
else
resolve(data);
});
});
}
let params = ...
var data = await writeToDynamoDB(params)
You can find sample code I wrote (in Typescript) at https://github.com/sebsto/maxi80-alexa/blob/master/lambda/src/DDBController.ts