My Lambda is throwing a invoke error timeout - amazon-web-services

Normally my lambda runs fine, today I am starting to receive this error:
ERROR Invoke Error
{
"errorType": "TimeoutError",
"errorMessage": "Connection timed out after 120000ms",
"code": "TimeoutError",
"message": "Connection timed out after 120000ms",
"time": "2020-11-27T10:34:39.272Z",
"region": "eu-west-1",
"hostname": "lambda.eu-west-1.amazonaws.com",
"retryable": true,
"stack": [
"TimeoutError: Connection timed out after 120000ms",
" at ClientRequest.<anonymous> (/var/task/node_modules/aws-sdk/lib/http/node.js:83:34)",
" at Object.onceWrapper (events.js:286:20)",
" at ClientRequest.emit (events.js:198:13)",
" at ClientRequest.EventEmitter.emit (domain.js:448:20)",
" at TLSSocket.emitRequestTimeout (_http_client.js:673:40)",
" at Object.onceWrapper (events.js:286:20)",
" at TLSSocket.emit (events.js:198:13)",
" at TLSSocket.EventEmitter.emit (domain.js:448:20)",
" at TLSSocket.Socket._onTimeout (net.js:443:8)",
" at ontimeout (timers.js:436:11)"
]
}
I don't have any clue what might be causing this, I know that this lambda invokes several lambdas and awaits on all of them to conclude, I checked the logs on cloudwatch for each one and all of them had completed with success.
Here is a snippet of the code I use to trigger all those child lambdas and wait for them:
function invokeAll(){
const baseLambdaPayload = {
arg1: args.myarg1,
arg2: args.myarg2,
arg3: args.myarg3,
arg4: args.myarg4
};
lambdaInvocations = [
...lambdaInvocations,
triggerLamdba(
'lambda1', {
...baseLambdaPayload,
morePayload: await getMorePayload()
}),
triggerLamdba(
'lambda2', baseLambdaPayload),
triggerLamdba(
'lambda3', baseLambdaPayload),
triggerLamdba(
'lambda4', baseLambdaPayload)
];
return await Promise.all(lambdaInvocations);
}
const triggerLamdba = (name, payload) => {
const params = {
FunctionName: name,
InvocationType: 'RequestResponse',
Payload: JSON.stringify(payload)
};
return lambda.invoke(params).promise();
}
Any tips to where I should look into? Thanks.

Related

lambda aws failed to connect - connect ETIMEDOUT error to 2 external ips

I have an issue in the past 3 days with 3% of the requests to our lambdas.
They fail due to connection timeout to other aws services. see stack trace in the same lambda init
2021-10-30T16:37:33.310Z 7954e15a-8ae7-491e-880b-f5b532bde961 INFO TypeError: Unable to generate certificate due to
RequestError: Error: connect ETIMEDOUT 52.4.211.23:443
at /var/task/node_modules/cognito-express/lib/strategy.js:42:23
at bound (domain.js:416:15)
at runBound (domain.js:427:12)
at tryCatcher (/var/task/node_modules/bluebird/js/release/util.js:16:23)
at Promise._settlePromiseFromHandler (/var/task/node_modules/bluebird/js/release/promise.js:547:31)
at Promise._settlePromise (/var/task/node_modules/bluebird/js/release/promise.js:604:18)
at Promise._settlePromise0 (/var/task/node_modules/bluebird/js/release/promise.js:649:10)
at Promise._settlePromises (/var/task/node_modules/bluebird/js/release/promise.js:725:18)
at _drainQueueStep (/var/task/node_modules/bluebird/js/release/async.js:93:12)
at _drainQueue (/var/task/node_modules/bluebird/js/release/async.js:86:9)
at Async._drainQueues (/var/task/node_modules/bluebird/js/release/async.js:102:5)
at Immediate.Async.drainQueues [as _onImmediate] (/var/task/node_modules/bluebird/js/release/async.js:15:14)
at processImmediate (internal/timers.js:464:21)
at process.topLevelDomainCallback (domain.js:147:15)
at process.callbackTrampoline (internal/async_hooks.js:129:24
2021-10-30T16:44:18.380Z 25392661-b635-4b73-9aed-67e655f13364 ERROR Unhandled Promise Rejection
{
"errorType": "Runtime.UnhandledPromiseRejection",
"errorMessage": "SequelizeConnectionError: connect ETIMEDOUT",
"reason": {
"errorType": "SequelizeConnectionError",
"errorMessage": "connect ETIMEDOUT",
"name": "SequelizeConnectionError",
"parent": {
"errorType": "Error",
"errorMessage": "connect ETIMEDOUT",
"code": "ETIMEDOUT",
"errorno": "ETIMEDOUT",
"syscall": "connect",
"fatal": true,
"stack": [
"Error: connect ETIMEDOUT",
" at Connection._handleTimeoutError (/var/task/node_modules/mysql2/lib/connection.js:189:17)",
" at listOnTimeout (internal/timers.js:557:17)",
" at processTimers (internal/timers.js:500:7)"
]
},
"original": {
"errorType": "Error",
"errorMessage": "connect ETIMEDOUT",
"code": "ETIMEDOUT",
"errorno": "ETIMEDOUT",
"syscall": "connect",
"fatal": true,
"stack": [
"Error: connect ETIMEDOUT",
" at Connection._handleTimeoutError (/var/task/node_modules/mysql2/lib/connection.js:189:17)",
" at listOnTimeout (internal/timers.js:557:17)",
" at processTimers (internal/timers.js:500:7)"
]
},
"stack": [
"SequelizeConnectionError: connect ETIMEDOUT",
" at ConnectionManager.connect (/var/task/node_modules/sequelize/lib/dialects/mysql/connection-manager.js:126:17)",
" at processTicksAndRejections (internal/process/task_queues.js:95:5)",
" at async ConnectionManager._connect (/var/task/node_modules/sequelize/lib/dialects/abstract/connection-manager.js:318:24)",
" at async /var/task/node_modules/sequelize/lib/dialects/abstract/connection-manager.js:250:32",
" at async ConnectionManager.getConnection (/var/task/node_modules/sequelize/lib/dialects/abstract/connection-manager.js:280:7)",
" at async /var/task/node_modules/sequelize/lib/sequelize.js:613:26",
" at async MySQLQueryInterface.select (/var/task/node_modules/sequelize/lib/dialects/abstract/query-interface.js:953:12)",
" at async Function.findAll (/var/task/node_modules/sequelize/lib/model.js:1753:21)",
" at async /var/task/src/routes/root/index_routes.js:20:18"
]
},
"promise": {},
"stack": [
"Runtime.UnhandledPromiseRejection: SequelizeConnectionError: connect ETIMEDOUT",
" at process.<anonymous> (/var/runtime/index.js:35:15)",
" at process.emit (events.js:412:35)",
" at process.emit (domain.js:470:12)",
" at processPromiseRejections (internal/process/promises.js:245:33)",
" at processTicksAndRejections (internal/process/task_queues.js:96:32)"
]
}
here is the mysql init code
if (global.sequelize != null) {
console.count('\x1b[32mRESERCH: connection exported from globals instead of creation\x1b[0m');
module.exports = global.sequelize;
} else {
console.count('\x1b[31mRESERCH: new connection created\x1b[0m');
global.sequelize = new Sequelize(
s.sqlDbName,
s.sqlUsername,
s.sqlPassword, {
host: s.sqlDbHost,
dialect: 'mysql',
// to print out the query + it's time
// check if causes performance issues
benchmark: true,
pool: {
max: 5,
min: 0,
idle: 10000
}
});
It's only some of the requests but it's causing a lot of errors for our users.
couldn't detect the root cause.
seems to be solved by changing the lambda's VPC slightly
it had 2 subnets, public and private.
removed the public
Not sure why it worked. maybe it forces the lambda to connect to db and cognito from the internal ip

.StartExecution is not a function

I have created a lambda (so far so good).
Specifications:
Has access to stepfunctions
Runs smooth
version: Node.js 14.x
But when I try to invoke a step function from my lambda like this:
var AWS = require("aws-sdk");
var stepfunctions = new AWS.StepFunctions({apiVersion: '2016-11-23'});
const params = {
"input": "{}",
"name": srcKey,
"stateMachineArn": process.env.STATE_MACHINE_ARN
};
return stepfunctions.StartExecution(params);
I get the following error:
Response
{
"errorType": "TypeError",
"errorMessage": "stepfunctions.StartExecution is not a function",
"trace": [
"TypeError: stepfunctions.StartExecution is not a function",
" at /var/task/index.js:119:26",
" at wrapper (/var/task/node_modules/async/dist/async.js:273:20)",
" at Response.next (/var/task/node_modules/async/dist/async.js:4585:24)",
" at Response.<anonymous> (/var/task/node_modules/async/dist/async.js:326:20)",
" at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:369:18)",
" at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:106:20)",
" at Request.emit (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:78:10)",
" at Request.emit (/var/runtime/node_modules/aws-sdk/lib/request.js:688:14)",
" at Request.transition (/var/runtime/node_modules/aws-sdk/lib/request.js:22:10)",
" at AcceptorStateMachine.runTo (/var/runtime/node_modules/aws-sdk/lib/state_machine.js:14:12)"
]
}
Which makes no sense to me?
If I see this documentation:
https://docs.aws.amazon.com/step-functions/latest/apireference/API_StartExecution.html
And even this one:
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#constructor-property
The function should be available there.
What am I missing / doing wrong?
Thanks!
You have a typo...it's a lowercase s:
startExecution vs StartExecution
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#startExecution-property

Wrong SQS AWS message when I'm subscribed from a SNS Topic

I'm having problems with the next design:
When I'm receiving the message in my SQS Subscriber, the model of message it's wrong, example:
{
"Type" : "Notification",
"MessageId" : "7a6789f0-02f0-5ed3-8a11-deebcd08f145",
"TopicArn" : "arn:aws:sns:us-east-2:167186109795:name_sns_topic",
"Message" : "My JSON message",
"Timestamp" : "1987-04-23T17:17:44.897Z",
"SignatureVersion" : "1",
"Signature" : "string",
"SigningCertURL" : "url",
"UnsubscribeURL" : "url",
"MessageAttributes" : {
"X-Header1" : {"Type":"String","Value":"value1"},
"X-Header2" : {"Type":"String","Value":"value2"},
"X-Header3" : {"Type":"String","Value":"value3"},
"X-HeaderN" : {"Type":"String","Value":"value4"}
}
}
The common model when recieve message from SQS should be:
{
"Records": [
{
"messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
"receiptHandle": "MessageReceiptHandle",
"body": "Hello from SQS!",
"attributes": {
"ApproximateReceiveCount": "1",
"SentTimestamp": "1523232000000",
"SenderId": "123456789012",
"ApproximateFirstReceiveTimestamp": "1523232000001"
},
"messageAttributes": {},
"md5OfBody": "7b270e59b47ff90a553787216d55d91d",
"eventSource": "aws:sqs",
"eventSourceARN": "arn:{partition}:sqs:{region}:123456789012:MyQueue",
"awsRegion": "{region}"
}
]
}
In my handler Java Lambda (example code) is throwing an exception because the estructure of de message received is not SQS Event:
public class MyHandler implements RequestHandler<SQSEvent, String> {
#Override
public String handleRequest(SQSEvent event, Context context) {
LambdaLogger logger = context.getLogger();
for (SQSEvent.SQSMessage msg : event.getRecords()) {
logger.log("SQS message body: " + msg.getBody());
logger.log("Get attributes: " + msg.getMessageAttributes().toString());
msg.getMessageAttributes()
.forEach(
(k, v) -> {
logger.log("key: " + k + "value: " + v.getStringValue());
});
}
return "Successful";
}
}
How can I do for handle the message thats its receiving ?
In my opinion this isn't documented too well but it's not bad once you figure it out.
The first thing is that I don't use the predefined Lambda objects. I read everything into a String and take it from there. So the base of my Lamda function is:
public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
// copy InputStream to String, avoiding 3rd party libraries
ByteArrayOutputStream result = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = inputStream.read(buffer)) != -1) {
result.write(buffer, 0, length);
}
String jsonString = result.toString();
}
When you "go direct" from SNS to Lambda the message looks something like (some fields removed for sake of length):
{
"Records": [
{
"EventSource": "aws:sns",
"EventVersion": "1.0",
"Sns": {
"Type": "Notification",
"Subject": "the message subject",
"Message": "{\"message\": \"this is the message\", \"value\": 100}",
"Timestamp": "2020-04-24T21:44:28.220Z",
"SignatureVersion": "1"
}
}
]
}
I had sent in a test message in JSON with two simple fields. Using JsonPath the "message" field inside of everything is read with:
String snsMessage = JsonPath.read(jsonString, "$.Records[0].Sns.Message");
String realMessage = JsonPath.read(snsMessage, "$.message");
But when it goes SNS -> SQS -> Lambda (or, indeed any SNS -> SQS path) the SNS message is now mostly wrapped and escaped in an SQS message:
{
"Records": [
{
"messageId": "ca8c53e5-8417-4479-a720-d4ecf970ca68",
"body": "{\n \"Type\" : \"Notification\",\n \"Subject\" : \"the message subject\",\n \"Message\" : \"{\\\"message\\\": \\\"this is the message\\\", \\\"value\\\": 100}\"\n}",
"attributes": {
"ApproximateReceiveCount": "1"
},
"md5OfBody": "6a4840230aca6a7bf7934bf191a529b8",
"eventSource": "aws:sqs"
}
]
}
So in this case, the value is in Records[0].body but that contains another JSON object. I'll admit that there is likely an easier way but from what I found I had to parse 3 times:
String sqsBody = <as read in lambda>;
String recordBody = JsonPath.read(sqsBody, "$.Records[0].body");
String internalMessage = JsonPath.read(recordBody, "$.Message");
// now read out of the sns message
String theSnsMessage = JsonPath.read(message, "$.message");

aws lambda returning response card throwing a null fulfillmentState error in Amazon Lex?

I have written this function which returns fine when it is just returning as a String. I have followed the syntax for the response card very closely and it passes my test case in lambda. However when it's called through Lex it throws an error which i'll post below. It says fulfillmentState cannot be null but in the error it throws it shows that it is not null.
I have tried switching the order of the dialogue state and response card, i have tried switching the order of "type" and "fulfillmentState". Function:
def backup_phone(intent_request):
back_up_location = get_slots(intent_request)["BackupLocation"]
phone_os = get_slots(intent_request)["PhoneType"]
try:
from googlesearch import search
except ImportError:
print("No module named 'google' found")
# to search
query = "How to back up {} to {}".format(phone_os, back_up_location)
result_list = []
for j in search(query, tld="com", num=5, stop=5, pause=2):
result_list.append(j)
return {
"dialogAction": {
"fulfilmentState": "Fulfilled",
"type": "Close",
"contentType": "Plain Text",
'content': "Here you go",
},
'responseCard': {
'contentType': 'application/vnd.amazonaws.card.generic',
'version': 1,
'genericAttachments': [{
'title': "Please select one of the options",
'subTitle': "{}".format(query),
'buttons': [
{
"text": "{}".format(result_list[0]),
"value": "test"
},
]
}]
}
}
screenshot of test case passing in lambda: https://ibb.co/sgjC2WK
screenshot of error throw in Lex: https://ibb.co/yqwN42m
Text for the error in Lex:
"An error has occurred: Invalid Lambda Response: Received invalid response from Lambda: Can not construct instance of CloseDialogAction, problem: fulfillmentState must not be null for Close dialog action at [Source: {"dialogAction": {"fulfilmentState": "Fulfilled", "type": "Close", "contentType": "Plain Text", "content": "Here you go"}, "responseCard": {"contentType": "application/vnd.amazonaws.card.generic", "version": 1, "genericAttachments": [{"title": "Please select one of the options", "subTitle": "How to back up Iphone to windows", "buttons": [{"text": "https://www.easeus.com/iphone-data-transfer/how-to-backup-your-iphone-with-windows-10.html", "value": "test"}]}]}}; line: 1, column: 121]"
I fixed the issue by sending everything i was trying to return through a function, and then storing all the info in the correct syntax into an object, which i then returned from the function. Relevant code below:
def close(session_attributes, fulfillment_state, message, response_card):
response = {
'sessionAttributes': session_attributes,
'dialogAction': {
'type': 'Close',
'fulfillmentState': fulfillment_state,
'message': message,
"responseCard": response_card,
}
}
return response
def backup_phone(intent_request):
back_up_location = get_slots(intent_request)["BackupLocation"]
phone_os = get_slots(intent_request)["PhoneType"]
try:
from googlesearch import search
except ImportError:
print("No module named 'google' found")
# to search
query = "How to back up {} to {}".format(phone_os, back_up_location)
result_list = []
for j in search(query, tld="com", num=1, stop=1, pause=1):
result_list.append(j)
return close(intent_request['sessionAttributes'],
'Fulfilled',
{'contentType': 'PlainText',
'content': 'test'},
{'version': 1,
'contentType': 'application/vnd.amazonaws.card.generic',
'genericAttachments': [
{
'title': "{}".format(query.lower()),
'subTitle': "Please select one of the options",
"imageUrl": "",
"attachmentLinkUrl": "{}".format(result_list[0]),
'buttons': [
{
"text": "{}".format(result_list[0]),
"value": "Thanks"
},
]
}
]
}
)

Repeatedly send API request based on parameters in Postman

I have a database table with approx 4,000 records (currently). A response to an API call (POST, JSON) gives me data of this table for a maximum of 1,000 records per API call. A parameter ‘PageNo’ defines which of the 4,000 records are selected (e.g. PageNo = 1 gives me record 1-1000). The header data of the response includes a ‘PageCount’, in my example 4. I am able to retrieve that ‘PageCount’ and the test below loops through the PageNo (result in Postman Console = 1 2 3 4).
How I can call the same request repeatedly in a loop and use the values of the PageNo (i) as a parameter for that request like so:
{{baseUrl}}/v1/Units/Search?PageNo={{i}}
In my example I would expect the request to run 4 times with PageNo2 = 1, 2, 3, 4.
I am aware that I can use a CSV file and loop through the request in Collection Runner but PageCount changes (i.e. the number of records in the table change) and I need to run this loop frequently so creating a new CSV file for each loop is not really an option.
Postman Test:
pm.environment.set('Headers2', JSON.stringify(pm.response.headers));
var Headers2 = JSON.stringify(pm.response.headers);
pm.environment.set('PageCount2', JSON.parse(Headers2)[10].value);
var i;
for (i = 1; [i] <= pm.environment.get('PageCount2'); i++) {
console.log(i);
postman.setNextRequest('custom fields | json Copy');
}
Postman Request:
{
"Location":"{{TestingLocation}}",
"Fields":[
"StockNo",
"BrandDesc"
],
"Filters": {
"StatusCode":"{{TestingUnitSearchStatusCode}}"
},
"PageSize":1000,
"PageNo" : "{{i}}"
}
With postman.setNextRequest() you can set the calling request as the same request. But you need an exit strategy, otherwise that request would be called infinite times. This only works with the collection runner.
On your first(!) request, store the amount of pages and create a counter.
Increment the counter. If it is smaller than the amount of pages, set the current request as the next request.
Else, do not set a next request. The collection runner will continue with its normal flow. Optionally remove the pages and counter variables.
You can let the request detect that it is its first iteration by not having initialized the amount of pages and counter variables.
Performed a POC for your use-case using postman-echo API.
Pre-req script --> Takes care of the initial request to the endpoint to retrieve the PageSize and set it as an env var. Also initializes the iterationCount to 1 (as env var)
Test Script --> Checks for the current iteration number and performs tests.
Here's a working postman collection.
If you're familiar with newman, you can just run:
newman run <collection-name>
You can also import this collection in Postman app and use collection runner.
{
"info": {
"_postman_id": "1d7669a6-a1a1-4d01-a162-f8675e01d1c7",
"name": "Loop Req Collection",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
{
"name": "Run_Req",
"event": [
{
"listen": "test",
"script": {
"id": "dac5e805-548b-4bdc-a26e-f56500414208",
"exec": [
"var iterationCount = pm.environment.get(\"iterationCount\");",
"if (pm.environment.get(\"iterationCount\") <= pm.environment.get(\"pageSize\")) {",
" console.log(\"Iteration Number: \" + iterationCount);",
"",
" pm.test(\"Status code is 200\", function() {",
" pm.response.to.have.status(200);",
" });",
"",
" pm.test(\"Check value of pageNo in body is equal to iterationCount\", function() {",
" var jsonData = pm.response.json();",
" return jsonData.json.PageNo == iterationCount;",
"",
" });",
" iterationCount++;",
" pm.environment.set(\"iterationCount\", iterationCount);",
" postman.setNextRequest('Run_Req');",
"}",
"if (pm.environment.get(\"iterationCount\") > pm.environment.get(\"pageSize\")) {",
" console.log(\"Cleanup\");",
" pm.environment.unset(\"pageSize\");",
" pm.environment.set(\"iterationCount\", 1);",
" postman.setNextRequest('');",
" pm.test(\"Cleanup Success\", function() {",
" if (pm.environment.get(\"pageSize\") == null && pm.environment.get(\"iterationCount\") == null) {",
" return true;",
" }",
" });",
"}"
],
"type": "text/javascript"
}
},
{
"listen": "prerequest",
"script": {
"id": "3d43c6c7-4a9b-46cf-bd86-c1823af5a68e",
"exec": [
"if (pm.environment.get(\"pageSize\") == null) {",
" pm.sendRequest(\"https://postman-echo.com/response-headers?pageSize=4\", function(err, response) {",
" var pageSize = response.headers.get('pageSize');",
" var iterationCount = 1;",
" console.log(\"pre-req:pageSize= \" + pageSize);",
" console.log(\"pre-req:iterationCount= \" + iterationCount);",
" pm.environment.set(\"pageSize\", pageSize);",
" pm.environment.set(\"iterationCount\", iterationCount);",
" });",
"",
"}"
],
"type": "text/javascript"
}
}
],
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"name": "Content-Type",
"value": "application/json",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"Location\":\"{{TestingLocation}}\",\n \"Fields\":[\n \"StockNo\",\n \"BrandDesc\"\n ],\n \"Filters\": {\n \"StatusCode\":\"{{TestingUnitSearchStatusCode}}\"\n },\n \"PageSize\":1000,\n \"PageNo\" : \"{{iterationCount}}\"\n}"
},
"url": {
"raw": "https://postman-echo.com/post",
"protocol": "https",
"host": [
"postman-echo",
"com"
],
"path": [
"post"
]
}
},
"response": []
}
]
}