I am trying to write a data transformation for an Aurora Postgres data stream. The identity transformation gives me objects like this:
{
"type": "DatabaseActivityMonitoringRecords",
"version": "1.0",
"databaseActivityEvents": "AYADeLZrKReAa/7tgBmd/d06ybQAXwA...BABVoaWV1Fi8LyA==",
"key": "AADDDVAS ...pztPgaw=="
}
So it appears to me that I need to decrypt these. I have the arn of the key attached to the database in question but I cannot get the data transformation to work. Here is what I have so far:
console.log('Loading function');
const zlib = require('zlib');
const AWS = require('aws-sdk');
exports.handler = async (event, context) => {
/* Process the list of records and transform them */
const output = event.records.map((record) => {
const keyId = "arn:aws:kms:us-east-1:501..89:key/1...d7"; // I don't need this?
const CiphertextBlob = record.data;
const kmsClient = new AWS.KMS({region: 'us-east-1'});
const pt = kmsClient.decrypt({ CiphertextBlob }, (err, data) => {
if (err) console.log(err, err.stack); // an error occurred
else {
const { Plaintext } = data;
console.log(data);
return Plaintext;
}
});
return { old_data : record.data,};
});
console.log(`Processing completed. Successful records ${output.length}.`);
return { records: output };
};
This gives me the baffling error "errorMessage": "Error: Unable to stringify response body" I am following the example outlined here: https://docs.aws.amazon.com/kms/latest/developerguide/programming-encryption.html#decryption
Any idea what I am doing wrong here?
Related
What is the way to get the response body of the uploaded JSON file to the lambda function to print it?
I used the following code but it is specified for Content Type. Any suggestions for this please?
// console.log('Loading function');
const aws = require('aws-sdk');
const s3 = new aws.S3({ apiVersion: '2006-03-01' });
exports.handler = async (event, context) => {
//console.log('Received event:', JSON.stringify(event, null, 2));
// Get the object from the event and show its content type
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: bucket,
Key: key,
};
try {
const { ContentType } = await s3.getObject(params).promise();
console.log('CONTENT TYPE:', ContentType);
console.log('Body: ', );
console.log("response: " + "I want to print the response body here when a JSON file uploaded")
return ContentType;
} catch (err) {
console.log(err);
const message = `Error getting object ${key} from bucket ${bucket}. Error : ` + err;
console.log(message);
throw new Error(message);
}
};
return value of getObject contains Body field.
const { ContentType, Body } = await s3.getObject(params).promise();
console.log('CONTENT TYPE:', ContentType);
console.log('Body: ', Body);
I am using AWS and have an api which is called via API gateway which calls a node.js lambda function.
Very often but randomly I get 502 responses but when I immediately try again with the exact same request I get a normal response. So I decided to search the logs to see if I could find any issues.
The following is what I found for 1 of the requests:
RequestId: xxxxx Error: Runtime exited with error: signal: segmentation fault Runtime.ExitError
as well as:
xxxx ERROR Uncaught Exception
{
"errorType": "Error",
"errorMessage": "Quit inactivity timeout",
"code": "PROTOCOL_SEQUENCE_TIMEOUT",
"fatal": true,
"timeout": 30000,
"stack": [
"Error: Quit inactivity timeout",
" at Quit.<anonymous> (/opt/nodejs/node_modules/mysql/lib/protocol/Protocol.js:160:17)",
" at Quit.emit (node:events:527:28)",
" at Quit.emit (node:domain:475:12)",
" at Quit._onTimeout (/opt/nodejs/node_modules/mysql/lib/protocol/sequences/Sequence.js:124:8)",
" at Timer._onTimeout (/opt/nodejs/node_modules/mysql/lib/protocol/Timer.js:32:23)",
" at listOnTimeout (node:internal/timers:559:17)",
" at processTimers (node:internal/timers:502:7)"
]
}
the following is my reusable sql connector:
const CustomSecret = require('../secrets/CustomSecret');
const mysql = require("mysql");
module.exports = class MySqlConnect {
databaseCredObject;
constructor() {
}
async queryDb(sql, args) {
if (!this.databaseCredObject) {
await this.fetchSecret();
}
let connection = null;
const connection_settings = {
host: this.databaseCredObject.host,
user: this.databaseCredObject.username,
password: this.databaseCredObject.password,
database: 'logbook'
};
connection = mysql.createConnection(connection_settings);
return new Promise((resolve, reject) => {
connection.connect(function (err) {
if (err) {
console.log('error when connecting to db:', err);
} else {
console.log('Connected');
connection.query(sql, args, function (err, result) {
connection.end();
if (err) {
return reject(err);
}
return resolve(result);
});
}
});
});
}
async fetchSecret() {
const databaseCredString = await CustomSecret.getSecret('secretname', 'eu-west-2');
this.databaseCredObject = JSON.parse(databaseCredString);
}
}
Finally this is an example of my lambda function (shortened version):
const {compress, decompress} = require("compress-json");
const MySqlConnect = require("customPackagePath/MySqlConnect");
const CustomJwt = require("customPackagePath/CustomJwt");
const AWS = require("aws-sdk");
const warmer = require("lambda-warmer");
exports.handler = async (event) => {
if (await warmer(event)) {
console.log("Warming");
return 'warmed';
}
let responseCode = 200;
let response = {};
response.headers = {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*'
};
const bodyContent = JSON.parse(event.body);
const dataType = bodyContent.dataType;
const webAuth = new CustomJwt();
const decodedToken = webAuth.decodeToken(event.headers.Authorization);
const userUUID = decodedToken['uuid'];
const connection = new MySqlConnect();
let sql;
switch (dataType) {
case 'userPreferences':
sql = await connection.queryDb('SELECT * FROM user WHERE uuid = ?', [userUUID]);
break;
}
let data = [];
for (let index in sql) {
data.push(JSON.parse(JSON.stringify(sql[index])));
}
const returnData = {
data
};
let compressed = compress(returnData);
response.statusCode = responseCode;
response.body = JSON.stringify(compressed);
return response;
};
Now I am new to infrastructure stuff. But it seems to me that once a lambda function has been called, its not closing or ending correctly. Also I am using the lambda warmer to keep the functions warm as seen in the lambda code and not sure if that is causing any issues.
Appreciate any help with this as I can't seem to figure it out.
Thanks
After doing more research I decided to add this to my Lambda function:
exports.handler = async (event, context, callback) => {
and the return like this
callback(null, response);
and ever since this issue seems to have been resolved. I am not entirely sure why but for now its looking good :)
CURRENTLY
I am trying to get AWS Textract working for images supplied from a function in Google Scripts, that is sent to a Lambda resolved. I am following documentation on https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Textract.html#analyzeDocument-property
My Google Scripts code:
function googleFunction(id) {
let file = DriveApp.getFileById(id);
console.log("File is a " + file.getMimeType());
let blob = file.getBlob();
let params = {
doc: blob,
};
var options = {
method: "PUT",
"Content-Type": "application/json",
payload: JSON.stringify(params),
};
let response = UrlFetchApp.fetch("https://api-path/prod/resolver", options);
}
My Lambda resolver code:
"use strict";
const AWS = require("aws-sdk");
exports.handler = async (event) => {
let params = JSON.parse(event.body);
console.log("Parse as document...");
let textract = new AWS.Textract();
let doc = params["doc"];
let config = {
Document: {
Bytes: doc,
FeatureTypes: ["TABLES"],
}
};
textract.analyzeDocument(config, function (err, data) {
console.log("analyzing...");
if (err) {
console.log(err, err.stack);
}
// an error occurred
else {
console.log("data:" + JSON.stringfy(data));
} // successful response
});
};
ISSUE
File is successfully sent from Google Scripts to Lambda, but the following error is returned:
"errorType": "InvalidParameterType",
"errorMessage": "Expected params.Document.Bytes to be a string, Buffer, Stream, Blob, or typed array object"
Questions
Is there a way of verifying what the format of the doc variable is, to ensure it meets AWS Textract's requirements?
Can anyone see a possible cause for the errors being returned?
NOTES
Textract works fine when the same file is uploaded to an S3 bucked, and supplied in the config using:
S3Object: { Bucket: 'bucket_name', Name: 'file_name' }
I have confirmed the file is a JPEG
Got it working with 2 changes:
added getBytes() to Google side code
added Buffer.from() to AWS side code
My Google Scripts code:
function googleFunction(id) {
let file = DriveApp.getFileById(id);
console.log("File is a " + file.getMimeType());
let blob = file.getBlob().getBytes();
let params = {
doc: blob,
};
var options = {
method: "PUT",
"Content-Type": "application/json",
payload: JSON.stringify(params),
};
let response = UrlFetchApp.fetch("https://api-path/prod/resolver", options);
}
My Lambda resolver code:
"use strict";
const AWS = require("aws-sdk");
exports.handler = async (event) => {
let params = JSON.parse(event.body);
console.log("Parse as document...");
let textract = new AWS.Textract();
let doc = params["doc"];
let config = {
Document: {
Bytes: Buffer.from(doc),
FeatureTypes: ["TABLES"],
}
};
textract.analyzeDocument(config, function (err, data) {
console.log("analyzing...");
if (err) {
console.log(err, err.stack);
}
// an error occurred
else {
console.log("data:" + JSON.stringfy(data));
} // successful response
});
};
I tried every possible solution on the internet with no hope
What I am trying to do is simply use aws lambda functions (through the aws console) to fetch user fcm token from lets say DynamoDB (not included in the question), use that token to create endpointArn, send push to that specific device
I tested to send Using SNS console and the push gets to the device successfully but I failed to get it to the device using Lambda functions although it gives success status and message ID
Here is the code I used
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'us-east-1'});
const sns = new AWS.SNS()
const sampleMessage = {
"GCM": {
"notification": {
"body": "Sample message for Android endpoints",
"title":"Title Test"
}
}
}
exports.handler = async (event) => {
const snsPayload = JSON.stringify(sampleMessage);
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
const params = {
PlatformApplicationArn: '<Platform Arn>',
Token: '<FCM Token>'
};
try {
const endpointData = await sns.createPlatformEndpoint(params).promise();
const paramsMessage = {
Message: snsPayload,
TargetArn: endpointData.EndpointArn
};
var publishTextPromise = await sns.publish(paramsMessage).promise();
response.MessageId = publishTextPromise.MessageId;
response.result = 'Success';
}
catch (e) {
console.log(e.stack)
response.result = 'Error'
}
return response;
};
After some trials and errors I figured out the solution for my own question
1- The GCM part of the payload should be a string not a json
2- The message parameter should have an attribute that explicitly sets the mime type of the payload to Json
Taking all that into consideration
const GCM_data = {
'notification': {
'body': 'Hellow from lambda function',
'title': 'Notification Title'
}
}
const data = {
"GCM": JSON.stringify(GCM_data)
}
const snsPayload = JSON.stringify(data)
and the params should look like
const paramsMessage = {
Message: snsPayload,
TargetArn: endpointData.EndpointArn,
MessageStructure: 'json'
};
and this will work :)
I am using SES service of AWS for send email.
I use Python and boto3.
(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ses.html#SES.Client.send_email)
I know how to check the overall statistics.
(https://aws.amazon.com/ko/premiumsupport/knowledge-center/ses-email-opens-clicks/)
But I don't know how to do individual inquiry.
After sending an email, message_id and request_id are returned.
I think we're going to use message_id, request_id to search.
I don't know what to do.
Configure SES to send notifications (send, deliver, bounce, open) to SNS topics.
Write a Lambda function to listen to the SNS topics. When a notification is received, write the event to your table (I use DynamoDB)
Now you have all your email events in a table
Here is the Lambda function I use for writing SES events to DynamoDB
const tableName = "DEV_EmailAuditLogs";
const AWS = require("aws-sdk");
AWS.config.update({ region: "us-east-1" });
const dynamodb = new AWS.DynamoDB({ apiVersion: "2012-10-08" });
exports.handler = (event, context, callback) => {
console.log(JSON.stringify(event, null, 2));
console.log("From SNS:", event.Records[0].Sns.Message);
console.log("Arn Topic:", event.Records[0].Sns.TopicArn);
const json = event.Records[0].Sns.Message;
const parsed = JSON.parse(json);
const type = parsed.eventType;
const sent = parsed.mail.timestamp.slice(0, -1);
let to, subject, from;
const headers = parsed.mail.headers;
for (const h of headers) {
switch (h.name) {
case "To":
to = h.value;
break;
case "From":
from = h.value;
break;
case "Subject":
subject = h.value;
break;
}
}
const eventData = parsed[type.toLowerCase()];
let created;
if (eventData && eventData.timestamp) {
created = eventData.timestamp.slice(0, -1);
} else {
created = sent;
}
var params = {
TableName: tableName,
Item: {
to: { S: to },
from: { S: from },
created: { S: created },
sent: { S: sent },
type: { S: type },
version: { N: "1" }
}
};
if (subject){
params.Item.subject = { S: subject };
}
dynamodb.putItem(params, function(err, data) {
if (err) {
console.log(err);
} else {
console.log(data);
}
});
};