I have data in CloudWatch logs, where were sent ip addresses.
For example,
{ "requestId":"test", "ip": "0.0.0.0", "requestTime":"24/May/2022", "httpMethod":"POST", ...}
Also, I have Data Stream in Kinesis, where were sent data with status 200
{
"Records": [
{
"SequenceNumber": "test",
"Data": {
test
},
"PartitionKey": "test"
}
],
"NextShardIterator": "test"
"MillisBehindLatest": 0,
"ResponseMetadata": {
"RequestId": "test",
"HTTPStatusCode": 200,
"HTTPHeaders": {
...
},
"RetryAttempts": 1
}
}
Is it possible to add IP for each record?
Related
Is long polling available on GCP PubSub JS SDK?
I want to be able to process multiple PubSub messages at once for example:
There are 1000 messages being sent to the topic every in a span of 1 minute.
Given that, for the next 10 secs, there will be 50 messages to be sent to a topic. I want my subscription to a long poll for 10 secs so that instead of processing each message. It will wait for 10 secs and potentially got all the 50messages.
AWS JS SDK has this feature, I was hoping I can do it on GCP as well.
https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/sqs-examples-enable-long-polling.html
This is an example of how it will work on AWS:
The SQS queue have more than 5 messages.
The listener receiveMessage will get 5 messages at once in a single receive. event
// Load the AWS SDK for Node.js
var AWS = require("aws-sdk");
// Set the region
AWS.config.update({ region: "REGION" });
// Set the AWS Region
const REGION = "us-east-1"; //e.g. "us-east-1"
// Set the parameters
const queueURL =
"https://sqs.us-east-1.amazonaws.com/763335115465/long-polling-per-message"; // SQS_QUEUE_URL
const params = {
AttributeNames: ["SentTimestamp"],
MaxNumberOfMessages: 5,
MessageAttributeNames: ["All"],
QueueUrl: queueURL,
WaitTimeSeconds: 20,
};
// Create SQS service object
const sqs = new AWS.SQS({
region: REGION,
credentials: {
accessKeyId: "xx",
secretAccessKey: "xxx",
},
});
sqs.receiveMessage(params, function (err, data) {
console.log({ err, data: JSON.stringify(data) });
if (err) {
console.log("Receive Error", err);
} else if (data.Messages) {
var deleteParams = {
QueueUrl: queueURL,
ReceiptHandle: data.Messages[0].ReceiptHandle,
};
sqs.deleteMessage(deleteParams, function (err, data) {
if (err) {
console.log("Delete Error", err);
} else {
console.log("Message Deleted", data);
}
});
}
});
{
"ResponseMetadata": { "RequestId": "25295507-c4ae-5106-a499-0d7808c163b8" },
"Messages": [
{
"MessageId": "5dbd863e-2c50-49c8-9c4b-9f70e8db8d17",
"ReceiptHandle": "asdf",
"MD5OfBody": "78ef53e38c997c445f2fe1cc63c13139",
"Body": "Test5",
"Attributes": { "SentTimestamp": "1610991641728" }
},
{
"MessageId": "09baf624-f2ee-4173-83ed-e74c0516a7e6",
"ReceiptHandle": "asdf",
"MD5OfBody": "c454552d52d55d3ef56408742887362b",
"Body": "Test2",
"Attributes": { "SentTimestamp": "1610991983369" }
},
{
"MessageId": "1cac914f-d946-434a-87a0-974b14cc2eba",
"ReceiptHandle": "asdf",
"MD5OfBody": "b3f66ec1535de7702c38e94408fa4a17",
"Body": "Test3",
"Attributes": { "SentTimestamp": "1610991986299" }
},
{
"MessageId": "95c2c8ad-fc7a-451a-b967-8ce1736a4cab",
"ReceiptHandle": "asdf",
"MD5OfBody": "f178860b5109214d9f3debe19a7800d3",
"Body": "Test7",
"Attributes": { "SentTimestamp": "1610991998129" }
},
{
"MessageId": "3711fa29-9bbc-418d-a35f-7adbd7daa952",
"ReceiptHandle": "asd",
"MD5OfBody": "b6e30158b9d7d2dc8bb4f4123fe93c9b",
"Body": "Test10",
"Attributes": { "SentTimestamp": "1610992008975" }
}
]
}
The Cloud Pub/Sub JS library provides a streaming, per-message API for receiving messages from subscriptions. There isn't a way to tell the library to give you a batch of N messages, so you'll have to implement it yourself.
Yes, you have it and it's documented here. Set the value of the timeout that you want to cancel the subscription. So that, if you want to wait 10 more seconds, add 10.000 millis!
I am trying to get Cloudfront metrics using the JS SDK for AWS but I am not getting anything back.
I am not sure what I am doing wrong but I have isolated this NOT to be with:
Permissions (gave it a full admin account for testing purposes)
Region. North Virginia (for CloudFront)
Basic params like: StartDate, EndDate, DistributionID
My code is as below (simplified):
var AWS = require('aws-sdk');
AWS.config.update({
accessKeyId: "accessKeyId",
secretAccessKey: "secretAccessKey",
apiVersion: '2017-10-25',
region: 'us-east-1'
});
var cloudwatchmetrics = new AWS.CloudWatch();
var cloudFrontParams = {
"StartTime": 1518867432,
"EndTime": 1518868032,
"MetricDataQueries": [
{
"Id": "m1",
"MetricStat": {
"Metric": {
"Dimensions": [
{
"Name": "DistributionId",
"Value": "ValueOfDistribution"
},
{
"Name": "Region",
"Value": "Global"
}
],
"MetricName": "Requests",
"Namespace": "AWS/CloudFront"
},
"Stat": "Sum",
"Period": 3600
},
"ReturnData": true
}
]
};
cloudwatchmetrics.getMetricData(cloudFrontParams, function (err, data) {
if (err) {
console.log(err);
}else{
console.log(JSON.stringify(data));
}
});
This is what I get back (it's not erroring out):
{
"ResponseMetadata":{
"RequestId":"xxxxxxxxxxxxxxxxxxxxx"
},
"MetricDataResults":[
{
"Id":"m1",
"Label":"Requests",
"Timestamps":[
],
"Values":[
],
"StatusCode":"Complete",
"Messages":[
]
}
]
}
The issue was with the StartTime, it was too far back in time. What I have in the post translates to: Saturday, February 17, 2018
Hopefully this helps someone someday.
I'm generating a presigned URL using s3.getSignedUrl('putObject', params) and for my params
var params = {
Bucket: bucketName,
Key: photoId + "-" + photoNumber + "-of-" + numberImages + ".jpeg",
Expires: signedUrlExpireSeconds,
ContentType: contentType,
Metadata : { testkey1 : "hello" }
};
I'm trying to receive the Metadata in my S3 successful upload lambda function, however it's not appearing. Anyone know why? The upload is successful and for my printed logs, I'm receiving everything but the metadata tag in the event:
console.log(event);
"Records": [
{
"eventVersion": "2.1",
"eventSource": "aws:s3",
"awsRegion": "us-east-1",
"eventTime": "2020-01-15T06:51:57.171Z",
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId":
},
"requestParameters": {
"sourceIPAddress":
},
"responseElements": {
"x-amz-request-id": "4C32689CE5B70A82",
"x-amz-id-2": "AS0f97RHlLW2DF6tVfRwbTeoEpk2bEne/0LrWqHpLJRHY5GMBjy/NQOHqYAMhd2JjiiUcuw0nUTMJS8pDAch1Abc5xzzWVMv"
},
"s3": {
"s3SchemaVersion": "1.0",
"configurationId": "9a9a755e-e809-4dbf-abf8-3450aaa208ed",
"bucket": {
"name": ,
"ownerIdentity": {
"principalId": "A3SZPXLS03IWBG"
},
"arn":
},
"object": {
"key": "BcGMYe-1-of-1.jpeg",
"size": 19371,
"eTag": "45c719f2f6b5349cc360db9a13d0cee4",
"sequencer": "005E1EB6921E08F7E4"
}
}
}
]
This is s3 event message structure. the message structure originally doesn't contain metadata.
You need to get metadata in the lambda function in person.
You would get metadata through s3 head-object command with the bucket-name and object-key in the event received.
{
"Records":[
{
"eventVersion":"2.2",
"eventSource":"aws:s3",
"awsRegion":"us-west-2",
"eventTime":The time, in ISO-8601 format, for example, 1970-01-01T00:00:00.000Z, when Amazon S3 finished processing the request,
"eventName":"event-type",
"userIdentity":{
"principalId":"Amazon-customer-ID-of-the-user-who-caused-the-event"
},
"requestParameters":{
"sourceIPAddress":"ip-address-where-request-came-from"
},
"responseElements":{
"x-amz-request-id":"Amazon S3 generated request ID",
"x-amz-id-2":"Amazon S3 host that processed the request"
},
"s3":{
"s3SchemaVersion":"1.0",
"configurationId":"ID found in the bucket notification configuration",
"bucket":{
"name":"bucket-name",
"ownerIdentity":{
"principalId":"Amazon-customer-ID-of-the-bucket-owner"
},
"arn":"bucket-ARN"
},
"object":{
"key":"object-key",
"size":object-size,
"eTag":"object eTag",
"versionId":"object version if bucket is versioning-enabled, otherwise null",
"sequencer": "a string representation of a hexadecimal value used to determine event sequence,
only used with PUTs and DELETEs"
}
},
"glacierEventData": {
"restoreEventData": {
"lifecycleRestorationExpiryTime": "The time, in ISO-8601 format, for example, 1970-01-01T00:00:00.000Z, of Restore Expiry",
"lifecycleRestoreStorageClass": "Source storage class for restore"
}
}
}
]
}
I have written an Alexa smart home skills.
When I try to discover the device using the Alexa test or from the mobile app, the lambda is triggered.
The lambda is getting successfully executed, but I get below error in App or test in Alexa console.
I couldn't find any new Smart Home devices. If you’ve n't already,
please enable the smart home skill for your device from the Alexa App.
What could be the possible issue?
Since the lambda is getting successfully executed, I don't think there is any issue with language (English(IN)) or AWS region (EU-WEST-1) , where the lambda is deployed.
I didn't see any logs on Alexa developer console
Any pointers?
Response from Lambda function -
header =
{
namespace: 'Alexa.Discovery',
name: 'Discover.Response',
payloadVersion: '3',
messageId: '785f0173-6ddb-41d8-a785-de7159c7f7ca'
}
payload =
{
"endpoints": [
{
"endpointId": "d4b87cbe6c8e490493733f260b8c2c25",
"friendlyName": "Kitchen",
"description": "Demo",
"manufacturerName": "Man1",
"displayCategories": [
"LIGHT"
],
"cookie": {
"owner": "Owner1"
},
"capabilities": [
{
"type": "AlexaInterface",
"version": "3",
"interface": "Alexa"
},
{
"type": "AlexaInterface",
"version": "3",
"interface": "Alexa.PowerController",
"properties": {
"supported": [
{
"name": "powerState"
}
],
"proactivelyReported": true,
"retrievable": true
}
},
{
"type": "AlexaInterface",
"version": "3",
"interface": "Alexa.BrightnessController",
"properties": {
"supported": [
{
"name": "brightness"
}
],
"proactivelyReported": true,
"retrievable": true
}
}
]
}
]
}
We are wrapping header and payload in the response event.
context.succeed({ event: { header: header, payload: payload } });
So far I haven't found a way to view the logs either.
I had the same problem and I realized that I was putting wrong values in some properties or schema entities like ids.
In the same way, another thing that solved me on some occasion was to place the scheme in the following way:
context.succeed({
"event": {
"header": {
"namespace": "Alexa.Discovery",
"name": "Discover.Response",
"payloadVersion": "3",
"messageId": header.messageId
},
"payload": {
"endpoints": [
{
"endpointId": "demo_id",
...
,
"cookie": {},
"capabilities": [
{
"type": "AlexaInterface",
"interface": "Alexa",
"version": "3"
},
...
]
}
]
}
}
});
I have a OnDemand triggered webjob that I want to trigger through ADF copy activity using HTTP linked service. Here is the linked service:-
{
"name": "LS_WebJob",
"properties": {
"hubName": "yas-cdp-adf_hub",
"type": "Http",
"typeProperties": {
"url": "https://cust-app.scm.azurewebsites.net/api/triggeredwebjobs/ConsoleApplication1/run",
"authenticationType": "Basic",
"username": "$custdata-app",
"password": "**********"
}
}
}
Input Dataset
{
"name": "ZZ_Inp_Webjob",
"properties": {
"published": false,
"type": "Http",
"linkedServiceName": "LS_WebJob",
"typeProperties": {
"requestMethod": "Post",
"requestBody": "Hey Buddy"
},
"availability": {
"frequency": "Day",
"interval": 1,
"style": "StartOfInterval"
},
"external": true,
"policy": {}
}
}
Output Dataset
{
"name": "ZZ_Out_WebJob",
"properties": {
"published": false,
"type": "AzureBlob",
"linkedServiceName": "LS_ABLB",
"typeProperties": {
"fileName": "webjob.json",
"folderPath": "yc-cdp-container/Dummy/temp",
"format": {
"type": "TextFormat"
}
},
"availability": {
"frequency": "Day",
"interval": 1,
"style": "StartOfInterval"
}
}
}
Pipeline
{
"name": "ZZ-PL-WebJob",
"properties": {
"description": "This pipeline copies data from an HTTP Marina WiFi Source URL to Azure blob",
"activities": [
{
"type": "Copy",
"typeProperties": {
"source": {
"type": "HttpSource"
},
"sink": {
"type": "BlobSink",
"writeBatchSize": 0,
"writeBatchTimeout": "00:00:00"
}
},
"inputs": [
{
"name": "ZZ_Inp_Webjob"
}
],
"outputs": [
{
"name": "ZZ_Out_Webjob"
}
],
"policy": {
"timeout": "01:00:00",
"concurrency": 1
},
"scheduler": {
"frequency": "Day",
"interval": 1,
"style": "StartOfInterval"
},
"name": "WebjobSourceToAzureBlob",
"description": "Copy from an HTTP source to an Azure blob"
}
],
"start": "2017-04-10T01:00:00Z",
"end": "2017-04-10T01:00:00Z",
"isPaused": false,
"hubName": "yas-cdp-adf_hub",
"pipelineMode": "Scheduled"
}
}
My webjob is a simple C# application:-
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
Console.WriteLine("My Team Rocks!");
}
}
}
When I am executing the pipeline, the webjob is being successfully triggered. However the pipeline fails with HTTP 409 conflict error.
Copy activity encountered a user error at Source side:
ErrorCode=UserErrorFailedToReadHttpFile,'Type=Microsoft.DataTransfer.Common.Shared.HybridDeliveryException,Message=Failed
to read data from http source
file.,Source=Microsoft.DataTransfer.ClientLibrary,''Type=System.Net.WebException,Message=The
remote server returned an error: (409) Conflict.,Source=System,'.
Try adding in the gateway name to the linked service json. Refer to this link How to integrate a WebJob within an Azure Data Factory Pipeline .