Lambda retrieve SSM parameter in Localstack - amazon-web-services

Using the latest localstack-full Docker image, I have stored an encrypted SSM parameter like this:
aws --region us-east-1 --endpoint-url=http://localhost:4566 ssm put-parameter --name "dbpassword" --value "secret2" --type "SecureString"
Then I have implemented a lambda in Go that I can access via API gateway. The implementation looks like this:
package main
import (
"context"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ssm"
"log"
"os"
)
type Event struct {
Name string `json:"name"`
}
func HandleRequest(ctx context.Context, event Event) (events.APIGatewayProxyResponse, error) {
theAwsRegion := os.Getenv("AWS_REGION")
//customResolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
// return aws.Endpoint{
// PartitionID: "aws",
// URL: "localhost:4566",
// SigningRegion: "us-east-1",
// }, nil
//})
theConfig, err := config.LoadDefaultConfig(
ctx,
config.WithRegion(theAwsRegion),
//config.WithEndpointResolver(customResolver),
)
if err != nil {
log.Fatalf("Unable to load SDK config, %v", err)
}
ssmClient := ssm.NewFromConfig(theConfig)
if ssmClient != nil {
ssmOutput, err := ssmClient.GetParameter(ctx, &ssm.GetParameterInput{Name: aws.String("dbpassword"), WithDecryption: true})
if err != nil {
return events.APIGatewayProxyResponse{
StatusCode: 200,
Headers: map[string]string{"Context-Type": "text/plain"},
Body: "Error occurred getting password",
IsBase64Encoded: false,
}, nil
}
thePassword := ssmOutput.Parameter.Value
return events.APIGatewayProxyResponse{
StatusCode: 200,
Headers: map[string]string{"Context-Type": "text/plain"},
Body: "Got a SSM client " + *thePassword,
IsBase64Encoded: false,
}, nil
}
return events.APIGatewayProxyResponse{
StatusCode: 200,
Headers: map[string]string{"Context-Type": "text/plain"},
Body: "Failed to obtain SSM client",
IsBase64Encoded: false,
}, nil
}
func main() {
lambda.Start(HandleRequest)
}
However, the lambda only responds with "{}" as soon as the call to GetParameter is introduced.
An SSM client is successfully retrieved.
There is also code commented-out attempt to use a custom endpoint resolver, which produces the same result.
Is there a problem with Localstack or what am I doing wrong?
Retrieving the parameter using the AWS CLI works without problems.
Update: I have tried to access the Localstack container from the lambda container over port 4566 and were able to obtain a response from the Localstack container.

Related

Keep getting build fail and unable to save import for main.go file for lambda function using SAM CLI

I'm trying to import "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" for my main.go file for my lambda function but every time I save the import disappears.
I have some simple Golang code trying to update a visitor count by updating a dynamodb table.
The build keeps failing saying that attributevalue are undefined but I can't save the import for attribute value.
package main
import (
"context"
"log"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
)
func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-east-1"))
if err != nil {
log.Fatalf("unable to load SDK config, %v", err)
}
svc := dynamodb.NewFromConfig(cfg)
// Build the request with its input parameters
resp, err := svc.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
TableName: aws.String("table-name"),
Key: map[string]*dynamodb.attributevalue{
"ID": {
S: aws.String("visitors"),
},
},
UpdateExpression: aws.String("ADD visitors :inc"),
ExpressionAttributeValues: map[string]*dynamodb.attributevalue{
":inc": {
N: aws.String("1"),
},
},
})
if err != nil {
log.Fatalf("Got error callingUpdateItem: %s", err)
}
return events.APIGatewayProxyResponse{
Headers: map[string]string{
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "*",
"Access-Control-Allow-Headers": "*",
},
StatusCode: 200,
}, nil
}
func main() {
lambda.Start(handler)
}
When I do go mod vendor I get
github.com/aws/aws-sdk-go-v2/featrues/dynamodb: no required module provides package github.com/aws/aws-sdk-go-v2/featrues/dynamodb; to add it:
Doing go get github.com/aws/aws-sdk-go-v2/featrues/dynamodb I get
go: module github.com/aws/aws-sdk-go-v2#upgrade found (v1.16.11), but does not contain package github.com/aws/aws-sdk-go-v2/featrues/dynamodb
I am very new to Go and have no idea what to do to resolve this. Any help would be greatly appreciated.
if you check github.com/aws/aws-sdk-go-v2/feature. available packages are dynamodb/attributevalue or dynamodb/expression. Looks like you are using attribute value so the import should have "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"

AWS s3 NoSuchKey error after retrieving file which was just uploaded

I uploaded a file to my s3 bucket and tried to read the file immediately after upload. I most of the time get "err NoSuchKey: The specified key does not exist". i check the bucket using the console and the file actually exist.
After refreshing the page, the file is able to be read.
Aws region is US East (N Virginia).
File is uploaded with a private read.
export function uploadFile(absolutePath: string, fileBuffer: Buffer, callback: (err, result) => void) {
try {
let uploadParams: awsSdk.S3.PutObjectRequest = {
Bucket: cfg.aws[process.env.NODE_ENV].bucket,
Key: absolutePath,
Body: fileBuffer,
ACL: 'private',
CacheControl: 'public, max-age=2628000'
}
s3.upload(uploadParams, function (err, result) {
if (err) {
Util.logError('Aws Upload File', err)
}
return callback(err, result)
})
} catch (err) {
Util.logError('Aws Upload File', err)
return callback(err, null)
}
}
export function obtainObjectOutput(absolutePath: string, callback: (err, result: awsSdk.S3.GetObjectOutput) => void) {
let getParaams: awsSdk.S3.GetObjectRequest = {
Bucket: cfg.aws[process.env.NODE_ENV].bucket,
Key: absolutePath
}
s3.getObject(getParaams, (error, result) => {
(error) ? callback(error, null) : callback(null, result)
})
}
The number one reason that S3 GetObject fails after an upload is that the GetObject request actually happened before the upload completed, This is easy to do in async JavaScript.

Golang AWS API Gateway invalid character 'e' looking for beginning of value

I am trying to create an API Gateway connected to a lambda which parses an HTML template with handlebars and then returns it but I am getting this error when I run it locally and even on a test url using AWS.
{
"errorMessage": "invalid character 'e' looking for beginning of value",
"errorType": "SyntaxError"
}
This is my SAM Template
AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: data-template-renderer
Parameters:
Stage:
Type: String
AllowedValues:
- dev
- staging
- production
Description: environment values
Resources:
# Defining the API Resource here means we can define the stage name rather than
# always being forced to have staging/prod. Also means we can add a custom domain with
# to the API gateway within this template if needed. Unfortunately there is a side effect
# where it creates a stage named "Stage". This is a known bug and the issue can be
# found at https://github.com/awslabs/serverless-application-model/issues/191
DataTemplateRendererApi:
Type: AWS::Serverless::Api
Properties:
Name: !Sub "${Stage}-data-template-renderer"
StageName: !Ref Stage
DefinitionBody:
swagger: "2.0"
basePath: !Sub "/${Stage}"
info:
title: !Sub "${Stage}-data-template-renderer"
version: "1.0"
consumes:
- application/json
produces:
- application/json
- text/plain
- application/pdf
paths:
/render:
post:
x-amazon-apigateway-integration:
uri:
"Fn::Sub": "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${RenderTemplate.Arn}/invocations"
httpMethod: POST
type: aws_proxy
x-amazon-apigateway-binary-media-types:
- "*/*"
RenderTemplate:
Type: AWS::Serverless::Function
Properties:
Environment:
Variables:
ENV: !Ref Stage
Runtime: go1.x
CodeUri: build
Handler: render_template
FunctionName: !Sub 'render_template-${Stage}'
Timeout: 30
Tracing: Active
Events:
RenderTemplateEndpoint:
Type: Api
Properties:
RestApiId: !Ref DataTemplateRendererApi
Path: /render
Method: POST
Policies:
- !Ref S3AccessPolicy
- CloudWatchPutMetricPolicy: {}
S3AccessPolicy:
Type: AWS::IAM::ManagedPolicy
Properties:
ManagedPolicyName: data-template-renderer-s3-policy
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action: s3:GetObject
Resource: !Sub "arn:aws:s3:::*"
- Effect: Allow
Action: s3:PutObject
Resource: !Sub "arn:aws:s3:::*"
And below is the code that I am using for the Lambda. Please forgive the fact that it may not be the best Golang Code you have seen but I am a beginner when it comes to Golang, as I am primarily a PHP dev but the company I work for is creating a lot of Golang lambdas so I started learning it.
package main
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/aymerick/raymond"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/SebastiaanKlippert/go-wkhtmltopdf"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
var sess *session.Session
func init() {
// Setup AWS S3 Session (build once use every function)
sess = session.Must(session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
}))
}
func main() {
lambda.Start(handleRequest)
}
type TemplateRendererRequest struct {
Template string `json:"template"`
Bucket string `json:"bucket"`
Type string `json:"type"`
Data map[string]interface{} `json:"data"`
}
type EmailResponse struct {
Email string `json:"email"`
}
func handleRequest(ctx context.Context, request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
// Unmarshal json request body into a TemplateRendererRequest struct that mimics the json payload
requestData := TemplateRendererRequest{}
err := json.Unmarshal([]byte(request.Body), &requestData)
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
// Get the template object from S3
result, err := s3.New(sess).GetObject(&s3.GetObjectInput{
Bucket: aws.String(requestData.Bucket),
Key: aws.String(requestData.Template),
})
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
defer result.Body.Close()
// The S3 Object Body is of type io.ReadCloser
// below we create a bytes buffer and then convert it to a string so we can use it later
buf := new(bytes.Buffer)
buf.ReadFrom(result.Body)
templateString := buf.String()
// Parse template through Handlebars library
parsedTemplate, err := parseTemplate(templateString, requestData)
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
switch requestData.Type {
case "pdf":
return handlePDFRequest(parsedTemplate, requestData)
default:
return handleEmailRequest(parsedTemplate)
}
}
func handleEmailRequest(parsedTemplate string) (events.APIGatewayProxyResponse, error) {
// Put email parsed template in a struct to return in the form of JSON
emailResponse := EmailResponse{
Email: parsedTemplate,
}
// JSON encode the emailResponse
response, err := JSONMarshal(emailResponse)
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
return events.APIGatewayProxyResponse{
Body: string(response),
StatusCode: 200,
Headers: map[string]string{
"Content-Type": "application/json",
},
}, nil
}
func parseTemplate(templateString string, request TemplateRendererRequest) (string, error) {
result, err := raymond.Render(templateString, request.Data)
return result, err
}
func handlePDFRequest(parsedTemplate string, requestData TemplateRendererRequest) (events.APIGatewayProxyResponse, error) {
pdfBytes, err := GeneratePDF(parsedTemplate)
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
keyNameParts := strings.Split(requestData.Template, "/")
keyName := keyNameParts[len(keyNameParts)-1]
pdfName := fmt.Sprintf("%s_%s.pdf", keyName, time.Now().UTC().Format("20060102150405"))
_, err = s3.New(sess).PutObject(&s3.PutObjectInput{
Bucket: aws.String(requestData.Bucket),
Key: aws.String(pdfName),
Body: bytes.NewReader(pdfBytes),
})
b64Pdf := base64.StdEncoding.EncodeToString(pdfBytes)
return events.APIGatewayProxyResponse{
Body: b64Pdf,
StatusCode: 200,
Headers: map[string]string{
"Content-Type": "application/pdf",
},
IsBase64Encoded: true,
}, nil
}
func GeneratePDF(templateString string) ([]byte, error) {
pdfg, err := wkhtmltopdf.NewPDFGenerator()
if err != nil {
return nil, err
}
// Pass S3 Object body (as reader io.Reader) directly into wkhtmltopdf
pdfg.AddPage(wkhtmltopdf.NewPageReader(strings.NewReader(templateString)))
// Create PDF document in internal buffer
if err := pdfg.Create(); err != nil {
return nil, err
}
// Return PDF as bytes array
return pdfg.Bytes(), nil
}
// https://stackoverflow.com/questions/28595664/how-to-stop-json-marshal-from-escaping-and
func JSONMarshal(t interface{}) ([]byte, error) {
buffer := &bytes.Buffer{}
encoder := json.NewEncoder(buffer)
encoder.SetEscapeHTML(false)
err := encoder.Encode(t)
return buffer.Bytes(), err
}
I have tried to search high and low for a solution but couldn't find why am I getting this really weird error which is not very helpful.
Thanks for taking the time and helping
Thanks to #Anzel who pointed me in the right direction I decided to look at the request.Body and it seems that by adding the */* to the API Gateway Binary media types, now even the request comes in encoded and lo and behold the first letter was indeed an e as the message was saying.
So I changed this:
// Unmarshal json request body into a TemplateRendererRequest struct that mimics the json payload
requestData := TemplateRendererRequest{}
err := json.Unmarshal([]byte(request.Body), &requestData)
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
To this:
// Unmarshal json request body into a TemplateRendererRequest struct that mimics the json payload
requestData := TemplateRendererRequest{}
b64String, _ := base64.StdEncoding.DecodeString(request.Body)
rawIn := json.RawMessage(b64String)
bodyBytes, err := rawIn.MarshalJSON()
if err != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", err.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, err
}
jsonMarshalErr := json.Unmarshal(bodyBytes, &requestData)
if jsonMarshalErr != nil {
return events.APIGatewayProxyResponse{
Body: fmt.Errorf("Error: %s ", jsonMarshalErr.Error()).Error(),
StatusCode: 400,
Headers: map[string]string{
"Content-Type": "text/plain",
},
}, jsonMarshalErr
}
From what I saw online you could also change this:
rawIn := json.RawMessage(b64String)
bodyBytes, err := rawIn.MarshalJSON()
To this:
[]byte(b64String)
When I now do a CURL request and output it to a file I get the PDF correctly.

call an external api using apigateway

I'm writing a lambda in node.js that will call an api(post) and gives back the resulting body and the code is as below.
const AWS = require('aws-sdk');
const request = require('request');
exports.handle = function(e, ctx, callback) {
var bodyDetails = {
uri: "myURL",
json: {
"requestChannel": "web1" },
"method": "POST"
};
callback = ctx.done;
var data = e.bodyJson || {};
request(bodyDetails, function(error, response, body) {
if (!error && response.statusCode === 200) {
console.log(JSON.parse(JSON.stringify(body)));
jsonBody = JSON.parse(JSON.stringify(body));
console.log(body + "\t from suvccess") // Print the json response
callback(null, jsonBody); // Return the JSON object back to our API call
} else {
callback(error);
}
});
}
and I'm testing the same in my lambda console. by passing a blank json {} and I get back the correct response.
Now my next plan is to integrate this piece against API Gateway. So I've created an api for this in my apigateway and in that, I've created a resource named home. and in the home, I created a GET method. with the below details.
Integration type: Lambda Function
Use Lambda Proxy integration : checked
Lambda Region: us-east-1
Lambda Function: myWorkingLambdaName
when I tested this using the test option given by apigateway. I get the response as
Request: /home
Status: 502
Latency: 2942 ms
Response Body
{
"message": "Internal server error"
}
when I see my console I see the values of the success block printed, but the status code is 502. This is very confusing, please let me know where am I going wrong and how can I fix this.
Thanks
API Gateway expects the following properties to be returned from your Lambda:
{
"isBase64Encoded": true|false,
"statusCode": httpStatusCode,
"headers": { "headerName": "headerValue", ... },
"body": "..."
}
So, instead of callback(null, jsonBody), you should be calling callback like this:
callback(null, {
isBase64Encoded: false,
statusCode: 200,
headers: {
"Access-Control-Allow-Origin" : "*",
},
body: JSON.stringify(jsonBody),
})

How to get data from aws Dynamodb with using partition key only?

I am using aws-sdk-go library for DynamoDb connectivity in Golang.
My DynamoDb table have a Partition key DeviceId (String) and a Sort Key Time (Number). How can I write GetItemInput to get all data with a specific DeviceId?
params := &dynamodb.GetItemInput{
Key: map[string]*dynamodb.AttributeValue {
"DeviceId": {
S: aws.String("item_1"),
},
},
ExpressionAttributeNames: map[string]*string{
"DeviceId": "DeviceId",
},
TableName: aws.String("DbName"),
}
list, err := svc.GetItem(params)
You have to use Query or Scan operation, this is a simple example but you can read more on Amazon documentation here
In particular, Query operation
A Query operation finds items in a table or a secondary index using only primary key attribute values
var queryInput = &dynamodb.QueryInput{
TableName: aws.String(dynamoRestDataTableName),
KeyConditions: map[string]*dynamodb.Condition{
"DeviceId": {
ComparisonOperator: aws.String("EQ"),
AttributeValueList: []*dynamodb.AttributeValue{
{
S: aws.String("aDeviceId"),
},
},
},
},
}
var resp, err = dynamoSvc.Query(queryInput)
if err != nil {
return nil, err
}
Query operation can be used in that case
Following is one generic example for the same
compositeKey := entity.GetPrimaryKey(inputVar)
expressionAttributeValues := map[string]*dynamodb.AttributeValue{
":v1": {
S: aws.String(compositeKey.PartitionKey.Data.(string)),
},
}
queryInput := dynamodb.QueryInput{
TableName: &d.TableName,
KeyConditionExpression: aws.String("id = :v1"),
ExpressionAttributeValues: expressionAttributeValues,
}
queryOutput, err := d.DdbSession.Query(&queryInput)
if err != nil {
log.Error("error in fetching records ", err)
return nil, err
}
// unmarshal the query output - items to interface
err = dynamodbattribute.UnmarshalListOfMaps(queryOutput.Items, &yourInterface)
You can use getItem in JavaScript SDK v2 like this
const params = {
TableName: 'tableName',
Key: {
id: { S: id },
},
};
const result = await dynamoDb.getItem(params).promise();
if (result.Item === undefined) {
throw new Error('not found');
}
const item = AWS.DynamoDB.Converter.unmarshall(result.Item)
If JavaScript SDK can do this, I assume golang SDK can also. If not, that means AWS doesn't take all languages equally ?