I want to test AWS Lambda in local environment using DynamoDB Local and SAM CLI.
I create a simple user db table(id, name) and I'm trying to get the data.
I run "sam local start-api --env-vars test/env.json". When I access "http://localhost:3000/users/1", an error occured. Error message is below. I can't understand what this error message means. How do I fix this error?
{
"errorMessage": "InvalidParameter: 1 validation error(s) found.\n- minimum field size of 3, GetItemInput.TableName.\n",
"errorType": "ErrInvalidParams"
}
This is my code.
func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
// Environment variables
endpoint := os.Getenv("DYNAMODB_ENDPOINT")
tableName := os.Getenv("DYNAMODB_TABLE_NAME")
// Request
id, _ := request.PathParameters["id"]
// DynamoDB
sess := session.Must(session.NewSession())
config := aws.NewConfig().WithRegion("ap-northeast-1")
if len(endpoint) > 0 {
config = config.WithEndpoint(endpoint)
}
db := dynamodb.New(sess, config)
response, err := db.GetItem(&dynamodb.GetItemInput{
TableName: aws.String(tableName),
Key: map[string]*dynamodb.AttributeValue{
"Id": {
N: aws.String(string(id)),
},
},
AttributesToGet: []*string{
aws.String("Id"),
aws.String("Name"),
},
ConsistentRead: aws.Bool(true),
ReturnConsumedCapacity: aws.String("NONE"),
})
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
user := User{}
err = dynamodbattribute.Unmarshal(&dynamodb.AttributeValue{M: response.Item}, &user)
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
// Json
bytes, err := json.Marshal(user)
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
return events.APIGatewayProxyResponse{
Body: string(bytes),
StatusCode: 200,
}, nil
}
error message
That's been resolved.
First Problem: can't read from the environment variable
This was because the indent size deviation of template.yaml. Category "Events" and Category "Environment" had to be in line.
Second Problem: error with "Function 'UserGetFunction' timed out after 5 seconds"
This was because "localhost" expression. Rewriting "localhost" in env.json to "xxx.xxx.xxx.xxx" worked fine.("xxx.xxx.xxx.xxx" is ip address of your laptop)
Related
I am trying to connect my local MinIO instance running in docker. I have written a simple Go program to do the work. See following:
package main
import (
"bytes"
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
)
func main() {
endpoint := "localhost:9000"
accessKeyID := "1mVyDeEcrRU7qQ7h"
secretAccessKey := "vNi9SNfcHNxfRDETSvkeFLO210mzu7ee"
useSSL := false
awsSession := session.Must(session.NewSession(&aws.Config{
Region: aws.String("ap-south-1"),
Credentials: credentials.NewStaticCredentialsFromCreds(credentials.Value{
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
SessionToken: "",
}),
Endpoint: aws.String(endpoint),
DisableSSL: aws.Bool(!useSSL),
}))
ctx := context.Background()
file, err := downloadFileFromS3(ctx, awsSession, "mybucket", "data.txt")
if err != nil {
log.Fatal(err)
}
reader := bytes.NewReader(file)
println(reader)
}
func downloadFileFromS3(ctx context.Context, sess *session.Session, bucketName string, fileName string) ([]byte, error) {
downloader := s3manager.NewDownloader(sess)
buff := &aws.WriteAtBuffer{}
_, err := downloader.DownloadWithContext(ctx, buff,
&s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(fileName),
})
if err != nil {
return nil, err
}
return buff.Bytes(), nil
}
But this fails to do the job. Check error below:
GOROOT=/usr/local/go #gosetup
GOPATH=/Users/kuldeep/go #gosetup
/usr/local/go/bin/go build -o /private/var/folders/29/zkqqp9_d0fscymp5ph4rscl40000gp/T/GoLand/___2go_build_minio_client_example minio-client-example #gosetup
/private/var/folders/29/zkqqp9_d0fscymp5ph4rscl40000gp/T/GoLand/___2go_build_minio_client_example
2023/01/16 22:37:52 RequestError: send request failed
caused by: Get "http://mybucket.localhost:9000/data.txt": dial tcp: lookup mybucket.localhost: no such host
Process finished with the exit code 1
What should I do to make it work?
When trying to use EC2 client, I get the following error:
=== RUN TestEc2
time="2022-06-10T14:18:43-07:00" level=info msg="starting localstack"
time="2022-06-10T14:18:44-07:00" level=info msg="waiting for localstack to start..."
time="2022-06-10T14:19:01-07:00" level=info msg="localstack: finished waiting"
all_test.go:305: operation error EC2: RunInstances, https response error StatusCode: 401, RequestID: 080f9f11-67d7-4061-86d3-e581551458c1, api error AuthFailure: AWS was not able to validate the provided access credentials
--- FAIL: TestEc2 (18.47s)
FAIL
FAIL command-line-arguments 19.197s
FAIL
func TestEc2(t *testing.T) {
ls, err := localstack.NewInstance()
if err != nil {
log.Fatalf("could not connect to docker: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
if err := ls.StartWithContext(ctx); err != nil {
log.Fatalf("could not start Localstack: %v", err)
}
ec2Client = ec2.NewFromConfig(awsConfig(ctx, ls, localstack.EC2))
createInstance(t, ec2Client)
}
func awsConfig(ctx context.Context, l *localstack.Instance, s localstack.Service) aws.Config {
rf := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: l.EndpointV2(s),
SigningRegion: endpoints.UsEast1RegionID,
}, nil
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion("us-east-1"),
config.WithEndpointResolverWithOptions(rf),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("dummy", "dummy", "dummy")),
)
if err != nil {
log.Fatalf("could not load config: %v", err)
}
return cfg
}
func createInstance(t *testing.T, c *ec2.Client) types.Instance {
t.Helper()
result, err := c.RunInstances(
context.TODO(),
&ec2.RunInstancesInput{
ImageId: aws.String("ami-e7527ed7"),
InstanceType: types.InstanceTypeT2Micro,
MinCount: aws.Int32(1),
MaxCount: aws.Int32(1),
})
if err != nil {
t.Fatal(err)
}
if result == nil {
t.Fatal("empty RunInstance result")
}
if len(result.Instances) != 1 {
t.Fatalf("exptected 1 instance, got %d", len(result.Instances))
}
return result.Instances[0]
}
But if instead I use the deprecated EndpointResolverFunc everything seem to work OK
r := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: l.EndpointV2(s),
SigningRegion: endpoints.UsEast1RegionID,
}, nil
})
cfg, err := config.LoadDefaultConfig(ctx,
....
config.WithEndpointResolver(r),
)
Anyone has any idea what I might be missing?
I am trying to upload a file using APIGateway and Lambda (Go). Idea is to upload the file and then move that file to S3 bucket. But for some reason I keep getting an error. Here is my code.
func parse(request events.APIGatewayProxyRequest) {
var fileBytes []byte
var filename, contentType string
contentType, params, parseErr := mime.ParseMediaType(request.Headers["Content-Type"])
fmt.Printf("DEBUG:: contentType: %v\n", contentType)
if parseErr != nil || !strings.HasPrefix(contentType, "multipart/") {
logger.Instance.Error("unexpected error when retrieving a part of the message", zap.Error(parseErr))
return
}
fmt.Printf("DEBUG:: request body: %v\n", request.Body)
multipartReader := multipart.NewReader(strings.NewReader(request.Body), params["boundary"])
// defer request.Body.Close()
for {
part, err := multipartReader.NextPart()
if err == io.EOF {
break
}
if err != nil {
logger.Instance.Error("unexpected error when retrieving a part of the message", zap.Error(err))
return
}
defer part.Close()
fileBytes, err = ioutil.ReadAll(part)
if err != nil {
logger.Instance.Error("failed to read content of the part", zap.Error(err))
return
}
}
logger.Instance.Info("handler.parseForm", zap.String("ParseForm", filename))
logger.Instance.Info("handler.parseForm", zap.String("ParseForm", contentType))
logger.Instance.Info("handler.parseForm", zap.Int("ParseForm", len(fileBytes)))
}
I get the following Error:
{
"level": "error",
"ts": 1614221323.4001436,
"caller": "uploadsvc/uploadsvc.go:53",
"msg": "unexpected error when retrieving a part of the message",
**"error": "multipart: NextPart: bufio: buffer full",**
"stacktrace": "main.parse\n\t/home/sarvs/go-workspace/src/uploadsvc/uploadsvc.go:53\nmain.handler\n\t/home/sarvs/go-workspace/src/uploadsvc/uploadsvc.go:144\nreflect.Value.call\n\t/snap/go/7013/src/reflect/value.go:476\nreflect.Value.Call\n\t/snap/go/7013/src/reflect/value.go:337\ngithub.com/aws/aws-lambda-go/lambda.NewHandler.func1\n\t/home/sarvs/go/pkg/mod/github.com/aws/aws-lambda-go#v1.22.0/lambda/handler.go:124\ngithub.com/aws/aws-lambda-go/lambda.lambdaHandler.Invoke\n\t/home/sarvs/go/pkg/mod/github.com/aws/aws-lambda-go#v1.22.0/lambda/handler.go:24\ngithub.com/aws/aws-lambda-go/lambda.(*Function).Invoke\n\t/home/sarvs/go/pkg/mod/github.com/aws/aws-lambda-go#v1.22.0/lambda/function.go:64\nreflect.Value.call\n\t/snap/go/7013/src/reflect/value.go:476\nreflect.Value.Call\n\t/snap/go/7013/src/reflect/value.go:337\nnet/rpc.(*service).call\n\t/snap/go/7013/src/net/rpc/server.go:377"
}
I am using GO SDK and using the DynamnoDB BatchGetItem API.
I saw this code example -
https://github.com/aws/aws-sdk-go/blob/master/service/dynamodb/examples_test.go
Is there any other code example which shows unmarshalling of the response from the BatchGetItem API?
Let me share piece of the code. The key to understand it is that when you send GetBatchItem request to dynamodb, you specify map of table names and keys for that table, so response you get is a map of tables names and matched items
placeIDs := []string { "london_123", "sanfran_15", "moscow_9" }
type Place {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}
mapOfAttrKeys := []map[string]*dynamodb.AttributeValue{}
for _, place := range placeIDs {
mapOfAttrKeys = append(mapOfAttrKeys, map[string]*dynamodb.AttributeValue{
"id": &dynamodb.AttributeValue{
S: aws.String(place),
},
"attr": &dynamodb.AttributeValue{
S: aws.String("place"),
},
})
}
input := &dynamodb.BatchGetItemInput{
RequestItems: map[string]*dynamodb.KeysAndAttributes{
tableName: &dynamodb.KeysAndAttributes{
Keys: mapOfAttrKeys,
},
},
}
batch, err := db.BatchGetItem(input)
if err != nil {
panic(fmt.Errorf("batch load of places failed, err: %w", err))
}
for _, table := range batch.Responses {
for _, item := range table {
var place Place
err = dynamodbattribute.UnmarshalMap(item, &place)
if err != nil {
panic(fmt.Errorf("failed to unmarshall place from dynamodb response, err: %w", err))
}
places = append(places, place)
}
}
I'm trying to download a particular data chunk from S3. Following is the code snippet.
func DownloadFromS3() ([]byte, error) {
retries := 5
awsSession = session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: aws.Config{
MaxRetries: &retries,
LogLevel: aws.LogLevel(aws.LogDebugWithHTTPBody),
},
}))
// Create S3 service client
serviceS3 = s3.New(awsSession)
d := s3manager.NewDownloaderWithClient(serviceS3,func(d *s3manager.Downloader) {
d.Concurrency = 10 // should be ignored
d.PartSize = 1 // should be ignored
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("mybucket"),
Key: aws.String("key1"),
Range: aws.String("bytes=0-9"),
})
if err != nil {
return nil, err
}
return w.Bytes(), err
}
But this keeps downloading part by part continuously till the entire object is retrieved; without downloading only the specified part. Am I missing any configurations here?
Looks like an issue with the Go SDK; try the s3.GetObject instead of downloader.
This was an issue with previous AWS-Go-SDK. It's now fixed https://github.com/aws/aws-sdk-go/pull/1311