Localstack: AWS was not able to validate the provided access credentials - amazon-web-services

When trying to use EC2 client, I get the following error:
=== RUN TestEc2
time="2022-06-10T14:18:43-07:00" level=info msg="starting localstack"
time="2022-06-10T14:18:44-07:00" level=info msg="waiting for localstack to start..."
time="2022-06-10T14:19:01-07:00" level=info msg="localstack: finished waiting"
all_test.go:305: operation error EC2: RunInstances, https response error StatusCode: 401, RequestID: 080f9f11-67d7-4061-86d3-e581551458c1, api error AuthFailure: AWS was not able to validate the provided access credentials
--- FAIL: TestEc2 (18.47s)
FAIL
FAIL command-line-arguments 19.197s
FAIL
func TestEc2(t *testing.T) {
ls, err := localstack.NewInstance()
if err != nil {
log.Fatalf("could not connect to docker: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
if err := ls.StartWithContext(ctx); err != nil {
log.Fatalf("could not start Localstack: %v", err)
}
ec2Client = ec2.NewFromConfig(awsConfig(ctx, ls, localstack.EC2))
createInstance(t, ec2Client)
}
func awsConfig(ctx context.Context, l *localstack.Instance, s localstack.Service) aws.Config {
rf := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: l.EndpointV2(s),
SigningRegion: endpoints.UsEast1RegionID,
}, nil
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion("us-east-1"),
config.WithEndpointResolverWithOptions(rf),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("dummy", "dummy", "dummy")),
)
if err != nil {
log.Fatalf("could not load config: %v", err)
}
return cfg
}
func createInstance(t *testing.T, c *ec2.Client) types.Instance {
t.Helper()
result, err := c.RunInstances(
context.TODO(),
&ec2.RunInstancesInput{
ImageId: aws.String("ami-e7527ed7"),
InstanceType: types.InstanceTypeT2Micro,
MinCount: aws.Int32(1),
MaxCount: aws.Int32(1),
})
if err != nil {
t.Fatal(err)
}
if result == nil {
t.Fatal("empty RunInstance result")
}
if len(result.Instances) != 1 {
t.Fatalf("exptected 1 instance, got %d", len(result.Instances))
}
return result.Instances[0]
}
But if instead I use the deprecated EndpointResolverFunc everything seem to work OK
r := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: l.EndpointV2(s),
SigningRegion: endpoints.UsEast1RegionID,
}, nil
})
cfg, err := config.LoadDefaultConfig(ctx,
....
config.WithEndpointResolver(r),
)
Anyone has any idea what I might be missing?

Related

Can we use AWS S3 sdk to connect MinIO(running locally) programatically? If yes, how?

I am trying to connect my local MinIO instance running in docker. I have written a simple Go program to do the work. See following:
package main
import (
"bytes"
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
)
func main() {
endpoint := "localhost:9000"
accessKeyID := "1mVyDeEcrRU7qQ7h"
secretAccessKey := "vNi9SNfcHNxfRDETSvkeFLO210mzu7ee"
useSSL := false
awsSession := session.Must(session.NewSession(&aws.Config{
Region: aws.String("ap-south-1"),
Credentials: credentials.NewStaticCredentialsFromCreds(credentials.Value{
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
SessionToken: "",
}),
Endpoint: aws.String(endpoint),
DisableSSL: aws.Bool(!useSSL),
}))
ctx := context.Background()
file, err := downloadFileFromS3(ctx, awsSession, "mybucket", "data.txt")
if err != nil {
log.Fatal(err)
}
reader := bytes.NewReader(file)
println(reader)
}
func downloadFileFromS3(ctx context.Context, sess *session.Session, bucketName string, fileName string) ([]byte, error) {
downloader := s3manager.NewDownloader(sess)
buff := &aws.WriteAtBuffer{}
_, err := downloader.DownloadWithContext(ctx, buff,
&s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(fileName),
})
if err != nil {
return nil, err
}
return buff.Bytes(), nil
}
But this fails to do the job. Check error below:
GOROOT=/usr/local/go #gosetup
GOPATH=/Users/kuldeep/go #gosetup
/usr/local/go/bin/go build -o /private/var/folders/29/zkqqp9_d0fscymp5ph4rscl40000gp/T/GoLand/___2go_build_minio_client_example minio-client-example #gosetup
/private/var/folders/29/zkqqp9_d0fscymp5ph4rscl40000gp/T/GoLand/___2go_build_minio_client_example
2023/01/16 22:37:52 RequestError: send request failed
caused by: Get "http://mybucket.localhost:9000/data.txt": dial tcp: lookup mybucket.localhost: no such host
Process finished with the exit code 1
What should I do to make it work?

Unit testing client-go function that auths and returns a k8s clientset

I am writing the simplest fxn possible using client-go that justs performs in-cluster authentication and returns a pointer to the kubernetes.Clientset object
// getInClusterAuth performs in cluster authentication and returns the clientset
func getInClusterAuth() (*kubernetes.Clientset, error) {
// creates the in-cluster config
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
// create the clientset
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return clientset, nil
}
Since this is sth that not only does it run against an external system (a k8s api server) but it is also supposed to be running from within a Pod once deployed, what is the appropriate way of unit-testing it?
Could it be that it is an acceptable practice to cover the case in e2e or integration tests?
you can use k8s.io/client-go/kubernetes/fake.NewSimpleClientset to mock a clientSet in your unit test .
import (
"context"
"testing"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestHelloWorld(t *testing.T) {
clientset := fake.NewSimpleClientset(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "influxdb-v2",
Namespace: "default",
Annotations: map[string]string{},
},
}, &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "chronograf",
Namespace: "default",
Annotations: map[string]string{},
},
})
_, err := clientset.CoreV1().Pods(v1.NamespaceDefault).Get(context.Background(), "influxdb-v2", metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
t.Error(err)
} else {
t.Errorf("failed to get service from apiserver: %s", err)
}
}
p := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "helloooooo"}}
_, err = clientset.CoreV1().Pods(v1.NamespaceDefault).Create(context.Background(), p, metav1.CreateOptions{})
if err != nil {
t.Fatalf("error injecting pod add: %v", err)
}
}
this is a sample: a fakeclient case

RPC API Create Service: Requested entity was not found error

I have been putting together a Go gRPC client for Cloud Run to use for an application that I am working on. I am able to perform LIST operations on services(and other objects), but not CREATE, REPLACE, DELETE or GET.
I have verified, using tcpdump, that the requests from the client are going to the intended regional endpoint and not to the global endpoint which I am aware only supports LIST.
The error I get is "rpc error: code = NotFound desc = Requested entity was not found."
It is unclear to me from this error message being returned what specific entity was not found.
Is it the case that ALL RPC endpoints only support LIST operations currently? I can think of no other explanation for this behaviour. Thanks in advance!
Here is my code:
package main
import (
"context"
"fmt"
runclient "github.com/cristian-radu/cloud-run-grpc/pkg/client"
runpb "github.com/cristian-radu/cloud-run-grpc/pkg/pb"
"google.golang.org/api/option"
)
func main() {
ctx := context.Background()
servicesClient, err := runclient.NewServicesClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
service := runpb.Service{
ApiVersion: "serving.knative.dev/v1",
Kind: "Service",
Metadata: &runpb.ObjectMeta{
Name: "namespaces/{project-id}/services/test",
Namespace: "{project-id}",
Annotations: map[string]string{
"run.googleapis.com/ingress": "all",
},
},
Spec: &runpb.ServiceSpec{
Template: &runpb.RevisionTemplate{
Metadata: &runpb.ObjectMeta{
Annotations: map[string]string{
"autoscaling.knative.dev/maxScale": "10",
},
},
Spec: &runpb.RevisionSpec{
ServiceAccountName: "{service-account}#{project-id}.iam.gserviceaccount.com",
Containers: []*runpb.Container{
{
Image: "us-docker.pkg.dev/cloudrun/container/hello",
Ports: []*runpb.ContainerPort{
{
ContainerPort: 8080,
},
},
},
},
},
},
Traffic: []*runpb.TrafficTarget{
{
Percent: 100,
LatestRevision: true,
},
},
},
}
req := runpb.CreateServiceRequest{
Parent: "namespaces/{project-id}",
Service: &service,
}
_, err = servicesClient.CreateService(ctx, &req)
if err != nil {
fmt.Println(err)
}
authorizedDomainsClient, err := runclient.NewAuthorizedDomainsClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
listAuthorizedDomains := runpb.ListAuthorizedDomainsRequest{Parent: "namespaces/{project-id}"}
authorizedDomainsIterator := authorizedDomainsClient.ListAuthorizedDomains(ctx, &listAuthorizedDomains)
fmt.Printf("Authorized Domains: %v \n\n", authorizedDomainsIterator.Response)
configurationsClient, err := runclient.NewConfigurationsClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
listConfigurations := runpb.ListConfigurationsRequest{Parent: "namespaces/{project-id}"}
configurationsResp, err := configurationsClient.ListConfigurations(ctx, &listConfigurations)
if err != nil {
fmt.Println(err)
}
fmt.Printf("Configurations: %v \n\n", configurationsResp)
domainMappingsClient, err := runclient.NewDomainMappingsClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
listDomainMappings := runpb.ListDomainMappingsRequest{Parent: "namespaces/{project-id}"}
domainMappingsResp, err := domainMappingsClient.ListDomainMappings(ctx, &listDomainMappings)
if err != nil {
fmt.Println(err)
}
fmt.Printf("Domain Mappings: %v \n\n", domainMappingsResp)
revisionsClient, err := runclient.NewRevisionsClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
revisions := runpb.ListRevisionsRequest{Parent: "namespaces/{project-id}"}
revisionsResp, err := revisionsClient.ListRevisions(ctx, &revisions)
if err != nil {
fmt.Println(err)
}
fmt.Printf("Revisions: %v \n\n", revisionsResp)
routesClient, err := runclient.NewRoutesClient(ctx, option.WithEndpoint("us-central1-run.googleapis.com:443"))
if err != nil {
fmt.Println(err)
}
routes := runpb.ListRoutesRequest{Parent: "namespaces/{project-id}"}
routesResp, err := routesClient.ListRoutes(ctx, &routes)
if err != nil {
fmt.Println(err)
}
fmt.Printf("Routes: %v \n\n", routesResp)
}
The output I get:
rpc error: code = NotFound desc = Requested entity was not found.
Authorized Domains: <nil>
Configurations: api_version:"serving.knative.dev/v1" kind:"ConfigurationList" items:{api_version:"serving.knative.dev/v1" kind:"Configuration" metadata:{name:"test-1" namespace:"{project-number}" self_link:"/apis/serving.knative.dev/v1/namespaces/{project-number}/configurations/test-1" uid:"90cc5b51-c40e-4692-871f-f9beb8ab6bd2" resource_version:"AAXFqFkSE1k" generation:2 create_time:{seconds:1624114599 nanos:280472000} labels:{key:"cloud.googleapis.com/location" value:"us-central1"} labels:{key:"serving.knative.dev/route" value:"test-1"} labels:{key:"serving.knative.dev/service" value:"test-1"} labels:{key:"serving.knative.dev/serviceUid" value:"1bf35167-9557-41a1-8a83-fb6b9cc4f3ec"} annotations:{key:"client.knative.dev/user-image" value:"us-docker.pkg.dev/cloudrun/container/hello"} annotations:{key:"run.googleapis.com/client-name" value:"cloud-console"} annotations:{key:"serving.knative.dev/creator" value:"{service-account}#{project-id}.iam.gserviceaccount.com"} annotations:{key:"serving.knative.dev/lastModifier" value:"me#example.com"} owner_references:{api_version:"Service" name:"test-1" uid:"1bf35167-9557-41a1-8a83-fb6b9cc4f3ec" block_owner_deletion:true 5:"serving.knative.dev/v1" 7:1}} spec:{template:{metadata:{name:"test-1-00002-mer" annotations:{key:"autoscaling.knative.dev/maxScale" value:"100"} annotations:{key:"run.googleapis.com/client-name" value:"cloud-console"} annotations:{key:"run.googleapis.com/sandbox" value:"gvisor"}} spec:{container_concurrency:80 timeout_seconds:300 service_account_name:"{service-account}#{project-id}.iam.gserviceaccount.com" containers:{image:"us-docker.pkg.dev/cloudrun/container/hello" resources:{3:8080} ports:{name:"\n\x06memory\x12\x05512Mi"}}}}} status:{observed_generation:2 latest_created_revision_name:"test-1-00002-mer" latest_ready_revision_name:"test-1-00002-mer" conditions:{type:"Ready" status:"True" last_transition_time:{seconds:1624115077 nanos:363769000}}}} unreachable:"asia-east1" unreachable:"asia-east2" unreachable:"asia-northeast1" unreachable:"asia-northeast2" unreachable:"asia-northeast3" unreachable:"asia-south1" unreachable:"asia-south2" unreachable:"asia-southeast1" unreachable:"asia-southeast2" unreachable:"australia-southeast1" unreachable:"australia-southeast2" unreachable:"europe-central2" unreachable:"europe-north1" unreachable:"europe-west1" unreachable:"europe-west2" unreachable:"europe-west3" unreachable:"europe-west4" unreachable:"europe-west6" unreachable:"northamerica-northeast1" unreachable:"southamerica-east1" unreachable:"us-east1" unreachable:"us-east4" unreachable:"us-west1" unreachable:"us-west2" unreachable:"us-west3" unreachable:"us-west4"
Domain Mappings: api_version:"domains.cloudrun.com/v1" kind:"DomainMappingList" 6:"asia-east1" 6:"asia-east2" 6:"asia-northeast1" 6:"asia-northeast2" 6:"asia-northeast3" 6:"asia-south1" 6:"asia-south2" 6:"asia-southeast1" 6:"asia-southeast2" 6:"australia-southeast1" 6:"australia-southeast2" 6:"europe-central2" 6:"europe-north1" 6:"europe-west1" 6:"europe-west2" 6:"europe-west3" 6:"europe-west4" 6:"europe-west6" 6:"northamerica-northeast1" 6:"southamerica-east1" 6:"us-east1" 6:"us-east4" 6:"us-west1" 6:"us-west2" 6:"us-west3" 6:"us-west4"
Revisions: api_version:"serving.knative.dev/v1" kind:"RevisionList" items:{api_version:"serving.knative.dev/v1" kind:"Revision" metadata:{name:"test-1-00002-mer" namespace:"{project-number}" self_link:"/apis/serving.knative.dev/v1/namespaces/{project-number}/revisions/test-1-00002-mer" uid:"a9baacff-d6f8-48ac-bbdc-98ceff612292" resource_version:"AAXFH+YLhiQ" generation:1 create_time:{seconds:1624115070 nanos:453178000} labels:{key:"cloud.googleapis.com/location" value:"us-central1"} labels:{key:"serving.knative.dev/configuration" value:"test-1"} labels:{key:"serving.knative.dev/configurationGeneration" value:"2"} labels:{key:"serving.knative.dev/route" value:"test-1"} labels:{key:"serving.knative.dev/service" value:"test-1"} labels:{key:"serving.knative.dev/serviceUid" value:"1bf35167-9557-41a1-8a83-fb6b9cc4f3ec"} annotations:{key:"autoscaling.knative.dev/maxScale" value:"100"} annotations:{key:"run.googleapis.com/client-name" value:"cloud-console"} annotations:{key:"run.googleapis.com/sandbox" value:"gvisor"} annotations:{key:"serving.knative.dev/creator" value:"me#example.com"} owner_references:{api_version:"Configuration" name:"test-1" uid:"90cc5b51-c40e-4692-871f-f9beb8ab6bd2" block_owner_deletion:true 5:"serving.knative.dev/v1" 7:1}} spec:{container_concurrency:80 timeout_seconds:300 service_account_name:"{service-account}#{project-id}.iam.gserviceaccount.com" containers:{image:"us-docker.pkg.dev/cloudrun/container/hello" resources:{3:8080} ports:{name:"\n\x06memory\x12\x05512Mi"}}} status:{observed_generation:1 conditions:{type:"Ready" status:"True" last_transition_time:{seconds:1624115077 nanos:320431000}} conditions:{type:"Active" status:"True" last_transition_time:{seconds:1624115082 nanos:217762000} severity:"Info"} conditions:{type:"ContainerHealthy" status:"True" last_transition_time:{seconds:1624115077 nanos:320431000}} conditions:{type:"ResourcesAvailable" status:"True" last_transition_time:{seconds:1624115076 nanos:732413000}} log_url:"https://console.cloud.google.com/logs/viewer?project={project-id}&resource=cloud_run_revision/service_name/test-1/revision_name/test-1-00002-mer&advancedFilter=resource.type%3D%22cloud_run_revision%22%0Aresource.labels.service_name%3D%22test-1%22%0Aresource.labels.revision_name%3D%22test-1-00002-mer%22" image_digest:"us-docker.pkg.dev/cloudrun/container/hello#sha256:244fd3d0268aa56777020d0bd1b9c0b828f2f0a8f403dbcd26eba5610a450e2c"}} unreachable:"asia-east1" unreachable:"asia-east2" unreachable:"asia-northeast1" unreachable:"asia-northeast2" unreachable:"asia-northeast3" unreachable:"asia-south1" unreachable:"asia-south2" unreachable:"asia-southeast1" unreachable:"asia-southeast2" unreachable:"australia-southeast1" unreachable:"australia-southeast2" unreachable:"europe-central2" unreachable:"europe-north1" unreachable:"europe-west1" unreachable:"europe-west2" unreachable:"europe-west3" unreachable:"europe-west4" unreachable:"europe-west6" unreachable:"northamerica-northeast1" unreachable:"southamerica-east1" unreachable:"us-east1" unreachable:"us-east4" unreachable:"us-west1" unreachable:"us-west2" unreachable:"us-west3" unreachable:"us-west4"
Routes: api_version:"serving.knative.dev/v1" kind:"RouteList" items:{api_version:"serving.knative.dev/v1" kind:"Route" metadata:{name:"test-1" namespace:"{project-number}" self_link:"/apis/serving.knative.dev/v1/namespaces/{project-number}/routes/test-1" uid:"d9ce0d4d-2477-4d14-9920-2a26728c9b6e" resource_version:"AAXFIQRxbns" generation:1 create_time:{seconds:1624114599 nanos:247571000} labels:{key:"cloud.googleapis.com/location" value:"us-central1"} labels:{key:"serving.knative.dev/service" value:"test-1"} labels:{key:"serving.knative.dev/serviceUid" value:"1bf35167-9557-41a1-8a83-fb6b9cc4f3ec"} annotations:{key:"client.knative.dev/user-image" value:"us-docker.pkg.dev/cloudrun/container/hello"} annotations:{key:"run.googleapis.com/client-name" value:"cloud-console"} annotations:{key:"run.googleapis.com/ingress" value:"all"} annotations:{key:"run.googleapis.com/ingress-status" value:"all"} annotations:{key:"serving.knative.dev/creator" value:"{service-account}#{project-id}.iam.gserviceaccount.com"} annotations:{key:"serving.knative.dev/lastModifier" value:"me#example.com"} owner_references:{api_version:"Service" name:"test-1" uid:"1bf35167-9557-41a1-8a83-fb6b9cc4f3ec" block_owner_deletion:true 5:"serving.knative.dev/v1" 7:1}} spec:{target:{configuration_name:"test-1" percent:100}} status:{observed_generation:1 conditions:{type:"Ready" status:"True" last_transition_time:{seconds:1624115082 nanos:399315000}} traffic:{revision_name:"test-1-00002-mer" percent:100 latest_revision:true} url:"https://test-1-azcqr7ym2a-uc.a.run.app" address:{url:"https://test-1-azcqr7ym2a-uc.a.run.app"}}} unreachable:"asia-east1" unreachable:"asia-east2" unreachable:"asia-northeast1" unreachable:"asia-northeast2" unreachable:"asia-northeast3" unreachable:"asia-south1" unreachable:"asia-south2" unreachable:"asia-southeast1" unreachable:"asia-southeast2" unreachable:"australia-southeast1" unreachable:"australia-southeast2" unreachable:"europe-central2" unreachable:"europe-north1" unreachable:"europe-west1" unreachable:"europe-west2" unreachable:"europe-west3" unreachable:"europe-west4" unreachable:"europe-west6" unreachable:"northamerica-northeast1" unreachable:"southamerica-east1" unreachable:"us-east1" unreachable:"us-east4" unreachable:"us-west1" unreachable:"us-west2" unreachable:"us-west3" unreachable:"us-west4"

I can't read from the environment variable (aws-lambda in Go)

I want to test AWS Lambda in local environment using DynamoDB Local and SAM CLI.
I create a simple user db table(id, name) and I'm trying to get the data.
I run "sam local start-api --env-vars test/env.json". When I access "http://localhost:3000/users/1", an error occured. Error message is below. I can't understand what this error message means. How do I fix this error?
{
"errorMessage": "InvalidParameter: 1 validation error(s) found.\n- minimum field size of 3, GetItemInput.TableName.\n",
"errorType": "ErrInvalidParams"
}
This is my code.
func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
// Environment variables
endpoint := os.Getenv("DYNAMODB_ENDPOINT")
tableName := os.Getenv("DYNAMODB_TABLE_NAME")
// Request
id, _ := request.PathParameters["id"]
// DynamoDB
sess := session.Must(session.NewSession())
config := aws.NewConfig().WithRegion("ap-northeast-1")
if len(endpoint) > 0 {
config = config.WithEndpoint(endpoint)
}
db := dynamodb.New(sess, config)
response, err := db.GetItem(&dynamodb.GetItemInput{
TableName: aws.String(tableName),
Key: map[string]*dynamodb.AttributeValue{
"Id": {
N: aws.String(string(id)),
},
},
AttributesToGet: []*string{
aws.String("Id"),
aws.String("Name"),
},
ConsistentRead: aws.Bool(true),
ReturnConsumedCapacity: aws.String("NONE"),
})
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
user := User{}
err = dynamodbattribute.Unmarshal(&dynamodb.AttributeValue{M: response.Item}, &user)
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
// Json
bytes, err := json.Marshal(user)
if err != nil {
return events.APIGatewayProxyResponse{}, err
}
return events.APIGatewayProxyResponse{
Body: string(bytes),
StatusCode: 200,
}, nil
}
error message
That's been resolved.
First Problem: can't read from the environment variable
This was because the indent size deviation of template.yaml. Category "Events" and Category "Environment" had to be in line.
Second Problem: error with "Function 'UserGetFunction' timed out after 5 seconds"
This was because "localhost" expression. Rewriting "localhost" in env.json to "xxx.xxx.xxx.xxx" worked fine.("xxx.xxx.xxx.xxx" is ip address of your laptop)

golang: GET request for presigned url of S3 object with SSE fails with 403 error

I'm getting 403 Forbidden HTTP response when I make a GET request to a presigned url generated by the AWS Presign method in Go.
The error message is:
The request signature we calculated does not match the signature you provided. Check your key and signing method
X-Amz-SignedHeaders is: host;x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5
I'm writing the object to S3 like this:
type DocumentStore struct {
bucketName string
bucketEncryptionKeyAlias string
aws *session.Session
}
func (s *DocumentStore) PutDocument(ctx context.Context, envelope []byte, metadata map[string]string) (PutDocumentResult, error) {
uploader := s3manager.NewUploader(s.aws)
var objectKey = uuid.New().String()
if _, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
ContentType: aws.String("application/octet-stream"),
Body: bytes.NewReader(envelope),
ServerSideEncryption: aws.String(s3.ServerSideEncryptionAwsKms),
SSEKMSKeyId: aws.String(s.bucketEncryptionKeyAlias),
Metadata: aws.StringMap(metadata),
}); err != nil {
return PutDocumentResult{}, fmt.Errorf("put document failed on upload: %v", err.Error())
}
return PutDocumentResult{
BucketName: s.bucketName,
ObjectKey: objectKey,
}, nil
}
I'm signing the url like this:
func (s *DocumentStore) NewSignedGetURL(ctx context.Context, objectKey string, ttl time.Duration) (string, error) {
svc := s3.New(s.aws)
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
SSECustomerKey: aws.String(s.bucketEncryptionKeyAlias),
SSECustomerAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
})
url, err := req.Presign(ttl)
if err != nil {
return "", fmt.Errorf("failed to presign GetObjectRequest for key %q: %v", objectKey, err)
}
return url, nil
}
And I'm calling the methods like this:
result, err := target.PutDocument(context.TODO(), envelope, metadata)
if err != nil {
t.Errorf("PutDocument failed: %v", err)
return
}
getURL, err := target.NewSignedGetURL(context.TODO(), result.ObjectKey, time.Minute*5)
if err != nil {
t.Errorf("failed to sign url: %v", err)
return
}
req, _ := http.NewRequest("GET", getURL, nil)
req.Header.Add("x-amz-server-side-encryption-customer-algorithm", s3.ServerSideEncryptionAwsKms)
req.Header.Add("x-amz-server-side-encryption-customer-key", test.cfg.AWS.BucketKMSAlias)
req.Header.Add("x-amz-server-side-encryption-customer-key-md5", "")
resp, err := http.DefaultClient.Do(req.WithContext(context.TODO()))
if err != nil {
t.Errorf("failed to request object from signed url: %v", err)
return
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("failed to read object stream from S3: %v", err)
return
}
if resp.StatusCode != http.StatusOK {
t.Errorf("failed to get object. Http status: %s(%d)\n%s", resp.Status, resp.StatusCode, data)
return
}
I can read the download the file from the aws cli like this:
aws --profile dispatcher_stage --region us-east-1 s3 cp s3://[bucket-name]/0c/09179312-e283-431c-ab71-6a0c437177fe . --sse aws:kms --sse-kms-key-id alias/[key-alias-name]
What am I missing?
I figured it out: the GetObject request doesn't need the SSE parameters as long as the user has Decrypt permission on the KMS key. Here's the relevant changes:
I'm now signing the url like this:
func (s *DocumentStore) NewSignedGetURL(ctx context.Context, objectKey string, ttl time.Duration) (string, error) {
svc := s3.New(s.aws)
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
})
url, err := req.Presign(ttl)
if err != nil {
return "", fmt.Errorf("failed to presign GetObjectRequest for key %q: %v", objectKey, err)
}
return url, nil
}
And I'm downloading the object like this:
getURL, err := target.NewSignedGetURL(context.TODO(), result.ObjectKey, time.Minute*5)
if err != nil {
t.Errorf("failed to sign url: %v", err)
return
}
req, _ := http.NewRequest("GET", getURL, nil)
req.Header.Add("host", req.Host)
resp, err := http.DefaultClient.Do(req.WithContext(context.TODO()))
if err != nil {
t.Errorf("failed to request object from signed url: %v", err)
return
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("failed to read object stream from S3: %v", err)
return
}