I have a go services that generate a presigned url to upload a file:
sess, err := session.NewSession(&aws.Config{
Region: aws.String(os.Getenv(AwsRegionEnv))},
)
if err != nil {
return nil, err
}
svc := s3.New(sess)
req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(os.Getenv(BucketNameEnv)),
Key: aws.String(getFileName(file, customer)),
})
minutesTimeout, err := strconv.Atoi(os.Getenv(TimeoutURL))
if err != nil {
return nil, err
}
str, err := req.Presign(time.Duration(minutesTimeout) * time.Minute)
if err != nil {
return nil, err
}
So, I can upload a file using this presigned url using curl:
curl -vT test.pdf '<<URL PRESIGNED>>'
But, when I add an ACL, this not work, the modification are:
req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(os.Getenv(BucketNameEnv)),
Key: aws.String(getFileName(file, customer)),
ACL: aws.String(s3.ObjectCannedACLAuthenticatedRead),
})
When I tried upload a file using the presigned url with ACL, I got this error:
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>SignatureDoesNotMatch</Code>
<Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>
<AWSAccessKeyId>ASIAS...</AWSAccessKeyId>
<StringToSign>AWS4-HMAC-SHA256
20210316T135400Z
20210316/us-east-2/s3/aws4_request
d7ab7d377b719636610b11793e3e68e104a3f41fb9f9f5608138a8c2b19ceaf3</StringToSign>
<SignatureProvided>bd59fbb080..</SignatureProvided>
<StringToSignBytes>41 57 53...</StringToSignBytes>
<CanonicalRequest>PUT
/35527810/sampleVpacheco3.pdf
X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20210316T135400Z&X-Amz-Expires=300&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEN7%2F%2F...&X-Amz-SignedHeaders=host%3Bx-amz-acl
host:adl-digital-dev-document-manager.s3.us-east-2.amazonaws.com
x-amz-acl:
host;x-amz-acl
UNSIGNED-PAYLOAD</CanonicalRequest>
<CanonicalRequestBytes>50 55 54...</CanonicalRequestBytes>
<RequestId>0V6FWNNGK2QCDA1V</RequestId>
<HostId>rE4rkv...</HostId>
</Error>
Any idea how I can add ACL and can upload a file successfully?
Full URL is:
https://document-manager.s3.us-east-2.amazonaws.com/35527810/sampleVpacheco5.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIASN3IRSVR%2F20210316%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20210316T143240Z&X-Amz-Expires=300&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEN7%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJGMEQCIGW4j1R7H3wIxUAN8FytKbDTRne4pJGJ9I4ofpXeh%2FcaAiBFTdRNVug9WODzLdCoMcFRVzFZ%2FtGeaweeteSSTY6yMSqtAwgXEAIaDDE2NzE4NjEwOTc5NSIMh1JvraENxW8E5aBqKooDLx39b6Lx1%2Fw6AtGMSzlYRILNIXdB2Ouviq0pUlfPVCSFlZnPzo%2F%2B6%2B8ZcIpHM8E%2FDjEn1NF1lvcz9QKsuXJI94XuVCSRGiBBRvpIdm%2Ff001q3C%2FmZW2I1aMsfV518LTtEQigJ%2Fv80TPVSv7ZozoR9Zae4W3C3efjm2sJ%2BkVkI%2FBm7z6Vd97Q%2BbpVztf8Lp4GImDp1G72wtOP7wq9wSDYzFEzUja91r7g97py1Wzin6%2BXUNX68yAH%2BRePqyW6by4Lht8086B7YQcj6h77kxwE89C1NMYhKPiNl1y%2Ff4NukwWxW%2FTefqSW3Qr26eDfTV%2FVyR7%2FeNCf7OOtpkGZEmOnFbd%2FyY6wVOARcTdixQkPKKu2GAkz%2B8xuNY10uTGoh2vul3gUWBZF4Yl13R7kIq%2FPBb1UVl%2BatCwN%2BDBMj22cM4Pn%2BOJPyqxCjcfyIXwRsiYDTmmtiSIWrTvSEQaWf1Dc95lQVToA2ZsAxB8LO88%2FEz0t3FUpPw0ncgbLbHedcRYqvV62RDRQK%2FI9zjCz78KCBjqnAfzDcfP25%2BIr6ia4elbxSDOWIIv%2FjZOLlRDedHdqLKCDjYbgXoWrTQTt%2BZCRlV7UtJxo%2ByVeJvsjmb3BdI4IjI8wd8XjkV5qMejJbFcmFIQV7df0cdGY7U6nOO8gxGK9fj7Fb1Y0DtZaCxaZU8D0d2iTfUn8kl%2FT0GwSPDZqz1I6oJuG58KLR%2BVKRhuZrhTq8%2Fm98cLg7diuwt%2Bt1RwL%2BK9oonqHqXcE&X-Amz-SignedHeaders=host%3Bx-amz-acl&X-Amz-Signature=27d1fae2f60187dce85b175980c4e91334fe2a0f192d220244aa4a27e798ec9f
I Tired this:
add header x-amc-acl: -H "x-amc-acl: authenticated-read"
add header host: -H "host: 127.0.0.1"
Modify URL, changing host%3Bx-amz-acl by host;x-amc-acl
Thanks!
it worked for me after adding acl properties in query param
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
q := req.HTTPRequest.URL.Query()
q.Add("x-amz-acl", "public-read")
q.Add("Content-Type", contentType)
req.HTTPRequest.URL.RawQuery = q.Encode()
presigned, err := req.Presign(5 * time.Minute)
if err != nil {
fmt.Printf("Error presigning URL: %v\n", err)
}
fmt.Println(presigned)
Related
I am trying to create Amazon SQS from my local machine and facing errors like
Got an error while trying to create queue: NoCredentialProviders: no valid providers in chain. Deprecated.
For verbose messaging see aws.Config.CredentialsChainVerboseErrors
What I did:
Step-1:
I have set up my credentials in .aws/credentials file
[default]
aws_access_key_id = TestAccessKey
aws_secret_access_key = TestSecretAccessKey
Step-2:
My code in go lang like below
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sqs"
)
func CreateQueue(sess *session.Session, queueName string) (*sqs.CreateQueueOutput, error) {
sqsClient := sqs.New(sess)
result, err := sqsClient.CreateQueue(&sqs.CreateQueueInput{
QueueName: &queueName,
Attributes: map[string]*string{
"DelaySeconds": aws.String("0"),
"VisibilityTimeout": aws.String("60"),
},
})
if err != nil {
return nil, err
}
return result, nil
}
func main() {
sess, err := session.NewSessionWithOptions(session.Options{
Profile: "default",
Config: aws.Config{
Region: aws.String("us-east-1"),
},
})
if err != nil {
fmt.Printf("Failed to initialize new session: %v", err)
return
}
queueName := "my-new-queue"
createRes, err := CreateQueue(sess, queueName)
if err != nil {
fmt.Printf("Got an error while trying to create queue: %v", err)
return
}
fmt.Println("Created a new queue with url: " + *createRes.QueueUrl)
}
Step-3:
Try updating profile in Shared Credential File (.aws/credentials) as below:
[default]
aws_access_key_id=TestAccessKey
aws_secret_access_key=TestSecretAccessKey
region=us-east-1
I just added region at the end.
To verify if your CLI session has correct AWS credentials configured run this command aws sts get-caller-identity This command will show which profile is used. If this works fine then you can run any simple AWS commands something like S3.getBuckets() to verify if the dev environment is setup correctly.
These 2 solutions should give you enough input to figure out what's wrong.
Need to download a file from s3 bucket and store in to tmp directory in lambda function. After that need apply grep command into file using "os/exec". I tried to do this algorithm using golang.
I tried through following way but it is not successful approach.
func MyHandler(ctx context.Context, s3Event events.S3Event) {
for _, record := range s3Event.Records {
s3record := record.S3
bucketName := s3record.Bucket.Name
fileName := s3record.Object.Key
download_path := "/tmp/"
file, err := os.Create(download_path + fileName)
if err != nil {
fmt.Println(err)
}
defer file.Close()
sess, _ := session.NewSession(&aws.Config{Region: aws.String("us-east-1")})
downloader := s3manager.NewDownloader(sess)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(fileName),
})
if err != nil {
fmt.Println(err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}
}
Please verify may approach correct or suggest a correct approach to store the file from s3 bucket into tem folder of lambda function everything I need to do using golang.
I have following confusion also
Do we need to create directory folder in lambda function or it is a default directory already there.
Can we apply os/exec command into tem folder and can we grep the file.
Do we need to delete uploaded file from tem folder then How? or The file will be deleted a automatically.
I am using AWS S3 service to upload images. Yesterday I updated the SDK v1 to v2 and found that the image upload is failing with the following error:
operation error S3: PutObject, https response error StatusCode: 403, RequestID: XXXXXXXXXXX, HostID: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
UPDATED:
I have aws credentials on my home folder in linux in .aws folder in the following format:
[default]
aws_access_key_id = XXXXXXXXXXXXXXXXXXX
aws_secret_access_key = XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx
Here is the code:
package main
import (
"context"
"fmt"
"io"
"net/http"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func main() {
fileName := "test123.jpg"
filePath := "/BUCKET_NAME/uploads/aman/2021/6/25/"
res, err := http.Get("https://images.app.goo.gl/mpQ5nXYXjdUMKGgW7")
if err != nil || res.StatusCode != 200 {
// handle errors
}
defer res.Body.Close()
UploadFileInS3Bucket(res.Body, fileName, filePath)
}
func UploadFileInS3Bucket(file io.Reader, fileName, filePath string) {
cfg, err := awsconfig.LoadDefaultConfig(context.TODO(),
awsconfig.WithRegion("REGION"),
)
client := s3.NewFromConfig(cfg)
uploader := manager.NewUploader(client)
uploadResp, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(filePath),
Key: aws.String(fileName),
Body: file,
ContentType: aws.String("image"),
})
fmt.Println(uploadResp)
fmt.Println(err)
}
I did not change any credentials/buckets/regions in my code.However if I run the code with SDK v1 then it works fine & images are uploading.
What is going wrong with the SDK v2 ?
After spending a couple of days, I came to know that SDK V2 takes following format for Bucket & Key field:
fileName := "uploads/2021/6/25/test123.jpg"
filePath := "BUCKET_NAME"
Basically for these fields there is vice versa behaviour in SDK V1 & V2. Above is the V2. Below is the V1:
fileName := "test123.jpg"
filePath := "/BUCKET_NAME/uploads/2021/6/25/"
I have troubles sending emails with AWS golang sdk using SendRawEmail operation. Even though I get no errors and receive MessageId back from AWS, I do not receive the email.
Sending emails using SendEmail works fine and I receive the email.
My code:
session, err := session.NewSession()
if err != nil {
return err
}
svc := ses.New(session, &aws.Config{Region: aws.String("eu-west-1")})
messageContent := `From: "Alice" <xxx#xxx>
To: "Bob" <xxx#xxx>
Return-Path: <xxx#xxx>
Subject: Hello
Content-Language: en-US
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
This is a test email`
base64messageContent := base64.StdEncoding.EncodeToString([]byte(messageContent))
source := aws.String("xxx#xxx")
destinations := []*string{aws.String("xxx#xxx")}
message := ses.RawMessage{Data: []byte(base64messageContent)}
input := ses.SendRawEmailInput{Source: source, Destinations: destinations, RawMessage: &message}
output, err := svc.SendRawEmail(&input)
if err != nil {
return err
}
log.Println("Response from SES", output)
return nil
}
I am using my Gmail as the destination email, if that makes any difference.
Data in RawData should not be base64 encoded. As documentation states:
// Data is automatically base64 encoded/decoded by the SDK.
I used environment variables before and it worked fine.
Now I am migrating my config variables into a single file and I have AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID variables containing respective values that are loaded from this file.
I tried this code but receiving an error:
creds := credentials.NewStaticCredentials("123", conf.AWS_SECRET_ACCESS_KEY, conf.AWS_ACCESS_KEY_ID)
sess, err := session.NewSession(&aws.Config{Credentials: creds})
Here is the error
InvalidClientTokenId: The security token included in the request is invalid.
How do I properly inject my keys into the aws sdk calls?
Try re-ordering your args so that ACCESS_KEY is the 1st param and SECRET_KEY is the second:
creds := credentials.NewStaticCredentials(conf.AWS_ACCESS_KEY_ID, conf.AWS_SECRET_ACCESS_KEY, "")
Try adding the region as well:
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-west-2"),
Credentials: credentials.NewStaticCredentials(conf.AWS_ACCESS_KEY_ID, conf.AWS_SECRET_ACCESS_KEY, ""),
})
Or you can just temporaly set Environment variables.
package main
import (
"fmt"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
const (
AccessKeyId = "XXXXXXXXXXXXXXXXXX"
SecretAccessKey = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
Region = "eu-west-1"
Bucket = "XXXXX-XXXX-XXX"
)
func main() {
os.Setenv("AWS_ACCESS_KEY_ID", AccessKeyId)
os.Setenv("AWS_SECRET_ACCESS_KEY", SecretAccessKey)
filename := os.Args[1]
file, err := os.Open(filename)
if err != nil {
fmt.Println("Failed to open file", filename, err)
os.Exit(1)
}
defer file.Close()
conf := aws.Config{Region: aws.String(Region)}
sess := session.New(&conf)
svc := s3manager.NewUploader(sess)
fmt.Println("Uploading file to S3...")
result, err := svc.Upload(&s3manager.UploadInput{
Bucket: aws.String(Bucket),
Key: aws.String(filepath.Base(filename)),
Body: file,
})
if err != nil {
fmt.Println("error", err)
os.Exit(1)
}
}
Additionally, if you hadn't known, the SDK allows for the use of the shared config under .aws/config. You can put your values in there and then set the environment variable AWS_SDK_LOAD_CONFIG to a truthy value to load the shared config. An example shared config would look like this:
[default]
aws_access_key_id = AKID
aws_secret_access_key = SECRET
Then running:
AWS_SDK_LOAD_CONFIG=true go run main.go
Connect your sdk client using this generic service
var awsSession *session.Session
func init() {
initializeAwsSession()
}
func initializeAwsSession() {
awsSession = session.Must(session.NewSession(&aws.Config{
Region: aws.String("ap-southeast-1"),
Credentials: credentials.NewStaticCredentials("YOUR_ACCESS_KEY","YOUR SECRET_KEY", ""),
}))
}