I want to delete the key(bucket/userID/fileName) using AWS-SDK-Go.
But this code doesn't delete the userID key.
config := model.NewConfig()
sess, _ := session.NewSession(&aws.Config{
Region: aws.String(config.AWSS3Region)},
)
svc := s3.New(sess)
input := &s3.DeleteObjectInput{
Bucket: aws.String(config.AWSS3Bucket),
Key: aws.String(userID + "/"),
}
result, err := svc.DeleteObject(input)
I can delete bucket/userID/fileName but I can't delete bucket/userID.
Here is my code to delete Objects from S3.
import(
"context"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func configS3() *s3.Client {
creds := credentials.NewStaticCredentialsProvider(os.Getenv("S3_ACCESS_KEY_ID"), os.Getenv("S3_SECRET_ACCESS_KEY"), "")
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(creds), config.WithRegion(os.Getenv("S3_REGION")))
if err != nil {
log.Fatal(err)
}
return s3.NewFromConfig(cfg)
}
func DeleteImageFromS3(echoCtx echo.Context) error {
awsClient := configS3()
input := &s3.DeleteObjectInput{
Bucket: aws.String("mybucket"),
Key: aws.String("pic.jpg"),
}
_, err := awsClient.DeleteObject(context.TODO(), input)
if err != nil {
fmt.Println("Got an error deleting item:")
fmt.Println(err)
return
}
return echoCtx.JSON(http.StatusOK, "Object Deleted Successfully")
}
For further reference, please check this https://aws.github.io/aws-sdk-go-v2/docs/code-examples/s3/deleteobject/
Related
I am trying to list out all tags of my lambda functions, struggling a lot, please help me if anyone knows.
func main() {
svc := lambda.New(session.New())
input := &lambda.ListTagsInput{
Resource: aws.String("arn:aws:lambda:us-east-1:657907747545:function-function-name"),
I'm expecting to list all tag arn's for my lambda functions
You can use the following code:
package main
import (
"context"
awsutils "github.com/alessiosavi/GoGPUtils/aws"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/lambda"
"os"
"sync"
)
var lambdaClient *lambda.Client = nil
var once sync.Once
func init() {
once.Do(func() {
cfg, err := awsutils.New()
if err != nil {
panic(err)
}
lambdaClient = lambda.New(lambda.Options{Credentials: cfg.Credentials, Region: cfg.Region})
})
}
func ListLambdas() ([]string, error) {
f, err := lambdaClient.ListFunctions(context.Background(), &lambda.ListFunctionsInput{})
if err != nil {
return nil, err
}
var functions = make([]string, len(f.Functions))
for i, functionName := range f.Functions {
functions[i] = *functionName.FunctionName
}
continuationToken := f.NextMarker
for continuationToken != nil {
f, err = lambdaClient.ListFunctions(context.Background(), &lambda.ListFunctionsInput{Marker: continuationToken})
if err != nil {
return nil, err
}
continuationToken = f.NextMarker
for _, functionName := range f.Functions {
functions = append(functions, *functionName.FunctionName)
}
}
return functions, nil
}
func DescribeLambda(lambdaName string) (*lambda.GetFunctionOutput, error) {
function, err := lambdaClient.GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String(lambdaName)})
if err != nil {
return nil, err
}
return function, nil
}
func ListTags(lambdaARN string) (*lambda.ListTagsOutput, error) {
return lambdaClient.ListTags(context.Background(), &lambda.ListTagsInput{
Resource: aws.String(lambdaARN),
})
}
Then you can use the ListLambdas method in order to list all your lambda. After, you can iterate the slice returned and call the DescribeLambda method in order to get the lambdaARN, then you can call the ListTags.
You can refer to the following repository in order to understand how to work with AWS (lambda, S3, glue etc etc) in Golang: https://github.com/alessiosavi/GoGPUtils/tree/master/aws
Trying to upload a picture to Google Cloud Platform, I always get the same err "<?xml version='1.0' encoding='UTF-8'?><Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.</Message><StringToSign>GOOG4-RSA-SHA256 20.................951Z".
I did add a service-account to the bucket with the role Storage Admin and Storage Object Admin as you can see on the pic
I have generated a Key(for the service account) and downloaded it as .json file, then I generate a presignURL using this code:
// key is the downloaded .json key file from the GCP service-account
// the return string is the presignedURL
func getPresignedURL(path, key string) (string, error) {
sakeyFile := filepath.Join(path, key)
saKey, err := ioutil.ReadFile(sakeyFile)
if err != nil {
log.Fatalln(err)
}
cfg, err := google.JWTConfigFromJSON(saKey)
if err != nil {
log.Fatalln(err)
}
bucket := "mybucket"
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return "", fmt.Errorf("storage.NewClient: %v", err)
}
defer client.Close()
opts := &storage.SignedURLOptions{
Scheme: storage.SigningSchemeV4,
Method: "PUT",
Headers: []string{
"Content-Type:image/jpeg",
},
Expires: time.Now().Add(15 * time.Minute),
GoogleAccessID: cfg.Email,
PrivateKey: cfg.PrivateKey,
}
u, err := client.Bucket(bucket).SignedURL("mypic.jpeg", opts)
if err != nil {
return "", fmt.Errorf("Bucket(%q).SignedURL: %v", bucket, err)
}
return u, nil
}
The presignedURL looks good, something like this:
https://storage.googleapis.com/djedjepicbucket/mypic.jpeg?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=djedje%40picstorage-363707.iam.gserviceaccount.com%2F20220926%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20220926T081951Z&X-Goog-Expires=899&X-Goog Signature=3f330715d7a38ea08f99134a16f464fb............5ad800a7665dfb1440034ab1f5ab045252336&X-Goog-SignedHeaders=content-type%3Bhost
Then I read a file(picture) from disk and upload it using the presignURL
// the uri is the presignedURL
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
if err != nil {
return nil, err
}
_, err = io.Copy(body, file)
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
req, err := http.NewRequest("PUT", uri, body)
req.Header.Set("Content-Type", "image/jpeg")
return req, err
}
Then I exec the request
// the previous func
request, err := newfileUploadRequest(purl, extraParams, "picture", filepath.Join(path, "download.jpeg"))
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Fatal(err)
}
resp.Body.Close()
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
fmt.Println(body)
}
Unfortunatly, I always get the same error back
403
map[Alt-Svc:[h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"] Content-Length:[884] Content-Type:[application/xml; charset=UTF-8] Date:[Mon, 26 Sep 2022 08:22:19 GMT] Server:[UploadServer] X-Guploader-Uploadid:[ADPyc......................ECL_4W]]
<?xml version='1.0' encoding='UTF-8'?><Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.</Message><StringToSign>GOOG4-RSA-SHA256
20220926T081951Z
20220926/auto/storage/goog4_request
c5f36838af4......................8ffb56329c1eb27f</StringToSign><CanonicalRequest>PUT
/djedjepicbucket/mypic.jpeg
X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=djedje%40picstorage-363707.iam.gserviceaccount.com%2F20220926%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20220926T081951Z&X-Goog-Expires=899&X-Goog-SignedHeaders=content-type%3Bhost
content-type:multipart/form-data; boundary=5be13cc........................dd6aef6823
host:storage.googleapis.com
content-type;host
UNSIGNED-PAYLOAD</CanonicalRequest></Error>
Actually I have tryied many other ways as well but I basically always get this(more or less) same err back, Does someone have an Idea what am I forgetting(I am on that for 2 days now...) ? Thank you
=============================
I edited the question, that code works perfectly
Found the answer, in the both getPresignedURL() and newfileUploadRequest() func, the Header must be set to "Content-Type:application/octet-stream"(or "Content-Type:image/jpeg" for instance if the picture need to be display using its URL), then the pic is uploaded without issue.
I have a middleware to log this service access. But I'm confused to do the unit testing several times I surfed googling. I have not found the right way to solve this
package accesslog
import (
"net/http"
"time"
"github.com/go-chi/chi/middleware"
"transactionService/pkg/log"
)
func Handler(logger log.Logger) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = log.WithRequest(ctx, r)
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
start := time.Now()
defer func() {
logger.With(ctx, "duration", time.Since(start), "status", ww.Status()).
Infof("%s %s %s %d %d", r.Method, r.URL.Path, r.Proto, ww.Status(), ww.BytesWritten())
}()
next.ServeHTTP(ww, r.WithContext(ctx))
}
return http.HandlerFunc(fn)
}
}
solved, this is my code to solve it
package accesslog
import (
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/go-chi/chi"
"transactionService/pkg/log"
)
func TestHandler(t *testing.T) {
logger, _ := log.NewForTest()
r := chi.NewRouter()
r.Use(Handler(logger))
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("test"))
})
ts := httptest.NewServer(r)
defer ts.Close()
if resp, body := testRequest(t, ts, "GET", "/", nil); body != "root" && resp.StatusCode != 200 {
t.Fatalf(body)
}
}
func testRequest(t *testing.T, ts *httptest.Server, method, path string, body io.Reader) (*http.Response, string) {
req, err := http.NewRequest(method, ts.URL+path, body)
if err != nil {
t.Fatal(err)
return nil, ""
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
return nil, ""
}
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
return nil, ""
}
defer resp.Body.Close()
return resp, string(respBody)
}
I'm getting 403 Forbidden HTTP response when I make a GET request to a presigned url generated by the AWS Presign method in Go.
The error message is:
The request signature we calculated does not match the signature you provided. Check your key and signing method
X-Amz-SignedHeaders is: host;x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5
I'm writing the object to S3 like this:
type DocumentStore struct {
bucketName string
bucketEncryptionKeyAlias string
aws *session.Session
}
func (s *DocumentStore) PutDocument(ctx context.Context, envelope []byte, metadata map[string]string) (PutDocumentResult, error) {
uploader := s3manager.NewUploader(s.aws)
var objectKey = uuid.New().String()
if _, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
ContentType: aws.String("application/octet-stream"),
Body: bytes.NewReader(envelope),
ServerSideEncryption: aws.String(s3.ServerSideEncryptionAwsKms),
SSEKMSKeyId: aws.String(s.bucketEncryptionKeyAlias),
Metadata: aws.StringMap(metadata),
}); err != nil {
return PutDocumentResult{}, fmt.Errorf("put document failed on upload: %v", err.Error())
}
return PutDocumentResult{
BucketName: s.bucketName,
ObjectKey: objectKey,
}, nil
}
I'm signing the url like this:
func (s *DocumentStore) NewSignedGetURL(ctx context.Context, objectKey string, ttl time.Duration) (string, error) {
svc := s3.New(s.aws)
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
SSECustomerKey: aws.String(s.bucketEncryptionKeyAlias),
SSECustomerAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
})
url, err := req.Presign(ttl)
if err != nil {
return "", fmt.Errorf("failed to presign GetObjectRequest for key %q: %v", objectKey, err)
}
return url, nil
}
And I'm calling the methods like this:
result, err := target.PutDocument(context.TODO(), envelope, metadata)
if err != nil {
t.Errorf("PutDocument failed: %v", err)
return
}
getURL, err := target.NewSignedGetURL(context.TODO(), result.ObjectKey, time.Minute*5)
if err != nil {
t.Errorf("failed to sign url: %v", err)
return
}
req, _ := http.NewRequest("GET", getURL, nil)
req.Header.Add("x-amz-server-side-encryption-customer-algorithm", s3.ServerSideEncryptionAwsKms)
req.Header.Add("x-amz-server-side-encryption-customer-key", test.cfg.AWS.BucketKMSAlias)
req.Header.Add("x-amz-server-side-encryption-customer-key-md5", "")
resp, err := http.DefaultClient.Do(req.WithContext(context.TODO()))
if err != nil {
t.Errorf("failed to request object from signed url: %v", err)
return
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("failed to read object stream from S3: %v", err)
return
}
if resp.StatusCode != http.StatusOK {
t.Errorf("failed to get object. Http status: %s(%d)\n%s", resp.Status, resp.StatusCode, data)
return
}
I can read the download the file from the aws cli like this:
aws --profile dispatcher_stage --region us-east-1 s3 cp s3://[bucket-name]/0c/09179312-e283-431c-ab71-6a0c437177fe . --sse aws:kms --sse-kms-key-id alias/[key-alias-name]
What am I missing?
I figured it out: the GetObject request doesn't need the SSE parameters as long as the user has Decrypt permission on the KMS key. Here's the relevant changes:
I'm now signing the url like this:
func (s *DocumentStore) NewSignedGetURL(ctx context.Context, objectKey string, ttl time.Duration) (string, error) {
svc := s3.New(s.aws)
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
})
url, err := req.Presign(ttl)
if err != nil {
return "", fmt.Errorf("failed to presign GetObjectRequest for key %q: %v", objectKey, err)
}
return url, nil
}
And I'm downloading the object like this:
getURL, err := target.NewSignedGetURL(context.TODO(), result.ObjectKey, time.Minute*5)
if err != nil {
t.Errorf("failed to sign url: %v", err)
return
}
req, _ := http.NewRequest("GET", getURL, nil)
req.Header.Add("host", req.Host)
resp, err := http.DefaultClient.Do(req.WithContext(context.TODO()))
if err != nil {
t.Errorf("failed to request object from signed url: %v", err)
return
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("failed to read object stream from S3: %v", err)
return
}
I'm trying to download Objects from S3, the following is my code:
func listFile(bucket, prefix string) error {
svc := s3.New(sess)
params := &s3.ListObjectsInput{
Bucket: aws.String(bucket), // Required
Prefix: aws.String(prefix),
}
return svc.ListObjectsPages(params, func(p *s3.ListObjectsOutput, lastPage bool) bool {
for _, o := range p.Contents {
//log.Println(*o.Key)
log.Println(*o.Key)
download(bucket, *o.Key)
return true
}
return lastPage
})
}
func download(bucket, key string) {
logDir := conf.Cfg.Section("share").Key("LOG_DIR").MustString(".")
tmpLogPath := filepath.Join(logDir, bucket, key)
s3Svc := s3.New(sess)
downloader := s3manager.NewDownloaderWithClient(s3Svc, func(d *s3manager.Downloader) {
d.PartSize = 2 * 1024 * 1024 // 2MB per part
})
f, err := os.OpenFile(tmpLogPath, os.O_CREATE|os.O_WRONLY, 0644)
if _, err = downloader.Download(f, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}); err != nil {
log.Fatal(err)
}
f.Close()
}
func main() {
bucket := "mybucket"
key := "myprefix"
listFile(bucket, key)
}
I can get the objects list in the function listFile(), but a 404 returned when call download, why?
I had the same problem with recent versions of the library. Sometimes, the object key will be prefixed with a "./" that the SDK will remove by default making the download fail.
Try adding this to your aws.Config and see if it helps:
config := aws.Config{
...
DisableRestProtocolURICleaning: aws.Bool(true),
}
I submitted an issue.