I have the following struct:
type ProcessedRecords struct {
CustIndividualID string `json:"individual id"`
Household string `json:"Household"`
}
And I have a slice of many structs that share this value. I'm trying to submit them using the PutRecordBatch operation from the AWS SDK:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/firehose"
)
type ProcessedRecords struct {
CustIndividualID string `json:"individual id"`
Household string `json:"Household"`
}
func main() {
submitToFirehose(recordList)
}
func submitToFirehose(records []ProcessedRecords) {
streamName := "processed-stream"
sess := session.Must(session.NewSession())
// Create a Firehose client with additional configuration
firehoseService := firehose.New(sess, aws.NewConfig().WithRegion("us-east-1"))
recordsBatchInput := &firehose.PutRecordBatchInput{}
recordsBatchInput = recordsBatchInput.SetDeliveryStreamName(streamName)
recordsInput := []*firehose.Record{}
for i := 0; i < len(records); i++ {
if len(recordsInput) == 500 {
recordsBatchInput = recordsBatchInput.SetRecords(recordsInput)
resp, err := firehoseService.PutRecordBatch(recordsBatchInput)
if err != nil {
fmt.Printf("PutRecordBatch err: %v\n", err)
} else {
fmt.Printf("FailedPuts: %v\n", *resp.FailedPutCount)
}
recordsInput = []*firehose.Record{}
}
b, err := json.Marshal(records[i])
if err != nil {
log.Printf("Error: %v", err)
}
record := &firehose.Record{Data: b}
recordsInput = append(recordsInput, record)
}
}
This seems to work and it would appear that my Glue backend is setup correctly, however CustIndividualID is not being written to S3. I suspect it's because it's reading the json:"individual id" as the column name and not the CustIndividualID.
this is a problem because glue tables can't have spaces in the column name. What am I doing wrong?
Related
Trying to create a Lambda to interact with my DynamoDB.
This specific Lambda is to put/write an item to the DB:
package main
import (
"fmt"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
)
type Item struct {
Email string `json:"email"`
Password string `json:"password"`
Rname string `json:"rname"`
}
func Put() error {
// Create a session - London Region
session, err := session.NewSession(&aws.Config{
Region: aws.String("eu-west-2")},
)
if err != nil {
fmt.Println(err)
}
svc := dynamodb.New(session)
// Create instance of Item Struct
item := Item{
Email: "123#mail.com",
Password: "12345678",
Rname: "abcde",
}
// Marshall Item
av, err := dynamodbattribute.MarshalMap(item)
if err != nil {
fmt.Println("Got error marshalling map:")
fmt.Println(err)
}
// Create Item
input := &dynamodb.PutItemInput{
Item: av,
TableName: aws.String("accountsTable"),
}
_, err = svc.PutItem(input)
if err != nil {
fmt.Println("Got error calling PutItem:")
fmt.Println(err)
}
return err
}
func main() {
lambda.Start(Put())
}
However getting the error:
{
"errorMessage": "handler is nil",
"errorType": "errorString"
}
I have changed the handler in run time settings to main too so don't think that would be the issue.
Building with:
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -a main.go
and putting the zip of the executable into AWS via console (no IAC's)
Any help would be greatly appreciated to resolve this error! Thanks.
You need to pass function handle not function result to lambda.Start
Please update your main function with👇
func main() {
lambda.Start(Put)
}
Thanks in advance :) . I'm using the following code to get metadata from an s3 object after listing all the object in a bucket . But I don't know why it gives the error undefined: s3.HeadObject when running go run listObjects.go -bucket xxxx -prefix xxxx
I tried two solutions: giving the client as the one created from the config and from the context as in this link appears [1]. BUt both gave the same error. Can you give me any clue?
package main
import (
"context"
"flag"
"fmt"
"log"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
var (
bucketName string
objectPrefix string
objectDelimiter string
maxKeys int
)
func init() {
flag.StringVar(&bucketName, "bucket", "", "The `name` of the S3 bucket to list objects from.")
flag.StringVar(&objectPrefix, "prefix", "", "The optional `object prefix` of the S3 Object keys to list.")
flag.StringVar(&objectDelimiter, "delimiter", "",
"The optional `object key delimiter` used by S3 List objects to group object keys.")
flag.IntVar(&maxKeys, "max-keys", 0,
"The maximum number of `keys per page` to retrieve at once.")
}
// Lists all objects in a bucket using pagination
func main() {
flag.Parse()
if len(bucketName) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, bucket name required")
}
// Load the SDK's configuration from environment and shared config, and
// create the client with this.
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
log.Fatalf("failed to load SDK configuration, %v", err)
}
client := s3.NewFromConfig(cfg)
// Set the parameters based on the CLI flag inputs.
params := &s3.ListObjectsV2Input{
Bucket: &bucketName,
}
if len(objectPrefix) != 0 {
params.Prefix = &objectPrefix
}
if len(objectDelimiter) != 0 {
params.Delimiter = &objectDelimiter
}
// Create the Paginator for the ListObjectsV2 operation.
p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) {
if v := int32(maxKeys); v != 0 {
o.Limit = v
}
})
// Iterate through the S3 object pages, printing each object returned.
var i int
log.Println("Objects:")
for p.HasMorePages() {
i++
// Next Page takes a new context for each page retrieval. This is where
// you could add timeouts or deadlines.
page, err := p.NextPage(context.TODO())
if err != nil {
log.Fatalf("failed to get page %v, %v", i, err)
}
// Log the objects found
// Headobject function is called
for _, obj := range page.Contents {
input := &s3.HeadObjectInput{
Bucket: &bucketName,
Key: obj.Key,
}
result, err := &s3.HeadObject(client, input)
if err != nil {
panic(err)
}
fmt.Println("Object:", *obj.Key)
}
}
}
./listObjects.go:86:20: undefined: s3.HeadObject
1
Doing the headObject as an auxiliary method works
package main
import (
"context"
"flag"
"fmt"
"log"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
var (
bucketName string
objectPrefix string
objectDelimiter string
maxKeys int
)
func init() {
flag.StringVar(&bucketName, "bucket", "", "The `name` of the S3 bucket to list objects from.")
flag.StringVar(&objectPrefix, "prefix", "", "The optional `object prefix` of the S3 Object keys to list.")
flag.StringVar(&objectDelimiter, "delimiter", "",
"The optional `object key delimiter` used by S3 List objects to group object keys.")
flag.IntVar(&maxKeys, "max-keys", 0,
"The maximum number of `keys per page` to retrieve at once.")
}
// Lists all objects in a bucket using pagination
func main() {
flag.Parse()
if len(bucketName) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, bucket name required")
}
// Load the SDK's configuration from environment and shared config, and
// create the client with this.
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
log.Fatalf("failed to load SDK configuration, %v", err)
}
client := s3.NewFromConfig(cfg)
// Set the parameters based on the CLI flag inputs.
params := &s3.ListObjectsV2Input{
Bucket: &bucketName,
}
if len(objectPrefix) != 0 {
params.Prefix = &objectPrefix
}
if len(objectDelimiter) != 0 {
params.Delimiter = &objectDelimiter
}
// Create the Paginator for the ListObjectsV2 operation.
p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) {
if v := int32(maxKeys); v != 0 {
o.Limit = v
}
})
// Iterate through the S3 object pages, printing each object returned.
var i int
log.Println("Objects:")
for p.HasMorePages() {
i++
// Next Page takes a new context for each page retrieval. This is where
// you could add timeouts or deadlines.
page, err := p.NextPage(context.TODO())
if err != nil {
log.Fatalf("failed to get page %v, %v", i, err)
}
// Log the objects found
// Headobject function is called
for _, obj := range page.Contents {
fmt.Println("Object:", *obj.Key)
OpHeadObject(client, bucketName, *obj.Key)
}
}
}
func OpHeadObject(sess *s3.Client, bucketName, objectName string) {
input := &s3.HeadObjectInput{
Bucket: &bucketName,
Key: &objectName,
}
resp, err := sess.HeadObject(context.TODO(), input)
if err != nil {
panic(err)
}
fmt.Println(resp.StorageClass) // that you want.
}
Is there a way to return test page values returned from the AWS API paginators to test the code below? If not, I suppose it's better to split the tag checking into a function that can be tested in isolation?
Note: This is just an example, I realize there are input Filters on the I can apply to the API call to achieve the same thing demonstrated here.
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
)
type handler struct {
EC2 ec2iface.EC2API
}
func main() {
sess := session.New()
client := ec2.New(sess)
h := &handler{EC2: client}
tagged, err := h.findTagged()
if err != nil {
panic(err)
}
fmt.Println(tagged)
}
func (h *handler) findTagged() ([]string, error) {
defaults := []string{}
input := &ec2.DescribeVpcsInput{}
err := h.EC2.DescribeVpcsPages(input, func(page *ec2.DescribeVpcsOutput, lastPage bool) bool {
for _, p := range page.Vpcs {
for _, t := range p.Tags {
if aws.StringValue(t.Key) == "test" {
defaults = append(defaults, aws.StringValue(p.VpcId))
}
}
}
return false
})
return defaults, err
}
This is described on the official documentation (Unit Testing with the AWS SDK for Go V2 - How to mock the AWS SDK for Go V2 when unit testing your application
Extract from the page:
import "context"
import "fmt"
import "testing"
import "github.com/aws/aws-sdk-go-v2/service/s3"
// ...
type mockListObjectsV2Pager struct {
PageNum int
Pages []*s3.ListObjectsV2Output
}
func (m *mockListObjectsV2Pager) HasMorePages() bool {
return m.PageNum < len(m.Pages)
}
func (m *mockListObjectsV2Pager) NextPage(ctx context.Context, f ...func(*s3.Options)) (output *s3.ListObjectsV2Output, err error) {
if m.PageNum >= len(m.Pages) {
return nil, fmt.Errorf("no more pages")
}
output = m.Pages[m.PageNum]
m.PageNum++
return output, nil
}
func TestCountObjects(t *testing.T) {
pager := &mockListObjectsV2Pager{
Pages: []*s3.ListObjectsV2Output{
{
KeyCount: 5,
},
{
KeyCount: 10,
},
{
KeyCount: 15,
},
},
}
objects, err := CountObjects(context.TODO(), pager)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if expect, actual := 30, objects; expect != actual {
t.Errorf("expect %v, got %v", expect, actual)
}
}
I am new to AWS and Golang, and I am trying to create a lambda function, which will trigger AWS Athena query and email the result using AWS SES service. Even after searching for an hour, I couldn't find a working example of lambda function (in Golang) to perform a query on Athena and getting the output of the query.
While searching, I found code for the same in Java, Python and Node Js, but not in Golang.
Even the Go-SDK page redirects to Java example. But unfortunately, I don't even understand Java.
I have also looked into this AWS SDK for Go API Reference page. But I don't understand what is the flow of the program and which operation to select.
I have tried to create the program for this, this may be completely wrong, and I don't know what to do next. Below is the code -
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/athena"
)
func main() {
// Create a new session in the us-west-2 region.
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
// Create an Athena session.
client := athena.New(sess)
// Example sending a request using the StartQueryExecutionRequest method.
query := "SELECT * FROM table1 ;"
params := query
req, resp := client.StartQueryExecutionRequest(params)
err1 := req.Send()
if err1 == nil { // resp is now filled
fmt.Println(resp)
}
}
Appreciate if someone can help me to perform an Athena query and to get its result in Golang(Preferably) or can share some resource. Once I get it, I can then send an email using AWS SES.
Use this to get started.
// run as: go run main.go
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "textqldb.textqltable"
const outputBucket = "s3://bucket-name-here/"
func main() {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
return
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
query := "select * from " + table
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("query error: %v\n", err)
return
}
fmt.Println(resp)
}
#Everton's code is executing a query on Athena, and its responses are getting saved on S3 bucket and not getting returned. So, I have added the code to execute the Athena query and get the response back. Hope this may help others.
// run as: go run main.go
package main
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "<Database_Name>.<Table_Name>"
const outputBucket = "s3://bucket-name-here/"
// Execute the query and return the query ID
func executeQuery(query string) *string {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
fmt.Println("Response is: ", resp, " Error is:", err)
if err != nil {
fmt.Printf("Query Error: %v\n", err)
}
fmt.Println("Query Execution Response ID:", resp.QueryExecutionId)
return resp.QueryExecutionId
}
// Takes queryId as input and returns its response
func getQueryResults(QueryID *string) (*athena.GetQueryResultsResponse, error) {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
panic("config error")
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
params1 := &athena.GetQueryResultsInput{
QueryExecutionId: QueryID,
}
req := client.GetQueryResultsRequest(params1)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("Query Response Error: %v\n", err)
return nil, err
}
return resp, nil
}
func main() {
query := "select * from " + table
// Execute an Athena Query
QueryID := executeQuery(query)
// Get the response of the query
// Wait for some time for query completion
time.Sleep(15 * time.Second) // Otherwise create a loop and try for every x seconds
Resp, err := getQueryResults(QueryID)
if err != nil {
fmt.Printf("Error getting Query Response: %v\n", err)
} else {
fmt.Println(" \nRows:", Resp.ResultSet.Rows)
}
}
Based on the example given in the link blow on API Operation Pagination without Callbacks
https://aws.amazon.com/blogs/developer/context-pattern-added-to-the-aws-sdk-for-go/
I am trying to list all the Backups in dynamodb. But it seems like pagination is not working and it is just retrieving first page and not going to next page
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
func main() {
sess, sessErr := session.NewSession()
if sessErr != nil {
fmt.Println(sessErr)
fmt.Println("Cound not initilize session..returning..")
return
}
// Create DynamoDB client
dynamodbSvc := dynamodb.New(sess)
params := dynamodb.ListBackupsInput{}
ctx := context.Background()
p := request.Pagination{
NewRequest: func() (*request.Request, error) {
req, _ := dynamodbSvc.ListBackupsRequest(¶ms)
req.SetContext(ctx)
return req, nil
},
}
for p.Next() {
page := p.Page().(*dynamodb.ListBackupsOutput)
fmt.Println("Received", len(page.BackupSummaries), "objects in page")
for _, obj := range page.BackupSummaries {
fmt.Println(aws.StringValue(obj.BackupName))
}
}
//return p.Err()
} //end of main
Its a bit late but I'll just put it here in case I can help somebody.
Example:
var exclusiveStartARN *string
var backups []*dynamodb.BackupSummary
for {
backup, err := svc.ListBackups(&dynamodb.ListBackupsInput{
ExclusiveStartBackupArn:exclusiveStartARN,
})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
backups = append(backups, backup.BackupSummaries...)
if backup.LastEvaluatedBackupArn != nil {
exclusiveStartARN = backup.LastEvaluatedBackupArn
//max 5 times a second so we dont hit the limit
time.Sleep(200 * time.Millisecond)
continue
}
break
}
fmt.Println(len(backups))
Explaination:
The way that pagination is done is via ExclusiveStartBackupArn in the ListBackupsRequest. The ListBackupsResponse returns LastEvaluatedBackupArn if there are more pages, or nil if its the last/only page.
It could be that you're smashing into the API a bit with your usage
You can call ListBackups a maximum of 5 times per second.
What is the value of p.HasNextPage() in your p.Next() loop?