Unable to perform query on AWS athena using Golang SDK - amazon-web-services

I am new to AWS and Golang, and I am trying to create a lambda function, which will trigger AWS Athena query and email the result using AWS SES service. Even after searching for an hour, I couldn't find a working example of lambda function (in Golang) to perform a query on Athena and getting the output of the query.
While searching, I found code for the same in Java, Python and Node Js, but not in Golang.
Even the Go-SDK page redirects to Java example. But unfortunately, I don't even understand Java.
I have also looked into this AWS SDK for Go API Reference page. But I don't understand what is the flow of the program and which operation to select.
I have tried to create the program for this, this may be completely wrong, and I don't know what to do next. Below is the code -
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/athena"
)
func main() {
// Create a new session in the us-west-2 region.
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
// Create an Athena session.
client := athena.New(sess)
// Example sending a request using the StartQueryExecutionRequest method.
query := "SELECT * FROM table1 ;"
params := query
req, resp := client.StartQueryExecutionRequest(params)
err1 := req.Send()
if err1 == nil { // resp is now filled
fmt.Println(resp)
}
}
Appreciate if someone can help me to perform an Athena query and to get its result in Golang(Preferably) or can share some resource. Once I get it, I can then send an email using AWS SES.

Use this to get started.
// run as: go run main.go
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "textqldb.textqltable"
const outputBucket = "s3://bucket-name-here/"
func main() {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
return
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
query := "select * from " + table
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("query error: %v\n", err)
return
}
fmt.Println(resp)
}

#Everton's code is executing a query on Athena, and its responses are getting saved on S3 bucket and not getting returned. So, I have added the code to execute the Athena query and get the response back. Hope this may help others.
// run as: go run main.go
package main
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "<Database_Name>.<Table_Name>"
const outputBucket = "s3://bucket-name-here/"
// Execute the query and return the query ID
func executeQuery(query string) *string {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
fmt.Println("Response is: ", resp, " Error is:", err)
if err != nil {
fmt.Printf("Query Error: %v\n", err)
}
fmt.Println("Query Execution Response ID:", resp.QueryExecutionId)
return resp.QueryExecutionId
}
// Takes queryId as input and returns its response
func getQueryResults(QueryID *string) (*athena.GetQueryResultsResponse, error) {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
panic("config error")
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
params1 := &athena.GetQueryResultsInput{
QueryExecutionId: QueryID,
}
req := client.GetQueryResultsRequest(params1)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("Query Response Error: %v\n", err)
return nil, err
}
return resp, nil
}
func main() {
query := "select * from " + table
// Execute an Athena Query
QueryID := executeQuery(query)
// Get the response of the query
// Wait for some time for query completion
time.Sleep(15 * time.Second) // Otherwise create a loop and try for every x seconds
Resp, err := getQueryResults(QueryID)
if err != nil {
fmt.Printf("Error getting Query Response: %v\n", err)
} else {
fmt.Println(" \nRows:", Resp.ResultSet.Rows)
}
}

Related

How to fix error with AWS Lambda handler, DynamoDB Put req?

Trying to create a Lambda to interact with my DynamoDB.
This specific Lambda is to put/write an item to the DB:
package main
import (
"fmt"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
)
type Item struct {
Email string `json:"email"`
Password string `json:"password"`
Rname string `json:"rname"`
}
func Put() error {
// Create a session - London Region
session, err := session.NewSession(&aws.Config{
Region: aws.String("eu-west-2")},
)
if err != nil {
fmt.Println(err)
}
svc := dynamodb.New(session)
// Create instance of Item Struct
item := Item{
Email: "123#mail.com",
Password: "12345678",
Rname: "abcde",
}
// Marshall Item
av, err := dynamodbattribute.MarshalMap(item)
if err != nil {
fmt.Println("Got error marshalling map:")
fmt.Println(err)
}
// Create Item
input := &dynamodb.PutItemInput{
Item: av,
TableName: aws.String("accountsTable"),
}
_, err = svc.PutItem(input)
if err != nil {
fmt.Println("Got error calling PutItem:")
fmt.Println(err)
}
return err
}
func main() {
lambda.Start(Put())
}
However getting the error:
{
"errorMessage": "handler is nil",
"errorType": "errorString"
}
I have changed the handler in run time settings to main too so don't think that would be the issue.
Building with:
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -a main.go
and putting the zip of the executable into AWS via console (no IAC's)
Any help would be greatly appreciated to resolve this error! Thanks.
You need to pass function handle not function result to lambda.Start
Please update your main function with👇
func main() {
lambda.Start(Put)
}

Sending a POST Request to external API from GCP cloud function returns 500 but not when sent locally

I'm currently trying to send a POST request to an external API from a GCP Cloud Function. I've tested the function extensively locally and it fulfills the request every time and also works from Postman, but when I run the exact same code from within a cloud function, it returns a 500 from the external API every single time.
I'm genuinely at a loss as to why when sending the POST request from within the cloud function it fails every single time.
Does GCP add any headers that might interfere with an external API call or is there a configuration option within the cloud function settings that needs to be configured to allow an external POST request?
I've attempted to implement an http retry mechanism, but that did not work either.
Again, locally and from Postman, the exact same code is successful every time I run it.
Here is the code I use to generate and send the request:
package email
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/hashicorp/go-retryablehttp"
)
var FailedRequestErr = errors.New("failed request to moosend")
const (
successCode = 0
moosendHost = "api.moosend.com/v3"
dailyNewsletterMailingListID = "2e461f4c-99d1-4a8e-80ea-168b20bdaf5f"
mainEmail = "jason#functionalbits.io"
campaignNameBase = "Functional Bits Newsletter - Issue"
campaignSubjectBase = "Functional Bits Issue"
)
type CreatingADraftCampaignRequest struct {
Name string `json:"Name"`
Subject string `json:"Subject"`
SenderEmail string `json:"SenderEmail"`
ReplyToEmail string `json:"ReplyToEmail"`
IsAB string `json:"IsAB"`
ConfirmationToEmail string `json:"ConfirmationToEmail,omitempty"`
WebLocation string `json:"WebLocation,omitempty"`
MailingLists []MailingLists `json:"MailingLists,omitempty"`
SegmentID string `json:"SegmentID,omitempty"`
ABCampaignType string `json:"ABCampaignType,omitempty"`
TrackInGoogleAnalytics string `json:"TrackInGoogleAnalytics,omitempty"`
DontTrackLinkClicks string `json:"DontTrackLinkClicks,omitempty"`
SubjectB string `json:"SubjectB,omitempty"`
WebLocationB string `json:"WebLocationB,omitempty"`
SenderEmailB string `json:"SenderEmailB,omitempty"`
HoursToTest string `json:"HoursToTest,omitempty"`
ListPercentage string `json:"ListPercentage,omitempty"`
ABWinnerSelectionType string `json:"ABWinnerSelectionType,omitempty"`
}
type MailingLists struct {
MailingListID string `json:"MailingListId"`
SegmentID float64 `json:"SegmentId,omitempty"`
}
type CampaignResponse struct {
Code int32 `json:"Code"`
Err interface{} `json:"Error"`
Context interface{} `json:"Context"`
}
type MoosendAPI struct {
apiKey string
client *http.Client
}
func NewMoosendAPI(apiKey string) *MoosendAPI {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
standardClient := retryClient.StandardClient()
return &MoosendAPI{
apiKey: apiKey,
client: standardClient,
}
}
func (m *MoosendAPI) CreateDraftCampaign(issueNumber string, webLocation string) (*CampaignResponse, error) {
campaign := CreatingADraftCampaignRequest{
Name: fmt.Sprintf("%s %s", campaignNameBase, issueNumber),
Subject: fmt.Sprintf("%s %s", campaignSubjectBase, issueNumber),
IsAB: "false",
WebLocation: webLocation,
MailingLists: []MailingLists{{MailingListID: dailyNewsletterMailingListID}},
SenderEmail: mainEmail,
ReplyToEmail: mainEmail,
ConfirmationToEmail: mainEmail,
TrackInGoogleAnalytics: "true",
}
body, err := json.Marshal(&campaign)
if err != nil {
log.Println("error marshalling campaign request")
return nil, err
}
fullURL := fmt.Sprintf("https://%s/campaigns/create.json?apikey=%s", moosendHost, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, bytes.NewBuffer(body))
if err != nil {
log.Println("request error")
return nil, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
log.Printf("request: %+v", req)
resp, err := m.client.Do(req)
if resp.StatusCode != http.StatusOK {
return nil, FailedRequestErr
}
if err != nil {
log.Println("error sending request")
return nil, err
}
log.Printf("response: %+v", resp)
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return nil, err
}
var draftResponse CampaignResponse
if err := json.Unmarshal(respBody, &draftResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", draftResponse)
return nil, err
}
return &draftResponse, nil
}
func (m *MoosendAPI) SendCampaign(campaignID string) error {
fullURL := fmt.Sprintf("https://%s/campaigns/%s/send.json?apikey=%s", moosendHost, campaignID, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, nil)
if err != nil {
log.Println("error creating request")
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
resp, err := m.client.Do(req)
if err != nil {
log.Println("error sending request")
return err
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return err
}
var sendResponse CampaignResponse
if err := json.Unmarshal(respBody, &sendResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", sendResponse)
return err
}
return nil
}
Then how it's run in the main function code:
package function
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"github.com/Functional-Bits/emailer-service/internal/email"
"github.com/Functional-Bits/emailer-service/internal/publish"
)
func CampaignGenerator(ctx context.Context, m publish.PubSubMessage) error {
moosendAPIKey, ok := os.LookupEnv("MOOSEND_API_KEY")
if !ok {
log.Println("missing moosendAPIKey")
}
mAPI := email.NewMoosendAPI(moosendAPIKey)
var msg publish.IncomingMessage
if err := json.Unmarshal(m.Data, &msg); err != nil {
log.Println(err)
return err
}
log.Printf("received message: %+v", msg)
log.Printf("generating draft campaign for issue %s", msg.IssueNumber)
draftResponse, err := mAPI.CreateDraftCampaign(msg.IssueNumber, msg.FileURL)
if err != nil {
log.Println(err)
return err
}
log.Printf("draft response: %+v", draftResponse)
campaignID, ok := draftResponse.Context.(string)
if !ok {
log.Printf("response didn't contain an ID: %+v", draftResponse)
return errors.New("no campaign generated")
}
log.Printf("sending campgain %s", campaignID)
if err := mAPI.SendCampaign(campaignID); err != nil {
log.Println(err)
return err
}
log.Printf("campaign successfully sent for issue number %s", msg.IssueNumber)
return nil
}
When this code is run locally, It correctly makes the 2 calls and sends an email campaign. When run from the cloud function I get a 500 internal server error with no additional information as to why. Link to API docs.
I get the following response from the external API (from my cloud function logs)
response: &{
Status:500 Internal Server Error
StatusCode:500
Proto:HTTP/1.1
ProtoMajor:1
ProtoMinor:1
Header:map[Access-Control-Allow-Headers:[Content-Type, Accept, Cache-Control, X-Requested-With]
Access-Control-Allow-Methods:[GET, POST, OPTIONS, DELETE, PUT]
Access-Control-Allow-Origin:[*]
Cache-Control:[private]
Content-Length:[12750]
Content-Type:[text/html; charset=utf-8]
Date:[Sun, 12 Dec 2021 07:00:09 GMT]
Server:[Microsoft-IIS/10.0]
X-Aspnet-Version:[4.0.30319]
X-Powered-By:[ASP.NET]
X-Robots-Tag:[noindex, nofollow]
X-Server-Id:[1]]
Body:0xc0003f04c0
ContentLength:12750
TransferEncoding:[]
Close:false
Uncompressed:false
Trailer:map[]
Request:0xc000160b00
TLS:0xc000500630
}
The response causes an unmarshal error because no campaign ID is returned.

Golang AWS Firehose using json field name instead of struct field name

I have the following struct:
type ProcessedRecords struct {
CustIndividualID string `json:"individual id"`
Household string `json:"Household"`
}
And I have a slice of many structs that share this value. I'm trying to submit them using the PutRecordBatch operation from the AWS SDK:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/firehose"
)
type ProcessedRecords struct {
CustIndividualID string `json:"individual id"`
Household string `json:"Household"`
}
func main() {
submitToFirehose(recordList)
}
func submitToFirehose(records []ProcessedRecords) {
streamName := "processed-stream"
sess := session.Must(session.NewSession())
// Create a Firehose client with additional configuration
firehoseService := firehose.New(sess, aws.NewConfig().WithRegion("us-east-1"))
recordsBatchInput := &firehose.PutRecordBatchInput{}
recordsBatchInput = recordsBatchInput.SetDeliveryStreamName(streamName)
recordsInput := []*firehose.Record{}
for i := 0; i < len(records); i++ {
if len(recordsInput) == 500 {
recordsBatchInput = recordsBatchInput.SetRecords(recordsInput)
resp, err := firehoseService.PutRecordBatch(recordsBatchInput)
if err != nil {
fmt.Printf("PutRecordBatch err: %v\n", err)
} else {
fmt.Printf("FailedPuts: %v\n", *resp.FailedPutCount)
}
recordsInput = []*firehose.Record{}
}
b, err := json.Marshal(records[i])
if err != nil {
log.Printf("Error: %v", err)
}
record := &firehose.Record{Data: b}
recordsInput = append(recordsInput, record)
}
}
This seems to work and it would appear that my Glue backend is setup correctly, however CustIndividualID is not being written to S3. I suspect it's because it's reading the json:"individual id" as the column name and not the CustIndividualID.
this is a problem because glue tables can't have spaces in the column name. What am I doing wrong?

DynamoDB list all backups using AWS GoLang SDK

Based on the example given in the link blow on API Operation Pagination without Callbacks
https://aws.amazon.com/blogs/developer/context-pattern-added-to-the-aws-sdk-for-go/
I am trying to list all the Backups in dynamodb. But it seems like pagination is not working and it is just retrieving first page and not going to next page
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
func main() {
sess, sessErr := session.NewSession()
if sessErr != nil {
fmt.Println(sessErr)
fmt.Println("Cound not initilize session..returning..")
return
}
// Create DynamoDB client
dynamodbSvc := dynamodb.New(sess)
params := dynamodb.ListBackupsInput{}
ctx := context.Background()
p := request.Pagination{
NewRequest: func() (*request.Request, error) {
req, _ := dynamodbSvc.ListBackupsRequest(&params)
req.SetContext(ctx)
return req, nil
},
}
for p.Next() {
page := p.Page().(*dynamodb.ListBackupsOutput)
fmt.Println("Received", len(page.BackupSummaries), "objects in page")
for _, obj := range page.BackupSummaries {
fmt.Println(aws.StringValue(obj.BackupName))
}
}
//return p.Err()
} //end of main
Its a bit late but I'll just put it here in case I can help somebody.
Example:
var exclusiveStartARN *string
var backups []*dynamodb.BackupSummary
for {
backup, err := svc.ListBackups(&dynamodb.ListBackupsInput{
ExclusiveStartBackupArn:exclusiveStartARN,
})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
backups = append(backups, backup.BackupSummaries...)
if backup.LastEvaluatedBackupArn != nil {
exclusiveStartARN = backup.LastEvaluatedBackupArn
//max 5 times a second so we dont hit the limit
time.Sleep(200 * time.Millisecond)
continue
}
break
}
fmt.Println(len(backups))
Explaination:
The way that pagination is done is via ExclusiveStartBackupArn in the ListBackupsRequest. The ListBackupsResponse returns LastEvaluatedBackupArn if there are more pages, or nil if its the last/only page.
It could be that you're smashing into the API a bit with your usage
You can call ListBackups a maximum of 5 times per second.
What is the value of p.HasNextPage() in your p.Next() loop?

Fabric-sdk-go client.Query returns error: failed to create transactor: Channel_Cfg_Cache - cache is closed

I am trying to make a simple demo using the fabric-sdk-go. I wonder anybod know why the code below ("client.Query") returns the error "failed to create transactor: Channel_Cfg_Cache - cache is closed"?
func initSdkClient() (*channel.Client){
sdk, err := fabsdk.New(config.FromFile("config.yaml"))
if err != nil {
logger.Fatalf("Failed to create new SDK: %s", err)
}
defer sdk.Close()
//prepare channel client context using client context
clientChannelContext := sdk.ChannelContext(channelID, fabsdk.WithUser("User1"), fabsdk.WithOrg(orgName))
// Channel client is used to query and execute transactions (Org1 is default org)
client, err := channel.New(clientChannelContext)
if err != nil {
logger.Fatalf("Failed to create new channel client: %s", err)
}
return client
}
func queryCC(client *channel.Client, targetEndpoints ...string) []byte {
response, err := client.Query(channel.Request{ChaincodeID: ccID, Fcn: "invoke", Args: defaultQueryArgs},
channel.WithRetry(retry.DefaultChannelOpts),
channel.WithTargetEndpoints(targetEndpoints...),
)
if err != nil {
***logger.Fatalf("Failed to query funds: %s", err)*** // error: failed to create transactor: Channel_Cfg_Cache - cache is closed
}
return response.Payload
}
func main() {
client := initSdkClient()
existingValue := queryCC(client)
logger.Info(existingValue)
logger.Info("hello, world\n")
}
Probably because you're closing the SDK instance at the end of initSdkClient function. Rather close the SDK at the end of main function.
Do something like
type Setup struct {
sdk *fabsdk.FabricSDK
client *channel.Client
}
func (setup *Setup) initSdkClient() *channel.Client {
sdk, err := fabsdk.New(config.FromFile("config.yaml"))
if err != nil {
fmt.Errorf("Failed to create new SDK: %s", err)
}
setup.sdk = sdk
//prepare channel client context using client context
clientChannelContext := setup.sdk.ChannelContext(channelID, fabsdk.WithUser("User1"), fabsdk.WithOrg(orgName))
// Channel client is used to query and execute transactions (Org1 is default org)
client, err := channel.New(clientChannelContext)
if err != nil {
fmt.Errorf("Failed to create new channel client: %s", err)
}
setup.client = client
return client
}
func (setup *Setup) queryCC(client *channel.Client, targetEndpoints ...string) []byte {
response, err := setup.client.Query(channel.Request{ChaincodeID: ccID, Fcn: "invoke", Args: defaultQueryArgs},
channel.WithRetry(retry.DefaultChannelOpts),
channel.WithTargetEndpoints(targetEndpoints...),
)
if err != nil {
fmt.Errorf("Failed to query funds: %s", err)
}
return response.Payload
}
func main() {
var setup Setup
client := setup.initSdkClient()
defer setup.sdk.Close()
existingValue := setup.queryCC(client)
}