I want to log only error request and retries request/response.
I tried using
cfg := dax.DefaultConfig()
cfg.HostPorts = []string{daxConfig.URL}
cfg.Region = daxConfig.Region
cfg.LogLevel = aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors
But this doesn't log the request body or the response
How to log the request and response body ?
Create a custom logger with the any name you want I have used customErrorLogger that implements the 'aws.logger' interface. 'aws.logger' passed an argument to 'aws.NewClient' function.
type customErrorLogger struct{}
func (l customErrorLogger ) Log(args ...interface{}) {
}
func main() {
sess, err := session.NewSession()
if err != nil {
fmt.Println("Error creating session:", err)
return
}
cfg := dax.DefaultConfig()
cfg.HostPorts = []string{daxConfig.URL}
cfg.Region = daxConfig.Region
cfg.LogLevel = aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors
client := dax.New(sess, cfg)
client.Config.Logger = customErrorLogger {}
}
Related
I wrote a lambda function in Go that would handle a form submission and send an email with the captured form data to a configured email address.
I am using AWS SES, or trying to.
My emailService.go has the following code:
package aj
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"net/smtp"
"os"
"path/filepath"
"strings"
"go.uber.org/zap"
)
var (
host = os.Getenv("EMAIL_HOST")
username = os.Getenv("EMAIL_USERNAME")
password = os.Getenv("EMAIL_PASSWORD")
portNumber = os.Getenv("EMAIL_PORT")
from = os.Getenv("SOURCE_EMAIL")
)
type EmailService struct {
auth smtp.Auth
}
type EmailSender interface {
SendEmail(logger *zap.Logger, ctx context.Context, m *Message) error
}
func NewEmailService() *EmailService {
auth := smtp.PlainAuth("", username, password, host)
return &EmailService{auth}
}
func (s *EmailService) SendEmail(logger *zap.Logger, ctx context.Context, m *Message) error {
return smtp.SendMail(fmt.Sprintf("%s:%s", host, portNumber), s.auth, from, m.To, m.ToBytes())
}
func NewMessage(s, b string) *Message {
return &Message{Subject: s, Body: b, Attachments: make(map[string][]byte)}
}
func (m *Message) AttachFile(src string) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
_, fileName := filepath.Split(src)
m.Attachments[fileName] = b
return nil
}
func (m *Message) ToBytes() []byte {
buf := bytes.NewBuffer(nil)
withAttachments := len(m.Attachments) > 0
buf.WriteString(fmt.Sprintf("Subject: %s\n", m.Subject))
buf.WriteString(fmt.Sprintf("To: %s\n", strings.Join(m.To, ",")))
if len(m.CC) > 0 {
buf.WriteString(fmt.Sprintf("Cc: %s\n", strings.Join(m.CC, ",")))
}
if len(m.BCC) > 0 {
buf.WriteString(fmt.Sprintf("Bcc: %s\n", strings.Join(m.BCC, ",")))
}
buf.WriteString("MIME-Version: 1.0\n")
writer := multipart.NewWriter(buf)
boundary := writer.Boundary()
if withAttachments {
buf.WriteString(fmt.Sprintf("Content-Type: multipart/mixed; boundary=%s\n", boundary))
buf.WriteString(fmt.Sprintf("--%s\n", boundary))
} else {
buf.WriteString("Content-Type: text/plain; charset=utf-8\n")
}
buf.WriteString(m.Body)
if withAttachments {
for k, v := range m.Attachments {
buf.WriteString(fmt.Sprintf("\n\n--%s\n", boundary))
buf.WriteString(fmt.Sprintf("Content-Type: %s\n", http.DetectContentType(v)))
buf.WriteString("Content-Transfer-Encoding: base64\n")
buf.WriteString(fmt.Sprintf("Content-Disposition: attachment; filename=%s\n", k))
b := make([]byte, base64.StdEncoding.EncodedLen(len(v)))
base64.StdEncoding.Encode(b, v)
buf.Write(b)
buf.WriteString(fmt.Sprintf("\n--%s", boundary))
}
buf.WriteString("--")
}
return buf.Bytes()
}
As you can see, I am using the standard library net/smtp package. Is that Ok to do when you want to send an email through SES? Of course, I am passing my SES SMTP credentials for the necessary/required environment variables.
Right now, I'm not getting any errors, but I am also not receiving emails even though the count on "emails sent" on the SES Dashboard (within the AWS Console) increments each time I send a request to my lambda function.
Am I going about this the wrong way? It is not working because I must use the SES package within the AWS SDK?
In the HTTP handler, I'm doing:
err = r.ParseMultipartForm(10 << 20)
if err != nil {
logger.Error("error parsing form data", zap.Error(err))
http.Error(w, "error parsing form data", http.StatusBadRequest)
return
}
I believe there's a limit of 10Mb on SES emails, and I believe the 10 << 20 ensure I don't go over? Correct me if I'm wrong please.
I'm currently trying to send a POST request to an external API from a GCP Cloud Function. I've tested the function extensively locally and it fulfills the request every time and also works from Postman, but when I run the exact same code from within a cloud function, it returns a 500 from the external API every single time.
I'm genuinely at a loss as to why when sending the POST request from within the cloud function it fails every single time.
Does GCP add any headers that might interfere with an external API call or is there a configuration option within the cloud function settings that needs to be configured to allow an external POST request?
I've attempted to implement an http retry mechanism, but that did not work either.
Again, locally and from Postman, the exact same code is successful every time I run it.
Here is the code I use to generate and send the request:
package email
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/hashicorp/go-retryablehttp"
)
var FailedRequestErr = errors.New("failed request to moosend")
const (
successCode = 0
moosendHost = "api.moosend.com/v3"
dailyNewsletterMailingListID = "2e461f4c-99d1-4a8e-80ea-168b20bdaf5f"
mainEmail = "jason#functionalbits.io"
campaignNameBase = "Functional Bits Newsletter - Issue"
campaignSubjectBase = "Functional Bits Issue"
)
type CreatingADraftCampaignRequest struct {
Name string `json:"Name"`
Subject string `json:"Subject"`
SenderEmail string `json:"SenderEmail"`
ReplyToEmail string `json:"ReplyToEmail"`
IsAB string `json:"IsAB"`
ConfirmationToEmail string `json:"ConfirmationToEmail,omitempty"`
WebLocation string `json:"WebLocation,omitempty"`
MailingLists []MailingLists `json:"MailingLists,omitempty"`
SegmentID string `json:"SegmentID,omitempty"`
ABCampaignType string `json:"ABCampaignType,omitempty"`
TrackInGoogleAnalytics string `json:"TrackInGoogleAnalytics,omitempty"`
DontTrackLinkClicks string `json:"DontTrackLinkClicks,omitempty"`
SubjectB string `json:"SubjectB,omitempty"`
WebLocationB string `json:"WebLocationB,omitempty"`
SenderEmailB string `json:"SenderEmailB,omitempty"`
HoursToTest string `json:"HoursToTest,omitempty"`
ListPercentage string `json:"ListPercentage,omitempty"`
ABWinnerSelectionType string `json:"ABWinnerSelectionType,omitempty"`
}
type MailingLists struct {
MailingListID string `json:"MailingListId"`
SegmentID float64 `json:"SegmentId,omitempty"`
}
type CampaignResponse struct {
Code int32 `json:"Code"`
Err interface{} `json:"Error"`
Context interface{} `json:"Context"`
}
type MoosendAPI struct {
apiKey string
client *http.Client
}
func NewMoosendAPI(apiKey string) *MoosendAPI {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
standardClient := retryClient.StandardClient()
return &MoosendAPI{
apiKey: apiKey,
client: standardClient,
}
}
func (m *MoosendAPI) CreateDraftCampaign(issueNumber string, webLocation string) (*CampaignResponse, error) {
campaign := CreatingADraftCampaignRequest{
Name: fmt.Sprintf("%s %s", campaignNameBase, issueNumber),
Subject: fmt.Sprintf("%s %s", campaignSubjectBase, issueNumber),
IsAB: "false",
WebLocation: webLocation,
MailingLists: []MailingLists{{MailingListID: dailyNewsletterMailingListID}},
SenderEmail: mainEmail,
ReplyToEmail: mainEmail,
ConfirmationToEmail: mainEmail,
TrackInGoogleAnalytics: "true",
}
body, err := json.Marshal(&campaign)
if err != nil {
log.Println("error marshalling campaign request")
return nil, err
}
fullURL := fmt.Sprintf("https://%s/campaigns/create.json?apikey=%s", moosendHost, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, bytes.NewBuffer(body))
if err != nil {
log.Println("request error")
return nil, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
log.Printf("request: %+v", req)
resp, err := m.client.Do(req)
if resp.StatusCode != http.StatusOK {
return nil, FailedRequestErr
}
if err != nil {
log.Println("error sending request")
return nil, err
}
log.Printf("response: %+v", resp)
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return nil, err
}
var draftResponse CampaignResponse
if err := json.Unmarshal(respBody, &draftResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", draftResponse)
return nil, err
}
return &draftResponse, nil
}
func (m *MoosendAPI) SendCampaign(campaignID string) error {
fullURL := fmt.Sprintf("https://%s/campaigns/%s/send.json?apikey=%s", moosendHost, campaignID, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, nil)
if err != nil {
log.Println("error creating request")
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
resp, err := m.client.Do(req)
if err != nil {
log.Println("error sending request")
return err
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return err
}
var sendResponse CampaignResponse
if err := json.Unmarshal(respBody, &sendResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", sendResponse)
return err
}
return nil
}
Then how it's run in the main function code:
package function
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"github.com/Functional-Bits/emailer-service/internal/email"
"github.com/Functional-Bits/emailer-service/internal/publish"
)
func CampaignGenerator(ctx context.Context, m publish.PubSubMessage) error {
moosendAPIKey, ok := os.LookupEnv("MOOSEND_API_KEY")
if !ok {
log.Println("missing moosendAPIKey")
}
mAPI := email.NewMoosendAPI(moosendAPIKey)
var msg publish.IncomingMessage
if err := json.Unmarshal(m.Data, &msg); err != nil {
log.Println(err)
return err
}
log.Printf("received message: %+v", msg)
log.Printf("generating draft campaign for issue %s", msg.IssueNumber)
draftResponse, err := mAPI.CreateDraftCampaign(msg.IssueNumber, msg.FileURL)
if err != nil {
log.Println(err)
return err
}
log.Printf("draft response: %+v", draftResponse)
campaignID, ok := draftResponse.Context.(string)
if !ok {
log.Printf("response didn't contain an ID: %+v", draftResponse)
return errors.New("no campaign generated")
}
log.Printf("sending campgain %s", campaignID)
if err := mAPI.SendCampaign(campaignID); err != nil {
log.Println(err)
return err
}
log.Printf("campaign successfully sent for issue number %s", msg.IssueNumber)
return nil
}
When this code is run locally, It correctly makes the 2 calls and sends an email campaign. When run from the cloud function I get a 500 internal server error with no additional information as to why. Link to API docs.
I get the following response from the external API (from my cloud function logs)
response: &{
Status:500 Internal Server Error
StatusCode:500
Proto:HTTP/1.1
ProtoMajor:1
ProtoMinor:1
Header:map[Access-Control-Allow-Headers:[Content-Type, Accept, Cache-Control, X-Requested-With]
Access-Control-Allow-Methods:[GET, POST, OPTIONS, DELETE, PUT]
Access-Control-Allow-Origin:[*]
Cache-Control:[private]
Content-Length:[12750]
Content-Type:[text/html; charset=utf-8]
Date:[Sun, 12 Dec 2021 07:00:09 GMT]
Server:[Microsoft-IIS/10.0]
X-Aspnet-Version:[4.0.30319]
X-Powered-By:[ASP.NET]
X-Robots-Tag:[noindex, nofollow]
X-Server-Id:[1]]
Body:0xc0003f04c0
ContentLength:12750
TransferEncoding:[]
Close:false
Uncompressed:false
Trailer:map[]
Request:0xc000160b00
TLS:0xc000500630
}
The response causes an unmarshal error because no campaign ID is returned.
I am new to AWS and Golang, and I am trying to create a lambda function, which will trigger AWS Athena query and email the result using AWS SES service. Even after searching for an hour, I couldn't find a working example of lambda function (in Golang) to perform a query on Athena and getting the output of the query.
While searching, I found code for the same in Java, Python and Node Js, but not in Golang.
Even the Go-SDK page redirects to Java example. But unfortunately, I don't even understand Java.
I have also looked into this AWS SDK for Go API Reference page. But I don't understand what is the flow of the program and which operation to select.
I have tried to create the program for this, this may be completely wrong, and I don't know what to do next. Below is the code -
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/athena"
)
func main() {
// Create a new session in the us-west-2 region.
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
// Create an Athena session.
client := athena.New(sess)
// Example sending a request using the StartQueryExecutionRequest method.
query := "SELECT * FROM table1 ;"
params := query
req, resp := client.StartQueryExecutionRequest(params)
err1 := req.Send()
if err1 == nil { // resp is now filled
fmt.Println(resp)
}
}
Appreciate if someone can help me to perform an Athena query and to get its result in Golang(Preferably) or can share some resource. Once I get it, I can then send an email using AWS SES.
Use this to get started.
// run as: go run main.go
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "textqldb.textqltable"
const outputBucket = "s3://bucket-name-here/"
func main() {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
return
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
query := "select * from " + table
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("query error: %v\n", err)
return
}
fmt.Println(resp)
}
#Everton's code is executing a query on Athena, and its responses are getting saved on S3 bucket and not getting returned. So, I have added the code to execute the Athena query and get the response back. Hope this may help others.
// run as: go run main.go
package main
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/athena"
)
const table = "<Database_Name>.<Table_Name>"
const outputBucket = "s3://bucket-name-here/"
// Execute the query and return the query ID
func executeQuery(query string) *string {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
fmt.Printf("config error: %v\n", err)
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
resultConf := &athena.ResultConfiguration{
OutputLocation: aws.String(outputBucket),
}
params := &athena.StartQueryExecutionInput{
QueryString: aws.String(query),
ResultConfiguration: resultConf,
}
req := client.StartQueryExecutionRequest(params)
resp, err := req.Send(context.TODO())
fmt.Println("Response is: ", resp, " Error is:", err)
if err != nil {
fmt.Printf("Query Error: %v\n", err)
}
fmt.Println("Query Execution Response ID:", resp.QueryExecutionId)
return resp.QueryExecutionId
}
// Takes queryId as input and returns its response
func getQueryResults(QueryID *string) (*athena.GetQueryResultsResponse, error) {
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
panic("config error")
}
cfg.Region = endpoints.UsEast2RegionID
client := athena.New(cfg)
params1 := &athena.GetQueryResultsInput{
QueryExecutionId: QueryID,
}
req := client.GetQueryResultsRequest(params1)
resp, err := req.Send(context.TODO())
if err != nil {
fmt.Printf("Query Response Error: %v\n", err)
return nil, err
}
return resp, nil
}
func main() {
query := "select * from " + table
// Execute an Athena Query
QueryID := executeQuery(query)
// Get the response of the query
// Wait for some time for query completion
time.Sleep(15 * time.Second) // Otherwise create a loop and try for every x seconds
Resp, err := getQueryResults(QueryID)
if err != nil {
fmt.Printf("Error getting Query Response: %v\n", err)
} else {
fmt.Println(" \nRows:", Resp.ResultSet.Rows)
}
}
I'm not in charge of maintaining the code base under test, but I have this method to try to test, which hits a real server:
func (a *APICoreSt) CallServer(context interface{}, apiType APIType, apiURL string, header map[string]string, jsonBody []byte) CallResultSt {
var (
Url = a.BaseURL + apiURL
err error
res *http.Response
resBody json.RawMessage
hc = &http.Client{}
req = new(http.Request)
)
req, err = http.NewRequest(string(apiType), Url, bytes.NewBuffer(jsonBody))
if err != nil {
//Use a map instead of errorSt so that it doesn't create a heavy dependency.
errorSt := map[string]string{
"Code": "ez020300007",
"Message": "The request failed to be created.",
}
logger.Instance.LogError(err.Error())
err, _ := json.Marshal(errorSt)
return CallResultSt{DetailObject: err, IsSucceeded: false}
}
for k, v := range header {
req.Header.Set(k, v)
}
res, err = hc.Do(req)
if res != nil {
resBody, err = ioutil.ReadAll(res.Body)
res.Body = ioutil.NopCloser(bytes.NewBuffer(resBody))
}
return CallResultSt{resBody, logger.Instance.CheckAndHandleErr(context, res)}
}
Clearly, since this is a unit test, I don't want to be making this call for real, and the test depends on external factors. Is there any way to inject any fakes/mocks, without touching this func itself (it's literally not my job to touch it)?
I am stuck over testing with mocking, Here is my route for handler:
r.Handle("/users/{userID}", negroni.New(
negroni.HandlerFunc(validateTokenMiddleware),
negroni.Wrap(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
getUserDetailsHandler(w, r, db)
})),
)).Methods("GET")
And here is my handler:
func getUserDetailsHandler(w http.ResponseWriter, r *http.Request, db *sql.DB) {
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
//Create UserDetailsView instance
var userview UserDetailsView
//Get varibale from mux
vars := mux.Vars(r)
//UserID fetches userId from vars
userID := vars["userID"]
//Get user Information by wpUsersID
wuis := store.NewWpUserInformationStore(db)
userInformation, _:= wuis.GetByID(uID)
json.NewEncoder(w).Encode(userview);
//Print result
w.WriteHeader(http.StatusOK)
}
And i mock the function which is in store package named as GetByID which is looks like this :
type wpUserInfoMockStore struct {
mock.Mock
}
func (m *wpUserInfoMockStore) GetByID(user *WpUserInformation) error {
rets := m.Called(user)
return rets.Error(0)
}
//InitMockStore store
func InitMockStore() *wpUserInfoMockStore {
s := new(wpUserInfoMockStore)
//store = s
return s
}
And i write test case for handler but i got an error cannot convert getUserDetailsHandler (type func(http.ResponseWriter, *http.Request, *sql.DB)) to type http.HandlerFunc but i can not find why is it happened, here i'm using reference for this https://github.com/sohamkamani/blog_example__go_web_db and here is my test case code:
func TestGetUserDetailsTes(t *testing.T) {
// Initialize the mock store
mockStore := store.InitMockStore()
mockStore.On("GetByID").Return([]*store.WpUserInformation{{
21,
sql.NullString{String: "john"},
sql.NullString{String: "Sorensen"},
0}}, nil).Once()
req, err := http.NewRequest("GET", "", nil)
//if requests gives error
if err != nil {
panic(err.Error())
}
//parameters for generateTestUserJWT are set
testUser.ID = "22"
testUser.UserName = "johns"
testUser.Depot = "NYC"
//JWT generated
refToken, err := generateTestJWT(testUser, false)
//handling error while generating token
if err != nil {
panic(err.Error())
}
//token returned is concatenated with Bearer string
newToken = "Bearer " + refToken
//request authorization header is set
req.Header.Set("Authorization", newToken)
req.Header.Set("Latitude", "123.12")
req.Header.Set("Longitude", "456.45")
//response is set
w := httptest.NewRecorder()
hf := http.HandlerFunc(getUserDetailsHandler)
hf.ServeHTTP(w, req)
//if response code is not statusOK then test fails
if w.Code != http.StatusOK {
t.Errorf("/users/{userID} GET request failed, got: %d, want: %d.", w.Code, http.StatusOK)
}
}
As you see i test handler without url like req, err := http.NewRequest("GET", "", nil) but when i used link inside then i can not able to use mock functions, here what am i missing/fault please help me out.
Thank you.
Use a middleware handler for generating the function. Pass a handler in your main function which will call your middleware returning http.handler. That way you can pass db object to your main data and which will call the middle ware returning handler.
func getUserDetailsHandler(w http.ResponseWriter, r *http.Request, db *sql.DB) http.HandlerFunc{
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
//Create UserDetailsView instance
var userview UserDetailsView
//Get varibale from mux
vars := mux.Vars(r)
//UserID fetches userId from vars
userID := vars["userID"]
//Get user Information by wpUsersID
wuis := store.NewWpUserInformationStore(db)
userInformation, _:= wuis.GetByID(uID)
json.NewEncoder(w).Encode(userview);
//Print result
w.WriteHeader(http.StatusOK)
}
}