I wrote a lambda function in Go that would handle a form submission and send an email with the captured form data to a configured email address.
I am using AWS SES, or trying to.
My emailService.go has the following code:
package aj
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"net/smtp"
"os"
"path/filepath"
"strings"
"go.uber.org/zap"
)
var (
host = os.Getenv("EMAIL_HOST")
username = os.Getenv("EMAIL_USERNAME")
password = os.Getenv("EMAIL_PASSWORD")
portNumber = os.Getenv("EMAIL_PORT")
from = os.Getenv("SOURCE_EMAIL")
)
type EmailService struct {
auth smtp.Auth
}
type EmailSender interface {
SendEmail(logger *zap.Logger, ctx context.Context, m *Message) error
}
func NewEmailService() *EmailService {
auth := smtp.PlainAuth("", username, password, host)
return &EmailService{auth}
}
func (s *EmailService) SendEmail(logger *zap.Logger, ctx context.Context, m *Message) error {
return smtp.SendMail(fmt.Sprintf("%s:%s", host, portNumber), s.auth, from, m.To, m.ToBytes())
}
func NewMessage(s, b string) *Message {
return &Message{Subject: s, Body: b, Attachments: make(map[string][]byte)}
}
func (m *Message) AttachFile(src string) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
_, fileName := filepath.Split(src)
m.Attachments[fileName] = b
return nil
}
func (m *Message) ToBytes() []byte {
buf := bytes.NewBuffer(nil)
withAttachments := len(m.Attachments) > 0
buf.WriteString(fmt.Sprintf("Subject: %s\n", m.Subject))
buf.WriteString(fmt.Sprintf("To: %s\n", strings.Join(m.To, ",")))
if len(m.CC) > 0 {
buf.WriteString(fmt.Sprintf("Cc: %s\n", strings.Join(m.CC, ",")))
}
if len(m.BCC) > 0 {
buf.WriteString(fmt.Sprintf("Bcc: %s\n", strings.Join(m.BCC, ",")))
}
buf.WriteString("MIME-Version: 1.0\n")
writer := multipart.NewWriter(buf)
boundary := writer.Boundary()
if withAttachments {
buf.WriteString(fmt.Sprintf("Content-Type: multipart/mixed; boundary=%s\n", boundary))
buf.WriteString(fmt.Sprintf("--%s\n", boundary))
} else {
buf.WriteString("Content-Type: text/plain; charset=utf-8\n")
}
buf.WriteString(m.Body)
if withAttachments {
for k, v := range m.Attachments {
buf.WriteString(fmt.Sprintf("\n\n--%s\n", boundary))
buf.WriteString(fmt.Sprintf("Content-Type: %s\n", http.DetectContentType(v)))
buf.WriteString("Content-Transfer-Encoding: base64\n")
buf.WriteString(fmt.Sprintf("Content-Disposition: attachment; filename=%s\n", k))
b := make([]byte, base64.StdEncoding.EncodedLen(len(v)))
base64.StdEncoding.Encode(b, v)
buf.Write(b)
buf.WriteString(fmt.Sprintf("\n--%s", boundary))
}
buf.WriteString("--")
}
return buf.Bytes()
}
As you can see, I am using the standard library net/smtp package. Is that Ok to do when you want to send an email through SES? Of course, I am passing my SES SMTP credentials for the necessary/required environment variables.
Right now, I'm not getting any errors, but I am also not receiving emails even though the count on "emails sent" on the SES Dashboard (within the AWS Console) increments each time I send a request to my lambda function.
Am I going about this the wrong way? It is not working because I must use the SES package within the AWS SDK?
In the HTTP handler, I'm doing:
err = r.ParseMultipartForm(10 << 20)
if err != nil {
logger.Error("error parsing form data", zap.Error(err))
http.Error(w, "error parsing form data", http.StatusBadRequest)
return
}
I believe there's a limit of 10Mb on SES emails, and I believe the 10 << 20 ensure I don't go over? Correct me if I'm wrong please.
Related
I am creating an API endpoint to handle form submissions.
The form takes the following:
Name
Email
Phone
Photo files (up to 5)
Then basically, sends an email to some email address with the photos as attachments.
I want to write tests for my handler to make sure everything is working well, however, I am struggling.
CODE:
Below is my HTTP handler (will run in AWS lambda, not that it matters).
package aj
import (
"fmt"
"mime"
"net/http"
"go.uber.org/zap"
)
const expectedContentType string = "multipart/form-data"
func FormSubmissionHandler(logger *zap.Logger, emailSender EmailSender) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// enforce a multipart/form-data content-type
contentType := r.Header.Get("content-type")
mediatype, _, err := mime.ParseMediaType(contentType)
if err != nil {
logger.Error("error when parsing the mime type", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if mediatype != expectedContentType {
logger.Error("unsupported content-type", zap.Error(err))
http.Error(w, fmt.Sprintf("api expects %v content-type", expectedContentType), http.StatusUnsupportedMediaType)
return
}
err = r.ParseMultipartForm(32 << 20)
if err != nil {
logger.Error("error parsing form data", zap.Error(err))
http.Error(w, "error parsing form data", http.StatusBadRequest)
return
}
name := r.PostForm.Get("name")
if name == "" {
fmt.Println("inside if statement for name")
logger.Error("name not set", zap.Error(err))
http.Error(w, "api expects name to be set", http.StatusBadRequest)
return
}
email := r.PostForm.Get("email")
if email == "" {
logger.Error("email not set", zap.Error(err))
http.Error(w, "api expects email to be set", http.StatusBadRequest)
return
}
phone := r.PostForm.Get("phone")
if phone == "" {
logger.Error("phone not set", zap.Error(err))
http.Error(w, "api expects phone to be set", http.StatusBadRequest)
return
}
emailService := NewEmailService()
m := NewMessage("Test", "Body message.")
err = emailService.SendEmail(logger, r.Context(), m)
if err != nil {
logger.Error("an error occurred sending the email", zap.Error(err))
http.Error(w, "error sending email", http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusOK)
})
}
The now updated test giving me trouble is:
package aj
import (
"bytes"
"context"
"encoding/json"
"fmt"
"image/jpeg"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"go.uber.org/zap"
)
type StubEmailService struct {
sendEmail func(logger *zap.Logger, ctx context.Context, email *Message) error
}
func (s *StubEmailService) SendEmail(logger *zap.Logger, ctx context.Context, email *Message) error {
return s.sendEmail(logger, ctx, email)
}
func TestFormSubmissionHandler(t *testing.T) {
// create the logger
logger, _ := zap.NewProduction()
t.Run("returns 400 (bad request) when name is not set in the body", func(t *testing.T) {
// set up a pipe avoid buffering
pipeReader, pipeWriter := io.Pipe()
// this writer is going to transform what we pass to it to multipart form data
// and write it to our io.Pipe
multipartWriter := multipart.NewWriter(pipeWriter)
go func() {
// close it when it has done its job
defer multipartWriter.Close()
// create a form field writer for name
nameField, err := multipartWriter.CreateFormField("name")
if err != nil {
t.Error(err)
}
// write string to the form field writer for name
nameField.Write([]byte("John Doe"))
// we create the form data field 'photo' which returns another writer to write the actual file
fileField, err := multipartWriter.CreateFormFile("photo", "test.png")
if err != nil {
t.Error(err)
}
// read image file as array of bytes
fileBytes, err := os.ReadFile("../../00-test-image.jpg")
// create an io.Reader
reader := bytes.NewReader(fileBytes)
// convert the bytes to a jpeg image
image, err := jpeg.Decode(reader)
if err != nil {
t.Error(err)
}
// Encode() takes an io.Writer. We pass the multipart field 'photo' that we defined
// earlier which, in turn, writes to our io.Pipe
err = jpeg.Encode(fileField, image, &jpeg.Options{Quality: 75})
if err != nil {
t.Error(err)
}
}()
formData := HandleFormRequest{Name: "John Doe", Email: "john.doe#example.com", Phone: "07542147833"}
// create the stub patient store
emailService := StubEmailService{
sendEmail: func(_ *zap.Logger, _ context.Context, email *Message) error {
if !strings.Contains(email.Body, formData.Name) {
t.Errorf("expected email.Body to contain %s", formData.Name)
}
return nil
},
}
// create a request to pass to our handler
req := httptest.NewRequest(http.MethodPost, "/handler", pipeReader)
// set the content type
req.Header.Set("content-type", "multipart/form-data")
// create a response recorder
res := httptest.NewRecorder()
// get the handler
handler := FormSubmissionHandler(logger, &emailService)
// our handler satisfies http.handler, so we can call its serve http method
// directly and pass in our request and response recorder
handler.ServeHTTP(res, req)
// assert status code is what we expect
assertStatusCode(t, res.Code, http.StatusBadRequest)
})
}
func assertStatusCode(t testing.TB, got, want int) {
t.Helper()
if got != want {
t.Errorf("handler returned wrong status code: got %v want %v", got, want)
}
}
As mentioned in the test name, I want to make sure a Name property is coming through with the request.
When I run go test ./... -v I get:
=== RUN TestFormSubmissionHandler/returns_400_(bad_request)_when_name_is_not_set_in_the_body
{"level":"error","ts":1675459283.4969518,"caller":"aj/handler.go:33","msg":"error parsing form data","error":"no multipart boundary param in Content-Type","stacktrace":"github.com/jwankhalaf-dh/ajgenerators.co.uk__form-handler/api/aj.FormSubmissionHandler.func1\n\t/home/j/code/go/src/github.com/jwankhalaf-dh/ajgenerators.co.uk__form-handler/api/aj/handler.go:33\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2109\ngithub.com/jwankhalaf-dh/ajgenerators.co.uk__form-handler/api/aj.TestFormSubmissionHandler.func3\n\t/home/j/code/go/src/github.com/jwankhalaf-dh/ajgenerators.co.uk__form-handler/api/aj/handler_test.go:132\ntesting.tRunner\n\t/usr/local/go/src/testing/testing.go:1446"}
I understand the error, but I am not sure how to overcome it.
My next test would be to test the same thing but for email, and then phone, then finally, I'd like to test file data, but I'm not sure how.
Thanks to Adrian and Cerise I was able to correctly construct the multipart/form-data in the request (updated code is in the question).
However, it was still not working, and the reason was, I was doing:
// set the content type
req.Header.Set("content-type", "multipart/form-data")
instead of:
// set the content type
req.Header.Add("content-type", multipartWriter.FormDataContentType())
This bounty has ended. Answers to this question are eligible for a +50 reputation bounty. Bounty grace period ends in 21 hours.
Abhisek Roy wants to draw more attention to this question:
Looking for a workaround to the current issue.
I was implementing RDS IAM authentication from my Golang service and came across an issue. I am using the init function to call create a function where I have created a sync for every 10 mins where every 10 mins the token should get refreshed, it is creating the new token but it's not passing the same to the main function. When the existing thread that was initiated between DB and the service gets killed it is unable to re-authenticate the connection for the second time. Some observations that I have noticed were-
When initially deploying the application it is creating the connection to RDS successfully but after some time as soon as the initial thread between service and DB gets killed it's unable to authenticate the reason which I checked was- the address where the token is getting stored. While making an API call the service picks the token from a static address where the initial token gets stored at the time of service deployment. However, while the token gets refreshed every 10 mins its getting stored on dynamic addresses from where the service is unable to pick up the token.
Here is the go file where actually I am calling and creating the DB function-
package db
import (
"fmt"
"log"
"os"
"time"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/rds/rdsutils"
"github.com/go-co-op/gocron"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
"github.com/jinzhu/gorm"
_ "github.com/lib/pq"
)
const DatabaseVersion = 1
const Success = "Database connection successful"
const database = "postgres"
// Service ...
type Service interface {
InitDatabaseConnection(db **gorm.DB)
CreateDatabaseConnection(db **gorm.DB)
Migrate(db *gorm.DB) error
}
type service struct {
}
// NewService ...
func NewService() Service {
return &service{}
}
func (s *service) InitDatabaseConnection(db **gorm.DB) {
log.Print("Creating Db Connection")
s.CreateDatabaseConnection(db)
//todo: add if; if env for iam run the below code
// todo: change 1 min to 10 mins
startTime := time.Now().Local().Add(time.Minute + time.Duration(1))
sch := gocron.NewScheduler(time.UTC)
sch.Every(600).Seconds().StartAt(startTime).Do(func() {
log.Printf("refreshing rds creds")
s.CreateDatabaseConnection(db)
})
sch.StartAsync()
}
func (s *service) CreateDatabaseConnection(db **gorm.DB) {
configuration, _ := common.New()
var dbURI = ConnectDataBase(configuration.UserName, configuration.Password,
configuration.Port, configuration.DbName, configuration.Host, configuration.SearchPath)
var err error
*db, err = gorm.Open(database, dbURI)
if err != nil {
panic(err)
}
print("\n \n", dbURI)
print("\n \n in server.go \n \n ", &db)
if err != nil {
log.Println(err.Error())
os.Exit(3)
} else {
log.Println(Success)
}
(*db).DB().SetMaxOpenConns(6)
(*db).DB().SetConnMaxLifetime(600 * time.Second)
(*db).DB().SetMaxIdleConns(2)
(*db).DB().SetConnMaxIdleTime(100 * time.Second)
}
func ConnectDataBase(username string, password string, port int, dbName string, host string, searchPath string) string {
var dbURI string
if searchPath == "" {
searchPath = "public"
}
dbEndpoint := fmt.Sprintf("%s:%d", host, port)
sess := session.Must(session.NewSession())
creds := sess.Config.Credentials
authToken, err := rdsutils.BuildAuthToken(dbEndpoint, "us-east-1", username, creds)
print("\n \n", authToken)
if err != nil {
panic(err)
}
dbURI = fmt.Sprintf("host=%s port=%d user=%s dbname=%s sslmode=require password=%s search_path=%s",
host, port, username, dbName, authToken, searchPath) //Build connection string
// if password != "" && username != "" {
// dbURI = fmt.Sprintf("host=%s port=%d user=%s dbname=%s sslmode=require password=%s search_path=%s",
// host, port, username, dbName, authToken, searchPath) //Build connection string
// } else {
// dbURI = fmt.Sprintf("host=%s port=%d dbname=%s sslmode=disable ",
// host, port, dbName) //Build connection string
// }
return dbURI
}
func (s *service) Migrate(db *gorm.DB) error {
migrationSourceURL := "file://../resources/db/migrations/"
database := db.DB()
row := db.Table("schema_migrations").Limit(1).Row()
var version int8
var dirty bool
err := row.Scan(&version, &dirty)
if err == nil {
log.Printf("database is currently on version : %v \n", version)
}
log.Println("Migrating database to version ", DatabaseVersion)
driver, err := postgres.WithInstance(database, &postgres.Config{})
if err != nil {
return err
}
m, err := migrate.NewWithDatabaseInstance(migrationSourceURL, "postgres", driver)
if err != nil {
return err
}
m.Log = LogService{}
err = m.Migrate(uint(DatabaseVersion))
if err != nil {
log.Println(err.Error())
return err
}
return nil
}
The common property above is from this piece of code-
package common
import (
"fmt"
"log"
"os"
"github.com/spf13/viper"
)
type ApiResponse struct {
Code int `json:"code"`
Message string `json:"message"`
}
type Constants struct {
Port int `json:"port"`
Host string `json:"host"`
DbName string `json:"dbName"`
UserName string `json:"userName"`
Password string `json:"password"`
AuthUserName string `json:"authUserName"`
AuthPassword string `json:"authPassword"`
SearchPath string `json:"searchPath"`
AwsRegion string `json:"awsRegion"`
}
func New() (*Constants, error) {
fmt.Println("reached constants")
config := Constants{}
constants, err := initViper()
config = constants
if err != nil {
return &config, err
}
return &config, nil
}
func initViper() (Constants, error) {
// temporary, will be removed later when auto deploy is available
env := os.Getenv("CS_ENV")
var configName string
if env == "develop" {
configName = "config-develop"
} else {
configName = "config-local"
}
viper.SetConfigName(configName) // Constants fileName without the .TOML or .YAML extension
viper.AddConfigPath("../resources/config") // Search the root directory for the configuration file
err := viper.ReadInConfig() // Find and read the config file
if err != nil {
log.Println(err.Error()) // Handle errors reading the config file
return Constants{}, err
}
if err = viper.ReadInConfig(); err != nil {
log.Panicf("Error reading config file, %s", err)
}
var constants Constants
err = viper.Unmarshal(&constants)
return constants, err
}
Also attaching the main function go file where I am calling the DB function for connecting the service to DB-
package main
import (
"bufio"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strings"
"time"
gokitlog "github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/gorilla/mux"
"github.com/jinzhu/gorm"
_ "github.com/lib/pq"
)
func runServer(config *common.Constants) {
var logger gokitlog.Logger
logger = gokitlog.NewLogfmtLogger(gokitlog.NewSyncWriter(os.Stderr))
logger = gokitlog.With(logger, "ts", gokitlog.DefaultTimestampUTC)
dbService := db2.NewService()
var db *gorm.DB
dbService.InitDatabaseConnection(&db)
err := dbService.Migrate(db)
if err != nil {
fmt.Println("Migrations scripts did not run :", err.Error())
if err.Error() != "no change" {
fmt.Println("failed to run migrations hence exiting", err.Error())
os.Exit(1)
}
}
level.Info(logger).Log("Starting Darwin ==========>")
repo, err := allocation.NewUserAllocationRepository(db, logger)
if err != nil {
level.Error(logger).Log("exit", err)
os.Exit(3)
}
userAllocationService := allocation.NewUserAllocationService(repo, logger)
endpoints := allocation.MakeEndpoints(userAllocationService)
router := mux.NewRouter()
sh := http.StripPrefix("/swaggerui/", http.FileServer(http.Dir("../resources/swaggerui/")))
router.PathPrefix("/swaggerui/").Handler(sh)
httpLogger := gokitlog.With(logger, "component", "http")
subRouter := router.PathPrefix("/api/v1").Subrouter()
subRouter.NotFoundHandler = http.HandlerFunc(notFound)
subRouter.Use(checkBasicAuth(config))
subRouter.HandleFunc("/healthCheck", healthCheck).Methods("GET")
allocation.MakeHandler(subRouter, httpLogger, endpoints)
http.Handle("/", subRouter)
log.Print("Running server on port 8080")
f, err := os.OpenFile("../performance.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Printf("error opening file: %v", err)
log.Fatal(http.ListenAndServe(":8080", router))
} else {
by := bufio.NewWriter(f)
defer f.Close()
log.Fatal(http.ListenAndServe(":8080", Logger(by, router)))
}
}
func healthCheck(w http.ResponseWriter, r *http.Request) {
resp := map[string]string{
"Status": "Success",
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(resp)
}
func notFound(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json;")
w.WriteHeader(http.StatusNotFound)
json.NewEncoder(w).Encode(map[string]string{"error": "api not found"})
}
func authHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json;")
w.WriteHeader(http.StatusUnauthorized)
json.NewEncoder(w).Encode(map[string]string{"error": "Not authorized"})
}
func checkBasicAuth(config *common.Constants) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.RequestURI() != "/api/v1/healthCheck" {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
w.Header().Set("Content-Type", "application/json;")
s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
if len(s) != 2 {
authHandler(w, r)
return
}
b, err := base64.StdEncoding.DecodeString(s[1])
if err != nil {
authHandler(w, r)
return
}
pair := strings.SplitN(string(b), ":", 2)
if len(pair) != 2 {
authHandler(w, r)
return
}
if pair[0] != config.AuthUserName || pair[1] != config.AuthPassword {
authHandler(w, r)
return
}
}
next.ServeHTTP(w, r)
})
}
}
func Logger(out *bufio.Writer, h http.Handler) http.Handler {
logger := log.New(out, "", 0)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
o := &responseObserver{ResponseWriter: w, status: http.StatusOK}
start := time.Now()
h.ServeHTTP(o, r)
endTime := time.Now()
if !strings.Contains(r.URL.String(), "/health") && strings.Contains(r.URL.String(), "/api") {
logger.Printf("Method: %s Path: %s Status: %d ExecutionTime: %d ms", r.Method, r.URL, o.status, endTime.Sub(start).Milliseconds())
out.Flush()
}
})
}
type responseObserver struct {
http.ResponseWriter
status int
}
func (r *responseObserver) WriteHeader(status int) {
r.status = status
r.ResponseWriter.WriteHeader(status)
}
I am building a dApp using #cosmjs.
I want to make a transaction on the COSMOS chain on my dApp running on the JUNO chain.
When I tried to get the signingClient, I got the following running error.
TypeError: Failed to fetch
at http (http://localhost:3000/static/js/bundle.js:53443:12)
at HttpClient.execute (http://localhost:3000/static/js/bundle.js:53471:65)
at Tendermint34Client.detectVersion (http://localhost:3000/static/js/bundle.js:55135:35)
at Tendermint34Client.create (http://localhost:3000/static/js/bundle.js:55128:33)
at Tendermint34Client.connect (http://localhost:3000/static/js/bundle.js:55115:33)
at SigningCosmWasmClient.connectWithSigner (http://localhost:3000/static/js/bundle.js:23403:64)
at http://localhost:3000/static/js/bundle.js:127476:51
at Generator.next (<anonymous>)
at fulfilled (http://localhost:3000/static/js/bundle.js:462877:24)
Here is my code;
import { SigningCosmWasmClient } from "#cosmjs/cosmwasm-stargate";
import { GasPrice }from "#cosmjs/stargate";
.....
const config = {
chainName: "Cosmos Hub",
chainId: "cosmoshub-4",
rpcEndpoint: "https://rpc-cosmoshub.whispernode.com",
restEndpoint: "",
faucetEndpoint: "",
addressPrefix: "cosmos",
microDenom: "uatom",
coinDecimals: "6",
gasPrice: "0.025",
}
.....
await window.keplr?.enable(config.chainId);
const offlineSigner= window.getOfflineSigner?.(
config.chainId
);
const account = await offlineSigner?.getAccounts();
let wasmChainClient = null;
if (offlineSigner) {
try {
wasmChainClient = await SigningCosmWasmClient.connectWithSigner(
config.rpcEndpoint,
offlineSigner,
{
gasPrice: GasPrice.fromString(
`${config.gasPrice}${config.microDenom}`
),
}
);
} catch (e) {
console.error("wallets", e);
}
}
const result= {
account: account?.[0],
client: wasmChainClient,
};
console.log(result)
Is this the problem of rpc endpoint?
I have tried other several rpc endpoints but all of them failed.
I really don't know why this happens.
I would be very thankful if anyone could help me with solving this issue.
This happens because your browser blocks request with different origin. Your origin is http://localhost:3000, and you requested https://rpc-cosmoshub.keplr.app. So, your browser blocks the response of the requested origin if there is no access-control-allow-origin header in the response. You can learn more here (https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS/Errors).
I have a workaround program that adds necessary headers to a response.
You have to have golang to run it.
package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
)
var hostname string
var port int
func main() {
// flags declaration using flag package
flag.StringVar(&hostname, "H", "https://rpc-cosmoshub.keplr.app", "Specify hostname")
flag.IntVar(&port, "p", 8081, "Specify port")
flag.Parse() // after declaring flags we
http.HandleFunc("/", serveCorsProxy)
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
}
// Serve a reverse proxy for a given url
func serveCorsProxy(res http.ResponseWriter, req *http.Request) {
proxyRequest, err := http.NewRequest(req.Method, hostname, req.Body)
proxyRequest.URL.Path = req.URL.Path
proxyRequest.URL.RawQuery = req.URL.RawQuery
if err != nil {
fmt.Printf("create request error: %v", err)
return
}
response, err := http.DefaultClient.Do(proxyRequest)
if err != nil {
fmt.Printf("proxy request error: %v", err)
return
}
setHeaders(response, &res)
body, err := ioutil.ReadAll(response.Body)
if err != nil {
fmt.Printf("response read error: %v", err)
return
}
res.WriteHeader(response.StatusCode)
_, _ = res.Write(body)
}
func setHeaders(src *http.Response, dest *http.ResponseWriter) {
header := (*dest).Header()
for name, values := range (*src).Header {
for _, value := range values {
header.Set(name, value)
}
}
header.Set("access-control-allow-headers", "Accept,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With")
header.Set("access-control-allow-methods", "GET, POST, OPTIONS")
header.Set("access-control-allow-origin", "*")
header.Set("access-control-expose-headers", "Content-Length,Content-Range")
header.Set("access-control-max-age", "1728000")
}
You have to save it to file main.go and run by go run main.go -H https://rpc-cosmoshub.keplr.app -p 3001.
After that you can access the RPC on localhost:3001
I'm currently trying to send a POST request to an external API from a GCP Cloud Function. I've tested the function extensively locally and it fulfills the request every time and also works from Postman, but when I run the exact same code from within a cloud function, it returns a 500 from the external API every single time.
I'm genuinely at a loss as to why when sending the POST request from within the cloud function it fails every single time.
Does GCP add any headers that might interfere with an external API call or is there a configuration option within the cloud function settings that needs to be configured to allow an external POST request?
I've attempted to implement an http retry mechanism, but that did not work either.
Again, locally and from Postman, the exact same code is successful every time I run it.
Here is the code I use to generate and send the request:
package email
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/hashicorp/go-retryablehttp"
)
var FailedRequestErr = errors.New("failed request to moosend")
const (
successCode = 0
moosendHost = "api.moosend.com/v3"
dailyNewsletterMailingListID = "2e461f4c-99d1-4a8e-80ea-168b20bdaf5f"
mainEmail = "jason#functionalbits.io"
campaignNameBase = "Functional Bits Newsletter - Issue"
campaignSubjectBase = "Functional Bits Issue"
)
type CreatingADraftCampaignRequest struct {
Name string `json:"Name"`
Subject string `json:"Subject"`
SenderEmail string `json:"SenderEmail"`
ReplyToEmail string `json:"ReplyToEmail"`
IsAB string `json:"IsAB"`
ConfirmationToEmail string `json:"ConfirmationToEmail,omitempty"`
WebLocation string `json:"WebLocation,omitempty"`
MailingLists []MailingLists `json:"MailingLists,omitempty"`
SegmentID string `json:"SegmentID,omitempty"`
ABCampaignType string `json:"ABCampaignType,omitempty"`
TrackInGoogleAnalytics string `json:"TrackInGoogleAnalytics,omitempty"`
DontTrackLinkClicks string `json:"DontTrackLinkClicks,omitempty"`
SubjectB string `json:"SubjectB,omitempty"`
WebLocationB string `json:"WebLocationB,omitempty"`
SenderEmailB string `json:"SenderEmailB,omitempty"`
HoursToTest string `json:"HoursToTest,omitempty"`
ListPercentage string `json:"ListPercentage,omitempty"`
ABWinnerSelectionType string `json:"ABWinnerSelectionType,omitempty"`
}
type MailingLists struct {
MailingListID string `json:"MailingListId"`
SegmentID float64 `json:"SegmentId,omitempty"`
}
type CampaignResponse struct {
Code int32 `json:"Code"`
Err interface{} `json:"Error"`
Context interface{} `json:"Context"`
}
type MoosendAPI struct {
apiKey string
client *http.Client
}
func NewMoosendAPI(apiKey string) *MoosendAPI {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
standardClient := retryClient.StandardClient()
return &MoosendAPI{
apiKey: apiKey,
client: standardClient,
}
}
func (m *MoosendAPI) CreateDraftCampaign(issueNumber string, webLocation string) (*CampaignResponse, error) {
campaign := CreatingADraftCampaignRequest{
Name: fmt.Sprintf("%s %s", campaignNameBase, issueNumber),
Subject: fmt.Sprintf("%s %s", campaignSubjectBase, issueNumber),
IsAB: "false",
WebLocation: webLocation,
MailingLists: []MailingLists{{MailingListID: dailyNewsletterMailingListID}},
SenderEmail: mainEmail,
ReplyToEmail: mainEmail,
ConfirmationToEmail: mainEmail,
TrackInGoogleAnalytics: "true",
}
body, err := json.Marshal(&campaign)
if err != nil {
log.Println("error marshalling campaign request")
return nil, err
}
fullURL := fmt.Sprintf("https://%s/campaigns/create.json?apikey=%s", moosendHost, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, bytes.NewBuffer(body))
if err != nil {
log.Println("request error")
return nil, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
log.Printf("request: %+v", req)
resp, err := m.client.Do(req)
if resp.StatusCode != http.StatusOK {
return nil, FailedRequestErr
}
if err != nil {
log.Println("error sending request")
return nil, err
}
log.Printf("response: %+v", resp)
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return nil, err
}
var draftResponse CampaignResponse
if err := json.Unmarshal(respBody, &draftResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", draftResponse)
return nil, err
}
return &draftResponse, nil
}
func (m *MoosendAPI) SendCampaign(campaignID string) error {
fullURL := fmt.Sprintf("https://%s/campaigns/%s/send.json?apikey=%s", moosendHost, campaignID, m.apiKey)
req, err := http.NewRequest(http.MethodPost, fullURL, nil)
if err != nil {
log.Println("error creating request")
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
resp, err := m.client.Do(req)
if err != nil {
log.Println("error sending request")
return err
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("error reading response body")
return err
}
var sendResponse CampaignResponse
if err := json.Unmarshal(respBody, &sendResponse); err != nil {
log.Println("error unmarshalling response")
log.Printf("%+v", sendResponse)
return err
}
return nil
}
Then how it's run in the main function code:
package function
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"github.com/Functional-Bits/emailer-service/internal/email"
"github.com/Functional-Bits/emailer-service/internal/publish"
)
func CampaignGenerator(ctx context.Context, m publish.PubSubMessage) error {
moosendAPIKey, ok := os.LookupEnv("MOOSEND_API_KEY")
if !ok {
log.Println("missing moosendAPIKey")
}
mAPI := email.NewMoosendAPI(moosendAPIKey)
var msg publish.IncomingMessage
if err := json.Unmarshal(m.Data, &msg); err != nil {
log.Println(err)
return err
}
log.Printf("received message: %+v", msg)
log.Printf("generating draft campaign for issue %s", msg.IssueNumber)
draftResponse, err := mAPI.CreateDraftCampaign(msg.IssueNumber, msg.FileURL)
if err != nil {
log.Println(err)
return err
}
log.Printf("draft response: %+v", draftResponse)
campaignID, ok := draftResponse.Context.(string)
if !ok {
log.Printf("response didn't contain an ID: %+v", draftResponse)
return errors.New("no campaign generated")
}
log.Printf("sending campgain %s", campaignID)
if err := mAPI.SendCampaign(campaignID); err != nil {
log.Println(err)
return err
}
log.Printf("campaign successfully sent for issue number %s", msg.IssueNumber)
return nil
}
When this code is run locally, It correctly makes the 2 calls and sends an email campaign. When run from the cloud function I get a 500 internal server error with no additional information as to why. Link to API docs.
I get the following response from the external API (from my cloud function logs)
response: &{
Status:500 Internal Server Error
StatusCode:500
Proto:HTTP/1.1
ProtoMajor:1
ProtoMinor:1
Header:map[Access-Control-Allow-Headers:[Content-Type, Accept, Cache-Control, X-Requested-With]
Access-Control-Allow-Methods:[GET, POST, OPTIONS, DELETE, PUT]
Access-Control-Allow-Origin:[*]
Cache-Control:[private]
Content-Length:[12750]
Content-Type:[text/html; charset=utf-8]
Date:[Sun, 12 Dec 2021 07:00:09 GMT]
Server:[Microsoft-IIS/10.0]
X-Aspnet-Version:[4.0.30319]
X-Powered-By:[ASP.NET]
X-Robots-Tag:[noindex, nofollow]
X-Server-Id:[1]]
Body:0xc0003f04c0
ContentLength:12750
TransferEncoding:[]
Close:false
Uncompressed:false
Trailer:map[]
Request:0xc000160b00
TLS:0xc000500630
}
The response causes an unmarshal error because no campaign ID is returned.
This question already has answers here:
How do you create a new instance of a struct from its type at run time in Go?
(5 answers)
Closed 9 months ago.
Working example files included at the end.
I have a package to assist in testing api handlers by creating test http contexts.
The issue is in the AssertJSONResponseBody. The problem is that once the concrete type and value are extracted from the interface, it contains a pointer to the original object. Any changes to the extracted object affect the original object. This then makes the following equal comparison useless because essentially they are pointing to the same value.
Here is the struct:
type TestRequest struct {
Recorder *httptest.ResponseRecorder
Context *gin.Context
t *testing.T
}
func (r *TestRequest) AssertJSONResponseBody(expectedObj interface{}, mesgAndArgs ...interface{}) bool {
outObject := reflect.Indirect(reflect.ValueOf(expectedObj)).Addr().Interface()
// Set break point at next line and compare expectedObject with outObject.
// Then, step over the line and watch the values for both objects change.
// When the decoder unmarshals the json the original object is changed
// because of the pointer in the outobject.
err := json.NewDecoder(r.Recorder.Body).Decode(outObject)
if err != nil {
return assert.Error(r.t, err)
}
return assert.Equal(r.t, expectedObj, outObject, mesgAndArgs...)
}
How do I create a new instance of the underlying type without coupling it to the original value via a pointer?
Here are the working example files.
APIHandler/main.go
package main
import (
"fmt"
"log"
"net/http"
"github.com/gin-gonic/gin"
)
const (
JSONBindingError string = "An error occurred binding the request."
EmailRequiredError string = "Email is required."
)
func main() {
router := gin.Default()
v1 := router.Group("/api/v1/contacts")
{
v1.POST("/", CreateContactHandler)
}
router.Run()
fmt.Printf("hello, world\n")
}
func CreateContactHandler(c *gin.Context) {
request := CreateContactRequest{}
err := c.Bind(&request)
if err != nil {
log.Println("ERROR:", JSONBindingError, err)
apiError := APIError{StatusCode: http.StatusBadRequest, Message: JSONBindingError}
c.JSON(http.StatusBadRequest, apiError)
return
}
if request.Contact.Email == "" {
log.Println("ERROR:", http.StatusBadRequest, EmailRequiredError)
apiError := APIError{StatusCode: http.StatusBadRequest, Message: EmailRequiredError}
c.JSON(http.StatusBadRequest, apiError)
return
}
// Successful client request
// resp := h.Client.CreateContact(request)
// c.JSON(resp.StatusCode, resp.Body)
}
type CreateContactRequest struct {
Contact Contact
}
type Contact struct {
Name string
Email string
}
type CreateContactResponse struct {
Message string
}
type APIError struct {
StatusCode int `json:"status"` // Should match the response status code
Message string `json:"message"`
}
type APIResponse struct {
StatusCode int
Body interface{}
}
APIHandler/helpers/http.go
package helpers
import (
"bytes"
"encoding/json"
"net/http/httptest"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/gin-gonic/gin"
)
// TestRequest is a struct to facilitate
// HTTP Context testing with gin handlers
type TestRequest struct {
Recorder *httptest.ResponseRecorder
Context *gin.Context
t *testing.T
}
// NewTestRequest returns a new TestRequest
func NewTestRequest(t *testing.T) *TestRequest {
rec := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(rec)
ctx.Request = httptest.NewRequest("GET", "/", nil)
return &TestRequest{
Recorder: rec,
Context: ctx,
t: t,
}
}
// SetJSONRequestBody returns a new TestRequest where the request is a post.
// Takes an interface to marshal into JSON and set as the request body.
func (r *TestRequest) SetJSONRequestBody(obj interface{}) *TestRequest {
json, err := json.Marshal(obj)
assert.NoError(r.t, err)
r.Context.Request = httptest.NewRequest("POST", "/", bytes.NewBuffer(json))
r.Context.Request.Header.Add("Content-Type", "application/json")
return r
}
// AssertStatusCode asserts that the recorded status
// is the same as the submitted status.
// The message and the args are added to the message
// when the assertion fails.
func (r *TestRequest) AssertStatusCode(expectedCode int, msgAndArgs ...interface{}) bool {
return assert.Equal(r.t, expectedCode, r.Recorder.Code, msgAndArgs...)
}
// AssertJSONResponseBody asserts that the recorded
// response body unmarshals to the given interface.
// The message and the args are added to the message
// when the assertion fails.
func (r *TestRequest) AssertJSONResponseBody(expectedObj interface{}, masgAndArgs ...interface{}) bool {
out := reflect.Indirect(reflect.ValueOf(expectedObj)).Addr().Interface()
err := json.NewDecoder(r.Recorder.Body).Decode(out)
if err != nil {
return assert.Error(r.t, err)
}
return assert.Equal(r.t, expectedObj, out, masgAndArgs...)
}
APIHandler/main_test.go
package main
import (
"APIHandler/helpers"
"net/http"
"testing"
)
func TestSingleContactCreate(t *testing.T) {
tests := []struct {
toCreate interface{}
handlerExpected APIError
clientReturn APIResponse
statusCode int
}{
// when there is a JSON binding error
{toCreate: "",
handlerExpected: APIError{StatusCode: http.StatusBadRequest, Message: EmailRequiredError},
clientReturn: APIResponse{},
statusCode: http.StatusBadRequest},
// when email is missing
{toCreate: CreateContactRequest{
Contact: Contact{
Name: "test",
}},
handlerExpected: APIError{StatusCode: http.StatusBadRequest, Message: JSONBindingError},
clientReturn: APIResponse{},
statusCode: http.StatusBadRequest},
}
// act
for i, test := range tests {
req := helpers.NewTestRequest(t)
req.SetJSONRequestBody(test.toCreate)
CreateContactHandler(req.Context)
// assert
req.AssertStatusCode(test.statusCode, "Test %d", i)
req.AssertJSONResponseBody(&test.handlerExpected, "Test %d", i)
}
}
Here's the resolution per #mkopriva:
func (r *TestRequest) AssertJSONResponseBody(expectedObj interface{}, masgAndArgs ...interface{}) bool {
elem := reflect.TypeOf(expectedObj)
theType := elem.Elem()
newInstance := reflect.New(theType)
out := newInstance.Interface()
err := json.NewDecoder(r.Recorder.Body).Decode(out)
if err != nil {
return assert.Error(r.t, err)
}
return assert.Equal(r.t, expectedObj, out, masgAndArgs...)
}