I have the following IAM Policy:
{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":"arn:aws:sts::<account>:assumed-role/custom_role/<role>"},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"sts:ExternalId":"<account>"}}}]}
but the "AWS" portion can also be an array:
"AWS": [
"arn:aws:sts::<account>:assumed-role/custom_role/<role_1>",
"arn:aws:sts::<account>:assumed-role/custom_role/<role_2>"
]
What I need is a regex that can parse both structures and return the list of arn:aws:sts as a list of strings... how can I accomplish that using regex in Golang?
I tried to use json.Unmarshal but the object structure is different between []string and string
Edit:
I have the following snippet:
re := regexp.MustCompile(`arn:aws:sts::[a-z0-9]*:assumed-role/custom_role/[a-z0-9]-*`)
result := re.FindAll([]byte(arn), 10)
for _, res := range result {
fmt.Println(string(res))
}
>>> `arn:aws:sts::<account_id>:assumed-role/custom_role/`
Using JSON decoder
You can decode the AWS key directly into a custom type implementing the "json.Unmarshaler" interface and decode both inputs correctly.
Demo
type AWSRoles []string
func (r *AWSRoles) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err == nil {
*r = append(*r, s)
return nil
}
var ss []string
if err := json.Unmarshal(b, &ss); err == nil {
*r = ss
return nil
}
return errors.New("cannot unmarshal neither to a string nor a slice of strings")
}
type AWSPolicy struct {
Statement []struct {
Principal struct {
AWSRoles AWSRoles `json:"AWS"`
} `json:"Principal"`
} `json:"Statement"`
}
Here's a test for it
var testsAWSPolicyParsing = []struct {
name string
input []byte
wantRoles []string
}{
{
name: "unique role",
input: []byte(`{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":"arn:aws:sts::<account>:assumed-role/custom_role/<role>"},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"sts:ExternalId":"<account>"}}}]}`),
wantRoles: []string{"arn:aws:sts::<account>:assumed-role/custom_role/<role>"},
},
{
name: "multiple roles",
input: []byte(`{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["arn:aws:sts::<account>:assumed-role/custom_role/<role_1>","arn:aws:sts::<account>:assumed-role/custom_role/<role_2>"]},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"sts:ExternalId":"<account>"}}}]}`),
wantRoles: []string{
"arn:aws:sts::<account>:assumed-role/custom_role/<role_1>",
"arn:aws:sts::<account>:assumed-role/custom_role/<role_2>",
},
},
}
func TestParseAWSPolicy(t *testing.T) {
for _, tc := range testsAWSPolicyParsing {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var p AWSPolicy
err := json.Unmarshal(tc.input, &p)
if err != nil {
t.Fatal("unexpected error parsing AWSRoles policy", err)
}
if l := len(p.Statement); l != 1 {
t.Fatalf("unexpected Statement length. want 1, got %d", l)
}
if got := p.Statement[0].Principal.AWSRoles; !reflect.DeepEqual(got, tc.wantRoles) {
t.Fatalf("roles are not the same, got %v, want %v", got, tc.wantRoles)
}
})
}
}
Using a Regex
If you still want to use a regex, this one would parse it as long as:
AWS account has only numbers [0-9]
the custom role name has only alphanumeric characters and underscores
var awsRolesRegex = regexp.MustCompile("arn:aws:sts::[a-z0-9]+:assumed-role/custom_role/[a-zA-Z0-9_]+")
Demo
Related
I have a function which gets all PDF files in a directory and returns the files if there are some.
func GetPdfFiles(path string) ([]string, error) {
var files []string
err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
if strings.HasSuffix(path, ".pdf") {
files = append(files, path)
}
return nil
})
if err != nil {
return nil, err
}
if files == nil {
return nil, errors.New("No PdfFiles found.")
}
return files, nil
}
My Test function gets the error: nil, from the filepath.Walk function which requires a anonymous function that returns an error, but it should get the error from the if statements like in the case of the second testcase with no files it should return errors.New("No PdfFiles found.").
How can i test it correctly.
func TestGetPdfFiles(t *testing.T) {
type args struct {
path string
}
cwd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
tests := []struct {
name string
args args
want []string
wantErr bool
}{
{name: "2PdfFiles", args: args{path: fmt.Sprintf("%s/testData/", cwd)}, want: []string{fmt.Sprintf("%s/testData/test-1.pdf", cwd), fmt.Sprintf("%s/testData/test-2.pdf", cwd)}, wantErr: false},
{name: "noPdfFiles", args: args{path: fmt.Sprintf("%s", cwd)}, want: nil, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetPdfFiles(tt.args.path)
if (err != nil) != tt.wantErr {
t.Errorf("GetPdfFiles() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetPdfFiles() got = %v, want %v", got, tt.want)
}
})
}
}
You use dependency injection, and modify your function to accept an implementation of fs.FS. That lets your tests pass it a mock file system.
https://bitfieldconsulting.com/golang/filesystems
https://www.gopherguides.com/articles/golang-1.16-io-fs-improve-test-performance
Or, perhaps simpler for your use case, modify your GetPdfFiles() to accept a directory walker function with the same signature as path.WalkDir():
package main
import (
"io/fs"
"path"
"path/filepath"
"strings"
)
func GetPdfFiles(root string) ([]string, error) {
return GetPdfFilesWithWalker(root, filepath.WalkDir)
}
type DirectoryWalker = func(string, fs.WalkDirFunc) error
func GetPdfFilesWithWalker(root string, walk DirectoryWalker) (fns []string, err error) {
collectPdfFiles := func(fn string, info fs.DirEntry, err error) error {
if ext := strings.ToLower(path.Ext(fn)); ext == ".pdf" {
fns = append(fns, fn)
}
return nil
}
err = walk(root, collectPdfFiles)
return fns, err
}
Now your GetPdfFiles() is a do-nothing wrapper that injects the default implementation (from path/filepath), and the core is in GetPdfFilesWithWalker(), against which you write your tests, passing in a suitable mock.
you can even construct a mock that will return errors, so you can test your error handling.
Your mock directory walker can be as simple as something like this (especially since you only use the path passed to the callback:
func MockDirectoryWalker(root string, visit fs.WalkDirFunc) (err error) {
paths := [][]string{
{root, "a"},
{root, "a", "a.pdf"},
{root, "a", "b.txt"},
{root,"a", "b"},
{root, "a", "b", "c.pdf"},
{root, "a", "b", "d.txt"},
}
for _, p := range paths {
fqn := path.Join(p...)
var di fs.DirEntry
visit(fqn, di, nil)
}
return err
}
How about using T.TempDir() for a unique empty directory every time. This will guarantee repeatable results. The testing package will also take care of cleaning that:
func TestGetPdfFiles(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
want []string
wantErr bool
}{
{name: "noPdfFiles", args: args{path: t.TempDir()}, want: nil, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetPdfFiles(tt.args.path)
if (err != nil) != tt.wantErr {
t.Errorf("GetPdfFiles() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetPdfFiles() got = %v, want %v", got, tt.want)
}
})
}
}
I guess, there isn't really a good way to do it.
so i removed the error checks and just return an empty array, it would've been too messy.
Also thanks to #Qasim Sarfraz for the T.TempDir idea.
func GetPdfFiles(path string) []string {
var files []string
filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
if strings.HasSuffix(path, ".pdf") {
files = append(files, path)
}
return nil
})
return files
}
func TestGetPdfFiles(t *testing.T) {
type args struct {
path string
}
cwd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
tests := []struct {
name string
args args
want []string
wantErr bool
}{
{name: "2PdfFiles", args: args{path: fmt.Sprintf("%s/testData/", cwd)}, want: []string{fmt.Sprintf("%s/testData/test-1.pdf", cwd), fmt.Sprintf("%s/testData/test-2.pdf", cwd)}},
{name: "noPdfFiles", args: args{path: t.TempDir()}, want: nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GetPdfFiles(tt.args.path)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetPdfFiles() got = %v, want %v", got, tt.want)
}
})
}
}
I am trying to "funnel" my cloudwatch logs through Kinesis and then to lambda for processing, however I cannot find a way to decode/parse the incoming logs.
So far I have tried this:
Method 1 using cloudwatch "class"
func function(request events.KinesisEvent) error {
for _, record := range request.Records {
fmt.Println(record.EventName)
fmt.Println(string(record.Kinesis.Data))
rawData := events.CloudwatchLogsRawData{
Data: string(record.Kinesis.Data),
}
parse, err := rawData.Parse()
fmt.Println(parse)
fmt.Println(err)
}
return nil
}
func main() {
lambda.Start(function)
}
Method 2 manual decoding
var logData events.CloudwatchLogsData
func Base64Decode(message []byte) (b []byte, err error) {
var l int
b = make([]byte, base64.StdEncoding.DecodedLen(len(message)))
l, err = base64.StdEncoding.Decode(b, message)
if err != nil {
return
}
return b[:l], nil
}
func Parse(rawData []byte, d events.CloudwatchLogsData) (err error) {
data, err := Base64Decode(rawData)
if err != nil {
return
}
zr, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return
}
defer zr.Close()
fmt.Println(zr)
dec := json.NewDecoder(zr)
err = dec.Decode(&d)
return
}
func function(request events.KinesisEvent) error {
for _, record := range request.Records {
fmt.Println(record.EventName)
fmt.Println(string(record.Kinesis.Data))
err = Parse(record.Kinesis.Data, logData)
fmt.Println(err)
fmt.Println(logData)
}
return nil
}
func main() {
lambda.Start(function)
}
Both of them I get the same error:
illegal base64 data at input byte 0
So as my understanding the log format received in in Base64 and compressed, but I cannot find anything online specifically for Go.
EDIT:
Added logData type
// CloudwatchLogsData is an unmarshal'd, ungzip'd, cloudwatch logs event
type CloudwatchLogsData struct {
Owner string `json:"owner"`
LogGroup string `json:"logGroup"`
LogStream string `json:"logStream"`
SubscriptionFilters []string `json:"subscriptionFilters"`
MessageType string `json:"messageType"`
LogEvents []CloudwatchLogsLogEvent `json:"logEvents"`
}
The Base64 decoded and decompressed data is formatted as JSON with the following structure: (According to AWS: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html)
{
"owner": "111111111111",
"logGroup": "logGroup_name",
"logStream": "111111111111_logGroup_name_us-east-1",
"subscriptionFilters": [
"Destination"
],
"messageType": "DATA_MESSAGE",
"logEvents": [
{
"id": "31953106606966983378809025079804211143289615424298221568",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
},
{
"id": "31953106606966983378809025079804211143289615424298221569",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
},
{
"id": "31953106606966983378809025079804211143289615424298221570",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
}
]
}
Ok, turns out that I did not have to decode from base64, but simply uncompress the data
func Unzip(data []byte) error {
rdata := bytes.NewReader(data)
r, err := gzip.NewReader(rdata)
if err != nil {
return err
}
uncompressedData, err := ioutil.ReadAll(r)
if err != nil {
return err
}
fmt.Println(string(uncompressedData))
return nil
}
The uncompressedData is the JSON string of the cloudwatch log
I am building a simple function that calls an API that returns a Post using GraphQL (https://github.com/machinebox/graphql). I wrapped the logic in a service that looks like this:
type Client struct {
gcl graphqlClient
}
type graphqlClient interface {
Run(ctx context.Context, req *graphql.Request, resp interface{}) error
}
func (c *Client) GetPost(id string) (*Post, error) {
req := graphql.NewRequest(`
query($id: String!) {
getPost(id: $id) {
id
title
}
}
`)
req.Var("id", id)
var resp getPostResponse
if err := c.gcl.Run(ctx, req, &resp); err != nil {
return nil, err
}
return resp.Post, nil
}
Now I'd like to add test tables for the GetPost function with a fail case when id is set to empty string which causes an error in the downstream call c.gcl.Run.
What I am struggling with is the way the gcl client can be mocked and forced to return the error (when no real API call happens).
My test so far:
package apiClient
import (
"context"
"errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/google/go-cmp/cmp"
"github.com/machinebox/graphql"
"testing"
)
type graphqlClientMock struct {
graphqlClient
HasError bool
Response interface{}
}
func (g graphqlClientMock) Run(_ context.Context, _ *graphql.Request, response interface{}) error {
if g.HasError {
return errors.New("")
}
response = g.Response
return nil
}
func newTestClient(hasError bool, response interface{}) *Client {
return &Client{
gcl: graphqlClientMock{
HasError: hasError,
Response: response,
},
}
}
func TestClient_GetPost(t *testing.T) {
tt := []struct{
name string
id string
post *Post
hasError bool
response getPostResponse
}{
{
name: "empty id",
id: "",
post: nil,
hasError: true,
},
{
name: "existing post",
id: "123",
post: &Post{id: aws.String("123")},
response: getPostResponse{
Post: &Post{id: aws.String("123")},
},
},
}
for _, tc := range tt {
t.Run(tc.name, func(t *testing.T) {
client := newTestClient(tc.hasError, tc.response)
post, err := client.GetPost(tc.id)
if err != nil {
if tc.hasError == false {
t.Error("unexpected error")
}
} else {
if tc.hasError == true {
t.Error("expected error")
}
if cmp.Equal(post, &tc.post) == false {
t.Errorf("Response data do not match: %s", cmp.Diff(post, tc.post))
}
}
})
}
}
I am not sure if passing the response to the mock like this is the right way to do it. Also, I'm struggling to set the right value to the response, since an interface{} type is passed and I don't know how to convert it to the getPostResponse and set the value to Post there.
Your test cases should not go beyond the implementation. I'm specifically referring to the empty-vs-nonempty input or any kind of input really.
Let's take a look at the code you want to test:
func (c *Client) GetPost(id string) (*Post, error) {
req := graphql.NewRequest(`
query($id: String!) {
getPost(id: $id) {
id
title
}
}
`)
req.Var("id", id)
var resp getPostResponse
if err := c.gcl.Run(ctx, req, &resp); err != nil {
return nil, err
}
return resp.Post, nil
}
Nothing in the implementation above is doing anything based on the id parameter value and therefore nothing in your tests for this piece of code should really care about what input is passed in, if it is irrelevant to the implementation it should also be irrelevant to the tests.
Your GetPost has basically two code branches that are taken based on a single factor, i.e. the "nilness" of the returned err variable. This means that as far as your implementation is concerned there are only two possible outcomes, based on what err value Run returns, and therefore there should only be two test cases, a 3rd or 4th test case would be just a variation, if not an outright copy, of the first two.
Your test client is also doing some unnecessary stuff, the main one being its name, i.e. what you have there is not a mock so calling it that is not helpful. Mocks usually do a lot more than just return predefined values, they ensure that methods are called, in the expected order and with the expected arguments, etc. And actually you don't need a mock here at all so it's a good thing it isn't one.
With that in mind, here's what I would suggest you do with your test client.
type testGraphqlClient struct {
resp interface{} // non-pointer value of the desired response, or nil
err error // the error to be returned by Run, or nil
}
func (g testGraphqlClient) Run(_ context.Context, _ *graphql.Request, resp interface{}) error {
if g.err != nil {
return g.err
}
if g.resp != nil {
// use reflection to set the passed in response value
// (i haven't tested this so there may be a bug or two)
reflect.ValueOf(resp).Elem().Set(reflect.ValueOf(g.resp))
}
return nil
}
... and here are the necessary test cases, all two of them:
func TestClient_GetPost(t *testing.T) {
tests := []struct {
name string
post *Post
err error
client testGraphqlClient
}{{
name: "return error from client",
err: errors.New("bad input"),
client: testGraphqlClient{err: errors.New("bad input")},
}, {
name: "return post from client",
post: &Post{id: aws.String("123")},
client: testGraphqlClient{resp: getPostResponse{Post: &Post{id: aws.String("123")}}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := Client{gql: tt.client}
post, err := client.GetPost("whatever")
if !cmp.Equal(err, tt.err) {
t.Errorf("got error=%v want error=%v", err, tt.err)
}
if !cmp.Equal(post, tt.post) {
t.Errorf("got post=%v want post=%v", post, tt.post)
}
})
}
}
... there's a bit of repetition going on here, the need to spell out the post and err twice but that's a small price to pay when compared to a more sophisticated/complicated test setup that would populate the test client from the test case's expected output fields.
Addendum:
If you were to update GetPost in such a way that it checks for the empty id and returns an error before it sends a request to graphql then your initial setup would make much more sense:
func (c *Client) GetPost(id string) (*Post, error) {
if id == "" {
return nil, errors.New("empty id")
}
req := graphql.NewRequest(`
query($id: String!) {
getPost(id: $id) {
id
title
}
}
`)
req.Var("id", id)
var resp getPostResponse
if err := c.gcl.Run(ctx, req, &resp); err != nil {
return nil, err
}
return resp.Post, nil
}
... and updating the test cases accordingly:
func TestClient_GetPost(t *testing.T) {
tests := []struct {
name string
id string
post *Post
err error
client testGraphqlClient
}{{
name: "return empty id error",
id: "",
err: errors.New("empty id"),
client: testGraphqlClient{},
}, {
name: "return error from client",
id: "nonemptyid",
err: errors.New("bad input"),
client: testGraphqlClient{err: errors.New("bad input")},
}, {
name: "return post from client",
id: "nonemptyid",
post: &Post{id: aws.String("123")},
client: testGraphqlClient{resp: getPostResponse{Post: &Post{id: aws.String("123")}}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := Client{gql: tt.client}
post, err := client.GetPost(tt.id)
if !cmp.Equal(err, tt.err) {
t.Errorf("got error=%v want error=%v", err, tt.err)
}
if !cmp.Equal(post, tt.post) {
t.Errorf("got post=%v want post=%v", post, tt.post)
}
})
}
}
I am using GO SDK and using the DynamnoDB BatchGetItem API.
I saw this code example -
https://github.com/aws/aws-sdk-go/blob/master/service/dynamodb/examples_test.go
Is there any other code example which shows unmarshalling of the response from the BatchGetItem API?
Let me share piece of the code. The key to understand it is that when you send GetBatchItem request to dynamodb, you specify map of table names and keys for that table, so response you get is a map of tables names and matched items
placeIDs := []string { "london_123", "sanfran_15", "moscow_9" }
type Place {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}
mapOfAttrKeys := []map[string]*dynamodb.AttributeValue{}
for _, place := range placeIDs {
mapOfAttrKeys = append(mapOfAttrKeys, map[string]*dynamodb.AttributeValue{
"id": &dynamodb.AttributeValue{
S: aws.String(place),
},
"attr": &dynamodb.AttributeValue{
S: aws.String("place"),
},
})
}
input := &dynamodb.BatchGetItemInput{
RequestItems: map[string]*dynamodb.KeysAndAttributes{
tableName: &dynamodb.KeysAndAttributes{
Keys: mapOfAttrKeys,
},
},
}
batch, err := db.BatchGetItem(input)
if err != nil {
panic(fmt.Errorf("batch load of places failed, err: %w", err))
}
for _, table := range batch.Responses {
for _, item := range table {
var place Place
err = dynamodbattribute.UnmarshalMap(item, &place)
if err != nil {
panic(fmt.Errorf("failed to unmarshall place from dynamodb response, err: %w", err))
}
places = append(places, place)
}
}
I use the following code which works ok.
This is working example
https://play.golang.org/p/wjvJtDNvJAQ
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
type requester interface {
HTTPRequest(c string, i string, mtd string, url string) (p []byte, e error)
}
type impl struct {
client *http.Client
}
// ----This is the function which I need to mock
func (s *ServiceInfo) wrapperFN() {
// Function 1 - get the values
v1, v2 := s.json.parseJson()
// call to http function
s.req.HTTPRequest(v1, v2, "POST", "http://www.mocky.io/v2/5c20eccc2e00005c001e0c84")
}
func (i impl) HTTPRequest(c string, ci string, mtd string, url string) (p []byte, e error) {
req, err := http.NewRequest(mtd, url, nil)
if err != nil {
return nil, err
}
req.SetBasicAuth(c, ci)
res, err := i.client.Do(req)
if err != nil {
return nil, err
}
token, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
fmt.Println("success")
return token, nil
}
type parser interface {
parseJson() (string, string)
}
type jsonP struct {
data string
}
func (s jsonP) parseJson() (string, string) {
var result map[string]interface{}
json.Unmarshal([]byte(s.data), &result)
b := result["person"].(map[string]interface{})
for key, value := range b {
return key, value.(string)
}
return "", ""
}
type ServiceInfo struct {
req requester
json parser
}
// When in production pass in concrete implementations.
func NewServiceInfo(http requester, json parser) *ServiceInfo {
return &ServiceInfo{
req: http,
json: json,
}
}
func main() {
httpClient := http.Client{}
js := `{"person":{"p1":"username","p2":"password"},"customers":"10"}`
j := jsonP{data: js}
s := NewServiceInfo(impl{client: &httpClient}, j)
s.wrapperFN()
}
Now i want to test it wrapperFN , what I try I've changed the code to use interface , which works.
This is just example to give a point ( the real code much more complicated)
The problem that I dont understand how to mock function inside wrapperFN like parseJson() , in the real world warpperFN contains several function which I need to mock ,because just calling them in the test will provide error.
How it's best to mock function like parseJson() & HTTPRequest? and assume that inside wrapperFN there is additional functions which is not related...
I need to know if this is the best practice for testing function.
This is the test (which im not sure how to make it right)
package main
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestServiceInfo_wrapperFN(t *testing.T) {
tests := []struct {
name string
s *ServiceInfo
}{
{
name: "wrapper test",
s: &ServiceInfo{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var testHandler http.Handler
srv := httptest.NewServer(testHandler)
defer srv.Close()
iReq := &impl{
client: srv.Client(),
}
v := &ServiceInfo{http: *iReq}
v.wrapperFN()
})
}
}