Unable to decode CloudWatch Logs from Kinesis stream - amazon-web-services

Question

I am trying to "funnel" my cloudwatch logs through Kinesis and then to lambda for processing, however I cannot find a way to decode/parse the incoming logs.
So far I have tried this:
Method 1 using cloudwatch "class"
func function(request events.KinesisEvent) error {
for _, record := range request.Records {
fmt.Println(record.EventName)
fmt.Println(string(record.Kinesis.Data))
rawData := events.CloudwatchLogsRawData{
Data: string(record.Kinesis.Data),
}
parse, err := rawData.Parse()
fmt.Println(parse)
fmt.Println(err)
}
return nil
}
func main() {
lambda.Start(function)
}
Method 2 manual decoding
var logData events.CloudwatchLogsData
func Base64Decode(message []byte) (b []byte, err error) {
var l int
b = make([]byte, base64.StdEncoding.DecodedLen(len(message)))
l, err = base64.StdEncoding.Decode(b, message)
if err != nil {
return
}
return b[:l], nil
}
func Parse(rawData []byte, d events.CloudwatchLogsData) (err error) {
data, err := Base64Decode(rawData)
if err != nil {
return
}
zr, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return
}
defer zr.Close()
fmt.Println(zr)
dec := json.NewDecoder(zr)
err = dec.Decode(&d)
return
}
func function(request events.KinesisEvent) error {
for _, record := range request.Records {
fmt.Println(record.EventName)
fmt.Println(string(record.Kinesis.Data))
err = Parse(record.Kinesis.Data, logData)
fmt.Println(err)
fmt.Println(logData)
}
return nil
}
func main() {
lambda.Start(function)
}
Both of them I get the same error:
illegal base64 data at input byte 0
So as my understanding the log format received in in Base64 and compressed, but I cannot find anything online specifically for Go.
EDIT:
Added logData type
// CloudwatchLogsData is an unmarshal'd, ungzip'd, cloudwatch logs event
type CloudwatchLogsData struct {
Owner string `json:"owner"`
LogGroup string `json:"logGroup"`
LogStream string `json:"logStream"`
SubscriptionFilters []string `json:"subscriptionFilters"`
MessageType string `json:"messageType"`
LogEvents []CloudwatchLogsLogEvent `json:"logEvents"`
}
The Base64 decoded and decompressed data is formatted as JSON with the following structure: (According to AWS: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html)
{
"owner": "111111111111",
"logGroup": "logGroup_name",
"logStream": "111111111111_logGroup_name_us-east-1",
"subscriptionFilters": [
"Destination"
],
"messageType": "DATA_MESSAGE",
"logEvents": [
{
"id": "31953106606966983378809025079804211143289615424298221568",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
},
{
"id": "31953106606966983378809025079804211143289615424298221569",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
},
{
"id": "31953106606966983378809025079804211143289615424298221570",
"timestamp": 1432826855000,
"message": "{\"eventVersion\":\"1.03\",\"userIdentity\":{\"type\":\"Root\"}"
}
]
}

Answer 1

Ok, turns out that I did not have to decode from base64, but simply uncompress the data
func Unzip(data []byte) error {
rdata := bytes.NewReader(data)
r, err := gzip.NewReader(rdata)
if err != nil {
return err
}
uncompressedData, err := ioutil.ReadAll(r)
if err != nil {
return err
}
fmt.Println(string(uncompressedData))
return nil
}
The uncompressedData is the JSON string of the cloudwatch log