Firestore exception occurred in retry method that was not classified as transient - google-cloud-platform

I'm using GCP with node 16 and firestore. I'm getting data of subcollection and update by id. In console.log(subCollect.id); and console.log(subCollect.data()); the information logged is correct. In update command I receive the folowing error
var subCollecPromises = [];
var parentPromises = [];
var querySnapshot = await db.collectionGroup("mysubcollection")
.where('name', '==', "subname")
.where('sequence', '==', 2).get();
querySnapshot.forEach(doc => {
var data = doc.data();
if(data.enable){
var childRef;
var parentRef;
childRef = doc.ref.parent;
parentRef = childRef.parent;
subCollecPromises.push(doc.ref.get());
parentPromises.push(parentRef.get());
}
});
const arrsubCollecSnap = await Promise.all(subCollecPromises);
const arrsubParentSnap = await Promise.all(parentPromises);
for (let index = 0; index < arrsubParentSnap.length; index++) {
const item = arrsubParentSnap[index];
var parentData = item.data();
var subCollect = arrsubCollecSnap[index];
console.log(subCollect.id);
console.log(subCollect.data());
await db.collection("mysubcollection").doc(subCollect.id)
.update({sequence: 3, datetime: new Date()});
await sendMail(parentData.mail);
}
Error in update:
Error: 5 NOT_FOUND: no entity to update: app: "dev~myapp-backend"
path <
Element {
type: "mysubcollection"
name: "Vuxx2Hy9xprtm7tZFyne"
}
>
code: 5,
details: 'no entity to update: app: "dev~myapp-backend"\n' +
'path <\n' +
' Element {\n' +
' type: "mysubcollection"\n' +
' name: "Vuxx2Hy9xprtm7tZFyne"\n' +
' }\n' +
'>\n',
metadata: Metadata {
internalRepr: Map(1) { 'content-type' => [Array] },
options: {}
},
note: 'Exception occurred in retry method that was not classified as transient'

The correct way for update a subcollection is starts at parent collection:
await db.collection("parentCollection").doc(parentId).collection("mysubcollection").doc(subCollect.id).update({sequence: 3, datetime: new Date()});

Related

Flutter aws amplify not returning data when calling graphql api

On button click I have programmed to call a graphql api which is connected to a Lambda function and the function is pulling data from a dynamodb table. The query does not produce any error, but it doesn't give me any results as well. I have also checked the cloudwatch logs and I dont see any traces of the function being called. Not sure on the careless mistake I am making here.
Here is my api
void findUser() async {
try {
String graphQLDocument = '''query getUserById(\$userId: ID!) {
getUserById(userId: \$id) {
id
name
}
}''';
var operation = Amplify.API.query(
request: GraphQLRequest<String>(
document: graphQLDocument,
variables: {'id': 'USER-14160000000'}));
var response = await operation.response;
var data = response.data;
print('Query result: ' + data);
} on ApiException catch (e) {
print('Query failed: $e');
}
}
Here is my lambda function -
const getUserById = require('./user-queries/getUserById');
exports.handler = async (event) => {
var userId = event.arguments.userId;
var name = event.arguments.name;
var avatarUrl = event.arguments.avatarUrl;
//console.log('Received Event - ', JSON.stringify(event,3));
console.log(userId);
switch(event.info.fieldName) {
case "getUserById":
return getUserById(userId);
}
};
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region: 'ca-central-1'});
async function getUserById(userId) {
const params = {
TableName:"Bol-Table",
KeyConditionExpression: 'pk = :hashKey and sk = :sortKey',
ExpressionAttributeValues: {
':hashKey': userId,
':sortKey': 'USER'
}
};
try {
const Item = await docClient.query(params).promise();
console.log(Item);
return {
id: Item.Items[0].pk,
name: Item.Items[0].details.displayName,
avatarUrl: Item.Items[0].details.avatarUrl,
createdAt: Item.Items[0].details.createdAt,
updatedAt: Item.Items[0].details.updatedAt
};
} catch(err) {
console.log("BOL Error: ", err);
}
}
module.exports = getUserById;
Upon button click I get this
Moving my comment to an answer:
Can you try changing your graphQLDocumnet to
String graphQLDocument = '''query getUserById(\$id: ID!) {
getUserById(userId: \$id) {
id
name
}
}''';
Your variable is $userId and then $id. Try calling it $id in both places like in your variables object.
Your flutter code is working fine but in lambda from the aws is returning blank string "" to not to print anything

How to point AWS-SDK DynamoDB to a serverless DynamoDB local

I am trying to write a script that will loop thru an array of items for a DynamoDB table and run a batch write command. My functionality is good, but I am having trouble with DynamoDB. Would be great if I could point my AWS.DynamoDB.DocumentClient() to my localhost running DynamoDB. Any tips?
Would also consider a way to just run the commands via the aws cli but I am not sure how to do that. I am running Node.js so it maybe possible?
Here is my code:
var AWS = require('aws-sdk');
AWS.config.update({ region: 'eu-central-1' });
var DynamoDB = new AWS.DynamoDB.DocumentClient()
DynamoDB.endpoint = 'http://localhost:8000';
const allItems = require('./resource.json');
const tableName = 'some-table-name';
console.log({ tableName, allItems });
var batches = [];
var currentBatch = [];
var count = 0;
for (let i = 0; i < allItems.length; i++) {
//push item to the current batch
count++;
currentBatch.push(allItems[i]);
if (count % 25 === 0) {
batches.push(currentBatch);
currentBatch = [];
}
}
//if there are still items left in the curr batch, add to the collection of batches
if (currentBatch.length > 0 && currentBatch.length !== 25) {
batches.push(currentBatch);
}
var completedRequests = 0;
var errors = false;
//request handler for DynamoDB
function requestHandler(err, data) {
console.log('In the request handler...');
return function (err, data) {
completedRequests++;
errors = errors ? true : err;
//log error
if (errors) {
console.error(JSON.stringify(err, null, 2));
console.error('Request caused a DB error.');
console.error('ERROR: ' + err);
console.error(JSON.stringify(err, null, 2));
} else {
var res = {
statusCode: 200,
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true,
},
body: JSON.stringify(data),
isBase64Encoded: false,
};
console.log(`Success: returned ${data}`);
return res;
}
if (completedRequests == batches.length) {
return errors;
}
};
}
//Make request
var params;
for (let j = 0; j < batches.length; j++) {
//items go in params.RequestedItems.id array
//format for the items is {PutRequest : {Item: ITEM_OBJECT}}
params = '{"RequestItems": {"' + tableName + '": []}}';
params = JSON.parse(params);
params.RequestItems[tableName] = batches[j];
console.log('before db.batchWriteItem: ', params);
//send to db
DynamoDB.batchWrite(params, requestHandler(params));
}
I figured it out and will leave this here for anyone that may need it.
var { DynamoDB } = require('aws-sdk');
var db = new DynamoDB.DocumentClient({
region: 'localhost',
endpoint: 'http://localhost:8000',
});

How to choose different Lambda function while Start Streaming to Amazon Elasticsearch Service

Following this Streaming CloudWatch Logs Data to Amazon Elasticsearch Service, it's working fine to stream cloud watch log to ELK having one log group and one Lambda function.
But now I want to change target lambda function for my other logs group, but I am not able to do that as there is no option in AWS console.
Any Help will be appreciated.
Thanks
I was streaming to ELK using the AWS console option which is Start Streaming to Amazon Elasticsearch Service, But I failed to change or choose different lambda function as there is only lambda function can be selected for any log group using this option.
So, I create new lambda function and set stream target to AWS lambda function,
Here is the code that all you need, Node version for lambda function is 4.* as it was some issue with the new version but the pulse point is it does not require any extra NPM packages.
// v1.1.2
var https = require('https');
var zlib = require('zlib');
var crypto = require('crypto');
var endpoint = 'search-my-test.us-west-2.es.amazonaws.com';
exports.handler = function(input, context) {
// decode input from base64
var zippedInput = new Buffer(input.awslogs.data, 'base64');
// decompress the input
zlib.gunzip(zippedInput, function(error, buffer) {
if (error) { context.fail(error); return; }
// parse the input from JSON
var awslogsData = JSON.parse(buffer.toString('utf8'));
// transform the input to Elasticsearch documents
var elasticsearchBulkData = transform(awslogsData);
// skip control messages
if (!elasticsearchBulkData) {
console.log('Received a control message');
context.succeed('Control message handled successfully');
return;
}
// post documents to the Amazon Elasticsearch Service
post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
console.log('Response: ' + JSON.stringify({
"statusCode": statusCode
}));
if (error) {
console.log('Error: ' + JSON.stringify(error, null, 2));
if (failedItems && failedItems.length > 0) {
console.log("Failed Items: " +
JSON.stringify(failedItems, null, 2));
}
context.fail(JSON.stringify(error));
} else {
console.log('Success: ' + JSON.stringify(success));
context.succeed('Success');
}
});
});
};
function transform(payload) {
if (payload.messageType === 'CONTROL_MESSAGE') {
return null;
}
var bulkRequestBody = '';
payload.logEvents.forEach(function(logEvent) {
var timestamp = new Date(1 * logEvent.timestamp);
// index name format: cwl-YYYY.MM.DD
var indexName = [
'prod-background-wo-' + timestamp.getUTCFullYear(), // year
('0' + (timestamp.getUTCMonth() + 1)).slice(-2), // month
('0' + timestamp.getUTCDate()).slice(-2) // day
].join('.');
var source = buildSource(logEvent.message, logEvent.extractedFields);
source['response_time'] = source["end"] - source["start"];
source['#id'] = logEvent.id;
source['#timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
source['#message'] = logEvent.message;
source['#owner'] = payload.owner;
source['#log_group'] = payload.logGroup;
source['#log_stream'] = payload.logStream;
var action = { "index": {} };
action.index._index = indexName;
action.index._type = payload.logGroup;
action.index._id = logEvent.id;
bulkRequestBody += [
JSON.stringify(action),
JSON.stringify(source),
].join('\n') + '\n';
});
return bulkRequestBody;
}
function buildSource(message, extractedFields) {
if (extractedFields) {
var source = {};
for (var key in extractedFields) {
if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
var value = extractedFields[key];
if (isNumeric(value)) {
source[key] = 1 * value;
continue;
}
jsonSubString = extractJson(value);
if (jsonSubString !== null) {
source['$' + key] = JSON.parse(jsonSubString);
}
source[key] = value;
}
}
return source;
}
jsonSubString = extractJson(message);
if (jsonSubString !== null) {
return JSON.parse(jsonSubString);
}
return {};
}
function extractJson(message) {
var jsonStart = message.indexOf('{');
if (jsonStart < 0) return null;
var jsonSubString = message.substring(jsonStart);
return isValidJson(jsonSubString) ? jsonSubString : null;
}
function isValidJson(message) {
try {
JSON.parse(message);
} catch (e) { return false; }
return true;
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
function post(body, callback) {
var requestParams = buildRequest(endpoint, body);
var request = https.request(requestParams, function(response) {
var responseBody = '';
response.on('data', function(chunk) {
responseBody += chunk;
});
response.on('end', function() {
var info = JSON.parse(responseBody);
var failedItems;
var success;
if (response.statusCode >= 200 && response.statusCode < 299) {
failedItems = info.items.filter(function(x) {
return x.index.status >= 300;
});
success = {
"attemptedItems": info.items.length,
"successfulItems": info.items.length - failedItems.length,
"failedItems": failedItems.length
};
}
var error = response.statusCode !== 200 || info.errors === true ? {
"statusCode": response.statusCode,
"responseBody": responseBody
} : null;
callback(error, success, response.statusCode, failedItems);
});
}).on('error', function(e) {
callback(e);
});
request.end(requestParams.body);
}
function buildRequest(endpoint, body) {
var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
var region = endpointParts[2];
var service = endpointParts[3];
var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
var date = datetime.substr(0, 8);
var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
var kRegion = hmac(kDate, region);
var kService = hmac(kRegion, service);
var kSigning = hmac(kService, 'aws4_request');
var request = {
host: endpoint,
method: 'POST',
path: '/_bulk',
body: body,
headers: {
'Content-Type': 'application/json',
'Host': endpoint,
'Content-Length': Buffer.byteLength(body),
'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
'X-Amz-Date': datetime
}
};
var canonicalHeaders = Object.keys(request.headers)
.sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
.map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
.join('\n');
var signedHeaders = Object.keys(request.headers)
.map(function(k) { return k.toLowerCase(); })
.sort()
.join(';');
var canonicalString = [
request.method,
request.path, '',
canonicalHeaders, '',
signedHeaders,
hash(request.body, 'hex'),
].join('\n');
var credentialString = [ date, region, service, 'aws4_request' ].join('/');
var stringToSign = [
'AWS4-HMAC-SHA256',
datetime,
credentialString,
hash(canonicalString, 'hex')
] .join('\n');
request.headers.Authorization = [
'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
'SignedHeaders=' + signedHeaders,
'Signature=' + hmac(kSigning, stringToSign, 'hex')
].join(', ');
return request;
}
function hmac(key, str, encoding) {
return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}
function hash(str, encoding) {
return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}

Streaming Insert/Update in Google Cloud - BigQuery

I am trying to Stream Salesforce data to Google Cloud Bigquery. Managed to implement the Insert stream, for eg: whenever a new Lead got created in SF, the same will be inserted to Biguery tables. checking on, is there a way i can go for Upsert of data. I knew that there is a streaming Buffer which wont allow to do any DML operation on the inserted data since those will be on streaming buffer for a short duration.
Really Appreciate any hints on Upsert part
Edited - June6th 2019
Using below cloud function for Inserting record
/**
* Responds to any HTTP request.
*
* #param {!express:Request} req HTTP request context.
* #param {!express:Response} res HTTP response context.
*/
exports.helloWorld = (req, res) => {
let message = req.query.mes || req.body.mes || 'Hello World!';
res.status(200).send(req.body);
var d =JSON.stringify(req.body);
console.log(d);
var e = d.replace(/:""/g, '');
var f = e.replace(/\\/g, '');
var g = f.replace(/"{n /g, '');
var h = g.replace(/n}"/g, '');
var i = h.replace(/n /g, '');
console.log(i);
const {BigQuery} = require('#google-cloud/bigquery');
const bigquery = new BigQuery();
var instance = "DEMO";
var table = "HTTP";
bigquery
.dataset(instance)
.table(table)
.insert(JSON.parse(i),
{'ignoreUnknownValues':true, 'raw':false})
.then ((data) => {
console.log('Inserted 1 rows');
console.log(data);
})
};
Code which worked for Update
exports.helloWorld = (req, res) => {
let message = req.query.mes || req.body.mes || 'Hello World!';
res.status(200).send(req.body);
var d =JSON.stringify(req.body);
console.log(d);
var e = d.replace(/:""/g, '');
var f = e.replace(/\\/g, '');
var g = f.replace(/"{n /g, '');
var h = g.replace(/n}"/g, '');
var i = h.replace(/n /g, '');
console.log(i);
var j = JSON.parse(i);
var k = JSON.stringify(j.Id);
var id = k.replace(/"/g, '');
console.log(k);
console.log(id);
const {BigQuery} = require('#google-cloud/bigquery');
const bigquery = new BigQuery();
var instance = "DEMO";
var table = "LEADS_STG";
bigquery
.dataset(instance)
.table(table)
.insert(JSON.parse(i),
{'ignoreUnknownValues':true, 'raw':false})
.then ((data) => {
console.log('Inserted 1 rows');
console.log(data);
})
/*const bigqueryClient = new BigQuery();*/
var delayInMilliseconds = 1000;
setTimeout(function() {
bigquery.query({
query: [
'MERGE DEMO.LEADS_D T USING (SELECT ID,NAME,LEADSOURCE,COMPANY FROM DEMO.LEADS_STG where ID= ? order by LASTMODIFIEDDATE DESC LIMIT 1) S ON T.ID = S.ID WHEN MATCHED THEN UPDATE SET NAME = S.NAME, LEADSOURCE = S.LEADSOURCE, COMPANY = S.COMPANY WHEN NOT MATCHED THEN INSERT ( ID, NAME, LEADSOURCE, COMPANY) VALUES( ID, NAME,LEADSOURCE,COMPANY)'
].join(' '),
params: [
id
]
}, function(err, rows) {});
}, delayInMilliseconds);
};

AWS DynamoDB returns validation error when called from AWS Lambda

I'm using AWS Lambda and try to write something to AWS DynamoDB. I use the following code:
var tableName = "locations";
var item = {
deviceId: {
S: event.deviceId
},
timestamps: {
S: event.timestamp
}
}
var params = {
TableName: tableName,
Item: item
};
dynamo.putItem(params, function(err, data) {
if (err) {
context.fail(new Error('Error ' + err));
} else {
context.success(null);
}
});
And I get the following error:
returns Error ValidationException: One or more parameter values were invalid: Type mismatch for key deviceId expected: S actual: M
This happened because the aws sdk for Nodejs had changed!
If you are using:
var doc = require('dynamodb-doc');
var dynamo = new doc.DynamoDB();
Then the parameters to the putItem call (and most other calls) have changed and instead needs to be:
var tableName = "locations";
var item = {
deviceId: event.deviceId,
timestamp: event.timestamp,
latitude: Number(event.latitude),
longitude: Number(event.longitude)
}
var params = {
TableName: tableName,
Item: item
};
Read all about the new sdk here: https://github.com/awslabs/dynamodb-document-js-sdk