I created a lambda function in AWS (Python) using "upload .zip"
I lost those files and I need to make some changes, is there is any way to download that .zip?
Yes!
Navigate over to your lambda function settings and on the top right you will have a button called "Actions". In the drop down menu select "export" and in the popup click "Download deployment package" and the function will download in a .zip file.
Action button on top-right
A popup from CTA above (Tap "Download deployment package" here)
Update: Added link to script by sambhaji-sawant. Fixed Typos, improved answer and script based on comments!
You can use aws-cli to download the zip of any lambda.
First you need to get the URL to the lambda zip
$ aws lambda get-function --function-name $functionName --query 'Code.Location'
Then you need to use wget/curl to download the zip from the URL.
$ wget -O myfunction.zip URL_from_step_1
Additionally you can list all functions on your AWS account using
$ aws lambda list-functions
I made a simple bash script to parallel download all the lambda functions from your AWS account. You can see it
here :)
Note: You will need to setup aws-cli before using the above commands (or any aws-cli command) using aws configure
Full guide here
You can use shell script available here
If you want to download all the functions in the given region here is my workaround.
I have created a simple node script to download function. Install all the required npm packages and set your AWS CLI to the region you want before running the script.
let download = require('download-file');
let extract = require('extract-zip');
let cmd = require('node-cmd');
let downloadFile = async function (dir, filename, url) {
let options = {
directory: dir,
filename: filename
}
return new Promise((success, failure) => {
download(url, options, function (err) {
if (err) {
failure(err)
} else {
success('done');
}
})
})
}
let extractZip = async function (source, target) {
return new Promise((success, failure) => {
extract(source, { dir: target }, function (err) {
if (err) {
failure(err)
} else {
success('done');
}
})
})
}
let getAllFunctionList = async function () {
return new Promise((success, failure) => {
cmd.get(
'aws lambda list-functions',
function (err, data, stderr) {
if (err || stderr) {
failure(err || stderr)
} else {
success(data)
}
}
);
})
}
let getFunctionDescription = async function (name) {
return new Promise((success, failure) => {
cmd.get(
`aws lambda get-function --function-name ${name}`,
function (err, data, stderr) {
if (err || stderr) {
failure(err || stderr)
} else {
success(data)
}
}
);
})
}
let init = async function () {
try {
let { Functions: getAllFunctionListResult } = JSON.parse(await getAllFunctionList());
let getFunctionDescriptionResult, downloadFileResult, extractZipResult;
getAllFunctionListResult.map(async (f) => {
var { Code: { Location: getFunctionDescriptionResult } } = JSON.parse(await getFunctionDescription(f.FunctionName));
downloadFileResult = await downloadFile('./functions', `${f.FunctionName}.zip`, getFunctionDescriptionResult)
extractZipResult = await extractZip(`./functions/${f.FunctionName}.zip`, `/Users/malhar/Desktop/get-lambda-functions/final/${f.FunctionName}`)
console.log('done', f.FunctionName);
})
} catch (e) {
console.log('error', e);
}
}
init()
Here is a bash script that I used, it downloads all the functions in the default region:
download_code () {
local OUTPUT=$1
OUTPUT=`sed -e 's/,$//' -e 's/^"//' -e 's/"$//g' <<<"$OUTPUT"`
url=$(aws lambda get-function --function-name get-marvel-movies-from-opensearch --query 'Code.Location' )
wget $url -O $OUTPUT.zip
}
FUNCTION_LIST=$(aws lambda list-functions --query Functions[*].FunctionName)
for run in $FUNCTION_LIST
do
download_code $run
done
echo "Finished!!!!"
You can find a python script to download all the lambda functions here.
import os
import sys
from urllib.request import urlopen
import zipfile
from io import BytesIO
import boto3
def get_lambda_functions_code_url():
client = boto3.client("lambda")
lambda_functions = [n["FunctionName"] for n in client.list_functions()["Functions"]]
functions_code_url = []
for fn_name in lambda_functions:
fn_code = client.get_function(FunctionName=fn_name)["Code"]
fn_code["FunctionName"] = fn_name
functions_code_url.append(fn_code)
return functions_code_url
def download_lambda_function_code(fn_name, fn_code_link, dir_path):
function_path = os.path.join(dir_path, fn_name)
if not os.path.exists(function_path):
os.mkdir(function_path)
with urlopen(fn_code_link) as lambda_extract:
with zipfile.ZipFile(BytesIO(lambda_extract.read())) as zfile:
zfile.extractall(function_path)
if __name__ == "__main__":
inp = sys.argv[1:]
print("Destination folder {}".format(inp))
if inp and os.path.exists(inp[0]):
dest = os.path.abspath(inp[0])
fc = get_lambda_functions_code_url()
print("There are {} lambda functions".format(len(fc)))
for i, f in enumerate(fc):
print("Downloading Lambda function {} {}".format(i+1, f["FunctionName"]))
download_lambda_function_code(f["FunctionName"], f["Location"], dest)
else:
print("Destination folder doesn't exist")
Related
Problem:
I am trying to get the data from a text file stored in s3, I get it right in intent handler using a sync await but I want to get string in localisation file as I am trying to implement the solution in 2 languages.
I am getting err saying skill does not respond correctly.
This is file.js
const AWS = require('aws-sdk');
//========================
// This step is not required if you are running your code inside lambda or in
// the local environment that has AWS set up
//========================
const s3 = new AWS.S3();
async function getS3Object (bucket, objectKey) {
try {
const params = {
Bucket: 'my-bucket',
Key: 'file.txt',
};
const data = await s3.getObject(params).promise();
let dat = data.Body.toString('utf-8');
return dat;
} catch (e) {
throw new Error(`Could not retrieve file from S3: ${e.message}`);
}
}
module.exports = getS3Object;
this is the localisation.js file code
const dataText = require('file.js');
async let textTitle = await dataText().then(); **// this does not work**
module.exports = {
en: {
translation: {
WELCOME_BACK_MSG : textTitle,
}
},
it: {
translation: {
WELCOME_MSG: textTitle,
}
}
}
The problem is that in your localisation.js file you are trying to export something that is obtained via an asynchronous function call, but you cannot do that directly, module.exports is assigned and returned synchronously. Please, see for instance this SO question and answer for an in-deep background.
As you are mentioning Alexa skill, and for the name of the file, localisation.js, I assume you are trying something similar to the solution proposed in this GitHub repository.
Analyzing the content of the index.js file they provide, it seems the library is using i18next for localisation.
The library provides the concept of backend if you need to load your localisation information from an external resource.
You can implement a custom backend, although the library offers one that could fit your needs, i18next-http-backend.
As indicated in the documentation, you can configure the library to fetch your localization resources with this backend with something like the following:
import i18next from 'i18next';
import Backend from 'i18next-http-backend';
i18next
.use(Backend)
.init({
backend: {
// for all available options read the backend's repository readme file
loadPath: '/locales/{{lng}}/{{ns}}.json'
}
});
Here in SO you can find a more complete example.
You need to provide a similar configuration to the localisation interceptor provided in the Alexa skill example project, perhaps something like:
import HttpApi from 'i18next-http-backend';
/**
* This request interceptor will bind a translation function 't' to the handlerInput
*/
const LocalizationInterceptor = {
process(handlerInput) {
const localisationClient = i18n
.use(HttpApi)
.init({
lng: Alexa.getLocale(handlerInput.requestEnvelope),
// resources: languageStrings,
backend: {
loadPath: 'https://your-bucket.amazonaws.com/locales/{{lng}}/translations.json',
crossDomain: true,
},
returnObjects: true
});
localisationClient.localise = function localise() {
const args = arguments;
const value = i18n.t(...args);
if (Array.isArray(value)) {
return value[Math.floor(Math.random() * value.length)];
}
return value;
};
handlerInput.t = function translate(...args) {
return localisationClient.localise(...args);
}
}
};
Please, be aware that instead of a text file you need to return a valid son file with the appropriate translations:
{
"WELCOME_MSG" : "Welcome!!",
"WELCOME_BACK_MSG" : "Welcome back!!"
}
I have followed this blog to update the code of a lambda function using a jar file stored in a S3 bucket. the execution was succeded, but it is not updating the code of target lambda function
Code snippet
console.log('Loading function');
var AWS = require('aws-sdk');
var lambda = new AWS.Lambda();
exports.handler = function(event, context) {
var functionName = "runJarFile";
var bucket = "jarfiletest2";
var key = "lambda-java-example-0.0.1-SNAPSHOT.jar.zip";
console.log("uploaded to lambda function: " + functionName);
var params = {
FunctionName: functionName,
S3Key: key,
S3Bucket: bucket,
Publish: true
};
lambda.updateFunctionCode(params, function(err, data) {
if (err) {
console.log(err, err.stack);
context.fail(err);
} else {
console.log(data);
context.succeed(data);
}
});
};
Thanks in advance
It's difficult to comment on this without knowing the details about the destination function. What's the output of the GetFunction API call of that Lambda, before and after calling the UpdateFunctionConfig call?
I am interested to see the SHA-256 hash of the code, and the last modified timestamp off that API call before and after calling UpdateFunctionConfig:
{
...
"CodeSha256": "5tT2qgzYUHoqwR616pZ2dpkn/0J1FrzJmlKidWaaCgk=",
"LastModified": "2019-09-24T18:20:35.054+0000"
...
}
If the values are exactly the same, can you add this check as per the blog post to see if the bucket and the object exists?
if (bucket == "YOUR_BUCKET_NAME" && key == "YOUR_CODE.zip" && version) {
// your code
} else {
context.succeed("skipping zip " + key + " in bucket " + bucket + " with version " + version);
}
Pls try to remove 'Publish: true' to call the latest version not the specified version
When I deploy my Cloud Function to GCP (written in Node.js), how can I see my file system environment for debugging purposes? What if I want to know what my current directory is or what files are present alongside my application?
When we deploy a Cloud Function, the full Node.js environment is present. We can run arbitrary Node.js logic within. This includes logging information which will then show in the Stackdriver logs. We can thus log our current working directory path as well as a list of all the files in our current directory. We can use this as a diagnostic aid. Here is an example:
const fs = require('fs');
exports.helloWorld = (req, res) => {
console.log(`CWD: ${process.cwd()}`);
fs.readdir('.', function (err, files) {
if (err) {
return console.log('Unable to scan directory: ' + err);
}
files.forEach(function (file) {
console.log(file);
});
res.status(200).send('Done!');
});
};
You can incorporate this logic in your own apps for testing.
And here is an alternate version which shows a recursive listing of all files and sub directories.
const fs = require('fs');
const walk = function(dir) {
var results = [];
var list = fs.readdirSync(dir);
list.forEach(function(file) {
file = dir + '/' + file;
var stat = fs.statSync(file);
if (stat && stat.isDirectory()) {
results = results.concat(walk(file));
} else {
results.push(file);
}
});
return results;
}
exports.helloWorld = (req, res) => {
let message = req.query.message || req.body.message || 'Hello World!';
console.log(`CWD: ${process.cwd()}`);
console.log(`Dir Listing: ${walk('.')}`);
res.status(200).send('Done!');
};
All credit to the above algorithm to node.js fs.readdir recursive directory search.
Trigger a cloud function whenever a new file is uploaded to cloud storage bucket. This function should call a dataproc job written in pyspark to read the file and load it to BigQuery.
I want to know how to call a google dataproc job from cloud function. Please suggest.
I was able to create a simple Cloud Function that triggers Dataproc Job on GCS create file event. In this example, the file in GCS contains a Pig query to execute. However you can follow Dataproc API documentation to create a PySpark version.
index.js:
exports.submitJob = (event, callback) => {
const google = require('googleapis');
const projectId = 'my-project'
const clusterName = 'my-cluster'
const file = event.data;
if (file.name) {
google.auth.getApplicationDefault(function (err, authClient, projectId) {
if (err) {
throw err;
}
const queryFileUri = "gs://" + file.bucket + "/" + file.name
console.log("Using queryFileUri: ", queryFileUri);
if (authClient.createScopedRequired && authClient.createScopedRequired()) {
authClient = authClient.createScoped([
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/userinfo.email'
]);
}
const dataproc = google.dataproc({ version: 'v1beta2', auth: authClient });
dataproc.projects.regions.jobs.submit({
projectId: projectId,
region: "global",
resource: {
"job": {
"placement": {"clusterName": clusterName},
"pigJob": {
"queryFileUri": queryFileUri,
}
}
}
}, function(err, response) {
if (err) {
console.error("Error submitting job: ", err);
}
console.log("Dataproc response: ", response);
callback();
});
});
} else {
throw "Skipped processing file!";
}
callback();
};
Make sure to set Function to execute to submitJob.
package.json:
{
"name": "sample-cloud-storage",
"version": "0.0.1",
"dependencies":{ "googleapis": "^21.3.0" }
}
The following blogpost gave me many ideas how to get started:
https://cloud.google.com/blog/big-data/2016/04/scheduling-dataflow-pipelines-using-app-engine-cron-service-or-cloud-functions
How do I launch a Cloud Dataflow job from a Google Cloud Function? I'd like to use Google Cloud Functions as a mechanism to enable cross-service composition.
I've included a very basic example of the WordCount sample below. Please note that you'll need to include a copy of the java binary in your Cloud Function deployment, since it is not in the default environment. Likewise, you'll need to package your deploy jar with your Cloud Function as well.
module.exports = {
wordcount: function (context, data) {
const spawn = require('child_process').spawn;
const child = spawn(
'jre1.8.0_73/bin/java',
['-cp',
'MY_JAR.jar',
'com.google.cloud.dataflow.examples.WordCount',
'--jobName=fromACloudFunction',
'--project=MY_PROJECT',
'--runner=BlockingDataflowPipelineRunner',
'--stagingLocation=gs://STAGING_LOCATION',
'--inputFile=gs://dataflow-samples/shakespeare/*',
'--output=gs://OUTPUT_LOCATION'
],
{ cwd: __dirname });
child.stdout.on('data', function(data) {
console.log('stdout: ' + data);
});
child.stderr.on('data', function(data) {
console.log('error: ' + data);
});
child.on('close', function(code) {
console.log('closing code: ' + code);
});
context.success();
}
}
You could further enhance this example by using the non-blocking runner and having the function return the Job ID, so that you can poll for job completion separately. This pattern should be valid for other SDKs as well, so long as their dependencies can be packaged into the Cloud Function.
The best way is to launch is via cloud function but be careful, if you are using the cloud function for google cloud storage, then for every file uploaded a dataflow job will be launched.
const { google } = require('googleapis');
const templatePath = "gs://template_dir/df_template;
const project = "<project_id>";
const tempLoc = "gs://tempLocation/";
exports.PMKafka = (data, context, callback) => {
const file = data;
console.log(`Event ${context.eventId}`);
console.log(`Event Type: ${context.eventType}`);
console.log(`Bucket Name: ${file.bucket}`);
console.log(`File Name: ${file.name}`);
console.log(`Metageneration: ${file.metageneration}`);
console.log(`Created: ${file.timeCreated}`);
console.log(`Updated: ${file.updated}`);
console.log(`Uploaded File Name - gs://${file.bucket}/${file.name}`);
google.auth.getApplicationDefault(function (err, authClient, projectId) {
if (err) {
throw err;
}
if (authClient.createScopedRequired && authClient.createScopedRequired()) {
authClient = authClient.createScoped(authScope);
}
const dataflow = google.dataflow({ version: 'v1b3', auth: authClient });
var inputDict= {
inputFile: `gs://${file.bucket}/${file.name}`,
...
...
<other_runtime_parameters>
};
var env = {
tempLocation: tempLoc
};
var resource_opts = {
parameters: inputDict,
environment: env,
jobName: config.jobNamePrefix + "-" + new Date().toISOString().toLowerCase().replace(":","-").replace(".","-")
};
var opts = {
gcsPath: templatePath,
projectId: project,
resource: resource_opts
}
console.log(`Dataflow Run Time Options - ${JSON.stringify(opts)}`)
dataflow.projects.templates.launch(opts, function (err, response) {
if (err) {
console.error("problem running dataflow template, error was: ", err);
slack.publishMessage(null, null, false, err);
return;
}
console.log("Dataflow template response: ", response);
var jobid = response["data"]["job"]["id"];
console.log("Dataflow Job ID: ", jobid);
});
callback();
});
};