I am trying to copy multiple files from one s3 bucket to another s3 bucket using lambda function but it is just copying 2 files in destination s3 bucket.
Here is my code:
# using python and boto3
import json
import boto3
s3_client = boto3.client('s3')
def lambda_handler(event, context):
source_bucket_name = event['Records'][0]['s3']['bucket']['name']
file_name = event['Records'][0]['s3']['object']['key']
destination_bucket_name = 'nishantnkd'
copy_object = {'Bucket': source_bucket_name, 'Key': file_name}
s3_client.copy_object(CopySource=copy_object,
Bucket=destination_bucket_name, Key=file_name)
return {'statusCode': 3000,
'body': json.dumps('File has been Successfully Copied')}
I presume that the Amazon S3 bucket is configured to trigger the AWS Lambda function when a new object is created.
When the Lambda function is triggered, it is possible that multiple event records are sent to the function. Therefore, it should loop through the event records like this:
# using python and boto3
import json
import boto3
s3_client = boto3.client('s3')
def lambda_handler(event, context):
for record in event['Records']: # This loop added
source_bucket_name = record['s3']['bucket']['name']
file_name = urllib.parse.unquote_plus(record['s3']['object']['key']) # Note this change too
destination_bucket_name = 'nishantnkd'
copy_object = {'Bucket': source_bucket_name, 'Key': file_name}
s3_client.copy_object(CopySource=copy_object, Bucket=destination_bucket_name, Key=file_name)
return {'statusCode': 3000,
'body': json.dumps('File has been Successfully Copied')}
Related
I'm working on unit tests for my lambda which is getting some files from S3, processing them and loading data from them to DynamoDB. I created botocore stubbers that are used during tests, but I got botocore.exceptions.NoCredentialsError: Unable to locate credentials
My lambda handler code
s3_client = boto3.client('s3')
ddb_client = boto3.resource('dynamodb', region_name='eu-west-1')
def lambda_handler(event, context):
for record in event['Records']:
s3_event = record.get('s3')
bucket = s3_event.get('bucket', {}).get('name', '')
file_key = s3_event.get('object', {}).get('key', '')
file = s3_client.get_object(Bucket=bucket, Key=file_key)
and tests file:
class TestLambda(unittest.TestCase):
def setUp(self) -> None:
self.session = botocore.session.get_session()
# S3 Stubber Set Up
self.s3_client = self.session.create_client('s3', region_name='eu-west-1')
self.s3_stubber = Stubber(self.s3_client)
# DDB Stubber Set Up
self.ddb_resource = boto3.resource('dynamodb', region_name='eu-west-1')
self.ddb_stubber = Stubber(self.ddb_resource.meta.client)
def test_s3_to_ddb_handler(self) -> None:
event = {}
with self.s3_stubber:
with self.ddb_stubber:
response = s3_to_ddb_handler.s3_to_ddb_handler(event, ANY)
Issue seems to be that actual call to AWS resources is done which shouldnt be the case and stubber should be used, how can I force that?
You need to call .activate() on your Stubber instances: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/stubber.html#botocore.stub.Stubber.activate
i have a lambda to copy files (sample code below) , once files are copied , i delete from source. is there a way to move files instead and also do it recursively, to maintain folder or prefix in s3?
import json
import boto3
s3_client=boto3.client('s3')
def lambda_handler(event, context):
source_bucket_name=event['Records'][0]['s3']['bucket']['name']
file_name=event['Records'][0]['s3']['object']['key']
destination_bucket_name='gfg-destination-bucket'
copy_object={'Bucket':source_bucket_name,'Key':file_name}
s3_client.copy_object(CopySource=copy_object,Bucket=destination_bucket_name,Key=file_name)
return {
'statusCode': 200,
'body': json.dumps('Success')
There is no direct method for moving file from source S3 bucket to Destination S3 Bucket.
So you have to Copy file then delete from source
import json
import boto3
def lambda_handler(event, context):
# TODO implement
SOURCE_BUCKET='source-test-file'
DESTINATION_BUCKET='destiation-test-file'
s3_resource = boto3.resource('s3')
s3_client = boto3.client('s3')
bucket = s3_resource.Bucket(SOURCE_BUCKET)
dest_bucket = s3_resource.Bucket(DESTINATION_BUCKET)
#count = bucket.objects.filter(Prefix=SOURCE_BUCKET)
count = bucket.objects.all()
#count total files in bucket
totalCount = 0
print('result :',len(list(count)))
length = len(list(count))
if (length==0):
print('result :',len(list(count)))
print("bucket is empety")
else:
print("bucket is not empety")
for object in bucket.objects.all():
print("print :",object.key)
key1 = object.key
sourceObject = { 'Bucket' : 'source-test-file', 'Key': object.key}
destObject = { 'Bucket' : 'destiation-test-file', 'Key': object.key}
s3_resource.meta.client.copy(sourceObject, 'destiation-test-file',key1)
#Delete the File After Copying It to the Target Directory
s3_resource.Object('sourceObject','*.txt').delete()
totalCount += 1
print("file Moved")
I am able to copy a file from one bucket to another, but not sure if i'm doing this wrong but i can't delete the file . any thoughts?
import boto3
import os
from requests_aws4auth import AWS4Auth
session = boto3.Session()
credentials = session.get_credentials()
aws4auth = AWS4Auth(credentials.access_key,credentials.secret_key,region, service, session_token=credentials.token)
s3 = boto3.resource('s3')
name = event['Records'][0]['s3']['bucket']['name']
key = event['Records'][0]['s3']['object']['key']
s3.meta.client.copy({'Bucket': name, 'key': key}, targetBucket, key)
s3.meta.client.delete({{'Bucket': name, 'key': key}})
Since you are creating s3 = boto3.resource('s3'), you may use it to delete the object.
For this you would create Object and then used its delete method. For example:
s3 = boto3.resource('s3')
object_to_be_deleted = s3.Object(name, key)
object_to_be_deleted.delete()
Also since you are using lambda, make sure that your function's execution role has permissions to delete the object or there are no bucket policies which prohibit such an action.
I would suggest you to use boto3 client() rather than resource(). Anyways, here is what I tried and worked for me:
To copy file
import boto3
client = boto3.client('s3')
copy_source = {'Bucket': 'from-bucket-s3', 'Key': 'cfn.json'}
client.copy(copy_source, 'to-bucket-s3', 'other-cfn.json')
To delete file
import boto3
client = boto3.client('s3')
client.delete_object(Bucket='to-bucket-s3', Key='other-cfn.json')
boto3 client() supports vast number of APIs than resource()
I have a lambda function that generates a list of untagged buckets in AWS environment. Currently I send the output to a slack channel directly. Instead I would like to have my lambda dump the output to a csv file and send it as a report. Here is the code for it, let me know if you need any other details.
import boto3
from botocore.exceptions import ClientError
import urllib3
import json
http = urllib3.PoolManager()
def lambda_handler(event, context):
#Printing the S3 buckets with no tags
s3 = boto3.client('s3')
s3_re = boto3.resource('s3')
buckets = []
print('Printing buckets with no tags..')
for bucket in s3_re.buckets.all():
s3_bucket = bucket
s3_bucket_name = s3_bucket.name
try:
response = s3.get_bucket_tagging(Bucket=s3_bucket_name)
except ClientError:
buckets.append(bucket)
print(bucket)
for bucket in buckets:
data = {"text": "%s bucket has no tags" % (bucket)}
r = http.request("POST", "https://hooks.slack.com/services/~/~/~",
body = json.dumps(data),
headers = {"Content-Type": "application/json"})
I am using Lambda to read image files when they are uploaded to S3 through a S3 trigger. The following is my code:
import json
import numpy as np
import face_recognition as fr
def lambda_handler(event, context):
for record in event['Records']:
bucket=record['s3']['bucket']['name']
key = record['s3']['object']['key']
print(bucket,key)
This correctly prints the bucket name and key. However how do I read the image so that I can run face-recognition module on the image. Can i generate the arn for each uploaded image and use it to read the same?
You can read the image from S3 directly:
s3 = boto3.client('s3')
resp = s3.get_object(Bucket=bucket, Key=key)
image_bytes = resp['Body'].read()