So Im writing a lambda fuction, witch is triggered by an S3 PUT,
import datetime
import boto3
import botocore
#boto3.set_stream_logger('botocore', level='DEBUG')
def lambda_handler(event, context):
src_bucket_name=event['Records'][0]['s3']['bucket']['name']
print src_bucket_name
file = event['Records'][0]['s3']['object']['key']
split_string = file.split('/')
file_string = split_string[-1].split('_')
fecha_str = event['Records'][0]['eventTime']
fecha_real=datetime.datetime.strptime(fecha_str, '%Y-%m-%dT%H:%M:%S.%fZ')+ datetime.timedelta(hours=-6)
new_path='PATH/'+file_string[0].lower()+'/'+str(fecha_real.year)+'/'+str(fecha_real.month)+'/'+split_string[-1]
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
copy_source = {
'Bucket': src_bucket_name,
'Key': file
}
s3.meta.client.copy(copy_source, DST_BUCKET_NAME, new_path)
when I run the code I get
ClientError: An error occurred (404) when calling the HeadObject operation: Not Found
the file does exists
file in source_bucket
Could you please tell what am I doing wrong?
EDIT:
I gave admin permissions to the role I'm using and still having the same error.
UPDATE-CLOSED:
I deleted the role, made a new one, change the code on the copy part to this:
copy_source = {
'Bucket': src_bucket_name,
'Key': file
}
r = s3_client.copy_object(
Bucket=[DST_BUCKET_NAME],
CopySource=copy_source,
Key=new_path
)
and it worked!
Related
While trying to implement S3 to s3 copy by using Lambda function I am getting below error.
"errorType": "AttributeError", "stackTrace": [
" File "/var/task/lambda_function.py", line 6, in lambda_handler\n file_name=event.Records[0].s3.bucket.name\n
Below is the code.
import boto3 def lambda_handler(event, context):
file_name = event['Records'][0]['s3']['object']['key']
#file_name=event.Records[0].s3.bucket.name
service_name='s3'
region_name='us-east-2'
aws_access_key_id=''
aws_secret_access_key=''
s3=boto3.resource(
service_name=service_name,
region_name=region_name,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key
)
copy_source = {
'Bucket': 'lamdbasourcebucket',
'Key': file_name
}
bucket_name = 'lambdatargetbucket2022'
s3.meta.client.copy(copy_source,bucket_name,file_name)
Can anyone help me on how to resolve the issue.
I have a file with urls in my s3 bucket. I would like to use a python lambda function to upload the url files to s3 bucket.
For example my uploaded file to s3 contains:
http://...
http://...
Each line corresponds to a file to be uploaded into s3.
Here is the code:
import json
import urllib.parse
import boto3
import requests
import os
from gzip import GzipFile
from io import TextIOWrapper
import requests
print('Loading functions')
s3 = boto3.client('s3')
def get_file_seqs(response):
try:
size = response['ResponseMetadata']['HTTPHeaders']['content-length']
print("[+] Size retrieved")
return size
except:
print("[-] Size can not be retrieved")
def lambda_handler(event, context):
# Defining bucket objects
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
#get file from s3
print('[+] Getting file from S3 bucket')
response = s3.get_object(Bucket=bucket, Key=key)
try:
#checking file size
print('[+] Checking file size')
file_size = get_file_seqs(response)
if file_size == 0:
print('File size is equal to 0')
return False
else:
#create new directories
print('[+] Creating new directories')
bucket_name = "triggersnextflow"
directories = ['backups/sample/', 'backups/control/']
#loop to create new dirs
for dirs in directories:
s3.put_object(Bucket = bucket_name, Key = dirs, Body = '')
#NOW I WOULD LIKE TO DOWNLOAD THE FILES FROM THE URLS INSIDE S3 OBJECT
#return true
return True
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
Download an S3 object to a file:
import boto3
s3 = boto3.resource('s3')
s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt')
You will find great resource of information here:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.download_file
I'm using the below Lambda code to read data from an S3 bucket that triggers the lambda function once a file is created into the S3 bucket.
import json
import urllib.parse
import boto3
print('Loading function')
s3 = boto3.client('s3')
def lambda_handler(event, context):
#print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
print("bucket= ", bucket)
print("key= ", key)
try:
response = s3.get_object(Bucket=bucket, Key=key)
print("CONTENT TYPE: " + response['ContentType'])
return response['ContentType']
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
The uploaded file is a csv file that contains 50 comma delimited 50 records.
I need to stream this received data into an AWS Kinesis Data Stream called test-stream once the file is uploaded to the S3 bucket.. Any help please?
Thanks..
this is basic thing and it seems obvious but am stuck in this one
am using boto3, I have this access key and secret key,
upload file route:
#app.route('/up')
def up():
main = request.files['mainimg']
bucket=<bucketname>
if main:
upload_to_aws(main)
the upload_to_aws function (from github):
import os
import boto3
from werkzeug.utils import secure_filename
def upload_to_aws(file, acl="public-read"):
filename = secure_filename(file.filename)
s3 = boto3.client(
's3',
aws_access_key_id=os.environ.get('FASO_S3_ACCESS_KEY'),
aws_secret_access_key=os.environ.get('FASO_S3_SECRET_KEY')
)
try:
s3.upload_fileobj(
file,
"fasofashion",
file.filename,
ExtraArgs={
"ACL": acl,
"ContentType": file.content_type
}
)
print('uploadeed')
except Exception as e:
# This is a catch all exception, edit this part to fit your needs.
print("Something Happened: ", e)
return e
I keep getting these errors:
Access denied file must be a string
File must be a string
I have tried below code but I am not able to convert the data from json to csv. Can someone please help me?
import boto3
import botocore
import csv
def lambda_handler(event, context):
BUCKET_NAME = 'name of the bucket' # replace with your bucket name
KEY = 'OUTPUT.csv' # replace with your object key
json_data = [{"id":"1","name":"test"},{"id":"2","name":"good"}]
with open("data.csv", "w") as file:
csv_file = csv.writer(file)
csv_file.writerow(['id', 'name'])
for item in data:
csv_file.writerow([item.get('id'),item.get('name')])
csv_binary = open('data.csv', 'rb').read()
try:
obj = s3.Object(BUCKET_NAME, KEY)
obj.put(Body=csv_binary)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
print("The object does not exist.")
else:
raise
s3client = boto3.client('s3')
try:
download_url = s3client.generate_presigned_url(
'get_object',
Params={
'Bucket': BUCKET_NAME,
'Key': KEY
},
ExpiresIn=3600
)
return {"csv_link": download_url}
except Exception as e:
raise utils_exception.ErrorResponse(400, e, Log)
Here is the response I am getting for the above code:
{
"errorMessage": "[Errno 30] Read-only file system: 'data.csv'",
"errorType": "OSError",
"stackTrace": [
" File \"/var/task/lambda_function.py\", line 8, in lambda_handler\n with open(\"data.csv\", \"wb\") as file:\n"
]
}
In AWS Lambda, you can only create files in the /tmp/ directory. Therefore, use:
with open("/tmp/data.csv", "w") as file:
A maximum of 512MB is provided, so it is a good idea to delete any temporary files so they do not interfere with future executions of the Lambda function.