def upload_file_to_aws(file_name):
"""Upload a file to an S3 bucket
:param file_name: File to upload
:return: True if file was uploaded, else False
"""
bucket = get_bucket()
# If S3 object_name was not specified, use file_name
object_name = file_name.split("/")[-1]
# Upload the file
s3_client = boto3.client(
"s3",
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
ce_logger.info(f"s3_client:{s3_client}")
try:
s3_client.upload_file(file_name, bucket, object_name)
except Exception as e:
logger.info(e)
return False
return True
When I run the above method locally, it can work normally and upload the file to s3,, but when I put this task in celery schedule, use the s3_client.upload_file(file_name, bucket, object_name) method. It will timeout. celery log is displayed
[the 2022-12-08 13:40:03, 205: INFO/ForkPoolWorker-1] project.invoice.tasks.download_gmail_attachment_create_invoice[14b98680-7f38-45fb-9dbe-283330c304b0]: s3_client:<botocore.client.S3 object at 0x7f0a8a5f35e0>
[13:45:11 2022-12-08, 162: INFO/ForkPoolWorker-1] project.invoice.tasks.download_gmail_attachment_create_invoice[14b98680-7f38-45fb-9dbe-283330c304b0]: Connect timeout on endpoint URL: "https://karbon-text.s3.ap-south-1.amazonaws.com/upload_aSZcLIM.pdf"
This is a scheduled task
app.conf.beat_schedule = {
"download_gmail_attachment_create_invoice": {
"task": "project.invoice.tasks.download_gmail_attachment_create_invoice",
"schedule": crontab(minute="*/10"),
},
}
#shared_task
def download_gmail_attachment_create_invoice():
file_name = get_file_name()
upload_file_to_aws(file_name)
Why does this happen
I want to upload files to aws s3 successfully in the celery schedule
Related
I have a file with urls in my s3 bucket. I would like to use a python lambda function to upload the url files to s3 bucket.
For example my uploaded file to s3 contains:
http://...
http://...
Each line corresponds to a file to be uploaded into s3.
Here is the code:
import json
import urllib.parse
import boto3
import requests
import os
from gzip import GzipFile
from io import TextIOWrapper
import requests
print('Loading functions')
s3 = boto3.client('s3')
def get_file_seqs(response):
try:
size = response['ResponseMetadata']['HTTPHeaders']['content-length']
print("[+] Size retrieved")
return size
except:
print("[-] Size can not be retrieved")
def lambda_handler(event, context):
# Defining bucket objects
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
#get file from s3
print('[+] Getting file from S3 bucket')
response = s3.get_object(Bucket=bucket, Key=key)
try:
#checking file size
print('[+] Checking file size')
file_size = get_file_seqs(response)
if file_size == 0:
print('File size is equal to 0')
return False
else:
#create new directories
print('[+] Creating new directories')
bucket_name = "triggersnextflow"
directories = ['backups/sample/', 'backups/control/']
#loop to create new dirs
for dirs in directories:
s3.put_object(Bucket = bucket_name, Key = dirs, Body = '')
#NOW I WOULD LIKE TO DOWNLOAD THE FILES FROM THE URLS INSIDE S3 OBJECT
#return true
return True
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
Download an S3 object to a file:
import boto3
s3 = boto3.resource('s3')
s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt')
You will find great resource of information here:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.download_file
I'm using the below Lambda code to read data from an S3 bucket that triggers the lambda function once a file is created into the S3 bucket.
import json
import urllib.parse
import boto3
print('Loading function')
s3 = boto3.client('s3')
def lambda_handler(event, context):
#print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
print("bucket= ", bucket)
print("key= ", key)
try:
response = s3.get_object(Bucket=bucket, Key=key)
print("CONTENT TYPE: " + response['ContentType'])
return response['ContentType']
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
The uploaded file is a csv file that contains 50 comma delimited 50 records.
I need to stream this received data into an AWS Kinesis Data Stream called test-stream once the file is uploaded to the S3 bucket.. Any help please?
Thanks..
I have the following function:
def upload_file(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket -> from aws docs
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_name
# Upload the file
s3_client = boto3.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name)
except ClientError as e:
logging.error(e)
return False
return True
I am trying to upload a html file to an S3 bucket acting as a webserver. When I manually upload the html file to S3, it works as expected, and displays the page when I navigate to the S3 bucket's URL.
If I programmatically upload the file using the above function, the html file will no longer be hosted, and my browser will attempt to download a XZ file.
Am I missing a parameter or something?
Courtesy of #jarmod, I learned I was setting an incorrect content-type.
Here is the updated function to upload an HTML file as text/html.
def upload_file(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket -> from aws docs
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_name
# Upload the file
s3_client = boto3.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name, ExtraArgs={'ContentType': "text/html"})
except ClientError as e:
logging.error(e)
return False
return True
I created a Python script that should upload a file from my local ec2 to the s3 bucket
import boto3
s3 = boto3.resource('s3')
data = open('backupFile.txt', 'rb')
s3.Bucket('mlsd').put_object(Key='backupFile.txt', Body=data)
I went to AWS account details and got the credentials.
I executed aws configure to set credentials on my EC2.
Hear is the output of the credentials using aws configure list:
I went to .aws/credentials and pasted access_key_id, secret_access_key, and token
I ensured that the token is not expired.
When I ran the script, I got the following output:
Not sure what the problem is.
Boto3 detects your credentials in possible locations, as described here, so it should find your access_key_id and secret_access_key
Make sure the user whose access_key_id you use has the access to S3 bucket.
I tried this code example and it works:
import logging
import boto3
from botocore.exceptions import ClientError
def upload_file(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_name
# Upload the file
s3_client = boto3.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name)
except ClientError as e:
logging.error(e)
return False
return True
I have developed an web app, in which files needs to be uploaded from local PC to AWS EC2 instance via flask web call and run the machine learning model in the back-end. But, could not find any related resources to do that.
Can we upload in AWS S3 instead and link ec2 EBS and S3?
If any help is provided then it will be useful to do this!
Use boto to upload files to s3.
In flash create an endpoint that will take the local file and push it to S3.
import boto3
from botocore.exceptions import ClientError
def upload_file(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_name
# Upload the file
s3_client = boto3.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name)
except ClientError as e:
logging.error(e)
return False
return True ```
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
Read This https://www.javatpoint.com/flask-file-uploading
Upload the file to tmp directory and then upload to S3.