I am using AWS CodeStar (Lambda + API Gateway) to build my serverless API. My lambda function works well in the Lambda console but strangely throws this error when I run the code on AWS CodeStar:
"message": "Internal server error"
Kindly help me with this issue.
import json
import os
import bz2
import pprint
import hashlib
import sqlite3
import re
from collections import namedtuple
from gzip import GzipFile
from io import BytesIO
from botocore.vendored import requests
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
def handler(event, context):
logger.info('## ENVIRONMENT VARIABLES')
logger.info(os.environ)
logger.info('## EVENT')
logger.info(event)
n = get_package_list()
n1 = str(n)
dat = {"total_pack":n1}
return {'statusCode': 200,
'headers': {'Content-Type': 'application/json'},
'body': json.dumps(dat)
}
def get_package_list():
url = "http://amazonlinux.us-east-2.amazonaws.com/2/core/2.0/x86_64/c60ceaf6dfa3bc10e730c9e803b51543250c8a12bb009af00e527a598394cd5e/repodata/primary.sqlite.gz"
db_filename = "dbfile"
resp = requests.get(url, stream=True)
remote_data = resp.raw.read()
cached_fh = BytesIO(remote_data)
compressed_fh = GzipFile(fileobj=cached_fh)
with open(os.path.join('/tmp',db_filename), "wb") as local_fh:
local_fh.write(compressed_fh.read())
package_obj_list = []
db = sqlite3.connect(os.path.join('/tmp',db_filename))
c = db.cursor()
c.execute('SELECT name FROM packages')
for package in c.fetchall():
package_obj_list.append(package)
no_of_packages = len(package_obj_list)
return no_of_packages
Expected Result: should return an Integer (no_of_packages).
Related
PYTHON
This is the Python code, We are streaming our large file from google cloud storage to cloud run. In this code the large csv file is splitting into chunks and those chunks are going into in_memory_file whenever we got the first chunks it should immediately start streaming it and the remaining chunks will stream in response but we are not able to stream those chunks from in_memory_file
import os
import signal
import sys
import json
import pandas as pd
import structlog
import time
import threading
import io
import csv
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import ThreadPoolExecutor
from types import FrameType
from io import BytesIO, StringIO
from google.cloud import storage
from google.oauth2 import service_account
from flask import Flask, Response, request
app = Flask(__name__)
# SigTerm Log
def getJSONLogger() -> structlog._config.BoundLoggerLazyProxy:
structlog.configure(
processors=[`enter code here`
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper("iso"),
structlog.processors.JSONRenderer(),
],
wrapper_class=structlog.stdlib.BoundLogger,
)
return structlog.get_logger()
logger = getJSONLogger()
# SigTerm Handler
def shutdown_handler(signal: int, frame: FrameType) -> None:
logger.info("Signal received, safely shutting down.")
print("Exiting process.", flush=True)
sys.exit(0)
signal.signal(signal.SIGTERM, shutdown_handler)
# Split files to chunks
def split_byte_size(size: int, uri: str, bucket: str, key: str) -> list:
byte_list = []
chunks = 50
start = 0
for i in range(size, size * chunks + 1, size):
stop = i // chunks
byte_list.append({"uri": uri, "start": start, "end": stop, "bucket": bucket, "key": key})
start = stop + 1
return byte_list
#Cloud Storage connection
project = 'XYZ'
service_account_credentials_path = 'key.json'
credentials = service_account.Credentials.from_service_account_file(service_account_credentials_path)
storage_client = storage.Client(project=project, credentials=credentials)
# Download objects as chunks
def downloader(input: dict) -> object:
bucket_object = storage_client.get_bucket(bucket_or_name=input["bucket"])
blob = bucket_object.blob(input["key"])
in_memory_file = io.BytesIO()
blob.download_to_file(in_memory_file, start=input['start'], end=input['end'])
#print("Chunk " + str(input['start']) + " to " + str(input['end']) + "completed")
return in_memory_file
#app.route("/chunk_data")
def chunk_data():
bucket_name = 'cloudrundemofile'
source_blob_name = 'demofile.csv'
bucket_object = storage_client.get_bucket(bucket_name)
blob = bucket_object.get_blob(source_blob_name)
split_bytes = split_byte_size(blob.size, project, bucket_name, source_blob_name)
print(split_bytes)
#Async Thread
with ThreadPoolExecutor(max_workers=5) as ex:
results = ex.map(downloader, split_bytes)
resp = Response(results, 206, mimetype='text/csv')
#resp.headers.add('Content-Range', 'bytes {0}-{1}/{2}'.format(start, start + length - 1, file_size))
return resp
#return "Success"
if __name__ == "__main__":
signal.signal(signal.SIGINT, shutdown_handler)
app.run(host="0.0.0.0", port=8080)
else:
signal.signal(signal.SIGTERM, shutdown_handler)**
I have written code in python that uploads a file to Google Drive, but after it uploads I cannot delete it from local drive, because I get error "Access Denied", but if I exit out of function then I can delete the file. So my question is how can I delete the file from inside the function?
GDriveUpload.py
import os
import httplib2
import ntpath
import oauth2client
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
# Copy your credentials here
_CLIENT_ID = 'YOUR_CLIENT_ID'
_CLIENT_SECRET = 'YOUR_CLIENT_SECRET'
_REFRESH_TOKEN = 'YOUR_REFRESH_TOKEN'
_PARENT_FOLDER_ID = 'YOUR_PARENT_FOLDER_ID'
_DATA_FILE = 'datafile.dat'
# ====================================================================================
# Upload file to Google Drive
def UploadFile(client_id, client_secret, refresh_token, parent_folder_id, local_file, DeleteOnExit=False):
cred = oauth2client.client.GoogleCredentials(None,client_id,client_secret,refresh_token,None,'https://accounts.google.com/o/oauth2/token',None)
http = cred.authorize(httplib2.Http())
drive_service = build('drive', 'v2', http=http)
media_body = MediaFileUpload(local_file, mimetype='application/octet-stream', chunksize=5242880, resumable=True)
body = {
'title': (ntpath.basename(local_file)),
'parents': [{'id': parent_folder_id}],
'mimeType': 'application/octet-stream'
}
request = drive_service.files().insert(body=body, media_body=media_body)
response = None
while response is None:
status, response = request.next_chunk()
if status:
print "Uploaded %.2f%%" % (status.progress() * 100)
if DeleteOnExit == True:
os.remove(local_file)
# ====================================================================================
if __name__ == '__main__':
UploadFile(_CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN, _PARENT_FOLDER_ID, _DATA_FILE, DeleteOnExit=True)
When I used flask-apscheduler(not apscheduler), I have some problems in my flask web project. Especially when I used db(flask-sqlalchemy) objects. The problem may be:
JOBS = [
{
'id': 'job1',
'func': 'app.monitor.views:test',
'args': (),
'trigger': 'interval',
'seconds': 2
}
]
./app/init.py:
from flask import Flask
from flask.ext.bootstrap import Bootstrap
from flask.ext.mail import Mail
from flask.ext.moment import Moment
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import LoginManager
from flask.ext.pagedown import PageDown
from flask_apscheduler import APScheduler
from celery import Celery
# from apscheduler.schedulers.blocking import BlockingScheduler
from config import config,Config
bootstrap = Bootstrap()
mail = Mail()
moment = Moment()
db = SQLAlchemy()
pagedown = PageDown()
celery = Celery(__name__, broker=Config.CELERY_BROKER_URL)
# https://pypi.python.org/pypi/Flask-APScheduler
scheduler = APScheduler()
login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config[config_name].init_app(app)
bootstrap.init_app(app)
mail.init_app(app)
moment.init_app(app)
db.init_app(app)
login_manager.init_app(app)
pagedown.init_app(app)
scheduler.init_app(app)
celery.conf.update(app.config)
if not app.debug and not app.testing and not app.config['SSL_DISABLE']:
from flask.ext.sslify import SSLify
sslify = SSLify(app)
from .monitor import monitor as monitor_1_0_blueprint
from .laser import laser as laser_1_0_blueprint
app.register_blueprint(monitor_blueprint,url_prefix='/monitor/api')
app.register_blueprint(laser_1_0_blueprint,url_prefix='/laser/api/v1.0')
return app
Error 1:db is : Error 2:db is :No handlers
could be found for logger "apscheduler.executors.default" Error 3:db
is : raise RuntimeError('working outside of application context')
RuntimeError: working outside of application context
The key to the problem is to get the db and app objects in flask-apscheduler jobs function(views.py):
from app import scheduler
def test():
#to Solve the log error problem
import logging
log = logging.getLogger('apscheduler.executors.default')
log.setLevel(logging.INFO) # DEBUG
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
#get the app object
app = scheduler.app
#get the db object and use it
with app.app_context():
print '........................',db
from app import scheduler#
def test():
#to Solve the log error problem
import logging
log = logging.getLogger('apscheduler.executors.default')
log.setLevel(logging.INFO) # DEBUG
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
#get the app object
app = scheduler.app
#get the db object and use it
with app.app_context():
print '........................',db
def test():
#to Solve the log error problem
import logging
log = logging.getLogger('apscheduler.executors.default')
log.setLevel(logging.INFO) # DEBUG
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
#get the app object
app = scheduler.app
#get the db object and use it
with app.app_context():
print '........................',db #the right db object
I am trying to print log on the local environment of Google App engine. It seems the way it should be but still i am not able to print the log. Need some helping hand here?
I need this output on the standard console.
import webapp2
from google.appengine.api import urlfetch
from Webx import WebxClass
import json
import logging
class SearchHandler(webapp2.RequestHandler):
def __init__(self,*args, **kwargs):
super(SearchHandler,self).__init__(*args, **kwargs)
self.result=[]
self.searchPortals = [WebxClass()]
self.asa = []
def handleCallBack(self,rpc,portalObject):
try:
rr = rpc.get_result()
if rr.status_code == 200:
if isinstance(portalObject, WebxClass):
resultList=portalObject.getResultList(rr.content)
self.result.extend(resultList)
except urlfetch.DownloadError:
self.result = 'Error while fetching from portal - ' + portalObject.getName()
def getSearchResult(self):
rpcs=[]
searchKeyword=self.request.get('searchString')
logging.error("------------------------------")
for portal in self.searchPortals:
rpc = urlfetch.create_rpc(deadline=5)
rpc.callback = lambda: self.handleCallBack(rpc, portal)
urlfetch.make_fetch_call(rpc, portal.getSearchURL(searchKeyword))
rpcs.append(rpc)
for rpc in rpcs:
rpc.wait()
self.response.status_int = 200
self.response.headers['Content-Type'] = 'application/json'
self.response.headers.add_header("Access-Control-Allow-Origin", "*")
self.response.write(json.dumps(self.result))
app = webapp2.WSGIApplication([
webapp2.Route(r'/search', methods=['GET'], handler='Torrent.SearchHandler:getSearchResult')
], debug=True)
def main():
logging.getLogger().setLevel(logging.DEBUG)
logging.debug("------------------------------")
app.run()
if __name__ == '__main__':
main()
I am new to google python api client.I am learning from https://developers.google.com/api-client-library/python/start/get_started.I want to make an api which converts python object into JSON data and sends to a servlet.
The python code of file api.py:
import os
import urllib2
import httplib2
import json
import requests
from apiclient.discovery import build
from oauth2client.client import flow_from_clientsecrets
from oauth2client.tools import run_flow
from oauth2client.file import Storage
from oauth2client import tools
api_version='1'
_file_="D:\API"
CLIENT_SECRETS = os.path.join(os.path.dirname(_file_))
flow=flow_from_clientsecrets(CLIENT_SECRETS,
scope=[
'https://www.googleapis.com/auth/devstorage.full_control',
],
http = httplib2.Http()
auth_http = credentials.authorize(http)
service=build('SendNotif',api_version,http=http)
req = urllib2.Request('http://example/notify')
req.add_header('Content-Type', 'application/json')
data={"message":"Hello User you are notified"}
data_json = json.dumps(data)
response = urllib2.urlopen(req, json.dumps(data))
The error shown is:
D:\API>python api.py
File "api.py", line 25
auth_http = credentials.authorize(http)
^
SyntaxError: invalid syntax
please do help in correcting me..
thanks in advance....
You're missing a closing parenthesis for this line:
flow=flow_from_clientsecrets(CLIENT_SECRETS,