I have the following script which I tried to get response status from request.get(url)
from flask import *
import requests
app = Flask(__name__)
#app.route('/')
def home():
r = requests.get('http://127.0.0.1:5000/welcome')
print r
return render_template('home.html')
#app.route('/welcome')
def wecome():
print "welcome page"
return "welcome to my page"
if __name__=='__main__':
app.run(debug=True)
When I access endpoint http://127.0.0.1:5000/ the browser keeps spinning none stop without any output result in terminal console as I expected even the error message.
However, if I change the requests url to something else externally as below, the response is showing up the status <Response [200]> in terminal and the page is loaded as normal.
from flask import *
import requests
app = Flask(__name__)
#app.route('/')
def home():
r = requests.get('https://api.github.com/emojis')
print r
return render_template('home.html')
#app.route('/welcome')
def wecome():
print "welcome page"
return "welcome to my page"
if __name__=='__main__':
app.run(debug=True)
what's going on with that?
How can I get response from internal url as I need it badly for my current work?
Related
When I run this, 404 error is been shown. Please help me solving this:
from flask import Flask
app = Flask(__name__)
#app.route("/")
def index():
return "<h1> Hello Puppy! </h1>"
#app.route("/information")
def info():
return "<h1> Puppies are Cute </h1>"
#app.route("/Puppy/<name>")
def puppy(name):
return "<h1> This is a page for {} </h1>".format(name)
if __name__ == '__main__':
app.run()
from flask import Flask
app = Flask(__name__)
#app.route('/hello/<name>')
def hello_name(name):
return 'Hello %s!' % name
if __name__ == '__main__':
app.run()
Enter http://localhost:5000/hello/<name> route.
here, name parameter is taken value from the route.
i.e. http://localhost:5000/hello/smit and its return Hello smit!
I'm experiencing an error with Flask. If I call the #app.route with the function, I retrieve 404 Not Found:
from flask import Flask, request
import requests
app = Flask(__name__)
#app.route('/webhook', methods=['GET', 'POST'])
def webhook():
return 'Hello!'
if __name__ == '__main__':
app.run("0.0.0.0", port=10101, debug=False)
However, if the function is not mentioned, I retrieve the 500 Internal Server Error:
from flask import Flask, request
import requests
app = Flask(__name__)
#app.route('/', methods=['GET', 'POST'])
def webhook():
return 'Hello!'
if __name__ == '__main__':
app.run("0.0.0.0", port=10101, debug=False)
Any help, please?
Your code runs fine. I just copy-pasted your original example and did a curl request to it with:
curl -X GET http://localhost:10101/webhook
curl -X POST --data "test=true" http://localhost:10101/webhook
Both return Hello!%
As suggested by #Sebastian Speitel - try enabling debug mode - that will give you an idea of what fails and why:
app.run("0.0.0.0", port=10101, debug=True)
I am trying to print log on the local environment of Google App engine. It seems the way it should be but still i am not able to print the log. Need some helping hand here?
I need this output on the standard console.
import webapp2
from google.appengine.api import urlfetch
from Webx import WebxClass
import json
import logging
class SearchHandler(webapp2.RequestHandler):
def __init__(self,*args, **kwargs):
super(SearchHandler,self).__init__(*args, **kwargs)
self.result=[]
self.searchPortals = [WebxClass()]
self.asa = []
def handleCallBack(self,rpc,portalObject):
try:
rr = rpc.get_result()
if rr.status_code == 200:
if isinstance(portalObject, WebxClass):
resultList=portalObject.getResultList(rr.content)
self.result.extend(resultList)
except urlfetch.DownloadError:
self.result = 'Error while fetching from portal - ' + portalObject.getName()
def getSearchResult(self):
rpcs=[]
searchKeyword=self.request.get('searchString')
logging.error("------------------------------")
for portal in self.searchPortals:
rpc = urlfetch.create_rpc(deadline=5)
rpc.callback = lambda: self.handleCallBack(rpc, portal)
urlfetch.make_fetch_call(rpc, portal.getSearchURL(searchKeyword))
rpcs.append(rpc)
for rpc in rpcs:
rpc.wait()
self.response.status_int = 200
self.response.headers['Content-Type'] = 'application/json'
self.response.headers.add_header("Access-Control-Allow-Origin", "*")
self.response.write(json.dumps(self.result))
app = webapp2.WSGIApplication([
webapp2.Route(r'/search', methods=['GET'], handler='Torrent.SearchHandler:getSearchResult')
], debug=True)
def main():
logging.getLogger().setLevel(logging.DEBUG)
logging.debug("------------------------------")
app.run()
if __name__ == '__main__':
main()
I was just wondering if there's any reasonable way to pass authentication cookies from webdriver.Firefox() instance to the spider itself? It would be helpful to perform some webdriver stuff and then go about scraping "business as usual". Something to the effect of:
def __init__(self):
BaseSpider.__init__(self)
self.selenium = webdriver.Firefox()
def __del__(self):
self.selenium.quit()
print self.verificationErrors
def parse(self, response):
# Initialize the webdriver, get login page
sel = self.selenium
sel.get(response.url)
sleep(3)
##### Transfer (sel) cookies to (self) and crawl normally??? #####
...
...
Transfer Cookies from Selenium to Scrapy Spider
Scrapying File
from selenium import webdriver
driver=webdriver.Firefox()
data=driver.get_cookies()
# write to temp file
with open('cookie.json', 'w') as outputfile:
json.dump(data, outputfile)
driver.close()
outputfile.close()
....
Spider
import os
if os.stat("cookie.json").st_size > 2:
with open('./cookie.json', 'r') as inputfile:
self.cookie = json.load(inputfile)
inputfile.close()
You can try to override BaseSpider.start_requests method to attach to starting requests needed cookies using scrapy.http.cookies.CookieJar.
See also: Scrapy - how to manage cookies/sessions
This works with chrome driver but not Firefox (Tested OK)
refer https://christopher.su/2015/selenium-chromedriver-ubuntu/ for installation.
import scrapy
from scrapy.spiders.init import InitSpider
from scrapy.http import Request
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pickle
class HybridSpider(InitSpider):
name = 'hybrid'
def init_request(self):
driver = webdriver.Chrome()`
driver.get('https://example.com')
driver.find_element_by_id('js-login').click()
driver.find_element_by_id('email').send_keys('mymail#example.net')
driver.find_element_by_id('password').send_keys('mypasssword',Keys.ENTER)
pickle.dump( driver.get_cookies() , open(os.getenv("HOME")+"/my_cookies","wb"))
cookies = pickle.load(open(os.getenv("HOME")+"/my_cookies", "rb"))
FH = open(os.getenv("HOME")+"/my_urls", 'r')
for url in FH.readlines():
pass
yield Request(url,cookies=cookies,callback=self.parse)
def parse(self, response):
pass
Haven't tried directly passing the cookies like
yield Request(url,cookies=driver.get_cookies(),callback=self.parse)
Might work too..
driver = webdriver.Chrome()
Then perform the login or interact with the page through the browser. Now when using the crawler in scrapy, set the cookies parameter:
request = Request(URL, cookies=driver.get_cookies(), callback=self.mycallback)