Scrapy post request form data - python-2.7

Question

I want to get the search result using scrapy post request after giving the input to CP Number as 16308 https://www.icsi.in/Student/Default.aspx?TabID=100 .
Here is my scrapy spider code :--
def parse(self, response):
head=response.xpath('//span[#id="dnn_ctlHeader_dnnBreadcrumb_lblBreadCrumb"]/span[#class="SkinObject"]/text()').extract_first()
view_gen = response.xpath('//input[#id="__VIEWSTATEGENERATOR"]/#value').extract_first()
dnn= response.xpath('//input[#id="__dnnVariable"]/#value').extract_first()
view_state = response.xpath('//input[#id="__VIEWSTATE"]/#value').extract_first()
view_val = response.xpath('//input[#id="__EVENTVALIDATION"]/#value').extract_first()
data={
'__VIEWSTATEGENERATOR':view_gen,
'__dnnVariable':dnn,
'__VIEWSTATE':view_state,
'__EVENTVALIDATION':view_val,
'dnn$ctr410$MemberSearch$txtCpNumber':'16803',
'dnn$ctr410$MemberSearch$ddlMemberType':'0'
}
yield scrapy.FormRequest(response.url,formdata=data,callback=self.fun)
Response
DEBUG: Crawled (200) https://www.icsi.in/Student/Default.aspx?tabid=100&error=An%20unexpected%20error%20has%20occurred&content=0> (referer: https://www.icsi.in/Student/Default.aspx?TabID=100)
[]

Answer 1

Response DEBUG: Crawled (200)
https://www.icsi.in/Student/Default.aspx?tabid=100&error=An%20unexpected%20error%20has%20occurred&content=0>
(referer: https://www.icsi.in/Student/Default.aspx?TabID=100) []
Your question is how to avoid getting this error right? Try to be more specific in the future.
When you want to scrape a webpage you have to inspect it all on your browser, see all parameters that are being sent with the request and make sure you are doing the same on your spider. You got lots of parameters on your code, but not all.
See my code bellow which actually solves your problem:
import scrapy
class MySpider(scrapy.Spider):
name = 'icsi'
start_urls = ['https://www.icsi.in/Student/Default.aspx?TabID=100']
search_action_url = 'https://www.icsi.in/Student/Default.aspx?TabID=100'
def parse(self, response):
formdata = dict()
for input in response.css('form#Form input'):
name = input.xpath('./#name').get()
value = input.xpath('./#value').get()
formdata[name] = str(value) if value else ''
formdata['dnn$ctr410$MemberSearch$txtCpNumber'] = '16308'
formdata['__EVENTTARGET'] = 'dnn$ctr410$MemberSearch$btnSearch'
return scrapy.FormRequest(self.search_action_url, formdata=formdata, callback=self.parse_search)
def parse_search(self, response):
scrapy.shell.inspect_response(response, self)
return
You were missing the parameter __EVENTTARGET, which informs the site you hit the button "Search".