Yahoo Finance Historical data downloader url is not working - cookies

Question

I have used the following url to fetch the historical data from yahoo finance. From last 16th May, 2017 the url is not working.
http://real-chart.finance.yahoo.com/table.csv?s=AAL&a=04&b=01&c=2017&d=04&e=02&f=2017&g=d&ignore=.csv
Seems like they have changed the url and the new url is:
https://query1.finance.yahoo.com/v7/finance/download/AAL?period1=1494873000&period2=1494959400&interval=1d&events=history&crumb=l0aEtuOKocj
In the above changed URL has a session cookie which is crumb. Is there any idea how to get this cookie programmatically(in JAVA)?

Answer 1

I recently wrote a simple python script to download the history of a single stock.
Here an example how to invoke it:
python get_quote_history.py --symbol=IBM --from=2017-01-01 --to=2017-05-25 -o IBM.csv
This will download IBM historical prices from 2017-01-01 to 2017-05-25 and save them in IBM.csv file.
import re
import urllib2
import calendar
import datetime
import getopt
import sys
import time
crumble_link = 'https://finance.yahoo.com/quote/{0}/history?p={0}'
crumble_regex = r'CrumbStore":{"crumb":"(.*?)"}'
cookie_regex = r'Set-Cookie: (.*?); '
quote_link = 'https://query1.finance.yahoo.com/v7/finance/download/{}?period1={}&period2={}&interval=1d&events=history&crumb={}'
def get_crumble_and_cookie(symbol):
link = crumble_link.format(symbol)
response = urllib2.urlopen(link)
match = re.search(cookie_regex, str(response.info()))
cookie_str = match.group(1)
text = response.read()
match = re.search(crumble_regex, text)
crumble_str = match.group(1)
return crumble_str, cookie_str
def download_quote(symbol, date_from, date_to):
time_stamp_from = calendar.timegm(datetime.datetime.strptime(date_from, "%Y-%m-%d").timetuple())
time_stamp_to = calendar.timegm(datetime.datetime.strptime(date_to, "%Y-%m-%d").timetuple())
attempts = 0
while attempts < 5:
crumble_str, cookie_str = get_crumble_and_cookie(symbol)
link = quote_link.format(symbol, time_stamp_from, time_stamp_to, crumble_str)
#print link
r = urllib2.Request(link, headers={'Cookie': cookie_str})
try:
response = urllib2.urlopen(r)
text = response.read()
print "{} downloaded".format(symbol)
return text
except urllib2.URLError:
print "{} failed at attempt # {}".format(symbol, attempts)
attempts += 1
time.sleep(2*attempts)
return ""
if __name__ == '__main__':
print get_crumble_and_cookie('KO')
from_arg = "from"
to_arg = "to"
symbol_arg = "symbol"
output_arg = "o"
opt_list = (from_arg+"=", to_arg+"=", symbol_arg+"=")
try:
options, args = getopt.getopt(sys.argv[1:],output_arg+":",opt_list)
except getopt.GetoptError as err:
print err
for opt, value in options:
if opt[2:] == from_arg:
from_val = value
elif opt[2:] == to_arg:
to_val = value
elif opt[2:] == symbol_arg:
symbol_val = value
elif opt[1:] == output_arg:
output_val = value
print "downloading {}".format(symbol_val)
text = download_quote(symbol_val, from_val, to_val)
with open(output_val, 'wb') as f:
f.write(text)
print "{} written to {}".format(symbol_val, output_val)

Answer 2

Andrea Galeazzi's excellent answer; with added options for splits and dividends, and twisted for python 3.
Also changed so "to:date" is included in the returned results, previously code returned up to but not including "to:date". Just different!
And be aware that Yahoo made minor changes in price rounding, column order, and split syntax.
## Downloaded from
## https://stackoverflow.com/questions/44044263/yahoo-finance-historical-data-downloader-url-is-not-working
## Modified for Python 3
## Added --event=history|div|split default = history
## changed so "to:date" is included in the returned results
## usage: download_quote(symbol, date_from, date_to, events).decode('utf-8')
import re
from urllib.request import urlopen, Request, URLError
import calendar
import datetime
import getopt
import sys
import time
crumble_link = 'https://finance.yahoo.com/quote/{0}/history?p={0}'
crumble_regex = r'CrumbStore":{"crumb":"(.*?)"}'
cookie_regex = r'Set-Cookie: (.*?); '
quote_link = 'https://query1.finance.yahoo.com/v7/finance/download/{}?period1={}&period2={}&interval=1d&events={}&crumb={}'
def get_crumble_and_cookie(symbol):
link = crumble_link.format(symbol)
response = urlopen(link)
match = re.search(cookie_regex, str(response.info()))
cookie_str = match.group(1)
text = response.read().decode("utf-8")
match = re.search(crumble_regex, text)
crumble_str = match.group(1)
return crumble_str , cookie_str
def download_quote(symbol, date_from, date_to,events):
time_stamp_from = calendar.timegm(datetime.datetime.strptime(date_from, "%Y-%m-%d").timetuple())
next_day = datetime.datetime.strptime(date_to, "%Y-%m-%d") + datetime.timedelta(days=1)
time_stamp_to = calendar.timegm(next_day.timetuple())
attempts = 0
while attempts < 5:
crumble_str, cookie_str = get_crumble_and_cookie(symbol)
link = quote_link.format(symbol, time_stamp_from, time_stamp_to, events,crumble_str)
#print link
r = Request(link, headers={'Cookie': cookie_str})
try:
response = urlopen(r)
text = response.read()
print ("{} downloaded".format(symbol))
return text
except URLError:
print ("{} failed at attempt # {}".format(symbol, attempts))
attempts += 1
time.sleep(2*attempts)
return b''
if __name__ == '__main__':
print (get_crumble_and_cookie('KO'))
from_arg = "from"
to_arg = "to"
symbol_arg = "symbol"
event_arg = "event"
output_arg = "o"
opt_list = (from_arg+"=", to_arg+"=", symbol_arg+"=", event_arg+"=")
try:
options, args = getopt.getopt(sys.argv[1:],output_arg+":",opt_list)
except getopt.GetoptError as err:
print (err)
symbol_val = ""
from_val = ""
to_val = ""
output_val = ""
event_val = "history"
for opt, value in options:
if opt[2:] == from_arg:
from_val = value
elif opt[2:] == to_arg:
to_val = value
elif opt[2:] == symbol_arg:
symbol_val = value
elif opt[2:] == event_arg:
event_val = value
elif opt[1:] == output_arg:
output_val = value
print ("downloading {}".format(symbol_val))
text = download_quote(symbol_val, from_val, to_val,event_val)
if text:
with open(output_val, 'wb') as f:
f.write(text)
print ("{} written to {}".format(symbol_val, output_val))

Answer 3

Got it to work, now I just have to parse the csv. Thought I'd share since I was having trouble with the syntax.
Dim crumb As String: crumb = "xxxx"
Dim cookie As String: cookie = "yyyy"
Dim urlStock As String: urlStock = "https://query1.finance.yahoo.com/v7/finance/download/SIRI?" & _
"period1=1274158800&" & _
"period2=1495059477&" & _
"interval=1d&events=history&crumb=" & crumb
Dim http As MSXML2.XMLHTTP: Set http = New MSXML2.ServerXMLHTTP
http.Open "GET", urlStock, False
http.setRequestHeader "Cookie", cookie
http.send

Answer 4

You can manually save the crumb/cookie pair in Chrome or you can use something like this to generate it. Then, just set the cookie header in java and pass the corresponding crumb in the URL

Answer 5

I have wrote a lightweight script that pulls together alot of the suggestions in this thread to fix this problem. https://github.com/AndrewRPorter/yahoo-historical
However, there are much better solutions such as, https://github.com/ranaroussi/fix-yahoo-finance
Hope these resources help!

Answer 6

I developed the following solution for this issue in Excel/VBA. The key challenge was the creation of the Crumb / Cookie pair. Once that is created you can re-use it for calls to Yahoo for the historical prices.
See here the key code for the Crumb / Cookie
Sub GetYahooRequest(strCrumb As String, strCookie As String)
'This routine will use a sample request to Yahoo to obtain a valid Cookie and Crumb
Dim strUrl As String: strUrl = "https://finance.yahoo.com/lookup?s=%7B0%7D"
Dim objRequest As WinHttp.WinHttpRequest
Set objRequest = New WinHttp.WinHttpRequest
With objRequest
.Open "GET", strUrl, True
.setRequestHeader "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"
.send
.waitForResponse
strCrumb = strExtractCrumb(.responseText)
strCookie = Split(.getResponseHeader("Set-Cookie"), ";")(0)
End With
End Sub
See the following Yahoo Historical Price Extract on my website for a Sample Excel workbook that demonstrates how to extract Yahoo Historical prices

Answer 7

Great answer Andrea, I have added to your code to allow for downloads of multiple stocks. (python 2.7)
file1: down.py
import os
myfile = open("ticker.csv", "r")
lines = myfile.readlines()
for line in lines:
ticker = line.strip();
cmd = "python get_quote_history.py --symbol=%s --from=2017-01-01 --to=2017-05-25 -o %s.csv" %(ticker,ticker)
os.system(cmd)
file2: ticker.csv
AAPL
MSFT
file3: get_quote_history.py