How to login to a site say http://www.example.com/ given I have a username and a password and retreive Cookies for further usage in Python 3?
Here is a code snippet I used in python to log into a web page:
base_url = "http://www.example.com"
store = 01234
username = 'myusername'
password = 'mypassword'
post_data = {
'hdnAction': 'LOGIN',
'txtStoreID': '%s' % store,
'txtLogin': '%s' % username,
'txtpassword': '%s' % password,
'txtNumLines':'10',
'btnLogin':'Login',
'hdnCount':'0'
}
params = urllib.urlencode(post_data)
request = urllib2.Request(base_url + "/", params)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
#print "getting url: %s" % request.get_full_url()
response = urllib2.urlopen(request, params, 300)
xmldata = response.read()
redirects = re.compile('''window\.location\.href="(.*?)"''', re.DOTALL).findall(xmldata)
cookie = response.headers.get('Set-Cookie').split(';')[0].strip() + ";"
if len(redirects) > 0 :
request = urllib2.Request(base_url + "%s" % redirects[0])
print "getting url: %s" % request.get_full_url()
request.add_header('cookie', cookie)
response = urllib2.urlopen(request, None, 300)
xmldata = response.read()
Those things in post data are elements on the HTML FORM tag. Look at thier "name" attributes.
Related
I wish to create new a data when any user was make any requests like Create Export Delete or Update but now I am working on Export.. I have two urls one is having data of tables (average_data urls) and one url which is have (audit_logs url) a data of user which will be created after when any user download an pdf format of the data .. so basically I wish to show which user on which time and which action he does it will get it on the audit_logs url
I am making request of post in data/views but getting error bad request at /audit_logs/create/
views.py
def averagePDF(request):
global fromDate
global toDate
global site
global device
global parameters
global interval
global data
headers = {
'authorization': "Bearer X...............",
}
devices_url = "http://127.0.0.1:8000/stations/list"
devices_params = {
'id': device
}
devices = requests.request("GET", devices_url, headers=headers, params=devices_params)
response = HttpResponse()
response['Content-Disposition'] = 'attachment; filename=Average Data.pdf'
elements = []
company_name = Paragraph(devices.json()[0]['site'], header_style)
elements.append(company_name)
report_data = Paragraph("Date: " + fromDate + " to " + toDate + " Station: " + devices.json()[0]['station'], title_style)
elements.append(report_data)
styles = getSampleStyleSheet()
styleN = styles['Normal']
styleN.wordWrap = 'CJK'
file_data = []
header = []
header.append("Timestamp")
header.append("Station")
for parameter in parameters:
header.append(parameter)
file_data.append(header)
data2 = [[Paragraph(cell, styleN) for cell in row] for row in file_data]
width = (PAGE_WIDTH-50) / len(header)
table_header = Table(data2, colWidths=width, style=table_header_style)
elements.append(table_header)
table_data = []
for key, values in data.items():
raw_data = []
raw_data.append(str(key))
raw_data.append(devices.json()[0]['station'])
for value in values:
raw_data.append(str(value))
table_data.append(raw_data)
table_data2 = [[Paragraph(cell, styleN)for cell in row] for row in table_data]
tableRaw = LongTable(table_data2, colWidths=width, style=table_style)
elements.append(tableRaw)
doc.title = "Average Data"
meta_data = request.META.get('HTTP_X_FORWARDED_FOR')
if meta_data:
ip = meta_data.split(',')[-1].strip()
else:
ip = request.META.get('REMOTE_ADDR')
**now=datetime.datetime.now()
# date_time = now.strftime('%Y-%m-%dT%H:%M:%S.%f')
username=str(request.user)
action_type="Export"
action="Export"
ip_address=ip
audit_url="http://127.0.0.1:8000/audit_logs/create/"
audit_parms={
"username":username,
"action_type":action_type,
"action":action,
"ip_address":ip_address
}
audit_obj=requests.post(audit_url, headers=headers, params=audit_parms)
print(audit_obj.json())**
when I am posting response its give me following response
{'username': 'abcpadmin', 'action_type': 'Export', 'action': 'Export', 'ip_address': '127.0.0.1'}
Bad Request: /audit_logs/create/
[15/Nov/2021 16:08:24] "POST /audit_logs/create/?username=abcpadmin&action_type=Export&action=Export&ip_address=127.0.0.1 HTTP/1.1" 400 160
<Response [400]>
{'username': ['This field is required.'], 'action_type': ['This field is required.'], 'action': ['This field is required.'], 'ip_address': ['This field is required.']}
When you use requests for post in your case, you need pass data instead of params
It should be :
audit_obj=requests.post(audit_url, headers=headers, data=audit_parms)
Technically, when you do requests.post(url=your_url, params=your_params), the url will be like https://localhost?key=value with key value in params dictionary.
You should take a look at djangorestframework
https://www.django-rest-framework.org/tutorial/2-requests-and-responses/
This is the recommended way to write APIs with django.
Some background. This code takes a data from a data collection script and then posts it to a secured REST API in Django. I have an api key generated and it is the only line in the file api.key. I also have the url to post to in the file post.url (it looks like http://example.com/api/ and then I concatenate the proper api node name on the end).
The code is below for my solar data api node (posts data collected from solar panels)
import gather_solar as gs
import requests
import json
import os
def post_solar():
print("DEBUG: start solar")
data = gs.gather_solar()
api_key = None
url = None
try:
here = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(here, 'api.key')
file = open(filename, "r")
api_key = file.readline()
api_key.replace('\n', '')
except Exception as e:
print("ERROR: " + str(e))
try:
here = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(here, 'post.url') #server will use different url
file = open(filename, "r")
url = file.readline()
url.replace('\n', '')
except Exception as e:
print("ERROR: " + str(e))
if api_key is not None and url is not None:
authorization = "Token " + api_key
authorization.replace('\n', '')
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Authorization': authorization
}
json_data = json.dumps(data)
url += "solar/"
url.replace('\n', '')
print(url)
req = requests.Request('POST', url, data=json_data, headers=headers)
prepared = req.prepare()
print("DEBUG: POST Headers: " + str(prepared.headers))
print("DEBUG: POST Body: " + str(prepared.body))
s = requests.Session()
response = s.send(prepared)
print("DEBUG: Response Code: " + str(response.status_code))
print("DEBUG: Response Headers: " + str(response.headers))
print("DEBUG: Response Data: " + str(response.json()))
else:
print("DEBUG: Error with API key")
print("DEBUG: end solar")
I am running code on an ubuntu server via AWS and have Apache 2 setup and running. However, whenever I run this script, I get an error saying that my token is invalid and it shows the token as "Token abcd...abcd\n" . This is especially frustrating because I don't have this issue when I run the script locally (visual studio code on Win10) As you can see I have tried to remove any newlines wherever possible but it doesn't seem to be helping. Any help would be greatly appreciated
replace() does not change the string (strings are immutable); it returns a new string. So:
api_key = file.readline()
api_key.replace('\n', '')
leaves the \n on api_key and you are ignoring the new string returned by replace().
You could just string the line before assignment:
api_key = file.readline().strip()
#app.route('/login')
def showLogin():
state = ''.join(random.choice(string.ascii_uppercase + string.digits)
for x in xrange(32))
login_session['state'] = state
# return "The current session state is %s" % login_session['state']
return render_template('login.html', STATE=state)
#app.route('/gconnect', methods=['POST'])
def gconnect():
# Validate state token
if request.args.get('state') != login_session['state']:
response = make_response(json.dumps('Invalid state parameter.'), 401)
response.headers['Content-Type'] = 'application/json'
return response
# Obtain authorization code
code = request.data
try:
# Upgrade the authorization code into a credentials object
oauth_flow = flow_from_clientsecrets('client_secrets.json', scope='')
oauth_flow.redirect_uri = 'postmessage'
credentials = oauth_flow.step2_exchange(code)
except FlowExchangeError:
response = make_response(
json.dumps('Failed to upgrade the authorization code.'), 401)
response.headers['Content-Type'] = 'application/json'
return response
# Check that the access token is valid.
access_token = credentials.access_token
url = ('https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=%s'
% access_token)
h = httplib2.Http()
result = json.loads(h.request(url, 'GET')[1])
# If there was an error in the access token info, abort.
if result.get('error') is not None:
response = make_response(json.dumps(result.get('error')), 500)
response.headers['Content-Type'] = 'application/json'
return response
# Verify that the access token is used for the intended user.
gplus_id = credentials.id_token['sub']
if result['user_id'] != gplus_id:
response = make_response(
json.dumps("Token's user ID doesn't match given user ID."), 401)
response.headers['Content-Type'] = 'application/json'
return response
# Verify that the access token is valid for this app.
if result['issued_to'] != CLIENT_ID:
response = make_response(
json.dumps("Token's client ID does not match app's."), 401)
print "Token's client ID does not match app's."
response.headers['Content-Type'] = 'application/json'
return response
stored_access_token = login_session.get('access_token')
stored_gplus_id = login_session.get('gplus_id')
if stored_access_token is not None and gplus_id == stored_gplus_id:
response = make_response(json.dumps('Current user is connected'), 200)
response.headers['Content-Type'] = 'application/json'
return response
# Store the access token in the session for later use.
login_session['access_token'] = credentials.access_token
login_session['gplus_id'] = gplus_id
# Get user info
userinfo_url = "https://www.googleapis.com/oauth2/v1/userinfo"
params = {'access_token': credentials.access_token, 'alt': 'json'}
answer = requests.get(userinfo_url, params=params)
data = answer.json()
login_session['username'] = data['name']
login_session['picture'] = data['picture']
login_session['email'] = data['email']
# See if a user exists, if it doesn't make a new one
user_id = getUserID(login_session['email'])
if not user_id:
user_id = createUser(login_session)
login_session['user_id'] = user_id
output = ''
output += '<h1>Welcome, '
output += login_session['username']
output += '!</h1>'
output += '<img src="'
output += login_session['picture']
output += ' " style = "width: 300px; height: 300px;border-radius: 150px;'
output += '-webkit-border-radius: 150px;-moz-border-radius: 150px;"> '
print "done!"
return output
I ran this OAuth code on my local machine. It worked but on Amazon Lightsail instance it gave the following error:
*Error: invalid_request Permission denied to generate login hint for target domain.
I even corrected my Redirect Uris and JAVA Script Origins to .com but it still gives the same error even on erasing the cookies.
I read the documentation yesterday and done some coding with python to fetch data in the following way. It's working fine.
import logging as log
import adal
import requests
import json
import datetime
from pprint import pprint
# Details of workspace. Fill in details for your workspace.
resource_group = 'Test'
workspace = 'FirstMyWorkspace'
# Details of query. Modify these to your requirements.
query = "Type=*"
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(hours=24)
num_results = 2 # If not provided, a default of 10 results will be used.
# IDs for authentication. Fill in values for your service principal.
subscription_id = '{subscription_id}'
tenant_id = '{tenant_id}'
application_id = '{application_id}'
application_key = '{application_key}'
# URLs for authentication
authentication_endpoint = 'https://login.microsoftonline.com/'
resource = 'https://management.core.windows.net/'
# Get access token
context = adal.AuthenticationContext('https://login.microsoftonline.com/' + tenant_id)
token_response = context.acquire_token_with_client_credentials('https://management.core.windows.net/', application_id, application_key)
access_token = token_response.get('accessToken')
# Add token to header
headers = {
"Authorization": 'Bearer ' + access_token,
"Content-Type": 'application/json'
}
# URLs for retrieving data
uri_base = 'https://management.azure.com'
uri_api = 'api-version=2015-11-01-preview'
uri_subscription = 'https://management.azure.com/subscriptions/' + subscription_id
uri_resourcegroup = uri_subscription + '/resourcegroups/'+ resource_group
uri_workspace = uri_resourcegroup + '/providers/Microsoft.OperationalInsights/workspaces/' + workspace
uri_search = uri_workspace + '/search'
# Build search parameters from query details
search_params = {
"query": query,
"top": num_results
}
# Build URL and send post request
uri = uri_search + '?' + uri_api
response = requests.post(uri, json=search_params,headers=headers)
# Response of 200 if successful
if response.status_code == 200:
# Parse the response to get the ID and status
data = response.json()
if data.get("__metadata", {}).get("resultType", "") == "error":
log.warn("oms_fetcher;fetch_job;error: " + ''.join('{}={}, '.format(key, val) for key, val in
data.get("error", {}).items()))
else:
print data["value"]
search_id = data["id"].split("/")
id = search_id[len(search_id)-1]
status = data["__metadata"]["Status"]
print status
# If status is pending, then keep checking until complete
while status == "Pending":
# Build URL to get search from ID and send request
uri_search = uri_search + '/' + id
uri = uri_search + '?' + uri_api
response = requests.get(uri, headers=headers)
# Parse the response to get the status
data = response.json()
status = data["__metadata"]["Status"]
print id
else:
# Request failed
print (response.status_code)
response.raise_for_status()
Today I went to the same webpage that I have followed yesterday but there is a different documentation today. So do I need to follow the new documentation? I tried new documentation too but got into an issue
url = "https://api.loganalytics.io/v1/workspaces/{workspace_id}/query"
headers = {
"X-Api-Key": "{api_key}",
"Content-Type": 'application/json'
}
search_param = {
}
res = requests.post(url=url, json=search_param, headers=headers)
print res.status_code
print res.json()
{u'error': {u'innererror': {u'message': u'The given API Key is not
valid for the request', u'code': u'UnsupportedKeyError'}, u'message':
u'Valid authentication was not provided', u'code':
u'AuthorizationRequiredError'}}
Here is the link to documentation
The api_key is not oms primary key on Portal. You could check example in this link. The token should like below:
Authorization: Bearer <access token>
So, you need modify X-Api-Key": "{api_key} to Authorization: Bearer <access token>.
You need create a service principal firstly, please check this link.
Then, you could use the sp to get token, please check this link.
Note: You could your code to get token, but you need modify the resource to https://api.loganalytics.io. Like below:
# Get access token
context = adal.AuthenticationContext('https://login.microsoftonline.com/' + tenant_id)
token_response = context.acquire_token_with_client_credentials('https://api.loganalytics.io', application_id, application_key)
access_token = token_response.get('accessToken')
# Add token to header
headers = {
"Authorization": 'Bearer ' + access_token,
"Content-Type": 'application/json'
}
Working Prototype to Query OMS or Log Analytic workspace.
import adal
import requests
import json
import datetime
from pprint import pprint
# Details of workspace. Fill in details for your workspace.
resource_group = 'xxx'
workspace = 'xxx'
workspaceid = 'xxxx'
# Details of query. Modify these to your requirements.
query = "AzureActivity | limit 10"
# IDs for authentication. Fill in values for your service principal.
subscription_id = 'xxxx'
# subscription_id = 'xxxx'
tenant_id = 'xxxx'
application_id = 'xxxx'
application_key = 'xxxxx'
# Get access token
context = adal.AuthenticationContext('https://login.microsoftonline.com/' + tenant_id)
token_response = context.acquire_token_with_client_credentials('https://api.loganalytics.io', application_id, application_key)
access_token = token_response.get('accessToken')
# Add token to header
headers = {
"Authorization": 'Bearer ' + access_token,
"Content-Type": 'application/json'
}
search_params = {
"query": query
}
url = "https://api.loganalytics.io/v1/workspaces/{workspaceID}/query"
res = requests.post(url=url, json=search_params, headers=headers)
print (res.status_code)
print (res.json())
This is my code. Instead of all the garbage values that I receive, how can I just output the text of the tweet, username and the timestamp?
import oauth2 as oauth
import urllib2 as urllib
access_token_key = "secret"
access_token_secret = "secret
consumer_key = "secret"
consumer_secret = "secret"
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
def fetchsamples():
url = "https://userstream.twitter.com/1.1/user.json"
parameters = []
response = twitterreq(url, "GET", parameters)
for line in response:
print line.strip()
if __name__ == '__main__':
fetchsamples()
The output I receive is like this
{"created_at":"Mon Dec 09 15:10:41 +0000 2013","id":410063960465891329,"id_str":"410063960465891329","text":"Django Hacker, Next Gen E-Commerce Technology http://t.co/de6gdIqJvL #Hyderabad","source":"\u003ca href=\"http://jobs.hasgeek.com\" rel=\"nofollow\"\u003eHasGeek Job Board\u003c/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":260654014,"id_str":"260654014","name":"HasGeek Job Board","screen_name":"hasjob","location":"","url":"http://jobs.hasgeek.in","description":"Job listings from the HasGeek Job Board.","protected":false,"followers_count":1530,"friends_count":1,"listed_count":32,"created_at":"Fri Mar 04 09:21:06 +0000 2011","favourites_count":0,"utc_offset":19800,"time_zone":"Mumbai","geo_enabled":true,"verified":false,"statuses_count":10528,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http://abs.twimg.com/images/themes/theme1/bg.png","profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme1/bg.png","profile_background_tile":false,"profile_image_url":"http://pbs.twimg.com/profile_images/1271127705/logo-star_normal.png","profile_image_url_https":"https://pbs.twimg.com/profile_images/1271127705/logo-star_normal.png","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"Hyderabad","indices":[69,79]}],"symbols":[],"urls":[{"url":"http://t.co/de6gdIqJvL","expanded_url":"http://hsgk.in/1aOhn8V","display_url":"hsgk.in/1aOhn8V","indices":[46,68]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"}
It's json. In python, at a very basic level, you can import the json library, load the tweet as a python object from json and print the fields as below.
import json
...
tweetObj = json.loads(response)
print tweetObj['user']['id_str'], tweetObj['created_at'], tweetObj['text']