I have users that need really high throttles so they can use the system a lot. Is there an easy way to give them higher throttles than the rest of the users?
I've looked around but haven't found anything.
I figured out a way to do this by extending the UserRateThrottle and adding special users to my settings file.
This class just overrides the allow_request method, adding some special logic to see if usernames are listed in the OVERRIDE_THROTTLE_RATES variable:
class ExceptionalUserRateThrottle(UserRateThrottle):
def allow_request(self, request, view):
"""
Give special access to a few special accounts.
Mirrors code in super class with minor tweaks.
"""
if self.rate is None:
return True
self.key = self.get_cache_key(request, view)
if self.key is None:
return True
self.history = self.cache.get(self.key, [])
self.now = self.timer()
# Adjust if user has special privileges.
override_rate = settings.REST_FRAMEWORK['OVERRIDE_THROTTLE_RATES'].get(
request.user.username,
None,
)
if override_rate is not None:
self.num_requests, self.duration = self.parse_rate(override_rate)
# Drop any requests from the history which have now passed the
# throttle duration
while self.history and self.history[-1] <= self.now - self.duration:
self.history.pop()
if len(self.history) >= self.num_requests:
return self.throttle_failure()
return self.throttle_success()
To use this, just set your DEFAULT_THROTTLE_CLASS to this class, then put some special users into OVERRIDE_THROTTLE_RATES like so:
'DEFAULT_THROTTLE_CLASSES': (
'rest_framework.throttling.AnonRateThrottle',
'cl.api.utils.ExceptionalUserRateThrottle',
),
'DEFAULT_THROTTLE_RATES': {
'anon': '100/day',
'user': '1000/hour',
},
'OVERRIDE_THROTTLE_RATES': {
'scout': '10000/hour',
'scout_test': '10000/hour',
},
I have found the solution after customized Django REST Throttling,
Its Blocking particular user after 3 login attempts (Block user_id that presents in my application). Block IP address after 6 login attempts for anonymous user.
prevent.py :-
#!/usr/bin/python
from collections import Counter
from rest_framework.throttling import SimpleRateThrottle
from django.contrib.auth.models import User
class UserLoginRateThrottle(SimpleRateThrottle):
scope = 'loginAttempts'
def get_cache_key(self, request, view):
user = User.objects.filter(username=request.data.get('username'))
ident = user[0].pk if user else self.get_ident(request)
return self.cache_format % {
'scope': self.scope,
'ident': ident
}
def allow_request(self, request, view):
"""
Implement the check to see if the request should be throttled.
On success calls `throttle_success`.
On failure calls `throttle_failure`.
"""
if self.rate is None:
return True
self.key = self.get_cache_key(request, view)
if self.key is None:
return True
self.history = self.cache.get(self.key, [])
self.now = self.timer()
while self.history and self.history[-1] <= self.now - self.duration:
self.history.pop()
if len(self.history) >= self.num_requests:
return self.throttle_failure()
if len(self.history) >= 3:
data = Counter(self.history)
for key, value in data.items():
if value == 2:
return self.throttle_failure()
return self.throttle_success(request)
def throttle_success(self, request):
"""
Inserts the current request's timestamp along with the key
into the cache.
"""
user = User.objects.filter(username=request.data.get('username'))
if user:
self.history.insert(0, user[0].id)
self.history.insert(0, self.now)
self.cache.set(self.key, self.history, self.duration)
return True
Views.py :-
from .prevent import UserLoginRateThrottle
....
....
....
class ObtainAuthToken(auth_views.ObtainAuthToken):
throttle_classes = (UserLoginRateThrottle,)/use this method here your login view
def post(self, request, *args, **kwargs):
....
....
Settings.py :-
Django-rest-framework
REST_FRAMEWORK = {
...
...
...
'DEFAULT_THROTTLE_CLASSES': (
'rest_framework.throttling.UserRateThrottle',
),
'DEFAULT_THROTTLE_RATES': {
'loginAttempts': '6/hr',
'user': '1000/min',
}
}
I know this a pretty old thread and the accepted answer was helpful for me as well. I wanted to show how you can have multiple user rate throttles in place, in this case additional ones for the root users
from rest_framework.settings import api_settings
from django.core.exceptions import ImproperlyConfigured
class RootRateThrottle(UserRateThrottle):
"""
Limits the rate of API calls that may be made by a given user.
The user id will be used as a unique cache key if the user is
authenticated. For anonymous requests, the IP address of the request will
be used.
"""
def get_cache_key(self, request, view):
if request.user.is_authenticated:
ident = request.user.pk
else:
ident = self.get_ident(request)
self.rate = self.get_rate(request)
logger.debug(
"Throttling rate for %s: %s", request.user, self.rate
)
self.num_requests, self.duration = self.parse_rate(self.rate)
return self.cache_format % {
'scope': self.scope,
'ident': ident
}
def get_rate(self, request=None):
"""
Determine the string representation of the allowed request rate.
"""
if not getattr(self, 'scope', None):
msg = ("You must set either `.scope` or `.rate` for '%s' throttle" %
self.__class__.__name__)
raise ImproperlyConfigured(msg)
if request and request.user.is_superuser:
throttle_rates = settings.REST_FRAMEWORK["ROOT_THROTTLE_RATES"]
else:
throttle_rates = api_settings.DEFAULT_THROTTLE_RATES
try:
return throttle_rates[self.scope]
except KeyError:
msg = "No default throttle rate set for '%s' scope" % self.scope
raise ImproperlyConfigured(msg)
class ByMinuteRateThrottle(RootRateThrottle):
scope = 'minute'
class ByHourRateThrottle(RootRateThrottle):
scope = 'hour'
class ByDayRateThrottle(RootRateThrottle):
scope = 'day'
the settings part then looks like this
'DEFAULT_THROTTLE_CLASSES': [
'threedi_api.throttling.ByMinuteRateThrottle',
'threedi_api.throttling.ByHourRateThrottle',
'threedi_api.throttling.ByDayRateThrottle',
],
'DEFAULT_THROTTLE_RATES': {
'minute': '100/min',
'hour': '1000/hour',
'day': '5000/day',
},
'ROOT_THROTTLE_RATES': {
'minute': '200/min',
'hour': '2000/hour',
'day': '10000/day',
},
Related
How can I prevent Django rest throttling count the request when the user request is invalid or the server failed to complete the process?
For example, I need params from the user, but when the user does not give the params, Django rest throttling still counts it.
Is there any solution to skipping the throttling counter when the request is not successful?
Example
class OncePerHourAnonThrottle(AnonRateThrottle):
rate = "1/hour"
class Autoliker(APIView):
throttle_classes = [OncePerHourAnonThrottle]
def get(self, request):
content = {"status": "get"}
return Response(content)
def post(self, request):
post_url = request.POST.get("url", None)
print(post_url)
content = {"status": "post"}
return Response(content)
def throttled(self, request, wait):
raise Throttled(
detail={
"message": "request limit exceeded",
"availableIn": f"{wait} seconds",
"throttleType": "type",
}
)
You can create a decorator to do so.
class OncePerHourAnonThrottle(AnonRateThrottle):
rate = "1/hour"
def allow_request(self, request, view):
"""
This function is copy of SimpleRateThrottle.allow_request
The only difference is, instead of executing self.throttle_success
it directly returns True and doesn't mark this request as success yet.
"""
if self.rate is None:
return True
self.key = self.get_cache_key(request, view)
if self.key is None:
return True
self.history = self.cache.get(self.key, [])
self.now = self.timer()
# Drop any requests from the history which have now passed the
# throttle duration
while self.history and self.history[-1] <= self.now - self.duration:
self.history.pop()
if len(self.history) >= self.num_requests:
return False
return True
def rate_limiter(view_function):
#wraps(view_function)
def inner(view_obj, request, *args, **kwargs):
throttle = OncePerHourAnonThrottle()
allowed = throttle.allow_request(request, None)
if not allowed:
raise exceptions.Throttled(throttle.wait())
try:
response = view_function(view_obj, request, *args, **kwargs)
except Exception as exc:
response = view_obj.handle_exception(exc)
if response.status_code == 200:
# now if everything goes OK, count this request as success
throttle.throttle_success()
return response
return inner
class Autoliker(APIView):
#rate_limiter
def post(requests):
# view logic
pass
This is the basic idea how you can do it, now you can make it a generic decorator or even class based decorator.
I have been programming an API using Django and djangorestframework for deployment in Google App Engine. The API is basically a package registry, so you can create, update, get, and delete packages with the API.
All the endpoints seem to work except for one. The only endpoint that doesn't work is one that display a paginated list of all packages in the online registry.
All the endpoints are working, but for some reason, when I hit the specific endpoint '/packages/', GCP gives me the error
400. That’s an error.
Your client has issued a malformed or illegal request. That’s all we know
When I run the application locally on my computer, all the endpoints work perfectly. The application only stops working for that specific route when I deploy it on Google App Engine.The API payload should be:
[
{
"Name": "*"
}
]
I am completely lost on this one and would appreciate any help.
VIEWS.py
import django.db.utils
from rest_framework.decorators import api_view
from rest_framework.response import Response
from django.core.paginator import Paginator, EmptyPage
import registry.models
from .serializers import *
from .models import *
# Create your views here.
#api_view(['GET'])
def apiOverview(request):
api_urls = {
'List': '/package-list/',
'Create': '/package-create/',
'Update': '/package-update/',
'Delete': '/package-delete/',
'Rate': '/package-rate/',
'Download': '/package-download/'
}
return Response(api_urls)
#api_view(['GET'])
def packages_middleware(request):
print(request)
print(type(request))
print(request.data)
print(type(request.data))
# determine offset query parameter
offset = request.GET.get('Offset')
if offset is None:
offset = 0
else:
offset = int(offset)
# capturing request body
response = []
queries = request.data
if len(queries) < 1:
return Response({'message: empty request body array'}, status=400)
else:
if len(queries) == 1 and queries[0]['Name'] == '*':
response = list(PackageMetadata.objects.all().values())
else:
for query in queries:
if 'Name' in query.keys() and 'Version' in query.keys():
for x in list(PackageMetadata.objects.filter(Name__icontains=query['Name']).filter(
Version__contains=query['Version']).values()):
response.append(x)
else:
response.append({
'message': 'query {q} is missing at least one attribute - remember to check spelling and capitalization'.format(q=query)
})
paginator = Paginator(response, 10)
try:
return Response(paginator.page(offset + 1).object_list, headers={'Offset': offset + 1})
except EmptyPage:
return Response(paginator.page(1).object_list, headers={'Offset': 2})
#api_view(['GET', 'PUT', 'DELETE'])
def package_middleware(request, pk):
try:
package = Package.objects.get(Metadata__ID__exact=str(pk))
if request.method == 'GET':
# CHECK THAT CONTENT FIELD IS SET
serializer = PackageSerializer(package, many=False)
return Response(serializer.data, status=200)
elif request.method == 'PUT':
payload = request.data
if 'Metadata' in payload and 'Data' in payload:
if payload['Metadata'] != PackageMetadataSerializer(package.Metadata, many=False).data:
return Response({"message": "name, version and ID must match"}, status=400)
else:
# CHECK THAT ONLY ONE DATA FIELD IS SET
serializer_data = PackageDataSerializer(data=payload['Data'], many=False)
if serializer_data.is_valid(raise_exception=True):
try:
serializer_data.update(instance=package.Data, validated_data=serializer_data.validated_data)
except django.db.utils.IntegrityError:
return Response(
{"message": "both Content and URL must be included in query, but exactly one can be set"},
status=400)
return Response(status=200)
else:
return Response(
{"message": "incorrect request body schema - remember to check spelling and capitalization"},
status=400)
else:
package.Metadata.delete()
package.Data.delete()
package.delete()
return Response({"message": "package deleted"}, status=200)
except registry.models.Package.DoesNotExist:
return Response({"message": "package not found"}, status=400)
#api_view(['POST'])
def create_package_middleware(request):
payload = request.data
print(payload)
if 'Metadata' in payload and 'Data' in payload:
serializer_metadata = PackageMetadataSerializer(data=payload['Metadata'], many=False)
serializer_data = PackageDataSerializer(data=payload['Data'], many=False)
if serializer_data.is_valid(raise_exception=True) and serializer_metadata.is_valid(raise_exception=True):
try:
metadata = PackageMetadata.objects.create(ID=serializer_metadata.data.get('ID'),
Name=serializer_metadata.data.get('Name'),
Version=serializer_metadata.data.get('Version'))
except django.db.utils.IntegrityError:
return Response({"message": "duplicate key-value (Name, Version) violates uniqueness constraint"},
status=403)
try:
data = PackageData.objects.create(Content=serializer_data.data.get('Content'),
URL=serializer_data.data.get('URL'))
except django.db.utils.IntegrityError:
metadata.delete()
return Response(
{"message": "both Content and URL must be included in query, but exactly one can be set"},
status=400)
Package.objects.create(Data=data, Metadata=metadata)
serializer_metadata = PackageMetadataSerializer(metadata, many=False)
return Response(serializer_metadata.data, status=200)
else:
return Response({"message": "incorrect request body schema - remember to check spelling and capitalization"},
status=400)
#api_view(['DELETE'])
def byName_middleware(request, name):
if name == '*':
return Response({"message": "query name reserved"}, status=400)
querySet = Package.objects.filter(Metadata__Name__iexact=name)
if len(querySet) == 0:
return Response({"message": "package not found"})
else:
for package in querySet:
package.Metadata.delete()
package.Data.delete()
package.delete()
return Response(status=200)
URLS.py
from django.urls import path, include
from . import views
urlpatterns = [
path('', views.apiOverview),
path('packages/', views.packages_middleware, name='packages_middleware'),
path('package/<str:pk>', views.package_middleware, name='package'),
path('package/', views.create_package_middleware, name='create'),
path('package/byName/<str:name>', views.byName_middleware, name='byName')
]
MODELS.py
from django.db import models
# Create your models here.
class PackageData(models.Model):
Content = models.TextField(blank=True, null=True) # actual zip file
URL = models.CharField(max_length=500, blank=True, null=True) # url of package
# class Meta:
# constraints = [
# models.CheckConstraint(
# name="%(app_label)s_%(class)s_content_or_url",
# check=models.Q(Content__isnull=True, URL__isnull=False) | models.Q(Content__isnull=False, URL__isnull=True)
# )
# ]
class PackageMetadata(models.Model):
class Meta:
constraints = [
models.UniqueConstraint(fields=['Name', 'Version'], name='unique_package')
]
ID = models.CharField(primary_key=True, max_length=50)
Name = models.CharField(max_length=50)
Version = models.CharField(max_length=50)
class Package(models.Model):
Data = models.ForeignKey(PackageData, on_delete=models.CASCADE)
Metadata = models.ForeignKey(PackageMetadata, on_delete=models.CASCADE)
class PackageRating(models.Model):
BusFactor = models.DecimalField(max_digits=10, decimal_places=9)
Correctness = models.DecimalField(max_digits=10, decimal_places=9)
GoodPinningPractice = models.DecimalField(max_digits=10, decimal_places=9)
LicenseScore = models.DecimalField(max_digits=10, decimal_places=9)
RampUp = models.DecimalField(max_digits=10, decimal_places=9)
ResponsiveMaintainer = models.DecimalField(max_digits=10, decimal_places=9)
class PackageQuery(models.Model):
Name = models.CharField(max_length=50)
Version = models.CharField(max_length=50)
You are making a GET call to /packages/ which is handled by the route definition packages_middleware(request).
In that function you have queries = request.data and the rest of your logic is dependent on queries. This document says request.data is for POST, PATCH, PUT. If that is correct, then that seems like where your error lies.
You can put a print statement just before and after that line and see if your App prints the statement after. That will help you confirm if that is the issue.
You should also print the contents of queries. Since you're not actually sending a body with your GET request, it is possible that the rest of your logic fails because the value of queries is not what you're expecting.
I've been playing with Django Two Factor authentication for the last two days or so, and I have it partially working. I am trying to figure out a way to remove the QR Token Generator. I have tried subclassing the setup view, but the form wizard is causing me some grief. The wizard is confusing me. I know in a regular form, how to remove radio buttons, but in this case, I can't seem to locate the source of the Token Generator.
The SetupView...
#class_view_decorator(never_cache)
#class_view_decorator(login_required)
class SetupView(IdempotentSessionWizardView):
"""
View for handling OTP setup using a wizard.
The first step of the wizard shows an introduction text, explaining how OTP
works and why it should be enabled. The user has to select the verification
method (generator / call / sms) in the second step. Depending on the method
selected, the third step configures the device. For the generator method, a
QR code is shown which can be scanned using a mobile phone app and the user
is asked to provide a generated token. For call and sms methods, the user
provides the phone number which is then validated in the final step.
"""
success_url = 'two_factor:setup_complete'
qrcode_url = 'two_factor:qr'
template_name = 'two_factor/core/setup.html'
session_key_name = 'django_two_factor-qr_secret_key'
initial_dict = {}
form_list = (
('welcome', Form),
('method', MethodForm),
('generator', TOTPDeviceForm),
('sms', PhoneNumberForm),
('call', PhoneNumberForm),
('validation', DeviceValidationForm),
('yubikey', YubiKeyDeviceForm),
)
condition_dict = {
'generator': lambda self: self.get_method() == 'generator',
'call': lambda self: self.get_method() == 'call',
'sms': lambda self: self.get_method() == 'sms',
'validation': lambda self: self.get_method() in ('sms', 'call'),
'yubikey': lambda self: self.get_method() == 'yubikey',
}
idempotent_dict = {
'yubikey': False,
}
def get_method(self):
method_data = self.storage.validated_step_data.get('method', {})
return method_data.get('method', None)
def get(self, request, *args, **kwargs):
"""
Start the setup wizard. Redirect if already enabled.
"""
if default_device(self.request.user):
return redirect(self.success_url)
return super(SetupView, self).get(request, *args, **kwargs)
def get_form_list(self):
"""
Check if there is only one method, then skip the MethodForm from form_list
"""
form_list = super(SetupView, self).get_form_list()
available_methods = get_available_methods()
if len(available_methods) == 1:
form_list.pop('method', None)
method_key, _ = available_methods[0]
self.storage.validated_step_data['method'] = {'method': method_key}
return form_list
def render_next_step(self, form, **kwargs):
"""
In the validation step, ask the device to generate a challenge.
"""
next_step = self.steps.next
if next_step == 'validation':
try:
self.get_device().generate_challenge()
kwargs["challenge_succeeded"] = True
except Exception:
logger.exception("Could not generate challenge")
kwargs["challenge_succeeded"] = False
return super(SetupView, self).render_next_step(form, **kwargs)
def done(self, form_list, **kwargs):
"""
Finish the wizard. Save all forms and redirect.
"""
# Remove secret key used for QR code generation
try:
del self.request.session[self.session_key_name]
except KeyError:
pass
# TOTPDeviceForm
if self.get_method() == 'generator':
form = [form for form in form_list if isinstance(form, TOTPDeviceForm)][0]
device = form.save()
# PhoneNumberForm / YubiKeyDeviceForm
elif self.get_method() in ('call', 'sms', 'yubikey'):
device = self.get_device()
device.save()
else:
raise NotImplementedError("Unknown method '%s'" % self.get_method())
django_otp.login(self.request, device)
return redirect(self.success_url)
def get_form_kwargs(self, step=None):
kwargs = {}
if step == 'generator':
kwargs.update({
'key': self.get_key(step),
'user': self.request.user,
})
if step in ('validation', 'yubikey'):
kwargs.update({
'device': self.get_device()
})
metadata = self.get_form_metadata(step)
if metadata:
kwargs.update({
'metadata': metadata,
})
return kwargs
def get_device(self, **kwargs):
"""
Uses the data from the setup step and generated key to recreate device.
Only used for call / sms -- generator uses other procedure.
"""
method = self.get_method()
kwargs = kwargs or {}
kwargs['name'] = 'default'
kwargs['user'] = self.request.user
if method in ('call', 'sms'):
kwargs['method'] = method
kwargs['number'] = self.storage.validated_step_data\
.get(method, {}).get('number')
return PhoneDevice(key=self.get_key(method), **kwargs)
if method == 'yubikey':
kwargs['public_id'] = self.storage.validated_step_data\
.get('yubikey', {}).get('token', '')[:-32]
try:
kwargs['service'] = ValidationService.objects.get(name='default')
except ValidationService.DoesNotExist:
raise KeyError("No ValidationService found with name 'default'")
except ValidationService.MultipleObjectsReturned:
raise KeyError("Multiple ValidationService found with name 'default'")
return RemoteYubikeyDevice(**kwargs)
def get_key(self, step):
self.storage.extra_data.setdefault('keys', {})
if step in self.storage.extra_data['keys']:
return self.storage.extra_data['keys'].get(step)
key = random_hex(20).decode('ascii')
self.storage.extra_data['keys'][step] = key
return key
def get_context_data(self, form, **kwargs):
context = super(SetupView, self).get_context_data(form, **kwargs)
if self.steps.current == 'generator':
key = self.get_key('generator')
rawkey = unhexlify(key.encode('ascii'))
b32key = b32encode(rawkey).decode('utf-8')
self.request.session[self.session_key_name] = b32key
context.update({
'QR_URL': reverse(self.qrcode_url)
})
elif self.steps.current == 'validation':
context['device'] = self.get_device()
context['cancel_url'] = resolve_url(settings.LOGIN_REDIRECT_URL)
return context
def process_step(self, form):
if hasattr(form, 'metadata'):
self.storage.extra_data.setdefault('forms', {})
self.storage.extra_data['forms'][self.steps.current] = form.metadata
return super(SetupView, self).process_step(form)
def get_form_metadata(self, step):
self.storage.extra_data.setdefault('forms', {})
return self.storage.extra_data['forms'].get(step, None)
Seems to reference a MethodForm...
class MethodForm(forms.Form):
method = forms.ChoiceField(label=_("Method"),
initial='generator',
widget=forms.RadioSelect)
def __init__(self, **kwargs):
super(MethodForm, self).__init__(**kwargs)
self.fields['method'].choices = get_available_methods()
I can't seem to track back to where the list of choices is defined, clearly in this case it is saying generator is the initial choice in setup, but I can't figure out how to remove the generator option as a list of valid choices. I also tried to remove generator from the form_list but this didn't seem to make a difference either.
If there's an easier way to remove the Token Generator option and a different approach that's altogether different, I'm open to that too.
Thanks in advance for any thoughts.
Found it. It was in the models.py....
def get_available_methods():
methods = [('generator', _('Token generator'))]
methods.extend(get_available_phone_methods())
methods.extend(get_available_yubikey_methods())
return methods
I removed the ('generator', _('Token generator')) reference from inside the brackets ( list ) and it removed the Token Generator option.
I've tried a lot before posting this question, I'm not against down votes, at least let me know WHY you are down voting.
I have built an Airflow plugin to fetch data from Cloud composer Airflow environment and accessing the cloud composer is working great from browsers as it needs the user to sign in before accessing any of the Airflow endpoints.
In my use case, I need to trigger the endpoints via code. Is there a way in which I can do this.
Below is the Airflow-Flask plugin being used
from datetime import datetime
import json
import os
import six
import time
from flask import Blueprint, request, Response
from sqlalchemy import or_
from airflow import settings
from airflow.exceptions import AirflowException, AirflowConfigException
from airflow.models import DagBag, DagRun
from airflow.utils.state import State
from airflow.utils.dates import date_range as utils_date_range
from airflow.www.app import csrf
airflow_api_blueprint = Blueprint('airflow_api', __name__, url_prefix='/api/v1')
class ApiInputException(Exception):
pass
class ApiResponse:
def __init__(self):
pass
STATUS_OK = 200
STATUS_BAD_REQUEST = 400
STATUS_UNAUTHORIZED = 401
STATUS_NOT_FOUND = 404
STATUS_SERVER_ERROR = 500
#staticmethod
def standard_response(status, payload):
json_data = json.dumps({
'response': payload
})
resp = Response(json_data, status=status, mimetype='application/json')
return resp
#staticmethod
def success(payload):
return ApiResponse.standard_response(ApiResponse.STATUS_OK, payload)
#staticmethod
def error(status, error):
return ApiResponse.standard_response(status, {
'error': error
})
#staticmethod
def bad_request(error):
return ApiResponse.error(ApiResponse.STATUS_BAD_REQUEST, error)
#staticmethod
def not_found(error='Resource not found'):
return ApiResponse.error(ApiResponse.STATUS_NOT_FOUND, error)
#staticmethod
def unauthorized(error='Not authorized to access this resource'):
return ApiResponse.error(ApiResponse.STATUS_UNAUTHORIZED, error)
#staticmethod
def server_error(error='An unexpected problem occurred'):
return ApiResponse.error(ApiResponse.STATUS_SERVER_ERROR, error)
#airflow_api_blueprint.before_request
def verify_authentication():
authorization = request.headers.get('authorization')
try:
api_auth_key = settings.conf.get('AIRFLOW_API_PLUGIN', 'AIRFLOW_API_AUTH')
except AirflowConfigException:
return
if authorization != api_auth_key:
return ApiResponse.unauthorized("You are not authorized to use this resource")
def format_dag_run(dag_run):
return {
'run_id': dag_run.run_id,
'dag_id': dag_run.dag_id,
'state': dag_run.get_state(),
'start_date': (None if not dag_run.start_date else str(dag_run.start_date)),
'end_date': (None if not dag_run.end_date else str(dag_run.end_date)),
'external_trigger': dag_run.external_trigger,
'execution_date': str(dag_run.execution_date)
}
def find_dag_runs(session, dag_id, dag_run_id, execution_date):
qry = session.query(DagRun)
qry = qry.filter(DagRun.dag_id == dag_id)
qry = qry.filter(or_(DagRun.run_id == dag_run_id, DagRun.execution_date == execution_date))
return qry.order_by(DagRun.execution_date).all()
#airflow_api_blueprint.route('/dags', methods=['GET'])
def dags_index():
dagbag = DagBag()
dags = []
for dag_id in dagbag.dags:
payload = {
'dag_id': dag_id,
'full_path': None,
'is_active': False,
'last_execution': None,
}
dag = dagbag.get_dag(dag_id)
if dag:
payload['full_path'] = dag.full_filepath
payload['is_active'] = (not dag.is_paused)
payload['last_execution'] = str(dag.latest_execution_date)
if request.args.get('dag_id') is not None:
if request.args.get('dag_id') not in payload['dag_id']:
continue
dags.append(payload)
return ApiResponse.success({'dags': dags})
#airflow_api_blueprint.route('/dag_runs', methods=['GET'])
def get_dag_runs():
dag_runs = []
session = settings.Session()
query = session.query(DagRun)
if request.args.get('state') is not None:
query = query.filter(DagRun.state == request.args.get('state'))
if request.args.get('external_trigger') is not None:
# query = query.filter(DagRun.external_trigger == (request.args.get('external_trigger') is True))
query = query.filter(DagRun.external_trigger == (request.args.get('external_trigger') in ['true', 'True']))
if request.args.get('prefix') is not None:
query = query.filter(DagRun.run_id.ilike('{}%'.format(request.args.get('prefix'))))
if request.args.get('dag_id') is not None:
query = query.filter(DagRun.dag_id.ilike('{}%'.format(request.args.get('dag_id'))))
runs = query.order_by(DagRun.execution_date).all()
for run in runs:
dag_runs.append(format_dag_run(run))
session.close()
return ApiResponse.success({'dag_runs': dag_runs})
#csrf.exempt
#airflow_api_blueprint.route('/dag_runs', methods=['POST'])
def create_dag_run():
# decode input
data = request.get_json(force=True)
# ensure there is a dag id
if 'dag_id' not in data or data['dag_id'] is None:
return ApiResponse.bad_request('Must specify the dag id to create dag runs for')
dag_id = data['dag_id']
limit = 500
partial = False
if 'limit' in data and data['limit'] is not None:
try:
limit = int(data['limit'])
if limit <= 0:
return ApiResponse.bad_request('Limit must be a number greater than 0')
if limit > 500:
return ApiResponse.bad_request('Limit cannot exceed 500')
except ValueError:
return ApiResponse.bad_request('Limit must be an integer')
if 'partial' in data and data['partial'] in ['true', 'True', True]:
partial = True
# ensure there is run data
start_date = datetime.now()
end_date = datetime.now()
if 'start_date' in data and data['start_date'] is not None:
try:
start_date = datetime.strptime(data['start_date'], '%Y-%m-%dT%H:%M:%S')
except ValueError:
error = '\'start_date\' has invalid format \'{}\', Ex format: YYYY-MM-DDThh:mm:ss'
return ApiResponse.bad_request(error.format(data['start_date']))
if 'end_date' in data and data['end_date'] is not None:
try:
end_date = datetime.strptime(data['end_date'], '%Y-%m-%dT%H:%M:%S')
except ValueError:
error = '\'end_date\' has invalid format \'{}\', Ex format: YYYY-MM-DDThh:mm:ss'
return ApiResponse.bad_request(error.format(data['end_date']))
# determine run_id prefix
prefix = 'manual_{}'.format(int(time.time()))
if 'prefix' in data and data['prefix'] is not None:
prefix = data['prefix']
if 'backfill' in prefix:
return ApiResponse.bad_request('Prefix cannot contain \'backfill\', Airflow will ignore dag runs using it')
# ensure prefix doesn't have an underscore appended
if prefix[:-1:] == "_":
prefix = prefix[:-1]
conf = None
if 'conf' in data and data['conf'] is not None:
if isinstance(data['conf'], six.string_types):
conf = data['conf']
else:
try:
conf = json.dumps(data['conf'])
except Exception:
return ApiResponse.bad_request('Could not encode specified conf JSON')
try:
session = settings.Session()
dagbag = DagBag('dags')
if dag_id not in dagbag.dags:
return ApiResponse.bad_request("Dag id {} not found".format(dag_id))
dag = dagbag.get_dag(dag_id)
# ensure run data has all required attributes and that everything is valid, returns transformed data
runs = utils_date_range(start_date=start_date, end_date=end_date, delta=dag._schedule_interval)
if len(runs) > limit and partial is False:
error = '{} dag runs would be created, which exceeds the limit of {}.' \
' Reduce start/end date to reduce the dag run count'
return ApiResponse.bad_request(error.format(len(runs), limit))
payloads = []
for exec_date in runs:
run_id = '{}_{}'.format(prefix, exec_date.isoformat())
if find_dag_runs(session, dag_id, run_id, exec_date):
continue
payloads.append({
'run_id': run_id,
'execution_date': exec_date,
'conf': conf
})
results = []
for index, run in enumerate(payloads):
if len(results) >= limit:
break
dag.create_dagrun(
run_id=run['run_id'],
execution_date=run['execution_date'],
state=State.RUNNING,
conf=conf,
external_trigger=True
)
results.append(run['run_id'])
session.close()
except ApiInputException as e:
return ApiResponse.bad_request(str(e))
except ValueError as e:
return ApiResponse.server_error(str(e))
except AirflowException as e:
return ApiResponse.server_error(str(e))
except Exception as e:
return ApiResponse.server_error(str(e))
return ApiResponse.success({'dag_run_ids': results})
#airflow_api_blueprint.route('/dag_runs/<dag_run_id>', methods=['GET'])
def get_dag_run(dag_run_id):
session = settings.Session()
runs = DagRun.find(run_id=dag_run_id, session=session)
if len(runs) == 0:
return ApiResponse.not_found('Dag run not found')
dag_run = runs[0]
session.close()
return ApiResponse.success({'dag_run': format_dag_run(dag_run)})
For programmatic auth to Composer's Airflow webserver, you will have to authenticate through IAP.
https://cloud.google.com/composer/docs/how-to/using/triggering-with-gcf contains a js example of this.
https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/iap/make_iap_request.py is a Python example, but note that it is missing a timeout on the final IAP request (https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1812).
Hi I use python for my facebook app and want to restrict my app from gaining permissions to the users friends. How can I accomplish this? Some code is conf.py
# Facebook Application ID and Secret.
FACEBOOK_APP_ID = '103297833078853'
FACEBOOK_APP_SECRET = 'd1f7a3dfeb650b6826456a5660134f58'
# Canvas Page name.
FACEBOOK_CANVAS_NAME = 'cyberfaze'
# A random token for use with the Real-time API.
FACEBOOK_REALTIME_VERIFY_TOKEN = 'RANDOM TOKEN'
# The external URL this application is available at where the Real-time API will
# send it's pings.
EXTERNAL_HREF = 'http://cyberfaze.appspot.com/'
# Facebook User IDs of admins. The poor mans admin system.
ADMIN_USER_IDS = ['5526183']
Here is main.py
#!/usr/bin/env python
# coding: utf-8
# Copyright 2011 Facebook, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
# dummy config to enable registering django template filters
os.environ[u'DJANGO_SETTINGS_MODULE'] = u'conf'
from google.appengine.dist import use_library
use_library('django', '1.2')
from django.template.defaultfilters import register
from django.utils import simplejson as json
from functools import wraps
from google.appengine.api import urlfetch, taskqueue
from google.appengine.ext import db, webapp
from google.appengine.ext.webapp import util, template
from google.appengine.runtime import DeadlineExceededError
from random import randrange
from uuid import uuid4
import Cookie
import base64
import cgi
import conf
import datetime
import hashlib
import hmac
import logging
import time
import traceback
import urllib
def htmlescape(text):
"""Escape text for use as HTML"""
return cgi.escape(
text, True).replace("'", ''').encode('ascii', 'xmlcharrefreplace')
#register.filter(name=u'get_name')
def get_name(dic, index):
"""Django template filter to render name"""
return dic[index].name
#register.filter(name=u'get_picture')
def get_picture(dic, index):
"""Django template filter to render picture"""
return dic[index].picture
def select_random(lst, limit):
"""Select a limited set of random non Falsy values from a list"""
final = []
size = len(lst)
while limit and size:
index = randrange(min(limit, size))
size = size - 1
elem = lst[index]
lst[index] = lst[size]
if elem:
limit = limit - 1
final.append(elem)
return final
_USER_FIELDS = u'name,email,picture,friends'
class User(db.Model):
user_id = db.StringProperty(required=True)
access_token = db.StringProperty(required=True)
name = db.StringProperty(required=True)
picture = db.StringProperty(required=True)
email = db.StringProperty()
friends = db.StringListProperty()
dirty = db.BooleanProperty()
def refresh_data(self):
"""Refresh this user's data using the Facebook Graph API"""
me = Facebook().api(u'/me',
{u'fields': _USER_FIELDS, u'access_token': self.access_token})
self.dirty = False
self.name = me[u'name']
self.email = me.get(u'email')
self.picture = me[u'picture']
self.friends = [user[u'id'] for user in me[u'friends'][u'data']]
return self.put()
class Run(db.Model):
user_id = db.StringProperty(required=True)
location = db.StringProperty(required=True)
distance = db.FloatProperty(required=True)
date = db.DateProperty(required=True)
#staticmethod
def find_by_user_ids(user_ids, limit=50):
if user_ids:
return Run.gql(u'WHERE user_id IN :1', user_ids).fetch(limit)
else:
return []
#property
def pretty_distance(self):
return u'%.2f' % self.distance
class RunException(Exception):
pass
class FacebookApiError(Exception):
def __init__(self, result):
self.result = result
def __str__(self):
return self.__class__.__name__ + ': ' + json.dumps(self.result)
class Facebook(object):
"""Wraps the Facebook specific logic"""
def __init__(self, app_id=conf.FACEBOOK_APP_ID,
app_secret=conf.FACEBOOK_APP_SECRET):
self.app_id = app_id
self.app_secret = app_secret
self.user_id = None
self.access_token = None
self.signed_request = {}
def api(self, path, params=None, method=u'GET', domain=u'graph'):
"""Make API calls"""
if not params:
params = {}
params[u'method'] = method
if u'access_token' not in params and self.access_token:
params[u'access_token'] = self.access_token
result = json.loads(urlfetch.fetch(
url=u'https://' + domain + u'.facebook.com' + path,
payload=urllib.urlencode(params),
method=urlfetch.POST,
headers={
u'Content-Type': u'application/x-www-form-urlencoded'})
.content)
if isinstance(result, dict) and u'error' in result:
raise FacebookApiError(result)
return result
def load_signed_request(self, signed_request):
"""Load the user state from a signed_request value"""
try:
sig, payload = signed_request.split(u'.', 1)
sig = self.base64_url_decode(sig)
data = json.loads(self.base64_url_decode(payload))
expected_sig = hmac.new(
self.app_secret, msg=payload, digestmod=hashlib.sha256).digest()
# allow the signed_request to function for upto 1 day
if sig == expected_sig and \
data[u'issued_at'] > (time.time() - 86400):
self.signed_request = data
self.user_id = data.get(u'user_id')
self.access_token = data.get(u'oauth_token')
except ValueError, ex:
pass # ignore if can't split on dot
#property
def user_cookie(self):
"""Generate a signed_request value based on current state"""
if not self.user_id:
return
payload = self.base64_url_encode(json.dumps({
u'user_id': self.user_id,
u'issued_at': str(int(time.time())),
}))
sig = self.base64_url_encode(hmac.new(
self.app_secret, msg=payload, digestmod=hashlib.sha256).digest())
return sig + '.' + payload
#staticmethod
def base64_url_decode(data):
data = data.encode(u'ascii')
data += '=' * (4 - (len(data) % 4))
return base64.urlsafe_b64decode(data)
#staticmethod
def base64_url_encode(data):
return base64.urlsafe_b64encode(data).rstrip('=')
class CsrfException(Exception):
pass
class BaseHandler(webapp.RequestHandler):
facebook = None
user = None
csrf_protect = True
def initialize(self, request, response):
"""General initialization for every request"""
super(BaseHandler, self).initialize(request, response)
try:
self.init_facebook()
self.init_csrf()
self.response.headers[u'P3P'] = u'CP=HONK' # iframe cookies in IE
except Exception, ex:
self.log_exception(ex)
raise
def handle_exception(self, ex, debug_mode):
"""Invoked for unhandled exceptions by webapp"""
self.log_exception(ex)
self.render(u'error',
trace=traceback.format_exc(), debug_mode=debug_mode)
def log_exception(self, ex):
"""Internal logging handler to reduce some App Engine noise in errors"""
msg = ((str(ex) or ex.__class__.__name__) +
u': \n' + traceback.format_exc())
if isinstance(ex, urlfetch.DownloadError) or \
isinstance(ex, DeadlineExceededError) or \
isinstance(ex, CsrfException) or \
isinstance(ex, taskqueue.TransientError):
logging.warn(msg)
else:
logging.error(msg)
def set_cookie(self, name, value, expires=None):
"""Set a cookie"""
if value is None:
value = 'deleted'
expires = datetime.timedelta(minutes=-50000)
jar = Cookie.SimpleCookie()
jar[name] = value
jar[name]['path'] = u'/'
if expires:
if isinstance(expires, datetime.timedelta):
expires = datetime.datetime.now() + expires
if isinstance(expires, datetime.datetime):
expires = expires.strftime('%a, %d %b %Y %H:%M:%S')
jar[name]['expires'] = expires
self.response.headers.add_header(*jar.output().split(u': ', 1))
def render(self, name, **data):
"""Render a template"""
if not data:
data = {}
data[u'js_conf'] = json.dumps({
u'appId': conf.FACEBOOK_APP_ID,
u'canvasName': conf.FACEBOOK_CANVAS_NAME,
u'userIdOnServer': self.user.user_id if self.user else None,
})
data[u'logged_in_user'] = self.user
data[u'message'] = self.get_message()
data[u'csrf_token'] = self.csrf_token
data[u'canvas_name'] = conf.FACEBOOK_CANVAS_NAME
self.response.out.write(template.render(
os.path.join(
os.path.dirname(__file__), 'templates', name + '.html'),
data))
def init_facebook(self):
"""Sets up the request specific Facebook and User instance"""
facebook = Facebook()
user = None
# initial facebook request comes in as a POST with a signed_request
if u'signed_request' in self.request.POST:
facebook.load_signed_request(self.request.get('signed_request'))
# we reset the method to GET because a request from facebook with a
# signed_request uses POST for security reasons, despite it
# actually being a GET. in webapp causes loss of request.POST data.
self.request.method = u'GET'
self.set_cookie(
'u', facebook.user_cookie, datetime.timedelta(minutes=1440))
elif 'u' in self.request.cookies:
facebook.load_signed_request(self.request.cookies.get('u'))
# try to load or create a user object
if facebook.user_id:
user = User.get_by_key_name(facebook.user_id)
if user:
# update stored access_token
if facebook.access_token and \
facebook.access_token != user.access_token:
user.access_token = facebook.access_token
user.put()
# refresh data if we failed in doing so after a realtime ping
if user.dirty:
user.refresh_data()
# restore stored access_token if necessary
if not facebook.access_token:
facebook.access_token = user.access_token
if not user and facebook.access_token:
me = facebook.api(u'/me', {u'fields': _USER_FIELDS})
try:
friends = [user[u'id'] for user in me[u'friends'][u'data']]
user = User(key_name=facebook.user_id,
user_id=facebook.user_id, friends=friends,
access_token=facebook.access_token, name=me[u'name'],
email=me.get(u'email'), picture=me[u'picture'])
user.put()
except KeyError, ex:
pass # ignore if can't get the minimum fields
self.facebook = facebook
self.user = user
def init_csrf(self):
"""Issue and handle CSRF token as necessary"""
self.csrf_token = self.request.cookies.get(u'c')
if not self.csrf_token:
self.csrf_token = str(uuid4())[:8]
self.set_cookie('c', self.csrf_token)
if self.request.method == u'POST' and self.csrf_protect and \
self.csrf_token != self.request.POST.get(u'_csrf_token'):
raise CsrfException(u'Missing or invalid CSRF token.')
def set_message(self, **obj):
"""Simple message support"""
self.set_cookie('m', base64.b64encode(json.dumps(obj)) if obj else None)
def get_message(self):
"""Get and clear the current message"""
message = self.request.cookies.get(u'm')
if message:
self.set_message() # clear the current cookie
return json.loads(base64.b64decode(message))
def user_required(fn):
"""Decorator to ensure a user is present"""
#wraps(fn)
def wrapper(*args, **kwargs):
handler = args[0]
if handler.user:
return fn(*args, **kwargs)
handler.redirect(u'/')
return wrapper
class RecentRunsHandler(BaseHandler):
"""Show recent runs for the user and friends"""
def get(self):
if self.user:
friends = {}
for friend in select_random(
User.get_by_key_name(self.user.friends), 30):
friends[friend.user_id] = friend
self.render(u'runs',
friends=friends,
user_recent_runs=Run.find_by_user_ids(
[self.user.user_id], limit=5),
friends_runs=Run.find_by_user_ids(friends.keys()),
)
else:
self.render(u'welcome')
class UserRunsHandler(BaseHandler):
"""Show a specific user's runs, ensure friendship with the logged in user"""
#user_required
def get(self, user_id):
if self.user.friends.count(user_id) or self.user.user_id == user_id:
user = User.get_by_key_name(user_id)
if not user:
self.set_message(type=u'error',
content=u'That user does not use Run with Friends.')
self.redirect(u'/')
return
self.render(u'user',
user=user,
runs=Run.find_by_user_ids([user_id]),
)
else:
self.set_message(type=u'error',
content=u'You are not allowed to see that.')
self.redirect(u'/')
class RunHandler(BaseHandler):
"""Add a run"""
#user_required
def post(self):
try:
location = self.request.POST[u'location'].strip()
if not location:
raise RunException(u'Please specify a location.')
distance = float(self.request.POST[u'distance'].strip())
if distance < 0:
raise RunException(u'Invalid distance.')
date_year = int(self.request.POST[u'date_year'].strip())
date_month = int(self.request.POST[u'date_month'].strip())
date_day = int(self.request.POST[u'date_day'].strip())
if date_year < 0 or date_month < 0 or date_day < 0:
raise RunException(u'Invalid date.')
date = datetime.date(date_year, date_month, date_day)
run = Run(
user_id=self.user.user_id,
location=location,
distance=distance,
date=date,
)
run.put()
title = run.pretty_distance + u' miles #' + location
publish = u'<a onclick=\'publishRun(' + \
json.dumps(htmlescape(title)) + u')\'>Post to facebook.</a>'
self.set_message(type=u'success',
content=u'Added your run. ' + publish)
except RunException, e:
self.set_message(type=u'error', content=unicode(e))
except KeyError:
self.set_message(type=u'error',
content=u'Please specify location, distance & date.')
except ValueError:
self.set_message(type=u'error',
content=u'Please specify a valid distance & date.')
except Exception, e:
self.set_message(type=u'error',
content=u'Unknown error occured. (' + unicode(e) + u')')
self.redirect(u'/')
class RealtimeHandler(BaseHandler):
"""Handles Facebook Real-time API interactions"""
csrf_protect = False
def get(self):
if (self.request.GET.get(u'setup') == u'1' and
self.user and conf.ADMIN_USER_IDS.count(self.user.user_id)):
self.setup_subscription()
self.set_message(type=u'success',
content=u'Successfully setup Real-time subscription.')
elif (self.request.GET.get(u'hub.mode') == u'subscribe' and
self.request.GET.get(u'hub.verify_token') ==
conf.FACEBOOK_REALTIME_VERIFY_TOKEN):
self.response.out.write(self.request.GET.get(u'hub.challenge'))
logging.info(
u'Successful Real-time subscription confirmation ping.')
return
else:
self.set_message(type=u'error',
content=u'You are not allowed to do that.')
self.redirect(u'/')
def post(self):
body = self.request.body
if self.request.headers[u'X-Hub-Signature'] != (u'sha1=' + hmac.new(
self.facebook.app_secret,
msg=body,
digestmod=hashlib.sha1).hexdigest()):
logging.error(
u'Real-time signature check failed: ' + unicode(self.request))
return
data = json.loads(body)
if data[u'object'] == u'user':
for entry in data[u'entry']:
taskqueue.add(url=u'/task/refresh-user/' + entry[u'id'])
logging.info('Added task to queue to refresh user data.')
else:
logging.warn(u'Unhandled Real-time ping: ' + body)
def setup_subscription(self):
path = u'/' + conf.FACEBOOK_APP_ID + u'/subscriptions'
params = {
u'access_token': conf.FACEBOOK_APP_ID + u'|' +
conf.FACEBOOK_APP_SECRET,
u'object': u'user',
u'fields': _USER_FIELDS,
u'callback_url': conf.EXTERNAL_HREF + u'realtime',
u'verify_token': conf.FACEBOOK_REALTIME_VERIFY_TOKEN,
}
response = self.facebook.api(path, params, u'POST')
logging.info(u'Real-time setup API call response: ' + unicode(response))
class RefreshUserHandler(BaseHandler):
"""Used as an App Engine Task to refresh a single user's data if possible"""
csrf_protect = False
def post(self, user_id):
logging.info('Refreshing user data for ' + user_id)
user = User.get_by_key_name(user_id)
if not user:
return
try:
user.refresh_data()
except FacebookApiError:
user.dirty = True
user.put()
def main():
routes = [
(r'/', RecentRunsHandler),
(r'/user/(.*)', UserRunsHandler),
(r'/run', RunHandler),
(r'/realtime', RealtimeHandler),
(r'/task/refresh-user/(.*)', RefreshUserHandler),
]
application = webapp.WSGIApplication(routes,
debug=os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'))
util.run_wsgi_app(application)
if __name__ == u'__main__':
main()
I think, and I'm still working with this example project myself, that if you modify this line AND strip out any "friend" references then it should work:
_USER_FIELDS = u'name,email,picture,friends'
becomes
_USER_FIELDS = u'name,email,picture'
It will be a lot of work to strip out all the "friend" references, but from what I can tell that's the only string that the app uses to request user information from the graph api.