I looked at all the responses on post AWS S3 - How to fix 'The request signature we calculated does not match the signature' error? but I still have issues with my Python API sign 4 error. It still returns me with error "The request signature we calculated does not match with the signature you provided".
Here are the details:
While I am able to sign successfully: https://abcdef.mycompany.com/issues/P1230 but can't get https://abcdef.mycompany.com/issues?q=assignedFolder%3A(abc123) signed, and keep getting error
import json, hashlib, hmac, datetime, uuid, sys, urllib3, boto3, time, urllib.parse, os, requests
def get_extract(docid):
method = 'GET'
service = 'myservice'
host = 'abcdef.mycompany.com'
region = 'us-east-1'
endpoint = 'https://abcdef.mycompany.com'
request_parameters = '?q=assignedFolder%3A(abc123)' ## docid is abc123: hard-coding for example purpose
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
access_key = 'mykey123'
secret_key = 'mykey123'
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
canonical_uri = '/issues'
canonical_querystring = request_parameters
canonical_headers = 'host:' + host + '\n' + 'x-amz-date:' + amzdate + '\n'
signed_headers = 'host;x-amz-date'
payload_hash = hashlib.sha256(('').encode('utf-8')).hexdigest()
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
headers = {'x-amz-date':amzdate,'Host': host,'Authorization':authorization_header}
request_url = endpoint + canonical_uri + canonical_querystring
http = urllib3.PoolManager()
res = http.request('GET',request_url,headers=headers)
print(len(res.data.decode('utf-8')))
print(res.data.decode('utf-8'))
Error:
<InvalidSignatureException>
<Message>The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.</Message>
</InvalidSignatureException>
Need some help with this request, have been struggling to get this working for last 4-5 days. Any help is appreciated
I recently upgraded to Django 2.2.2 and Python 3.6.8, and my filter_horizontal feature in Django admin has disappeared.
I tried viewing my admin in Chrome incognito mode, as some answers suggest, and I also tried changing verbose_name strings to unicode. However, none of these worked.
Here is an example of a model for which I am attempting to show filter_horizontal. This worked on my app prior to the upgrades.
admin.py
class ResearchAdmin(admin.ModelAdmin):
filter_horizontal = ('criteria', 'state')
def save_model(self, request, obj, form, change):
obj.save()
# Update cache for tasks containing the saved goal
tasks = Task.objects.filter(research__id=obj.pk).values_list('pk', flat=True)
for t in tasks:
cache.delete_many(['task%s' % t, 'task%s_research' % t])
models.py
"""
Clinical Research model
"""
def __str__(self):
return "%s" % (self.name)
name = models.CharField(max_length=50, null=True, blank=True)
type = models.CharField(max_length=50, null=True, blank=True)
cta = models.CharField(max_length=50, null=True, blank=True)
description = models.TextField(null=True, blank=True)
picture = models.ImageField(upload_to='images/%Y/%m/%d', null=True, blank=True, help_text="Upload portrait image for modal study description")
layout = models.CharField(max_length=1, choices=LAYOUT_TYPE, null=False, blank=False, default='r')
criteria = models.ManyToManyField('Answer', blank=True, db_index=True, help_text="Answers required for qualified patients")
required_contact = models.ManyToManyField('ContactField', blank=True, db_index=True, help_text="Contact info for patient to enter")
email = models.EmailField(null=True, blank=True, help_text="Sponsor email for notifying of screened patients")
link = models.URLField(null=True, blank=True)
state = models.ManyToManyField('State', blank=True, help_text="Qualifying states")
lat = models.CharField(max_length=60, null=True, blank=True)
lng = models.CharField(max_length=60, null=True, blank=True)
distance = models.PositiveIntegerField(null=True, blank=True, help_text="Maximum distance from user in miles to show")
class Meta:
verbose_name = u"Research"
verbose_name_plural = u"Research"
There are no error messages, but the filter_horizontal frontend doesn't show up in admin for the criteria and state fields.
##collectstatic##
As #iain-shelvington suggested, this issue may be due to some kind of interference with the cached front-end code required to display the filter_horizontal format. I have tried running in Google Incognito mode, clearing catch, and running collectstatic --clear, and none of these work. Moreover, there aren't any differences between the admin static files pre- and post- upgrade.
#SylvainBiehler pointed out that django_gulp may be overriding collectstatic. I disabled django_gulp and ran ./manage.py collectstatic --clear all admin files are now updated post Django upgrade.
##Comparing admin files pre- and post- Django upgrade##
I was able to spin up a version of my app pre-Django upgrade, and the filter_horizontal capability works in the older version. There are some differences in the construction of the Criteria field from Chrome console:
Old Version (works)
Select element prior to choices:
<select multiple="multiple" class="selectfilter" id="id_criteria" name="criteria">
Javascript after choices:
<script type="text/javascript">addEvent(window, "load", function(e) {SelectFilter.init("id_criteria", "criteria", 0, "https://xxxxxxx/static/admin/"); });</script>
New Version (broken)
Slightly different select element. No javascript after choices:
<select name="criteria" id="id_criteria" multiple class="selectfilter" data-field-name="criteria" data-is-stacked="0">
This seems to be causing the issue, but I have no idea how to fix it. Any ideas?
##Analyzing admin JS in console##
SelectBox.js loads in the console and declares var = SelectBox = {... which is never called.
SelectFilter.js also loads but the function is never called:
/*
SelectFilter2 - Turns a multiple-select box into a filter interface.
Requires core.js, SelectBox.js and addevent.js.
*/
(function($) {
function findForm(node) {
// returns the node of the form containing the given node
if (node.tagName.toLowerCase() != 'form') {
return findForm(node.parentNode);
}
return node;
}
window.SelectFilter = {
init: function(field_id, field_name, is_stacked, admin_static_prefix) {
if (field_id.match(/__prefix__/)){
// Don't intialize on empty forms.
return;
}
var from_box = document.getElementById(field_id);
from_box.id += '_from'; // change its ID
from_box.className = 'filtered';
var ps = from_box.parentNode.getElementsByTagName('p');
for (var i=0; i<ps.length; i++) {
if (ps[i].className.indexOf("info") != -1) {
// Remove <p class="info">, because it just gets in the way.
from_box.parentNode.removeChild(ps[i]);
} else if (ps[i].className.indexOf("help") != -1) {
// Move help text up to the top so it isn't below the select
// boxes or wrapped off on the side to the right of the add
// button:
from_box.parentNode.insertBefore(ps[i], from_box.parentNode.firstChild);
}
}
// <div class="selector"> or <div class="selector stacked">
var selector_div = quickElement('div', from_box.parentNode);
selector_div.className = is_stacked ? 'selector stacked' : 'selector';
// <div class="selector-available">
var selector_available = quickElement('div', selector_div, '');
selector_available.className = 'selector-available';
var title_available = quickElement('h2', selector_available, interpolate(gettext('Available %s') + ' ', [field_name]));
quickElement('img', title_available, '', 'src', admin_static_prefix + 'img/icon-unknown.gif', 'width', '10', 'height', '10', 'class', 'help help-tooltip', 'title', interpolate(gettext('This is the list of available %s. You may choose some by selecting them in the box below and then clicking the "Choose" arrow between the two boxes.'), [field_name]));
var filter_p = quickElement('p', selector_available, '', 'id', field_id + '_filter');
filter_p.className = 'selector-filter';
var search_filter_label = quickElement('label', filter_p, '', 'for', field_id + "_input");
var search_selector_img = quickElement('img', search_filter_label, '', 'src', admin_static_prefix + 'img/selector-search.gif', 'class', 'help-tooltip', 'alt', '', 'title', interpolate(gettext("Type into this box to filter down the list of available %s."), [field_name]));
filter_p.appendChild(document.createTextNode(' '));
var filter_input = quickElement('input', filter_p, '', 'type', 'text', 'placeholder', gettext("Filter"));
filter_input.id = field_id + '_input';
selector_available.appendChild(from_box);
var choose_all = quickElement('a', selector_available, gettext('Choose all'), 'title', interpolate(gettext('Click to choose all %s at once.'), [field_name]), 'href', 'javascript: (function(){ SelectBox.move_all("' + field_id + '_from", "' + field_id + '_to"); SelectFilter.refresh_icons("' + field_id + '");})()', 'id', field_id + '_add_all_link');
choose_all.className = 'selector-chooseall';
// <ul class="selector-chooser">
var selector_chooser = quickElement('ul', selector_div, '');
selector_chooser.className = 'selector-chooser';
var add_link = quickElement('a', quickElement('li', selector_chooser, ''), gettext('Choose'), 'title', gettext('Choose'), 'href', 'javascript: (function(){ SelectBox.move("' + field_id + '_from","' + field_id + '_to"); SelectFilter.refresh_icons("' + field_id + '");})()', 'id', field_id + '_add_link');
add_link.className = 'selector-add';
var remove_link = quickElement('a', quickElement('li', selector_chooser, ''), gettext('Remove'), 'title', gettext('Remove'), 'href', 'javascript: (function(){ SelectBox.move("' + field_id + '_to","' + field_id + '_from"); SelectFilter.refresh_icons("' + field_id + '");})()', 'id', field_id + '_remove_link');
remove_link.className = 'selector-remove';
// <div class="selector-chosen">
var selector_chosen = quickElement('div', selector_div, '');
selector_chosen.className = 'selector-chosen';
var title_chosen = quickElement('h2', selector_chosen, interpolate(gettext('Chosen %s') + ' ', [field_name]));
quickElement('img', title_chosen, '', 'src', admin_static_prefix + 'img/icon-unknown.gif', 'width', '10', 'height', '10', 'class', 'help help-tooltip', 'title', interpolate(gettext('This is the list of chosen %s. You may remove some by selecting them in the box below and then clicking the "Remove" arrow between the two boxes.'), [field_name]));
var to_box = quickElement('select', selector_chosen, '', 'id', field_id + '_to', 'multiple', 'multiple', 'size', from_box.size, 'name', from_box.getAttribute('name'));
to_box.className = 'filtered';
var clear_all = quickElement('a', selector_chosen, gettext('Remove all'), 'title', interpolate(gettext('Click to remove all chosen %s at once.'), [field_name]), 'href', 'javascript: (function() { SelectBox.move_all("' + field_id + '_to", "' + field_id + '_from"); SelectFilter.refresh_icons("' + field_id + '");})()', 'id', field_id + '_remove_all_link');
clear_all.className = 'selector-clearall';
from_box.setAttribute('name', from_box.getAttribute('name') + '_old');
// Set up the JavaScript event handlers for the select box filter interface
addEvent(filter_input, 'keyup', function(e) { SelectFilter.filter_key_up(e, field_id); });
addEvent(filter_input, 'keydown', function(e) { SelectFilter.filter_key_down(e, field_id); });
addEvent(from_box, 'change', function(e) { SelectFilter.refresh_icons(field_id) });
addEvent(to_box, 'change', function(e) { SelectFilter.refresh_icons(field_id) });
addEvent(from_box, 'dblclick', function() { SelectBox.move(field_id + '_from', field_id + '_to'); SelectFilter.refresh_icons(field_id); });
addEvent(to_box, 'dblclick', function() { SelectBox.move(field_id + '_to', field_id + '_from'); SelectFilter.refresh_icons(field_id); });
addEvent(findForm(from_box), 'submit', function() { SelectBox.select_all(field_id + '_to'); });
SelectBox.init(field_id + '_from');
SelectBox.init(field_id + '_to');
// Move selected from_box options to to_box
SelectBox.move(field_id + '_from', field_id + '_to');
if (!is_stacked) {
// In horizontal mode, give the same height to the two boxes.
var j_from_box = $(from_box);
var j_to_box = $(to_box);
var resize_filters = function() { j_to_box.height($(filter_p).outerHeight() + j_from_box.outerHeight()); }
if (j_from_box.outerHeight() > 0) {
resize_filters(); // This fieldset is already open. Resize now.
} else {
// This fieldset is probably collapsed. Wait for its 'show' event.
j_to_box.closest('fieldset').one('show.fieldset', resize_filters);
}
}
// Initial icon refresh
SelectFilter.refresh_icons(field_id);
},
refresh_icons: function(field_id) {
var from = $('#' + field_id + '_from');
var to = $('#' + field_id + '_to');
var is_from_selected = from.find('option:selected').length > 0;
var is_to_selected = to.find('option:selected').length > 0;
// Active if at least one item is selected
$('#' + field_id + '_add_link').toggleClass('active', is_from_selected);
$('#' + field_id + '_remove_link').toggleClass('active', is_to_selected);
// Active if the corresponding box isn't empty
$('#' + field_id + '_add_all_link').toggleClass('active', from.find('option').length > 0);
$('#' + field_id + '_remove_all_link').toggleClass('active', to.find('option').length > 0);
},
filter_key_up: function(event, field_id) {
var from = document.getElementById(field_id + '_from');
// don't submit form if user pressed Enter
if ((event.which && event.which == 13) || (event.keyCode && event.keyCode == 13)) {
from.selectedIndex = 0;
SelectBox.move(field_id + '_from', field_id + '_to');
from.selectedIndex = 0;
return false;
}
var temp = from.selectedIndex;
SelectBox.filter(field_id + '_from', document.getElementById(field_id + '_input').value);
from.selectedIndex = temp;
return true;
},
filter_key_down: function(event, field_id) {
var from = document.getElementById(field_id + '_from');
// right arrow -- move across
if ((event.which && event.which == 39) || (event.keyCode && event.keyCode == 39)) {
var old_index = from.selectedIndex;
SelectBox.move(field_id + '_from', field_id + '_to');
from.selectedIndex = (old_index == from.length) ? from.length - 1 : old_index;
return false;
}
// down arrow -- wrap around
if ((event.which && event.which == 40) || (event.keyCode && event.keyCode == 40)) {
from.selectedIndex = (from.length == from.selectedIndex + 1) ? 0 : from.selectedIndex + 1;
}
// up arrow -- wrap around
if ((event.which && event.which == 38) || (event.keyCode && event.keyCode == 38)) {
from.selectedIndex = (from.selectedIndex == 0) ? from.length - 1 : from.selectedIndex - 1;
}
return true;
}
}
})(django.jQuery);
##INSTALLED APPS##
########## APP CONFIGURATION
INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS
DJANGO_APPS = (
'django_gulp',
# Default Django apps:
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Useful template tags:
# 'django.contrib.humanize',
# Python-Social-Auth
'social_django',
# Admin panel and documentation:
'django.contrib.admin',
'django.core.management',
# 'django.contrib.admindocs',
# for task queue
# For django-storages static store on AWS S3
'storages',
# Other django apps
'rest_framework',
)
# Apps specific for this project go here.
LOCAL_APPS = (
'base',
'myproject',
'users',
'campaigns',
)
I copied the static/admin files from Django.contrib using the following command in terminal:
cp -a /Users/username/.virtualenvs/rs/lib/python3.9/site-packages/django/contrib/admin/. /Users/username/Documents/myproject/static/
I then ran collectstatic to upload the files to S3, where they are stored for production. This actually worked, but seems a bit hacky. I must not be doing something right for these files to not update upon Django upgrade.
I'm trying to put tags after creating AMI from an instance using boto3 and getting an error:
botocore.exceptions.ParamValidationError: Parameter validation failed:
Unknown parameter in input: "TagSpecifications", must be one of:
BlockDeviceMappings, Description, DryRun, InstanceId, Name, NoReboot
Here is my code, can you please check what I'm doing wrong?
It works for snapshot but failing for image.
import xlrd
import boto3
import datetime
client = boto3.client('ec2')
# Give the location of the file
loc = ("/Users/user1/Documents/aws-python/aws-tag-test (1).xlsx")
# To open Workbook
wb = xlrd.open_workbook(loc)
sheet = wb.sheet_by_index(0)
# For row 0 and column 0
#print (sheet.cell_value(0, 0))
nowtime = datetime.datetime.now()
nowdate = (nowtime.strftime("%Y-%m-%d %H-%M"))
print (nowdate)
#print (nowtime)
server_ids = []
instancename =[]
for i in range (1,sheet.nrows):
server_ids.append(sheet.cell_value(i,1))
instancename.append(sheet.cell_value(i,0))
#print (sheet.cell_value(i,1))
# excel closed
for i in range (len(server_ids)):
print(server_ids[i], instancename[i])
response = client.create_image(
Description = 'ami ' + instancename[i] + ' ' + str(nowdate),
InstanceId = server_ids[i],
Name = 'ami ' + instancename[i] + ' ' + str(nowdate),
NoReboot = True,
DryRun=False,
TagSpecifications=[
{
'ResourceType': 'image',
'Tags': [
{
'Key': 'Name',
'Value': 'ami-' + instancename[i] + '-' + str(nowdate)
},
{
'Key': 'date',
'Value': datetime.datetime.now().strftime("%Y-%m-%d")
}
]
},
]
)
#)
print(response)
Really appreciate your help.
Yes, it is now available. Not sure when, but it was definitely added sometime after the original comments.
Here is my sample code i want to show all my products in widget so i took as dictionary in for loop, but i am not getting the all products in the dictionary last product is overwrite the previous one.Can any one please help me where i did a mistake.Thank you in advance.
Given below is my code:
products_list = {"Givers":{"price": 140,"Quantity":100,"Discount":0,"Total_price":14000,"Paid_amount":14000},"New_Member":{"price": 1200,"Quantity":10,"Discount":0,"Total_price":12000,"Paid_amount":12000},"YearMembers_Ribbons":{"price": 110,"Quantity":0,"Discount":0,"Total_price":1100,"Paid_amount":1100}}
for key,value in products_list.items():
self.widget_orders_info = QtGui.QWidget()
self.widget_orders_info.setWindowFlags(QtCore.Qt.FramelessWindowHint)
self.widget_orders_info.setStyleSheet("QWidget {background-color: blue;color: white;}")
self.order_info_vbox = QtGui.QVBoxLayout(self.widget_orders_info)
self.title_hbox = QtGui.QHBoxLayout()
self.orderdetail_label = QtGui.QLabel("Order_Details(22HRS)",alignment =QtCore.Qt.AlignCenter,objectName="small")
self.close_pushbtn = QtGui.QPushButton("Close")
self.title_hbox.addWidget(self.orderdetail_label)
self.title_hbox.addStretch()
self.title_hbox.addWidget(self.close_pushbtn)
self.close_pushbtn.clicked.connect(self.widget_orders_info.close)
self.order_info_vbox.addLayout(self.title_hbox)
self.line = QtGui.QFrame(frameShape=QtGui.QFrame.HLine)
self.order_info_vbox.addWidget(self.line)
self.order_hbox = QtGui.QHBoxLayout()
keys = value.keys()
values = value.values()
self.line1 = QtGui.QFrame(frameShape=QtGui.QFrame.VLine)
self.order_hbox.addWidget(self.line1)
self.items_Vboxlay = QtGui.QVBoxLayout()
self.vbox2 = QtGui.QVBoxLayout()
self.item_label = QtGui.QLabel(str(key), objectName="small")
self.vbox2.addWidget(self.item_label)
self.hbox1 = QtGui.QHBoxLayout()
self.price = QtGui.QLabel(keys[1]+": " +str(values[1]))
self.qnty =QtGui.QLabel(keys[4]+": "+str(values[4]))
self.discount = QtGui.QLabel(keys[0]+ ": " +u'\u20B9 '+ str(values[0]))
self.totalprice =QtGui.QLabel(keys[2]+ ": "+u'\u20B9 '+ str(values[2]))
self.paid = QtGui.QLabel(keys[3] + ": "+u'\u20B9 '+ str(values[3]))
self.hbox1.addWidget(self.price)
self.hbox1.addWidget(self.qnty)
self.hbox1.addWidget(self.discount)
self.hbox1.addWidget(self.totalprice)
self.hbox1.addWidget(self.paid)
self.line4 = QtGui.QFrame();self.line4.setFrameShape(QtGui.QFrame.HLine)
self.vbox2.addLayout(self.hbox1)
self.vbox2.addLayout(self.h2box)
self.vbox2.addWidget(self.line1)
self.items_Vboxlay.addLayout(self.vbox2)
self.order_hbox.addLayout(self.items_Vboxlay)
self.order_info_vbox.addLayout(self.order_hbox)
self.add_hbox = QtGui.QHBoxLayout()
self.print_btn = QtGui.QPushButton("Print")
self.print_btn.clicked.connect(partial(self.print_function,type))
self.add_hbox.setAlignment(QtCore.Qt.AlignCenter)
self.add_hbox.addWidget(self.print_btn)
self.order_info_vbox.addLayout(self.add_hbox)
self.widget_orders_info.show()
self.widget_orders_info.resize(500,300)
Sorry, I have Python-3 andPyQt5. I think my example should help you.
import sys
from functools import partial
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
class Window(QMainWindow):
def __init__(self):
super().__init__()
products_list = {
"Givers":{
"price": 140,
"Quantity": 100,
"Discount": 0,
"Total_price":14000,
"Paid_amount":14000},
"New_Member":{
"price": 1200,
"Quantity": 10,
"Discount": 0,
"Total_price":12000,
"Paid_amount":12000},
"YearMembers_Ribbons":{
"price": 110,
"Quantity": 0,
"Discount": 0,
"Total_price":1100,
"Paid_amount":1100}
}
# *1 +++
self.widget_orders_info = QWidget()
self.widget_orders_info.setWindowFlags(Qt.FramelessWindowHint)
self.widget_orders_info.setStyleSheet("QWidget {background-color: blue;color: white;}")
self.order_info_vbox = QVBoxLayout(self.widget_orders_info)
self.title_hbox = QHBoxLayout()
self.orderdetail_label = QLabel("Order_Details(22HRS)",
alignment=Qt.AlignCenter,
objectName="small")
self.close_pushbtn = QPushButton("Close")
self.title_hbox.addWidget(self.orderdetail_label)
self.title_hbox.addStretch()
self.title_hbox.addWidget(self.close_pushbtn)
self.close_pushbtn.clicked.connect(self.widget_orders_info.close)
self.order_info_vbox.addLayout(self.title_hbox)
self.line = QFrame(frameShape=QFrame.HLine)
self.order_info_vbox.addWidget(self.line)
self.order_hbox = QHBoxLayout()
# *2 +++
self.line1 = QFrame(frameShape = QFrame.VLine)
self.order_hbox.addWidget(self.line1)
self.items_Vboxlay = QVBoxLayout()
self.vbox2 = QVBoxLayout()
for key, value in products_list.items():
# self.widget_orders_info = QWidget()
# ...
# transferred from the `for` loop, see `* 1 +++`
# ...
# self.order_hbox = QHBoxLayout()
# keys = value.keys()
# values = value.values()
keys = [ vk for vk in value.keys()]
values = [ vv for vv in value.values()]
# self.line1 = QFrame(frameShape = QFrame.VLine)
# ...
# transferred from the `for` loop, see `* 2 +++`
# ...
# self.vbox2 = QVBoxLayout()
self.item_label = QLabel(str(key), objectName="small")
self.vbox2.addWidget(self.item_label)
self.hbox1 = QHBoxLayout()
self.price = QLabel(keys[1] + ": " + str(values[1]))
self.qnty = QLabel(keys[4] + ": " + str(values[4]))
self.discount = QLabel(keys[0] + ": " + u'\u20B9 ' + str(values[0]))
self.totalprice = QLabel(keys[2] + ": " + u'\u20B9 ' + str(values[2]))
self.paid = QLabel(keys[3] + ": " + u'\u20B9 ' + str(values[3]))
self.hbox1.addWidget(self.price)
self.hbox1.addWidget(self.qnty)
self.hbox1.addWidget(self.discount)
self.hbox1.addWidget(self.totalprice)
self.hbox1.addWidget(self.paid)
self.line4 = QFrame()
self.line4.setFrameShape(QFrame.HLine)
self.vbox2.addLayout(self.hbox1)
# self.vbox2.addLayout(self.h2box) # ??????????????????????????
self.vbox2.addWidget(self.line1)
self.items_Vboxlay.addLayout(self.vbox2)
self.order_hbox.addLayout(self.items_Vboxlay)
self.order_info_vbox.addLayout(self.order_hbox)
self.add_hbox = QHBoxLayout()
self.print_btn = QPushButton("Print")
self.print_btn.clicked.connect(partial(self.print_function, type))
self.add_hbox.setAlignment(Qt.AlignCenter)
self.add_hbox.addWidget(self.print_btn)
self.order_info_vbox.addLayout(self.add_hbox)
self.widget_orders_info.show()
self.widget_orders_info.resize(500,300)
def print_function(self, t):
print("--- t ->", t)
if __name__ == '__main__':
import sys
app = QApplication(sys.argv)
w = Window()
# w.show()
sys.exit(app.exec_())
I am using Firehose with a lambda decorator to ingest vpc flow logs into Redshift.
(VPC Flow Logs -> Kinesis Data Stream -> Kinesis Firehose -> Lambda Decorator -> Redshift) The volume of traffic is high which causes the lambda to error out with task timed out when reingesting unprocessed records back into firehose. The lambda has the max timeout and 3GB of memory.
I believe the issue is related to lambda's 6mb payload size. Is there a way to batch or reduce the payload to ensure the function doesn't error out? Thanks in advance.
import base64
import json
import gzip
import StringIO
import boto3
import datetime
def transformLogEvent(log_event):
version = log_event['extractedFields']['version']
accountid = log_event['extractedFields']['account_id']
interfaceid = log_event['extractedFields']['interface_id']
srcaddr = log_event['extractedFields']['srcaddr']
dstaddr = log_event['extractedFields']['dstaddr']
srcport = log_event['extractedFields']['srcport']
dstport = log_event['extractedFields']['dstport']
protocol = log_event['extractedFields']['protocol']
packets = log_event['extractedFields']['packets']
bytes = log_event['extractedFields']['bytes']
starttime = datetime.datetime.fromtimestamp(int(log_event['extractedFields']['start'])).strftime('%Y-%m-%d %H:%M:%S')
endtime = datetime.datetime.fromtimestamp(int(log_event['extractedFields']['end'])).strftime('%Y-%m-%d %H:%M:%S')
action = log_event['extractedFields']['action']
logstatus = log_event['extractedFields']['log_status']
row = '"' + str(version) + '"' + "," + '"' + str(accountid) + '"' + "," + '"' + str(interfaceid) + '"' + "," + '"' + str(srcaddr) + '"' + "," + '"' + str(dstaddr) + '"' + "," + '"' + str(srcport) + '"' + "," + '"' + str(dstport) + '"' + "," + '"' + str(protocol) + '"' + "," + '"' + str(packets) + '"' + "," + '"' + str(bytes) + '"' + "," + '"' + str(starttime) + '"' + "," + '"' + str(endtime) + '"' + "," + '"' + str(action) + '"' + "," + '"' + str(logstatus) + '"' + "\n"
#print(row)
return row
def processRecords(records):
for r in records:
data = base64.b64decode(r['data'])
striodata = StringIO.StringIO(data)
try:
with gzip.GzipFile(fileobj=striodata, mode='r') as f:
data = json.loads(f.read())
except IOError:
# likely the data was re-ingested into firehose
pass
recId = r['recordId']
# re-ingested data into firehose
if type(data) == str:
yield {
'data': data,
'result': 'Ok',
'recordId': recId
}
elif data['messageType'] != 'DATA_MESSAGE':
yield {
'result': 'ProcessingFailed',
'recordId': recId
}
else:
data = ''.join([transformLogEvent(e) for e in data['logEvents']])
#print(data)
data = base64.b64encode(data)
yield {
'data': data,
'result': 'Ok',
'recordId': recId
}
def putRecords(streamName, records, client, attemptsMade, maxAttempts):
failedRecords = []
codes = []
errMsg = ''
try:
response = client.put_record_batch(DeliveryStreamName=streamName, Records=records)
except Exception as e:
failedRecords = records
errMsg = str(e)
# if there are no failedRecords (put_record_batch succeeded), iterate over the response to gather results
if not failedRecords and response['FailedPutCount'] > 0:
for idx, res in enumerate(response['RequestResponses']):
if not res['ErrorCode']:
continue
codes.append(res['ErrorCode'])
failedRecords.append(records[idx])
errMsg = 'Individual error codes: ' + ','.join(codes)
if len(failedRecords) > 0:
if attemptsMade + 1 < maxAttempts:
print('Some records failed while calling PutRecords, retrying. %s' % (errMsg))
putRecords(streamName, failedRecords, client, attemptsMade + 1, maxAttempts)
else:
raise RuntimeError('Could not put records after %s attempts. %s' % (str(maxAttempts), errMsg))
def handler(event, context):
streamARN = ''
region = ''
streamName = ''
records = list(processRecords(event['records']))
projectedSize = 0
recordsToReingest = []
for idx, rec in enumerate(records):
if rec['result'] == 'ProcessingFailed':
continue
projectedSize += len(rec['data']) + len(rec['recordId'])
# 4000000 instead of 6291456 to leave ample headroom for the stuff we didn't account for
if projectedSize > 4000000:
recordsToReingest.append({
'Data': rec['data']
})
records[idx]['result'] = 'Dropped'
del(records[idx]['data'])
if len(recordsToReingest) > 0:
client = boto3.client('firehose', region_name=region)
putRecords(streamName, recordsToReingest, client, attemptsMade=0, maxAttempts=20)
print('Reingested %d records out of %d' % (len(recordsToReingest), len(event['records'])))
else:
print('No records to be reingested')
return {"records": records}