Related
In Django i have the results i want but it returns separated data in same ids
How to groupby ids in a list of dicts?
Use pandas to fix this, but it doesn't work quite right. What I need is a simple dictionary list where if there is a repeated id, the information that is different is added as the value of that key and if there are several values, it is stored in a list. So as I show below in the result I want
i have this:
<
QuerySet[{
'id': 7086098,
'action_plan': None,
'comment': None
}, {
'id': 7105838,
'action_plan': 'foo',
'comment': None
}, {
'id': 7105838,
'action_plan': 'foos2',
'comment': None
}, {
'id': 7169339,
'action_plan': 'xxxxxx',
'comment': None
}, {
'id': 7169346,
'action_plan': 'report',
'comment': None
}, {
'id': 7169346,
'action_plan': 'zxczxczxczc',
'comment': None
}, {
'id': 7622793,
'action_plan': 'foofoo',
'comment': None
}, {
'id': 7622793,
'action_plan': 'role play',
'comment': None
}, {
'id': 7723661,
'action_plan': 'google',
'comment': 'chrome'
}, {
'id': 7723661,
'action_plan': 'netscape',
'comment': None
}, {
'id': 7723661,
'action_plan': 'urra',
'comment': 'firefox'
}, {
'id': 7723661,
'action_plan': 'sdasd',
'comment': None
}] >
i want to get this:
[{
'id': 7086098,
'action_plan': None,
'comment': None
}, {
'id': 7105838,
'action_plan': ['foo', 'foos2'],
'comment': [None, None]
}, {
'id': 7169339,
'action_plan': 'xxxxxx',
'comment': None
}, {
'id': 7169346,
'action_plan': ['report', 'zxczxczxczc'],
'comment': [None, None]
}, {
'id': 7622793,
'action_plan': ['foofoo', 'role play'],
'comment': [None, None]
}, {
'id': 7723661,
'action_plan': ['google', 'netscape', 'urra', 'sdasd'],
'comment': ['chrome', None, 'firefox', None]
}]
I am trying to automate the EMR cluster creation through boto3. Unfortunately, I'm getting the following warning:
The Auto Scaling policy for instance group ig-MI0ANZ0C3WNN in Amazon EMR cluster j-BS3Y2OAO65R6 (qidv2_historical_3.0.1) could not attach and failed at 2021-09-20 17:41 UTC.
I cannot figure out what is the issue is. This was adapted from an aws cli command which didn't raise any warnings or issues, but after transitioning to boto3, was getting this autoscaling policy warning
cluster_id = self.boto_client().run_job_flow(
Name=self.cluster_name,
LogUri='s3n://aws-logs',
JobFlowRole='EMR_EC2_DefaultRole',
ReleaseLabel=self.release_label,
Applications=[{'Name': 'Spark'},{'Name': 'Hive'},{'Name': 'Hadoop'},{'Name': 'Pig'},{'Name': 'Hue'},
{'Name': 'Zeppelin'},{'Name': 'Livy'},{'Name': 'JupyterHub'},{'Name': 'Tensorflow'}
],
AutoScalingRole='EMR_AutoScaling_DefaultRole',
BootstrapActions=[
{
'Name': 'Custom action',
'ScriptBootstrapAction': {
'Path': 's3://ml-data/emr-bootstrap_spk3.0.1.sh'
}
}
],
ServiceRole='EMR_DefaultRole',
ScaleDownBehavior='TERMINATE_AT_TASK_COMPLETION',
EbsRootVolumeSize=25,
Steps=[
{
'Name': 'Setup Debugging',
'ActionOnFailure': 'TERMINATE_CLUSTER',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': ['state-pusher-script']
}
},
{
'Name': 'Setup - Sync with S3',
'ActionOnFailure': 'CANCEL_AND_WAIT',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': ['aws', 's3', 'sync',
's3://ch-ml-data/',
'/mnt/src/']
}
},
{
'Name': 'Spark Application',
'ActionOnFailure': 'CANCEL_AND_WAIT',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': ['cd /mnt/src; bash spark_jobs/qid_pipeline_historical_run.sh']
}
}
],
Configurations=[
{
'Classification': 'zeppelin-env',
'Properties': {},
'Configurations': [
{
'Classification': 'export',
'Properties': {
'ZEPPELIN_PORT': '8890',
'HADOOP_CONF_DIR': '/etc/hadoop/conf',
'ZEPPELIN_LOG_DIR': '/var/log/zeppelin',
'ZEPPELIN_PID': '$ZEPPELIN_PID_DIR/zeppelin.pid',
'MASTER': 'yarn-client',
'SPARK_SUBMIT_OPTIONS': "$SPARK_SUBMIT_OPTIONS --conf '\''spark.executorEnv.PYTHONPATH=/usr/lib/spark/python/lib/py4j-src.zip:/usr/lib/spark/python/:<CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-src.zip'\'' --conf spark.yarn.isPython=true",
'PYSPARK_DRIVER_PYTHON': '/mnt/anaconda3/envs/question-identification-v2/bin/python',
'ZEPPELIN_NOTEBOOK_USER': 'user',
'CLASSPATH': ':/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar',
'ZEPPELIN_PID_DIR': '/var/run/zeppelin',
'PYSPARK_PYTHON': '/mnt/anaconda3/envs/question-identification-v2/bin/python',
'SPARK_HOME': '/usr/lib/spark',
'ZEPPELIN_NOTEBOOK_S3_BUCKET': 'ch-ml-data',
'ZEPPELIN_WAR_TEMPDIR': '/var/run/zeppelin/webapps',
'ZEPPELIN_CONF_DIR': '/etc/zeppelin/conf',
'ZEPPELIN_NOTEBOOK_STORAGE': 'org.apache.zeppelin.notebook.repo.S3NotebookRepo',
'ZEPPELIN_NOTEBOOK_DIR': '/var/lib/zeppelin/notebook',
'ZEPPELIN_ADDR': '0.0.0.0'
}
}
]
},
{
'Classification': 'hive-site',
'Properties': {
'hive.metastore.client.factory.class': 'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory'
}
},
{
'Classification': 'spark-hive-site',
'Properties': {
'hive.metastore.client.factory.class': 'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory'
}
}
],
Instances={
'Ec2KeyName': 'emr-temporary',
'KeepJobFlowAliveWhenNoSteps': False,
'TerminationProtected': False,
'Ec2SubnetId': 'subnet-063735e4fa63e3bac',
'AdditionalSlaveSecurityGroups': ["sg-012970517d0a88bae", "sg-01813cf2115b55874", "sg-04563fc7e8ed9e1ec", "sg-07ab30655981361ad"],
'ServiceAccessSecurityGroup': 'sg-00dd6e63d7004176d',
'EmrManagedSlaveSecurityGroup': 'sg-048b83d1a20550b43',
'EmrManagedMasterSecurityGroup': 'sg-017402b74e879aaa5',
'AdditionalMasterSecurityGroups': ["sg-012970517d0a88bae", "sg-01813cf2115b55874", "sg-04563fc7e8ed9e1ec", "sg-07ab30655981361ad"],
'InstanceGroups': [
{
'Name': 'Task',
'InstanceRole': 'TASK',
'InstanceType': 'i3.2xlarge',
'InstanceCount': 1
},
{
'Name': 'Master - 1',
'InstanceRole': 'MASTER',
'InstanceType': 'i3.2xlarge',
'InstanceCount': 1,
},
{
'Name': 'Core - 2',
'InstanceRole': 'CORE',
'InstanceType': 'i3.2xlarge',
'InstanceCount': 1,
'Market': 'SPOT',
'AutoScalingPolicy': {
'Constraints': {
'MinCapacity': 3,
'MaxCapacity': 100
},
'Rules': [
{
'Name': 'memory',
'Description': '',
'Action': {
'SimpleScalingPolicyConfiguration': {
'ScalingAdjustment': 10,
'CoolDown': 300,
'AdjustmentType': 'CHANGE_IN_CAPACITY'
}
},
'Trigger': {
'CloudWatchAlarmDefinition': {
'MetricName': 'YARNMemoryAvailablePercentage',
'ComparisonOperator': 'LESS_THAN',
'Statistic': 'AVERAGE',
'Period': 300,
'EvaluationPeriods': 2,
'Unit': 'PERCENT',
'Namespace': 'AWS/ElasticMapReduce',
'Threshold': 25,
'Dimensions': [
{
'Value': '${emr.clusterId}',
'Key': 'JobFlowId'
}
]
}
}
},
{
'Name': 'mem',
'Description': '',
'Action': {
'SimpleScalingPolicyConfiguration': {
'ScalingAdjustment': -5,
'CoolDown': 300,
'AdjustmentType': 'CHANGE_IN_CAPACITY'
}
},
'Trigger': {
'CloudWatchAlarmDefinition': {
'MetricName': 'YARNMemoryAvailablePercentage',
'ComparisonOperator': 'GREATER_THAN_OR_EQUAL',
'Statistic': 'AVERAGE',
'Period': 300,
'EvaluationPeriods': 18,
'Unit': 'PERCENT',
'Namespace': 'AWS/ElasticMapReduce',
'Threshold': 50,
'Dimensions': [
{
'Value': '${emr.clusterId}',
'Key': 'JobFlowId'
}
],
}
}
}
]
}
}
]
}
)
I have following boto3 script:
Import boto3
ChangeBatch={
'Changes': [
{
'Action': 'DELETE',
'ResourceRecordSet': {
'Name': 'test.example.com.',
'Region': 'us-west-1',
'SetIdentifier': 'test1',
'AliasTarget': {
'HostedZoneId': '**675',
'DNSName': 'testexample.example.com.',
'EvaluateTargetHealth': 'True'
},
'HealthCheckId': '**-**-**-675'
}
}
]
}
When I run the above code it does not delete anything. This is a latency based routing policy. Not sure what am I doing wrong I checked online and looked at aws documentation this is the suggested way to delete recordset.
I figured it out. It was missing type.
Import boto3
ChangeBatch={
'Changes': [
{
'Action': 'DELETE',
'ResourceRecordSet': {
'Name': 'test.example.com.',
'Region': 'us-west-1',
'Type': 'A'
'SetIdentifier': 'test1',
'AliasTarget': {
'HostedZoneId': '**675',
'DNSName': 'testexample.example.com.',
'EvaluateTargetHealth': 'True'
},
'HealthCheckId': '**-**-**-675'
}
}
]
}
I have been working to send a list to user containing some data. I am following facebook's doc to setup my request payload. However, I am getting the following error:
{'error': {
'message': '(#100) Invalid URL button fields provided. Please check documentation for details.',
'type': 'OAuthException',
'code': 100,
'error_subcode': 2018125, 'fbtrace_id': 'GZFFcM+j5e/'}}
Here is my JSON Payload:
{'recipient': {'id': 'MY_MESSENGER_ID'},
'message':
{'attachment':
{'type': 'template',
'payload':
{'template_type': 'list',
'top_element_style': 'compact',
'elements':
[{'title': 'Hello 1', 'subtitle': 'Subtitle 1',
'buttons':
[{'title': 'View', 'type': 'web_url',
'url': 'https://www.medium.com/',
'messenger_extensions': 'false',
'webview_height_ratio': 'full',
'fallback_url': 'https://www.medium.com/'}],
'default_action':
{'title': 'View', 'type': 'web_url',
'url': 'https://www.medium.com/',
'messenger_extensions': 'false',
'webview_height_ratio': 'full',
'fallback_url': 'https://www.medium.com/'}},
{'title': 'Hello 2', 'subtitle': 'Subtitle 2',
'image_url': 'https://cdn-images-1.medium.com/1*Vkf6A8Mb0wBoL3Fw1u0paA.jpeg',
'buttons':
[{'title': 'View', 'type': 'web_url',
'url': 'https://www.medium.com/',
'messenger_extensions': 'false',
'webview_height_ratio': 'full',
'fallback_url': 'https://www.medium.com/'}],
'default_action':
{'title': 'View', 'type': 'web_url',
'url': 'https://www.medium.com/',
'messenger_extensions': 'false',
'webview_height_ratio': 'full',
'fallback_url': 'https://www.medium.com/'}}]}}}}
I have checked, re-checked it multiple times. PLUS, I have sent the facebook's example json from the doc but I have got the same reply. Please take a look and let me know where I am stuck!
This is my end url:
"https://graph.facebook.com/v2.6/me/messages?access_token="
Thanks in advance!
Your request has two issues:
For default_action, you can set fallback_url only if messenger_extensions:true
default_action cannot have a title prop.
Try this:
{
"recipient": {
"id": "{{PSID}}"
},
"message": {
"attachment": {
"type": "template",
"payload": {
"template_type": "list",
"top_element_style": "compact",
"elements": [{
"title": "Hello 1",
"subtitle": "Subtitle 1",
"buttons": [{
"title": "View",
"type": "web_url",
"url": "https://www.medium.com/",
"messenger_extensions": "false",
"webview_height_ratio": "full"
}],
"default_action": {
"type": "web_url",
"url": "https://www.medium.com/",
"messenger_extensions": "false",
"webview_height_ratio": "full"
}
},
{
"title": "Hello 2",
"subtitle": "Subtitle 2",
"image_url": "https://cdn-images-1.medium.com/1*Vkf6A8Mb0wBoL3Fw1u0paA.jpeg",
"buttons": [{
"title": "View",
"type": "web_url",
"url": "https://www.medium.com/",
"messenger_extensions": "false",
"webview_height_ratio": "full"
}],
"default_action": {
"type": "web_url",
"url": "https://www.medium.com/",
"messenger_extensions": "false",
"webview_height_ratio": "full"
}
}
]
}
}
}
}
I am trying to get the SnapshotId of the below output with no success. I can get the value of the AMI description and the value of AMI_ID.
{
'Images': [
{
'Architecture': 'i386'|'x86_64',
'CreationDate': 'string',
'ImageId': 'string',
'ImageLocation': 'string',
'ImageType': 'machine'|'kernel'|'ramdisk',
'Public': True|False,
'KernelId': 'string',
'OwnerId': 'string',
'Platform': 'Windows',
'ProductCodes': [
{
'ProductCodeId': 'string',
'ProductCodeType': 'devpay'|'marketplace'
},
],
'RamdiskId': 'string',
'State': 'pending'|'available'|'invalid'|'deregistered'|'transient'|'failed'|'error',
'BlockDeviceMappings': [
{
'DeviceName': 'string',
'VirtualName': 'string',
'Ebs': {
'Encrypted': True|False,
'DeleteOnTermination': True|False,
'Iops': 123,
'SnapshotId': 'string',
'VolumeSize': 123,
'VolumeType': 'standard'|'io1'|'gp2'|'sc1'|'st1'
},
'NoDevice': 'string'
},
],
'Description': 'string',
'EnaSupport': True|False,
'Hypervisor': 'ovm'|'xen',
'ImageOwnerAlias': 'string',
'Name': 'string',
'RootDeviceName': 'string',
'RootDeviceType': 'ebs'|'instance-store',
'SriovNetSupport': 'string',
'StateReason': {
'Code': 'string',
'Message': 'string'
},
'Tags': [
{
'Key': 'string',
'Value': 'string'
},
],
'VirtualizationType': 'hvm'|'paravirtual'
},
]
}
Using the following code:
import boto3
client = boto3.client('ec2', region_name='us-east-1')
def verifica_imagem(imagem):
amiresponse = client.describe_images(
Filters=[
{
'Name': 'description',
'Values': [
imagem,
]
},
],
DryRun=False
)
try:
data = str(amiresponse['Images'][0]['Description'])
ami_id = str(amiresponse['Images'][0]['ImageId'])
snapshot_id = str(amiresponse['Images'][0]['SnapshotId'])
except:
print "AMI not exists! Exiting...."
return 1
verifica_imagem('IMAGE_XXXXXXX')
I can't understand how to use the key of SnapshotId. I have tried:
snapshot_id = str(amiresponse['Images']['BlockDeviceMappings']['Ebs'][0]['SnapshotId']) but is not working too.
The value of Images and BlockDeviceMappings is an array and Ebs is a dict.
Use this to fetch the value of SnapshotId,
snapshot_id = amiresponse['Images'][0]['BlockDeviceMappings'][0]['Ebs']['SnapshotId']