When I log into my AWS account and go to the Billing -> Bills dashboard, I see my Total Bill for the month and previous months, $X USD. Boto3's cost explorer will return my monthly AmortizedCost, but number returned is significantly less than my Total Bill?
def billing(account):
today = date.today()
start = today.replace(day = 1) - relativedelta(months = 12)
client = boto3.client('ce',
region_name = 'us-east-1',
aws_access_key_id = account['aws_access_key_id'],
aws_secret_access_key = account['aws_secret_access_key'])
response = client.get_cost_and_usage(
TimePeriod={
'Start': str(start),
'End': str(today),
},
Granularity = 'MONTHLY',
Metrics=[
'AmortizedCost',
]
)
print("{}".format(account['Account']))
for item in response['ResultsByTime']:
print("{}: {}".format( item['TimePeriod']['Start'], item['Total']['AmortizedCost']['Amount']))
How do I get the total amount I see on the Billing -> Bills dashboard?
I'm trying cloudwatch, but I'm not having any success.
Related
I know this page lists up the instance types which based on Nitro system but I would like to know the list in a dynamic way with CLI. (for example, using aws ec2 describe-instances). Is it possible to get Nitro based instance type other than parsing the static page? If so, could you tell me the how?
You'd have to write a bit of additional code to get that information. aws ec2 describe-instances will give you InstanceType property. You should use a programming language to parse the JSON, extract InstanceType and then call describe-instances like so: https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instance-types.html?highlight=nitro
From the JSON you get back, extract hypervisor. That'll give you Nitro if the instance is Nitro.
Here's a Python code that might work. I have not tested it fully but you can tweak this to get the results you want.
"""List all EC2 instances"""
import boto3
def ec2_connection():
"""Connect to AWS using API"""
region = 'us-east-2'
aws_key = 'xxx'
aws_secret = 'xxx'
session = boto3.Session(
aws_access_key_id = aws_key,
aws_secret_access_key = aws_secret
)
ec2 = session.client('ec2', region_name = region)
return ec2
def get_reservations(ec2):
"""Get a list of instances as a dictionary"""
response = ec2.describe_instances()
return response['Reservations']
def process_instances(reservations, ec2):
"""Print a colorful list of IPs and instances"""
if len(reservations) == 0:
print('No instance found. Quitting')
return
for reservation in reservations:
for instance in reservation['Instances']:
# get friendly name of the server
# only try this for mysql1.local server
friendly_name = get_friendly_name(instance)
if friendly_name.lower() != 'mysql1.local':
continue
# get the hypervisor based on the instance type
instance_type = get_instance_info(instance['InstanceType'], ec2)
# print findings
print(f'{friendly_name} // {instance["InstanceType"]} is {instance_type}')
break
def get_instance_info(instance_type, ec2):
"""Get hypervisor from the instance type"""
response = ec2.describe_instance_types(
InstanceTypes=[instance_type]
)
return response['InstanceTypes'][0]['Hypervisor']
def get_friendly_name(instance):
"""Get friendly name of the instance"""
tags = instance['Tags']
for tag in tags:
if tag['Key'] == 'Name':
return tag['Value']
return 'Unknown'
def run():
"""Main method to call"""
ec2 = ec2_connection()
reservations = get_reservations(ec2)
process_instances(reservations, ec2)
if __name__ == '__main__':
run()
print('Done')
In the above answer , the statement "From the JSON you get back, extract hypervisor. That'll give you Nitro if the instance is Nitro " is not longer accurate.
As per the latest AWS documentation,
hypervisor - The hypervisor type of the instance (ovm | xen ). The value xen is used for both Xen and Nitro hypervisors.
Cleaned up, verified working code below:
# Get all instance types that run on Nitro hypervisor
import boto3
def get_nitro_instance_types():
"""Get all instance types that run on Nitro hypervisor"""
ec2 = boto3.client('ec2', region_name = 'us-east-1')
response = ec2.describe_instance_types(
Filters=[
{
'Name': 'hypervisor',
'Values': [
'nitro',
]
},
],
)
instance_types = []
for instance_type in response['InstanceTypes']:
instance_types.append(instance_type['InstanceType'])
return instance_types
get_nitro_instance_types()
Example output as of 12/06/2022 below:
['r5dn.8xlarge', 'x2iedn.xlarge', 'r6id.2xlarge', 'r6gd.medium',
'm5zn.2xlarge', 'r6idn.16xlarge', 'c6a.48xlarge', 'm5a.16xlarge',
'im4gn.2xlarge', 'c6gn.16xlarge', 'c6in.24xlarge', 'r5ad.24xlarge',
'r6i.xlarge', 'c6i.32xlarge', 'x2iedn.2xlarge', 'r6id.xlarge',
'i3en.24xlarge', 'i3en.12xlarge', 'm5d.8xlarge', 'c6i.8xlarge',
'r6g.large', 'm6gd.4xlarge', 'r6a.2xlarge', 'x2iezn.4xlarge',
'c6i.large', 'r6in.24xlarge', 'm6gd.xlarge', 'm5dn.2xlarge',
'd3en.2xlarge', 'c6id.8xlarge', 'm6a.large', 'is4gen.xlarge',
'r6g.8xlarge', 'm6idn.large', 'm6a.2xlarge', 'c6i.4xlarge',
'i4i.16xlarge', 'm5zn.6xlarge', 'm5.8xlarge', 'm6id.xlarge',
'm5n.16xlarge', 'c6g.16xlarge', 'r5n.12xlarge', 't4g.nano',
'm5ad.12xlarge', 'r6in.12xlarge', 'm6idn.12xlarge', 'g5.2xlarge',
'trn1.32xlarge', 'x2gd.8xlarge', 'is4gen.4xlarge', 'r6gd.xlarge',
'r5a.xlarge', 'r5a.2xlarge', 'c5ad.24xlarge', 'r6a.xlarge',
'r6g.medium', 'm6id.12xlarge', 'r6idn.2xlarge', 'c5n.2xlarge',
'g5.4xlarge', 'm5d.xlarge', 'i3en.3xlarge', 'r5.24xlarge',
'r6gd.2xlarge', 'c5d.large', 'm6gd.12xlarge', 'm6id.2xlarge',
'm6i.large', 'z1d.2xlarge', 'm5a.4xlarge', 'm5a.2xlarge',
'c6in.xlarge', 'r6id.16xlarge', 'c7g.8xlarge', 'm5dn.12xlarge',
'm6gd.medium', 'im4gn.8xlarge', 'm5dn.large', 'c5ad.4xlarge',
'r6g.16xlarge', 'c6a.24xlarge', 'c6a.16xlarge']
"""List all EC2 instances"""
import boto3
def ec2_connection():
"""Connect to AWS using API"""
region = 'us-east-2'
aws_key = 'xxx'
aws_secret = 'xxx'
session = boto3.Session(
aws_access_key_id = aws_key,
aws_secret_access_key = aws_secret
)
ec2 = session.client('ec2', region_name = region)
return ec2
def get_reservations(ec2):
"""Get a list of instances as a dictionary"""
response = ec2.describe_instances()
return response['Reservations']
def process_instances(reservations, ec2):
"""Print a colorful list of IPs and instances"""
if len(reservations) == 0:
print('No instance found. Quitting')
return
for reservation in reservations:
for instance in reservation['Instances']:
# get friendly name of the server
# only try this for mysql1.local server
friendly_name = get_friendly_name(instance)
if friendly_name.lower() != 'mysql1.local':
continue
# get the hypervisor based on the instance type
instance_type = get_instance_info(instance['InstanceType'], ec2)
# print findings
print(f'{friendly_name} // {instance["InstanceType"]} is {instance_type}')
break
def get_instance_info(instance_type, ec2):
"""Get hypervisor from the instance type"""
response = ec2.describe_instance_types(
InstanceTypes=[instance_type]
)
return response['InstanceTypes'][0]['Hypervisor']
def get_friendly_name(instance):
"""Get friendly name of the instance"""
tags = instance['Tags']
for tag in tags:
if tag['Key'] == 'Name':
return tag['Value']
return 'Unknown'
def run():
"""Main method to call"""
ec2 = ec2_connection()
reservations = get_reservations(ec2)
process_instances(reservations, ec2)
if name == 'main':
run()
print('Done')
I try to do Amazon S3 Bucket resource tagging with AWS Lambda. I need Lambda function to take tags from users and apply them to newly uploaded objects in an Amazon S3 Bucket.
I think get_user_tags in handler is the faulty part
def lambda_handler(event, context):
#Get uncompressed CloudWatch Event data for parsing API calls
def get_unc_cw_event_data(event):
cw_data_dict = dict()
cw_data_dict = event['detail']
return cw_data_dict
#Get resource tags assigned to a specified IAM role
#Returns a list of key:string,value:string resource tag dictionaries
def get_user_tags(user_name):
try:
client = boto3.client('iam')
response = dict()
response = client.list_user_tags(
UserName=user_name
)
except botocore.exceptions.ClientError as error:
print("Boto3 API returned error: ", error)
print("No Tags Applied To: ", resource_id)
no_tags = list()
return no_tags
return response['Tags']
#Get resource tags stored in AWS SSM Parameter Store
#Returns a list of key:string,value:string resource tag dictionaries
def get_ssm_parameter_tags(role_name, user_name):
tag_list = list()
try:
path_string = "/auto-tag/" + role_name + "/" + user_name + "/tag"
ssm_client = boto3.client('ssm')
get_parameter_response = ssm_client.get_parameters_by_path(
Path=path_string,
Recursive=True,
WithDecryption=True
)
for parameter in get_parameter_response['Parameters']:
tag_dictionary = dict()
path_components = parameter['Name'].split("/")
tag_key = path_components[-1]
tag_dictionary['Key'] = tag_key
tag_dictionary['Value'] = parameter['Value']
tag_list.append(tag_dictionary)
return tag_list
except botocore.exceptions.ClientError as error:
print("Boto3 API returned error: ", error)
tag_list.clear()
return tag_list
#Apply tags to resource
def set_resource_tags(resource_id, resource_tags):
# Is this an EC2 resource?
if re.search("^i-", resource_id):
try:
client = boto3.client('ec2')
response = client.create_tags(
Resources=[
resource_id
],
Tags=resource_tags
)
response = client.describe_volumes(
Filters=[
{
'Name': 'attachment.instance-id',
'Values': [
resource_id
]
}
]
)
try:
for volume in response['Volumes']:
ec2 = boto3.resource('ec2')
ec2_vol = ec2.Volume(volume['VolumeId'])
vol_tags = ec2_vol.create_tags(
Tags=resource_tags
)
except botocore.exceptions.ClientError as error:
print("Boto3 API returned error: ", error)
print("No Tags Applied To: ", response['Volumes'])
return False
except botocore.exceptions.ClientError as error:
print("Boto3 API returned error: ", error)
print("No Tags Applied To: ", resource_id)
return False
return True
else:
return False
data_dict = get_unc_cw_event_data(event)
#data_dict = get_cw_event_data(event)
user_id_arn = data_dict['userIdentity']['arn']
user_id_components = user_id_arn.split("/")
user_id = user_id_components[-1]
#role_arn = data_dict['userIdentity']['sessionContext']['sessionIssuer']['arn']
role_arn = data_dict['userIdentity']['arn']
role_components = role_arn.split("/")
role_name = role_components[-1]
resource_date = data_dict['eventTime']
resource_role_tags = list()
resource_role_tags = get_role_tags(role_name)
resource_parameter_tags = list()
resource_parameter_tags = get_ssm_parameter_tags(role_name, user_id)
resource_tags = list()
resource_tags = resource_role_tags + resource_parameter_tags
created_by = dict()
created_by['Key'] = 'Created by'
created_by['Value'] = user_id
resource_tags.append(created_by)
roleName = dict()
roleName['Key'] = 'Role Name'
roleName['Value'] = role_name
resource_tags.append(roleName)
date_created = dict()
date_created['Key'] = 'Date created'
date_created['Value'] = resource_date
resource_tags.append(date_created)
if 'instancesSet' in data_dict['responseElements']:
for item in data_dict['responseElements']['instancesSet']['items']:
resource_id = item['instanceId']
if set_resource_tags(resource_id, resource_tags):
return {
'statusCode': 200,
'Resource ID': resource_id,
'body': json.dumps(resource_tags)
}
else:
return {
'statusCode': 500,
'No tags applied to Resource ID': resource_id,
'Lambda function name': context.function_name,
'Lambda function version': context.function_version
}
else:
return {
'statusCode': 200,
'No resources to tag': event['id']
}
And the resulting error is this:
Response { "errorMessage": "Bad handler
'resource-auto-tagger-main/source/resource-auto-tagger': not enough
values to unpack (expected 2, got 1)", "errorType":
"Runtime.MalformedHandlerName", "stackTrace": []
I run my first AWS Lambda, so will be happy about any help.
You can certainly use an AWS Lambda function to tag objects that are located in an Amazon S3 bucket. There is a development article that shows this use case. It's implemented in Java; however, it will point you in the right direction and you can port to your programming language.
Creating an Amazon Web Services Lambda function that tags digital assets located in Amazon S3 buckets
In this example, the Lambda function reads all objects in a given Amazon S3 bucket. For each object in the bucket, it passes the image to the Amazon Rekognition service to generate a series of labels. Each label is used to create a tag that is applied to the image. After you execute the Lambda function, it automatically creates tags based on all images in a given Amazon S3 bucket and applies them to the images.
I am trying to get instances that have tag name 'ttl' and tag values 'older than today'.
Our users tag their instances based on a future date (yyyy.mm.dd) so that the script should not automatically delete them before that date. The filter below should be able to get only instances whose tag-value is smaller than today's date. Is there a way to filter the tage values based on this logic -> get me instances that are < today ?
today = = datetime.today().strftime('%Y.%m.%d')
filters = [{'Name': 'tag:ttl','Values': ['<today']},{'Name': 'instance-state-name','Values': ['running']}]
The full code looks like this:
import boto3
import logging
#setup simple logging for INFO
logger = logging.getLogger()
logger.setLevel(logging.INFO)
#define the connection
ec2 = boto3.resource('ec2')
def lambda_handler(event, context):
filters = [{'Name': 'tag:ttl','Values': ['<today']},{'Name': 'instance-state-name','Values': ['running']}]
#locate all running instances
RunningInstances = [instance.id for instance in instances]
print (RunningInstances)
#make sure there are actually instances to shut down.
if len(RunningInstances) > 0:
shuttingDown = ec2.instances.filter(InstanceIds=RunningInstances).stop()
print "shuttingDown"
else:
print "Nothing to see here"
You can't compare dates in the query as you specified. You need to write some code comparing the dates in your python script and take action. Roughly like below:
import datetime
import boto3
# Connect to EC2
ec2 = boto3.resource('ec2')
def get_ttl(instance):
for tag in instance.tags:
if 'ttl'in tag['Key']:
ttl = tag['Value']
return ttl
def lambda_handler(event,context):
running_instances = ec2.instances.filter(Filters=[{
'Name': 'instance-state-name',
'Values': ['running']}])
for instance in running_instances:
ttl = get_ttl(instance)
if ttl:
if datetime.date.today() > datetime.datetime.strptime(ttl, "%Y-%m-%d").date():
print('stopping instance')
else:
print("nothing to do here")
Basically, it resolves around comparing dates
In [30]: datestring = '2021-02-22'
In [31]: datetime.date.today() == datetime.datetime.strptime(datestring, "%Y-%m-%d").date()
Out[31]: False
In [32]: datetime.date.today() < datetime.datetime.strptime(datestring, "%Y-%m-%d").date()
Out[32]: True
In [33]: datetime.date.today() > datetime.datetime.strptime(datestring, "%Y-%m-%d").date()
Out[33]: False
Is there a way to send EMR logs to CloudWatch instead of S3. We would like to have all our services logs in one location. Seems like the only thing you can do is set up alarms for monitoring but that doesn't cover logging.
https://docs.aws.amazon.com/emr/latest/ManagementGuide/UsingEMR_ViewingMetrics.html
Would I have to install CloudWatch agent on the nodes in the cluster https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AgentReference.html
you can install the CloudWatch agent via EMR’s bootstrap configuration, and configure it to watch log directories. It then starts to push logs to Amazon CloudWatch Logs
You can read the logs from s3 and push them to the cloudwatch using boto3 and delete them from s3 if you do not need. In some use-cases stdout.gz log will be needed to be in the cloudwatch for monitoring purposes.
boto3 documentation on put_log_events
import boto3
import botocore.session
import logging
import time
import datetime
import gzip
def get_session(service_name):
session = botocore.session.get_session()
aws_access_key_id = session.get_credentials().access_key
aws_secret_access_key = session.get_credentials().secret_key
aws_session_token = session.get_credentials().token
region = session.get_config_variable('region')
return boto3.client(
service_name = service_name,
region_name = region,
aws_access_key_id = aws_access_key_id,
aws_secret_access_key = aws_secret_access_key,
aws_session_token = aws_session_token
)
def get_log_file(s3, bucket, key):
log_file = None
try:
obj = s3.get_object(Bucket=bucket, Key=key)
compressed_body = obj['Body'].read()
log_file = gzip.decompress(compressed_body)
except Exception as e:
logger.error(f"Error reading from bucket : {e}")
raise
return log_file
def create_log_events(logs, batch_size):
log_event_batch = []
log_event_batch_collection = []
try:
for line in logs.splitlines():
log_event = {'timestamp': int(round(time.time() * 1000)), 'message':line.decode('utf-8')}
if len(log_event_batch) < batch_size:
log_event_batch.append(log_event)
else:
log_event_batch_collection.append(log_event_batch)
log_event_batch = []
log_event_batch.append(log_event)
except Exception as e:
logger.error(f"Error creating log events : {e}")
raise
log_event_batch_collection.append(log_event_batch)
return log_event_batch_collection
def create_log_stream_and_push_log_events(logs, log_group, log_stream, log_event_batch_collection, delay):
response = logs.create_log_stream(logGroupName=log_group, logStreamName=log_stream)
seq_token = None
try:
for log_event_batch in log_event_batch_collection:
log_event = {
'logGroupName': log_group,
'logStreamName': log_stream,
'logEvents': log_event_batch
}
if seq_token:
log_event['sequenceToken'] = seq_token
response = logs.put_log_events(**log_event)
seq_token = response['nextSequenceToken']
time.sleep(delay)
except Exception as e:
logger.error(f"Error pushing log events : {e}")
raise
The caller function
def main():
s3 = get_session('s3')
logs = get_session('logs')
BUCKET_NAME = 'Your_Bucket_Name'
KEY = 'logs/emr/Path_To_Log/stdout.gz'
BATCH_SIZE = 10000 #According to boto3 docs
PUSH_DELAY = 0.2 #According to boto3 docs
LOG_GROUP='test_log_group' #Destination log group
LOG_STREAM='{}-{}'.format(time.strftime('%Y-%m-%d'),'logstream.log')
log_file = get_log_file(s3, BUCKET_NAME, KEY)
log_event_batch_collection = create_log_events(log_file, BATCH_SIZE)
create_log_stream_and_push_log_events(logs, LOG_GROUP, LOG_STREAM, log_event_batch_collection, PUSH_DELAY)
I am aware of the HTTP Data Collector API that can be used to pull data into Azure Log analytics, my ask here is on AWS Cloudwatch data to Azure. We have Azure hosted application and an external AWS hosted Serverless Lamda functions and we want to import the logs of those 13 serverless functions into Azure. I know from the documentation and there is a python function that can be used as a AWS Lamda function and the python example is in MSFT documentation. But what I am failing to understand is what Json format that AWS cloud collector needs to create so they can send it to Azure Log Analytics. Any examples on this ? Any help on how this can be done. I have come across this blog also but that is splunk specific. https://www.splunk.com/blog/2017/02/03/how-to-easily-stream-aws-cloudwatch-logs-to-splunk.html
Hey never mind I was able to dig a little deeper and I found that in AWS I can STREAM the Logs from one Lambda to other Lambda function thru subscription. Once that was setthen all I did was consumed that and on the fly created the JSON and sent it to Azure Logs. In case if you or anyone is interested in it, following is the code:-
import json
import datetime
import hashlib
import hmac
import base64
import boto3
import datetime
import gzip
from botocore.vendored import requests
from datetime import datetime
Update the customer ID to your Log Analytics workspace ID
customer_id = "XXXXXXXYYYYYYYYYYYYZZZZZZZZZZ"
For the shared key, use either the primary or the secondary Connected Sources client authentication key
shared_key = "XXXXXXXXXXXXXXXXXXXXXXXXXX"
The log type is the name of the event that is being submitted
log_type = 'AWSLambdafuncLogReal'
json_data = [{
"slot_ID": 12345,
"ID": "5cdad72f-c848-4df0-8aaa-ffe033e75d57",
"availability_Value": 100,
"performance_Value": 6.954,
"measurement_Name": "last_one_hour",
"duration": 3600,
"warning_Threshold": 0,
"critical_Threshold": 0,
"IsActive": "true"
},
{
"slot_ID": 67890,
"ID": "b6bee458-fb65-492e-996d-61c4d7fbb942",
"availability_Value": 100,
"performance_Value": 3.379,
"measurement_Name": "last_one_hour",
"duration": 3600,
"warning_Threshold": 0,
"critical_Threshold": 0,
"IsActive": "false"
}]
#body = json.dumps(json_data)
#####################
######Functions######
#####################
Build the API signature
def build_signature(customer_id, shared_key, date, content_length, method, content_type, resource):
x_headers = 'x-ms-date:' + date
string_to_hash = method + "\n" + str(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource
bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
decoded_key = base64.b64decode(shared_key)
encoded_hash = base64.b64encode(
hmac.new(decoded_key, bytes_to_hash, digestmod=hashlib.sha256).digest()).decode()
authorization = "SharedKey {}:{}".format(customer_id,encoded_hash)
return authorization
Build and send a request to the POST API
def post_data(customer_id, shared_key, body, log_type):
method = 'POST'
content_type = 'application/json'
resource = '/api/logs'
rfc1123date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
print (rfc1123date)
content_length = len(body)
signature = build_signature(customer_id, shared_key, rfc1123date, content_length, method, content_type, resource)
uri = 'https://' + customer_id + '.ods.opinsights.azure.com' + resource + '?api-version=2016-04-01'
headers = {
'content-type': content_type,
'Authorization': signature,
'Log-Type': log_type,
'x-ms-date': rfc1123date
}
response = requests.post(uri,data=body, headers=headers)
if (response.status_code >= 200 and response.status_code <= 299):
print("Accepted")
else:
print("Response code: {}".format(response.status_code))
print(response.text)
def lambda_handler(event, context):
cloudwatch_event = event["awslogs"]["data"]
decode_base64 = base64.b64decode(cloudwatch_event)
decompress_data = gzip.decompress(decode_base64)
log_data = json.loads(decompress_data)
print(log_data)
awslogdata = json.dumps(log_data)
post_data(customer_id, shared_key, awslogdata, log_type)