How to set or not set audio selector in AWS media convert? - amazon-web-services

I got this error, when creating job for AWS media convert:
Invalid selector_sequence_id [0] specified for audio_description [1].
I do not even need sound for my output mp4 video.
My intention is to loop for 2 second an image (png or jpg) and add a fade effect for the first frames.
How would you change the sent json?
{
"middlewareStack": {},
"input": {
"Queue": "arn:aws:mediaconvert:eu-central-1:634617701827:queues/Default",
"UserMetadata": {},
"Role": "arn:aws:iam::634617701827:role/service-role/MediaConvert_Default_Role",
"Settings": {
"TimecodeConfig": {
"Anchor": "00:00:00:00",
"Source": "EMBEDDED"
},
"OutputGroups": [
{
"Name": "File Group",
"Outputs": [
{
"Preset": "createPromoVideo",
"Extension": "mp4",
"NameModifier": "_fade",
"VideoDescription": {
"CodecSettings": {
"FilterGraph": "fade=out:150:30"
},
"ScalingBehavior": "DEFAULT",
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"Height": 1080,
"Width": 1080
},
"AudioDescriptions": [
{
"AudioSelector": {
"SelectorSettings": [
{
"AudioSelectorName": "Default"
}
]
},
"CodecSettings": {
"Codec": "AAC",
"AacSettings": {
"Bitrate": 96000,
"CodingMode": "CODING_MODE_2_0",
"SampleRate": 48000
}
}
}
]
}
],
"OutputGroupSettings": {
"Type": "FILE_GROUP_SETTINGS",
"FileGroupSettings": {
"Destination": "s3://t44-post-cover/8fui.mp4",
"DestinationSettings": {
"S3Settings": {
"AccessControl": {
"CannedAcl": "PUBLIC_READ"
}
}
}
}
}
}
],
"Inputs": [
{
"FileInput": "s3://t44-post-cover/8fui",
"VideoSelector": {
"ColorSpace": "FOLLOW"
},
"FilterEnable": "AUTO",
"TimecodeSource": "ZEROBASED",
"InputClippings": [
{
"StartTimecode": "00:00:00:00",
"EndTimecode": "00:00:02:00"
}
],
"FilterGraph": "fade=in:0:30",
"AudioSelectors": {
"Default": {
"DefaultSelection": "DEFAULT"
}
}
}
]
},
"AccelerationSettings": {
"Mode": "DISABLED"
},
"StatusUpdateInterval": "SECONDS_60",
"Priority": 0
}
}

AWS MediaConvert requires you to have at least one Audio Selector.
Just provide it with this simple one:
"Inputs": [
...
{
"AudioSelectors": {
"Audio Selector 1": {
"Offset": 0,
"DefaultSelection": "DEFAULT",
"SelectorType": "LANGUAGE_CODE",
"ProgramSelection": 1,
"LanguageCode": "ENM"
}
},
...
},
UPDATE:
A more barebones one:
"Inputs": [
...
{
"AudioSelectors": {
"Audio Selector 1": {
DefaultSelection: 'DEFAULT',
},
}
},
...
},

Related

Elasticsearch query to match a pattern and replace it with regex

I have to get the TOP 5 hit APIs/Urls and display with the http statuses of each url and I am able to do it in a way with the below extraction query
{
"query": {
"bool": {
"must": [
{
"range": {
"#timestamp": {
"from": "now-15m",
"to": "now",
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"aggregations": {
"Url": {
"terms": {
"field": "data.url.keyword",
"size": 5,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
},
"aggregations": {
"Status": {
"terms": {
"field": "data.response.status",
"size": 5,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
}
}
}
}
}
}
Output :
"aggregations": {
"Url": {
"doc_count_error_upper_bound": 940,
"sum_other_doc_count": 52374,
"buckets": [
{
"Status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"doc_count": 3,
"key": 200
},
{
"doc_count": 254,
"key": 400
}
]
},
"doc_count": 3515,
"key": "/account/me/"
},
{
"Status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"doc_count": 3376,
"key": 200
}
]
},
"doc_count": 3385,
"key": "/PlanDetails"
},
{
"Status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"doc_count": 3282,
"key": 200
}
]
},
"doc_count": 3282,
"key": "/evaluation"
},
{
"Status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"doc_count": 3205,
"key": 200
}
]
},
"doc_count": 3205,
"key": "/user/me"
},
{
"Status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"doc_count": 3055,
"key": 200
}
]
},
"doc_count": 3055,
"key": "/user"
}
]
}
}
}
But I have few URL where user ID comes inbetween the URLs.
For eg URLs like
/account/me/3417375321062014.cust
DJXXODKNPA1581975RI/PlanDetails
KXVEIPBYSR1597110677RI/payment
Because of distinct user ID inside the URL, elastic search considers these as separate one.
User ID is a mix of alphabets and numbers, sometimes only numbers and few only alphabets.
I need to replace these userID's in URL and count them as one
Lets say there are two hits with 200 status for /payment
DJXXODKNPA1583201975RI/payment
KXVEIPBYSR1597110677RI/payment
I should be able to replace the user ID with * and count as one API so I get output like
*/payment 200:2
But now it considers these as distinct URL/API, because of this I am unable to get the correct Top 5 Hit urls/APIs.
Any idea on this would be a great help. Thanks

How to configure dynamically a AWS Sage Maker task with AWS Step Function

I'm trying to build an ML pipeline using AWS Step Function.
I would like to configure the 'CreateHyperParameterTuningJob' dynamically depending on the input of the task.
Here is a screenshot of the State Machine that I'm trying to build:
ML State Machine
When I try to create this State Machine, I got the following error:
The value for the field 'MaxParallelTrainingJobs' must be an INTEGER
I'm struggling to figure out what is the issue here.
Do you have any suggestion to make the SM configuration dynamic with Step Function? Is it even possible?
Here is the input data passed to the 'Run training job' task:
{
"client_id": "test",
"training_job_definition": {
"AlgorithmSpecification": {
"TrainingImage": "433757028032.dkr.ecr.us-west-2.amazonaws.com/xgboost:latest",
"TrainingInputMode": "File"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "ml.m5.large",
"VolumeSizeInGB": 5
},
"StaticHyperParameters": {
"num_round": 750
},
"StoppingCondition": {
"MaxRuntimeInSeconds": 900
},
"InputDataConfig": [
{
"ChannelName": "train",
"CompressionType": "None",
"ContentType": "csv",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": "..."
}
}
},
{
"ChannelName": "validation",
"CompressionType": "None",
"ContentType": "csv",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": "..."
}
}
}
],
"OutputDataConfig": {
"S3OutputPath": "..."
},
"RoleArn": "arn:aws:iam::679298748479:role/landingzone_sagemaker_role"
},
"hyper_parameter_tuning_job_config": {
"HyperParameterTuningJobObjective": {
"MetricName": "validation:rmse",
"Type": "Minimize"
},
"Strategy": "Bayesian",
"ResourceLimits": {
"MaxParallelTrainingJobs": 2,
"MaxNumberOfTrainingJobs": 10
},
"ParameterRanges": {
"ContinuousParameterRanges": [
{
"Name": "eta",
"MinValue": 0.01,
"MaxValue": 0.04
},
{
"Name": "gamma",
"MinValue": 0,
"MaxValue": 100
},
{
"Name": "subsample",
"MinValue": 0.6,
"MaxValue": 1
},
{
"Name": "lambda",
"MinValue": 0,
"MaxValue": 5
},
{
"Name": "alpha",
"MinValue": 0,
"MaxValue": 2
}
],
"IntegerParameterRanges": [
{
"Name": "max_depth",
"MinValue": 5,
"MaxValue": 10
}
]
}
}
}
Here is JSON file that describes the State Machine:
{
"StartAt": "Generate Training Dataset",
"States": {
"Generate Training Dataset": {
"Resource": "arn:aws:lambda:uswest-2:012345678912:function:StepFunctionsSample-SageMaLambdaForDataGeneration-1TF67BUE5A12U",
"Type": "Task",
"Next": "Run training job"
},
"Run training job": {
"Resource": "arn:aws:states:::sagemaker:createHyperParameterTuningJob.sync",
"Parameters": {
"HyperParameterTuningJobName.$": "$.execution_date",
"HyperParameterTuningJobConfig": {
"HyperParameterTuningJobObjective": {
"MetricName": "$.hyper_parameter_tuning_job_config.HyperParameterTuningJobObjective.MetricName",
"Type": "Minimize"
},
"Strategy": "$.hyper_parameter_tuning_job_config.Strategy",
"ResourceLimits": {
"MaxParallelTrainingJobs": "$.hyper_parameter_tuning_job_config.ResourceLimits.MaxParallelTrainingJobs",
"MaxNumberOfTrainingJobs": "$.hyper_parameter_tuning_job_config.ResourceLimits.MaxNumberOfTrainingJobs"
},
"ParameterRanges": "$.hyper_parameter_tuning_job_config.ParameterRanges"
},
"TrainingJobDefinition": {
"AlgorithmSpecification": "$.training_job_definition.AlgorithmSpecification",
"StoppingCondition": "$.training_job_definition.StoppingCondition",
"ResourceConfig": "$.training_job_definition.ResourceConfig",
"RoleArn": "$.training_job_definition.RoleArn",
"InputDataConfig": "$.training_job_definition.InputDataConfig",
"OutputDataConfig": "$.training_job_definition.OutputDataConfig",
"StaticHyperParameters": "$.training_job_definition.StaticHyperParameters"
},
"HyperParameterTuningJobConfig.ResourceLimits": ""
},
"Type": "Task",
"End": true
}
}
}

Thrift Rate Limiting with Envoy

I'm extending Istio to provide Thrift features. The Istio component I'm working on right now is Pilot (Envoy config service). I've extended it with basic Thrift routing so that it can provide an Envoy configuration to route a listener to the correct cluster. My development environment looks something like:
Now I'm trying to add rate limiting. I'm following the docs. I've patched my config with a RouteAction that doesn't get applied, because there is no rate limit filter. This works fine, and passes traffic as before:
...
{
"filters": [
{
"name": "envoy.filters.network.thrift_proxy",
"typed_config": {
"#type": "type.googleapis.com/envoy.config.filter.network.thrift_proxy.v2alpha1.ThriftProxy",
"stat_prefix": "10.97.28.169_9090",
"transport": "HEADER",
"protocol": "BINARY",
"route_config": {
"name": "outbound|9090||backend.default.svc.cluster.local",
"routes": [
{
"match": {
"method_name": ""
},
"route": {
"cluster": "outbound|9090||backend.default.svc.cluster.local",
"rate_limits": [
{
"actions": [
{
"request_headers": {
"header_name": ":method-name",
"descriptor_key": "method-name"
}
}
]
}
]
}
}
]
}
}
}
]
}
],
"deprecated_v1": {
"bind_to_port": false
},
"listener_filters_timeout": "0.100s",
"traffic_direction": "OUTBOUND",
"continue_on_listener_filters_timeout": true
},
"last_updated": "2019-10-30T16:26:39.203Z"
},
...
I've built a function to create an envoy.filters.thrift.rate_limit filter that makes Envoy to call the rate limit service I've set up (I've tried both GoogleGrpc and EnvoyGrpc):
func buildThriftRatelimit(ratelimitServiceUri, domain string) *thrift_ratelimit.RateLimit {
var thriftRateLimit *thrift_ratelimit.RateLimit
timeout := 2000 * time.Millisecond
thriftRateLimit = &thrift_ratelimit.RateLimit{
Domain: domain,
Timeout: &timeout,
FailureModeDeny: false,
RateLimitService: &ratelimit.RateLimitServiceConfig{
GrpcService: &core.GrpcService{
TargetSpecifier: &core.GrpcService_GoogleGrpc_{
GoogleGrpc: &core.GrpcService_GoogleGrpc{
StatPrefix: ratelimitServiceUri,
TargetUri: ratelimitServiceUri,
},
},
},
},
}
thriftRateLimit.Validate()
if err := thriftRateLimit.Validate(); err != nil {
panic(err)
}
return thriftRateLimit
}
Which produces:
...
{
"filters": [
{
"name": "envoy.filters.network.thrift_proxy",
"typed_config": {
"#type": "type.googleapis.com/envoy.config.filter.network.thrift_proxy.v2alpha1.ThriftProxy",
"stat_prefix": "10.97.28.169_9090",
"transport": "HEADER",
"protocol": "BINARY",
"route_config": {
"name": "outbound|9090||backend.default.svc.cluster.local",
"routes": [
{
"match": {
"method_name": ""
},
"route": {
"cluster": "outbound|9090||backend.default.svc.cluster.local",
"rate_limits": [
{
"actions": [
{
"request_headers": {
"header_name": ":method-name",
"descriptor_key": "method-name"
}
}
]
}
]
}
}
]
},
"thrift_filters": [
{
"name": "envoy.filters.thrift.rate_limit",
"typed_config": {
"#type": "type.googleapis.com/envoy.config.filter.thrift.rate_limit.v2alpha1.RateLimit",
"domain": "backend.default.svc.cluster.local",
"timeout": "2s",
"rate_limit_service": {
"grpc_service": {
"google_grpc": {
"target_uri": "istio-lyft-ratelimit.istio-system.svc.cluster.local:80",
"stat_prefix": "istio-lyft-ratelimit.istio-system.svc.cluster.local:80"
}
}
}
}
}
]
}
}
]
}
],
"deprecated_v1": {
"bind_to_port": false
},
"listener_filters_timeout": "0.100s",
"traffic_direction": "OUTBOUND",
"continue_on_listener_filters_timeout": true
},
"last_updated": "2019-10-30T16:26:39.203Z"
},
...
When the rate limit filter is applied, connections to the backend die and no error is returned to the client or displayed in Envoy's logs.
If Thrift filters are provided, you need to add the Router filter as the last filter in the chain like so:
...
{
"filters": [
{
"name": "envoy.filters.network.thrift_proxy",
"typed_config": {
"#type": "type.googleapis.com/envoy.config.filter.network.thrift_proxy.v2alpha1.ThriftProxy",
"stat_prefix": "10.97.28.169_9090",
"transport": "HEADER",
"protocol": "BINARY",
"route_config": {
"name": "outbound|9090||backend.default.svc.cluster.local",
"routes": [
{
"match": {
"method_name": ""
},
"route": {
"cluster": "outbound|9090||backend.default.svc.cluster.local",
"rate_limits": [
{
"actions": [
{
"request_headers": {
"header_name": ":method-name",
"descriptor_key": "method-name"
}
}
]
}
]
}
}
]
},
"thrift_filters": [
{
"name": "envoy.filters.thrift.rate_limit",
"typed_config": {
"#type": "type.googleapis.com/envoy.config.filter.thrift.rate_limit.v2alpha1.RateLimit",
"domain": "backend.default.svc.cluster.local",
"timeout": "2s",
"rate_limit_service": {
"grpc_service": {
"google_grpc": {
"target_uri": "istio-lyft-ratelimit.istio-system.svc.cluster.local:80",
"stat_prefix": "istio-lyft-ratelimit.istio-system.svc.cluster.local:80"
}
}
}
}
},
{
"name": "envoy.filters.thrift.router"
}
]
}
}
]
}
],
"deprecated_v1": {
"bind_to_port": false
},
"listener_filters_timeout": "0.100s",
"traffic_direction": "OUTBOUND",
"continue_on_listener_filters_timeout": true
},
"last_updated": "2019-10-30T16:26:39.203Z"
},
...

Unable to use more than two Action Packages to create multiple custom actions

I followed this link :
https://developers.google.com/assistant/sdk/guides/service/python/extend/custom-actions
And I successfully create three actions files. and it works individually. but when I test all together like deploying three action packages:
./gactions update --action_package MqttAct.json --action_package action.json --action_package MesureTemp.json --project rpi3-0001-ga-******
I found that only two packages will work. it's like we can't have more than two action packages?
I'm surprised you can even deploy two packages at the same time. The idea is that you have one action package, which can contain a variety of actions merged together.
Here is a merged action package, based on the documentation:
{
"manifest": {
"displayName": "Blinky light",
"invocationName": "Blinky light",
"category": "PRODUCTIVITY"
},
"actions": [
{
"name": "com.example.actions.BlinkLight",
"availability": {
"deviceClasses": [
{
"assistantSdkDevice": {}
}
]
},
"intent": {
"name": "com.example.intents.BlinkLight",
"parameters": [
{
"name": "number",
"type": "SchemaOrg_Number"
},
{
"name": "speed",
"type": "Speed"
}
],
"trigger": {
"queryPatterns": [
"blink ($Speed:speed)? $SchemaOrg_Number:number times",
"blink $SchemaOrg_Number:number times ($Speed:speed)?"
]
}
},
"fulfillment": {
"staticFulfillment": {
"templatedResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Blinking $number times"
}
},
{
"deviceExecution": {
"command": "com.example.commands.BlinkLight",
"params": {
"speed": "$speed",
"number": "$number"
}
}
}
]
}
}
}
},
{
"name": "com.example.actions.BlonkLight",
"availability": {
"deviceClasses": [
{
"assistantSdkDevice": {}
}
]
},
"intent": {
"name": "com.example.intents.BlonkLight",
"parameters": [
{
"name": "number",
"type": "SchemaOrg_Number"
},
{
"name": "speed",
"type": "Speed"
}
],
"trigger": {
"queryPatterns": [
"blonk ($Speed:speed)? $SchemaOrg_Number:number times",
"blonk $SchemaOrg_Number:number times ($Speed:speed)?"
]
}
},
"fulfillment": {
"staticFulfillment": {
"templatedResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Blonking $number times"
}
},
{
"deviceExecution": {
"command": "com.example.commands.BlonkLight",
"params": {
"speed": "$speed",
"number": "$number"
}
}
}
]
}
}
}
}
],
"types": [
{
"name": "$Speed",
"entities": [
{
"key": "slowly",
"synonyms": [
"slowly",
"slow"
]
},
{
"key": "normally",
"synonyms": [
"normally",
"regular"
]
},
{
"key": "quickly",
"synonyms": [
"quickly",
"fast",
"quick"
]
}
]
}
]
}

Hypen in response data is throwing error when setting as global variable for nested response

I have a post man request. Which returns a response data as:
[
{
"id": "7ca27c09-2b67-417e-b367-f97d49824a2f",
"tags": [],
"type": "ApplicationProfile",
"userId": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"profile": {
"User-Profile": {
"PHI": {
"gender": "Male",
"surgeryDate": {
"value": "2017-06-06"
},
"surgeryType": {
"value": "gastricBand"
},
"heightInches": "72",
"approvalPhase": "2",
"profileImageUrl": "",
"motivationalImageUrl": ""
},
"PII": {
"mail": "c11#c111.com",
"lastName": "",
"firstName": "name1",
"shouldReceiveNotifications": true
}
},
"motivationalGoals": [
{
"goal": "Improving health",
"goalId": "2957"
},
{
"goal": "Relieving pain",
"goalId": "2958"
},
{
"goal": "Feeling better",
"goalId": "2960"
}
]
},
"version": 35,
"createdBy": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"lastModifiedBy": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"applicationName": "Health Partner",
"createdDateTime": "2017-06-08T16:08:32.497Z",
"lastModifiedDateTime": "2017-06-09T13:23:03.503Z"
},
{
"id": "091b5ebd-b096-436e-a703-f97f0ae77baf",
"tags": [
"Application-Profile"
],
"type": "ApplicationProfile",
"userId": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"profile": {
"User-Profile": {
"PHI": {
"gender": "",
"surgeryDate": {
"value": "2017-06-01"
},
"surgeryType": {
"value": ""
},
"heightInches": "72",
"approvalPhase": "2",
"profileImageUrl": "",
"motivationalImageUrl": ""
},
"PII": {
"mail": "c11#c111.com",
"lastName": "Dev",
"firstName": "Kat",
"shouldReceiveNotifications": true
}
},
"motivationalGoals": [
{
"goal": "Improving health",
"goalId": "2957"
},
{
"goal": "Relieving pain",
"goalId": "2958"
},
{
"goal": "Feeling better",
"goalId": "2960"
}
]
},
"version": 41,
"createdBy": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"lastModifiedBy": "2f2c8684-874d-49ea-bfa4-977069c1e3e2",
"applicationName": "Health Partner Weightloss",
"createdDateTime": "2017-06-03T16:19:57.811Z",
"lastModifiedDateTime": "2017-06-08T16:07:49.555Z"
}
]
I need to extract the value of the emailid and set as global variable.
here is my code:
var jsonData1 = JSON.parse(responseBody);
postman.setGlobalVariable("jsonData1",jsonData1.profile.User-Profile.PII.mail);
But I am getting error. "User is undefined".
The problem here is that the hyphen / minus sign is interpreted as the subtraction operator.
Consider changing your API and using "userProfile" instead of "User-Profile".