AWS Lambda NodeJS launch EMR Spark Problem - amazon-web-services

Question

I am trying to launch an EMR cluster with the following script in Lambda (NodeJS-12).
var AWS = require('aws-sdk');
var emr = new AWS.EMR({apiVersion: '2009-03-31', region: 'ap-northeast-1'});
exports.handler = (event, context, callback) => {
var params = {
"Name": "LaunchEMR",
"Instances": {
"KeepJobFlowAliveWhenNoSteps": true,
"TerminationProtected": false,
"Ec2SubnetId": "subnet-XXXXXXXX",
"EmrManagedMasterSecurityGroup": "ElasticMapReduce-master",
"EmrManagedSlaveSecurityGroup": "ElasticMapReduce-slave",
"HadoopVersion": "2.8.5",
"InstanceGroups": [{
"Name": "Master",
"InstanceRole": "MASTER",
"InstanceCount": 1,
"InstanceType": "m3.xlarge",
"Market": "ON_DEMAND"
}, {
"Name": "Core",
"InstanceRole": "CORE",
"InstanceCount": 1,
"InstanceType": "m3.xlarge",
"Market": "ON_DEMAND"
}]
},
"Applications": [{
"Name": "Hadoop"
}, {
"Name": "Spark"
}],
"ServiceRole": "EMR_DefaultRole",
"JobFlowRole": "EMR_EC2_DefaultRole",
"ReleaseLabel": "emr-5.28.0"
};
emr.runJobFlow(params)
.on('success', function(response){ console.log("success => " + response); console.log(response); })
.on('error', function(response){ console.log("error => " + response); console.log(response); })
.on('complete', function(response){ console.log("complete => " + response); console.log(response); })
.send( function(err, data){
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
callback(null, {
statusCode: 200,
body: JSON.stringify(err),
});
});
};
I had tuned the IAM policies can I seem to be able to submit the request successfully to EMR, as there is no error in the Lambda cloud-watch log, and we can see the request in EMR events.
However, we will get a cluster termination event right after the request event.
Error Log
Amazon EMR Cluster j-31OE0F3OXGO4K (LaunchEMR) has terminated with errors at 2019-12-02 06:32 UTC with a reason of VALIDATION_ERROR.
The SubnetID and Security-Group are the same as my manually started clusters.
I added the "iam:PassRole" in "EMR_DefaultRole".
I also tried some other instance types.
Is there anything that I am missing?

Answer 1

I am able to successfully run it by taking these 3 lines off.
"Ec2SubnetId": "subnet-XXXXXXXX",
"EmrManagedMasterSecurityGroup": "ElasticMapReduce-master",
"EmrManagedSlaveSecurityGroup": "ElasticMapReduce-slave",
But I still have no idea why it works. (and why it doesn't)