Dataflow job not triggering on cloud from Composer(Airflow) - google-cloud-platform

Question

I am trying to execute apache beam pipeline from composer and facing below issue that the job does not get trigger on GCP.
Job log: (parameterized below stuff not to reveal company specific details:
NFO - Running command: java -jar /tmp/dataflow40103bb6-GcsToBqDataIngestion.jar --runner=DataflowRunner --project=<project_id> --zone=northamerica-northeast1-a --stagingLocation=gs:// --maxNumWorkers=1 --tempLocation=<> --region=northamerica-northeast1 --subnetwork=<network_link> --serviceAccount= --usePublicIps=false --pipelineConfig=pipeline_config/pgp_comm_apps.properties --workerMachineType=n1-standard-2 --env=dev --jobName=test-ingestion-7e20f260#-#{"workflow": "fds-test-dataflow", "task-id": "load-data", "execution-date": "2022-08-02T19:31:51.473861+00:00"}
DAG code:
import datetime
from airflow import models
# The DAG object; we'll need this to instantiate a DAG
from airflow import DAG
from airflow.contrib.operators.dataflow_operator import DataFlowJavaOperator
# Operators; we need this to operate!
from airflow.operators.bash_operator import BashOperator
default_dag_args = {
# The start_date describes when a DAG is valid / can be run. Set this to a
# fixed point in time rather than dynamically, since it is evaluated every
# time a DAG is parsed. See:
# https://airflow.apache.org/faq.html#what-s-the-deal-with-start-date
'owner': 'Airflow',
'depends_on_past': False,
'start_date': datetime.datetime(2022, 7, 27),
'email': ['test#test.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 0,
'dataflow_default_options': {
'project': '<prod_id>',
#"region": "northamerica-northeast1",
"zone": "northamerica-northeast1-a",
'stagingLocation': 'gs://location',
}
#'retry_delay': timedelta(minutes=30),
}
# Define a DAG (directed acyclic graph) of tasks.
# Any task you create within the context manager is automatically added to the
# DAG object.
with models.DAG(
'fds-test-dataflow',
catchup=False,
schedule_interval=None,
#schedule_interval=datetime.timedelta(days=1),
default_args=default_dag_args) as dag:
task = DataFlowJavaOperator(
task_id='load-data',
gcp_conn_id="gcp_connection",
job_name='test-ingestion',
jar='gs://path_to_jar',
delegate_to="<SA>",
location='northamerica-northeast1',
options={
'maxNumWorkers': '1',
'project': '<proj_id>',
'tempLocation': 'gs://location/',
'region': 'northamerica-northeast1',
"zone": "northamerica-northeast1-a",
'subnetwork': 'network',
'serviceAccount': 'SA',
'usePublicIps': 'false',
'pipelineConfig': 'pipeline_config/pgp_comm_apps.properties',
"currentTms": '"2022-06-28 10:00:00"',
'labels': {},
'workerMachineType': 'n1-standard-2',
'env': 'dev'
},
dag=dag,)
task