Dynamic glue operators in task group on runtime - airflow

Question

I am trying to create a dag for a ETL solution where we have 4 stages to the the pipeline. We have source_2_s3,staging,dim_population,fact_loads
Each of these stages have multiple glue jobs in them that can run in parallel. I am trying to run these stages as taskgroups and passing a list of jobnames as parameters so we can create the glue operators on runtime but I am only getting one gluejob for each task group? What am I doing wrong. My code is as follows:
The yaml file contains all jobs for each task group and the gluejob script location as keyvalue pair
from os import path
from airflow import DAG
from airflow.providers.amazon.aws.operators.glue import AwsGlueJobOperator
import yaml
from airflow.models import Variable
from airflow.utils.dates import days_ago
from airflow.operators.dummy import DummyOperator
from airflow.utils.task_group import TaskGroup
AwsGlueJobOperator.ui_color = "#F3D5C5"
DAG_FOLDER_PATH = path.dirname(__file__)
ENVIRONMENT = Variable.get("environment")
CONFIG_FILE_NAME = f"dwh_config_{ENVIRONMENT}.yml"
with open(path.join(DAG_FOLDER_PATH, CONFIG_FILE_NAME), 'r') as fl:
cfg = yaml.safe_load(fl)["dwh"]
aws_conn_id = cfg["aws"]["aws_conn_id"]
glue_scripts_bucket_name = cfg["glue"]["app"]["bucket_name"]
src1_tg = cfg["glue"]["app"]["src1_taskgroup"]
src2_tg = cfg["glue"]["app"]["src2_taskgroup"]
staging_tg = cfg["glue"]["app"]["staging_taskgroup"]
dim_tg = cfg["glue"]["app"]["dim_taskgroup"]
fact_tg = cfg["glue"]["app"]["fact_taskgroup"]
def glue_operator(list_of_jobs):
for jobname, script_path in list_of_jobs.items():
return AwsGlueJobOperator(
task_id=jobname,
dag=dag,
aws_conn_id='aws_default',
region_name='eu-west-2',
job_name=jobname,
script_location=path.join("s3://", glue_scripts_bucket_name,
script_path)
)
with DAG(dag_id="CLIENTUSAGE_DAILY_DWH_REFRESH", schedule_interval="#daily", start_date=days_ago(1),
tags=['Clientusage']) as dag:
batch_start_job = DummyOperator(task_id="START")
batch_close_job = DummyOperator(task_id="END")
with TaskGroup(group_id='src1_tg ') as src1_taskgroup:
list = glue_operator(src1_tg)
with TaskGroup(group_id='src2_tg') as src2_taskgroup:
list = glue_operator(src2_tg)
with TaskGroup(group_id='staging_tg') as staging_taskgroup:
list = glue_operator(staging_tg)
with TaskGroup(group_id='dim_tg') as dim_taskgroup:
list = glue_operator(dim_tg)
with TaskGroup(group_id='fact_tg') as fact_taskgroup:
list = glue_operator(fact_tg)
batch_start_job >> [src1_taskgroup,
src2_taskgroup] >> staging_taskgroup >> dim_taskgroup >> fact_taskgroup
'''

Answer 1

The issue was my function for generating dags. I forgot the basics of functions :D. Had to append all operators to an array before returning it.
def glue_operator(list_of_jobs):
operators = []
for jobname, script_path in list_of_jobs.items():
operators.append(AwsGlueJobOperator(
task_id=jobname,
dag=dag,
aws_conn_id='aws_default',
region_name='eu-west-2',
job_name=jobname,
script_location=path.join("s3://", glue_scripts_bucket_name,
script_path)
))
return operators