fix: airflow's BashOperator's cwd cannot be templated

main
Ricard Illa 2023-06-26 10:04:28 +02:00
parent 06c76f0b65
commit 7d898c6297
2 changed files with 5 additions and 3 deletions

View File

@ -16,8 +16,11 @@ The following parameters are available:
* `input`: location of the CSV input file
* `beam_etl_path`: location of the apache beam pipeline
* `dbt_path`: location of the dbt project
* `products_table`: products_table table name
I decided not to configure the rest of the table locations because that makes
more sense to be defined in DBT.
Ideally, I would parametrize the dbt path as well. But the `cwd` parameter of
`BashOperator` is not a template and implementing that is not worth it for such
a minor improvement.

View File

@ -23,7 +23,6 @@ CSV_FNAME = (
CONFIG = {
"input": f"{ HOME }/gcs/data/{ CSV_FNAME }",
"beam_etl_path": "/etl/main.py",
"dbt_path": "/dbt",
"products_table": "sustainability_score.products",
}
@ -34,7 +33,7 @@ def dbt(cmd: str, attach_dag: DAG) -> BashOperator:
dag=attach_dag,
task_id=f"dbt_{ cmd }",
bash_command=f"dbt { cmd }",
cwd="{{ params.dbt_path }}",
cwd="/dbt",
env={
"POSTGRES_HOST": "{{ conn.get('pg_db').host }}",
"POSTGRES_USER": "{{ conn.get('pg_db').login }}",