fix: airflow's BashOperator's cwd cannot be templated
parent
06c76f0b65
commit
7d898c6297
|
@ -16,8 +16,11 @@ The following parameters are available:
|
||||||
|
|
||||||
* `input`: location of the CSV input file
|
* `input`: location of the CSV input file
|
||||||
* `beam_etl_path`: location of the apache beam pipeline
|
* `beam_etl_path`: location of the apache beam pipeline
|
||||||
* `dbt_path`: location of the dbt project
|
|
||||||
* `products_table`: products_table table name
|
* `products_table`: products_table table name
|
||||||
|
|
||||||
I decided not to configure the rest of the table locations because that makes
|
I decided not to configure the rest of the table locations because that makes
|
||||||
more sense to be defined in DBT.
|
more sense to be defined in DBT.
|
||||||
|
|
||||||
|
Ideally, I would parametrize the dbt path as well. But the `cwd` parameter of
|
||||||
|
`BashOperator` is not a template and implementing that is not worth it for such
|
||||||
|
a minor improvement.
|
||||||
|
|
|
@ -23,7 +23,6 @@ CSV_FNAME = (
|
||||||
CONFIG = {
|
CONFIG = {
|
||||||
"input": f"{ HOME }/gcs/data/{ CSV_FNAME }",
|
"input": f"{ HOME }/gcs/data/{ CSV_FNAME }",
|
||||||
"beam_etl_path": "/etl/main.py",
|
"beam_etl_path": "/etl/main.py",
|
||||||
"dbt_path": "/dbt",
|
|
||||||
"products_table": "sustainability_score.products",
|
"products_table": "sustainability_score.products",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,7 +33,7 @@ def dbt(cmd: str, attach_dag: DAG) -> BashOperator:
|
||||||
dag=attach_dag,
|
dag=attach_dag,
|
||||||
task_id=f"dbt_{ cmd }",
|
task_id=f"dbt_{ cmd }",
|
||||||
bash_command=f"dbt { cmd }",
|
bash_command=f"dbt { cmd }",
|
||||||
cwd="{{ params.dbt_path }}",
|
cwd="/dbt",
|
||||||
env={
|
env={
|
||||||
"POSTGRES_HOST": "{{ conn.get('pg_db').host }}",
|
"POSTGRES_HOST": "{{ conn.get('pg_db').host }}",
|
||||||
"POSTGRES_USER": "{{ conn.get('pg_db').login }}",
|
"POSTGRES_USER": "{{ conn.get('pg_db').login }}",
|
||||||
|
|
Loading…
Reference in New Issue