fix: airflow's BashOperator's cwd cannot be templated
parent
06c76f0b65
commit
7d898c6297
|
@ -16,8 +16,11 @@ The following parameters are available:
|
|||
|
||||
* `input`: location of the CSV input file
|
||||
* `beam_etl_path`: location of the apache beam pipeline
|
||||
* `dbt_path`: location of the dbt project
|
||||
* `products_table`: products_table table name
|
||||
|
||||
I decided not to configure the rest of the table locations because that makes
|
||||
more sense to be defined in DBT.
|
||||
|
||||
Ideally, I would parametrize the dbt path as well. But the `cwd` parameter of
|
||||
`BashOperator` is not a template and implementing that is not worth it for such
|
||||
a minor improvement.
|
||||
|
|
|
@ -23,7 +23,6 @@ CSV_FNAME = (
|
|||
CONFIG = {
|
||||
"input": f"{ HOME }/gcs/data/{ CSV_FNAME }",
|
||||
"beam_etl_path": "/etl/main.py",
|
||||
"dbt_path": "/dbt",
|
||||
"products_table": "sustainability_score.products",
|
||||
}
|
||||
|
||||
|
@ -34,7 +33,7 @@ def dbt(cmd: str, attach_dag: DAG) -> BashOperator:
|
|||
dag=attach_dag,
|
||||
task_id=f"dbt_{ cmd }",
|
||||
bash_command=f"dbt { cmd }",
|
||||
cwd="{{ params.dbt_path }}",
|
||||
cwd="/dbt",
|
||||
env={
|
||||
"POSTGRES_HOST": "{{ conn.get('pg_db').host }}",
|
||||
"POSTGRES_USER": "{{ conn.get('pg_db').login }}",
|
||||
|
|
Loading…
Reference in New Issue