diff --git a/.gitignore b/.gitignore index 763513e..50a29b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .ipynb_checkpoints +pipeline/state diff --git a/pipeline/docker-compose.yml b/pipeline/docker-compose.yml new file mode 100644 index 0000000..269834a --- /dev/null +++ b/pipeline/docker-compose.yml @@ -0,0 +1,37 @@ +--- +version: "3.8" + +x-airflow-common: + &airflow-common + image: us-docker.pkg.dev/cloud-airflow-releaser/airflow-worker-scheduler-2-5-1/airflow-worker-scheduler-2-5-1:composer-2.3.1-airflow-2-5-1 + entrypoint: /usr/local/bin/airflow-entrypoint.sh + volumes: + - ./scripts/airflow-init.sh:/usr/local/bin/airflow-init.sh:ro + - ./scripts/airflow-entrypoint.sh:/usr/local/bin/airflow-entrypoint.sh:ro + - ../data:/home/airflow/gcs/data:ro + - ./state/airflow-data:/home/airflow/airflow + environment: + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + +services: + + airflow-init: + <<: *airflow-common + restart: "no" + entrypoint: /usr/local/bin/airflow-init.sh + + airflow-scheduler: + <<: *airflow-common + command: airflow scheduler + depends_on: + airflow-init: + condition: service_completed_successfully + + airflow-webserver: + <<: *airflow-common + command: airflow webserver + ports: + - 8080:8080 + depends_on: + airflow-init: + condition: service_completed_successfully diff --git a/pipeline/scripts/airflow-entrypoint.sh b/pipeline/scripts/airflow-entrypoint.sh new file mode 100755 index 0000000..b2c7bc9 --- /dev/null +++ b/pipeline/scripts/airflow-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -xe + +[ -f composer_requirements.txt ] && pip3 install --upgrade -r composer_requirements.txt +[ -f dev_requirements.txt ] && pip3 install -r dev_requirements.txt + +exec "$@" diff --git a/pipeline/scripts/airflow-init.sh b/pipeline/scripts/airflow-init.sh new file mode 100755 index 0000000..b6aa655 --- /dev/null +++ b/pipeline/scripts/airflow-init.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -xe + +sudo chown airflow:airflow airflow +airflow db init + +# Allow non-authenticated access to UI for Airflow 2.* +sed -i.bak -E 's/^(# )?(AUTH_ROLE_PUBLIC) = .*$/\2 = "Admin"/' /home/airflow/airflow/webserver_config.py