feat: added files for containerized airflow

.gitignore

@@ -1 +1,2 @@
 .ipynb_checkpoints
+pipeline/state

pipeline/docker-compose.yml

@@ -0,0 +1,37 @@
+---
+version: "3.8"
+
+x-airflow-common:
+  &airflow-common
+    image: us-docker.pkg.dev/cloud-airflow-releaser/airflow-worker-scheduler-2-5-1/airflow-worker-scheduler-2-5-1:composer-2.3.1-airflow-2-5-1
+    entrypoint: /usr/local/bin/airflow-entrypoint.sh
+    volumes:
+      - ./scripts/airflow-init.sh:/usr/local/bin/airflow-init.sh:ro
+      - ./scripts/airflow-entrypoint.sh:/usr/local/bin/airflow-entrypoint.sh:ro
+      - ../data:/home/airflow/gcs/data:ro
+      - ./state/airflow-data:/home/airflow/airflow
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+
+services:
+
+  airflow-init:
+    <<: *airflow-common
+    restart: "no"
+    entrypoint: /usr/local/bin/airflow-init.sh
+
+  airflow-scheduler:
+    <<: *airflow-common
+    command: airflow scheduler
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-webserver:
+    <<: *airflow-common
+    command: airflow webserver
+    ports:
+      - 8080:8080
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully

pipeline/scripts/airflow-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -xe
+
+[ -f composer_requirements.txt ] && pip3 install --upgrade -r composer_requirements.txt
+[ -f dev_requirements.txt ] && pip3 install -r dev_requirements.txt
+
+exec "$@"

pipeline/scripts/airflow-init.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -xe
+
+sudo chown airflow:airflow airflow
+airflow db init
+
+# Allow non-authenticated access to UI for Airflow 2.*
+sed -i.bak -E 's/^(# )?(AUTH_ROLE_PUBLIC) = .*$/\2 = "Admin"/' /home/airflow/airflow/webserver_config.py