From b5ab3ca687255756a50be84392118c4fa1a28aee Mon Sep 17 00:00:00 2001
From: Ricard Illa <ricard@trkkn.com>
Date: Wed, 21 Jun 2023 16:02:12 +0200
Subject: [PATCH] feat: added files for containerized airflow

---
 .gitignore                             |  1 +
 pipeline/docker-compose.yml            | 37 ++++++++++++++++++++++++++
 pipeline/scripts/airflow-entrypoint.sh |  8 ++++++
 pipeline/scripts/airflow-init.sh       |  9 +++++++
 4 files changed, 55 insertions(+)
 create mode 100644 pipeline/docker-compose.yml
 create mode 100755 pipeline/scripts/airflow-entrypoint.sh
 create mode 100755 pipeline/scripts/airflow-init.sh

diff --git a/.gitignore b/.gitignore
index 763513e..50a29b0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 .ipynb_checkpoints
+pipeline/state
diff --git a/pipeline/docker-compose.yml b/pipeline/docker-compose.yml
new file mode 100644
index 0000000..269834a
--- /dev/null
+++ b/pipeline/docker-compose.yml
@@ -0,0 +1,37 @@
+---
+version: "3.8"
+
+x-airflow-common:
+  &airflow-common
+    image: us-docker.pkg.dev/cloud-airflow-releaser/airflow-worker-scheduler-2-5-1/airflow-worker-scheduler-2-5-1:composer-2.3.1-airflow-2-5-1
+    entrypoint: /usr/local/bin/airflow-entrypoint.sh
+    volumes:
+      - ./scripts/airflow-init.sh:/usr/local/bin/airflow-init.sh:ro
+      - ./scripts/airflow-entrypoint.sh:/usr/local/bin/airflow-entrypoint.sh:ro
+      - ../data:/home/airflow/gcs/data:ro
+      - ./state/airflow-data:/home/airflow/airflow
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+
+services:
+
+  airflow-init:
+    <<: *airflow-common
+    restart: "no"
+    entrypoint: /usr/local/bin/airflow-init.sh
+
+  airflow-scheduler:
+    <<: *airflow-common
+    command: airflow scheduler
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-webserver:
+    <<: *airflow-common
+    command: airflow webserver
+    ports:
+      - 8080:8080
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
diff --git a/pipeline/scripts/airflow-entrypoint.sh b/pipeline/scripts/airflow-entrypoint.sh
new file mode 100755
index 0000000..b2c7bc9
--- /dev/null
+++ b/pipeline/scripts/airflow-entrypoint.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -xe
+
+[ -f composer_requirements.txt ] && pip3 install --upgrade -r composer_requirements.txt
+[ -f dev_requirements.txt ] && pip3 install -r dev_requirements.txt
+
+exec "$@"
diff --git a/pipeline/scripts/airflow-init.sh b/pipeline/scripts/airflow-init.sh
new file mode 100755
index 0000000..b6aa655
--- /dev/null
+++ b/pipeline/scripts/airflow-init.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -xe
+
+sudo chown airflow:airflow airflow
+airflow db init
+
+# Allow non-authenticated access to UI for Airflow 2.*
+sed -i.bak -E 's/^(# )?(AUTH_ROLE_PUBLIC) = .*$/\2 = "Admin"/' /home/airflow/airflow/webserver_config.py