Using Official Airflow Image apache/airflow:1.10.11

.dockerignore

@@ -1 +1,6 @@
 .git
+.github
+*.yml
+Makefile
+README.md
+LICENSE

.github/workflows/ci.yml

@@ -6,13 +6,13 @@ on:
       - master
   pull_request:
     branches:
-    - master
+      - master
 
 jobs:
   ci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
       - run: docker build -t "${PWD##*/}" .
-      - run: docker run "${PWD##*/}" python -V
-      - run: docker run "${PWD##*/}" version
+      - run: docker run "${PWD##*/}" python -V | grep '3.6'
+      - run: docker run "${PWD##*/}" version | grep '1.10.11'

.gitignore

@@ -24,3 +24,6 @@ sftp-config.json
 
 # Python
 __pycache__
+
+# IDE
+.idea/

Dockerfile

@@ -1,86 +1,39 @@
-# VERSION 1.10.9
-# AUTHOR: Matthieu "Puckel_" Roisil
+# VERSION 1.10.11
+# AUTHOR: Swapnil Gusani
 # DESCRIPTION: Basic Airflow container
-# BUILD: docker build --rm -t puckel/docker-airflow .
-# SOURCE: https://github.com/puckel/docker-airflow
-
-FROM python:3.7-slim-buster
-LABEL maintainer="Puckel_"
-
-# Never prompt the user for choices on installation/configuration of packages
-ENV DEBIAN_FRONTEND noninteractive
-ENV TERM linux
-
-# Airflow
-ARG AIRFLOW_VERSION=1.10.9
-ARG AIRFLOW_USER_HOME=/usr/local/airflow
-ARG AIRFLOW_DEPS=""
-ARG PYTHON_DEPS=""
-ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
-
-# Define en_US.
-ENV LANGUAGE en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
-ENV LC_CTYPE en_US.UTF-8
-ENV LC_MESSAGES en_US.UTF-8
-
-# Disable noisy "Handling signal" log messages:
-# ENV GUNICORN_CMD_ARGS --log-level WARNING
-
-RUN set -ex \
-    && buildDeps=' \
-        freetds-dev \
-        libkrb5-dev \
-        libsasl2-dev \
-        libssl-dev \
-        libffi-dev \
-        libpq-dev \
-        git \
-    ' \
-    && apt-get update -yqq \
-    && apt-get upgrade -yqq \
-    && apt-get install -yqq --no-install-recommends \
-        $buildDeps \
-        freetds-bin \
-        build-essential \
-        default-libmysqlclient-dev \
-        apt-utils \
-        curl \
-        rsync \
-        netcat \
-        locales \
-    && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
-    && locale-gen \
-    && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
-    && useradd -ms /bin/bash -d ${AIRFLOW_USER_HOME} airflow \
-    && pip install -U pip setuptools wheel \
-    && pip install pytz \
-    && pip install pyOpenSSL \
-    && pip install ndg-httpsclient \
-    && pip install pyasn1 \
-    && pip install apache-airflow[crypto,celery,postgres,hive,jdbc,mysql,ssh${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \
-    && pip install 'redis==3.2' \
-    && if [ -n "${PYTHON_DEPS}" ]; then pip install ${PYTHON_DEPS}; fi \
-    && apt-get purge --auto-remove -yqq $buildDeps \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean \
-    && rm -rf \
-        /var/lib/apt/lists/* \
-        /tmp/* \
-        /var/tmp/* \
-        /usr/share/man \
-        /usr/share/doc \
-        /usr/share/doc-base
-
-COPY script/entrypoint.sh /entrypoint.sh
-COPY config/airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg
-
-RUN chown -R airflow: ${AIRFLOW_USER_HOME}
-
-EXPOSE 8080 5555 8793
+# BUILD: docker build --rm -t swapniel99/docker-airflow .
+# SOURCE: https://github.com/swapniel99/docker-airflow
+
+FROM apache/airflow:1.10.11
+
+USER root
+
+##System upgrade
+#RUN set -ex \
+#    # Upgrade packages
+#    && apt-get update -yqq \
+#    && apt-get upgrade -yqq \
+#    && pip install --upgrade pip \
+#    # Cleanup
+#    && apt-get autoremove -yqq --purge \
+#    && apt-get clean \
+#    && rm -rf \
+#        /var/lib/apt/lists/* \
+#        /root/.cache/pip \
+#        /tmp/* \
+#        /var/tmp/* \
+#        /usr/share/man \
+#        /usr/share/doc \
+#        /usr/share/doc-base
+
+# Copy Config Files
+COPY script/entrypoint_wrapper.sh /entrypoint_wrapper.sh
+COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+# Make airflow user owner
+RUN chown -R airflow: ${AIRFLOW_HOME}
+
+EXPOSE 5555 8793
 
 USER airflow
-WORKDIR ${AIRFLOW_USER_HOME}
-ENTRYPOINT ["/entrypoint.sh"]
-CMD ["webserver"]
+ENTRYPOINT ["/entrypoint_wrapper.sh"]

Makefile

@@ -0,0 +1,70 @@
+SERVICE = "scheduler"
+TITLE = "Airflow Containers"
+FILE = "docker-compose-LocalExecutor.yml"
+
+.PHONY: run
+
+build:
+	docker build -t docker-airflow .
+	docker tag docker-airflow-ascent:latest swapniel99/docker-airflow:latest
+
+up:
+	@echo "Starting $(TITLE)"
+	docker-compose -f $(FILE) up -d
+
+upf:
+	@echo "Starting $(TITLE)"
+	docker-compose -f $(FILE) up
+
+down:
+	@echo "Stopping $(TITLE)"
+	docker-compose -f $(FILE) down
+
+start:
+	@echo "Starting $(TITLE)"
+	docker-compose -f $(FILE) start
+
+stop:
+	@echo "Stopping $(TITLE)"
+	docker-compose -f $(FILE) stop
+
+restart:
+	@echo "Restarting $(TITLE)"
+	docker-compose -f $(FILE) restart
+
+downup: down print-newline up
+
+run:
+	docker-compose -f $(FILE) run --rm --entrypoint='' $(SERVICE) bash
+
+runr:
+	docker-compose -f $(FILE) run --rm --entrypoint='' -u root $(SERVICE) bash
+
+bash:
+	docker-compose -f $(FILE) exec $(SERVICE) bash
+
+bashr:
+	docker-compose -f $(FILE) exec -u root $(SERVICE) bash
+
+logs:
+	docker-compose -f $(FILE) logs --tail 50 --follow $(SERVICE)
+
+conf:
+	docker-compose -f $(FILE) config
+
+initdb:
+	docker-compose -f $(FILE) run --rm $(SERVICE) initdb
+
+upgradedb:
+	docker-compose -f $(FILE) run --rm $(SERVICE) upgradedb
+
+resetdb:
+	docker-compose -f $(FILE) run --rm $(SERVICE) resetdb
+
+rbacadmin:
+	# Change user details and password after login if using RBAC mode.
+	docker-compose -f $(FILE) exec webserver airflow create_user -r Admin -u admin -e admin@example.com -f Firstname -l Lastname -p admin123
+
+print-newline:
+	@echo ""
+	@echo ""

README.md

@@ -1,16 +1,20 @@
 # docker-airflow
-[![CI status](https://github.com/puckel/docker-airflow/workflows/CI/badge.svg?branch=master)](https://github.com/puckel/docker-airflow/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush)
-[![Docker Build status](https://img.shields.io/docker/build/puckel/docker-airflow?style=plastic)](https://hub.docker.com/r/puckel/docker-airflow/tags?ordering=last_updated)
+[![CI status](https://github.com/swapniel99/docker-airflow/workflows/CI/badge.svg?branch=master)](https://github.com/swapniel99/docker-airflow/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush)
+[![Docker Build status](https://img.shields.io/docker/build/swapniel99/docker-airflow?style=plastic)](https://hub.docker.com/r/swapniel99/docker-airflow/tags?ordering=last_updated)
 
-[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/puckel/docker-airflow/)
-[![Docker Pulls](https://img.shields.io/docker/pulls/puckel/docker-airflow.svg)]()
-[![Docker Stars](https://img.shields.io/docker/stars/puckel/docker-airflow.svg)]()
+[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/swapniel99/docker-airflow/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/swapniel99/docker-airflow.svg)]()
+[![Docker Stars](https://img.shields.io/docker/stars/swapniel99/docker-airflow.svg)]()
 
-This repository contains **Dockerfile** of [apache-airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/)'s [automated build](https://registry.hub.docker.com/u/puckel/docker-airflow/) published to the public [Docker Hub Registry](https://registry.hub.docker.com/).
+(Readme not fully updated.)
+
+Courtesy: [puckel/docker-airflow](https://github.com/puckel/docker-airflow)
+
+This repository contains **Dockerfile** of [apache-airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/)'s [automated build](https://registry.hub.docker.com/u/swapniel99/docker-airflow/) published to the public [Docker Hub Registry](https://registry.hub.docker.com/).
 
 ## Informations
 
-* Based on Python (3.7-slim-buster) official Image [python:3.7-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue
+* Based on Python (3.6-slim-buster) official Image [apache/airflow:1.10.11](https://hub.docker.com/r/apache/airflow) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue
 * Install [Docker](https://www.docker.com/)
 * Install [Docker Compose](https://docs.docker.com/compose/install/)
 * Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow)
@@ -19,26 +23,26 @@ This repository contains **Dockerfile** of [apache-airflow](https://github.com/a
 
 Pull the image from the Docker repository.
 
-    docker pull puckel/docker-airflow
+    docker pull swapniel99/docker-airflow
 
 ## Build
 
 Optionally install [Extra Airflow Packages](https://airflow.incubator.apache.org/installation.html#extra-package) and/or python dependencies at build time :
 
-    docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow .
-    docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow .
+    docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t swapniel99/docker-airflow .
+    docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t swapniel99/docker-airflow .
 
 or combined
 
-    docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow .
+    docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t swapniel99/docker-airflow .
 
-Don't forget to update the airflow images in the docker-compose files to puckel/docker-airflow:latest.
+Don't forget to update the airflow images in the docker-compose files to swapniel99/docker-airflow:latest.
 
 ## Usage
 
 By default, docker-airflow runs Airflow with **SequentialExecutor** :
 
-    docker run -d -p 8080:8080 puckel/docker-airflow webserver
+    docker run -d -p 8080:8080 swapniel99/docker-airflow webserver
 
 If you want to run another executor, use the other docker-compose.yml files provided in this repository.
 
@@ -54,7 +58,7 @@ NB : If you want to have DAGs example loaded (default=False), you've to set the
 
 `LOAD_EX=n`
 
-    docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow
+    docker run -d -p 8080:8080 -e LOAD_EX=y swapniel99/docker-airflow
 
 If you want to use Ad hoc query, make sure you've configured connections:
 Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) :
@@ -65,7 +69,7 @@ Go to Admin -> Connections and Edit "postgres_default" set this values (equivale
 
 For encrypted connection passwords (in Local or Celery Executor), you must have the same fernet_key. By default docker-airflow generates the fernet_key at startup, you have to set an environment variable in the docker-compose (ie: docker-compose-LocalExecutor.yml) file to set the same key accross containers. To generate a fernet_key :
 
-    docker run puckel/docker-airflow python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)"
+    docker run swapniel99/docker-airflow python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)"
 
 ## Configuring Airflow
 
@@ -111,16 +115,16 @@ This can be used to scale to a multi node setup using docker swarm.
 
 If you want to run other airflow sub-commands, such as `list_dags` or `clear` you can do so like this:
 
-    docker run --rm -ti puckel/docker-airflow airflow list_dags
+    docker run --rm -ti swapniel99/docker-airflow airflow list_dags
 
 or with your docker-compose set up like this:
 
     docker-compose -f docker-compose-CeleryExecutor.yml run --rm webserver airflow list_dags
 
 You can also use this to run a bash shell or any other command in the same environment that airflow would be run in:
 
-    docker run --rm -ti puckel/docker-airflow bash
-    docker run --rm -ti puckel/docker-airflow ipython
+    docker run --rm -ti swapniel99/docker-airflow bash
+    docker run --rm -ti swapniel99/docker-airflow ipython
 
 # Simplified SQL database configuration using PostgreSQL
 
@@ -163,7 +167,7 @@ it explicitly:
 | `REDIS_PROTO`     | `redis://`    | Protocol                       |
 | `REDIS_HOST`      | `redis`       | Redis server host              |
 | `REDIS_PORT`      | `6379`        | Redis server port              |
-| `REDIS_PASSWORD`  | empty         | If Redis is password protected |
+| `REDIS_PASSWORD`  | `redispass`   | If Redis is password protected |
 | `REDIS_DBNUM`     | `1`           | Database number                |
 
 You can also use those variables to adapt your compose file to match an existing Redis instance managed elsewhere.