Merge pull request #5 from viktor-pecheniuk/vpec/airflow

+ added airflow

.gitignore

@@ -1,3 +1,10 @@
 .idea
 target
-.DS_Store
+.DS_Store
+
+# python section
+*__pycache__
+*.py[cod]
+*$py.class
+*.pyc
+

README.md

@@ -21,7 +21,7 @@ Bigkube is about big data and Spark local development automation - automated dep
 #### Before deployment 
 
 1. Make sure minikube has --cpus=4 --memory=8192 flags set. Make sure the appropriate vm-driver is set (use ```minikube config set```).
-2. Run ```sbt assmbly``` repo's base dir. 
+2. Run ```sbt assembly``` repo's base dir. 
 
 #### Deployment steps
 
@@ -47,8 +47,19 @@ Note: bigkube.sh resolves all service accounts issues, secrets, config maps and
 
 Alongside with ```kubectl delete -f file.yaml``` you can use:
 1. ```./bigkube.sh --delete``` - deletes all bigkube infrastructure
-2. ```./bigkube.sh --spark-drop``` - deletes helmed spark operator 
+2. ```./bigkube.sh --spark-drop``` - deletes helmed spark operator
 
+---
+
+### **Airflow** inside of the bigkube with custom Spark Operator
+
+1. ```cd deployment```
+2. You need just run ```./bigkube.sh --airflow-init``` and all necessary **airflow** staff should be up and running
+3. ```minikube service airflow``` will open your browser with Airflow UI
+
+For more details, please visit this repository [Airflow](https://github.com/viktor-pecheniuk/airflow-sandbox)
+
+---
 
 #### Acknowledgments
 

airflow/.dockerignore

@@ -0,0 +1,4 @@
+**/.dockerignore
+**/.git
+**/.gitignore
+**/__pycache__

airflow/build/Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.7
+MAINTAINER "[email protected]"
+
+# Airflow setup
+ARG AIRFLOW_VERSION=1.10.2
+ARG JSON_LOGGER_VERSION=0.1.10
+ARG PROMETHEUS_CLI_VERSION=0.5.0
+ARG K8S_VERSION=9.0.0
+
+ENV AIRFLOW_HOME=/app/airflow
+ENV SLUGIFY_USES_TEXT_UNIDECODE=yes
+ENV PYTHONPATH=$AIRFLOW_HOME
+
+RUN apt-get update \
+  && apt-get install -y supervisor vim \
+  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+  && rm -rf /var/lib/apt/lists/* \
+  && pip install apache-airflow[postgres]==${AIRFLOW_VERSION} \
+  && pip install python-json-logger==${JSON_LOGGER_VERSION} \
+  && pip install prometheus-client==${PROMETHEUS_CLI_VERSION} \
+  && pip install kubernetes==${K8S_VERSION} \
+  && pip install psycopg2
+
+# replace owns configs
+COPY ./conf/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY ./conf/airflow.cfg $AIRFLOW_HOME/airflow.cfg
+
+COPY . $AIRFLOW_HOME/
+RUN rm -rf $AIRFLOW_HOME/conf
+
+# run tests before deploy
+RUN airflow initdb \
+  && cd $AIRFLOW_HOME && python -m unittest tests/unit/*
+
+EXPOSE 5001
+
+CMD ["/usr/bin/supervisord"]