docker-compose.yaml

version: "2.1"

x-airflow-service: &airflow-service
  image: puckel/docker-airflow:1.10.9
  env_file: airflow_env
  volumes:
    - ./dags:/usr/local/airflow/dags
    - ./sql:/usr/local/airflow/sql
    - ./scripts:/usr/local/airflow/scripts
    - ./dataset:/usr/local/airflow/dataset
    - ./creds:/usr/local/airflow/creds
    - ./requirements.txt:/requirements.txt
    #- ./creds/kaggle.json:/usr/local/airflow/.kaggle/kaggle.json
  restart: always
  environment:
    - LOAD_EX=n
    - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
    - EXECUTOR=Celery
    - POSTGRES_USER=airflow
    - POSTGRES_PASSWORD=airflow
    - POSTGRES_DB=airflow

services:
  redis:
    image: 'redis:5.0.5'
    restart: always


  postgres:
    image: postgres:9.6
    restart: always
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
      - PGDATA=/var/lib/postgresql/data/pgdata
    volumes:
      - ./pgdata:/var/lib/postgresql/data/pgdata

  webserver:
    <<: *airflow-service
    depends_on:
      - postgres
      - redis
    ports:
      - "80:8080"
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  flower:
    <<: *airflow-service
    depends_on:
      - redis
    ports:
      - "5555:5555"
    command: flower

  scheduler:
    <<: *airflow-service
    depends_on:
      - webserver
    command: scheduler

  worker:
    <<: *airflow-service
    depends_on:
      - scheduler
    command: worker