x-meltano-image: &meltano-image
  image: meltano-poc:v1.2 
  build: .
  platform: linux/amd64 # Required for compatibility with ECS Fargate

x-meltano-env: &meltano-env
  MELTANO_DATABASE_URI: ${MELTANO_DATABASE_URI}
  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} 
  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
  TARGET_REDSHIFT_CLUSTER_IDENTIFIER: ${TARGET_REDSHIFT_CLUSTER_IDENTIFIER} 
  TARGET_REDSHIFT_PASSWORD: ${TARGET_REDSHIFT_PASSWORD}
  TARGET_REDSHIFT_PROD_PASSWORD: ${TARGET_REDSHIFT_PROD_PASSWORD} 
  TAP_MYSQL_PASSWORD: ${TAP_MYSQL_PASSWORD} 
  TAP_MYSQL_PROD_PASSWORD: ${TAP_MYSQL_PROD_PASSWORD}

x-airflow-env: &airflow-env
  AIRFLOW__CORE__EXECUTOR: ${AIRFLOW__CORE__EXECUTOR}
  AIRFLOW__CORE__SQL_ALCHEMY_CONN: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN} 
  AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} 
  AIRFLOW__LOGGING__LOGGING_LEVEL: ${AIRFLOW__LOGGING__LOGGING_LEVEL}
  AIRFLOW__LOGGING__FAB_LOGGING_LEVEL: ${AIRFLOW__LOGGING__FAB_LOGGING_LEVEL}
  AIRFLOW__LOGGING__COLORED_CONSOLE_LOG: ${AIRFLOW__LOGGING__COLORED_CONSOLE_LOG}
  AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES} 
  AIRFLOW_HOME: ${AIRFLOW_HOME}

services:
  meltano:
    <<: *meltano-image
    command: dragon
    environment:
      <<: *meltano-env
    volumes:
      - meltano_elt_logs_data:/project/.meltano/logs/elt
    expose:
      - 5000
    ports:
      - 5000:5000
    depends_on:
      - meltano-system-db
    networks:
      - meltano
    restart: on-failure

  meltano-system-db:
    image: postgres
    environment:
      POSTGRES_PASSWORD: postgres # CHANGE ME
      POSTGRES_DB: meltano
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - meltano_postgresql_data:/var/lib/postgresql/data
    expose:
      - 5432
    networks:
      - meltano
      - airflow
    restart: unless-stopped

  airflow-scheduler:
    <<: *meltano-image
    # command: invoke airflow scheduler
    # command: /project/.meltano/utilities/airflow/venv/bin/airflow scheduler
    entrypoint: ["/project/.meltano/utilities/airflow/venv/bin/airflow"]
    command: ["scheduler"]
    environment:
      <<: 
        - *airflow-env
        - *meltano-env
    volumes:
      - meltano_elt_logs_data:/project/.meltano/logs/elt
      - ./orchestrate/airflow/airflow.cfg:/project/orchestrate/airflow/airflow.cfg:ro
    expose:
      - 8793
    depends_on:
      - meltano-system-db
      - airflow-metadata-db
    networks:
      - meltano
      - airflow
    restart: unless-stopped
  
  airflow-webserver:
    <<: *meltano-image
    # command: invoke airflow webserver
    # command: /project/.meltano/utilities/airflow/venv/bin/airflow webserver
    entrypoint: ["/project/.meltano/utilities/airflow/venv/bin/airflow"]
    command: ["webserver"]
    environment:
      <<: 
        - *airflow-env
        - *meltano-env
    volumes:
      - meltano_elt_logs_data:/project/.meltano/logs/elt
      - ./orchestrate/airflow/airflow.cfg:/project/orchestrate/airflow/airflow.cfg:ro
    expose:
      - 8080
    ports:
      - 8080:8080
    depends_on:
      - meltano-system-db
      - airflow-metadata-db
    networks:
      - meltano
      - airflow
    restart: unless-stopped
  
  airflow-metadata-db:
    image: postgres
    environment:
      POSTGRES_PASSWORD: postgres # CHANGE ME
      POSTGRES_DB: airflow
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - airflow_postgresql_data:/var/lib/postgresql/data
    expose:
      - 5432
    ports: # To access locally via DataGrip; changed to 5434 forwarding port instead of default postgres 5432 to avoid port conflict with meltano db
      - 5434:5432
    networks:
      - meltano
      - airflow
    restart: unless-stopped

  airflow-init:
    <<: *meltano-image
    entrypoint: ["/project/.meltano/utilities/airflow/venv/bin/airflow"]
    command: ["db", "init"]
    environment:
      <<: 
        - *airflow-env
        - *meltano-env
    depends_on:
      - airflow-metadata-db
    networks:
      - airflow



networks:
  meltano:
  airflow:

volumes:
  meltano_postgresql_data:
    driver: local
  meltano_elt_logs_data:
    driver: local
  airflow_postgresql_data:
    driver: local
