docker-compose.yaml

version: "3.7"

services:
  postgres:
    image: postgres:13.1-alpine
    hostname: postgres
    restart: always
    environment:
      POSTGRES_USER: ${PSQL_USERNAME}
      POSTGRES_PASSWORD: ${PSQL_PASSWORD}
      POSTGRES_DB: ${PSQL_DATABASE}
    shm_size: '1g'
    volumes:
      - "agr-literature-pg-data:/var/lib/postgresql/data"
      - "./postgresql.conf:/etc/postgresql.conf"
    networks:
      - agr-literature
    # ports needed by pgadmin to connect through host.docker.internal
    ports:
      - "${PSQL_PORT}:${PSQL_PORT}"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PSQL_USERNAME}"]
      interval: 10s
      timeout: 5s
      retries: 5
    command:
      postgres -c config_file=/etc/postgresql.conf -p ${PSQL_PORT}


  pgadmin:
    image: dpage/pgadmin4:5.1
    hostname: pgadmin
    restart: always
    environment:
      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD}
      PGADMIN_ACCESS_CONTROL_ALLOW_ORIGIN: "*"
      PGADMIN_LISTEN_PORT: 81
    ports:
      - "${PGADMIN_PORT}:81"
    volumes:
      - "agr-literature-pgadmin-data:/var/lib/pgadmin"
    networks:
      - agr-literature
    depends_on:
      - postgres
    extra_hosts:
      - "host.docker.internal:host-gateway"


  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
    hostname: elasticsearch
    restart: always
    environment:
      - node.name=aws-literature-es1
      - cluster.name=es-docker-cluster
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - "agr-literature-es-data:/usr/share/elasticsearch/data"
    ports:
      - "${ELASTICSEARCH_PORT}:${ELASTICSEARCH_PORT}"
    networks:
      - agr-literature

  api:
    restart: always
    build:
      context: .
      dockerfile: docker/api.dockerfile
      args:
        REG: ${REG}
        ALLIANCE_RELEASE: ${ALLIANCE_RELEASE}
    environment:
      API_PORT: "${API_PORT}"
      API_SERVER: "${API_SERVER}"
      PYTHONPATH: "/usr/local/bin/src/literature/"
      XML_PATH: "/usr/local/bin/src/literature/agr_literature_service/lit_processing/data_ingest/tmp/"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
      OKTA_CLIENT_ID: "${OKTA_CLIENT_ID}"
      OKTA_CLIENT_SECRET: "${OKTA_CLIENT_SECRET}"
      ENV_STATE: "${ENV_STATE}"
      ID_MATI_URL: "${ID_MATI_URL}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
      PERSISTENT_STORE_DB_USERNAME: "${PERSISTENT_STORE_DB_USERNAME}"
      PERSISTENT_STORE_DB_PASSWORD: "${PERSISTENT_STORE_DB_PASSWORD}"
      PERSISTENT_STORE_DB_HOST: "${PERSISTENT_STORE_DB_HOST}"
      PERSISTENT_STORE_DB_PORT: "${PERSISTENT_STORE_DB_PORT}"
      PERSISTENT_STORE_DB_NAME: "${PERSISTENT_STORE_DB_NAME}"
      RESOURCE_DESCRIPTOR_URL: "${RESOURCE_DESCRIPTOR_URL}"
      HOST: "${HOST}"
      OKTA_DOMAIN: "${OKTA_DOMAIN}"
      OKTA_API_AUDIENCE: "${OKTA_API_AUDIENCE}"
      ELASTICSEARCH_HOST: "${ELASTICSEARCH_HOST}"
      ELASTICSEARCH_PORT: "${ELASTICSEARCH_PORT}"
      ELASTICSEARCH_INDEX: "${ELASTICSEARCH_INDEX}"
      LOG_PATH: "${LOG_PATH}"
      LOG_URL: "${LOG_URL}"
      CRONTAB_EMAIL: "${CRONTAB_EMAIL}"
      SENDER_EMAIL: "${SENDER_EMAIL}"
      SENDER_PASSWORD: "${SENDER_PASSWORD}"
      REPLY_TO: "${REPLY_TO}"
      ATEAM_API_URL: "${ATEAM_API_URL}"
      SGD_API_URL: "${SGD_API_URL}"
    hostname: literature_api
    networks:
      - agr-literature
    volumes:
      - agr-logs:/logs
    ports:
      - "${API_PORT}:8080"

  xml_processing:
    restart: always
    build:
      context: .
      dockerfile: docker/xml_processing.dockerfile
      args:
        REG: ${REG}
        ALLIANCE_RELEASE: ${ALLIANCE_RELEASE}
    hostname: literature_xml_processing
    environment:
      API_PORT: "${API_PORT}"
      API_SERVER: "${API_SERVER}"
      PYTHONPATH: "/usr/local/bin/src/literature/"
      XML_PATH: "/usr/local/bin/src/literature/agr_literature_service/lit_processing/data_ingest/tmp/"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
      OKTA_CLIENT_ID: "${OKTA_CLIENT_ID}"
      OKTA_CLIENT_SECRET: "${OKTA_CLIENT_SECRET}"
      ENV_STATE: "${ENV_STATE}"
      ID_MATI_URL: "${ID_MATI_URL}"
      HOST: "${HOST}"
      OKTA_DOMAIN: "${OKTA_DOMAIN}"
      OKTA_API_AUDIENCE: "${OKTA_API_AUDIENCE}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
      RESOURCE_DESCRIPTOR_URL: "${RESOURCE_DESCRIPTOR_URL}"
      REG: "${REG}"
    networks:
      - agr-literature
    volumes:
      - agr-logs:/logs
    depends_on:
      - api

  automated_scripts:
    restart: always
    build:
      context: .
      dockerfile: docker/automated_scripts.dockerfile
      args:
        REG: ${REG}
        ALLIANCE_RELEASE: ${ALLIANCE_RELEASE}
    environment:
      PYTHONPATH: "/usr/src/app"
      XML_PATH: "/usr/src/app/agr_literature_service/lit_processing/data_ingest/tmp/"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
      OKTA_CLIENT_ID: "${OKTA_CLIENT_ID}"
      OKTA_CLIENT_SECRET: "${OKTA_CLIENT_SECRET}"
      ENV_STATE: "${ENV_STATE}"
      ID_MATI_URL: "${ID_MATI_URL}"
      HOST: "${HOST}"
      OKTA_DOMAIN: "${OKTA_DOMAIN}"
      OKTA_API_AUDIENCE: "${OKTA_API_AUDIENCE}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
      RESOURCE_DESCRIPTOR_URL: "${RESOURCE_DESCRIPTOR_URL}"
      REG: "${REG}"
      NCBI_API_KEY: "${NCBI_API_KEY}"
      LOG_PATH: "/var/log/automated_scripts/"
      LOG_URL: "${LOG_URL}"
      CRONTAB_EMAIL: "${CRONTAB_EMAIL}"
      SENDER_EMAIL: "${SENDER_EMAIL}"
      SENDER_PASSWORD: "${SENDER_PASSWORD}"
      REPLY_TO: "${REPLY_TO}"
      ELASTICSEARCH_HOST: "${ELASTICSEARCH_HOST}"
      ELASTICSEARCH_PORT: "${ELASTICSEARCH_PORT}"
      ELASTICSEARCH_INDEX: "${ELASTICSEARCH_INDEX}"
      API_PORT: "${API_PORT}"
      API_SERVER: "${API_SERVER}"
      ATEAM_API_URL: "${ATEAM_API_URL}"
      DEBEZIUM_CONNECTOR_HOST: "${DEBEZIUM_CONNECTOR_HOST}"
      DEBEZIUM_CONNECTOR_PORT: "${DEBEZIUM_CONNECTOR_PORT}"
      DEBEZIUM_INDEX_NAME: "${DEBEZIUM_INDEX_NAME}"
      DEBEZIUM_KSQLDB_HOST: "${DEBEZIUM_KSQLDB_HOST}"
      DEBEZIUM_KSQLDB_PORT: "${DEBEZIUM_KSQLDB_PORT}"
      TZ: "UTC"
      PDF2TEI_API_URL: "${PDF2TEI_API_URL}"
      PERSISTENT_STORE_DB_USERNAME: "${PERSISTENT_STORE_DB_USERNAME}"
      PERSISTENT_STORE_DB_PASSWORD: "${PERSISTENT_STORE_DB_PASSWORD}"
      PERSISTENT_STORE_DB_HOST: "${PERSISTENT_STORE_DB_HOST}"
      PERSISTENT_STORE_DB_PORT: "${PERSISTENT_STORE_DB_PORT}"
      PERSISTENT_STORE_DB_NAME: "${PERSISTENT_STORE_DB_NAME}"
    volumes:
      - "${LOG_PATH}:/var/log/automated_scripts"
      - /etc/localtime:/etc/localtime:ro
    networks:
      - agr-literature

  test_runner:
    build:
      context: .
      dockerfile: docker/test_runner.dockerfile
      args:
        REG: ${REG}
        ALLIANCE_RELEASE: ${ALLIANCE_RELEASE}
    hostname: test_runner
    environment:
      API_PORT: "${API_PORT}"
      API_SERVER: "${API_SERVER}"
      PYTHONPATH: "/usr/local/bin/src/literature/"
      XML_PATH: "/usr/local/bin/src/literature/agr_literature_service/lit_processing/tests/tmp/"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
      OKTA_CLIENT_ID: "${OKTA_CLIENT_ID}"
      OKTA_CLIENT_SECRET: "${OKTA_CLIENT_SECRET}"
      ENV_STATE: "${ENV_STATE}"
      ID_MATI_URL: "${ID_MATI_URL}"
      HOST: "${HOST}"
      OKTA_DOMAIN: "${OKTA_DOMAIN}"
      OKTA_API_AUDIENCE: "${OKTA_API_AUDIENCE}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
      ELASTICSEARCH_HOST: "${ELASTICSEARCH_HOST}"
      ELASTICSEARCH_PORT: "${ELASTICSEARCH_PORT}"
      ELASTICSEARCH_INDEX: "${ELASTICSEARCH_INDEX}"
      RESOURCE_DESCRIPTOR_URL: "${RESOURCE_DESCRIPTOR_URL}"
      REG: "${REG}"
    volumes:
      - agr-logs:/logs
      - .:/usr/local/bin/src/literature
    networks:
      - agr-literature

  dev_app:
    restart: always
    build:
      context: .
      dockerfile: docker/dev-app.dockerfile
      args:
        REG: ${REG}
        ALLIANCE_RELEASE: ${ALLIANCE_RELEASE}
    hostname: dev_app
    environment:
      API_PORT: "8080"
      API_SERVER: "localhost"
      PYTHONPATH: "/usr/local/bin/src/literature/"
      XML_PATH: "/usr/local/bin/src/literature/agr_literature_service/lit_processing/data_ingest/tmp/"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
      OKTA_CLIENT_ID: "${OKTA_CLIENT_ID}"
      OKTA_CLIENT_SECRET: "${OKTA_CLIENT_SECRET}"
      ENV_STATE: "${ENV_STATE}"
      ID_MATI_URL: "${ID_MATI_URL}"
      ATEAM_API_URL: "${ATEAM_API_URL}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
      ELASTICSEARCH_HOST: "${ELASTICSEARCH_HOST}"
      ELASTICSEARCH_PORT: "${ELASTICSEARCH_PORT}"
      ELASTICSEARCH_INDEX: "${ELASTICSEARCH_INDEX}"
      RESOURCE_DESCRIPTOR_URL: "${RESOURCE_DESCRIPTOR_URL}"
      HOST: "${HOST}"
      OKTA_DOMAIN: "${OKTA_DOMAIN}"
      OKTA_API_AUDIENCE: "${OKTA_API_AUDIENCE}"
      REG: "${REG}"
    ports:
      - "${API_PORT}:8080"
    networks:
      - agr-literature
    volumes:
      - agr-logs:/logs
      - .:/usr/local/bin/src/literature

  dbz_zookeeper:
    image: quay.io/debezium/zookeeper:${DEBEZIUM_VERSION}
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    networks:
      - agr-literature

  dbz_kafka:
    image: quay.io/debezium/kafka:${DEBEZIUM_VERSION}
    ports:
      - 9092:9092
    environment:
      - ZOOKEEPER_CONNECT=dbz_zookeeper:2181
      - KAFKA_MESSAGE_MAX_BYTES=200000000
      - KAFKA_SOCKET_REQUEST_MAX_BYTES=200000000
    networks:
      - agr-literature

  dbz_connector:
    build:
      context: .
      dockerfile: docker/debezium_connector.dockerfile
    ports:
      - 8083:8083
    environment:
      - BOOTSTRAP_SERVERS=dbz_kafka:9092
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=my_source_connect_configs
      - OFFSET_STORAGE_TOPIC=my_source_connect_offsets
      - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=false
      - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=false
    networks:
      - agr-literature

  dbz_setup:
    build:
      context: .
      dockerfile: docker/debezium_setup.dockerfile
    environment:
      ELASTICSEARCH_HOST: "${ELASTICSEARCH_HOST}"
      ELASTICSEARCH_PORT: "${ELASTICSEARCH_PORT}"
      DEBEZIUM_CONNECTOR_HOST: "${DEBEZIUM_CONNECTOR_HOST}"
      DEBEZIUM_CONNECTOR_PORT: "${DEBEZIUM_CONNECTOR_PORT}"
      DEBEZIUM_INDEX_NAME: "${DEBEZIUM_INDEX_NAME}"
      DEBEZIUM_KSQLDB_HOST: "${DEBEZIUM_KSQLDB_HOST}"
      DEBEZIUM_KSQLDB_PORT: "${DEBEZIUM_KSQLDB_PORT}"
      PSQL_USERNAME: "${PSQL_USERNAME}"
      PSQL_PASSWORD: "${PSQL_PASSWORD}"
      PSQL_HOST: "${PSQL_HOST}"
      PSQL_PORT: "${PSQL_PORT}"
      PSQL_DATABASE: "${PSQL_DATABASE}"
    networks:
      - agr-literature
    command:
      /setup.sh

  dbz_ksql_server:
    image: confluentinc/ksqldb-server:0.26.0
    depends_on:
      - dbz_kafka
      - dbz_connector
    ports:
      - "8088:8088"
    hostname: dbz_ksql_server
    environment:
      - KSQL_BOOTSTRAP_SERVERS=dbz_kafka:9092
      - KSQL_HOST_NAME=dbz_ksql_server
      - KSQL_APPLICATION_ID="cp-all-in-one"
      - KSQL_LISTENERS=http://0.0.0.0:8088
      - KSQL_CACHE_MAX_BYTES_BUFFERING=0
      - KSQL_OPTS=-Dmax.request.size=30000000 -Dmax.message.bytes=30000000 -Dmessage.max.bytes=30000000
    networks:
      - agr-literature

  # KSQLDB client for manual queries during development
  #
  # To start it:
  # - start the docker-compose service: $ docker-compose up -d dbz_ksql_cli
  # - run bash on the container: $ docker exec -it agr-literature-main-dbz_ksql_cli-1 bash
  # - determine the server IP address (for some reason this version of ksqldb cli does not work with host names): $ ping dbz_ksql_server
  # - start the ksql client connecting to the server: $ ksql http://<ksqldb_server_ip>:8088
  dbz_ksql_cli:
    image: confluentinc/ksqldb-cli:0.26.0
    depends_on:
      - dbz_ksql_server
    networks:
      - agr-literature
    entrypoint:
      - /bin/bash
    tty: true

  jenkins:
    image: jenkins/jenkins:2.346.1-lts-jdk11
    volumes:
      - /usr/share/jenkins_home:/var/jenkins_home
      - /var/run/docker.sock:/var/run/docker.sock
      - /usr/bin/docker:/usr/bin/docker
      - /usr/local/bin/docker-compose:/usr/bin/docker-compose
      - /usr/share/agr_env_files/.env.rdsdev_4001:/usr/share/agr_env_files/.env
    ports:
      - "49001:8080"
      - "50000:50000"
    restart: on-failure
    deploy:
      resources:
        limits:
          memory: 512m
        reservations:
          memory: 256m

  file_uploader:
    build:
      context: .
      dockerfile: docker/file_uploader.dockerfile
    environment:
      - OKTA_DOMAIN=$OKTA_DOMAIN
      - OKTA_API_AUDIENCE=$OKTA_API_AUDIENCE
      - OKTA_CLIENT_ID=$OKTA_CLIENT_ID
      - OKTA_CLIENT_SECRET=$OKTA_CLIENT_SECRET
      - API_PORT=$API_PORT
      - API_SERVER=$API_SERVER


volumes:
  agr-literature-pg-data:
  agr-literature-pgadmin-data:
  agr-literature-es-data:
  agr-logs:

networks:
  agr-literature: