docker-compose.test.py2.yml

# -*- coding: utf-8 -*-
#
# This file is part of hepcrawl.
# Copyright (C) 2017 CERN.
#
# hepcrawl is a free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

version: '2.1'

services:
  service_base: &service_base
    image: hepcrawl_base
    environment:
      - APP_BROKER_URL=pyamqp://guest:guest@rabbitmq:5672//
      - APP_CELERY_RESULT_BACKEND=redis://redis:6379/1
      - APP_CRAWLER_HOST_URL=http://scrapyd:6800
      - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results
      - APP_LAST_RUNS_PATH=/code/.scrapy/last_runs
      - APP_CRAWL_ONCE_PATH=/code/.scrapy
      - COVERAGE_PROCESS_START=/code/.coveragerc
      - APP_DOWNLOAD_BUCKET=downloaded
    tty: true
    volumes:
      - .:/code/
      - ./tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf

  functional_aps:
    <<: *service_base
    command: py.test tests/functional/aps
    depends_on:
      - scrapyd
      - localstack
      - aps-http-server.local

  functional_wsp:
    <<: *service_base
    command: py.test -vv tests/functional/wsp
    depends_on:
      scrapyd:
        condition: service_healthy
      ftp_server:
        condition: service_healthy

  functional_elsevier:
    <<: *service_base
    command: py.test -vv tests/functional/elsevier
    depends_on:
      scrapyd:
        condition: service_healthy
      localstack:
        condition: service_healthy
      elsevier-http-server.local:
        condition: service_healthy

  functional_desy:
    <<: *service_base
    command: py.test -vv tests/functional/desy
    depends_on:
      scrapyd:
        condition: service_healthy
      localstack:
        condition: service_healthy

  functional_arxiv:
    <<: *service_base
    command: py.test -vv tests/functional/arxiv
    depends_on:
      scrapyd:
        condition: service_healthy
      arxiv-http-server.local:
        condition: service_healthy

  functional_pos:
    <<: *service_base
    command: py.test -vv tests/functional/pos
    depends_on:
      scrapyd:
        condition: service_healthy
      http-server.local:
        condition: service_healthy

  unit:
    <<: *service_base
    command: bash -c "py.test tests/unit -vv && make -C docs clean && make -C docs html && python setup.py sdist && ls dist/*"
    links: []

  celery:
    <<: *service_base
    command: celery worker --events --app hepcrawl.testlib.tasks --loglevel=debug
    depends_on:
      rabbitmq:
        condition: service_healthy
      redis:
        condition: service_healthy

  scrapyd:
    <<: *service_base
    command: scrapyd --pidfile=/tmp/scrapyd.pid
    restart: "always"
    networks:
      default:
      ftp:
    depends_on:
      celery:
        condition: service_started
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: ['CMD', "curl", "-k", "http://localhost:6800/listprojects.json"]
    ports:
      - "1234:1234"

  scrapyd-deploy:
    <<: *service_base
    command: scrapyd-deploy
    depends_on:
      scrapyd:
        condition: service_healthy

  ftp_server:
    image: stilliard/pure-ftpd:hardened
    restart: "always"
    environment:
      - PUBLICHOST=1.2.3.4
      - ADDED_FLAGS="--pidfile /var/run/pure-ftpd.pid"
    networks:
      ftp:
        ipv4_address: 1.2.3.4
    volumes:
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/FFT:/home/ftpusers/bob/FFT
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/DESY:/home/ftpusers/bob/DESY
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "ls -l /var/run/pure-ftpd.pid"

  http-server.local:
    image: nginx:stable-alpine
    restart: "always"
    volumes:
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/ssl:/etc/nginx/ssl
      - ${PWD}/tests/functional/pos/fixtures/https_server/records:/etc/nginx/html/
    ports:
      - 443:443
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k https://localhost:443/187.html"

  functional_cds:
    <<: *service_base
    command: py.test -vv tests/functional/cds
    depends_on:
      scrapyd:
        condition: service_healthy
      cds-http-server.local:
        condition: service_healthy

  elsevier-http-server.local:
    image: nginx:stable-alpine
    restart: "always"
    volumes:
      - ${PWD}/tests/functional/elsevier/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/elsevier/fixtures/http_server/data:/etc/nginx/html/
    ports:
      - 80:80
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:80/arxiv-physics-hep-th.xml"

  arxiv-http-server.local:
    image: nginx:stable-alpine
    restart: "always"
    volumes:
      - ${PWD}/tests/functional/arxiv/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/arxiv/fixtures/http_server/records:/etc/nginx/html/
    ports:
      - 80:80
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:80/arxiv-physics-hep-th.xml"

  cds-http-server.local:
    image: nginx:stable-alpine
    restart: "always"
    volumes:
      - ${PWD}/tests/functional/cds/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/cds/fixtures/http_server/records:/etc/nginx/html/
    ports:
      - 80:80
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:80/cds-single.xml"

  aps-http-server.local:
    image: nginx:stable-alpine
    restart: "always"
    volumes:
      - ./tests/functional/aps/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ./tests/functional/aps/fixtures/http_server/records:/etc/nginx/html/
    ports:
      - 80:80

  rabbitmq:
    image: rabbitmq:3.9.11-management
    restart: "always"
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "rabbitmqctl status"

  redis:
    image: redis
    restart: "always"
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "redis-cli -h 127.0.0.1 ping| grep PONG"

  localstack:
    image: localstack/localstack:0.11.6
    restart: "always"
    ports:
      - '4566:4566'
    environment:
      - SERVICES=s3
      - DEBUG=1
      - DATA_DIR=/home/localstack/data
      - HOSTNAME_EXTERNAL=localstack
      - HOSTNAME=localstack
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k localstack:4566|grep running"

networks:
  ftp:
    ipam:
      config:
        - subnet: 1.0.0.0/8