pyspider/docker-compose.yml

version: "3"
services:
  redis:
    image: redis:alpine
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
    ports:
      - "6379:6379"
    networks:
      - cars

  mysql:
    image: mysql:5.6
    environment:
      - MYSQL_ROOT_PASSWORD=root
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
    ports:
      - "3306:3306"
    networks:
      - cars

  scheduler:
    image: binux/pyspider:latest
    command: '--taskdb "mysql+taskdb://root:root@mysql:3306/taskdb" --resultdb "mysql+resultdb://root:root@mysql:3306/resultdb" --projectdb "mysql+projectdb://root:root@mysql:3306/projectdb" --message-queue "redis://redis:6379/db"  scheduler --inqueue-limit 5000 --delete-time 43200'
    ports:
      - "23333:23333"
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role != manager
      restart_policy:
        condition: on-failure
    networks:
      - cars

  phantomjs:
    image: binux/pyspider:latest
    command: phantomjs
    env_file:
      - /home/docker/cars.env
    deploy:
      replicas: 2
      placement:
        constraints:
          - node.role != manager
      restart_policy:
        condition: on-failure
    ports:
      - "25555:25555"
    networks:
      - cars

  fetcher:
    image: binux/pyspider:latest
    command: '--message-queue "redis://redis:6379/db" --phantomjs-proxy "phantomjs:80" fetcher --xmlrpc'
    env_file:
      - /home/docker/cars.env
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role != manager
      restart_policy:
        condition: on-failure
    networks:
      - cars

  processor:
    image: binux/pyspider:latest
    command: '--projectdb "mysql+projectdb://root:root@mysql:3306/projectdb" --message-queue "redis://redis:6379/db" processor'
    deploy:
      replicas: 2
      placement:
        constraints:
          - node.role != manager
      restart_policy:
        condition: on-failure
    networks:
      - cars

  result-worker:
    image: binux/pyspider:latest
    command: '--taskdb "mysql+taskdb://root:root@mysql:3306/taskdb" --resultdb "mysql+resultdb://root:root@mysql:3306/resultdb" --projectdb "mysql+projectdb://root:root@mysql:3306/projectdb" --message-queue "redis://redis:6379/db" result_worker'
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
    networks:
      - cars

  webui:
    image: binux/pyspider:latest
    command: '--taskdb "mysql+taskdb://root:root@mysql:3306/taskdb" --resultdb "mysql+resultdb://root:root@mysql:3306/resultdb" --projectdb "mysql+projectdb://root:root@mysql:3306/projectdb" --message-queue "redis://redis:6379/db" webui --max-rate 0.2 --max-burst 3 --scheduler-rpc "http://scheduler:23333/" --fetcher-rpc "http://fetcher/"'
    env_file:
      - /home/docker/cars.env
    deploy:
      replicas: 4
      placement:
        constraints:
          - node.role == manager
      restart_policy:
        condition: on-failure
    networks:
      - cars
    ports:
      - "5000:5000"

networks:
  cars: