search

nclskfm / square-core

SQuARE: Software for question answering research.
https://square.ukp-lab.de
2 stars 0 forks source link

Batching queries skill manager #73

Closed sgrubeml closed 1 year ago

sgrubeml commented 1 year ago

What does this PR do?

This PR adds batching functionality to the evaluator for query calls to the skill manager. Instead of sending a query call for each data point, data points are "batched" into a single request and the skill manager returns the predictions for all data points at once.

Who can review?

@BenediktStuhrmann @melicoretta @nclskfm

Test locally

To test this PR locally

change square-core/skill-manager/Dockerfile to:

FROM python:3.10.8-slim-buster as base

ENV PYTHONUNBUFFERED=1
ENV PIP_NO_CACHE_DIR=1

# required to install packages from github
RUN apt-get -y update && apt-get -y install git

RUN pip install --upgrade pip

WORKDIR /app

COPY square-core/skill-manager/requirements.txt requirements.txt
RUN pip install -r requirements.txt

COPY square-skill-api square-skill-api
RUN pip install --editable square-skill-api

COPY square-core/skill-manager skill_manager

COPY square-core/skill-manager/logging.conf logging.conf

FROM base as test
RUN apt update && apt -y install docker.io

COPY requirements.dev.txt requirements.dev.txt 
RUN pip install -r requirements.dev.txt

COPY tests tests
COPY pytest.ini pytest.ini

FROM base as build
EXPOSE 8000

CMD ["uvicorn", "skill_manager.main:app", "--host", "0.0.0.0", "--port", "8000", "--log-config", "logging.conf"]

change square-core/skills/Dockefile to

FROM python:3.10.8-slim-buster as build

ENV PYTHONUNBUFFERED=1
ENV PIP_NO_CACHE_DIR=1

RUN apt-get -y update && apt-get -y install git

WORKDIR /app

RUN pip install --upgrade pip
COPY square-core/skills/requirements.txt requirements.txt
RUN pip install -r requirements.txt

COPY square-skill-api square-skill-api
RUN pip install --editable square-skill-api

COPY square-core/skills/main.py main.py
COPY square-core/skills/utils.py utils.py

COPY square-core/skills/extractive-qa/skill.py skill.py 
COPY square-core/skills/logging.conf logging.conf

EXPOSE 80

CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80", "--log-config", "logging.conf"]

change urls to call locally running skill-manager and extractive-qa skill.

Add the following docker-compose to the project root folder:

services:

  evaluator:
    image: ukpsquare/evaluator:latest
    build:
      context: evaluator
      target: build
    command: uvicorn evaluator.app.main:app --reload --host 0.0.0.0 --port 8081 --log-config logging.conf
    expose:
    - 8081
    ports:
      - 8081:8081
    env_file:
    - ./evaluator/.local.env
    environment:
      API_PREFIX: ""
      WEB_CONCURRENCY: "1"
      KEYCLOAK_BASE_URL: ""
      MODEL_API_URL: ""
      MONGO_HOST: mongodb
      MONGO_PORT: 27017
      VERIFY_ISSUER: "0"
      SQUARE_PRIVATE_KEY_FILE: /app/private_key.pem
      DATASET_DIR: /app/datasets/
      REDIS_HOST: redis
      REDIS_PORT: 6379
    depends_on:
    - mongodb
    - redis
    - rabbit
    volumes:
      - ./evaluator/evaluator:/app/evaluator
      - ./evaluator/datasets:/app/datasets/
      - ./evaluator/private_key.pem:/app/private_key.pem

  evaluation_worker:
    image: ukpsquare/evaluator:latest
    build:
      context: evaluator
      target: build
    command: celery -A evaluator.tasks worker --loglevel=info
    env_file:
    - ./evaluator/.local.env
    environment:
      API_PREFIX: ""
      WEB_CONCURRENCY: "1"
      KEYCLOAK_BASE_URL: ""
      MODEL_API_URL: ""
      MONGO_HOST: mongodb
      MONGO_PORT: 27017
      VERIFY_ISSUER: "0"
      SQUARE_PRIVATE_KEY_FILE: /app/private_key.pem
      DATASET_DIR: /app/datasets/
      REDIS_HOST: redis
      REDIS_PORT: 6379
    depends_on:
    - mongodb
    - redis
    - rabbit
    volumes:
      - ./evaluator/evaluator:/app/evaluator
      - ./evaluator//datasets:/app/datasets/
      - ./evaluator/private_key.pem:/app/private_key.pem

  mongodb:
    image: mongo:5.0.4
    restart: always
    volumes:
    - mongo-data:/data/db
    expose:
    - 27017
    ports:
    - 27017:27017
    env_file:
    - ./evaluator/.local.env

  redis:
    image: redis:latest
    ports:
    - 6379:6379
    env_file:
    - ./evaluator/.local.env
    command:
    - redis-server
    - --requirepass ${REDIS_PASSWORD}

  rabbit:
    hostname: rabbit
    image: rabbitmq:3-management
    ports:
      - 5672:5672
      - 15672:15672
    env_file:
      - ./evaluator/.local.env

  skill-manager:
    image: skill-manager-batching
    build:
      context: skill-manager
      target: build
    command: uvicorn skill_manager.main:app --reload --host 0.0.0.0 --port 8000 --log-config logging.conf --workers 2
    ports:
    - 8000:8000
    expose:
    - 8000
    env_file:
    - ./skill-manager/.local.env
    environment:
      API_PREFIX: ""
      WEB_CONCURRENCY: "1"
      KEYCLOAK_BASE_URL: ""
      MODEL_API_URL: ""
      MONGO_HOST: mongodb
      MONGO_PORT: 27017
      VERIFY_ISSUER: "0"
      REDIS_HOST: redis
      REDIS_PORT: 6379
      SQUARE_PRIVATE_KEY_FILE: /app/private_key.pem
    depends_on:
    - mongodb
    - redis
    volumes:
      - ./skill-manager/skill_manager:/app/skill_manager
      - ./evaluator/private_key.pem:/app/private_key.pem

  extractive-qa:
    image: skills-batching
    build:
      context: skills
      args:
        skill: extractive-qa
    environment:
      WEB_CONCURRENCY: "2"
      VERIFY_SSL: "1"
      SQUARE_API_URL: https://square.ukp-lab.de/api
      SQUARE_SKILL_MANAGER: http://skill-manager:8000/api
      KEYCLOAK_BASE_URL: ""
      REALM: square
      CLIENT_ID: ukp-extractive-qa
      SQUARE_PRIVATE_KEY_FILE: /app/private_key.pem
    volumes:
      - ./evaluator/private_key.pem:/app/private_key.pem
    ports:
      - 8083:80

volumes:
  mongo-data:

If skills not running locally (inference not possible), test batching by adding a mocked response to the extractive-qa skill in square-core/skills/extractive-qa/skill.py:

Mocked data response is for query with batch size 2, make sure to add only 2 data points to the query, for example by setting dataset = dataset[:2] in square-core/evaluator/evaluator/tasks/predict.py before setting the query_request variable.

Clone the square-skill-api repo to the parent folder of the square-core project. Folder structure should look like this:

root folder
-- square-core
-- square-skill-api

change the query attribute of the QueryRequest class in https://github.com/UKP-SQuARE/square-skill-api/blob/master/square_skill_api/models/request.py#L97 to this:

   query: Union[str, List[str]] = Field(
        ..., description="The input to the model that is entered by the user"
    )

Make sure to import Union and List.

Go to root folder and run the following command from your terminal:

docker build -t skill-manager-batching -f square-core/skill-manager/Dockerfile . && docker build -t skills-batching -f square-core/skills/Dockerfile .

go to the square-core folder and run the following command:

docker compose --env-file evaluator/.local.env up -d

I hope i didnt forget anything :)