sample-vllm-template/compose.yaml at main · DefangSamples/sample-vllm-template · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
name: vllm
services:
  mistral:
    restart: unless-stopped
    image: ghcr.io/mistralai/mistral-src/vllm:latest
    ports:
    - mode: host
      target: 8000
    command:
    - --host
    - 0.0.0.0
    - --model
    - TheBloke/Mistral-7B-Instruct-v0.2-AWQ
    - --quantization
    - awq
    - --dtype
    - auto
    - --tensor-parallel-size
    - '1'
    - --gpu-memory-utilization
    - '.95'
    - --max-model-len
    - '8000'
    deploy:
      resources:
        reservations:
          cpus: '2.0'
          memory: 8192M
          devices:
          - capabilities:
            - gpu
            count: 1
    healthcheck:
      test:
      - CMD
      - python3
      - -c
      - import sys, urllib.request;urllib.request.urlopen(sys.argv[1]).read()
      - http://localhost:8000/health
      interval: 1m
    environment:
    - HF_TOKEN
  ui:
    restart: unless-stopped
    build:
      context: ui
      dockerfile: Dockerfile
    ports:
    - mode: ingress
      target: 3000
      published: 3000
    deploy:
      resources:
        reservations:
          memory: 256M
    healthcheck:
      test:
      - CMD
      - wget
      - --spider
      - http://localhost:3000
      interval: 10s
      timeout: 2s
      retries: 10
    environment:
    - OPENAI_BASE_URL=http://mistral:8000/v1/
    depends_on:
    - mistral