Spilt ray serve

helm/rayservice/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: rayservice
+description: Model service Helm chart
+version: 0.1.0
+type: application

helm/rayservice/applications/episeg-1.yaml

@@ -0,0 +1,32 @@
+- name: episeg-1
+  import_path: models.semantic_segmentation:app
+  route_prefix: /episeg-1
+  runtime_env:
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+  deployments:
+    - name: SemanticSegmentation
+      max_ongoing_requests: 16
+      max_queued_requests: 32
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 4
+        target_ongoing_requests: 4
+      ray_actor_options:
+        num_cpus: 4
+        memory: 12884901888
+        num_gpus: 1
+        runtime_env:
+          env_vars:
+            MLFLOW_TRACKING_URI: http://mlflow.rationai-mlflow:5000
+      user_config:
+        tile_size: 1024
+        mpp: 0.468
+        max_batch_size: 8
+        batch_wait_timeout_s: 0.1
+        intra_op_num_threads: 4
+        trt_max_workspace_size: 6442450944
+        trt_cache_path: /mnt/cache/trt_cache/episeg-1/b8
+        trt_builder_optimization_level: 3
+        model:
+          _target_: providers.model_provider:mlflow
+          artifact_uri: mlflow-artifacts:/10/39f821ed5b964c71a603cc6db196f9fd/artifacts/checkpoints/epoch=19-step=32020/model.onnx/model.onnx

helm/rayservice/applications/heatmap-builder.yaml

@@ -0,0 +1,19 @@
+- name: heatmap-builder
+  import_path: builders.heatmap_builder:app
+  route_prefix: /heatmap-builder
+  runtime_env:
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+  deployments:
+    - name: HeatmapBuilder
+      max_ongoing_requests: 16
+      max_queued_requests: 32
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 4
+        target_ongoing_requests: 8
+      ray_actor_options:
+        num_cpus: 8
+        memory: 12884901888
+      user_config:
+        num_threads: 8
+        max_concurrent_tasks: 16

helm/rayservice/applications/prostate-classifier-1.yaml

@@ -0,0 +1,31 @@
+- name: prostate-classifier-1
+  import_path: models.binary_classifier:app
+  route_prefix: /prostate-classifier-1
+  runtime_env:
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+  deployments:
+    - name: BinaryClassifier
+      max_ongoing_requests: 512
+      max_queued_requests: 1024
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 4
+        target_ongoing_requests: 128
+      ray_actor_options:
+        num_cpus: 4
+        num_gpus: 1
+        memory: 12884901888
+        runtime_env:
+          env_vars:
+            MLFLOW_TRACKING_URI: http://mlflow.rationai-mlflow:5000
+      user_config:
+        tile_size: 512
+        max_batch_size: 256
+        batch_wait_timeout_s: 0.05
+        intra_op_num_threads: 4
+        trt_max_workspace_size: 8589934592
+        trt_cache_path: /mnt/cache/trt_cache/binary-classifier-1/b256
+        trt_builder_optimization_level: 3
+        model:
+          _target_: providers.model_provider:mlflow
+          artifact_uri: mlflow-artifacts:/65/aebc892f526047249b972f200bef4381/artifacts/checkpoints/epoch=0-step=6972/prostate_model_norm.onnx

helm/rayservice/applications/virchow2.yaml

@@ -0,0 +1,28 @@
+- name: virchow2
+  import_path: models.virchow2:app
+  route_prefix: /virchow2
+  runtime_env:
+    config:
+      setup_timeout_seconds: 1800
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+  deployments:
+    - name: Virchow2
+      max_ongoing_requests: 1024
+      max_queued_requests: 2048
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 4
+        target_ongoing_requests: 256
+      ray_actor_options:
+        num_cpus: 4
+        num_gpus: 1
+        memory: 8589934592
+        runtime_env:
+          env_vars:
+            HF_HOME: /mnt/huggingface_cache
+      user_config:
+        tile_size: 224
+        max_batch_size: 512
+        batch_wait_timeout_s: 0.1
+        model:
+          repo_id: paige-ai/Virchow2

helm/rayservice/templates/rayservice.yaml

@@ -0,0 +1,65 @@
+apiVersion: ray.io/v1
+kind: RayService
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace | default "rationai-jobs-ns" }}
+spec:
+  rayClusterConfig:
+    rayVersion: 2.53.0
+    enableInTreeAutoscaling: true
+    autoscalerOptions:
+      idleTimeoutSeconds: 60
+      securityContext:
+        runAsUser: 1000
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+    headGroupSpec:
+      rayStartParams:
+        num-cpus: "0"
+        dashboard-host: "0.0.0.0"
+      template:
+        spec:
+          securityContext:
+            fsGroupChangePolicy: OnRootMismatch
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: ray-head
+              image: rayproject/ray:2.53.0-py312
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  cpu: 0
+                  memory: 4Gi
+                requests:
+                  cpu: 0
+                  memory: 4Gi
+              env:
+                - name: HTTPS_PROXY
+                  value: http://proxy.ics.muni.cz:3128
+              ports:
+                - containerPort: 6379
+                  name: gcs-server
+                - containerPort: 8265
+                  name: dashboard
+                - containerPort: 10001
+                  name: client
+                - containerPort: 8000
+                  name: serve
+              securityContext:
+                runAsUser: 1000
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+    workerGroupSpecs:
+{{- range $workerName := .Values.workers }}
+{{- $workerContent := printf "workers/%s.yaml" $workerName | $.Files.Get | fromYaml }}
+      - {{ toYaml $workerContent | nindent 8 | trim }}
+{{- end }}
+  serveConfigV2: |
+    applications:
+{{- range $appName := .Values.applications }}
+{{ printf "applications/%s.yaml" $appName | $.Files.Get | indent 4 }}
+{{- end }}

helm/rayservice/values.yaml

@@ -0,0 +1,9 @@
+workers:
+  - cpu-workers
+  - mig20-workers
+
+applications:
+  - episeg-1
+  - heatmap-builder
+  - prostate-classifier-1
+  - virchow2

helm/rayservice/workers/cpu-workers.yaml

@@ -0,0 +1,66 @@
+groupName: cpu-workers
+replicas: 0
+minReplicas: 0
+maxReplicas: 2
+template:
+  spec:
+    securityContext:
+      fsGroupChangePolicy: OnRootMismatch
+      runAsNonRoot: true
+      seccompProfile:
+        type: RuntimeDefault
+    containers:
+      - name: ray-worker
+        image: cerit.io/rationai/model-service:2.53.0
+        imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: 8
+            memory: 16Gi
+          requests:
+            cpu: 8
+            memory: 16Gi
+        env:
+          - name: HTTPS_PROXY
+            value: http://proxy.ics.muni.cz:3128
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+          runAsUser: 1000
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "ray stop"]
+        volumeMounts:
+          - name: data
+            mountPath: /mnt/data
+          - name: public-data
+            mountPath: /mnt/data/Public
+          - name: projects
+            mountPath: /mnt/projects
+          - name: bioptic-tree
+            mountPath: /mnt/bioptic_tree
+          - name: trt-cache-volume
+            mountPath: /mnt/cache
+          - name: huggingface-cache
+            mountPath: /mnt/huggingface_cache
+    volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: data-ro
+      - name: public-data
+        persistentVolumeClaim:
+          claimName: rationai-data-ro-pvc-jobs
+      - name: projects
+        persistentVolumeClaim:
+          claimName: projects-rw
+      - name: bioptic-tree
+        persistentVolumeClaim:
+          claimName: bioptictree-ro
+      - name: trt-cache-volume
+        persistentVolumeClaim:
+          claimName: tensorrt-cache-pvc
+      - name: huggingface-cache
+        persistentVolumeClaim:
+          claimName: huggingface-cache-pvc

helm/rayservice/workers/mig20-workers.yaml

@@ -0,0 +1,76 @@
+groupName: mig20-gpu-workers
+replicas: 0
+minReplicas: 0
+maxReplicas: 4
+rayStartParams:
+  num-gpus: "1"
+template:
+  spec:
+    securityContext:
+      fsGroup: 1000
+      fsGroupChangePolicy: OnRootMismatch
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+    containers:
+      - name: ray-worker
+        image: cerit.io/rationai/model-service:2.54.0-gpu
+        imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: 8
+            memory: 24Gi
+            nvidia.com/mig-2g.20gb: 1
+          requests:
+            cpu: 8
+            memory: 24Gi
+        env:
+          - name: HTTPS_PROXY
+            value: http://proxy.ics.muni.cz:3128
+          - name: HF_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: huggingface-secret
+                key: token
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+          runAsUser: 1000
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "ray stop"]
+        volumeMounts:
+          - name: data
+            mountPath: /mnt/data
+          - name: public-data
+            mountPath: /mnt/data/Public
+          - name: projects
+            mountPath: /mnt/projects
+          - name: bioptic-tree
+            mountPath: /mnt/bioptic_tree
+          - name: trt-cache-volume
+            mountPath: /mnt/cache
+          - name: huggingface-cache
+            mountPath: /mnt/huggingface_cache
+    volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: data-ro
+      - name: public-data
+        persistentVolumeClaim:
+          claimName: rationai-data-ro-pvc-jobs
+      - name: projects
+        persistentVolumeClaim:
+          claimName: projects-rw
+      - name: bioptic-tree
+        persistentVolumeClaim:
+          claimName: bioptictree-ro
+      - name: trt-cache-volume
+        persistentVolumeClaim:
+          claimName: tensorrt-cache-pvc
+      - name: huggingface-cache
+        persistentVolumeClaim:
+          claimName: huggingface-cache-pvc