From 84f1df1b7bd0afd4003cf5dd5773d63b826198f2 Mon Sep 17 00:00:00 2001 From: Lianhao Lu Date: Thu, 12 Dec 2024 08:44:49 +0000 Subject: [PATCH 1/2] Workaround to acknowledge HF_TOKEN in model-downloader Signed-off-by: Lianhao Lu --- .../common/speecht5/templates/deployment.yaml | 16 ++++++++++++---- helm-charts/common/tei/templates/deployment.yaml | 14 +++++++++++--- .../common/teirerank/templates/deployment.yaml | 14 +++++++++++--- helm-charts/common/tgi/templates/deployment.yaml | 16 ++++++++++++---- .../common/vllm/templates/deployment.yaml | 14 +++++++++++--- .../common/whisper/templates/deployment.yaml | 14 +++++++++++--- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/helm-charts/common/speecht5/templates/deployment.yaml b/helm-charts/common/speecht5/templates/deployment.yaml index 0db70e67..1a926000 100644 --- a/helm-charts/common/speecht5/templates/deployment.yaml +++ b/helm-charts/common/speecht5/templates/deployment.yaml @@ -35,8 +35,8 @@ spec: - configMapRef: name: {{ include "speecht5.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false {{- if hasKey .Values.securityContext "runAsGroup" }} runAsGroup: {{ .Values.securityContext.runAsGroup }} {{- end }} @@ -51,16 +51,24 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.TTS_MODEL_PATH | quote }}; - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.VOCODER_MODEL| quote }}; + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download models {{ .Values.TTS_MODEL_PATH }} {{ .Values.VOCODER_MODEL }} ... "; + huggingface-cli download --cache-dir /data {{ .Values.TTS_MODEL_PATH | quote }}; + huggingface-cli download --cache-dir /data {{ .Values.VOCODER_MODEL| quote }}; + echo "Change model files mode ..."; chmod -R g+w /data/models--{{ replace "/" "--" .Values.TTS_MODEL_PATH }}; chmod -R g+w /data/models--{{ replace "/" "--" .Values.VOCODER_MODEL }} + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Release.Name }} diff --git a/helm-charts/common/tei/templates/deployment.yaml b/helm-charts/common/tei/templates/deployment.yaml index 10f13981..52ea2767 100644 --- a/helm-charts/common/tei/templates/deployment.yaml +++ b/helm-charts/common/tei/templates/deployment.yaml @@ -38,8 +38,8 @@ spec: - configMapRef: name: {{ include "tei.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false {{- if hasKey .Values.securityContext "runAsGroup" }} runAsGroup: {{ .Values.securityContext.runAsGroup }} {{- end }} @@ -54,14 +54,22 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) $(MODEL_ID); + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download model $(MODEL_ID) ... "; + huggingface-cli download --cache-dir /data $(MODEL_ID); + echo "Change model files mode ..."; chmod -R g+w /data/models--{{ replace "/" "--" .Values.EMBEDDING_MODEL_ID }} + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Chart.Name }} diff --git a/helm-charts/common/teirerank/templates/deployment.yaml b/helm-charts/common/teirerank/templates/deployment.yaml index 3387fad0..6b82b54b 100644 --- a/helm-charts/common/teirerank/templates/deployment.yaml +++ b/helm-charts/common/teirerank/templates/deployment.yaml @@ -38,8 +38,8 @@ spec: - configMapRef: name: {{ include "teirerank.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false {{- if hasKey .Values.securityContext "runAsGroup" }} runAsGroup: {{ .Values.securityContext.runAsGroup }} {{- end }} @@ -54,14 +54,22 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) $(MODEL_ID); + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download model $(MODEL_ID) ... "; + huggingface-cli download --cache-dir /data $(MODEL_ID); + echo "Change model files mode ..."; chmod -R g+w /data/models--{{ replace "/" "--" .Values.RERANK_MODEL_ID }} + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Chart.Name }} diff --git a/helm-charts/common/tgi/templates/deployment.yaml b/helm-charts/common/tgi/templates/deployment.yaml index 66d62dda..214cbddd 100644 --- a/helm-charts/common/tgi/templates/deployment.yaml +++ b/helm-charts/common/tgi/templates/deployment.yaml @@ -38,8 +38,8 @@ spec: - configMapRef: name: {{ include "tgi.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false {{- if hasKey .Values.securityContext "runAsGroup" }} runAsGroup: {{ .Values.securityContext.runAsGroup }} {{- end }} @@ -54,14 +54,22 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) $(MODEL_ID); - chmod -R g+w /data/models--{{ replace "/" "--" .Values.LLM_MODEL_ID }} + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download model $(MODEL_ID) ... "; + huggingface-cli download --cache-dir /data $(MODEL_ID); + echo "Change model files mode ..."; + chmod -R g+w /data/models--{{ replace "/" "--" .Values.LLM_MODEL_ID }}; + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Chart.Name }} diff --git a/helm-charts/common/vllm/templates/deployment.yaml b/helm-charts/common/vllm/templates/deployment.yaml index 71aef029..afa559cd 100644 --- a/helm-charts/common/vllm/templates/deployment.yaml +++ b/helm-charts/common/vllm/templates/deployment.yaml @@ -35,8 +35,8 @@ spec: - configMapRef: name: {{ include "vllm.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false capabilities: drop: - ALL @@ -48,14 +48,22 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.LLM_MODEL_ID | quote }}; + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download model {{ .Values.LLM_MODEL_ID }} ... "; + huggingface-cli download --cache-dir /data {{ .Values.LLM_MODEL_ID | quote }}; + echo "Change model files mode ..."; chmod -R g+w /data/models--{{ replace "/" "--" .Values.LLM_MODEL_ID }} + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Chart.Name }} diff --git a/helm-charts/common/whisper/templates/deployment.yaml b/helm-charts/common/whisper/templates/deployment.yaml index 4d4503d9..2b965f39 100644 --- a/helm-charts/common/whisper/templates/deployment.yaml +++ b/helm-charts/common/whisper/templates/deployment.yaml @@ -35,8 +35,8 @@ spec: - configMapRef: name: {{ include "whisper.fullname" . }}-config securityContext: - allowPrivilegeEscalation: false readOnlyRootFilesystem: true + allowPrivilegeEscalation: false {{- if hasKey .Values.securityContext "runAsGroup" }} runAsGroup: {{ .Values.securityContext.runAsGroup }} {{- end }} @@ -51,14 +51,22 @@ spec: seccompProfile: type: RuntimeDefault image: huggingface/downloader:0.17.3 - command: ['sh', '-c'] + command: ['sh', '-ec'] args: - | - huggingface-cli download --cache-dir /data --token $(HF_TOKEN) {{ .Values.ASR_MODEL_PATH | quote }}; + echo "Huggingface log in ..."; + huggingface-cli login --token $(HF_TOKEN); + echo "Download model {{ .Values.ASR_MODEL_PATH }} ... "; + huggingface-cli download --cache-dir /data {{ .Values.ASR_MODEL_PATH | quote }}; + echo "Change model files mode ..."; chmod -R g+w /data/models--{{ replace "/" "--" .Values.ASR_MODEL_PATH }} + # NOTE: Buggy logout command; + # huggingface-cli logout; volumeMounts: - mountPath: /data name: model-volume + - mountPath: /tmp + name: tmp {{- end }} containers: - name: {{ .Release.Name }} From c12565adc0c160b67cf2950bb3bf1d30d79c8db7 Mon Sep 17 00:00:00 2001 From: Lianhao Lu Date: Thu, 12 Dec 2024 11:07:36 +0000 Subject: [PATCH 2/2] tgi: Fix permission issue of non-root user Fix issue #639 Signed-off-by: Lianhao Lu --- helm-charts/common/tgi/templates/configmap.yaml | 1 + helm-charts/common/tgi/templates/deployment.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/helm-charts/common/tgi/templates/configmap.yaml b/helm-charts/common/tgi/templates/configmap.yaml index 82be971e..0b738587 100644 --- a/helm-charts/common/tgi/templates/configmap.yaml +++ b/helm-charts/common/tgi/templates/configmap.yaml @@ -19,6 +19,7 @@ data: no_proxy: {{ .Values.global.no_proxy | quote }} {{- if contains "tgi-gaudi" .Values.image.repository }} HABANA_LOGS: "/tmp/habana_logs" + TRITON_CACHE_DIR: "/tmp/triton_cache" {{- end }} NUMBA_CACHE_DIR: "/tmp" HF_HOME: "/tmp/.cache/huggingface" diff --git a/helm-charts/common/tgi/templates/deployment.yaml b/helm-charts/common/tgi/templates/deployment.yaml index 214cbddd..23d0bec7 100644 --- a/helm-charts/common/tgi/templates/deployment.yaml +++ b/helm-charts/common/tgi/templates/deployment.yaml @@ -100,6 +100,8 @@ spec: name: shm - mountPath: /tmp name: tmp + - mountPath: /usr/src/out + name: tokenizer ports: - name: http containerPort: {{ .Values.port }} @@ -136,6 +138,8 @@ spec: sizeLimit: {{ .Values.shmSize }} - name: tmp emptyDir: {} + - name: tokenizer + emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }}