merged master

Signed-off-by: Kevin Su <[email protected]>
flyteorg · Sep 25, 2024 · 193c662 · 193c662
2 parents d0463dc + c01a059
commit 193c662
Show file tree

Hide file tree

Showing 83 changed files with 3,281 additions and 348 deletions.
diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -94,7 +94,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar
       - name: Upload single binary image
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
@@ -133,7 +133,7 @@ jobs:
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
@@ -207,7 +207,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar

diff --git a/charts/flyte-core/templates/propeller/configmap.yaml b/charts/flyte-core/templates/propeller/configmap.yaml
@@ -48,9 +48,10 @@ data:
 {{- end }}
   storage.yaml: | {{ tpl (include "storage" .) $ | nindent 4 }}
   cache.yaml: |
-    cache:
-      max_size_mbs: {{ .Values.flytepropeller.cacheSizeMbs }}
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: {{ .Values.flytepropeller.cacheSizeMbs }}
+        target_gc_percent: 70
 {{- with .Values.configmap.task_logs }}
   task_logs.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
 {{- end }}

diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml
@@ -513,9 +513,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 1024
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 1024
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       logs:
@@ -1278,7 +1279,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1362,7 +1363,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml
@@ -177,9 +177,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 1024
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 1024
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       logs:
@@ -429,7 +430,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -513,7 +514,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml
@@ -544,9 +544,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 1024
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 1024
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       logs:
@@ -1408,7 +1409,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1492,7 +1493,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "33bc4dd986fdb015ce49d998deedf122e119579ec09db311e67276230678a70"
+        configChecksum: "16400bbf28ab823ca433a04379f00fecb11530c3c265dd1bdc3e30209ac6a6d"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml
@@ -180,9 +180,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 1024
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 1024
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       k8s-array:
@@ -437,7 +438,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "64a3f5e546eddd8126d03c005460b964f428a0da31c0bfac5f70c63fbf3d635"
+        configChecksum: "b11d6c5dd0dd16bced82fb44fd2fb31c2f27a134543ab7edaf060da42ab27e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -520,7 +521,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "64a3f5e546eddd8126d03c005460b964f428a0da31c0bfac5f70c63fbf3d635"
+        configChecksum: "b11d6c5dd0dd16bced82fb44fd2fb31c2f27a134543ab7edaf060da42ab27e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml
@@ -560,9 +560,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 1024
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 1024
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       k8s-array:
@@ -1431,7 +1432,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "64a3f5e546eddd8126d03c005460b964f428a0da31c0bfac5f70c63fbf3d635"
+        configChecksum: "b11d6c5dd0dd16bced82fb44fd2fb31c2f27a134543ab7edaf060da42ab27e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1514,7 +1515,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "64a3f5e546eddd8126d03c005460b964f428a0da31c0bfac5f70c63fbf3d635"
+        configChecksum: "b11d6c5dd0dd16bced82fb44fd2fb31c2f27a134543ab7edaf060da42ab27e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml
@@ -683,9 +683,10 @@ data:
       limits:
         maxDownloadMBs: 10
   cache.yaml: |
-    cache:
-      max_size_mbs: 0
-      target_gc_percent: 70
+    storage:
+      cache:
+        max_size_mbs: 0
+        target_gc_percent: 70
   task_logs.yaml: | 
     plugins:
       logs:
@@ -7182,7 +7183,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "84d449758d51dc641aff55ae07f4376a860b5038e8407cb9d2444c4f895d953"
+        configChecksum: "8a003fdbed4b3801328c26cb5c202e4ca113875347388366d29a302d610e7e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -7258,7 +7259,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.1
       annotations:
-        configChecksum: "84d449758d51dc641aff55ae07f4376a860b5038e8407cb9d2444c4f895d953"
+        configChecksum: "8a003fdbed4b3801328c26cb5c202e4ca113875347388366d29a302d610e7e4"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: Z0JZYkhxR1MxZUgwNHBTdg==
+  haSharedSecret: ZXlJVkhWYjdIMHhjamZadA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1413,7 +1413,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 964019f329e42d4f7d7309d49654078a05b00cb8c408efebd2c74cf0db8edc22
+        checksum/secret: 94a4c448ea7ad0892283bc4cfc6c506c83c9c5fe998587f4b2c55194c6a674e3
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -798,7 +798,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: b0dBaDJVclFUREF2SGZhRw==
+  haSharedSecret: OW1PbDdRY0t4RllhM3Nybg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1362,7 +1362,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 8d78585f9928848e9a693f26d3b9720ae3aa231dbaf09160298bb0afbe3046fd
+        checksum/secret: 1f30487909a5b2db21b8f92a734fcb321ab30f01694f4257333026e00d512053
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: b3IzQzNZU21jQ1podDVsTg==
+  haSharedSecret: MWVqaUwzWDZtUWY4TDdscA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 24329631e5f08f0c5538ab0abc8aa7e9cf9e48e1318684b879d449249ae072f3
+        checksum/secret: 53219c6f309435a180b4635448e130a2ec19b63b379a881dde73bf8ae957a1ad
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docs/conf.py b/docs/conf.py
@@ -287,7 +287,7 @@
     "python": ("https://docs.python.org/3", None),
     "numpy": ("https://numpy.org/doc/stable", None),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
-    "torch": ("https://pytorch.org/docs/master/", None),
+    "torch": ("https://pytorch.org/docs/main/", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/reference", None),
     "matplotlib": ("https://matplotlib.org", None),
     "pandera": ("https://pandera.readthedocs.io/en/stable/", None),

diff --git a/docs/deployment/agents/chatgpt.rst b/docs/deployment/agents/chatgpt.rst
@@ -78,45 +78,25 @@ Specify agent configuration
 Add the OpenAI API token
 -------------------------------
 
-1. Install flyteagent pod using helm:
+1. Install the flyteagent pod using helm:
 
-.. code-block::
+.. code-block:: bash
 
   helm repo add flyteorg https://flyteorg.github.io/flyte
   helm install flyteagent flyteorg/flyteagent --namespace flyte
 
-2. Get the base64 value of your OpenAI API token:
+2. Set Your OpenAI API Token as a Secret (Base64 Encoded):
 
-.. code-block::
+.. code-block:: bash
 
-  echo -n "<OPENAI_API_TOKEN>" | base64
+  SECRET_VALUE=$(echo -n "<OPENAI_API_TOKEN>" | base64) && \
+  kubectl patch secret flyteagent -n flyte --patch "{\"data\":{\"flyte_openai_api_key\":\"$SECRET_VALUE\"}}"
 
-3. Edit the flyteagent secret:
+3. Restart development:
 
-      .. code-block:: bash
-
-        kubectl edit secret flyteagent -n flyte
-
-      .. code-block:: yaml
-        :emphasize-lines: 3
-
-        apiVersion: v1
-        data:
-          flyte_openai_api_key: <BASE64_ENCODED_OPENAI_API_TOKEN>
-        kind: Secret
-        metadata:
-          annotations:
-            meta.helm.sh/release-name: flyteagent
-            meta.helm.sh/release-namespace: flyte
-          creationTimestamp: "2023-10-04T04:09:03Z"
-          labels:
-            app.kubernetes.io/managed-by: Helm
-          name: flyteagent
-          namespace: flyte
-          resourceVersion: "753"
-          uid: 5ac1e1b6-2a4c-4e26-9001-d4ba72c39e54
-        type: Opaque
+.. code-block:: bash
 
+  kubectl rollout restart deployment flyteagent -n flyte
 
 Upgrade the Flyte Helm release
 ------------------------------