Merge branch 'master' into flyte-streaming-deck

Signed-off-by: Future-Outlier <[email protected]>

.github/workflows/checks.yml

@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
   PRIORITIES: "P0"
@@ -32,8 +33,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
-          # TODO(monorepo): Enable lint flytecopilot
-          # - flytecopilot
+          - flytecopilot
           - flytectl
           - flyteidl
           - flyteplugins

.github/workflows/dependency-review.yml

@@ -0,0 +1,18 @@
+name: 'Dependency Review'
+on: [pull_request]
+permissions:
+  contents: read
+  pull-requests: write
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v4
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v4
+        with:
+          comment-summary-in-pr: on-failure
+          # Licenses need to come from https://spdx.org/licenses/
+          deny-licenses: GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later
+

.github/workflows/flytectl-install.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - flytectl/**
+      - 'release-v**'
   push:
     branches:
       - master

.github/workflows/flyteidl-buf-publish.yml

@@ -6,6 +6,7 @@ on:
       - artifacts-shell-2
       - artifacts
       - master
+      - 'release-v**'
     paths:
       - 'flyteidl/**'
 jobs:

.github/workflows/flyteidl-checks.yml

@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
 jobs:

.github/workflows/helm-charts.yaml

@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - 'release-v**'
   workflow_dispatch:
 
 jobs:

.github/workflows/publish.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Before Build
         run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.github/workflows/sandbox.yml

@@ -53,7 +53,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64

.github/workflows/single-binary.yml

@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - 'release-v**'
   workflow_dispatch:
 
 jobs:
@@ -72,7 +73,7 @@ jobs:
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64
@@ -83,7 +84,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -107,7 +108,7 @@ jobs:
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
         if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -143,7 +144,7 @@ jobs:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           load: true
@@ -175,7 +176,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow
+          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow pandas
           uv pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v4
@@ -188,7 +189,7 @@ jobs:
           flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
-              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
+              pyflyte -vv --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
@@ -239,7 +240,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

.github/workflows/tests.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
   pull_request:
 jobs:
   compile:
@@ -69,6 +70,9 @@ jobs:
       - name: Build the documentation
         working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
+        env:
+          # this is a fake key for algolia docsearch to make docs build process pass
+          DOCSEARCH_API_KEY: fake_docsearch_api_key
         run: |
           conda activate monodocs-env
           make docs

.github/workflows/validate-helm-charts.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - 'release-v**'
     paths:
       - "charts/**"
       - "deployment/**"

.gitignore

@@ -30,12 +30,11 @@ vendor/
 /docker/sandbox-bundled/images/tar
 **/bin/
 docs/_tags/
-docs/flytectl
-docs/protos
-docs/flytekit
+docs/api/flytectl
+docs/api/flytekit
+docs/api/flyteidl
 docs/flytesnacks
 docs/examples
 docs/_src
 docs/_projects
-docs/api
 docs/tests

CHANGELOG/CHANGELOG-v1.13.2.md

@@ -0,0 +1,53 @@
+# Flyte 1.13.2 Release Notes
+
+## What's Changed
+* Enable echo plugin by default by @pingsutw in https://github.com/flyteorg/flyte/pull/5679
+* Do not emit execution id label by default in single binary by @eapolinario in https://github.com/flyteorg/flyte/pull/5704
+* Using new offloaded metadata literal message for literal offloading by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5705
+* Improve error message for MismatchingTypes by @pingsutw in https://github.com/flyteorg/flyte/pull/5639
+* [Docs] Echo Task by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5707
+* Improve execution name readability by @wayner0628 in https://github.com/flyteorg/flyte/pull/5637
+* Configure imagePullPolicy to be Always pull on flyte sandbox environment by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5709
+* Set IsDefault to False for echo plugin by @pingsutw in https://github.com/flyteorg/flyte/pull/5713
+* Move default execution name generation to flyteadmin by @wayner0628 in https://github.com/flyteorg/flyte/pull/5714
+* Update helm/docs per changes in supported task discovery by @Sovietaced in https://github.com/flyteorg/flyte/pull/5694
+* [flyteagent] Add Logging for Agent Supported Task Types by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5718
+* extend pod customization to include init containers by @samhita-alla in https://github.com/flyteorg/flyte/pull/5685
+* Update "Try Serverless" language in Quickstart guide by @neverett in https://github.com/flyteorg/flyte/pull/5698
+* Refactor flyteadmin to pass proto structs as pointers by @Sovietaced in https://github.com/flyteorg/flyte/pull/5717
+* fix: Use deterministic execution names in scheduler by @pingsutw in https://github.com/flyteorg/flyte/pull/5724
+* [flyteagent] Enable `agent-service` in every configuration  by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5730
+* [Docs][flyteagent] Remove `agent-service` config in flyte agent documentation by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5731
+* Fix flytectl returning the oldest workflow when using --latest flag by @RRap0so in https://github.com/flyteorg/flyte/pull/5716
+* Remove explicit go toolchain versions by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5723
+* Add listing api to stow storage by @bgedik in https://github.com/flyteorg/flyte/pull/5741
+* Use latest upload/download-artifact action version by @Sovietaced in https://github.com/flyteorg/flyte/pull/5743
+* Introduced SMTP notification by @robert-ulbrich-mercedes-benz in https://github.com/flyteorg/flyte/pull/5535
+* Added literal offloading for array node map tasks by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5697
+* pytorch object.inv moved by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5755
+* [RFC] Binary IDL With MessagePack Bytes by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5742
+* Rename literal offloading flags and enable offloading in local config by @eapolinario in https://github.com/flyteorg/flyte/pull/5754
+* Add semver check for dev and beta version by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5757
+* Add comments for semver regex by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5758
+* Enable caching in flyte-core helmchart by @cpaulik in https://github.com/flyteorg/flyte/pull/5739
+* [Flyte][1][IDL] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5751
+* [Flyte][2][Literal Type For Scalar] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5761
+* Backoff on etcd errors by @EngHabu in https://github.com/flyteorg/flyte/pull/5710
+* [Docs][flyteagent] Update Databricks Agent Setup to V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5766
+* [Docs][flyteagent] Improve Agent Secret Setup by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5765
+* [RFC] Offloaded Raw Literals by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5103
+* Use pluggable clock for auto refresh cache and make unit tests fast by @Sovietaced in https://github.com/flyteorg/flyte/pull/5767
+* [Docs] Flyte Deck example by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5777
+* Update ImageSpec documentation by @pingsutw in https://github.com/flyteorg/flyte/pull/5748
+* Revert "Improve execution name readability" by @pingsutw in https://github.com/flyteorg/flyte/pull/5740
+* [Docs] Flyte Deck example V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5781
+* [flytepropeller][flyteadmin] Compiler unknown literal type error handling by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5651
+* [Temporary fix] Pin flytekit version for single binary builds by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5798
+* Fix propeller crash when inferring literal type for an offloaded literal by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5771
+
+## New Contributors
+* @wayner0628 made their first contribution in https://github.com/flyteorg/flyte/pull/5637
+* @robert-ulbrich-mercedes-benz made their first contribution in https://github.com/flyteorg/flyte/pull/5535
+* @cpaulik made their first contribution in https://github.com/flyteorg/flyte/pull/5739
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.13.1...v1.13.2

CODEOWNERS

@@ -1,3 +1,3 @@
 # Automatically request docs team for docs PR review
-/docs/          @neverett @ppiegaze
-
+/docs/              @ppiegaze
+/docs/deployment/   @davidmirror-ops

CONTRIBUTING.md

@@ -1,3 +1,15 @@
 # Contributing to Flyte
 
-For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html) section of the documentation.
+For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute/index.html) section of the documentation.
+
+## Recommendation Order (For Beginners)
+* Setup dev environment 
+* Read the following and run at least 5 examples. Pay close attention to the generated outputs, the Graph view, task  
+  logs, etc. Repeat with as many examples as you need to have an initial understanding of what an execution looks like:
+  * https://docs.flyte.org/en/latest/user_guide/introduction.html
+  * https://docs.flyte.org/en/latest/flytesnacks/userguide.html
+* Finish reading the [Concepts](https://docs.flyte.org/en/latest/user_guide/concepts/main_concepts/index.html)
+* Finish reading the [Control Plane](https://docs.flyte.org/en/latest/user_guide/concepts/control_plane/index.html)
+* Finish reading the [Component Architecture](https://docs.flyte.org/en/latest/user_guide/concepts/component_architecture/index.html)
+* Choose 2 good first issues from the following and start solving them with the knowledge you have read.
+* Familiar with using [ImageSpec to push images to localhost for development](https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/imagespec.html#image-spec-example)

Dockerfile

@@ -5,8 +5,8 @@ FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 FROM --platform=${BUILDPLATFORM} golang:1.22-bookworm AS flytebuilder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 WORKDIR /flyteorg/build
 
@@ -29,10 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r
 FROM debian:bookworm-slim
 
 ARG FLYTE_VERSION
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
 
-ENV DEBCONF_NONINTERACTIVE_SEEN true
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Install core packages
 RUN apt-get update && apt-get install --no-install-recommends --yes \

Dockerfile.datacatalog

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 

Dockerfile.flyteadmin

@@ -3,11 +3,11 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -41,7 +41,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytecopilot

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -32,7 +32,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
+LABEL org.opencontainers.image.source=https://github.com/lyft/flyteplugins
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytepropeller

@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -33,7 +33,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytepropeller
 
 COPY --from=builder /artifacts /bin