Merge remote-tracking branch 'origin' into katrina/cor-2229-enum-rese…

…rved

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/publish.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Before Build
         run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.github/workflows/sandbox.yml

@@ -53,7 +53,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64

.github/workflows/single-binary.yml

@@ -73,7 +73,7 @@ jobs:
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64
@@ -84,7 +84,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -108,7 +108,7 @@ jobs:
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
         if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -144,7 +144,7 @@ jobs:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           load: true
@@ -240,7 +240,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

Dockerfile

@@ -5,8 +5,8 @@ FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 FROM --platform=${BUILDPLATFORM} golang:1.22-bookworm AS flytebuilder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 WORKDIR /flyteorg/build
 
@@ -29,10 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r
 FROM debian:bookworm-slim
 
 ARG FLYTE_VERSION
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
 
-ENV DEBCONF_NONINTERACTIVE_SEEN true
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Install core packages
 RUN apt-get update && apt-get install --no-install-recommends --yes \

Dockerfile.datacatalog

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 

Dockerfile.flyteadmin

@@ -3,11 +3,11 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -41,7 +41,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytecopilot

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -32,7 +32,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
+LABEL org.opencontainers.image.source=https://github.com/lyft/flyteplugins
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytepropeller

@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -33,7 +33,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytepropeller
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytescheduler

@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -36,7 +36,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

docker/sandbox/Dockerfile

@@ -52,13 +52,13 @@ COPY --from=base /flyteorg/ /flyteorg/
 COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh
 
 ARG FLYTE_VERSION="latest"
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
 
 ARG FLYTE_TEST="release"
-ENV FLYTE_TEST "${FLYTE_TEST}"
+ENV FLYTE_TEST="${FLYTE_TEST}"
 
 # Update PATH variable
-ENV PATH "/flyteorg/bin:${PATH}"
+ENV PATH="/flyteorg/bin:${PATH}"
 
 # Declare volumes for k3s
 VOLUME /var/lib/kubelet

docs/community/contribute/contribute_code.rst

@@ -429,7 +429,7 @@ If not, we can start backends with a single command.
 Before running your workflow in the sandbox, make sure you're able to successfully run it locally.
 To deploy the workflow in the sandbox, you'll need to build a Flytekit image.
 Create a Dockerfile in your Flytekit directory with the minimum required configuration to run a task, as shown below.
-If your task requires additional components, such as plugins, you may find it useful to refer to the construction of the `officail flitekit image <https://github.com/flyteorg/flytekit/blob/master/Dockerfile>`__
+If your task requires additional components, such as plugins, you may find it useful to refer to the construction of the `official flytekit image <https://github.com/flyteorg/flytekit/blob/master/Dockerfile>`__
 
 .. code:: Dockerfile
 

docs/user_guide/data_types_and_io/accessing_attributes.md

@@ -11,6 +11,10 @@ Note that while this functionality may appear to be the normal behavior of Pytho
 Consequently, accessing attributes in this manner is, in fact, a specially implemented feature.
 This functionality facilitates the direct passing of output attributes within workflows, enhancing the convenience of working with complex data structures.
 
+```{important}
+Flytekit version >= v1.14.0 supports Pydantic BaseModel V2, you can do attribute access on Pydantic BaseModel V2 as well.
+```
+
 ```{note}
 To clone and run the example code on this page, see the [Flytesnacks repo][flytesnacks].
 ```
@@ -19,7 +23,7 @@ To begin, import the required dependencies and define a common task for subseque
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 1-10
+:lines: 1-9
 ```
 
 ## List
@@ -31,38 +35,38 @@ Flyte currently does not support output promise access through list slicing.
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 14-23
+:lines: 13-22
 ```
 
 ## Dictionary
 Access the output dictionary by specifying the key.
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 27-35
+:lines: 26-34
 ```
 
 ## Data class
 Directly access an attribute of a dataclass.
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 39-53
+:lines: 38-51
 ```
 
 ## Complex type
 Combinations of list, dict and dataclass also work effectively.
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 57-80
+:lines: 55-78
 ```
 
 You can run all the workflows locally as follows:
 
 ```{literalinclude} /examples/data_types_and_io/data_types_and_io/attribute_access.py
 :caption: data_types_and_io/attribute_access.py
-:lines: 84-88
+:lines: 82-86
 ```
 
 ## Failure scenario

docs/user_guide/data_types_and_io/dataclass.md

@@ -11,8 +11,24 @@ When you've multiple values that you want to send across Flyte entities, you can
 Flytekit uses the [Mashumaro library](https://github.com/Fatal1ty/mashumaro)
 to serialize and deserialize dataclasses.
 
+With the 1.14 release, `flytekit` adopted `MessagePack` as the 
+serialization format for dataclasses, overcoming a major limitation of serialization into a JSON string within a Protobuf `struct` datatype, like the previous versions do:
+
+to store `int` types, Protobuf's `struct` converts them to `float`, forcing users to write boilerplate code to work around this issue.
+
+:::{important}
+If you're using Flytekit version < v1.11.1, you will need to add `from dataclasses_json import dataclass_json` to your imports and decorate your dataclass with `@dataclass_json`.
+:::
+
 :::{important}
-If you're using Flytekit version below v1.11.1, you will need to add `from dataclasses_json import dataclass_json` to your imports and decorate your dataclass with `@dataclass_json`.
+Flytekit version < v1.14.0 will produce protobuf `struct` literal for dataclasses.
+
+Flytekit version >= v1.14.0 will produce msgpack bytes literal for dataclasses.
+
+If you're using Flytekit version >= v1.14.0 and you want to produce protobuf `struct` literal for dataclasses, you can 
+set environment variable  `FLYTE_USE_OLD_DC_FORMAT` to `true`.
+
+For more details, you can refer the MSGPACK IDL RFC: https://github.com/flyteorg/flyte/blob/master/rfc/system/5741-binary-idl-with-message-pack.md
 :::
 
 ```{note}

docs/user_guide/data_types_and_io/index.md

@@ -114,7 +114,7 @@ Here's a breakdown of these mappings:
       - Use ``pyspark.DataFrame`` as a type hint.
     * - ``pydantic.BaseModel``
       - ``Map``
-      - To utilize the type, install the ``flytekitplugins-pydantic`` plugin.
+      - To utilize the type, install the ``pydantic>2`` module.
       - Use ``pydantic.BaseModel`` as a type hint.
     * - ``torch.Tensor`` / ``torch.nn.Module``
       - File
@@ -144,6 +144,7 @@ flytefile
 flytedirectory
 structureddataset
 dataclass
+pydantic_basemodel
 accessing_attributes
 pytorch_type
 enum_type