Skip to content

Commit

Permalink
[Monorepo] Update manifest using the monorepo components (#4082)
Browse files Browse the repository at this point in the history 
* Use latest tag in script/release.sh

Signed-off-by: Eduardo Apolinario <[email protected]>

* Use a more idiomatic way of consuming grep results in call to kustomize in script/release.sh

Signed-off-by: Eduardo Apolinario <[email protected]>

* Rearrange how config docs are created

Signed-off-by: Eduardo Apolinario <[email protected]>

* Add compile_flytepropeller make target

Signed-off-by: Eduardo Apolinario <[email protected]>

* Move the binaries instead of copying them

---------

Signed-off-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>
  • Loading branch information
@eapolinario
eapolinario and eapolinario authored Sep 26, 2023
1 parent 8090b16 commit 8c05f4e
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 42 deletions.
2 changes: 1 addition & 1 deletion datacatalog/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ update_boilerplate:
.PHONY: compile
compile:
mkdir -p ./bin
go build -o datacatalog ./cmd/main.go && mv ./datacatalog ./bin
go build -o bin/datacatalog ./cmd/main.go

.PHONY: linux_compile
linux_compile: export CGO_ENABLED ?= 0
Expand Down
6 changes: 4 additions & 2 deletions flyteadmin/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ k8s_integration_execute:

.PHONY: compile
compile:
go build -o flyteadmin -ldflags=$(LD_FLAGS) ./cmd/ && mv ./flyteadmin ${GOPATH}/bin
mkdir -p ./bin
go build -o bin/flyteadmin -ldflags=$(LD_FLAGS) ./cmd/

.PHONY: compile_debug
compile_debug:
Expand All @@ -46,7 +47,8 @@ compile_debug:

.PHONY: compile_scheduler
compile_scheduler:
go build -o flytescheduler -ldflags=$(LD_FLAGS) ./cmd/scheduler/ && mv ./flytescheduler ${GOPATH}/bin
mkdir -p ./bin
go build -o bin/flytescheduler -ldflags=$(LD_FLAGS) ./cmd/scheduler/

.PHONY: compile_scheduler_debug
compile_scheduler_debug:
Expand Down
7 changes: 6 additions & 1 deletion flytepropeller/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ compile:
mkdir -p ./bin
go build -o bin/flytepropeller ./cmd/controller/main.go
go build -o bin/flytepropeller-manager ./cmd/manager/main.go
go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go && cp bin/kubectl-flyte ${GOPATH}/bin
go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go

.PHONY: compile_flytepropeller
compile_flytepropeller:
mkdir -p ./bin
go build -o bin/flytepropeller ./cmd/controller/main.go

cross_compile:
@glide install
Expand Down
22 changes: 8 additions & 14 deletions script/generate_config_docs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,14 @@ ROOT_DIR=${CUR_DIR}/..
OUTPUT_DIR="${ROOT_DIR}"/rsts/deployment/configuration/generated
GOBIN=${GOPATH:-~/go}/bin

# The version should same as https://github.com/flyteorg/flyte/blob/6b92b72f508d0603fa44153a4e30cf81be76adfd/script/release.sh#L5-L8
FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name)
FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name)
DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name)

pushd "$(mktemp -d)"
curl https://github.com/flyteorg/flyteadmin/releases/download/${FLYTEADMIN_TAG}/flyteadmin_${FLYTEADMIN_TAG}_linux_x86_64.tar.gz --output flyteadmin.tar.gz -s -L && tar -xvf flyteadmin.tar.gz
mv flyteadmin $GOBIN/flyteadmin
mv flytescheduler $GOBIN/scheduler
curl https://github.com/flyteorg/flytepropeller/releases/download/${FLYTEPROPELLER_TAG}/flytepropeller_${FLYTEPROPELLER_TAG}_linux_x86_64.tar.gz --output flytepropeller.tar.gz -s -L && tar -xvf flytepropeller.tar.gz
mv flytepropeller $GOBIN/flytepropeller
curl https://github.com/flyteorg/datacatalog/releases/download/${DATACATALOG_TAG}/datacatalog_${DATACATALOG_TAG}_linux_x86_64.tar.gz --output datacatalog.tar.gz -s -L && tar -xvf datacatalog.tar.gz
mv datacatalog $GOBIN/datacatalog
popd
make -C datacatalog compile
mv datacatalog/bin/datacatalog ${GOBIN}/datacatalog
make -C flyteadmin compile
mv flyteadmin/bin/flyteadmin ${GOBIN}/flyteadmin
make -C flyteadmin compile_scheduler
mv flyteadmin/bin/flytescheduler ${GOBIN}/scheduler
make -C flytepropeller compile_flytepropeller
mv flytepropeller/bin/flytepropeller ${GOBIN}/flytepropeller

output_config () {
CONFIG_NAME=$1
Expand Down
49 changes: 25 additions & 24 deletions script/release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,42 @@

set -ex

FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name)
DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name)
# Get latest release tag (minus the component prefix)
# TODO(monorepo): This only works if we have at least one component tag per release.
# In other words, if we have two consecutive releases the latest tag in the second release is going to point to an invalid
# tag (because there will not be images tagged with the previous release tag).
LATEST_TAG=$(git tag | sed 's#[^/]*/##' | sort | tail -n 1)

FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name)
FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name)
FLYTECOPILOT_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytecopilot/releases/latest" | jq -r .tag_name)
FLYTESTDLIB_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytestdlib/releases/latest" | jq -r .tag_name)

# bump latest release of flyte component in kustomize
grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG/g"
grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${DATACATALOG_TAG} # DATACATALOG_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${LATEST_TAG} # FLYTEADMIN_TAG/g" {}
grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${LATEST_TAG} # DATACATALOG_TAG/g" {}
grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g" {}
grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${LATEST_TAG} # FLYTEPROPELLER_TAG/g" {}

# bump latest release of flyte component in helm
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${LATEST_TAG} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${LATEST_TAG} # FLYTEADMIN_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${LATEST_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${LATEST_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${LATEST_TAG} # DATACATALOG_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${LATEST_TAG} # DATACATALOG_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTECONSOLE_TAG,tag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTECONSOLE_TAG,tag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${LATEST_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${LATEST_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte-core/values.yaml

sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${FLYTECOPILOT_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${FLYTECOPILOT_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTECOPILOT_TAG,tag: ${FLYTECOPILOT_TAG} # FLYTECOPILOT_TAG," ./charts/flyte-binary/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${LATEST_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${LATEST_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTECOPILOT_TAG,tag: ${LATEST_TAG} # FLYTECOPILOT_TAG," ./charts/flyte-binary/values.yaml

go get github.com/flyteorg/flyteadmin@${FLYTEADMIN_TAG}
go get github.com/flyteorg/flytepropeller@${FLYTEPROPELLER_TAG}
go get github.com/flyteorg/datacatalog@${DATACATALOG_TAG}
go get github.com/flyteorg/flytestdlib@${FLYTESTDLIB_TAG}
go get github.com/flyteorg/flyte/flyteadmin@${LATEST_TAG}
go get github.com/flyteorg/flyte/flytepropeller@${LATEST_TAG}
go get github.com/flyteorg/flyte/datacatalog@${LATEST_TAG}
go get github.com/flyteorg/flyte/flytestdlib@${LATEST_TAG}
go mod tidy

0 comments on commit 8c05f4e

Please sign in to comment.