Merge pull request #5741 from FederatedAI/feature-2.2.0-docker_compose

Feature 2.2.0 docker compose
FederatedAI · Nov 19, 2024 · 5a06d9e · 5a06d9e
2 parents f418070 + 60f939a
commit 5a06d9e
Show file tree

Hide file tree

Showing 22 changed files with 2,340 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -33,6 +33,7 @@ FATE can be deployed on a single node or on multiple nodes. Choose the deploymen
 ### Cluster deployment
 Deploying FATE to multiple nodes to achieve scalability, reliability and manageability.
 - [Cluster deployment by CLI](./deploy/cluster-deploy): Using CLI to deploy a FATE cluster.
+- [Docker-Compose deployment](./deploy/docker-compose): Using docker-compose to deploy FATE.
 
 ### Quick Start
 - [Training Demo with Only FATE Installed From Pypi](doc/2.0/fate/ml)

diff --git a/deploy/docker-compose/README_zh.md b/deploy/docker-compose/README_zh.md
diff --git a/deploy/docker-compose/docker-deploy/docker_deploy.sh b/deploy/docker-compose/docker-deploy/docker_deploy.sh
@@ -0,0 +1,273 @@
+#!/bin/bash
+
+# Copyright 2019-2022 VMware, Inc.
+# Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# you may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+BASEDIR=$(dirname "$0")
+cd $BASEDIR
+WORKINGDIR=$(pwd)
+
+# fetch fate-python image
+source ${WORKINGDIR}/.env
+source ${WORKINGDIR}/parties.conf
+cd ${WORKINGDIR}
+
+Deploy() {
+	if [ "$1" = "" ]; then
+		echo "No party id was provided, please check your arguments "
+		exit 1
+	fi
+
+	while [ "$1" != "" ]; do
+		case $1 in
+		splitting_proxy)
+			shift
+			DeployPartyInternal $@
+			break
+			;;
+		all)
+			for party in ${party_list[*]}; do
+				if [ "$2" != "" ]; then
+					case $2 in
+					--training)
+						DeployPartyInternal $party
+						if [ "${exchangeip}" != "" ]; then
+							DeployPartyInternal exchange
+						fi
+						;;
+					esac
+				else
+					DeployPartyInternal $party
+					if [ "${exchangeip}" != "" ]; then
+						DeployPartyInternal exchange
+					fi
+				fi
+			done
+			break
+			;;
+		*)
+			if [ "$2" != "" ]; then
+				case $2 in
+				--training)
+					DeployPartyInternal $1
+					break
+					;;
+				esac
+			else
+				DeployPartyInternal $1
+			fi
+			;;
+		esac
+		shift
+
+	done
+}
+
+Delete() {
+	if [ "$1" = "" ]; then
+		echo "No party id was provided, please check your arguments "
+		exit 1
+	fi
+
+	while [ "$1" != "" ]; do
+		case $1 in
+		all)
+			for party in ${party_list[*]}; do
+				if [ "$2" != "" ]; then
+					DeleteCluster $party $2
+				else
+					DeleteCluster $party
+				fi
+			done
+			if [ "${exchangeip}" != "" ]; then
+				DeleteCluster exchange
+			fi
+			break
+			;;
+		*)
+			DeleteCluster $@
+			break
+			;;
+		esac
+	done
+}
+
+DeployPartyInternal() {
+	target_party_id=$1
+	# should not use localhost at any case
+	target_party_ip="127.0.0.1"
+
+	# check configuration files
+	if [ ! -d ${WORKINGDIR}/outputs ]; then
+		echo "Unable to find outputs dir, please generate config files first."
+		return 1
+	fi
+	if [ ! -f ${WORKINGDIR}/outputs/confs-${target_party_id}.tar ]; then
+		echo "Unable to find deployment file of training for party $target_party_id, please generate it first."
+		return 0
+	fi
+	# extract the ip address of the target party
+	if [ "$target_party_id" = "exchange" ]; then
+		target_party_ip=${exchangeip}
+	elif [ "$2" != "" ]; then
+		target_party_ip="$2"
+	else
+		for ((i = 0; i < ${#party_list[*]}; i++)); do
+			if [ "${party_list[$i]}" = "$target_party_id" ]; then
+				target_party_ip=${party_ip_list[$i]}
+			fi
+		done
+	fi
+	# verify the target_party_ip
+	if [ "$target_party_ip" = "127.0.0.1" ]; then
+		echo "Unable to find Party: $target_party_id, please check you input."
+		return 1
+	fi
+	if [ "$3" != "" ]; then
+		user=$3
+	fi
+
+	handleLocally confs
+	if [ "$local_flag" == "true" ]; then
+		return 0
+	fi
+
+	scp -P ${SSH_PORT} ${WORKINGDIR}/outputs/confs-$target_party_id.tar $user@$target_party_ip:~/
+	#rm -f ${WORKINGDIR}/outputs/confs-$target_party_id.tar
+	echo "$target_party_ip training cluster copy is ok!"
+	ssh -p ${SSH_PORT} -tt $user@$target_party_ip << eeooff
+mkdir -p $dir
+rm -f $dir/confs-$target_party_id.tar
+mv ~/confs-$target_party_id.tar $dir
+cd $dir
+tar -xzf confs-$target_party_id.tar
+cd confs-$target_party_id
+docker-compose down
+docker volume rm -f confs-${target_party_id}_shared_dir_examples
+docker volume rm -f confs-${target_party_id}_shared_dir_fate
+docker volume rm -f confs-${target_party_id}_sdownload_dir
+docker volume rm -f confs-${target_party_id}_fate_flow_logs
+
+docker-compose up -d
+cd ../
+rm -f confs-${target_party_id}.tar
+exit
+eeooff
+	echo "party ${target_party_id} deploy is ok!"
+}
+
+DeleteCluster() {
+	target_party_id=$1
+	cluster_type=$2
+	target_party_serving_ip="127.0.0.1"
+	target_party_ip="127.0.0.1"
+
+	# extract the ip address of the target party
+	if [ "$target_party_id" == "exchange" ]; then
+		target_party_ip=${exchangeip}
+	else
+		for ((i = 0; i < ${#party_list[*]}; i++)); do
+			if [ "${party_list[$i]}" = "$target_party_id" ]; then
+				target_party_ip=${party_ip_list[$i]}
+			fi
+		done
+	fi
+
+	# echo "target_party_ip: $target_party_ip"
+
+	for ((i = 0; i < ${#party_list[*]}; i++)); do
+		if [ "${party_list[$i]}" = "$target_party_id" ]; then
+			target_party_serving_ip=${serving_ip_list[$i]}
+		fi
+	done
+
+	#	echo "target_party_ip: $target_party_ip"
+	#	echo "cluster_type: $cluster_type"
+
+	# delete training cluster
+	if [ "$cluster_type" == "--training" ]; then
+		ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
+cd $dir/confs-$target_party_id
+docker-compose down
+exit
+eeooff
+		echo "party $target_party_id training cluster is deleted!"
+	# delete training cluster and serving cluster
+	else
+		# if party is exchange then delete exchange cluster
+		if [ "$target_party_id" == "exchange" ]; then
+			ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
+cd $dir/confs-$target_party_id
+docker-compose down
+exit
+eeooff
+		else
+			if [ "$target_party_ip" != "" ]; then
+			ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
+cd $dir/confs-$target_party_id
+docker-compose down
+exit
+eeooff
+			fi
+			echo "party $target_party_id training cluster is deleted!"
+		fi
+	fi
+}
+
+ShowUsage() {
+	echo "Usage: "
+	echo "Deploy all parties or specified partie(s): bash docker_deploy.sh partyid1[partyid2...] | all"
+}
+
+handleLocally() {
+	type=$1
+	for ip in $(hostname -I); do
+		if [ "$target_party_ip" == "$ip" ]; then
+			mkdir -p $dir
+			tar -xf ${WORKINGDIR}/outputs/${type}-${target_party_id}.tar -C $dir
+			cd ${dir}/${type}-${target_party_id}
+			docker-compose down
+			docker-compose up -d
+			local_flag="true"
+			return 0
+		fi
+	done
+	local_flag="false"
+}
+
+main() {
+
+	if [ "$1" = "" ] || [ "$1" = "--help" ]; then
+		ShowUsage
+		exit 1
+	elif [ "$1" = "--delete" ] || [ "$1" = "--del" ]; then
+		shift
+		Delete $@
+	else
+		Deploy "$@"
+	fi
+
+	for ((i = 0; i < ${#party_list[*]}; i++)); do
+	  if [ $party_list[$i] != "exchange" ]; then
+        echo "Use  ${party_ip_list[$i]}:8080 to access fateboard of party: ${party_list[$i]}"
+      fi
+      if [[ "$computing" == "spark"* ]]; then
+        echo "   Use  ${party_ip_list[$i]}:8888 to access Spark of party: ${party_list[$i]}"
+      fi
+	done
+
+	exit 0
+}
+
+main $@