From 9b9bff78c1cbe4c20a89e29ec05f71ade5edee19 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 9 Apr 2020 17:32:25 +1200 Subject: [PATCH 01/91] chore(build): add setup.sh script --- .gitignore | 1 + setup.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 setup.sh diff --git a/.gitignore b/.gitignore index 1521c8b7..b992d0c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ dist +.tmp diff --git a/setup.sh b/setup.sh new file mode 100755 index 00000000..733875f7 --- /dev/null +++ b/setup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +PLATFORM=$1 +ARCH=$2 +DOCKER_VERSION=$3 + +PLATFORM="linux" +ARCH="x86_64" +DOCKER_VERSION="18.09.3" + +DOWNLOAD_FOLDER=".tmp/download" + +rm -rf "${DOWNLOAD_FOLDER}" +mkdir -pv "${DOWNLOAD_FOLDER}" + +if [ "${PLATFORM}" == 'win' ]; then + wget -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip" + unzip "${DOWNLOAD_FOLDER}/docker-binaries.zip" -d "${DOWNLOAD_FOLDER}" + mv "${DOWNLOAD_FOLDER}/docker/docker.exe" dist/ +else + wget -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz" + tar -xf "${DOWNLOAD_FOLDER}/docker-binaries.tgz" -C "${DOWNLOAD_FOLDER}" + mv "${DOWNLOAD_FOLDER}/docker/docker" dist/ +fi + +exit 0 From dd66547f0d57f9e00eb74276d7bf9555c15c748b Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 22 Apr 2020 15:54:49 +0300 Subject: [PATCH 02/91] chore(build): add skip_compile arg --- dev.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/dev.sh b/dev.sh index 79b4b4d9..2a99de92 100755 --- a/dev.sh +++ b/dev.sh @@ -2,18 +2,27 @@ LOG_LEVEL=DEBUG CAP_HOST_MANAGEMENT=1 #Enabled by default. Change this to anything else to disable this feature -EDGE=0 +EDGE=1 TMP="/tmp" GIT_COMMIT_HASH=`git rev-parse --short HEAD` GIT_BRANCH_NAME=`git rev-parse --abbrev-ref HEAD` IMAGE_NAME="portainerci/agent:${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" -if [[ $# -ne 1 ]] ; then - echo "Usage: $(basename $0) " - exit 1 + +MODE="swarm" +if [[ $# -gt 0 ]] ; then + MODE=$1 fi -MODE=$1 +SKIP_COMPILE=false +if [[ $# -eq 2 ]] ; then + SKIP_COMPILE=true +fi + +if [[ "${MODE}" == 'help' ]]; then + echo "Usage: $(basename $0) " + exit 0 +fi function compile() { echo "Compilation..." @@ -36,7 +45,7 @@ function deploy_local() { echo "Image build..." docker build --no-cache -t "${IMAGE_NAME}" -f build/linux/Dockerfile . -# docker push "${IMAGE_NAME}" + # docker push "${IMAGE_NAME}" echo "Deployment..." @@ -58,7 +67,7 @@ function deploy_local() { function deploy_swarm() { DOCKER_MANAGER=tcp://10.0.7.10 DOCKER_NODE=tcp://10.0.7.11 -# DOCKER_NODE2=tcp://10.0.7.12 + # DOCKER_NODE2=tcp://10.0.7.12 EDGE_ID="a1a1c817-7f89-43b1-97e5-508d96c00be9" # generated via uuidgen @@ -70,14 +79,14 @@ function deploy_swarm() { docker -H "${DOCKER_MANAGER}:2375" network rm portainer-agent-dev-net docker -H "${DOCKER_MANAGER}:2375" rmi -f "${IMAGE_NAME}" docker -H "${DOCKER_NODE}:2375" rmi -f "${IMAGE_NAME}" -# docker -H "${DOCKER_NODE2}:2375" rmi -f "${IMAGE_NAME}" + # docker -H "${DOCKER_NODE2}:2375" rmi -f "${IMAGE_NAME}" echo "Building image locally and exporting to Swarm cluster..." docker build --no-cache -t "${IMAGE_NAME}" -f build/linux/Dockerfile . docker save "${IMAGE_NAME}" -o "${TMP}/portainer-agent.tar" docker -H "${DOCKER_MANAGER}:2375" load -i "${TMP}/portainer-agent.tar" docker -H "${DOCKER_NODE}:2375" load -i "${TMP}/portainer-agent.tar" -# docker -H "${DOCKER_NODE2}:2375" load -i "${TMP}/portainer-agent.tar" + # docker -H "${DOCKER_NODE2}:2375" load -i "${TMP}/portainer-agent.tar" echo "Sleep..." sleep 5 @@ -99,13 +108,15 @@ function deploy_swarm() { --publish mode=host,published=80,target=80 \ "${IMAGE_NAME}" -# --mount type=volume,src=portainer_agent_data,dst=/data \ + # --mount type=volume,src=portainer_agent_data,dst=/data \ docker -H "${DOCKER_MANAGER}:2375" service logs -f portainer-agent-dev } function main() { - compile + if [[ $SKIP_COMPILE == false ]]; then + compile + fi if [[ "${MODE}" == 'local' ]]; then deploy_local From ad86920327c6446466b36615e23ff1dc8917d69b Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 22 Apr 2020 15:57:27 +0300 Subject: [PATCH 03/91] fix(build): revert edge var --- dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev.sh b/dev.sh index 2a99de92..ae9ba200 100755 --- a/dev.sh +++ b/dev.sh @@ -2,7 +2,7 @@ LOG_LEVEL=DEBUG CAP_HOST_MANAGEMENT=1 #Enabled by default. Change this to anything else to disable this feature -EDGE=1 +EDGE=0 TMP="/tmp" GIT_COMMIT_HASH=`git rev-parse --short HEAD` GIT_BRANCH_NAME=`git rev-parse --abbrev-ref HEAD` From a673cfb0addda754f98b29d31ec03c537b905c86 Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Tue, 14 Apr 2020 11:13:12 +0300 Subject: [PATCH 04/91] feat(edge-compute): create a basic stack manager --- exec/stack.go | 99 ++++++++++++++++++++++++++++++++++++++++++++++ exec/stack_test.go | 59 +++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 exec/stack.go create mode 100644 exec/stack_test.go diff --git a/exec/stack.go b/exec/stack.go new file mode 100644 index 00000000..9ae2790b --- /dev/null +++ b/exec/stack.go @@ -0,0 +1,99 @@ +package exec + +import ( + "bytes" + "os/exec" + "path" + "runtime" + + "github.com/portainer/portainer/api" +) + +// EdgeStackManager represents a service for managing stacks. +type EdgeStackManager struct { + binaryPath string +} + +// NewEdgeStackManager initializes a new EdgeStackManager service. +// It also updates the configuration of the Docker CLI binary. +func NewEdgeStackManager(binaryPath string) (*EdgeStackManager, error) { + manager := &EdgeStackManager{ + binaryPath: binaryPath, + } + + return manager, nil +} + +// Login executes the docker login command against a list of registries (including DockerHub). +func (manager *EdgeStackManager) Login() error { + // dockerhub *portainer.DockerHub, registries []portainer.Registry + // command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath) + // for _, registry := range registries { + // if registry.Authentication { + // registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL) + // runCommandAndCaptureStdErr(command, registryArgs, nil, "") + // } + // } + + // if dockerhub.Authentication { + // dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password) + // runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "") + // } + return nil +} + +// Logout executes the docker logout command. +func (manager *EdgeStackManager) Logout() error { + command := manager.prepareDockerCommand(manager.binaryPath) + args := []string{"logout"} + return runCommandAndCaptureStdErr(command, args, "") + +} + +// Deploy executes the docker stack deploy command. +func (manager *EdgeStackManager) Deploy(name, projectPath, entryPoint string, prune bool) error { + stackFilePath := path.Join(projectPath, entryPoint) + command := manager.prepareDockerCommand(manager.binaryPath) + + args := []string{} + if prune { + args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, name) + } else { + args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, name) + } + + stackFolder := path.Dir(stackFilePath) + return runCommandAndCaptureStdErr(command, args, stackFolder) +} + +// Remove executes the docker stack rm command. +func (manager *EdgeStackManager) Remove(name string) error { + command := manager.prepareDockerCommand(manager.binaryPath) + args := []string{"stack", "rm", name} + return runCommandAndCaptureStdErr(command, args, "") +} + +func runCommandAndCaptureStdErr(command string, args []string, workingDir string) error { + var stderr bytes.Buffer + cmd := exec.Command(command, args...) + cmd.Stderr = &stderr + cmd.Dir = workingDir + + err := cmd.Run() + if err != nil { + return portainer.Error(stderr.String()) + } + + return nil +} + +func (manager *EdgeStackManager) prepareDockerCommand(binaryPath string) string { + // Assume Linux as a default + command := path.Join(binaryPath, "docker") + + if runtime.GOOS == "windows" { + command = path.Join(binaryPath, "docker.exe") + } + + return command +} diff --git a/exec/stack_test.go b/exec/stack_test.go new file mode 100644 index 00000000..292cef36 --- /dev/null +++ b/exec/stack_test.go @@ -0,0 +1,59 @@ +package exec + +import ( + "testing" +) + +const dockerPath = "" + +func TestNewEdgeStackManager(t *testing.T) { + _, err := NewEdgeStackManager(dockerPath) + if err != nil { + t.Errorf("Failed creating manager: %v", err) + } +} + +func TestLogin(t *testing.T) { + manager, err := NewEdgeStackManager(dockerPath) + if err != nil { + t.Errorf("Failed creating manager: %v", err) + } + + err = manager.Login() + if err != nil { + t.Errorf("Failed login: %v", err) + } +} + +func TestLogout(t *testing.T) { + manager, err := NewEdgeStackManager(dockerPath) + if err != nil { + t.Errorf("Failed creating manager: %v", err) + } + + err = manager.Login() + if err != nil { + t.Errorf("Failed login: %v", err) + } + + err = manager.Logout() + if err != nil { + t.Errorf("Failed logout: %v", err) + } +} + +func TestDeploy(t *testing.T) { + // Deploy/Remove are harder to test because we need: + // 1. create manager + // 2. create stack with file (os) + // 3. run deploy + +} + +func TestRemove(t *testing.T) { + // Deploy/Remove are harder to test because we need: + // 1. create manager + // 2. create stack with file (os) + // 3. run deploy + // 4. remove +} From c668f34cd4ba11edd8c66a60c1e8e6ab944887fc Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Tue, 14 Apr 2020 11:13:45 +0300 Subject: [PATCH 05/91] feat(edge-compute): connect the manager to server --- agent.go | 9 +++++++++ cmd/agent/main.go | 14 ++++++++++++++ http/handler/handler.go | 1 + http/server.go | 18 +++++++++++------- os/options.go | 2 ++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/agent.go b/agent.go index 3d00aa40..f88e7c63 100644 --- a/agent.go +++ b/agent.go @@ -16,6 +16,7 @@ type ( EdgeInactivityTimeout string EdgeInsecurePoll bool LogLevel string + DockerBinaryPath string } // ClusterMember is the representation of an agent inside a cluster. @@ -134,6 +135,14 @@ type ( Scheduler interface { Schedule(schedules []Schedule) error } + + // EdgeStackManager is a service to manager edge stacks + EdgeStackManager interface { + Login() error + Logout() error + Deploy(name, projectPath, entryPoint string, prune bool) error + Remove(name string) error + } ) const ( diff --git a/cmd/agent/main.go b/cmd/agent/main.go index b882eb05..f33bc937 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,6 +8,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" + "github.com/portainer/agent/exec" "github.com/portainer/agent/filesystem" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" @@ -85,6 +86,7 @@ func main() { } var tunnelOperator agent.TunnelOperator + var edgeStackManager agent.EdgeStackManager if options.EdgeMode { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) @@ -107,6 +109,17 @@ func main() { if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } + + edgeStackManager, err = exec.NewEdgeStackManager(options.DockerBinaryPath) + if err != nil { + log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) + } + + err = edgeStackManager.Login() + if err != nil { + log.Fatalf("[ERROR] [main,edge,stack] [message: Edge stack manager failed to login] [error: %s]", err) + } + } systemService := ghw.NewSystemService(agent.HostRoot) @@ -132,6 +145,7 @@ func main() { AgentTags: agentTags, AgentOptions: options, EdgeMode: options.EdgeMode, + EdgeStackManager: edgeStackManager, } if options.EdgeMode { diff --git a/http/handler/handler.go b/http/handler/handler.go index d3dcd9e4..63dafd0b 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -46,6 +46,7 @@ type Config struct { AgentOptions *agent.Options Secured bool EdgeMode bool + EdgeStackManager agent.EdgeStackManager } var dockerAPIVersionRegexp = regexp.MustCompile(`(/v[0-9]\.[0-9]*)?`) diff --git a/http/server.go b/http/server.go index dcaada81..4fdb10e1 100644 --- a/http/server.go +++ b/http/server.go @@ -20,6 +20,7 @@ type APIServer struct { agentTags map[string]string agentOptions *agent.Options edgeMode bool + edgeStackManager agent.EdgeStackManager } // APIServerConfig represents a server configuration @@ -34,6 +35,7 @@ type APIServerConfig struct { AgentTags map[string]string AgentOptions *agent.Options EdgeMode bool + EdgeStackManager agent.EdgeStackManager } // NewAPIServer returns a pointer to a APIServer. @@ -48,19 +50,21 @@ func NewAPIServer(config *APIServerConfig) *APIServer { agentTags: config.AgentTags, agentOptions: config.AgentOptions, edgeMode: config.EdgeMode, + edgeStackManager: config.EdgeStackManager, } } // Start starts a new web server by listening on the specified listenAddr. func (server *APIServer) StartUnsecured() error { config := &handler.Config{ - SystemService: server.systemService, - ClusterService: server.clusterService, - TunnelOperator: server.tunnelOperator, - AgentTags: server.agentTags, - AgentOptions: server.agentOptions, - EdgeMode: server.edgeMode, - Secured: false, + SystemService: server.systemService, + ClusterService: server.clusterService, + TunnelOperator: server.tunnelOperator, + AgentTags: server.agentTags, + AgentOptions: server.agentOptions, + EdgeMode: server.edgeMode, + Secured: false, + EdgeStackManager: server.edgeStackManager, } h := handler.NewHandler(config) diff --git a/os/options.go b/os/options.go index e362e91f..4c1f186b 100644 --- a/os/options.go +++ b/os/options.go @@ -23,6 +23,7 @@ const ( EnvKeyEdgeInactivityTimeout = "EDGE_INACTIVITY_TIMEOUT" EnvKeyEdgeInsecurePoll = "EDGE_INSECURE_POLL" EnvKeyLogLevel = "LOG_LEVEL" + EnvKeyDockerBinaryPath = "DOCKER_BINARY_PATH" ) type EnvOptionParser struct{} @@ -44,6 +45,7 @@ func (parser *EnvOptionParser) Options() (*agent.Options, error) { EdgeInactivityTimeout: agent.DefaultEdgeSleepInterval, EdgeInsecurePoll: false, LogLevel: agent.DefaultLogLevel, + DockerBinaryPath: os.Getenv(EnvKeyDockerBinaryPath), } if os.Getenv(EnvKeyCapHostManagement) == "1" { From b342a0224f0a8439672f7e1a1195016924dc18ee Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 10:10:37 +0300 Subject: [PATCH 06/91] style(agent): Capitalize Edge stack --- agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent.go b/agent.go index f88e7c63..c162cd8d 100644 --- a/agent.go +++ b/agent.go @@ -136,7 +136,7 @@ type ( Schedule(schedules []Schedule) error } - // EdgeStackManager is a service to manager edge stacks + // EdgeStackManager is a service to manager Edge stacks EdgeStackManager interface { Login() error Logout() error From e14174cef3077f959575fb5c8b0bc03a90d81416 Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 10:11:06 +0300 Subject: [PATCH 07/91] fix(agent): remove login --- cmd/agent/main.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index f33bc937..15a26fa2 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -114,12 +114,6 @@ func main() { if err != nil { log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) } - - err = edgeStackManager.Login() - if err != nil { - log.Fatalf("[ERROR] [main,edge,stack] [message: Edge stack manager failed to login] [error: %s]", err) - } - } systemService := ghw.NewSystemService(agent.HostRoot) From d46138f64502e70236a653e02449cdaecd7d5597 Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 10:12:10 +0300 Subject: [PATCH 08/91] style(ege-stacks): remove commented code --- exec/stack.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/exec/stack.go b/exec/stack.go index 9ae2790b..94c1d6c2 100644 --- a/exec/stack.go +++ b/exec/stack.go @@ -26,19 +26,7 @@ func NewEdgeStackManager(binaryPath string) (*EdgeStackManager, error) { // Login executes the docker login command against a list of registries (including DockerHub). func (manager *EdgeStackManager) Login() error { - // dockerhub *portainer.DockerHub, registries []portainer.Registry - // command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath) - // for _, registry := range registries { - // if registry.Authentication { - // registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL) - // runCommandAndCaptureStdErr(command, registryArgs, nil, "") - // } - // } - - // if dockerhub.Authentication { - // dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password) - // runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "") - // } + // Not implemented yet. return nil } From b3a5f7d01d3467a57f77b0daca9accba321f4cbf Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 10:12:38 +0300 Subject: [PATCH 09/91] refactor(edge-stacks): remove unit test file --- exec/stack_test.go | 59 ---------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 exec/stack_test.go diff --git a/exec/stack_test.go b/exec/stack_test.go deleted file mode 100644 index 292cef36..00000000 --- a/exec/stack_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package exec - -import ( - "testing" -) - -const dockerPath = "" - -func TestNewEdgeStackManager(t *testing.T) { - _, err := NewEdgeStackManager(dockerPath) - if err != nil { - t.Errorf("Failed creating manager: %v", err) - } -} - -func TestLogin(t *testing.T) { - manager, err := NewEdgeStackManager(dockerPath) - if err != nil { - t.Errorf("Failed creating manager: %v", err) - } - - err = manager.Login() - if err != nil { - t.Errorf("Failed login: %v", err) - } -} - -func TestLogout(t *testing.T) { - manager, err := NewEdgeStackManager(dockerPath) - if err != nil { - t.Errorf("Failed creating manager: %v", err) - } - - err = manager.Login() - if err != nil { - t.Errorf("Failed login: %v", err) - } - - err = manager.Logout() - if err != nil { - t.Errorf("Failed logout: %v", err) - } -} - -func TestDeploy(t *testing.T) { - // Deploy/Remove are harder to test because we need: - // 1. create manager - // 2. create stack with file (os) - // 3. run deploy - -} - -func TestRemove(t *testing.T) { - // Deploy/Remove are harder to test because we need: - // 1. create manager - // 2. create stack with file (os) - // 3. run deploy - // 4. remove -} From ce7d1e2df0d065ff2aee92b8e9b7e572e0c3e951 Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 10:24:02 +0300 Subject: [PATCH 10/91] feat(edge-stack): remove docker binary env --- agent.go | 3 ++- cmd/agent/main.go | 2 +- os/options.go | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent.go b/agent.go index c162cd8d..18f33aaf 100644 --- a/agent.go +++ b/agent.go @@ -16,7 +16,6 @@ type ( EdgeInactivityTimeout string EdgeInsecurePoll bool LogLevel string - DockerBinaryPath string } // ClusterMember is the representation of an agent inside a cluster. @@ -222,4 +221,6 @@ const ( DataDirectory = "/data" // EdgeKeyFile is the name of the file used to persist the Edge key associated to the agent. EdgeKeyFile = "agent_edge_key" + // DockerBinaryPath is the path of the docker binary + DockerBinaryPath = "/app" ) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 15a26fa2..2eeb1d0f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -110,7 +110,7 @@ func main() { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } - edgeStackManager, err = exec.NewEdgeStackManager(options.DockerBinaryPath) + edgeStackManager, err = exec.NewEdgeStackManager(agent.DockerBinaryPath) if err != nil { log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) } diff --git a/os/options.go b/os/options.go index 4c1f186b..124b1a1a 100644 --- a/os/options.go +++ b/os/options.go @@ -45,7 +45,6 @@ func (parser *EnvOptionParser) Options() (*agent.Options, error) { EdgeInactivityTimeout: agent.DefaultEdgeSleepInterval, EdgeInsecurePoll: false, LogLevel: agent.DefaultLogLevel, - DockerBinaryPath: os.Getenv(EnvKeyDockerBinaryPath), } if os.Getenv(EnvKeyCapHostManagement) == "1" { From f505ddd3c9eb0eb869c553930d2fd07a6a21270c Mon Sep 17 00:00:00 2001 From: Chaim Lev Ari Date: Thu, 16 Apr 2020 11:51:12 +0300 Subject: [PATCH 11/91] refactor(edge-stack): use errors instead of portainer.error --- exec/stack.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/exec/stack.go b/exec/stack.go index 94c1d6c2..a6376eb9 100644 --- a/exec/stack.go +++ b/exec/stack.go @@ -2,11 +2,10 @@ package exec import ( "bytes" + "errors" "os/exec" "path" "runtime" - - "github.com/portainer/portainer/api" ) // EdgeStackManager represents a service for managing stacks. @@ -69,7 +68,7 @@ func runCommandAndCaptureStdErr(command string, args []string, workingDir string err := cmd.Run() if err != nil { - return portainer.Error(stderr.String()) + return errors.New(stderr.String()) } return nil From 9f1242d3be0fd40dc1ce4aa5e332e2709fc66b26 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 22 Apr 2020 12:39:06 +0300 Subject: [PATCH 12/91] feat(tunnel): start polling only if leader --- agent.go | 1 + cmd/agent/main.go | 49 +++++++++++++++++++++++++++++++++++++---------- docker/docker.go | 17 ++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/agent.go b/agent.go index 18f33aaf..5f72f09d 100644 --- a/agent.go +++ b/agent.go @@ -98,6 +98,7 @@ type ( GetInformationFromDockerEngine() (map[string]string, error) GetContainerIpFromDockerEngine(containerName string, ignoreNonSwarmNetworks bool) (string, error) GetServiceNameFromDockerEngine(containerName string) (string, error) + IsLeaderNode() (bool, error) } // TLSService is used to create TLS certificates to use enable HTTPS. diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2eeb1d0f..88e5a7d7 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -105,7 +105,7 @@ func main() { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } - err := enableEdgeMode(tunnelOperator, clusterService, options) + err := enableEdgeMode(tunnelOperator, clusterService, infoService, options) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } @@ -162,7 +162,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, options *agent.Options) error { +func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, infoService agent.InfoService, options *agent.Options) error { edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { @@ -177,16 +177,45 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } - if clusterService != nil { - tags := clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = clusterService.UpdateTags(tags) - if err != nil { - return err - } + if clusterService == nil { + return tunnelOperator.Start() } - return tunnelOperator.Start() + tags := clusterService.GetTags() + tags[agent.MemberTagEdgeKeySet] = "set" + err = clusterService.UpdateTags(tags) + if err != nil { + return err + } + isLeaderNode, err := infoService.IsLeaderNode() + if err != nil { + return err + } + if isLeaderNode { + return tunnelOperator.Start() + } + + ticker := time.NewTicker(time.Duration(5) * time.Second) + go func() { + for { + select { + case <-ticker.C: + isLeaderNode, err := infoService.IsLeaderNode() + if err != nil { + log.Printf("[ERROR] [edge,http,poll] [message: an error occured during short poll] [error: %s]", err) + } + + if isLeaderNode { + tunnelOperator.Start() + ticker.Stop() + } + // case <-operator.refreshSignal: + // log.Println("[DEBUG] [http,edge,poll] [message: shutting down Portainer short-polling client]") + // ticker.Stop() + // return + } + } + }() } log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") diff --git a/docker/docker.go b/docker/docker.go index 9607bcbd..32a6770c 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -109,3 +109,20 @@ func (service *InfoService) GetServiceNameFromDockerEngine(containerName string) return containerInspect.Config.Labels[serviceNameLabel], nil } + +// IsLeaderNode returns true if the node is the leader node of the swarm cluster +func (service *InfoService) IsLeaderNode() (bool, error) { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(agent.SupportedDockerAPIVersion)) + if err != nil { + return false, err + } + + defer cli.Close() + + node, _, err := cli.NodeInspectWithRaw(context.Background(), "self") + if err != nil { + return false, err + } + + return node.ManagerStatus.Leader, nil +} From 18bc8226373d1eb8f280c4da2fef7e67cb1e93a2 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 10:57:24 +0300 Subject: [PATCH 13/91] refactor(agent): create poll interval constant --- agent.go | 2 ++ cmd/agent/main.go | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agent.go b/agent.go index 5f72f09d..73820a9a 100644 --- a/agent.go +++ b/agent.go @@ -166,6 +166,8 @@ const ( DefaultEdgePollInterval = "5s" // DefaultEdgeSleepInterval is the default interval after which the agent will close the tunnel if no activity. DefaultEdgeSleepInterval = "5m" + // DefaultSwarmLeaderCheckInterval is the default interval used to check if current node was promoted to leader + DefaultSwarmLeaderCheckInterval = "1m" // SupportedDockerAPIVersion is the minimum Docker API version supported by the agent. SupportedDockerAPIVersion = "1.24" // HTTPTargetHeaderName is the name of the header used to specify a target node. diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 88e5a7d7..f0e123be 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -187,15 +187,17 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl if err != nil { return err } + isLeaderNode, err := infoService.IsLeaderNode() if err != nil { return err } + if isLeaderNode { return tunnelOperator.Start() } - ticker := time.NewTicker(time.Duration(5) * time.Second) + ticker := time.NewTicker(time.Duration(agent.DefaultSwarmLeaderCheckInterval) * time.Second) go func() { for { select { @@ -209,10 +211,6 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl tunnelOperator.Start() ticker.Stop() } - // case <-operator.refreshSignal: - // log.Println("[DEBUG] [http,edge,poll] [message: shutting down Portainer short-polling client]") - // ticker.Stop() - // return } } }() From 57344fd754708d995cf3286c9754082901d2ef9a Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 12:44:50 +0300 Subject: [PATCH 14/91] refactor(edge): move leader check to function --- cmd/agent/main.go | 63 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index f0e123be..65a211ad 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -188,32 +188,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } - isLeaderNode, err := infoService.IsLeaderNode() - if err != nil { - return err - } - - if isLeaderNode { - return tunnelOperator.Start() - } - - ticker := time.NewTicker(time.Duration(agent.DefaultSwarmLeaderCheckInterval) * time.Second) - go func() { - for { - select { - case <-ticker.C: - isLeaderNode, err := infoService.IsLeaderNode() - if err != nil { - log.Printf("[ERROR] [edge,http,poll] [message: an error occured during short poll] [error: %s]", err) - } - - if isLeaderNode { - tunnelOperator.Start() - ticker.Stop() - } - } - } - }() + return pollLeaderStatus(infoService, tunnelOperator) } log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") @@ -243,6 +218,42 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return nil } +func pollLeaderStatus(infoService agent.InfoService, tunnelOperator agent.TunnelOperator) error { + isLeaderNode, err := infoService.IsLeaderNode() + if err != nil { + return err + } + + if isLeaderNode { + return tunnelOperator.Start() + } + + pollFrequency, err := time.ParseDuration(agent.DefaultSwarmLeaderCheckInterval) + if err != nil { + return err + } + + ticker := time.NewTicker(time.Duration(pollFrequency) * time.Second) + go func() { + for { + select { + case <-ticker.C: + isLeaderNode, err := infoService.IsLeaderNode() + if err != nil { + log.Printf("[ERROR] [edge,http,poll] [message: an error occured during short poll] [error: %s]", err) + } + + if isLeaderNode { + tunnelOperator.Start() + ticker.Stop() + } + } + } + }() + + return nil +} + func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { edgeKey := options.EdgeKey From 95bdf1faea2d03f3fbebee2802cab29a94d77eb8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 12:54:50 +0300 Subject: [PATCH 15/91] feat(edge): add logs when leader is changed --- cmd/agent/main.go | 3 ++- dev.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 65a211ad..c8b5ed4b 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -240,10 +240,11 @@ func pollLeaderStatus(infoService agent.InfoService, tunnelOperator agent.Tunnel case <-ticker.C: isLeaderNode, err := infoService.IsLeaderNode() if err != nil { - log.Printf("[ERROR] [edge,http,poll] [message: an error occured during short poll] [error: %s]", err) + log.Printf("[ERROR] [main,edge,swarm] [message: an error occured during short poll] [error: %s]", err) } if isLeaderNode { + log.Printf("[DEBUG] [main,edge,swarm] [message: node was promoted to leader]") tunnelOperator.Start() ticker.Stop() } diff --git a/dev.sh b/dev.sh index ae9ba200..2a99de92 100755 --- a/dev.sh +++ b/dev.sh @@ -2,7 +2,7 @@ LOG_LEVEL=DEBUG CAP_HOST_MANAGEMENT=1 #Enabled by default. Change this to anything else to disable this feature -EDGE=0 +EDGE=1 TMP="/tmp" GIT_COMMIT_HASH=`git rev-parse --short HEAD` GIT_BRANCH_NAME=`git rev-parse --abbrev-ref HEAD` From 462ee4132c34eb026c33cb2823ba395fcc78f746 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 17:14:48 +0300 Subject: [PATCH 16/91] refactor(tunnel): move check for leader to tunnel --- cmd/agent/main.go | 68 ++++++++++--------------------------------- http/tunnel/tunnel.go | 58 +++++++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index c8b5ed4b..2dbd81e9 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -91,11 +91,14 @@ func main() { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) operatorConfig := &tunnel.OperatorConfig{ - APIServerAddr: apiServerAddr, - EdgeID: options.EdgeID, - PollFrequency: agent.DefaultEdgePollInterval, - InactivityTimeout: options.EdgeInactivityTimeout, - InsecurePoll: options.EdgeInsecurePoll, + APIServerAddr: apiServerAddr, + EdgeID: options.EdgeID, + PollFrequency: agent.DefaultEdgePollInterval, + InactivityTimeout: options.EdgeInactivityTimeout, + InsecurePoll: options.EdgeInsecurePoll, + IsCluster: clusterMode, + InfoService: infoService, + LeaderCheckFrequency: agent.DefaultSwarmLeaderCheckInterval, } log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) @@ -177,18 +180,16 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } - if clusterService == nil { - return tunnelOperator.Start() - } - - tags := clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = clusterService.UpdateTags(tags) - if err != nil { - return err + if clusterService != nil { + tags := clusterService.GetTags() + tags[agent.MemberTagEdgeKeySet] = "set" + err = clusterService.UpdateTags(tags) + if err != nil { + return err + } } - return pollLeaderStatus(infoService, tunnelOperator) + return tunnelOperator.Start() } log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") @@ -218,43 +219,6 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return nil } -func pollLeaderStatus(infoService agent.InfoService, tunnelOperator agent.TunnelOperator) error { - isLeaderNode, err := infoService.IsLeaderNode() - if err != nil { - return err - } - - if isLeaderNode { - return tunnelOperator.Start() - } - - pollFrequency, err := time.ParseDuration(agent.DefaultSwarmLeaderCheckInterval) - if err != nil { - return err - } - - ticker := time.NewTicker(time.Duration(pollFrequency) * time.Second) - go func() { - for { - select { - case <-ticker.C: - isLeaderNode, err := infoService.IsLeaderNode() - if err != nil { - log.Printf("[ERROR] [main,edge,swarm] [message: an error occured during short poll] [error: %s]", err) - } - - if isLeaderNode { - log.Printf("[DEBUG] [main,edge,swarm] [message: node was promoted to leader]") - tunnelOperator.Start() - ticker.Stop() - } - } - } - }() - - return nil -} - func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { edgeKey := options.EdgeKey diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index d5a61c34..14375630 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -27,22 +27,28 @@ type Operator struct { pollIntervalInSeconds float64 insecurePoll bool inactivityTimeout time.Duration + leaderCheckFrequency time.Duration edgeID string key *edgeKey httpClient *http.Client tunnelClient agent.ReverseTunnelClient scheduleManager agent.Scheduler + infoService agent.InfoService lastActivity time.Time refreshSignal chan struct{} + isCluster bool } // OperatorConfig represents the configuration used to create a new Operator. type OperatorConfig struct { - APIServerAddr string - EdgeID string - InactivityTimeout string - PollFrequency string - InsecurePoll bool + APIServerAddr string + EdgeID string + InactivityTimeout string + PollFrequency string + InsecurePoll bool + IsCluster bool + LeaderCheckFrequency string + InfoService agent.InfoService } // NewTunnelOperator creates a new reverse tunnel operator @@ -57,15 +63,23 @@ func NewTunnelOperator(config *OperatorConfig) (*Operator, error) { return nil, err } + leaderCheckFrequency, err := time.ParseDuration(config.LeaderCheckFrequency) + if err != nil { + return nil, err + } + return &Operator{ apiServerAddr: config.APIServerAddr, edgeID: config.EdgeID, pollIntervalInSeconds: pollFrequency.Seconds(), insecurePoll: config.InsecurePoll, inactivityTimeout: inactivityTimeout, + leaderCheckFrequency: leaderCheckFrequency, tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: make(chan struct{}), + infoService: config.InfoService, + isCluster: config.IsCluster, }, nil } @@ -122,17 +136,47 @@ func (operator *Operator) ResetActivityTimer() { // if needed as well as manage schedules. // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. +// If node is part of a cluster, it will start a check for leader status, and will run those loops only when the node +// is promoted to a leader func (operator *Operator) Start() error { if operator.key == nil { return errors.New("missing Edge key") } - operator.startStatusPollLoop() - operator.startActivityMonitoringLoop() + if !operator.isCluster { + log.Printf("[DEBUG] [http,edge,poll] [message: not a cluster, starting poll]") + operator.startStatusPollLoop() + operator.startActivityMonitoringLoop() + return nil + } + operator.startLeaderCheckLoop() return nil } +func (operator *Operator) startLeaderCheckLoop() { + ticker := time.NewTicker(operator.leaderCheckFrequency) + go func() { + for { + select { + case <-ticker.C: + isLeaderNode, err := operator.infoService.IsLeaderNode() + if err != nil { + log.Printf("[ERROR] [http,edge,poll] [message: an error occured during leader check] [error: %s]", err) + return + } + + if isLeaderNode { + log.Printf("[DEBUG] [http,edge,poll] [message: node is a leader, starting poll]") + operator.startStatusPollLoop() + operator.startActivityMonitoringLoop() + ticker.Stop() + } + } + } + }() +} + func (operator *Operator) restartStatusPollLoop() { close(operator.refreshSignal) operator.refreshSignal = make(chan struct{}) From 3578c21311d8db329ae74fd7bbf85ebe5e0e51b1 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 17:15:40 +0300 Subject: [PATCH 17/91] fix(tunnel): remove stop of loop --- http/tunnel/tunnel.go | 1 - 1 file changed, 1 deletion(-) diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index 14375630..f8b2bcfa 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -163,7 +163,6 @@ func (operator *Operator) startLeaderCheckLoop() { isLeaderNode, err := operator.infoService.IsLeaderNode() if err != nil { log.Printf("[ERROR] [http,edge,poll] [message: an error occured during leader check] [error: %s]", err) - return } if isLeaderNode { From ee3fa05feabd1e70de78ad38cf546a9b229884f6 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 26 Apr 2020 17:17:47 +0300 Subject: [PATCH 18/91] fix(main): remove not used service --- cmd/agent/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2dbd81e9..29a5cbb5 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -108,7 +108,7 @@ func main() { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } - err := enableEdgeMode(tunnelOperator, clusterService, infoService, options) + err := enableEdgeMode(tunnelOperator, clusterService, options) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } @@ -165,7 +165,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, infoService agent.InfoService, options *agent.Options) error { +func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, options *agent.Options) error { edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { From 2576a2575ea5314fc1c5a9454775f90fa753e9e0 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 27 Apr 2020 10:02:16 +0300 Subject: [PATCH 19/91] fix(tunnel): revent tunnel --- http/tunnel/tunnel.go | 57 ++++++------------------------------------- 1 file changed, 7 insertions(+), 50 deletions(-) diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index f8b2bcfa..d5a61c34 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -27,28 +27,22 @@ type Operator struct { pollIntervalInSeconds float64 insecurePoll bool inactivityTimeout time.Duration - leaderCheckFrequency time.Duration edgeID string key *edgeKey httpClient *http.Client tunnelClient agent.ReverseTunnelClient scheduleManager agent.Scheduler - infoService agent.InfoService lastActivity time.Time refreshSignal chan struct{} - isCluster bool } // OperatorConfig represents the configuration used to create a new Operator. type OperatorConfig struct { - APIServerAddr string - EdgeID string - InactivityTimeout string - PollFrequency string - InsecurePoll bool - IsCluster bool - LeaderCheckFrequency string - InfoService agent.InfoService + APIServerAddr string + EdgeID string + InactivityTimeout string + PollFrequency string + InsecurePoll bool } // NewTunnelOperator creates a new reverse tunnel operator @@ -63,23 +57,15 @@ func NewTunnelOperator(config *OperatorConfig) (*Operator, error) { return nil, err } - leaderCheckFrequency, err := time.ParseDuration(config.LeaderCheckFrequency) - if err != nil { - return nil, err - } - return &Operator{ apiServerAddr: config.APIServerAddr, edgeID: config.EdgeID, pollIntervalInSeconds: pollFrequency.Seconds(), insecurePoll: config.InsecurePoll, inactivityTimeout: inactivityTimeout, - leaderCheckFrequency: leaderCheckFrequency, tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: make(chan struct{}), - infoService: config.InfoService, - isCluster: config.IsCluster, }, nil } @@ -136,46 +122,17 @@ func (operator *Operator) ResetActivityTimer() { // if needed as well as manage schedules. // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. -// If node is part of a cluster, it will start a check for leader status, and will run those loops only when the node -// is promoted to a leader func (operator *Operator) Start() error { if operator.key == nil { return errors.New("missing Edge key") } - if !operator.isCluster { - log.Printf("[DEBUG] [http,edge,poll] [message: not a cluster, starting poll]") - operator.startStatusPollLoop() - operator.startActivityMonitoringLoop() - return nil - } + operator.startStatusPollLoop() + operator.startActivityMonitoringLoop() - operator.startLeaderCheckLoop() return nil } -func (operator *Operator) startLeaderCheckLoop() { - ticker := time.NewTicker(operator.leaderCheckFrequency) - go func() { - for { - select { - case <-ticker.C: - isLeaderNode, err := operator.infoService.IsLeaderNode() - if err != nil { - log.Printf("[ERROR] [http,edge,poll] [message: an error occured during leader check] [error: %s]", err) - } - - if isLeaderNode { - log.Printf("[DEBUG] [http,edge,poll] [message: node is a leader, starting poll]") - operator.startStatusPollLoop() - operator.startActivityMonitoringLoop() - ticker.Stop() - } - } - } - }() -} - func (operator *Operator) restartStatusPollLoop() { close(operator.refreshSignal) operator.refreshSignal = make(chan struct{}) From 2b792d078898ae54471c2dc06cc3bd8e2511d1b1 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 27 Apr 2020 14:51:51 +0300 Subject: [PATCH 20/91] refactor(tunnel): move config check loop to main --- agent.go | 12 ++++-- cmd/agent/main.go | 69 +++++++++++++++++++++++----------- docker/docker.go | 23 ++++++------ http/edge.go | 2 - http/handler/key/key_create.go | 2 - http/tunnel/tunnel.go | 19 ++++++++-- 6 files changed, 83 insertions(+), 44 deletions(-) diff --git a/agent.go b/agent.go index 73820a9a..f862a9d0 100644 --- a/agent.go +++ b/agent.go @@ -98,7 +98,6 @@ type ( GetInformationFromDockerEngine() (map[string]string, error) GetContainerIpFromDockerEngine(containerName string, ignoreNonSwarmNetworks bool) (string, error) GetServiceNameFromDockerEngine(containerName string) (string, error) - IsLeaderNode() (bool, error) } // TLSService is used to create TLS certificates to use enable HTTPS. @@ -124,6 +123,7 @@ type ( // the reverse tunnel. TunnelOperator interface { Start() error + Stop() error IsKeySet() bool SetKey(key string) error GetKey() string @@ -166,8 +166,8 @@ const ( DefaultEdgePollInterval = "5s" // DefaultEdgeSleepInterval is the default interval after which the agent will close the tunnel if no activity. DefaultEdgeSleepInterval = "5m" - // DefaultSwarmLeaderCheckInterval is the default interval used to check if current node was promoted to leader - DefaultSwarmLeaderCheckInterval = "1m" + // DefaultConfigCheckInterval is the default interval used to check if node config changed + DefaultConfigCheckInterval = "1m" // SupportedDockerAPIVersion is the minimum Docker API version supported by the agent. SupportedDockerAPIVersion = "1.24" // HTTPTargetHeaderName is the name of the header used to specify a target node. @@ -199,6 +199,8 @@ const ( // MemberTagKeyAgentPort is the name of the label storing information about the port exposed // by the agent. MemberTagKeyAgentPort = "AgentPort" + // MemberTagKeyIsLeader is the name of the label storing whether the node is a swarm leader + MemberTagKeyIsLeader = "NodeIsLeader" // MemberTagKeyNodeName is the name of the label storing information about the name of the // node where the agent is running. MemberTagKeyNodeName = "NodeName" @@ -214,6 +216,10 @@ const ( NodeRoleManager = "manager" // NodeRoleWorker represents a worker node. NodeRoleWorker = "worker" + // EngineStatusSwarm represents a swarm docker engine + EngineStatusSwarm = "swarm" + // EngineStatusStandalone represents a standalone docker engine + EngineStatusStandalone = "standalone" // TLSCertPath is the default path to the TLS certificate file. TLSCertPath = "cert.pem" // TLSKeyPath is the default path to the TLS key file. diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 29a5cbb5..0f7994f4 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -29,15 +29,17 @@ func main() { logutils.SetupLogger(options.LogLevel) var infoService agent.InfoService = docker.NewInfoService() - agentTags, err := retrieveInformationFromDockerEnvironment(infoService) + + agentTags, err := infoService.GetInformationFromDockerEngine() if err != nil { log.Fatalf("[ERROR] [main,docker] [message: Unable to retrieve information from Docker] [error: %s]", err) } + agentTags[agent.MemberTagKeyAgentPort] = options.AgentServerPort log.Printf("[DEBUG] [main,configuration] [Member tags: %+v]", agentTags) clusterMode := false - if agentTags[agent.MemberTagEngineStatus] == "swarm" { + if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm { clusterMode = true log.Println("[INFO] [main] [message: Agent running on a Swarm cluster node. Running in cluster mode]") } @@ -91,14 +93,11 @@ func main() { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) operatorConfig := &tunnel.OperatorConfig{ - APIServerAddr: apiServerAddr, - EdgeID: options.EdgeID, - PollFrequency: agent.DefaultEdgePollInterval, - InactivityTimeout: options.EdgeInactivityTimeout, - InsecurePoll: options.EdgeInsecurePoll, - IsCluster: clusterMode, - InfoService: infoService, - LeaderCheckFrequency: agent.DefaultSwarmLeaderCheckInterval, + APIServerAddr: apiServerAddr, + EdgeID: options.EdgeID, + PollFrequency: agent.DefaultEdgePollInterval, + InactivityTimeout: options.EdgeInactivityTimeout, + InsecurePoll: options.EdgeInsecurePoll, } log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) @@ -108,7 +107,7 @@ func main() { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } - err := enableEdgeMode(tunnelOperator, clusterService, options) + err := enableEdgeMode(tunnelOperator, clusterService, infoService, options) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } @@ -165,7 +164,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, options *agent.Options) error { +func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, infoService agent.InfoService, options *agent.Options) error { edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { @@ -188,8 +187,43 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } } + loopIntervalFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) + if err != nil { + return err + } + + ticker := time.NewTicker(time.Duration(loopIntervalFrequency)) + + go func() { + for { + select { + case <-ticker.C: + key := tunnelOperator.GetKey() + if key == "" { + continue + } + + agentTags, err := infoService.GetInformationFromDockerEngine() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured during docker config check] [error: %s]", err) + continue + } + + if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { + err = tunnelOperator.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured while starting poll] [error: %s]", err) + } + } else { + err = tunnelOperator.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured while stopping poll] [error: %s]", err) + } + } + } + } - return tunnelOperator.Start() + }() } log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") @@ -294,12 +328,3 @@ func parseOptions() (*agent.Options, error) { optionParser := os.NewEnvOptionParser() return optionParser.Options() } - -func retrieveInformationFromDockerEnvironment(infoService agent.InfoService) (map[string]string, error) { - agentTags, err := infoService.GetInformationFromDockerEngine() - if err != nil { - return nil, err - } - - return agentTags, nil -} diff --git a/docker/docker.go b/docker/docker.go index 32a6770c..318d3916 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -41,12 +41,20 @@ func (service *InfoService) GetInformationFromDockerEngine() (map[string]string, info[agent.MemberTagKeyNodeName] = dockerInfo.Name if dockerInfo.Swarm.NodeID == "" { - info[agent.MemberTagEngineStatus] = "standalone" + info[agent.MemberTagEngineStatus] = agent.EngineStatusStandalone } else { - info[agent.MemberTagEngineStatus] = "swarm" + info[agent.MemberTagEngineStatus] = agent.EngineStatusSwarm info[agent.MemberTagKeyNodeRole] = agent.NodeRoleWorker if dockerInfo.Swarm.ControlAvailable { info[agent.MemberTagKeyNodeRole] = agent.NodeRoleManager + isLeader, err := service.isLeaderNode(cli, dockerInfo.Swarm.NodeID) + if err != nil { + return nil, err + } + + if isLeader { + info[agent.MemberTagKeyIsLeader] = "1" + } } } @@ -111,15 +119,8 @@ func (service *InfoService) GetServiceNameFromDockerEngine(containerName string) } // IsLeaderNode returns true if the node is the leader node of the swarm cluster -func (service *InfoService) IsLeaderNode() (bool, error) { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(agent.SupportedDockerAPIVersion)) - if err != nil { - return false, err - } - - defer cli.Close() - - node, _, err := cli.NodeInspectWithRaw(context.Background(), "self") +func (service *InfoService) isLeaderNode(cli *client.Client, nodeID string) (bool, error) { + node, _, err := cli.NodeInspectWithRaw(context.Background(), nodeID) if err != nil { return false, err } diff --git a/http/edge.go b/http/edge.go index 3a43e054..f8a75644 100644 --- a/http/edge.go +++ b/http/edge.go @@ -77,8 +77,6 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { go server.propagateKeyInCluster(tags[agent.MemberTagKeyNodeName], key) } - go server.tunnelOperator.Start() - w.Write([]byte("Agent setup OK. You can close this page.")) server.Shutdown() } diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index c2c62b2b..30dfe842 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -55,7 +55,5 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe } } - go handler.tunnelOperator.Start() - return response.Empty(w) } diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index d5a61c34..854ebbae 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -65,7 +65,7 @@ func NewTunnelOperator(config *OperatorConfig) (*Operator, error) { inactivityTimeout: inactivityTimeout, tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), - refreshSignal: make(chan struct{}), + refreshSignal: nil, }, nil } @@ -126,16 +126,27 @@ func (operator *Operator) Start() error { if operator.key == nil { return errors.New("missing Edge key") } - + if operator.refreshSignal != nil { + return nil + } + operator.refreshSignal = make(chan struct{}) operator.startStatusPollLoop() operator.startActivityMonitoringLoop() return nil } +// Stop stops the poll loop +func (operator *Operator) Stop() error { + if operator.refreshSignal != nil { + close(operator.refreshSignal) + operator.refreshSignal = nil + } + return nil +} + func (operator *Operator) restartStatusPollLoop() { - close(operator.refreshSignal) - operator.refreshSignal = make(chan struct{}) + operator.Stop() operator.startStatusPollLoop() } From 987b847fef6d6a781a938f5422db99e6582355f5 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 28 Apr 2020 08:38:41 +0300 Subject: [PATCH 21/91] fix(edge): start docker polling also on local --- cmd/agent/main.go | 77 +++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 0f7994f4..9a7ac17d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -187,43 +187,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } } - loopIntervalFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) - if err != nil { - return err - } - - ticker := time.NewTicker(time.Duration(loopIntervalFrequency)) - - go func() { - for { - select { - case <-ticker.C: - key := tunnelOperator.GetKey() - if key == "" { - continue - } - - agentTags, err := infoService.GetInformationFromDockerEngine() - if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured during docker config check] [error: %s]", err) - continue - } - - if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { - err = tunnelOperator.Start() - if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured while starting poll] [error: %s]", err) - } - } else { - err = tunnelOperator.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured while stopping poll] [error: %s]", err) - } - } - } - } - }() } log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") @@ -250,6 +214,47 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl } }() + loopIntervalFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) + if err != nil { + return err + } + + ticker := time.NewTicker(time.Duration(loopIntervalFrequency)) + + go func() { + for { + select { + case <-ticker.C: + log.Printf("[DEBUG] [main,edge,poll] [message: checking docker config and key]") + + key := tunnelOperator.GetKey() + if key == "" { + continue + } + + agentTags, err := infoService.GetInformationFromDockerEngine() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured during docker config check] [error: %s]", err) + continue + } + + if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { + log.Printf("[DEBUG] [main,edge,poll] [message: this is either a leader or a standalone agent, starting tunnel operator]") + err = tunnelOperator.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured while starting poll] [error: %s]", err) + } + } else { + log.Printf("[DEBUG] [main,edge,poll] [message: this is neither a leader nor a standalone agent, starting tunnel operator]") + err = tunnelOperator.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,poll] [message: an error occured while stopping poll] [error: %s]", err) + } + } + } + } + }() + return nil } From 2ff79ac7ed3ee37a813050fcad00565cd3e92719 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 28 Apr 2020 08:54:41 +0300 Subject: [PATCH 22/91] fix(tunnel): start key server only if key is not set --- cmd/agent/main.go | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 9a7ac17d..2ec42e47 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -187,32 +187,31 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } } + } else { + log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") + edgeServer := http.NewEdgeServer(tunnelOperator, clusterService) - } - - log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - edgeServer := http.NewEdgeServer(tunnelOperator, clusterService) - - go func() { - log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) + go func() { + log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) - err := edgeServer.Start(options.EdgeServerAddr, options.EdgeServerPort) - if err != nil { - log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) - } + err := edgeServer.Start(options.EdgeServerAddr, options.EdgeServerPort) + if err != nil { + log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) + } - log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") - }() + log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") + }() - go func() { - timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) - <-timer1.C + go func() { + timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) + <-timer1.C - if !tunnelOperator.IsKeySet() { - log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) - edgeServer.Shutdown() - } - }() + if !tunnelOperator.IsKeySet() { + log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) + edgeServer.Shutdown() + } + }() + } loopIntervalFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { From 59a101535043803bbe810b19e145d43bf98ef146 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:26:13 +0300 Subject: [PATCH 23/91] fix(main): replace docker check category --- cmd/agent/main.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2ec42e47..315ff3e4 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -224,7 +224,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl for { select { case <-ticker.C: - log.Printf("[DEBUG] [main,edge,poll] [message: checking docker config and key]") + log.Printf("[DEBUG] [main,edge,docker] [message: checking docker config and key]") key := tunnelOperator.GetKey() if key == "" { @@ -233,21 +233,21 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl agentTags, err := infoService.GetInformationFromDockerEngine() if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured during docker config check] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured during docker config check] [error: %s]", err) continue } if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { - log.Printf("[DEBUG] [main,edge,poll] [message: this is either a leader or a standalone agent, starting tunnel operator]") + log.Printf("[DEBUG] [main,edge,docker] [message: this is either a leader or a standalone agent, starting tunnel operator]") err = tunnelOperator.Start() if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured while starting poll] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) } } else { - log.Printf("[DEBUG] [main,edge,poll] [message: this is neither a leader nor a standalone agent, starting tunnel operator]") + log.Printf("[DEBUG] [main,edge,docker] [message: this is neither a leader nor a standalone agent, starting tunnel operator]") err = tunnelOperator.Stop() if err != nil { - log.Printf("[ERROR] [main,edge,poll] [message: an error occured while stopping poll] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping poll] [error: %s]", err) } } } From b4c2de6ada589eff1ebcd32cc9d122a586eda6cf Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:39:27 +0300 Subject: [PATCH 24/91] fix(main): replace debug messages --- cmd/agent/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 315ff3e4..6dffb47b 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -224,8 +224,6 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl for { select { case <-ticker.C: - log.Printf("[DEBUG] [main,edge,docker] [message: checking docker config and key]") - key := tunnelOperator.GetKey() if key == "" { continue @@ -238,18 +236,19 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl } if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { - log.Printf("[DEBUG] [main,edge,docker] [message: this is either a leader or a standalone agent, starting tunnel operator]") err = tunnelOperator.Start() if err != nil { log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) } + } else { - log.Printf("[DEBUG] [main,edge,docker] [message: this is neither a leader nor a standalone agent, starting tunnel operator]") err = tunnelOperator.Stop() if err != nil { log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping poll] [error: %s]", err) } } + + log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %s]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") } } }() From 3cba6517244b21892e1d696116263d5c104c78f2 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:39:25 +0300 Subject: [PATCH 25/91] fix(main): change error messages Co-authored-by: Anthony Lapenna --- cmd/agent/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 6dffb47b..25cc842e 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -231,24 +231,24 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl agentTags, err := infoService.GetInformationFromDockerEngine() if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured during docker config check] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) continue } if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { err = tunnelOperator.Start() if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting the short-poll process] [error: %s]", err) } } else { err = tunnelOperator.Stop() if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping poll] [error: %s]", err) + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) } } - log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %s]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") + log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") } } }() From 7d5aa67f397cebe694a7c8ad9114fabec32521ec Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:42:30 +0300 Subject: [PATCH 26/91] style(docker): remove comment --- docker/docker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker.go b/docker/docker.go index 318d3916..c18551bf 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -118,7 +118,6 @@ func (service *InfoService) GetServiceNameFromDockerEngine(containerName string) return containerInspect.Config.Labels[serviceNameLabel], nil } -// IsLeaderNode returns true if the node is the leader node of the swarm cluster func (service *InfoService) isLeaderNode(cli *client.Client, nodeID string) (bool, error) { node, _, err := cli.NodeInspectWithRaw(context.Background(), nodeID) if err != nil { From 34614f455bb7396a34196189b515a4cb113e21ce Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:40:33 +0300 Subject: [PATCH 27/91] refactor(main): remove unneeded cast Co-authored-by: Anthony Lapenna --- cmd/agent/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 25cc842e..cca44d53 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -218,7 +218,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl return err } - ticker := time.NewTicker(time.Duration(loopIntervalFrequency)) + ticker := time.NewTicker(loopIntervalFrequency) go func() { for { From dd056bd5ce7bd420fbb696e8d4ea8198ea7e41a9 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:44:33 +0300 Subject: [PATCH 28/91] fix(main): fix logs --- cmd/agent/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index cca44d53..10cc4313 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -172,7 +172,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl } if edgeKey != "" { - log.Println("[DEBUG] [main,edge] [message: Edge key available. Starting tunnel operator.]") + log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") err := tunnelOperator.SetKey(edgeKey) if err != nil { From 50d5e78c8395322ea5bf52d6196fa61500a70d38 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:45:26 +0300 Subject: [PATCH 29/91] refactor(main): rename runtime check frequency var --- cmd/agent/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 10cc4313..e2f02ef9 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -213,12 +213,12 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl }() } - loopIntervalFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) + runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { return err } - ticker := time.NewTicker(loopIntervalFrequency) + ticker := time.NewTicker(runtimeCheckFrequency) go func() { for { From 6a3f6a87ecf2dd10405ef37cbd9a511e2db6600e Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:55:31 +0300 Subject: [PATCH 30/91] refactor(edge): build enableEdgeMode from functions --- cmd/agent/main.go | 163 ++++++++++++++++++++++++++-------------------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index e2f02ef9..2e0c0468 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -165,7 +165,6 @@ func startAPIServer(config *http.APIServerConfig) error { } func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, infoService agent.InfoService, options *agent.Options) error { - edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { return err @@ -174,86 +173,18 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl if edgeKey != "" { log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") - err := tunnelOperator.SetKey(edgeKey) + err := associateEdgeKey(tunnelOperator, clusterService, edgeKey) if err != nil { return err } - if clusterService != nil { - tags := clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = clusterService.UpdateTags(tags) - if err != nil { - return err - } - } } else { log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - edgeServer := http.NewEdgeServer(tunnelOperator, clusterService) - - go func() { - log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) - - err := edgeServer.Start(options.EdgeServerAddr, options.EdgeServerPort) - if err != nil { - log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) - } - log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") - }() - - go func() { - timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) - <-timer1.C - - if !tunnelOperator.IsKeySet() { - log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) - edgeServer.Shutdown() - } - }() - } - - runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) - if err != nil { - return err + serveEdgeUI(tunnelOperator, clusterService, options) } - ticker := time.NewTicker(runtimeCheckFrequency) - - go func() { - for { - select { - case <-ticker.C: - key := tunnelOperator.GetKey() - if key == "" { - continue - } - - agentTags, err := infoService.GetInformationFromDockerEngine() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) - continue - } - - if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { - err = tunnelOperator.Start() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting the short-poll process] [error: %s]", err) - } - - } else { - err = tunnelOperator.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) - } - } - - log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") - } - } - }() - - return nil + return startRuntimeConfigCheckProcess(tunnelOperator, infoService) } func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { @@ -331,3 +262,91 @@ func parseOptions() (*agent.Options, error) { optionParser := os.NewEnvOptionParser() return optionParser.Options() } + +func associateEdgeKey(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, edgeKey string) error { + err := tunnelOperator.SetKey(edgeKey) + if err != nil { + return err + } + + if clusterService != nil { + tags := clusterService.GetTags() + tags[agent.MemberTagEdgeKeySet] = "set" + err = clusterService.UpdateTags(tags) + if err != nil { + return err + } + } + + return nil +} + +func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, options *agent.Options) { + edgeServer := http.NewEdgeServer(tunnelOperator, clusterService) + + go func() { + log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) + + err := edgeServer.Start(options.EdgeServerAddr, options.EdgeServerPort) + if err != nil { + log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) + } + + log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") + }() + + go func() { + timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) + <-timer1.C + + if !tunnelOperator.IsKeySet() { + log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) + edgeServer.Shutdown() + } + }() +} + +func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService) error { + + runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) + if err != nil { + return err + } + + ticker := time.NewTicker(runtimeCheckFrequency) + + go func() { + for { + select { + case <-ticker.C: + key := tunnelOperator.GetKey() + if key == "" { + continue + } + + agentTags, err := infoService.GetInformationFromDockerEngine() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) + continue + } + + if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { + err = tunnelOperator.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) + } + + } else { + err = tunnelOperator.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) + } + } + + log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %b]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") + } + } + }() + + return nil +} From e0afaa06471a4e423fc700893e85f80354e86526 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 16:13:42 +0300 Subject: [PATCH 31/91] refactor(docker): construct docker info from functions --- docker/docker.go | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index c18551bf..7f444746 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -41,21 +41,14 @@ func (service *InfoService) GetInformationFromDockerEngine() (map[string]string, info[agent.MemberTagKeyNodeName] = dockerInfo.Name if dockerInfo.Swarm.NodeID == "" { - info[agent.MemberTagEngineStatus] = agent.EngineStatusStandalone + getStandaloneInfo(info) } else { - info[agent.MemberTagEngineStatus] = agent.EngineStatusSwarm - info[agent.MemberTagKeyNodeRole] = agent.NodeRoleWorker - if dockerInfo.Swarm.ControlAvailable { - info[agent.MemberTagKeyNodeRole] = agent.NodeRoleManager - isLeader, err := service.isLeaderNode(cli, dockerInfo.Swarm.NodeID) - if err != nil { - return nil, err - } - - if isLeader { - info[agent.MemberTagKeyIsLeader] = "1" - } + + err := getSwarmInformation(info, dockerInfo, cli) + if err != nil { + return nil, err } + } return info, nil @@ -118,11 +111,25 @@ func (service *InfoService) GetServiceNameFromDockerEngine(containerName string) return containerInspect.Config.Labels[serviceNameLabel], nil } -func (service *InfoService) isLeaderNode(cli *client.Client, nodeID string) (bool, error) { - node, _, err := cli.NodeInspectWithRaw(context.Background(), nodeID) - if err != nil { - return false, err +func getStandaloneInfo(info map[string]string) { + info[agent.MemberTagEngineStatus] = agent.EngineStatusStandalone +} + +func getSwarmInformation(info map[string]string, dockerInfo types.Info, cli *client.Client) error { + info[agent.MemberTagEngineStatus] = agent.EngineStatusSwarm + info[agent.MemberTagKeyNodeRole] = agent.NodeRoleWorker + if dockerInfo.Swarm.ControlAvailable { + info[agent.MemberTagKeyNodeRole] = agent.NodeRoleManager + + node, _, err := cli.NodeInspectWithRaw(context.Background(), dockerInfo.Swarm.NodeID) + if err != nil { + return err + } + + if node.ManagerStatus.Leader { + info[agent.MemberTagKeyIsLeader] = "1" + } } - return node.ManagerStatus.Leader, nil + return nil } From ea0ba382fff55e002d06223253fe8523575dfcc2 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 11:03:13 +0300 Subject: [PATCH 32/91] refactor(main): move log command --- cmd/agent/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2e0c0468..a946ee33 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -330,6 +330,8 @@ func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoSer continue } + log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") + if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { err = tunnelOperator.Start() if err != nil { @@ -342,8 +344,6 @@ func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoSer log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) } } - - log.Printf("[DEBUG] [main,edge,docker] [] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %b]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") } } }() From 4a53fd97add255dc42df682bb7e157006724ddca Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 11:48:47 +0300 Subject: [PATCH 33/91] chore(build): copy the whole dist folder to docker --- build/linux/Dockerfile | 2 +- build/windows/Dockerfile | 2 +- dev.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/linux/Dockerfile b/build/linux/Dockerfile index 60d0221b..3a2dcdd0 100644 --- a/build/linux/Dockerfile +++ b/build/linux/Dockerfile @@ -2,7 +2,7 @@ FROM portainer/base WORKDIR /app -COPY dist/agent /app/ +COPY dist /app/ COPY static /app/static ENTRYPOINT ["./agent"] diff --git a/build/windows/Dockerfile b/build/windows/Dockerfile index c4b27405..8712e6ca 100644 --- a/build/windows/Dockerfile +++ b/build/windows/Dockerfile @@ -4,7 +4,7 @@ USER ContainerAdministrator WORKDIR /app -COPY dist/agent.exe /app/ +COPY dist /app/ COPY static /app/static ENTRYPOINT ["C:/app/agent.exe"] diff --git a/dev.sh b/dev.sh index 2a99de92..d6e3c735 100755 --- a/dev.sh +++ b/dev.sh @@ -27,7 +27,6 @@ fi function compile() { echo "Compilation..." - rm -rf dist/* cd cmd/agent GOOS="linux" GOARCH="amd64" CGO_ENABLED=0 go build -a --installsuffix cgo --ldflags '-s' rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi @@ -54,6 +53,7 @@ function deploy_local() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ + -e EDGE_KEY="aHR0cDovLzE3Mi4xNy4wLjE6OTAwMHwxNzIuMTcuMC4xOjgwMDB8ZWU6YTQ6YTg6ZDY6YzQ6Njk6MjY6MGI6N2Y6MDk6YTU6YTk6N2Y6NzQ6YTk6ZDZ8Nw" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker/volumes:/var/lib/docker/volumes \ -v /:/host \ @@ -100,6 +100,7 @@ function deploy_swarm() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ + -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ --mode global \ --mount type=bind,src=//var/run/docker.sock,dst=/var/run/docker.sock \ --mount type=bind,src=//var/lib/docker/volumes,dst=/var/lib/docker/volumes \ From 0f0f3d0f5c6ec8eafee96b0d3b4fe9bf529d600e Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 22 Apr 2020 15:45:06 +0300 Subject: [PATCH 34/91] feat(http): receive stack status --- http/tunnel/poll.go | 14 ++++++++++++++ http/tunnel/stack.go | 0 2 files changed, 14 insertions(+) create mode 100644 http/tunnel/stack.go diff --git a/http/tunnel/poll.go b/http/tunnel/poll.go index b9205c99..de16c12f 100644 --- a/http/tunnel/poll.go +++ b/http/tunnel/poll.go @@ -17,12 +17,18 @@ import ( const clientDefaultPollTimeout = 5 +type stackStatus struct { + EdgeStackID int + Version int +} + type pollStatusResponse struct { Status string `json:"status"` Port int `json:"port"` Schedules []agent.Schedule `json:"schedules"` CheckinInterval float64 `json:"checkin"` Credentials string `json:"credentials"` + Stacks []stackStatus `json:"stacks"` } func (operator *Operator) createHTTPClient(timeout float64) { @@ -104,6 +110,14 @@ func (operator *Operator) poll() error { go operator.restartStatusPollLoop() } + if responseData.Stacks != nil { + err := operator.handleStacks(responseData.Stacks) + if err != nil { + log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) + return err + } + } + return nil } diff --git a/http/tunnel/stack.go b/http/tunnel/stack.go new file mode 100644 index 00000000..e69de29b From 482eb9deeca81ce2a97e5392e7984071979f134a Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 11:42:48 +0300 Subject: [PATCH 35/91] refactor(docker): rename dockerStackService --- agent.go | 6 ++-- cmd/agent/main.go | 6 ++-- exec/stack.go | 21 ++++++------ http/handler/handler.go | 18 +++++----- http/server.go | 76 ++++++++++++++++++++--------------------- 5 files changed, 62 insertions(+), 65 deletions(-) diff --git a/agent.go b/agent.go index f862a9d0..1b48db1d 100644 --- a/agent.go +++ b/agent.go @@ -136,11 +136,11 @@ type ( Schedule(schedules []Schedule) error } - // EdgeStackManager is a service to manager Edge stacks - EdgeStackManager interface { + // DockerStackService is a service to manager Edge stacks + DockerStackService interface { Login() error Logout() error - Deploy(name, projectPath, entryPoint string, prune bool) error + Deploy(name, stackFileContent string, prune bool) error Remove(name string) error } ) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index a946ee33..6fff9772 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -88,7 +88,6 @@ func main() { } var tunnelOperator agent.TunnelOperator - var edgeStackManager agent.EdgeStackManager if options.EdgeMode { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) @@ -112,9 +111,9 @@ func main() { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } - edgeStackManager, err = exec.NewEdgeStackManager(agent.DockerBinaryPath) + dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) if err != nil { - log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) } } @@ -141,7 +140,6 @@ func main() { AgentTags: agentTags, AgentOptions: options, EdgeMode: options.EdgeMode, - EdgeStackManager: edgeStackManager, } if options.EdgeMode { diff --git a/exec/stack.go b/exec/stack.go index a6376eb9..b3760bfc 100644 --- a/exec/stack.go +++ b/exec/stack.go @@ -8,15 +8,15 @@ import ( "runtime" ) -// EdgeStackManager represents a service for managing stacks. -type EdgeStackManager struct { +// DockerStackService represents a service for managing stacks. +type DockerStackService struct { binaryPath string } -// NewEdgeStackManager initializes a new EdgeStackManager service. +// NewDockerStackService initializes a new DockerStackService service. // It also updates the configuration of the Docker CLI binary. -func NewEdgeStackManager(binaryPath string) (*EdgeStackManager, error) { - manager := &EdgeStackManager{ +func NewDockerStackService(binaryPath string) (*DockerStackService, error) { + manager := &DockerStackService{ binaryPath: binaryPath, } @@ -24,13 +24,13 @@ func NewEdgeStackManager(binaryPath string) (*EdgeStackManager, error) { } // Login executes the docker login command against a list of registries (including DockerHub). -func (manager *EdgeStackManager) Login() error { +func (manager *DockerStackService) Login() error { // Not implemented yet. return nil } // Logout executes the docker logout command. -func (manager *EdgeStackManager) Logout() error { +func (manager *DockerStackService) Logout() error { command := manager.prepareDockerCommand(manager.binaryPath) args := []string{"logout"} return runCommandAndCaptureStdErr(command, args, "") @@ -38,8 +38,7 @@ func (manager *EdgeStackManager) Logout() error { } // Deploy executes the docker stack deploy command. -func (manager *EdgeStackManager) Deploy(name, projectPath, entryPoint string, prune bool) error { - stackFilePath := path.Join(projectPath, entryPoint) +func (manager *DockerStackService) Deploy(name, stackFilePath string, prune bool) error { command := manager.prepareDockerCommand(manager.binaryPath) args := []string{} @@ -54,7 +53,7 @@ func (manager *EdgeStackManager) Deploy(name, projectPath, entryPoint string, pr } // Remove executes the docker stack rm command. -func (manager *EdgeStackManager) Remove(name string) error { +func (manager *DockerStackService) Remove(name string) error { command := manager.prepareDockerCommand(manager.binaryPath) args := []string{"stack", "rm", name} return runCommandAndCaptureStdErr(command, args, "") @@ -74,7 +73,7 @@ func runCommandAndCaptureStdErr(command string, args []string, workingDir string return nil } -func (manager *EdgeStackManager) prepareDockerCommand(binaryPath string) string { +func (manager *DockerStackService) prepareDockerCommand(binaryPath string) string { // Assume Linux as a default command := path.Join(binaryPath, "docker") diff --git a/http/handler/handler.go b/http/handler/handler.go index 63dafd0b..9cf2e6f5 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -38,15 +38,15 @@ type Handler struct { // Config represents a server handler configuration // used to create a new handler type Config struct { - SystemService agent.SystemService - ClusterService agent.ClusterService - SignatureService agent.DigitalSignatureService - TunnelOperator agent.TunnelOperator - AgentTags map[string]string - AgentOptions *agent.Options - Secured bool - EdgeMode bool - EdgeStackManager agent.EdgeStackManager + SystemService agent.SystemService + ClusterService agent.ClusterService + SignatureService agent.DigitalSignatureService + TunnelOperator agent.TunnelOperator + AgentTags map[string]string + AgentOptions *agent.Options + Secured bool + EdgeMode bool + DockerStackService agent.DockerStackService } var dockerAPIVersionRegexp = regexp.MustCompile(`(/v[0-9]\.[0-9]*)?`) diff --git a/http/server.go b/http/server.go index 4fdb10e1..cdff27af 100644 --- a/http/server.go +++ b/http/server.go @@ -11,60 +11,60 @@ import ( // APIServer is the web server exposing the API of an agent. type APIServer struct { - addr string - port string - systemService agent.SystemService - clusterService agent.ClusterService - signatureService agent.DigitalSignatureService - tunnelOperator agent.TunnelOperator - agentTags map[string]string - agentOptions *agent.Options - edgeMode bool - edgeStackManager agent.EdgeStackManager + addr string + port string + systemService agent.SystemService + clusterService agent.ClusterService + signatureService agent.DigitalSignatureService + tunnelOperator agent.TunnelOperator + agentTags map[string]string + agentOptions *agent.Options + edgeMode bool + dockerStackService agent.DockerStackService } // APIServerConfig represents a server configuration // used to create a new API server type APIServerConfig struct { - Addr string - Port string - SystemService agent.SystemService - ClusterService agent.ClusterService - SignatureService agent.DigitalSignatureService - TunnelOperator agent.TunnelOperator - AgentTags map[string]string - AgentOptions *agent.Options - EdgeMode bool - EdgeStackManager agent.EdgeStackManager + Addr string + Port string + SystemService agent.SystemService + ClusterService agent.ClusterService + SignatureService agent.DigitalSignatureService + TunnelOperator agent.TunnelOperator + AgentTags map[string]string + AgentOptions *agent.Options + EdgeMode bool + DockerStackService agent.DockerStackService } // NewAPIServer returns a pointer to a APIServer. func NewAPIServer(config *APIServerConfig) *APIServer { return &APIServer{ - addr: config.Addr, - port: config.Port, - systemService: config.SystemService, - clusterService: config.ClusterService, - signatureService: config.SignatureService, - tunnelOperator: config.TunnelOperator, - agentTags: config.AgentTags, - agentOptions: config.AgentOptions, - edgeMode: config.EdgeMode, - edgeStackManager: config.EdgeStackManager, + addr: config.Addr, + port: config.Port, + systemService: config.SystemService, + clusterService: config.ClusterService, + signatureService: config.SignatureService, + tunnelOperator: config.TunnelOperator, + agentTags: config.AgentTags, + agentOptions: config.AgentOptions, + edgeMode: config.EdgeMode, + dockerStackService: config.DockerStackService, } } // Start starts a new web server by listening on the specified listenAddr. func (server *APIServer) StartUnsecured() error { config := &handler.Config{ - SystemService: server.systemService, - ClusterService: server.clusterService, - TunnelOperator: server.tunnelOperator, - AgentTags: server.agentTags, - AgentOptions: server.agentOptions, - EdgeMode: server.edgeMode, - Secured: false, - EdgeStackManager: server.edgeStackManager, + SystemService: server.systemService, + ClusterService: server.clusterService, + TunnelOperator: server.tunnelOperator, + AgentTags: server.agentTags, + AgentOptions: server.agentOptions, + EdgeMode: server.edgeMode, + Secured: false, + DockerStackService: server.dockerStackService, } h := handler.NewHandler(config) From 33ebcd73c8d393e67ea98e71efaf7463fbcd4dc5 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 11:43:19 +0300 Subject: [PATCH 36/91] feat(http): create portainer client --- http/portainerclient/portainer_client.go | 117 +++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 http/portainerclient/portainer_client.go diff --git a/http/portainerclient/portainer_client.go b/http/portainerclient/portainer_client.go new file mode 100644 index 00000000..74fa5320 --- /dev/null +++ b/http/portainerclient/portainer_client.go @@ -0,0 +1,117 @@ +package portainerclient + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "strconv" + "time" + + "github.com/portainer/agent" +) + +// PortainerClient is used to execute HTTP requests against the agent API +type PortainerClient struct { + httpClient *http.Client + serverAddress string + endpointID string + edgeID string +} + +// NewPortainerClient returns a pointer to a new PortainerClient instance +func NewPortainerClient(serverAddress, endpointID, edgeID string) *PortainerClient { + return &PortainerClient{ + serverAddress: serverAddress, + endpointID: endpointID, + httpClient: &http.Client{ + Timeout: time.Second * 3, + }, + } +} + +type stackConfigResponse struct { + StackFileContent string + Prune bool +} + +// GetEdgeStackConfig fetches the Edge stack config +func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool, error) { + requestURL := fmt.Sprintf("http://%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) + + req, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return "", false, err + } + + req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, client.edgeID) + + resp, err := client.httpClient.Do(req) + if err != nil { + return "", false, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("[ERROR] [http,client] [response_code: %d] [message: GetEdgeStackConfig operation failed]", resp.StatusCode) + return "", false, errors.New("GetEdgeStackConfig operation failed") + } + + var data stackConfigResponse + err = json.NewDecoder(resp.Body).Decode(&data) + if err != nil { + return "", false, err + } + + return data.StackFileContent, data.Prune, nil +} + +type setEdgeStackStatusPayload struct { + Error string + Type int + EndpointID int +} + +// SetEdgeStackStatus set the Edge stack status on portainer server +func (client *PortainerClient) SetEdgeStackStatus(edgeStackID, edgeStackStatus int, error string) error { + endpointID, err := strconv.Atoi(client.endpointID) + if err != nil { + return err + } + + payload := setEdgeStackStatusPayload{ + Error: error, + Type: edgeStackStatus, + EndpointID: endpointID, + } + + data, err := json.Marshal(payload) + if err != nil { + return err + } + + requestURL := fmt.Sprintf("http://%s/api/edge_stacks/%d/status", client.serverAddress, edgeStackID) + + req, err := http.NewRequest(http.MethodPut, requestURL, bytes.NewReader(data)) + if err != nil { + return err + } + + req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, client.edgeID) + + resp, err := client.httpClient.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("[ERROR] [http,client] [response_code: %d] [message: SetEdgeStackStatus operation failed]", resp.StatusCode) + return errors.New("SetEdgeStackStatus operation failed") + } + + return nil +} From 610407fb545fd043424d726748ef9f9591172cc0 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:21:56 +0300 Subject: [PATCH 37/91] refactor(edge): move key management to service --- agent.go | 10 ++- cmd/agent/main.go | 52 +++++++++------ http/edge.go | 6 +- http/handler/handler.go | 7 +- http/handler/key/handler.go | 4 +- http/handler/key/key_create.go | 4 +- http/handler/key/key_inspect.go | 4 +- http/key/key.go | 115 ++++++++++++++++++++++++++++++++ http/server.go | 5 ++ http/tunnel/key.go | 39 ----------- http/tunnel/poll.go | 30 ++++++--- http/tunnel/stack.go | 0 http/tunnel/tunnel.go | 63 +++++------------ 13 files changed, 212 insertions(+), 127 deletions(-) create mode 100644 http/key/key.go delete mode 100644 http/tunnel/key.go delete mode 100644 http/tunnel/stack.go diff --git a/agent.go b/agent.go index 1b48db1d..3a049dcb 100644 --- a/agent.go +++ b/agent.go @@ -124,11 +124,17 @@ type ( TunnelOperator interface { Start() error Stop() error + CloseTunnel() error + ResetActivityTimer() + } + + // EdgeKeyService is a service that manages edge key + EdgeKeyService interface { IsKeySet() bool SetKey(key string) error GetKey() string - CloseTunnel() error - ResetActivityTimer() + GetPortainerConfig() (string, string, error) + GetTunnelConfig() (string, string, error) } // Scheduler is used to manage schedules diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 6fff9772..877afb59 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,11 +8,13 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" - "github.com/portainer/agent/exec" "github.com/portainer/agent/filesystem" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" "github.com/portainer/agent/http/client" + + // "github.com/portainer/agent/http/edgestacks" + "github.com/portainer/agent/http/key" "github.com/portainer/agent/http/tunnel" "github.com/portainer/agent/logutils" "github.com/portainer/agent/net" @@ -87,10 +89,16 @@ func main() { defer clusterService.Leave() } + var edgeKeyService agent.EdgeKeyService var tunnelOperator agent.TunnelOperator if options.EdgeMode { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) + edgeKeyService, err = key.NewService() + if err != nil { + log.Fatalf("[ERROR] [main,edge,key] [message: Unable to create key service] [error: %s]", err) + } + operatorConfig := &tunnel.OperatorConfig{ APIServerAddr: apiServerAddr, EdgeID: options.EdgeID, @@ -101,20 +109,25 @@ func main() { log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) - tunnelOperator, err = tunnel.NewTunnelOperator(operatorConfig) + // dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) + // if err != nil { + // log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) + // } + + // edgeStackManager, err := edgestacks.NewManager(dockerStackService, edgeKeyService, options.EdgeID) + // if err != nil { + // log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) + // } + + tunnelOperator, err = tunnel.NewTunnelOperator(edgeKeyService, operatorConfig) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } - err := enableEdgeMode(tunnelOperator, clusterService, infoService, options) + err = enableEdgeMode(tunnelOperator, clusterService, edgeKeyService, infoService, options) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } - - dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) - if err != nil { - log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) - } } systemService := ghw.NewSystemService(agent.HostRoot) @@ -135,6 +148,7 @@ func main() { Port: options.AgentServerPort, SystemService: systemService, ClusterService: clusterService, + EdgeKeyService: edgeKeyService, SignatureService: signatureService, TunnelOperator: tunnelOperator, AgentTags: agentTags, @@ -162,7 +176,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, infoService agent.InfoService, options *agent.Options) error { +func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, infoService agent.InfoService, options *agent.Options) error { edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { return err @@ -171,7 +185,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl if edgeKey != "" { log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") - err := associateEdgeKey(tunnelOperator, clusterService, edgeKey) + err := associateEdgeKey(keyService, clusterService, edgeKey) if err != nil { return err } @@ -179,10 +193,10 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl } else { log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - serveEdgeUI(tunnelOperator, clusterService, options) + serveEdgeUI(tunnelOperator, clusterService, keyService, options) } - return startRuntimeConfigCheckProcess(tunnelOperator, infoService) + return startRuntimeConfigCheckProcess(tunnelOperator, infoService, keyService) } func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { @@ -261,8 +275,8 @@ func parseOptions() (*agent.Options, error) { return optionParser.Options() } -func associateEdgeKey(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, edgeKey string) error { - err := tunnelOperator.SetKey(edgeKey) +func associateEdgeKey(edgeKeyService agent.EdgeKeyService, clusterService agent.ClusterService, edgeKey string) error { + err := edgeKeyService.SetKey(edgeKey) if err != nil { return err } @@ -279,8 +293,8 @@ func associateEdgeKey(tunnelOperator agent.TunnelOperator, clusterService agent. return nil } -func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, options *agent.Options) { - edgeServer := http.NewEdgeServer(tunnelOperator, clusterService) +func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, options *agent.Options) { + edgeServer := http.NewEdgeServer(tunnelOperator, clusterService, keyService) go func() { log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) @@ -297,14 +311,14 @@ func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.Clust timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) <-timer1.C - if !tunnelOperator.IsKeySet() { + if !keyService.IsKeySet() { log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) edgeServer.Shutdown() } }() } -func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService) error { +func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService, keyService agent.EdgeKeyService) error { runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { @@ -317,7 +331,7 @@ func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoSer for { select { case <-ticker.C: - key := tunnelOperator.GetKey() + key := keyService.GetKey() if key == "" { continue } diff --git a/http/edge.go b/http/edge.go index f8a75644..5f79d750 100644 --- a/http/edge.go +++ b/http/edge.go @@ -18,13 +18,15 @@ type EdgeServer struct { httpServer *http.Server tunnelOperator agent.TunnelOperator clusterService agent.ClusterService + keyService agent.EdgeKeyService } // NewEdgeServer returns a pointer to a new instance of EdgeServer. -func NewEdgeServer(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService) *EdgeServer { +func NewEdgeServer(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService) *EdgeServer { return &EdgeServer{ tunnelOperator: tunnelOperator, clusterService: clusterService, + keyService: keyService, } } @@ -59,7 +61,7 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { return } - err = server.tunnelOperator.SetKey(key) + err = server.keyService.SetKey(key) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/http/handler/handler.go b/http/handler/handler.go index 9cf2e6f5..e1cb3119 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -32,6 +32,7 @@ type Handler struct { hostHandler *host.Handler pingHandler *ping.Handler securedProtocol bool + edgeKeyService agent.EdgeKeyService tunnelOperator agent.TunnelOperator } @@ -41,6 +42,7 @@ type Config struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService + EdgeKeyService agent.EdgeKeyService TunnelOperator agent.TunnelOperator AgentTags map[string]string AgentOptions *agent.Options @@ -61,12 +63,13 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.TunnelOperator, config.ClusterService, notaryService, config.EdgeMode), + keyHandler: key.NewHandler(config.TunnelOperator, config.ClusterService, notaryService, config.EdgeKeyService, config.EdgeMode), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), securedProtocol: config.Secured, tunnelOperator: config.TunnelOperator, + edgeKeyService: config.EdgeKeyService, } } @@ -76,7 +79,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, request *http.Request) { return } - if !h.securedProtocol && !h.tunnelOperator.IsKeySet() { + if !h.securedProtocol && !h.edgeKeyService.IsKeySet() { httperror.WriteError(rw, http.StatusForbidden, "Unable to use the unsecured agent API without Edge key", errors.New("edge key not set")) return } diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index d76e1128..189a9a60 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -14,6 +14,7 @@ import ( type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator + edgeKeyService agent.EdgeKeyService clusterService agent.ClusterService edgeMode bool } @@ -22,9 +23,10 @@ type Handler struct { // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, notaryService *security.NotaryService, edgeMode bool) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, notaryService *security.NotaryService, edgeKeyService agent.EdgeKeyService, edgeMode bool) *Handler { h := &Handler{ Router: mux.NewRouter(), + edgeKeyService: edgeKeyService, tunnelOperator: tunnelOperator, clusterService: clusterService, edgeMode: edgeMode, diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index 30dfe842..9fb61cc6 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -29,7 +29,7 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } - if handler.tunnelOperator.IsKeySet() { + if handler.edgeKeyService.IsKeySet() { return &httperror.HandlerError{http.StatusConflict, "An Edge key is already associated to this agent", errors.New("Edge key already associated")} } @@ -41,7 +41,7 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} } - err = handler.tunnelOperator.SetKey(payload.Key) + err = handler.edgeKeyService.SetKey(payload.Key) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to associate Edge key", err} } diff --git a/http/handler/key/key_inspect.go b/http/handler/key/key_inspect.go index 37f4fe92..c6c1693e 100644 --- a/http/handler/key/key_inspect.go +++ b/http/handler/key/key_inspect.go @@ -17,11 +17,11 @@ func (handler *Handler) keyInspect(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } - if !handler.tunnelOperator.IsKeySet() { + if !handler.edgeKeyService.IsKeySet() { return &httperror.HandlerError{http.StatusNotFound, "No key associated to this agent", errors.New("Edge key unavailable")} } - edgeKey := handler.tunnelOperator.GetKey() + edgeKey := handler.edgeKeyService.GetKey() return response.JSON(w, keyInspectResponse{ Key: edgeKey, diff --git a/http/key/key.go b/http/key/key.go new file mode 100644 index 00000000..74cb3a3b --- /dev/null +++ b/http/key/key.go @@ -0,0 +1,115 @@ +package key + +import ( + "encoding/base64" + "errors" + "fmt" + "strings" + + "github.com/portainer/agent" + "github.com/portainer/agent/filesystem" +) + +type edgeKey struct { + PortainerInstanceURL string + TunnelServerAddr string + TunnelServerFingerprint string + EndpointID string +} + +// Service is a service that manages edge key +type Service struct { + key *edgeKey +} + +// NewService creates a new instance of Service +func NewService() (*Service, error) { + return &Service{}, nil +} + +// SetKey parses and associate a key to the service +func (service *Service) SetKey(key string) error { + edgeKey, err := parseEdgeKey(key) + if err != nil { + return err + } + + err = filesystem.WriteFile(agent.DataDirectory, agent.EdgeKeyFile, []byte(key), 0444) + if err != nil { + return err + } + + service.key = edgeKey + + return nil +} + +// GetKey returns the key associated to the service +func (service *Service) GetKey() string { + var encodedKey string + + if service.key != nil { + encodedKey = encodeKey(service.key) + } + + return encodedKey +} + +// GetPortainerConfig returns portainer url and endpoint id +func (service *Service) GetPortainerConfig() (string, string, error) { + if service.key == nil { + return "", "", errors.New("Key is not set") + } + + key := service.key + return key.PortainerInstanceURL, key.EndpointID, nil +} + +// GetTunnelConfig returns tunnel url and tunnel fingerprint +func (service *Service) GetTunnelConfig() (string, string, error) { + if service.key == nil { + return "", "", errors.New("Key is not set") + } + + key := service.key + return key.TunnelServerAddr, key.TunnelServerFingerprint, nil +} + +// IsKeySet checks if a key is associated to the service +func (service *Service) IsKeySet() bool { + if service.key == nil { + return false + } + return true +} + +// parseEdgeKey decodes a base64 encoded key and extract the decoded information from the following +// format: ||| +// are expected in the user:password format +func parseEdgeKey(key string) (*edgeKey, error) { + decodedKey, err := base64.RawStdEncoding.DecodeString(key) + if err != nil { + return nil, err + } + + keyInfo := strings.Split(string(decodedKey), "|") + + if len(keyInfo) != 4 { + return nil, errors.New("invalid key format") + } + + edgeKey := &edgeKey{ + PortainerInstanceURL: keyInfo[0], + TunnelServerAddr: keyInfo[1], + TunnelServerFingerprint: keyInfo[2], + EndpointID: keyInfo[3], + } + + return edgeKey, nil +} + +func encodeKey(edgeKey *edgeKey) string { + keyInfo := fmt.Sprintf("%s|%s|%s|%s", edgeKey.PortainerInstanceURL, edgeKey.TunnelServerAddr, edgeKey.TunnelServerFingerprint, edgeKey.EndpointID) + encodedKey := base64.RawStdEncoding.EncodeToString([]byte(keyInfo)) + return encodedKey +} diff --git a/http/server.go b/http/server.go index cdff27af..f62749f9 100644 --- a/http/server.go +++ b/http/server.go @@ -16,6 +16,7 @@ type APIServer struct { systemService agent.SystemService clusterService agent.ClusterService signatureService agent.DigitalSignatureService + edgeKeyService agent.EdgeKeyService tunnelOperator agent.TunnelOperator agentTags map[string]string agentOptions *agent.Options @@ -31,6 +32,7 @@ type APIServerConfig struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService + EdgeKeyService agent.EdgeKeyService TunnelOperator agent.TunnelOperator AgentTags map[string]string AgentOptions *agent.Options @@ -46,6 +48,7 @@ func NewAPIServer(config *APIServerConfig) *APIServer { systemService: config.SystemService, clusterService: config.ClusterService, signatureService: config.SignatureService, + edgeKeyService: config.EdgeKeyService, tunnelOperator: config.TunnelOperator, agentTags: config.AgentTags, agentOptions: config.AgentOptions, @@ -63,6 +66,7 @@ func (server *APIServer) StartUnsecured() error { AgentTags: server.agentTags, AgentOptions: server.agentOptions, EdgeMode: server.edgeMode, + EdgeKeyService: server.edgeKeyService, Secured: false, DockerStackService: server.dockerStackService, } @@ -87,6 +91,7 @@ func (server *APIServer) StartSecured() error { config := &handler.Config{ SystemService: server.systemService, ClusterService: server.clusterService, + EdgeKeyService: server.edgeKeyService, SignatureService: server.signatureService, TunnelOperator: server.tunnelOperator, AgentTags: server.agentTags, diff --git a/http/tunnel/key.go b/http/tunnel/key.go deleted file mode 100644 index 02e2f74c..00000000 --- a/http/tunnel/key.go +++ /dev/null @@ -1,39 +0,0 @@ -package tunnel - -import ( - "encoding/base64" - "errors" - "fmt" - "strings" -) - -// parseEdgeKey decodes a base64 encoded key and extract the decoded information from the following -// format: ||| -// are expected in the user:password format -func parseEdgeKey(key string) (*edgeKey, error) { - decodedKey, err := base64.RawStdEncoding.DecodeString(key) - if err != nil { - return nil, err - } - - keyInfo := strings.Split(string(decodedKey), "|") - - if len(keyInfo) != 4 { - return nil, errors.New("invalid key format") - } - - edgeKey := &edgeKey{ - PortainerInstanceURL: keyInfo[0], - TunnelServerAddr: keyInfo[1], - TunnelServerFingerprint: keyInfo[2], - EndpointID: keyInfo[3], - } - - return edgeKey, nil -} - -func encodeKey(edgeKey *edgeKey) string { - keyInfo := fmt.Sprintf("%s|%s|%s|%s", edgeKey.PortainerInstanceURL, edgeKey.TunnelServerAddr, edgeKey.TunnelServerFingerprint, edgeKey.EndpointID) - encodedKey := base64.RawStdEncoding.EncodeToString([]byte(keyInfo)) - return encodedKey -} diff --git a/http/tunnel/poll.go b/http/tunnel/poll.go index de16c12f..a35536e8 100644 --- a/http/tunnel/poll.go +++ b/http/tunnel/poll.go @@ -48,7 +48,12 @@ func (operator *Operator) createHTTPClient(timeout float64) { } func (operator *Operator) poll() error { - pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", operator.key.PortainerInstanceURL, operator.key.EndpointID) + portainerURL, endpointID, err := operator.edgeKeyService.GetPortainerConfig() + if err != nil { + return err + } + + pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", portainerURL, endpointID) req, err := http.NewRequest("GET", pollURL, nil) if err != nil { return err @@ -110,13 +115,13 @@ func (operator *Operator) poll() error { go operator.restartStatusPollLoop() } - if responseData.Stacks != nil { - err := operator.handleStacks(responseData.Stacks) - if err != nil { - log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) - return err - } - } + // if responseData.Stacks != nil { + // err := operator.handleStacks(responseData.Stacks) + // if err != nil { + // log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) + // return err + // } + // } return nil } @@ -132,9 +137,14 @@ func (operator *Operator) createTunnel(encodedCredentials string, remotePort int return err } + tunnelServerAddr, tunnelServerFingerprint, err := operator.edgeKeyService.GetTunnelConfig() + if err != nil { + return err + } + tunnelConfig := agent.TunnelConfig{ - ServerAddr: operator.key.TunnelServerAddr, - ServerFingerpint: operator.key.TunnelServerFingerprint, + ServerAddr: tunnelServerAddr, + ServerFingerpint: tunnelServerFingerprint, Credentials: string(credentials), RemotePort: strconv.Itoa(remotePort), LocalAddr: operator.apiServerAddr, diff --git a/http/tunnel/stack.go b/http/tunnel/stack.go deleted file mode 100644 index e69de29b..00000000 diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index 854ebbae..7e91371f 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -13,13 +13,6 @@ import ( const tunnelActivityCheckInterval = 30 * time.Second -type edgeKey struct { - PortainerInstanceURL string - TunnelServerAddr string - TunnelServerFingerprint string - EndpointID string -} - // Operator is used to poll a Portainer instance and to establish a reverse tunnel if needed. // It also takes care of closing the tunnel after a set period of inactivity. type Operator struct { @@ -28,12 +21,13 @@ type Operator struct { insecurePoll bool inactivityTimeout time.Duration edgeID string - key *edgeKey httpClient *http.Client tunnelClient agent.ReverseTunnelClient scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} + // edgeStackManager agent.EdgeStackManager + edgeKeyService agent.EdgeKeyService } // OperatorConfig represents the configuration used to create a new Operator. @@ -46,7 +40,7 @@ type OperatorConfig struct { } // NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(config *OperatorConfig) (*Operator, error) { +func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, config *OperatorConfig) (*Operator, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err @@ -66,45 +60,11 @@ func NewTunnelOperator(config *OperatorConfig) (*Operator, error) { tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, + edgeKeyService: edgeKeyService, + // edgeStackManager: edgeStackManager, }, nil } -// SetKey parses and associate a key to the operator -func (operator *Operator) SetKey(key string) error { - edgeKey, err := parseEdgeKey(key) - if err != nil { - return err - } - - err = filesystem.WriteFile(agent.DataDirectory, agent.EdgeKeyFile, []byte(key), 0444) - if err != nil { - return err - } - - operator.key = edgeKey - - return nil -} - -// GetKey returns the key associated to the operator -func (operator *Operator) GetKey() string { - var encodedKey string - - if operator.key != nil { - encodedKey = encodeKey(operator.key) - } - - return encodedKey -} - -// IsKeySet checks if a key is associated to the operator -func (operator *Operator) IsKeySet() bool { - if operator.key == nil { - return false - } - return true -} - // CloseTunnel closes the reverse tunnel managed by the operator func (operator *Operator) CloseTunnel() error { return operator.tunnelClient.CloseTunnel() @@ -123,7 +83,7 @@ func (operator *Operator) ResetActivityTimer() { // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. func (operator *Operator) Start() error { - if operator.key == nil { + if !operator.edgeKeyService.IsKeySet() { return errors.New("missing Edge key") } if operator.refreshSignal != nil { @@ -150,8 +110,13 @@ func (operator *Operator) restartStatusPollLoop() { operator.startStatusPollLoop() } -func (operator *Operator) startStatusPollLoop() { - log.Printf("[DEBUG] [http,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, operator.key.PortainerInstanceURL) +func (operator *Operator) startStatusPollLoop() error { + portainerURL, _, err := operator.edgeKeyService.GetPortainerConfig() + if err != nil { + return err + } + + log.Printf("[DEBUG] [http,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, portainerURL) ticker := time.NewTicker(time.Duration(operator.pollIntervalInSeconds) * time.Second) go func() { @@ -170,6 +135,8 @@ func (operator *Operator) startStatusPollLoop() { } } }() + + return nil } func (operator *Operator) startActivityMonitoringLoop() { From 3da8bc5fa20084df27d6660821543881a9238758 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 3 May 2020 15:22:20 +0300 Subject: [PATCH 38/91] feat(edge): manage stacks --- agent.go | 9 +- cmd/agent/main.go | 22 +-- http/edgestacks/edgestacks.go | 226 +++++++++++++++++++++++ http/portainerclient/portainer_client.go | 7 +- http/tunnel/poll.go | 23 ++- http/tunnel/tunnel.go | 8 +- 6 files changed, 267 insertions(+), 28 deletions(-) create mode 100644 http/edgestacks/edgestacks.go diff --git a/agent.go b/agent.go index 3a049dcb..f6bb790b 100644 --- a/agent.go +++ b/agent.go @@ -142,13 +142,20 @@ type ( Schedule(schedules []Schedule) error } - // DockerStackService is a service to manager Edge stacks + // DockerStackService is a service to manager deploy and remove docker stacks DockerStackService interface { Login() error Logout() error Deploy(name, stackFileContent string, prune bool) error Remove(name string) error } + + // EdgeStackManager is a service to manage Edge stacks + EdgeStackManager interface { + UpdateStacksStatus(stacks map[int]int) error + Start() error + Stop() + } ) const ( diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 877afb59..e463548a 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,12 +8,12 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" + "github.com/portainer/agent/exec" "github.com/portainer/agent/filesystem" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" "github.com/portainer/agent/http/client" - - // "github.com/portainer/agent/http/edgestacks" + "github.com/portainer/agent/http/edgestacks" "github.com/portainer/agent/http/key" "github.com/portainer/agent/http/tunnel" "github.com/portainer/agent/logutils" @@ -109,17 +109,17 @@ func main() { log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) - // dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) - // if err != nil { - // log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) - // } + dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) + if err != nil { + log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) + } - // edgeStackManager, err := edgestacks.NewManager(dockerStackService, edgeKeyService, options.EdgeID) - // if err != nil { - // log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) - // } + edgeStackManager, err := edgestacks.NewManager(dockerStackService, edgeKeyService, options.EdgeID) + if err != nil { + log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) + } - tunnelOperator, err = tunnel.NewTunnelOperator(edgeKeyService, operatorConfig) + tunnelOperator, err = tunnel.NewTunnelOperator(edgeKeyService, edgeStackManager, operatorConfig) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go new file mode 100644 index 00000000..18c2db35 --- /dev/null +++ b/http/edgestacks/edgestacks.go @@ -0,0 +1,226 @@ +package edgestacks + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/portainer/agent" + "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/http/portainerclient" +) + +const baseDir = "/tmp/edge_stacks" + +type edgeStackID int + +type edgeStack struct { + ID edgeStackID + Version int + FileFolder string + FileName string + Prune bool + Status edgeStackStatus + Action edgeStackAction +} + +type edgeStackStatus int + +const ( + _ edgeStackStatus = iota + statusPending + statusDone + statusError +) + +type edgeStackAction int + +const ( + _ edgeStackAction = iota + actionDeploy + actionUpdate + actionDelete + actionIdle +) + +type edgeStackStatusType int + +const ( + _ edgeStackStatusType = iota + edgeStackStatusOk + edgeStackStatusError + edgeStackStatusAcknowledged +) + +// EdgeStackManager represents a service for managing Edge stacks +type EdgeStackManager struct { + stacks map[edgeStackID]*edgeStack + stopSignal chan struct{} + dockerStackService agent.DockerStackService + keyService agent.EdgeKeyService + edgeID string +} + +// NewManager creates a new instance of EdgeStackManager +func NewManager(dockerStackService agent.DockerStackService, keyService agent.EdgeKeyService, edgeID string) (*EdgeStackManager, error) { + return &EdgeStackManager{ + dockerStackService: dockerStackService, + keyService: keyService, + edgeID: edgeID, + stacks: map[edgeStackID]*edgeStack{}, + stopSignal: nil, + }, nil +} + +// UpdateStacksStatus updates stacks version and status +func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { + for stackID, version := range stacks { + stack, ok := manager.stacks[edgeStackID(stackID)] + if ok { + if stack.Version == version { + continue + } + log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to update %d] \n", stackID) + + stack.Action = actionUpdate + stack.Version = version + stack.Status = statusPending + } else { + log.Printf("[DEBUG] [stacksmanager,update] [message: received new stack %d] \n", stackID) + + stack = &edgeStack{ + Action: actionDeploy, + ID: edgeStackID(stackID), + Status: statusPending, + Version: version, + } + } + + cli, err := manager.createPortainerClient() + if err != nil { + return err + } + + fileContent, prune, err := cli.GetEdgeStackConfig(int(stack.ID)) + if err != nil { + return err + } + + stack.Prune = prune + + folder := fmt.Sprintf("%s/%d", baseDir, stackID) + fileName := "docker-compose.yml" + err = filesystem.WriteFile(folder, fileName, []byte(fileContent), 644) + if err != nil { + return err + } + + stack.FileFolder = folder + stack.FileName = fileName + + manager.stacks[stack.ID] = stack + + err = cli.SetEdgeStackStatus(int(stack.ID), int(edgeStackStatusAcknowledged), "") + if err != nil { + return err + } + } + + for stackID, stack := range manager.stacks { + if _, ok := stacks[int(stackID)]; !ok { + stack.Action = actionDelete + stack.Status = statusPending + + manager.stacks[stackID] = stack + } + } + + return nil +} + +// Stop stops the manager +func (manager *EdgeStackManager) Stop() { + if manager.stopSignal == nil { + return + } + + close(manager.stopSignal) + manager.stopSignal = nil +} + +// Start starts the loop checking for stacks to deploy +func (manager *EdgeStackManager) Start() error { + if manager.stopSignal != nil { + return errors.New("Manager is already started") + } + + manager.stopSignal = make(chan struct{}) + + go (func() { + for { + select { + case <-manager.stopSignal: + log.Println("[DEBUG] [http,edge,stacksmanager] [message: shutting down Edge stack manager]") + return + default: + stack := manager.next() + if stack == nil { + timer1 := time.NewTimer(1 * time.Minute) + <-timer1.C + continue + } + + stackName := fmt.Sprintf("edge_%d", stack.ID) + stackFileLocation := fmt.Sprintf("%s/%s", stack.FileFolder, stack.FileName) + + stack.Status = statusDone + stack.Action = actionIdle + responseStatus := int(edgeStackStatusOk) + errorMessage := "" + + err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v] \n", err) + stack.Status = statusError + responseStatus = int(edgeStackStatusError) + errorMessage = err.Error() + } else { + log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d] \n", stack.ID, stack.Version) + } + + manager.stacks[stack.ID] = stack + + cli, err := manager.createPortainerClient() + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v] \n", err) + } + + err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v] \n", err) + } + } + } + })() + + return nil +} + +func (manager *EdgeStackManager) next() *edgeStack { + for _, stack := range manager.stacks { + if stack.Status == statusPending { + return stack + } + } + return nil +} + +func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.PortainerClient, error) { + portainerURL, endpointID, err := manager.keyService.GetPortainerConfig() + if err != nil { + return nil, err + } + + return portainerclient.NewPortainerClient(portainerURL, endpointID, manager.edgeID), nil +} diff --git a/http/portainerclient/portainer_client.go b/http/portainerclient/portainer_client.go index 74fa5320..665db570 100644 --- a/http/portainerclient/portainer_client.go +++ b/http/portainerclient/portainer_client.go @@ -26,6 +26,7 @@ func NewPortainerClient(serverAddress, endpointID, edgeID string) *PortainerClie return &PortainerClient{ serverAddress: serverAddress, endpointID: endpointID, + edgeID: edgeID, httpClient: &http.Client{ Timeout: time.Second * 3, }, @@ -39,7 +40,7 @@ type stackConfigResponse struct { // GetEdgeStackConfig fetches the Edge stack config func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool, error) { - requestURL := fmt.Sprintf("http://%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) + requestURL := fmt.Sprintf("%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { @@ -55,7 +56,7 @@ func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("[ERROR] [http,client] [response_code: %d] [message: GetEdgeStackConfig operation failed]", resp.StatusCode) + log.Printf("[ERROR] [http,portainerclient] [response_code: %d] [message: GetEdgeStackConfig operation failed] \n", resp.StatusCode) return "", false, errors.New("GetEdgeStackConfig operation failed") } @@ -92,7 +93,7 @@ func (client *PortainerClient) SetEdgeStackStatus(edgeStackID, edgeStackStatus i return err } - requestURL := fmt.Sprintf("http://%s/api/edge_stacks/%d/status", client.serverAddress, edgeStackID) + requestURL := fmt.Sprintf("%s/api/edge_stacks/%d/status", client.serverAddress, edgeStackID) req, err := http.NewRequest(http.MethodPut, requestURL, bytes.NewReader(data)) if err != nil { diff --git a/http/tunnel/poll.go b/http/tunnel/poll.go index a35536e8..a2b3aa11 100644 --- a/http/tunnel/poll.go +++ b/http/tunnel/poll.go @@ -18,8 +18,8 @@ import ( const clientDefaultPollTimeout = 5 type stackStatus struct { - EdgeStackID int - Version int + ID int + Version int } type pollStatusResponse struct { @@ -115,13 +115,18 @@ func (operator *Operator) poll() error { go operator.restartStatusPollLoop() } - // if responseData.Stacks != nil { - // err := operator.handleStacks(responseData.Stacks) - // if err != nil { - // log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) - // return err - // } - // } + if responseData.Stacks != nil { + stacks := map[int]int{} + for _, stack := range responseData.Stacks { + stacks[stack.ID] = stack.Version + } + + err := operator.edgeStackManager.UpdateStacksStatus(stacks) + if err != nil { + log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) + return err + } + } return nil } diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index 7e91371f..3edc6197 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -26,8 +26,8 @@ type Operator struct { scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} - // edgeStackManager agent.EdgeStackManager - edgeKeyService agent.EdgeKeyService + edgeStackManager agent.EdgeStackManager + edgeKeyService agent.EdgeKeyService } // OperatorConfig represents the configuration used to create a new Operator. @@ -40,7 +40,7 @@ type OperatorConfig struct { } // NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, config *OperatorConfig) (*Operator, error) { +func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, edgeStackManager agent.EdgeStackManager, config *OperatorConfig) (*Operator, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err @@ -61,7 +61,7 @@ func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, config *OperatorConf scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, edgeKeyService: edgeKeyService, - // edgeStackManager: edgeStackManager, + edgeStackManager: edgeStackManager, }, nil } From a097c218583d886f99d13849edca4b96b725b869 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 11:09:58 +0300 Subject: [PATCH 39/91] feat(main): start edge stack manager --- cmd/agent/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index e463548a..e7495092 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -115,6 +115,11 @@ func main() { } edgeStackManager, err := edgestacks.NewManager(dockerStackService, edgeKeyService, options.EdgeID) + if err != nil { + log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to create stack manager] [error: %s]", err) + } + + err = edgeStackManager.Start() if err != nil { log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) } From a62094f6121f73f2e8e48d663c874765dd92b87e Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 11:46:46 +0300 Subject: [PATCH 40/91] fix(http): change portainer response payload --- http/portainerclient/portainer_client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/portainerclient/portainer_client.go b/http/portainerclient/portainer_client.go index 665db570..47e6d09a 100644 --- a/http/portainerclient/portainer_client.go +++ b/http/portainerclient/portainer_client.go @@ -71,7 +71,7 @@ func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool type setEdgeStackStatusPayload struct { Error string - Type int + Status int EndpointID int } @@ -84,7 +84,7 @@ func (client *PortainerClient) SetEdgeStackStatus(edgeStackID, edgeStackStatus i payload := setEdgeStackStatusPayload{ Error: error, - Type: edgeStackStatus, + Status: edgeStackStatus, EndpointID: endpointID, } From 41840e3178e85b596afb7b074574c04df7daca9a Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 12:57:19 +0300 Subject: [PATCH 41/91] feat(edgestacks): remove stacks --- http/edgestacks/edgestacks.go | 75 +++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go index 18c2db35..e96fd8f4 100644 --- a/http/edgestacks/edgestacks.go +++ b/http/edgestacks/edgestacks.go @@ -129,6 +129,7 @@ func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { for stackID, stack := range manager.stacks { if _, ok := stacks[int(stackID)]; !ok { + log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to delete %d] \n", stackID) stack.Action = actionDelete stack.Status = statusPending @@ -174,31 +175,10 @@ func (manager *EdgeStackManager) Start() error { stackName := fmt.Sprintf("edge_%d", stack.ID) stackFileLocation := fmt.Sprintf("%s/%s", stack.FileFolder, stack.FileName) - stack.Status = statusDone - stack.Action = actionIdle - responseStatus := int(edgeStackStatusOk) - errorMessage := "" - - err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) - if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v] \n", err) - stack.Status = statusError - responseStatus = int(edgeStackStatusError) - errorMessage = err.Error() - } else { - log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d] \n", stack.ID, stack.Version) - } - - manager.stacks[stack.ID] = stack - - cli, err := manager.createPortainerClient() - if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v] \n", err) - } - - err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) - if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v] \n", err) + if stack.Action == actionDeploy || stack.Action == actionUpdate { + manager.deployStack(stack, stackName, stackFileLocation) + } else if stack.Action == actionDelete { + manager.deleteStack(stack, stackName, stackFileLocation) } } } @@ -216,6 +196,35 @@ func (manager *EdgeStackManager) next() *edgeStack { return nil } +func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { + stack.Status = statusDone + stack.Action = actionIdle + responseStatus := int(edgeStackStatusOk) + errorMessage := "" + + err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v] \n", err) + stack.Status = statusError + responseStatus = int(edgeStackStatusError) + errorMessage = err.Error() + } else { + log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d] \n", stack.ID, stack.Version) + } + + manager.stacks[stack.ID] = stack + + cli, err := manager.createPortainerClient() + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v] \n", err) + } + + err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) + if err != nil { + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v] \n", err) + } +} + func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.PortainerClient, error) { portainerURL, endpointID, err := manager.keyService.GetPortainerConfig() if err != nil { @@ -224,3 +233,19 @@ func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.Porta return portainerclient.NewPortainerClient(portainerURL, endpointID, manager.edgeID), nil } + +func (manager *EdgeStackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { + err := filesystem.RemoveFile(stackFileLocation) + if err != nil { + log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed deleting edge stack file] [error: %v] \n", err) + return + } + + err = manager.dockerStackService.Remove(stackName) + if err != nil { + log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed removing stack] [error: %v] \n", err) + return + } + + delete(manager.stacks, stack.ID) +} From 50993929009c458d876c0992be4517113535af95 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 13:04:07 +0300 Subject: [PATCH 42/91] feat(edgestacks): add debug messages --- http/edgestacks/edgestacks.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go index e96fd8f4..b015999e 100644 --- a/http/edgestacks/edgestacks.go +++ b/http/edgestacks/edgestacks.go @@ -197,6 +197,7 @@ func (manager *EdgeStackManager) next() *edgeStack { } func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { + log.Printf("[DEBUG] [stacksmanager,update] [message: deploying stack %d] \n", stack.ID) stack.Status = statusDone stack.Action = actionIdle responseStatus := int(edgeStackStatusOk) @@ -235,6 +236,7 @@ func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.Porta } func (manager *EdgeStackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { + log.Printf("[DEBUG] [stacksmanager,update] [message: removing stack %d] \n", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed deleting edge stack file] [error: %v] \n", err) From 1236527540f570295b1339b1a141b5fe775e49bc Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 4 May 2020 18:49:08 +0300 Subject: [PATCH 43/91] feat(main): start stacks manager when agent is ready --- agent.go | 2 +- cmd/agent/main.go | 35 ++++++++++++++++++++++++----------- http/edgestacks/edgestacks.go | 10 +++++----- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/agent.go b/agent.go index f6bb790b..8087ab75 100644 --- a/agent.go +++ b/agent.go @@ -154,7 +154,7 @@ type ( EdgeStackManager interface { UpdateStacksStatus(stacks map[int]int) error Start() error - Stop() + Stop() error } ) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index e7495092..95a5a2d4 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -119,17 +119,12 @@ func main() { log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to create stack manager] [error: %s]", err) } - err = edgeStackManager.Start() - if err != nil { - log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to start stack manager] [error: %s]", err) - } - tunnelOperator, err = tunnel.NewTunnelOperator(edgeKeyService, edgeStackManager, operatorConfig) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) } - err = enableEdgeMode(tunnelOperator, clusterService, edgeKeyService, infoService, options) + err = enableEdgeMode(tunnelOperator, clusterService, edgeKeyService, infoService, edgeStackManager, options) if err != nil { log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) } @@ -181,7 +176,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, infoService agent.InfoService, options *agent.Options) error { +func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, infoService agent.InfoService, edgeStackManager agent.EdgeStackManager, options *agent.Options) error { edgeKey, err := retrieveEdgeKey(options, clusterService) if err != nil { return err @@ -201,7 +196,7 @@ func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.Cl serveEdgeUI(tunnelOperator, clusterService, keyService, options) } - return startRuntimeConfigCheckProcess(tunnelOperator, infoService, keyService) + return startRuntimeConfigCheckProcess(tunnelOperator, infoService, keyService, edgeStackManager) } func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { @@ -323,7 +318,7 @@ func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.Clust }() } -func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService, keyService agent.EdgeKeyService) error { +func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService, keyService agent.EdgeKeyService, edgeStackManager agent.EdgeStackManager) error { runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { @@ -347,9 +342,12 @@ func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoSer continue } - log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentTags[agent.MemberTagKeyIsLeader] == "1") + isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" + isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm + + log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) - if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusStandalone || agentTags[agent.MemberTagKeyIsLeader] == "1" { + if !isSwarm || isLeader { err = tunnelOperator.Start() if err != nil { log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) @@ -360,6 +358,21 @@ func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoSer if err != nil { log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) } + + } + + if isSwarm && isLeader { + err = edgeStackManager.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,stack] [message: an error occured while starting the Edge stack manager] [error: %s]", err) + } + + } else { + err = edgeStackManager.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,stack] [message: an error occured while stopping the Edge stack manager] [error: %s]", err) + } + } } } diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go index b015999e..8d192ac5 100644 --- a/http/edgestacks/edgestacks.go +++ b/http/edgestacks/edgestacks.go @@ -141,13 +141,13 @@ func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { } // Stop stops the manager -func (manager *EdgeStackManager) Stop() { - if manager.stopSignal == nil { - return +func (manager *EdgeStackManager) Stop() error { + if manager.stopSignal != nil { + close(manager.stopSignal) + manager.stopSignal = nil } - close(manager.stopSignal) - manager.stopSignal = nil + return nil } // Start starts the loop checking for stacks to deploy From 7f0cd67241dd6985c7f957313314451b83e3c8f1 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 11:36:52 +0300 Subject: [PATCH 44/91] refactor(edge): create an edge manager --- agent.go | 10 ++ cmd/agent/main.go | 248 +-------------------------- edge/edge.go | 286 ++++++++++++++++++++++++++++++++ http/handler/handler.go | 33 ++-- http/handler/key/handler.go | 6 +- http/handler/key/key_create.go | 4 +- http/handler/key/key_inspect.go | 4 +- http/server.go | 79 ++++----- 8 files changed, 360 insertions(+), 310 deletions(-) create mode 100644 edge/edge.go diff --git a/agent.go b/agent.go index 8087ab75..d441fb77 100644 --- a/agent.go +++ b/agent.go @@ -137,6 +137,16 @@ type ( GetTunnelConfig() (string, string, error) } + // EdgeManager is a service to manager edge tasks + EdgeManager interface { + Enable(edgeKey string) error + IsKeySet() bool + SetKey(key string) error + GetKey() string + + ResetActivityTimer() + } + // Scheduler is used to manage schedules Scheduler interface { Schedule(schedules []Schedule) error diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 95a5a2d4..ab26e8a7 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,14 +8,9 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" - "github.com/portainer/agent/exec" - "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/edge" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" - "github.com/portainer/agent/http/client" - "github.com/portainer/agent/http/edgestacks" - "github.com/portainer/agent/http/key" - "github.com/portainer/agent/http/tunnel" "github.com/portainer/agent/logutils" "github.com/portainer/agent/net" "github.com/portainer/agent/os" @@ -89,44 +84,16 @@ func main() { defer clusterService.Leave() } - var edgeKeyService agent.EdgeKeyService - var tunnelOperator agent.TunnelOperator + var edgeManager agent.EdgeManager if options.EdgeMode { - apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) - - edgeKeyService, err = key.NewService() - if err != nil { - log.Fatalf("[ERROR] [main,edge,key] [message: Unable to create key service] [error: %s]", err) - } - - operatorConfig := &tunnel.OperatorConfig{ - APIServerAddr: apiServerAddr, - EdgeID: options.EdgeID, - PollFrequency: agent.DefaultEdgePollInterval, - InactivityTimeout: options.EdgeInactivityTimeout, - InsecurePoll: options.EdgeInsecurePoll, - } - - log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) - - dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) - if err != nil { - log.Fatalf("[ERROR] [main,edge,docker] [message: Unable to start docker stack service] [error: %s]", err) - } - - edgeStackManager, err := edgestacks.NewManager(dockerStackService, edgeKeyService, options.EdgeID) - if err != nil { - log.Fatalf("[ERROR] [main,edge,stack] [message: Unable to create stack manager] [error: %s]", err) - } - - tunnelOperator, err = tunnel.NewTunnelOperator(edgeKeyService, edgeStackManager, operatorConfig) + edgeManager, err = edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) if err != nil { - log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to create tunnel operator] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) } - err = enableEdgeMode(tunnelOperator, clusterService, edgeKeyService, infoService, edgeStackManager, options) + err = edgeManager.Enable(options.EdgeKey) if err != nil { - log.Fatalf("[ERROR] [main,edge,rtunnel] [message: Unable to start agent in Edge mode] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to enable edge mode] [error: %s]", err) } } @@ -148,9 +115,8 @@ func main() { Port: options.AgentServerPort, SystemService: systemService, ClusterService: clusterService, - EdgeKeyService: edgeKeyService, + EdgeManager: edgeManager, SignatureService: signatureService, - TunnelOperator: tunnelOperator, AgentTags: agentTags, AgentOptions: options, EdgeMode: options.EdgeMode, @@ -176,207 +142,7 @@ func startAPIServer(config *http.APIServerConfig) error { return server.StartSecured() } -func enableEdgeMode(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, infoService agent.InfoService, edgeStackManager agent.EdgeStackManager, options *agent.Options) error { - edgeKey, err := retrieveEdgeKey(options, clusterService) - if err != nil { - return err - } - - if edgeKey != "" { - log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") - - err := associateEdgeKey(keyService, clusterService, edgeKey) - if err != nil { - return err - } - - } else { - log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - - serveEdgeUI(tunnelOperator, clusterService, keyService, options) - } - - return startRuntimeConfigCheckProcess(tunnelOperator, infoService, keyService, edgeStackManager) -} - -func retrieveEdgeKey(options *agent.Options, clusterService agent.ClusterService) (string, error) { - edgeKey := options.EdgeKey - - if options.EdgeKey != "" { - log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") - edgeKey = options.EdgeKey - } - - if edgeKey == "" { - var keyRetrievalError error - - edgeKey, keyRetrievalError = retrieveEdgeKeyFromFilesystem() - if keyRetrievalError != nil { - return "", keyRetrievalError - } - - if edgeKey == "" && clusterService != nil { - edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(clusterService) - if keyRetrievalError != nil { - return "", keyRetrievalError - } - } - } - - return edgeKey, nil -} - -func retrieveEdgeKeyFromFilesystem() (string, error) { - var edgeKey string - - edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) - - keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) - if err != nil { - return "", err - } - - if keyFileExists { - filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) - if err != nil { - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") - edgeKey = string(filesystemKey) - } - - return edgeKey, nil -} - -func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { - var edgeKey string - - member := clusterService.GetMemberWithEdgeKeySet() - if member != nil { - httpCli := client.NewAPIClient() - - memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) - memberKey, err := httpCli.GetEdgeKey(memberAddr) - if err != nil { - log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") - edgeKey = memberKey - } - - return edgeKey, nil -} - func parseOptions() (*agent.Options, error) { optionParser := os.NewEnvOptionParser() return optionParser.Options() } - -func associateEdgeKey(edgeKeyService agent.EdgeKeyService, clusterService agent.ClusterService, edgeKey string) error { - err := edgeKeyService.SetKey(edgeKey) - if err != nil { - return err - } - - if clusterService != nil { - tags := clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = clusterService.UpdateTags(tags) - if err != nil { - return err - } - } - - return nil -} - -func serveEdgeUI(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService, options *agent.Options) { - edgeServer := http.NewEdgeServer(tunnelOperator, clusterService, keyService) - - go func() { - log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", options.EdgeServerAddr, options.EdgeServerPort) - - err := edgeServer.Start(options.EdgeServerAddr, options.EdgeServerPort) - if err != nil { - log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) - } - - log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") - }() - - go func() { - timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) - <-timer1.C - - if !keyService.IsKeySet() { - log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) - edgeServer.Shutdown() - } - }() -} - -func startRuntimeConfigCheckProcess(tunnelOperator agent.TunnelOperator, infoService agent.InfoService, keyService agent.EdgeKeyService, edgeStackManager agent.EdgeStackManager) error { - - runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) - if err != nil { - return err - } - - ticker := time.NewTicker(runtimeCheckFrequency) - - go func() { - for { - select { - case <-ticker.C: - key := keyService.GetKey() - if key == "" { - continue - } - - agentTags, err := infoService.GetInformationFromDockerEngine() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) - continue - } - - isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" - isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm - - log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) - - if !isSwarm || isLeader { - err = tunnelOperator.Start() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) - } - - } else { - err = tunnelOperator.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) - } - - } - - if isSwarm && isLeader { - err = edgeStackManager.Start() - if err != nil { - log.Printf("[ERROR] [main,edge,stack] [message: an error occured while starting the Edge stack manager] [error: %s]", err) - } - - } else { - err = edgeStackManager.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,stack] [message: an error occured while stopping the Edge stack manager] [error: %s]", err) - } - - } - } - } - }() - - return nil -} diff --git a/edge/edge.go b/edge/edge.go new file mode 100644 index 00000000..66149bd8 --- /dev/null +++ b/edge/edge.go @@ -0,0 +1,286 @@ +package edge + +import ( + "fmt" + "log" + "time" + + "github.com/portainer/agent" + "github.com/portainer/agent/exec" + "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/http" + "github.com/portainer/agent/http/client" + "github.com/portainer/agent/http/edgestacks" + "github.com/portainer/agent/http/key" + "github.com/portainer/agent/http/tunnel" +) + +type EdgeManager struct { + clusterService agent.ClusterService + dockerStackService agent.DockerStackService + infoService agent.InfoService + keyService agent.EdgeKeyService + stackManager agent.EdgeStackManager + tunnelOperator agent.TunnelOperator + serverAddr string + serverPort string +} + +func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { + apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) + + operatorConfig := &tunnel.OperatorConfig{ + APIServerAddr: apiServerAddr, + EdgeID: options.EdgeID, + PollFrequency: agent.DefaultEdgePollInterval, + InactivityTimeout: options.EdgeInactivityTimeout, + InsecurePoll: options.EdgeInsecurePoll, + } + + log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) + + keyService, err := key.NewService() + if err != nil { + return nil, err + } + + dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) + if err != nil { + return nil, err + } + + edgeStackManager, err := edgestacks.NewManager(dockerStackService, keyService, options.EdgeID) + if err != nil { + return nil, err + } + + tunnelOperator, err := tunnel.NewTunnelOperator(keyService, edgeStackManager, operatorConfig) + if err != nil { + return nil, err + } + + return &EdgeManager{ + clusterService: clusterService, + dockerStackService: dockerStackService, + infoService: infoService, + keyService: keyService, + stackManager: edgeStackManager, + tunnelOperator: tunnelOperator, + serverAddr: options.EdgeServerAddr, + serverPort: options.EdgeServerPort, + }, nil +} + +func (manager *EdgeManager) Enable(edgeKey string) error { + edgeKey, err := manager.retrieveEdgeKey(edgeKey) + if err != nil { + return err + } + + if edgeKey != "" { + log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") + + err := manager.associateEdgeKey(edgeKey) + if err != nil { + return err + } + + } else { + log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") + + manager.serveEdgeUI() + } + + return manager.startRuntimeConfigCheckProcess() + +} + +func (manager *EdgeManager) IsKeySet() bool { + return manager.keyService.IsKeySet() +} + +func (manager *EdgeManager) GetKey() string { + return manager.keyService.GetKey() +} + +func (manager *EdgeManager) SetKey(key string) error { + return manager.keyService.SetKey(key) +} + +func (manager *EdgeManager) ResetActivityTimer() { + manager.tunnelOperator.ResetActivityTimer() +} + +func (manager *EdgeManager) retrieveEdgeKey(edgeKey string) (string, error) { + + if edgeKey != "" { + log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") + return edgeKey, nil + } + + var keyRetrievalError error + + edgeKey, keyRetrievalError = manager.retrieveEdgeKeyFromFilesystem() + if keyRetrievalError != nil { + return "", keyRetrievalError + } + + if edgeKey == "" && manager.clusterService != nil { + edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(manager.clusterService) + if keyRetrievalError != nil { + return "", keyRetrievalError + } + } + + return edgeKey, nil +} + +func (manager *EdgeManager) retrieveEdgeKeyFromFilesystem() (string, error) { + var edgeKey string + + edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) + + keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) + if err != nil { + return "", err + } + + if keyFileExists { + filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) + if err != nil { + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") + edgeKey = string(filesystemKey) + } + + return edgeKey, nil +} + +func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { + var edgeKey string + + member := clusterService.GetMemberWithEdgeKeySet() + if member != nil { + httpCli := client.NewAPIClient() + + memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) + memberKey, err := httpCli.GetEdgeKey(memberAddr) + if err != nil { + log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") + edgeKey = memberKey + } + + return edgeKey, nil +} + +func (manager *EdgeManager) associateEdgeKey(edgeKey string) error { + err := manager.keyService.SetKey(edgeKey) + if err != nil { + return err + } + + if manager.clusterService != nil { + tags := manager.clusterService.GetTags() + tags[agent.MemberTagEdgeKeySet] = "set" + err = manager.clusterService.UpdateTags(tags) + if err != nil { + return err + } + } + + return nil +} + +func (manager *EdgeManager) serveEdgeUI() { + edgeServer := http.NewEdgeServer(manager.tunnelOperator, manager.clusterService, manager.keyService) + + go func() { + log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", manager.serverAddr, manager.serverPort) + + err := edgeServer.Start(manager.serverAddr, manager.serverPort) + if err != nil { + log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) + } + + log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") + }() + + go func() { + timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) + <-timer1.C + + if !manager.keyService.IsKeySet() { + log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) + edgeServer.Shutdown() + } + }() +} + +func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { + + runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) + if err != nil { + return err + } + + ticker := time.NewTicker(runtimeCheckFrequency) + + go func() { + for { + select { + case <-ticker.C: + key := manager.keyService.GetKey() + if key == "" { + continue + } + + agentTags, err := manager.infoService.GetInformationFromDockerEngine() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) + continue + } + + isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" + isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm + + log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) + + if !isSwarm || isLeader { + err = manager.tunnelOperator.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) + } + + } else { + err = manager.tunnelOperator.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) + } + + } + + if isSwarm && isLeader { + err = manager.stackManager.Start() + if err != nil { + log.Printf("[ERROR] [main,edge,stack] [message: an error occured while starting the Edge stack manager] [error: %s]", err) + } + + } else { + err = manager.stackManager.Stop() + if err != nil { + log.Printf("[ERROR] [main,edge,stack] [message: an error occured while stopping the Edge stack manager] [error: %s]", err) + } + + } + } + } + }() + + return nil +} diff --git a/http/handler/handler.go b/http/handler/handler.go index e1cb3119..cfbfa2dd 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -32,23 +32,21 @@ type Handler struct { hostHandler *host.Handler pingHandler *ping.Handler securedProtocol bool - edgeKeyService agent.EdgeKeyService - tunnelOperator agent.TunnelOperator + edgeManager agent.EdgeManager } // Config represents a server handler configuration // used to create a new handler type Config struct { - SystemService agent.SystemService - ClusterService agent.ClusterService - SignatureService agent.DigitalSignatureService - EdgeKeyService agent.EdgeKeyService - TunnelOperator agent.TunnelOperator - AgentTags map[string]string - AgentOptions *agent.Options - Secured bool - EdgeMode bool - DockerStackService agent.DockerStackService + SystemService agent.SystemService + ClusterService agent.ClusterService + SignatureService agent.DigitalSignatureService + EdgeManager agent.EdgeManager + TunnelOperator agent.TunnelOperator + AgentTags map[string]string + AgentOptions *agent.Options + Secured bool + EdgeMode bool } var dockerAPIVersionRegexp = regexp.MustCompile(`(/v[0-9]\.[0-9]*)?`) @@ -63,13 +61,12 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.TunnelOperator, config.ClusterService, notaryService, config.EdgeKeyService, config.EdgeMode), + keyHandler: key.NewHandler(config.TunnelOperator, config.ClusterService, notaryService, config.EdgeManager, config.EdgeMode), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), securedProtocol: config.Secured, - tunnelOperator: config.TunnelOperator, - edgeKeyService: config.EdgeKeyService, + edgeManager: config.EdgeManager, } } @@ -79,13 +76,13 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, request *http.Request) { return } - if !h.securedProtocol && !h.edgeKeyService.IsKeySet() { + if !h.securedProtocol && !h.edgeManager.IsKeySet() { httperror.WriteError(rw, http.StatusForbidden, "Unable to use the unsecured agent API without Edge key", errors.New("edge key not set")) return } - if h.tunnelOperator != nil { - h.tunnelOperator.ResetActivityTimer() + if h.edgeManager != nil { + h.edgeManager.ResetActivityTimer() } request.URL.Path = dockerAPIVersionRegexp.ReplaceAllString(request.URL.Path, "") diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index 189a9a60..4b871fca 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -14,7 +14,7 @@ import ( type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator - edgeKeyService agent.EdgeKeyService + edgeManager agent.EdgeManager clusterService agent.ClusterService edgeMode bool } @@ -23,10 +23,10 @@ type Handler struct { // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, notaryService *security.NotaryService, edgeKeyService agent.EdgeKeyService, edgeMode bool) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, notaryService *security.NotaryService, edgeManager agent.EdgeManager, edgeMode bool) *Handler { h := &Handler{ Router: mux.NewRouter(), - edgeKeyService: edgeKeyService, + edgeManager: edgeManager, tunnelOperator: tunnelOperator, clusterService: clusterService, edgeMode: edgeMode, diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index 9fb61cc6..013d2412 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -29,7 +29,7 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } - if handler.edgeKeyService.IsKeySet() { + if handler.edgeManager.IsKeySet() { return &httperror.HandlerError{http.StatusConflict, "An Edge key is already associated to this agent", errors.New("Edge key already associated")} } @@ -41,7 +41,7 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} } - err = handler.edgeKeyService.SetKey(payload.Key) + err = handler.edgeManager.SetKey(payload.Key) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to associate Edge key", err} } diff --git a/http/handler/key/key_inspect.go b/http/handler/key/key_inspect.go index c6c1693e..99a88633 100644 --- a/http/handler/key/key_inspect.go +++ b/http/handler/key/key_inspect.go @@ -17,11 +17,11 @@ func (handler *Handler) keyInspect(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } - if !handler.edgeKeyService.IsKeySet() { + if !handler.edgeManager.IsKeySet() { return &httperror.HandlerError{http.StatusNotFound, "No key associated to this agent", errors.New("Edge key unavailable")} } - edgeKey := handler.edgeKeyService.GetKey() + edgeKey := handler.edgeManager.GetKey() return response.JSON(w, keyInspectResponse{ Key: edgeKey, diff --git a/http/server.go b/http/server.go index f62749f9..4339a36e 100644 --- a/http/server.go +++ b/http/server.go @@ -11,64 +11,56 @@ import ( // APIServer is the web server exposing the API of an agent. type APIServer struct { - addr string - port string - systemService agent.SystemService - clusterService agent.ClusterService - signatureService agent.DigitalSignatureService - edgeKeyService agent.EdgeKeyService - tunnelOperator agent.TunnelOperator - agentTags map[string]string - agentOptions *agent.Options - edgeMode bool - dockerStackService agent.DockerStackService + addr string + port string + systemService agent.SystemService + clusterService agent.ClusterService + signatureService agent.DigitalSignatureService + edgeManager agent.EdgeManager + agentTags map[string]string + agentOptions *agent.Options + edgeMode bool } // APIServerConfig represents a server configuration // used to create a new API server type APIServerConfig struct { - Addr string - Port string - SystemService agent.SystemService - ClusterService agent.ClusterService - SignatureService agent.DigitalSignatureService - EdgeKeyService agent.EdgeKeyService - TunnelOperator agent.TunnelOperator - AgentTags map[string]string - AgentOptions *agent.Options - EdgeMode bool - DockerStackService agent.DockerStackService + Addr string + Port string + SystemService agent.SystemService + ClusterService agent.ClusterService + SignatureService agent.DigitalSignatureService + EdgeManager agent.EdgeManager + AgentTags map[string]string + AgentOptions *agent.Options + EdgeMode bool } // NewAPIServer returns a pointer to a APIServer. func NewAPIServer(config *APIServerConfig) *APIServer { return &APIServer{ - addr: config.Addr, - port: config.Port, - systemService: config.SystemService, - clusterService: config.ClusterService, - signatureService: config.SignatureService, - edgeKeyService: config.EdgeKeyService, - tunnelOperator: config.TunnelOperator, - agentTags: config.AgentTags, - agentOptions: config.AgentOptions, - edgeMode: config.EdgeMode, - dockerStackService: config.DockerStackService, + addr: config.Addr, + port: config.Port, + systemService: config.SystemService, + clusterService: config.ClusterService, + signatureService: config.SignatureService, + edgeManager: config.EdgeManager, + agentTags: config.AgentTags, + agentOptions: config.AgentOptions, + edgeMode: config.EdgeMode, } } // Start starts a new web server by listening on the specified listenAddr. func (server *APIServer) StartUnsecured() error { config := &handler.Config{ - SystemService: server.systemService, - ClusterService: server.clusterService, - TunnelOperator: server.tunnelOperator, - AgentTags: server.agentTags, - AgentOptions: server.agentOptions, - EdgeMode: server.edgeMode, - EdgeKeyService: server.edgeKeyService, - Secured: false, - DockerStackService: server.dockerStackService, + SystemService: server.systemService, + ClusterService: server.clusterService, + AgentTags: server.agentTags, + AgentOptions: server.agentOptions, + EdgeMode: server.edgeMode, + EdgeManager: server.edgeManager, + Secured: false, } h := handler.NewHandler(config) @@ -91,12 +83,11 @@ func (server *APIServer) StartSecured() error { config := &handler.Config{ SystemService: server.systemService, ClusterService: server.clusterService, - EdgeKeyService: server.edgeKeyService, SignatureService: server.signatureService, - TunnelOperator: server.tunnelOperator, AgentTags: server.agentTags, AgentOptions: server.agentOptions, EdgeMode: server.edgeMode, + EdgeManager: server.edgeManager, Secured: true, } From 8d80d769c0cf016922eaf58d64e05e848ddc07dd Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 12:43:28 +0300 Subject: [PATCH 45/91] refactor(edge): move key management to edge manager --- agent.go | 13 ++------- edge/edge.go | 37 +++++++------------------- {http/key => edge}/key.go | 44 ++++++++++++------------------ http/edge.go | 10 +++---- http/edgestacks/edgestacks.go | 27 ++++++++++++------- http/tunnel/poll.go | 15 +++-------- http/tunnel/tunnel.go | 50 +++++++++++++++++++---------------- 7 files changed, 79 insertions(+), 117 deletions(-) rename {http/key => edge}/key.go (69%) diff --git a/agent.go b/agent.go index d441fb77..e19989bb 100644 --- a/agent.go +++ b/agent.go @@ -122,21 +122,12 @@ type ( // TunnelOperator is a service that is used to communicate with a Portainer instance and to manage // the reverse tunnel. TunnelOperator interface { - Start() error + Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error Stop() error CloseTunnel() error ResetActivityTimer() } - // EdgeKeyService is a service that manages edge key - EdgeKeyService interface { - IsKeySet() bool - SetKey(key string) error - GetKey() string - GetPortainerConfig() (string, string, error) - GetTunnelConfig() (string, string, error) - } - // EdgeManager is a service to manager edge tasks EdgeManager interface { Enable(edgeKey string) error @@ -163,7 +154,7 @@ type ( // EdgeStackManager is a service to manage Edge stacks EdgeStackManager interface { UpdateStacksStatus(stacks map[int]int) error - Start() error + Start(portainerURL, endpointID string) error Stop() error } ) diff --git a/edge/edge.go b/edge/edge.go index 66149bd8..7af483af 100644 --- a/edge/edge.go +++ b/edge/edge.go @@ -11,7 +11,6 @@ import ( "github.com/portainer/agent/http" "github.com/portainer/agent/http/client" "github.com/portainer/agent/http/edgestacks" - "github.com/portainer/agent/http/key" "github.com/portainer/agent/http/tunnel" ) @@ -19,11 +18,11 @@ type EdgeManager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService infoService agent.InfoService - keyService agent.EdgeKeyService stackManager agent.EdgeStackManager tunnelOperator agent.TunnelOperator serverAddr string serverPort string + key *edgeKey } func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { @@ -39,22 +38,17 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) - keyService, err := key.NewService() - if err != nil { - return nil, err - } - dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) if err != nil { return nil, err } - edgeStackManager, err := edgestacks.NewManager(dockerStackService, keyService, options.EdgeID) + edgeStackManager, err := edgestacks.NewManager(dockerStackService, options.EdgeID) if err != nil { return nil, err } - tunnelOperator, err := tunnel.NewTunnelOperator(keyService, edgeStackManager, operatorConfig) + tunnelOperator, err := tunnel.NewTunnelOperator(edgeStackManager, operatorConfig) if err != nil { return nil, err } @@ -63,7 +57,6 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService clusterService: clusterService, dockerStackService: dockerStackService, infoService: infoService, - keyService: keyService, stackManager: edgeStackManager, tunnelOperator: tunnelOperator, serverAddr: options.EdgeServerAddr, @@ -95,18 +88,6 @@ func (manager *EdgeManager) Enable(edgeKey string) error { } -func (manager *EdgeManager) IsKeySet() bool { - return manager.keyService.IsKeySet() -} - -func (manager *EdgeManager) GetKey() string { - return manager.keyService.GetKey() -} - -func (manager *EdgeManager) SetKey(key string) error { - return manager.keyService.SetKey(key) -} - func (manager *EdgeManager) ResetActivityTimer() { manager.tunnelOperator.ResetActivityTimer() } @@ -180,7 +161,7 @@ func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, er } func (manager *EdgeManager) associateEdgeKey(edgeKey string) error { - err := manager.keyService.SetKey(edgeKey) + err := manager.SetKey(edgeKey) if err != nil { return err } @@ -198,7 +179,7 @@ func (manager *EdgeManager) associateEdgeKey(edgeKey string) error { } func (manager *EdgeManager) serveEdgeUI() { - edgeServer := http.NewEdgeServer(manager.tunnelOperator, manager.clusterService, manager.keyService) + edgeServer := http.NewEdgeServer(manager, manager.clusterService) go func() { log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", manager.serverAddr, manager.serverPort) @@ -215,7 +196,7 @@ func (manager *EdgeManager) serveEdgeUI() { timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) <-timer1.C - if !manager.keyService.IsKeySet() { + if !manager.IsKeySet() { log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) edgeServer.Shutdown() } @@ -235,7 +216,7 @@ func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { for { select { case <-ticker.C: - key := manager.keyService.GetKey() + key := manager.GetKey() if key == "" { continue } @@ -252,7 +233,7 @@ func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) if !isSwarm || isLeader { - err = manager.tunnelOperator.Start() + err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) if err != nil { log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) } @@ -266,7 +247,7 @@ func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { } if isSwarm && isLeader { - err = manager.stackManager.Start() + err = manager.stackManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) if err != nil { log.Printf("[ERROR] [main,edge,stack] [message: an error occured while starting the Edge stack manager] [error: %s]", err) } diff --git a/http/key/key.go b/edge/key.go similarity index 69% rename from http/key/key.go rename to edge/key.go index 74cb3a3b..5ef66efc 100644 --- a/http/key/key.go +++ b/edge/key.go @@ -1,4 +1,4 @@ -package key +package edge import ( "encoding/base64" @@ -17,18 +17,8 @@ type edgeKey struct { EndpointID string } -// Service is a service that manages edge key -type Service struct { - key *edgeKey -} - -// NewService creates a new instance of Service -func NewService() (*Service, error) { - return &Service{}, nil -} - -// SetKey parses and associate a key to the service -func (service *Service) SetKey(key string) error { +// SetKey parses and associate a key to the manager +func (manager *EdgeManager) SetKey(key string) error { edgeKey, err := parseEdgeKey(key) if err != nil { return err @@ -39,45 +29,45 @@ func (service *Service) SetKey(key string) error { return err } - service.key = edgeKey + manager.key = edgeKey return nil } -// GetKey returns the key associated to the service -func (service *Service) GetKey() string { +// GetKey returns the key associated to the manager +func (manager *EdgeManager) GetKey() string { var encodedKey string - if service.key != nil { - encodedKey = encodeKey(service.key) + if manager.key != nil { + encodedKey = encodeKey(manager.key) } return encodedKey } // GetPortainerConfig returns portainer url and endpoint id -func (service *Service) GetPortainerConfig() (string, string, error) { - if service.key == nil { +func (manager *EdgeManager) GetPortainerConfig() (string, string, error) { + if manager.key == nil { return "", "", errors.New("Key is not set") } - key := service.key + key := manager.key return key.PortainerInstanceURL, key.EndpointID, nil } // GetTunnelConfig returns tunnel url and tunnel fingerprint -func (service *Service) GetTunnelConfig() (string, string, error) { - if service.key == nil { +func (manager *EdgeManager) GetTunnelConfig() (string, string, error) { + if manager.key == nil { return "", "", errors.New("Key is not set") } - key := service.key + key := manager.key return key.TunnelServerAddr, key.TunnelServerFingerprint, nil } -// IsKeySet checks if a key is associated to the service -func (service *Service) IsKeySet() bool { - if service.key == nil { +// IsKeySet checks if a key is associated to the manager +func (manager *EdgeManager) IsKeySet() bool { + if manager.key == nil { return false } return true diff --git a/http/edge.go b/http/edge.go index 5f79d750..3091df9b 100644 --- a/http/edge.go +++ b/http/edge.go @@ -16,17 +16,15 @@ import ( // EdgeServer expose an UI to associate an Edge key with the agent. type EdgeServer struct { httpServer *http.Server - tunnelOperator agent.TunnelOperator + edgeManager agent.EdgeManager clusterService agent.ClusterService - keyService agent.EdgeKeyService } // NewEdgeServer returns a pointer to a new instance of EdgeServer. -func NewEdgeServer(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, keyService agent.EdgeKeyService) *EdgeServer { +func NewEdgeServer(edgeManager agent.EdgeManager, clusterService agent.ClusterService) *EdgeServer { return &EdgeServer{ - tunnelOperator: tunnelOperator, clusterService: clusterService, - keyService: keyService, + edgeManager: edgeManager, } } @@ -61,7 +59,7 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { return } - err = server.keyService.SetKey(key) + err = server.edgeManager.SetKey(key) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go index 8d192ac5..fbba7d16 100644 --- a/http/edgestacks/edgestacks.go +++ b/http/edgestacks/edgestacks.go @@ -58,15 +58,16 @@ type EdgeStackManager struct { stacks map[edgeStackID]*edgeStack stopSignal chan struct{} dockerStackService agent.DockerStackService - keyService agent.EdgeKeyService edgeID string + portainerURL string + endpointID string + isEnabled bool } // NewManager creates a new instance of EdgeStackManager -func NewManager(dockerStackService agent.DockerStackService, keyService agent.EdgeKeyService, edgeID string) (*EdgeStackManager, error) { +func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*EdgeStackManager, error) { return &EdgeStackManager{ dockerStackService: dockerStackService, - keyService: keyService, edgeID: edgeID, stacks: map[edgeStackID]*edgeStack{}, stopSignal: nil, @@ -75,6 +76,10 @@ func NewManager(dockerStackService agent.DockerStackService, keyService agent.Ed // UpdateStacksStatus updates stacks version and status func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { + if !manager.isEnabled { + return nil + } + for stackID, version := range stacks { stack, ok := manager.stacks[edgeStackID(stackID)] if ok { @@ -145,17 +150,21 @@ func (manager *EdgeStackManager) Stop() error { if manager.stopSignal != nil { close(manager.stopSignal) manager.stopSignal = nil + manager.isEnabled = false } return nil } // Start starts the loop checking for stacks to deploy -func (manager *EdgeStackManager) Start() error { +func (manager *EdgeStackManager) Start(portainerURL, endpointID string) error { if manager.stopSignal != nil { - return errors.New("Manager is already started") + return nil } + manager.portainerURL = portainerURL + manager.endpointID = endpointID + manager.isEnabled = true manager.stopSignal = make(chan struct{}) go (func() { @@ -227,12 +236,10 @@ func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackF } func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.PortainerClient, error) { - portainerURL, endpointID, err := manager.keyService.GetPortainerConfig() - if err != nil { - return nil, err + if manager.portainerURL == "" || manager.endpointID == "" || manager.edgeID == "" { + return nil, errors.New("Client parameters are invalid") } - - return portainerclient.NewPortainerClient(portainerURL, endpointID, manager.edgeID), nil + return portainerclient.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil } func (manager *EdgeStackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { diff --git a/http/tunnel/poll.go b/http/tunnel/poll.go index a2b3aa11..9dc9f3be 100644 --- a/http/tunnel/poll.go +++ b/http/tunnel/poll.go @@ -48,12 +48,8 @@ func (operator *Operator) createHTTPClient(timeout float64) { } func (operator *Operator) poll() error { - portainerURL, endpointID, err := operator.edgeKeyService.GetPortainerConfig() - if err != nil { - return err - } - pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", portainerURL, endpointID) + pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", operator.portainerURL, operator.endpointID) req, err := http.NewRequest("GET", pollURL, nil) if err != nil { return err @@ -142,14 +138,9 @@ func (operator *Operator) createTunnel(encodedCredentials string, remotePort int return err } - tunnelServerAddr, tunnelServerFingerprint, err := operator.edgeKeyService.GetTunnelConfig() - if err != nil { - return err - } - tunnelConfig := agent.TunnelConfig{ - ServerAddr: tunnelServerAddr, - ServerFingerpint: tunnelServerFingerprint, + ServerAddr: operator.tunnelServerAddr, + ServerFingerpint: operator.tunnelServerFingerprint, Credentials: string(credentials), RemotePort: strconv.Itoa(remotePort), LocalAddr: operator.apiServerAddr, diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index 3edc6197..e9d4da44 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -16,18 +16,21 @@ const tunnelActivityCheckInterval = 30 * time.Second // Operator is used to poll a Portainer instance and to establish a reverse tunnel if needed. // It also takes care of closing the tunnel after a set period of inactivity. type Operator struct { - apiServerAddr string - pollIntervalInSeconds float64 - insecurePoll bool - inactivityTimeout time.Duration - edgeID string - httpClient *http.Client - tunnelClient agent.ReverseTunnelClient - scheduleManager agent.Scheduler - lastActivity time.Time - refreshSignal chan struct{} - edgeStackManager agent.EdgeStackManager - edgeKeyService agent.EdgeKeyService + apiServerAddr string + pollIntervalInSeconds float64 + insecurePoll bool + inactivityTimeout time.Duration + edgeID string + httpClient *http.Client + tunnelClient agent.ReverseTunnelClient + scheduleManager agent.Scheduler + lastActivity time.Time + refreshSignal chan struct{} + edgeStackManager agent.EdgeStackManager + portainerURL string + endpointID string + tunnelServerAddr string + tunnelServerFingerprint string } // OperatorConfig represents the configuration used to create a new Operator. @@ -40,7 +43,7 @@ type OperatorConfig struct { } // NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, edgeStackManager agent.EdgeStackManager, config *OperatorConfig) (*Operator, error) { +func NewTunnelOperator(edgeStackManager agent.EdgeStackManager, config *OperatorConfig) (*Operator, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err @@ -60,7 +63,6 @@ func NewTunnelOperator(edgeKeyService agent.EdgeKeyService, edgeStackManager age tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, - edgeKeyService: edgeKeyService, edgeStackManager: edgeStackManager, }, nil } @@ -82,13 +84,20 @@ func (operator *Operator) ResetActivityTimer() { // if needed as well as manage schedules. // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. -func (operator *Operator) Start() error { - if !operator.edgeKeyService.IsKeySet() { - return errors.New("missing Edge key") +func (operator *Operator) Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error { + if portainerURL == "" || endpointID == "" || tunnelServerAddr == "" || tunnelServerFingerprint == "" { + return errors.New("Tunnel operator parameters are invalid") } + + operator.portainerURL = portainerURL + operator.endpointID = endpointID + operator.tunnelServerAddr = tunnelServerAddr + operator.tunnelServerFingerprint = tunnelServerFingerprint + if operator.refreshSignal != nil { return nil } + operator.refreshSignal = make(chan struct{}) operator.startStatusPollLoop() operator.startActivityMonitoringLoop() @@ -111,12 +120,7 @@ func (operator *Operator) restartStatusPollLoop() { } func (operator *Operator) startStatusPollLoop() error { - portainerURL, _, err := operator.edgeKeyService.GetPortainerConfig() - if err != nil { - return err - } - - log.Printf("[DEBUG] [http,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, portainerURL) + log.Printf("[DEBUG] [http,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, operator.portainerURL) ticker := time.NewTicker(time.Duration(operator.pollIntervalInSeconds) * time.Second) go func() { From cadb1e7051e944c0e6d569c31e7f0bc5491ef040 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 12:53:01 +0300 Subject: [PATCH 46/91] refactor(edge): move key related functionality to key --- edge/edge.go | 90 +--------------------------------- edge/key.go | 79 +++++++++++++++++++++++++++++ http/edge.go | 7 --- http/handler/handler.go | 2 +- http/handler/key/handler.go | 4 +- http/handler/key/key_create.go | 11 ----- 6 files changed, 82 insertions(+), 111 deletions(-) diff --git a/edge/edge.go b/edge/edge.go index 7af483af..d4462a1b 100644 --- a/edge/edge.go +++ b/edge/edge.go @@ -7,9 +7,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/exec" - "github.com/portainer/agent/filesystem" "github.com/portainer/agent/http" - "github.com/portainer/agent/http/client" "github.com/portainer/agent/http/edgestacks" "github.com/portainer/agent/http/tunnel" ) @@ -73,7 +71,7 @@ func (manager *EdgeManager) Enable(edgeKey string) error { if edgeKey != "" { log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") - err := manager.associateEdgeKey(edgeKey) + err := manager.SetKey(edgeKey) if err != nil { return err } @@ -92,92 +90,6 @@ func (manager *EdgeManager) ResetActivityTimer() { manager.tunnelOperator.ResetActivityTimer() } -func (manager *EdgeManager) retrieveEdgeKey(edgeKey string) (string, error) { - - if edgeKey != "" { - log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") - return edgeKey, nil - } - - var keyRetrievalError error - - edgeKey, keyRetrievalError = manager.retrieveEdgeKeyFromFilesystem() - if keyRetrievalError != nil { - return "", keyRetrievalError - } - - if edgeKey == "" && manager.clusterService != nil { - edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(manager.clusterService) - if keyRetrievalError != nil { - return "", keyRetrievalError - } - } - - return edgeKey, nil -} - -func (manager *EdgeManager) retrieveEdgeKeyFromFilesystem() (string, error) { - var edgeKey string - - edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) - - keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) - if err != nil { - return "", err - } - - if keyFileExists { - filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) - if err != nil { - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") - edgeKey = string(filesystemKey) - } - - return edgeKey, nil -} - -func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { - var edgeKey string - - member := clusterService.GetMemberWithEdgeKeySet() - if member != nil { - httpCli := client.NewAPIClient() - - memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) - memberKey, err := httpCli.GetEdgeKey(memberAddr) - if err != nil { - log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") - edgeKey = memberKey - } - - return edgeKey, nil -} - -func (manager *EdgeManager) associateEdgeKey(edgeKey string) error { - err := manager.SetKey(edgeKey) - if err != nil { - return err - } - - if manager.clusterService != nil { - tags := manager.clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = manager.clusterService.UpdateTags(tags) - if err != nil { - return err - } - } - - return nil -} - func (manager *EdgeManager) serveEdgeUI() { edgeServer := http.NewEdgeServer(manager, manager.clusterService) diff --git a/edge/key.go b/edge/key.go index 5ef66efc..54d9cc50 100644 --- a/edge/key.go +++ b/edge/key.go @@ -4,10 +4,12 @@ import ( "encoding/base64" "errors" "fmt" + "log" "strings" "github.com/portainer/agent" "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/http/client" ) type edgeKey struct { @@ -31,6 +33,15 @@ func (manager *EdgeManager) SetKey(key string) error { manager.key = edgeKey + if manager.clusterService != nil { + tags := manager.clusterService.GetTags() + tags[agent.MemberTagEdgeKeySet] = "set" + err = manager.clusterService.UpdateTags(tags) + if err != nil { + return err + } + } + return nil } @@ -103,3 +114,71 @@ func encodeKey(edgeKey *edgeKey) string { encodedKey := base64.RawStdEncoding.EncodeToString([]byte(keyInfo)) return encodedKey } + +func (manager *EdgeManager) retrieveEdgeKey(edgeKey string) (string, error) { + + if edgeKey != "" { + log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") + return edgeKey, nil + } + + var keyRetrievalError error + + edgeKey, keyRetrievalError = retrieveEdgeKeyFromFilesystem() + if keyRetrievalError != nil { + return "", keyRetrievalError + } + + if edgeKey == "" && manager.clusterService != nil { + edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(manager.clusterService) + if keyRetrievalError != nil { + return "", keyRetrievalError + } + } + + return edgeKey, nil +} + +func retrieveEdgeKeyFromFilesystem() (string, error) { + var edgeKey string + + edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) + + keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) + if err != nil { + return "", err + } + + if keyFileExists { + filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) + if err != nil { + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") + edgeKey = string(filesystemKey) + } + + return edgeKey, nil +} + +func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { + var edgeKey string + + member := clusterService.GetMemberWithEdgeKeySet() + if member != nil { + httpCli := client.NewAPIClient() + + memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) + memberKey, err := httpCli.GetEdgeKey(memberAddr) + if err != nil { + log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") + edgeKey = memberKey + } + + return edgeKey, nil +} diff --git a/http/edge.go b/http/edge.go index 3091df9b..f57a7a02 100644 --- a/http/edge.go +++ b/http/edge.go @@ -67,13 +67,6 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { if server.clusterService != nil { tags := server.clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = server.clusterService.UpdateTags(tags) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - go server.propagateKeyInCluster(tags[agent.MemberTagKeyNodeName], key) } diff --git a/http/handler/handler.go b/http/handler/handler.go index cfbfa2dd..bd70395d 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -61,7 +61,7 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.TunnelOperator, config.ClusterService, notaryService, config.EdgeManager, config.EdgeMode), + keyHandler: key.NewHandler(config.TunnelOperator, notaryService, config.EdgeManager, config.EdgeMode), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index 4b871fca..e3c0dec5 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -15,7 +15,6 @@ type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator edgeManager agent.EdgeManager - clusterService agent.ClusterService edgeMode bool } @@ -23,12 +22,11 @@ type Handler struct { // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, clusterService agent.ClusterService, notaryService *security.NotaryService, edgeManager agent.EdgeManager, edgeMode bool) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager agent.EdgeManager, edgeMode bool) *Handler { h := &Handler{ Router: mux.NewRouter(), edgeManager: edgeManager, tunnelOperator: tunnelOperator, - clusterService: clusterService, edgeMode: edgeMode, } diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index 013d2412..2a4979fb 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -5,8 +5,6 @@ import ( "log" "net/http" - "github.com/portainer/agent" - "github.com/portainer/libhttp/request" httperror "github.com/portainer/libhttp/error" @@ -46,14 +44,5 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to associate Edge key", err} } - if handler.clusterService != nil { - tags := handler.clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" - err = handler.clusterService.UpdateTags(tags) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update agent cluster tags", err} - } - } - return response.Empty(w) } From 84804e22f94b2c1c52cd85f2f87508af1fb39c6c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 13:05:21 +0300 Subject: [PATCH 47/91] feat(edge): start config check when key is set --- edge/edge.go | 79 ++++++++++++++++++++++++++++------------------------ edge/key.go | 2 +- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/edge/edge.go b/edge/edge.go index d4462a1b..fdb64398 100644 --- a/edge/edge.go +++ b/edge/edge.go @@ -82,8 +82,7 @@ func (manager *EdgeManager) Enable(edgeKey string) error { manager.serveEdgeUI() } - return manager.startRuntimeConfigCheckProcess() - + return nil } func (manager *EdgeManager) ResetActivityTimer() { @@ -122,58 +121,64 @@ func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { return err } + err = manager.checkRuntimeConfig() + if err != nil { + return err + } + ticker := time.NewTicker(runtimeCheckFrequency) go func() { for { select { case <-ticker.C: - key := manager.GetKey() - if key == "" { - continue - } - - agentTags, err := manager.infoService.GetInformationFromDockerEngine() + err := manager.checkRuntimeConfig() if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured during Docker runtime configuration check] [error: %s]", err) - continue + log.Printf("[ERROR] [main,edge,runtime] [message: an error occured during Docker runtime configuration check] [error: %s]", err) } + } + } + }() - isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" - isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm + return nil +} - log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) +func (manager *EdgeManager) checkRuntimeConfig() error { + agentTags, err := manager.infoService.GetInformationFromDockerEngine() + if err != nil { + return err + } - if !isSwarm || isLeader { - err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while starting poll] [error: %s]", err) - } + isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" + isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm - } else { - err = manager.tunnelOperator.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,docker] [message: an error occured while stopping the short-poll process] [error: %s]", err) - } + log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) - } + if !isSwarm || isLeader { + err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) + if err != nil { + return err + } - if isSwarm && isLeader { - err = manager.stackManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) - if err != nil { - log.Printf("[ERROR] [main,edge,stack] [message: an error occured while starting the Edge stack manager] [error: %s]", err) - } + } else { + err = manager.tunnelOperator.Stop() + if err != nil { + return err + } + } - } else { - err = manager.stackManager.Stop() - if err != nil { - log.Printf("[ERROR] [main,edge,stack] [message: an error occured while stopping the Edge stack manager] [error: %s]", err) - } + if isSwarm && isLeader { + err = manager.stackManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) + if err != nil { + return err + } - } - } + } else { + err = manager.stackManager.Stop() + if err != nil { + return err } - }() + } return nil } diff --git a/edge/key.go b/edge/key.go index 54d9cc50..c6141d6a 100644 --- a/edge/key.go +++ b/edge/key.go @@ -42,7 +42,7 @@ func (manager *EdgeManager) SetKey(key string) error { } } - return nil + return manager.startRuntimeConfigCheckProcess() } // GetKey returns the key associated to the manager From 114c5fac17c65a62d76fc0502f59ce04290968cd Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 13:17:55 +0300 Subject: [PATCH 48/91] style(edge): add comments for functions --- edge/edge.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/edge/edge.go b/edge/edge.go index fdb64398..ca5ca706 100644 --- a/edge/edge.go +++ b/edge/edge.go @@ -12,6 +12,7 @@ import ( "github.com/portainer/agent/http/tunnel" ) +// EdgeManager manages Edge functionality type EdgeManager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService @@ -23,6 +24,7 @@ type EdgeManager struct { key *edgeKey } +// NewEdgeManager creates an instance of EdgeManager func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) @@ -62,6 +64,7 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService }, nil } +// Enable enables the manager func (manager *EdgeManager) Enable(edgeKey string) error { edgeKey, err := manager.retrieveEdgeKey(edgeKey) if err != nil { @@ -85,6 +88,7 @@ func (manager *EdgeManager) Enable(edgeKey string) error { return nil } +// ResetActivityTimer resets the activity timer func (manager *EdgeManager) ResetActivityTimer() { manager.tunnelOperator.ResetActivityTimer() } From cc7f071b9e87db75ac06bbc6f28d55a59a9208f8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 13:31:37 +0300 Subject: [PATCH 49/91] refactor(edge): move edge package to internal --- cmd/agent/main.go | 2 +- {edge => internal/edge}/edge.go | 0 {edge => internal/edge}/key.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {edge => internal/edge}/edge.go (100%) rename {edge => internal/edge}/key.go (100%) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index ab26e8a7..90a45dae 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,9 +8,9 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" - "github.com/portainer/agent/edge" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" + "github.com/portainer/agent/internal/edge" "github.com/portainer/agent/logutils" "github.com/portainer/agent/net" "github.com/portainer/agent/os" diff --git a/edge/edge.go b/internal/edge/edge.go similarity index 100% rename from edge/edge.go rename to internal/edge/edge.go diff --git a/edge/key.go b/internal/edge/key.go similarity index 100% rename from edge/key.go rename to internal/edge/key.go From 81c431190af9d23de36de3c4ca2f1ed52c4c444c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 15:42:23 +0300 Subject: [PATCH 50/91] refactor(http): move portainer client to client package --- http/{portainerclient => client}/portainer_client.go | 4 ++-- http/edgestacks/edgestacks.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename http/{portainerclient => client}/portainer_client.go (94%) diff --git a/http/portainerclient/portainer_client.go b/http/client/portainer_client.go similarity index 94% rename from http/portainerclient/portainer_client.go rename to http/client/portainer_client.go index 47e6d09a..2bb4f3ce 100644 --- a/http/portainerclient/portainer_client.go +++ b/http/client/portainer_client.go @@ -1,4 +1,4 @@ -package portainerclient +package client import ( "bytes" @@ -56,7 +56,7 @@ func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("[ERROR] [http,portainerclient] [response_code: %d] [message: GetEdgeStackConfig operation failed] \n", resp.StatusCode) + log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: GetEdgeStackConfig operation failed] \n", resp.StatusCode) return "", false, errors.New("GetEdgeStackConfig operation failed") } diff --git a/http/edgestacks/edgestacks.go b/http/edgestacks/edgestacks.go index fbba7d16..48994bb1 100644 --- a/http/edgestacks/edgestacks.go +++ b/http/edgestacks/edgestacks.go @@ -8,7 +8,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/filesystem" - "github.com/portainer/agent/http/portainerclient" + "github.com/portainer/agent/http/client" ) const baseDir = "/tmp/edge_stacks" @@ -235,11 +235,11 @@ func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackF } } -func (manager *EdgeStackManager) createPortainerClient() (*portainerclient.PortainerClient, error) { +func (manager *EdgeStackManager) createPortainerClient() (*client.PortainerClient, error) { if manager.portainerURL == "" || manager.endpointID == "" || manager.edgeID == "" { return nil, errors.New("Client parameters are invalid") } - return portainerclient.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil + return client.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil } func (manager *EdgeStackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { From 6f8f26c5d2a5c8749124d55919928f7d7fd7eec6 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 16:01:13 +0300 Subject: [PATCH 51/91] refactor(edge): rename runtime variables --- internal/edge/edge.go | 10 +++++----- internal/edge/key.go | 20 -------------------- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index ca5ca706..5f40ffed 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -153,12 +153,12 @@ func (manager *EdgeManager) checkRuntimeConfig() error { return err } - isLeader := agentTags[agent.MemberTagKeyIsLeader] == "1" - isSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm + agentRunsOnLeaderNode := agentTags[agent.MemberTagKeyIsLeader] == "1" + agentRunsOnSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm - log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], isLeader) + log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) - if !isSwarm || isLeader { + if !agentRunsOnSwarm || agentRunsOnLeaderNode { err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) if err != nil { return err @@ -171,7 +171,7 @@ func (manager *EdgeManager) checkRuntimeConfig() error { } } - if isSwarm && isLeader { + if agentRunsOnSwarm && agentRunsOnLeaderNode { err = manager.stackManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) if err != nil { return err diff --git a/internal/edge/key.go b/internal/edge/key.go index c6141d6a..27d507e3 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -56,26 +56,6 @@ func (manager *EdgeManager) GetKey() string { return encodedKey } -// GetPortainerConfig returns portainer url and endpoint id -func (manager *EdgeManager) GetPortainerConfig() (string, string, error) { - if manager.key == nil { - return "", "", errors.New("Key is not set") - } - - key := manager.key - return key.PortainerInstanceURL, key.EndpointID, nil -} - -// GetTunnelConfig returns tunnel url and tunnel fingerprint -func (manager *EdgeManager) GetTunnelConfig() (string, string, error) { - if manager.key == nil { - return "", "", errors.New("Key is not set") - } - - key := manager.key - return key.TunnelServerAddr, key.TunnelServerFingerprint, nil -} - // IsKeySet checks if a key is associated to the manager func (manager *EdgeManager) IsKeySet() bool { if manager.key == nil { From 662d331ecf84987be8edeb620abf11bc948945df Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 5 May 2020 15:57:50 +0300 Subject: [PATCH 52/91] fix(agent): change description of docker stack service Co-authored-by: Anthony Lapenna --- agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent.go b/agent.go index e19989bb..e143aab8 100644 --- a/agent.go +++ b/agent.go @@ -143,7 +143,7 @@ type ( Schedule(schedules []Schedule) error } - // DockerStackService is a service to manager deploy and remove docker stacks + // DockerStackService is a service used to deploy and remove Docker stacks DockerStackService interface { Login() error Logout() error From 071a12796378657987a9ad3d4ca0b49a3409b079 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 08:50:47 +0300 Subject: [PATCH 53/91] refactor(edge): remove edge manager interface --- agent.go | 10 --------- cmd/agent/main.go | 41 ++++++++++++++++++++++++++++++++----- dev.sh | 4 ++-- http/edge.go | 5 +++-- http/handler/handler.go | 5 +++-- http/handler/key/handler.go | 5 +++-- http/server.go | 5 +++-- internal/edge/edge.go | 31 ---------------------------- 8 files changed, 50 insertions(+), 56 deletions(-) diff --git a/agent.go b/agent.go index e143aab8..d5392f9a 100644 --- a/agent.go +++ b/agent.go @@ -128,16 +128,6 @@ type ( ResetActivityTimer() } - // EdgeManager is a service to manager edge tasks - EdgeManager interface { - Enable(edgeKey string) error - IsKeySet() bool - SetKey(key string) error - GetKey() string - - ResetActivityTimer() - } - // Scheduler is used to manage schedules Scheduler interface { Schedule(schedules []Schedule) error diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 90a45dae..fed7a006 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -84,17 +84,23 @@ func main() { defer clusterService.Leave() } - var edgeManager agent.EdgeManager + edgeManager, err := edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) + if err != nil { + log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) + } + if options.EdgeMode { - edgeManager, err = edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) - if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) - } err = edgeManager.Enable(options.EdgeKey) if err != nil { log.Fatalf("[ERROR] [main,edge] [message: Unable to enable edge mode] [error: %s]", err) } + + if !edgeManager.IsKeySet() { + log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") + + serveEdgeUI(edgeManager, clusterService, options.EdgeServerAddr, options.EdgeServerPort) + } } systemService := ghw.NewSystemService(agent.HostRoot) @@ -146,3 +152,28 @@ func parseOptions() (*agent.Options, error) { optionParser := os.NewEnvOptionParser() return optionParser.Options() } + +func serveEdgeUI(edgeManager *edge.EdgeManager, clusterService agent.ClusterService, serverAddr, serverPort string) { + edgeServer := http.NewEdgeServer(edgeManager, clusterService) + + go func() { + log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", serverAddr, serverPort) + + err := edgeServer.Start(serverAddr, serverPort) + if err != nil { + log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) + } + + log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") + }() + + go func() { + timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) + <-timer1.C + + if !edgeManager.IsKeySet() { + log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) + edgeServer.Shutdown() + } + }() +} diff --git a/dev.sh b/dev.sh index d6e3c735..4821b6ab 100755 --- a/dev.sh +++ b/dev.sh @@ -45,6 +45,7 @@ function deploy_local() { echo "Image build..." docker build --no-cache -t "${IMAGE_NAME}" -f build/linux/Dockerfile . # docker push "${IMAGE_NAME}" + # -e EDGE_KEY="aHR0cDovLzE3Mi4xNy4wLjE6OTAwMHwxNzIuMTcuMC4xOjgwMDB8ZWU6YTQ6YTg6ZDY6YzQ6Njk6MjY6MGI6N2Y6MDk6YTU6YTk6N2Y6NzQ6YTk6ZDZ8Nw" \ echo "Deployment..." @@ -53,7 +54,6 @@ function deploy_local() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ - -e EDGE_KEY="aHR0cDovLzE3Mi4xNy4wLjE6OTAwMHwxNzIuMTcuMC4xOjgwMDB8ZWU6YTQ6YTg6ZDY6YzQ6Njk6MjY6MGI6N2Y6MDk6YTU6YTk6N2Y6NzQ6YTk6ZDZ8Nw" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker/volumes:/var/lib/docker/volumes \ -v /:/host \ @@ -92,6 +92,7 @@ function deploy_swarm() { sleep 5 echo "Deployment..." + # -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ docker -H "${DOCKER_MANAGER}:2375" network create --driver overlay portainer-agent-dev-net docker -H "${DOCKER_MANAGER}:2375" service create --name portainer-agent-dev \ @@ -100,7 +101,6 @@ function deploy_swarm() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ - -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ --mode global \ --mount type=bind,src=//var/run/docker.sock,dst=/var/run/docker.sock \ --mount type=bind,src=//var/lib/docker/volumes,dst=/var/lib/docker/volumes \ diff --git a/http/edge.go b/http/edge.go index f57a7a02..ed94df64 100644 --- a/http/edge.go +++ b/http/edge.go @@ -8,6 +8,7 @@ import ( "time" "github.com/portainer/agent/http/client" + "github.com/portainer/agent/internal/edge" "github.com/gorilla/mux" "github.com/portainer/agent" @@ -16,12 +17,12 @@ import ( // EdgeServer expose an UI to associate an Edge key with the agent. type EdgeServer struct { httpServer *http.Server - edgeManager agent.EdgeManager + edgeManager *edge.EdgeManager clusterService agent.ClusterService } // NewEdgeServer returns a pointer to a new instance of EdgeServer. -func NewEdgeServer(edgeManager agent.EdgeManager, clusterService agent.ClusterService) *EdgeServer { +func NewEdgeServer(edgeManager *edge.EdgeManager, clusterService agent.ClusterService) *EdgeServer { return &EdgeServer{ clusterService: clusterService, edgeManager: edgeManager, diff --git a/http/handler/handler.go b/http/handler/handler.go index bd70395d..48215b04 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -17,6 +17,7 @@ import ( "github.com/portainer/agent/http/handler/websocket" "github.com/portainer/agent/http/proxy" "github.com/portainer/agent/http/security" + "github.com/portainer/agent/internal/edge" httperror "github.com/portainer/libhttp/error" ) @@ -32,7 +33,7 @@ type Handler struct { hostHandler *host.Handler pingHandler *ping.Handler securedProtocol bool - edgeManager agent.EdgeManager + edgeManager *edge.EdgeManager } // Config represents a server handler configuration @@ -41,7 +42,7 @@ type Config struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService - EdgeManager agent.EdgeManager + EdgeManager *edge.EdgeManager TunnelOperator agent.TunnelOperator AgentTags map[string]string AgentOptions *agent.Options diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index e3c0dec5..be9884ac 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" "github.com/portainer/agent/http/security" + "github.com/portainer/agent/internal/edge" httperror "github.com/portainer/libhttp/error" ) @@ -14,7 +15,7 @@ import ( type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator - edgeManager agent.EdgeManager + edgeManager *edge.EdgeManager edgeMode bool } @@ -22,7 +23,7 @@ type Handler struct { // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager agent.EdgeManager, edgeMode bool) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.EdgeManager, edgeMode bool) *Handler { h := &Handler{ Router: mux.NewRouter(), edgeManager: edgeManager, diff --git a/http/server.go b/http/server.go index 4339a36e..7b595078 100644 --- a/http/server.go +++ b/http/server.go @@ -7,6 +7,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/http/handler" + "github.com/portainer/agent/internal/edge" ) // APIServer is the web server exposing the API of an agent. @@ -16,7 +17,7 @@ type APIServer struct { systemService agent.SystemService clusterService agent.ClusterService signatureService agent.DigitalSignatureService - edgeManager agent.EdgeManager + edgeManager *edge.EdgeManager agentTags map[string]string agentOptions *agent.Options edgeMode bool @@ -30,7 +31,7 @@ type APIServerConfig struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService - EdgeManager agent.EdgeManager + EdgeManager *edge.EdgeManager AgentTags map[string]string AgentOptions *agent.Options EdgeMode bool diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 5f40ffed..26930456 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -7,7 +7,6 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/exec" - "github.com/portainer/agent/http" "github.com/portainer/agent/http/edgestacks" "github.com/portainer/agent/http/tunnel" ) @@ -78,11 +77,6 @@ func (manager *EdgeManager) Enable(edgeKey string) error { if err != nil { return err } - - } else { - log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - - manager.serveEdgeUI() } return nil @@ -93,31 +87,6 @@ func (manager *EdgeManager) ResetActivityTimer() { manager.tunnelOperator.ResetActivityTimer() } -func (manager *EdgeManager) serveEdgeUI() { - edgeServer := http.NewEdgeServer(manager, manager.clusterService) - - go func() { - log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", manager.serverAddr, manager.serverPort) - - err := edgeServer.Start(manager.serverAddr, manager.serverPort) - if err != nil { - log.Fatalf("[ERROR] [main,edge,http] [message: Unable to start Edge server] [error: %s]", err) - } - - log.Println("[INFO] [main,edge,http] [message: Edge server shutdown]") - }() - - go func() { - timer1 := time.NewTimer(agent.DefaultEdgeSecurityShutdown * time.Minute) - <-timer1.C - - if !manager.IsKeySet() { - log.Printf("[INFO] [main,edge,http] [message: Shutting down Edge UI server as no key was specified after %d minutes]", agent.DefaultEdgeSecurityShutdown) - edgeServer.Shutdown() - } - }() -} - func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) From 5dcd7e960873dbfdad23b2006ffe26d6c1b023ce Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 09:06:39 +0300 Subject: [PATCH 54/91] refactor(edge): move stacks manager to internal --- agent.go | 7 ------ http/tunnel/poll.go | 2 +- http/tunnel/tunnel.go | 7 +++--- internal/edge/edge.go | 10 ++++----- {http => internal}/edgestacks/edgestacks.go | 24 ++++++++++----------- 5 files changed, 22 insertions(+), 28 deletions(-) rename {http => internal}/edgestacks/edgestacks.go (88%) diff --git a/agent.go b/agent.go index d5392f9a..1b4dcce0 100644 --- a/agent.go +++ b/agent.go @@ -140,13 +140,6 @@ type ( Deploy(name, stackFileContent string, prune bool) error Remove(name string) error } - - // EdgeStackManager is a service to manage Edge stacks - EdgeStackManager interface { - UpdateStacksStatus(stacks map[int]int) error - Start(portainerURL, endpointID string) error - Stop() error - } ) const ( diff --git a/http/tunnel/poll.go b/http/tunnel/poll.go index 9dc9f3be..6ae3f541 100644 --- a/http/tunnel/poll.go +++ b/http/tunnel/poll.go @@ -117,7 +117,7 @@ func (operator *Operator) poll() error { stacks[stack.ID] = stack.Version } - err := operator.edgeStackManager.UpdateStacksStatus(stacks) + err := operator.edgeStacksManager.UpdateStacksStatus(stacks) if err != nil { log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) return err diff --git a/http/tunnel/tunnel.go b/http/tunnel/tunnel.go index e9d4da44..ea5628e9 100644 --- a/http/tunnel/tunnel.go +++ b/http/tunnel/tunnel.go @@ -9,6 +9,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/chisel" "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/internal/edgestacks" ) const tunnelActivityCheckInterval = 30 * time.Second @@ -26,7 +27,7 @@ type Operator struct { scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} - edgeStackManager agent.EdgeStackManager + edgeStacksManager *edgestacks.Manager portainerURL string endpointID string tunnelServerAddr string @@ -43,7 +44,7 @@ type OperatorConfig struct { } // NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(edgeStackManager agent.EdgeStackManager, config *OperatorConfig) (*Operator, error) { +func NewTunnelOperator(edgeStacksManager *edgestacks.Manager, config *OperatorConfig) (*Operator, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err @@ -63,7 +64,7 @@ func NewTunnelOperator(edgeStackManager agent.EdgeStackManager, config *Operator tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, - edgeStackManager: edgeStackManager, + edgeStacksManager: edgeStacksManager, }, nil } diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 26930456..6e214f0c 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -7,8 +7,8 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/exec" - "github.com/portainer/agent/http/edgestacks" "github.com/portainer/agent/http/tunnel" + "github.com/portainer/agent/internal/edgestacks" ) // EdgeManager manages Edge functionality @@ -16,7 +16,7 @@ type EdgeManager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService infoService agent.InfoService - stackManager agent.EdgeStackManager + stacksManager *edgestacks.Manager tunnelOperator agent.TunnelOperator serverAddr string serverPort string @@ -56,7 +56,7 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService clusterService: clusterService, dockerStackService: dockerStackService, infoService: infoService, - stackManager: edgeStackManager, + stacksManager: edgeStackManager, tunnelOperator: tunnelOperator, serverAddr: options.EdgeServerAddr, serverPort: options.EdgeServerPort, @@ -141,13 +141,13 @@ func (manager *EdgeManager) checkRuntimeConfig() error { } if agentRunsOnSwarm && agentRunsOnLeaderNode { - err = manager.stackManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) + err = manager.stacksManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) if err != nil { return err } } else { - err = manager.stackManager.Stop() + err = manager.stacksManager.Stop() if err != nil { return err } diff --git a/http/edgestacks/edgestacks.go b/internal/edgestacks/edgestacks.go similarity index 88% rename from http/edgestacks/edgestacks.go rename to internal/edgestacks/edgestacks.go index 48994bb1..ee707743 100644 --- a/http/edgestacks/edgestacks.go +++ b/internal/edgestacks/edgestacks.go @@ -53,8 +53,8 @@ const ( edgeStackStatusAcknowledged ) -// EdgeStackManager represents a service for managing Edge stacks -type EdgeStackManager struct { +// Manager represents a service for managing Edge stacks +type Manager struct { stacks map[edgeStackID]*edgeStack stopSignal chan struct{} dockerStackService agent.DockerStackService @@ -64,9 +64,9 @@ type EdgeStackManager struct { isEnabled bool } -// NewManager creates a new instance of EdgeStackManager -func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*EdgeStackManager, error) { - return &EdgeStackManager{ +// NewManager creates a new instance of Manager +func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*Manager, error) { + return &Manager{ dockerStackService: dockerStackService, edgeID: edgeID, stacks: map[edgeStackID]*edgeStack{}, @@ -75,7 +75,7 @@ func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*Ed } // UpdateStacksStatus updates stacks version and status -func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { +func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { if !manager.isEnabled { return nil } @@ -146,7 +146,7 @@ func (manager *EdgeStackManager) UpdateStacksStatus(stacks map[int]int) error { } // Stop stops the manager -func (manager *EdgeStackManager) Stop() error { +func (manager *Manager) Stop() error { if manager.stopSignal != nil { close(manager.stopSignal) manager.stopSignal = nil @@ -157,7 +157,7 @@ func (manager *EdgeStackManager) Stop() error { } // Start starts the loop checking for stacks to deploy -func (manager *EdgeStackManager) Start(portainerURL, endpointID string) error { +func (manager *Manager) Start(portainerURL, endpointID string) error { if manager.stopSignal != nil { return nil } @@ -196,7 +196,7 @@ func (manager *EdgeStackManager) Start(portainerURL, endpointID string) error { return nil } -func (manager *EdgeStackManager) next() *edgeStack { +func (manager *Manager) next() *edgeStack { for _, stack := range manager.stacks { if stack.Status == statusPending { return stack @@ -205,7 +205,7 @@ func (manager *EdgeStackManager) next() *edgeStack { return nil } -func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { +func (manager *Manager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { log.Printf("[DEBUG] [stacksmanager,update] [message: deploying stack %d] \n", stack.ID) stack.Status = statusDone stack.Action = actionIdle @@ -235,14 +235,14 @@ func (manager *EdgeStackManager) deployStack(stack *edgeStack, stackName, stackF } } -func (manager *EdgeStackManager) createPortainerClient() (*client.PortainerClient, error) { +func (manager *Manager) createPortainerClient() (*client.PortainerClient, error) { if manager.portainerURL == "" || manager.endpointID == "" || manager.edgeID == "" { return nil, errors.New("Client parameters are invalid") } return client.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil } -func (manager *EdgeStackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { +func (manager *Manager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { log.Printf("[DEBUG] [stacksmanager,update] [message: removing stack %d] \n", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { From 282af517c59790aa11ca61ac81077acbe56a1a94 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 09:37:43 +0300 Subject: [PATCH 55/91] refactor(edge): init manager only when in edge mode --- cmd/agent/main.go | 7 +++---- internal/edge/edge.go | 47 ++++++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index fed7a006..17e053c6 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -84,16 +84,15 @@ func main() { defer clusterService.Leave() } - edgeManager, err := edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) + edgeManager, err := edge.NewEdgeManager() if err != nil { log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) } if options.EdgeMode { - - err = edgeManager.Enable(options.EdgeKey) + err = edgeManager.Init(options, advertiseAddr, clusterService, infoService) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to enable edge mode] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to init edge manager] [error: %s]", err) } if !edgeManager.IsKeySet() { diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 6e214f0c..bf62b060 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -18,13 +18,19 @@ type EdgeManager struct { infoService agent.InfoService stacksManager *edgestacks.Manager tunnelOperator agent.TunnelOperator - serverAddr string - serverPort string key *edgeKey + edgeMode bool } // NewEdgeManager creates an instance of EdgeManager -func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { +func NewEdgeManager() (*EdgeManager, error) { + + return &EdgeManager{}, nil +} + +// Enable enables the manager +func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) error { + apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) operatorConfig := &tunnel.OperatorConfig{ @@ -39,33 +45,27 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) if err != nil { - return nil, err + return err } + manager.dockerStackService = dockerStackService - edgeStackManager, err := edgestacks.NewManager(dockerStackService, options.EdgeID) + stacksManager, err := edgestacks.NewManager(dockerStackService, options.EdgeID) if err != nil { - return nil, err + return err } + manager.stacksManager = stacksManager - tunnelOperator, err := tunnel.NewTunnelOperator(edgeStackManager, operatorConfig) + tunnelOperator, err := tunnel.NewTunnelOperator(stacksManager, operatorConfig) if err != nil { - return nil, err + return err } + manager.tunnelOperator = tunnelOperator - return &EdgeManager{ - clusterService: clusterService, - dockerStackService: dockerStackService, - infoService: infoService, - stacksManager: edgeStackManager, - tunnelOperator: tunnelOperator, - serverAddr: options.EdgeServerAddr, - serverPort: options.EdgeServerPort, - }, nil -} + manager.infoService = infoService + manager.clusterService = clusterService + manager.edgeMode = true -// Enable enables the manager -func (manager *EdgeManager) Enable(edgeKey string) error { - edgeKey, err := manager.retrieveEdgeKey(edgeKey) + edgeKey, err := manager.retrieveEdgeKey(options.EdgeKey) if err != nil { return err } @@ -82,6 +82,11 @@ func (manager *EdgeManager) Enable(edgeKey string) error { return nil } +// IsEdgeModeEnabled returns true if edge mode is enabled +func (manager *EdgeManager) IsEdgeModeEnabled() bool { + return manager.edgeMode +} + // ResetActivityTimer resets the activity timer func (manager *EdgeManager) ResetActivityTimer() { manager.tunnelOperator.ResetActivityTimer() From 29ca2e133b430734b44a0c8d031575279d641d3c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 09:44:18 +0300 Subject: [PATCH 56/91] refactor(edge): check edgemode with edge manager --- cmd/agent/main.go | 7 +++---- http/handler/handler.go | 3 +-- http/handler/key/handler.go | 4 +--- http/handler/key/key_create.go | 2 +- http/handler/key/key_inspect.go | 2 +- http/server.go | 5 ----- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 17e053c6..b1b92261 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -105,7 +105,7 @@ func main() { systemService := ghw.NewSystemService(agent.HostRoot) var signatureService agent.DigitalSignatureService - if !options.EdgeMode { + if !edgeManager.IsEdgeModeEnabled() { signatureService = crypto.NewECDSAService(options.SharedSecret) tlsService := crypto.TLSService{} @@ -124,10 +124,9 @@ func main() { SignatureService: signatureService, AgentTags: agentTags, AgentOptions: options, - EdgeMode: options.EdgeMode, } - if options.EdgeMode { + if edgeManager.IsEdgeModeEnabled() { config.Addr = advertiseAddr } @@ -140,7 +139,7 @@ func main() { func startAPIServer(config *http.APIServerConfig) error { server := http.NewAPIServer(config) - if config.EdgeMode { + if config.EdgeManager.IsEdgeModeEnabled() { return server.StartUnsecured() } diff --git a/http/handler/handler.go b/http/handler/handler.go index 48215b04..964fa722 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -47,7 +47,6 @@ type Config struct { AgentTags map[string]string AgentOptions *agent.Options Secured bool - EdgeMode bool } var dockerAPIVersionRegexp = regexp.MustCompile(`(/v[0-9]\.[0-9]*)?`) @@ -62,7 +61,7 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.TunnelOperator, notaryService, config.EdgeManager, config.EdgeMode), + keyHandler: key.NewHandler(config.TunnelOperator, notaryService, config.EdgeManager), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index be9884ac..866db48d 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -16,19 +16,17 @@ type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator edgeManager *edge.EdgeManager - edgeMode bool } // NewHandler returns a pointer to an Handler // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.EdgeManager, edgeMode bool) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.EdgeManager) *Handler { h := &Handler{ Router: mux.NewRouter(), edgeManager: edgeManager, tunnelOperator: tunnelOperator, - edgeMode: edgeMode, } h.Handle("/key", diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index 2a4979fb..cc45f47f 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -23,7 +23,7 @@ func (payload *keyCreatePayload) Validate(r *http.Request) error { } func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - if !handler.edgeMode { + if !handler.edgeManager.IsEdgeModeEnabled() { return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } diff --git a/http/handler/key/key_inspect.go b/http/handler/key/key_inspect.go index 99a88633..0b9bd25f 100644 --- a/http/handler/key/key_inspect.go +++ b/http/handler/key/key_inspect.go @@ -13,7 +13,7 @@ type keyInspectResponse struct { } func (handler *Handler) keyInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - if !handler.edgeMode { + if !handler.edgeManager.IsEdgeModeEnabled() { return &httperror.HandlerError{http.StatusServiceUnavailable, "Edge key management is disabled on non Edge agent", errors.New("Edge key management is disabled")} } diff --git a/http/server.go b/http/server.go index 7b595078..048578eb 100644 --- a/http/server.go +++ b/http/server.go @@ -20,7 +20,6 @@ type APIServer struct { edgeManager *edge.EdgeManager agentTags map[string]string agentOptions *agent.Options - edgeMode bool } // APIServerConfig represents a server configuration @@ -34,7 +33,6 @@ type APIServerConfig struct { EdgeManager *edge.EdgeManager AgentTags map[string]string AgentOptions *agent.Options - EdgeMode bool } // NewAPIServer returns a pointer to a APIServer. @@ -48,7 +46,6 @@ func NewAPIServer(config *APIServerConfig) *APIServer { edgeManager: config.EdgeManager, agentTags: config.AgentTags, agentOptions: config.AgentOptions, - edgeMode: config.EdgeMode, } } @@ -59,7 +56,6 @@ func (server *APIServer) StartUnsecured() error { ClusterService: server.clusterService, AgentTags: server.agentTags, AgentOptions: server.agentOptions, - EdgeMode: server.edgeMode, EdgeManager: server.edgeManager, Secured: false, } @@ -87,7 +83,6 @@ func (server *APIServer) StartSecured() error { SignatureService: server.signatureService, AgentTags: server.agentTags, AgentOptions: server.agentOptions, - EdgeMode: server.edgeMode, EdgeManager: server.edgeManager, Secured: true, } From ca911b34a9470bb47974959db03466a837c20612 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 09:56:41 +0300 Subject: [PATCH 57/91] style(agent): remove new lines and fix logs --- http/client/portainer_client.go | 10 +++++----- internal/edgestacks/edgestacks.go | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/http/client/portainer_client.go b/http/client/portainer_client.go index 2bb4f3ce..b1b8beec 100644 --- a/http/client/portainer_client.go +++ b/http/client/portainer_client.go @@ -13,7 +13,7 @@ import ( "github.com/portainer/agent" ) -// PortainerClient is used to execute HTTP requests against the agent API +// PortainerClient is used to execute HTTP requests against the Portainer API type PortainerClient struct { httpClient *http.Client serverAddress string @@ -38,7 +38,7 @@ type stackConfigResponse struct { Prune bool } -// GetEdgeStackConfig fetches the Edge stack config +// GetEdgeStackConfig retrieves the configuration associated to an Edge stack func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool, error) { requestURL := fmt.Sprintf("%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) @@ -56,7 +56,7 @@ func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: GetEdgeStackConfig operation failed] \n", resp.StatusCode) + log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: GetEdgeStackConfig operation failed]", resp.StatusCode) return "", false, errors.New("GetEdgeStackConfig operation failed") } @@ -75,7 +75,7 @@ type setEdgeStackStatusPayload struct { EndpointID int } -// SetEdgeStackStatus set the Edge stack status on portainer server +// SetEdgeStackStatus updates the status of an Edge stack on the Portainer server func (client *PortainerClient) SetEdgeStackStatus(edgeStackID, edgeStackStatus int, error string) error { endpointID, err := strconv.Atoi(client.endpointID) if err != nil { @@ -110,7 +110,7 @@ func (client *PortainerClient) SetEdgeStackStatus(edgeStackID, edgeStackStatus i defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("[ERROR] [http,client] [response_code: %d] [message: SetEdgeStackStatus operation failed]", resp.StatusCode) + log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: SetEdgeStackStatus operation failed]", resp.StatusCode) return errors.New("SetEdgeStackStatus operation failed") } diff --git a/internal/edgestacks/edgestacks.go b/internal/edgestacks/edgestacks.go index ee707743..0c0f3a8b 100644 --- a/internal/edgestacks/edgestacks.go +++ b/internal/edgestacks/edgestacks.go @@ -86,13 +86,13 @@ func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { if stack.Version == version { continue } - log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to update %d] \n", stackID) + log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to update %d]", stackID) stack.Action = actionUpdate stack.Version = version stack.Status = statusPending } else { - log.Printf("[DEBUG] [stacksmanager,update] [message: received new stack %d] \n", stackID) + log.Printf("[DEBUG] [stacksmanager,update] [message: received new stack %d]", stackID) stack = &edgeStack{ Action: actionDeploy, @@ -134,7 +134,7 @@ func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { for stackID, stack := range manager.stacks { if _, ok := stacks[int(stackID)]; !ok { - log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to delete %d] \n", stackID) + log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to delete %d]", stackID) stack.Action = actionDelete stack.Status = statusPending @@ -206,7 +206,7 @@ func (manager *Manager) next() *edgeStack { } func (manager *Manager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [stacksmanager,update] [message: deploying stack %d] \n", stack.ID) + log.Printf("[DEBUG] [stacksmanager,update] [message: deploying stack %d]", stack.ID) stack.Status = statusDone stack.Action = actionIdle responseStatus := int(edgeStackStatusOk) @@ -214,24 +214,24 @@ func (manager *Manager) deployStack(stack *edgeStack, stackName, stackFileLocati err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v] \n", err) + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v]", err) stack.Status = statusError responseStatus = int(edgeStackStatusError) errorMessage = err.Error() } else { - log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d] \n", stack.ID, stack.Version) + log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d]", stack.ID, stack.Version) } manager.stacks[stack.ID] = stack cli, err := manager.createPortainerClient() if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v] \n", err) + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v]", err) } err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v] \n", err) + log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v]", err) } } @@ -243,16 +243,16 @@ func (manager *Manager) createPortainerClient() (*client.PortainerClient, error) } func (manager *Manager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [stacksmanager,update] [message: removing stack %d] \n", stack.ID) + log.Printf("[DEBUG] [stacksmanager,update] [message: removing stack %d]", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { - log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed deleting edge stack file] [error: %v] \n", err) + log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed deleting edge stack file] [error: %v]", err) return } err = manager.dockerStackService.Remove(stackName) if err != nil { - log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed removing stack] [error: %v] \n", err) + log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed removing stack] [error: %v]", err) return } From a63b22a9188bb808b54610ea2da423f88ef91c05 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 10:12:21 +0300 Subject: [PATCH 58/91] feat(tunnel): move tunnel to edge package --- internal/edge/edge.go | 5 ++--- .../tunnel.go => internal/edge/operator.go | 16 +++++++-------- .../poll.go => internal/edge/operator_poll.go | 20 +++++++++---------- 3 files changed, 20 insertions(+), 21 deletions(-) rename http/tunnel/tunnel.go => internal/edge/operator.go (81%) rename http/tunnel/poll.go => internal/edge/operator_poll.go (73%) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index bf62b060..51799034 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -7,7 +7,6 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/exec" - "github.com/portainer/agent/http/tunnel" "github.com/portainer/agent/internal/edgestacks" ) @@ -33,7 +32,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) - operatorConfig := &tunnel.OperatorConfig{ + operatorConfig := &OperatorConfig{ APIServerAddr: apiServerAddr, EdgeID: options.EdgeID, PollFrequency: agent.DefaultEdgePollInterval, @@ -55,7 +54,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } manager.stacksManager = stacksManager - tunnelOperator, err := tunnel.NewTunnelOperator(stacksManager, operatorConfig) + tunnelOperator, err := NewTunnelOperator(stacksManager, operatorConfig) if err != nil { return err } diff --git a/http/tunnel/tunnel.go b/internal/edge/operator.go similarity index 81% rename from http/tunnel/tunnel.go rename to internal/edge/operator.go index ea5628e9..32669ed4 100644 --- a/http/tunnel/tunnel.go +++ b/internal/edge/operator.go @@ -1,4 +1,4 @@ -package tunnel +package edge import ( "errors" @@ -121,7 +121,7 @@ func (operator *Operator) restartStatusPollLoop() { } func (operator *Operator) startStatusPollLoop() error { - log.Printf("[DEBUG] [http,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, operator.portainerURL) + log.Printf("[DEBUG] [internal,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, operator.portainerURL) ticker := time.NewTicker(time.Duration(operator.pollIntervalInSeconds) * time.Second) go func() { @@ -130,11 +130,11 @@ func (operator *Operator) startStatusPollLoop() error { case <-ticker.C: err := operator.poll() if err != nil { - log.Printf("[ERROR] [edge,http,poll] [message: an error occured during short poll] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,poll] [message: an error occured during short poll] [error: %s]", err) } case <-operator.refreshSignal: - log.Println("[DEBUG] [http,edge,poll] [message: shutting down Portainer short-polling client]") + log.Println("[DEBUG] [internal,edge,poll] [message: shutting down Portainer short-polling client]") ticker.Stop() return } @@ -148,7 +148,7 @@ func (operator *Operator) startActivityMonitoringLoop() { ticker := time.NewTicker(tunnelActivityCheckInterval) quit := make(chan struct{}) - log.Printf("[DEBUG] [http,edge,monitoring] [monitoring_interval_seconds: %f] [inactivity_timeout: %s] [message: starting activity monitoring loop]", tunnelActivityCheckInterval.Seconds(), operator.inactivityTimeout.String()) + log.Printf("[DEBUG] [internal,edge,monitoring] [monitoring_interval_seconds: %f] [inactivity_timeout: %s] [message: starting activity monitoring loop]", tunnelActivityCheckInterval.Seconds(), operator.inactivityTimeout.String()) go func() { for { @@ -160,15 +160,15 @@ func (operator *Operator) startActivityMonitoringLoop() { } elapsed := time.Since(operator.lastActivity) - log.Printf("[DEBUG] [http,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: tunnel activity monitoring]", elapsed.Seconds()) + log.Printf("[DEBUG] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: tunnel activity monitoring]", elapsed.Seconds()) if operator.tunnelClient.IsTunnelOpen() && elapsed.Seconds() > operator.inactivityTimeout.Seconds() { - log.Printf("[INFO] [http,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: shutting down tunnel after inactivity period]", elapsed.Seconds()) + log.Printf("[INFO] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: shutting down tunnel after inactivity period]", elapsed.Seconds()) err := operator.tunnelClient.CloseTunnel() if err != nil { - log.Printf("[ERROR] [http,edge,monitoring] [message: unable to shutdown tunnel] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,monitoring] [message: unable to shutdown tunnel] [error: %s]", err) } } diff --git a/http/tunnel/poll.go b/internal/edge/operator_poll.go similarity index 73% rename from http/tunnel/poll.go rename to internal/edge/operator_poll.go index 6ae3f541..dad03735 100644 --- a/http/tunnel/poll.go +++ b/internal/edge/operator_poll.go @@ -1,4 +1,4 @@ -package tunnel +package edge import ( "crypto/tls" @@ -68,7 +68,7 @@ func (operator *Operator) poll() error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("[DEBUG] [http,edge,poll] [response_code: %d] [message: Poll request failure]", resp.StatusCode) + log.Printf("[DEBUG] [internal,edge,poll] [response_code: %d] [message: Poll request failure]", resp.StatusCode) return errors.New("short poll request failed") } @@ -78,34 +78,34 @@ func (operator *Operator) poll() error { return err } - log.Printf("[DEBUG] [http,edge,poll] [status: %s] [port: %d] [schedule_count: %d] [checkin_interval_seconds: %f]", responseData.Status, responseData.Port, len(responseData.Schedules), responseData.CheckinInterval) + log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [port: %d] [schedule_count: %d] [checkin_interval_seconds: %f]", responseData.Status, responseData.Port, len(responseData.Schedules), responseData.CheckinInterval) if responseData.Status == "IDLE" && operator.tunnelClient.IsTunnelOpen() { - log.Printf("[DEBUG] [http,edge,poll] [status: %s] [message: Idle status detected, shutting down tunnel]", responseData.Status) + log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [message: Idle status detected, shutting down tunnel]", responseData.Status) err := operator.tunnelClient.CloseTunnel() if err != nil { - log.Printf("[ERROR] [http,edge,poll] [message: Unable to shutdown tunnel] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,poll] [message: Unable to shutdown tunnel] [error: %s]", err) } } if responseData.Status == "REQUIRED" && !operator.tunnelClient.IsTunnelOpen() { - log.Println("[DEBUG] [http,edge,poll] [message: Required status detected, creating reverse tunnel]") + log.Println("[DEBUG] [internal,edge,poll] [message: Required status detected, creating reverse tunnel]") err := operator.createTunnel(responseData.Credentials, responseData.Port) if err != nil { - log.Printf("[ERROR] [http,edge,poll] [message: Unable to create tunnel] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,poll] [message: Unable to create tunnel] [error: %s]", err) return err } } err = operator.scheduleManager.Schedule(responseData.Schedules) if err != nil { - log.Printf("[ERROR] [http,edge,cron] [message: an error occured during schedule management] [err: %s]", err) + log.Printf("[ERROR] [internal,edge,cron] [message: an error occured during schedule management] [err: %s]", err) } if responseData.CheckinInterval != operator.pollIntervalInSeconds { - log.Printf("[DEBUG] [http,edge,poll] [old_interval: %f] [new_interval: %f] [message: updating poll interval]", operator.pollIntervalInSeconds, responseData.CheckinInterval) + log.Printf("[DEBUG] [internal,edge,poll] [old_interval: %f] [new_interval: %f] [message: updating poll interval]", operator.pollIntervalInSeconds, responseData.CheckinInterval) operator.pollIntervalInSeconds = responseData.CheckinInterval operator.createHTTPClient(responseData.CheckinInterval) go operator.restartStatusPollLoop() @@ -119,7 +119,7 @@ func (operator *Operator) poll() error { err := operator.edgeStacksManager.UpdateStacksStatus(stacks) if err != nil { - log.Printf("[ERROR] [http,edge,stacks] [message: an error occured during stack management] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: an error occured during stack management] [error: %s]", err) return err } } From 269c3c2639ec2e4adf327bb027adb8fb87bd546f Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 10:18:40 +0300 Subject: [PATCH 59/91] refactor(edgestacks): move stacks manager to edge --- internal/edge/edge.go | 13 +++-- internal/edge/operator.go | 5 +- .../edgestacks.go => edge/stack.go} | 50 +++++++++---------- 3 files changed, 33 insertions(+), 35 deletions(-) rename internal/{edgestacks/edgestacks.go => edge/stack.go} (70%) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 51799034..1c44d6bd 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -7,7 +7,6 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/exec" - "github.com/portainer/agent/internal/edgestacks" ) // EdgeManager manages Edge functionality @@ -15,7 +14,7 @@ type EdgeManager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService infoService agent.InfoService - stacksManager *edgestacks.Manager + stacksManager *StacksManager tunnelOperator agent.TunnelOperator key *edgeKey edgeMode bool @@ -40,7 +39,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c InsecurePoll: options.EdgeInsecurePoll, } - log.Printf("[DEBUG] [main,edge,configuration] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) + log.Printf("[DEBUG] [internal,edge] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) if err != nil { @@ -48,7 +47,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } manager.dockerStackService = dockerStackService - stacksManager, err := edgestacks.NewManager(dockerStackService, options.EdgeID) + stacksManager, err := NewStacksManager(dockerStackService, options.EdgeID) if err != nil { return err } @@ -70,7 +69,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } if edgeKey != "" { - log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") + log.Println("[DEBUG] [internal,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") err := manager.SetKey(edgeKey) if err != nil { @@ -111,7 +110,7 @@ func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { case <-ticker.C: err := manager.checkRuntimeConfig() if err != nil { - log.Printf("[ERROR] [main,edge,runtime] [message: an error occured during Docker runtime configuration check] [error: %s]", err) + log.Printf("[ERROR] [internal,edge,runtime] [message: an error occured during Docker runtime configuration check] [error: %s]", err) } } } @@ -129,7 +128,7 @@ func (manager *EdgeManager) checkRuntimeConfig() error { agentRunsOnLeaderNode := agentTags[agent.MemberTagKeyIsLeader] == "1" agentRunsOnSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm - log.Printf("[DEBUG] [main,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) + log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) if !agentRunsOnSwarm || agentRunsOnLeaderNode { err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) diff --git a/internal/edge/operator.go b/internal/edge/operator.go index 32669ed4..6ea9b978 100644 --- a/internal/edge/operator.go +++ b/internal/edge/operator.go @@ -9,7 +9,6 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/chisel" "github.com/portainer/agent/filesystem" - "github.com/portainer/agent/internal/edgestacks" ) const tunnelActivityCheckInterval = 30 * time.Second @@ -27,7 +26,7 @@ type Operator struct { scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} - edgeStacksManager *edgestacks.Manager + edgeStacksManager *StacksManager portainerURL string endpointID string tunnelServerAddr string @@ -44,7 +43,7 @@ type OperatorConfig struct { } // NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(edgeStacksManager *edgestacks.Manager, config *OperatorConfig) (*Operator, error) { +func NewTunnelOperator(edgeStacksManager *StacksManager, config *OperatorConfig) (*Operator, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err diff --git a/internal/edgestacks/edgestacks.go b/internal/edge/stack.go similarity index 70% rename from internal/edgestacks/edgestacks.go rename to internal/edge/stack.go index 0c0f3a8b..b93da973 100644 --- a/internal/edgestacks/edgestacks.go +++ b/internal/edge/stack.go @@ -1,4 +1,4 @@ -package edgestacks +package edge import ( "errors" @@ -53,8 +53,8 @@ const ( edgeStackStatusAcknowledged ) -// Manager represents a service for managing Edge stacks -type Manager struct { +// StacksManager represents a service for managing Edge stacks +type StacksManager struct { stacks map[edgeStackID]*edgeStack stopSignal chan struct{} dockerStackService agent.DockerStackService @@ -64,9 +64,9 @@ type Manager struct { isEnabled bool } -// NewManager creates a new instance of Manager -func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*Manager, error) { - return &Manager{ +// NewStacksManager creates a new instance of StacksManager +func NewStacksManager(dockerStackService agent.DockerStackService, edgeID string) (*StacksManager, error) { + return &StacksManager{ dockerStackService: dockerStackService, edgeID: edgeID, stacks: map[edgeStackID]*edgeStack{}, @@ -75,7 +75,7 @@ func NewManager(dockerStackService agent.DockerStackService, edgeID string) (*Ma } // UpdateStacksStatus updates stacks version and status -func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { +func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { if !manager.isEnabled { return nil } @@ -86,13 +86,13 @@ func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { if stack.Version == version { continue } - log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to update %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [message: received stack to update %d]", stackID) stack.Action = actionUpdate stack.Version = version stack.Status = statusPending } else { - log.Printf("[DEBUG] [stacksmanager,update] [message: received new stack %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [message: received new stack %d]", stackID) stack = &edgeStack{ Action: actionDeploy, @@ -134,7 +134,7 @@ func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { for stackID, stack := range manager.stacks { if _, ok := stacks[int(stackID)]; !ok { - log.Printf("[DEBUG] [stacksmanager,update] [message: received stack to delete %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [message: received stack to delete %d]", stackID) stack.Action = actionDelete stack.Status = statusPending @@ -146,7 +146,7 @@ func (manager *Manager) UpdateStacksStatus(stacks map[int]int) error { } // Stop stops the manager -func (manager *Manager) Stop() error { +func (manager *StacksManager) Stop() error { if manager.stopSignal != nil { close(manager.stopSignal) manager.stopSignal = nil @@ -157,7 +157,7 @@ func (manager *Manager) Stop() error { } // Start starts the loop checking for stacks to deploy -func (manager *Manager) Start(portainerURL, endpointID string) error { +func (manager *StacksManager) Start(portainerURL, endpointID string) error { if manager.stopSignal != nil { return nil } @@ -171,7 +171,7 @@ func (manager *Manager) Start(portainerURL, endpointID string) error { for { select { case <-manager.stopSignal: - log.Println("[DEBUG] [http,edge,stacksmanager] [message: shutting down Edge stack manager]") + log.Println("[DEBUG] [internal,edge,stack] [message: shutting down Edge stack manager]") return default: stack := manager.next() @@ -196,7 +196,7 @@ func (manager *Manager) Start(portainerURL, endpointID string) error { return nil } -func (manager *Manager) next() *edgeStack { +func (manager *StacksManager) next() *edgeStack { for _, stack := range manager.stacks { if stack.Status == statusPending { return stack @@ -205,8 +205,8 @@ func (manager *Manager) next() *edgeStack { return nil } -func (manager *Manager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [stacksmanager,update] [message: deploying stack %d]", stack.ID) +func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { + log.Printf("[DEBUG] [internal,edge,stack] [message: deploying stack %d]", stack.ID) stack.Status = statusDone stack.Action = actionIdle responseStatus := int(edgeStackStatusOk) @@ -214,45 +214,45 @@ func (manager *Manager) deployStack(stack *edgeStack, stackName, stackFileLocati err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed deploying stack] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: failed deploying stack] [error: %v]", err) stack.Status = statusError responseStatus = int(edgeStackStatusError) errorMessage = err.Error() } else { - log.Printf("[DEBUG] [http,edge,stacksmanager] [message: deployed stack id: %v, version: %d]", stack.ID, stack.Version) + log.Printf("[DEBUG] [internal,edge,stack] [message: deployed stack id: %v, version: %d]", stack.ID, stack.Version) } manager.stacks[stack.ID] = stack cli, err := manager.createPortainerClient() if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed creating portainer client] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: failed creating portainer client] [error: %v]", err) } err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) if err != nil { - log.Printf("[ERROR] [http,edge,stacksmanager] [message: failed setting edge stack status] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: failed setting edge stack status] [error: %v]", err) } } -func (manager *Manager) createPortainerClient() (*client.PortainerClient, error) { +func (manager *StacksManager) createPortainerClient() (*client.PortainerClient, error) { if manager.portainerURL == "" || manager.endpointID == "" || manager.edgeID == "" { return nil, errors.New("Client parameters are invalid") } return client.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil } -func (manager *Manager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [stacksmanager,update] [message: removing stack %d]", stack.ID) +func (manager *StacksManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { + log.Printf("[DEBUG] [internal,edge,stack] [message: removing stack %d]", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { - log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed deleting edge stack file] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: failed deleting edge stack file] [error: %v]", err) return } err = manager.dockerStackService.Remove(stackName) if err != nil { - log.Printf("[ERROR] [edge,stacksmanager, delete] [message: failed removing stack] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: failed removing stack] [error: %v]", err) return } From aa47a2dae428391468a876ffeab84720680cdde1 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:05:44 +0300 Subject: [PATCH 60/91] refactor(poll): move poll mechanism to poll service --- internal/edge/edge.go | 18 +- internal/edge/operator.go | 180 ------------------ internal/edge/operator_poll.go | 156 ---------------- internal/edge/poll.go | 325 +++++++++++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 345 deletions(-) delete mode 100644 internal/edge/operator.go delete mode 100644 internal/edge/operator_poll.go create mode 100644 internal/edge/poll.go diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 1c44d6bd..c8c5b9ad 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -15,7 +15,7 @@ type EdgeManager struct { dockerStackService agent.DockerStackService infoService agent.InfoService stacksManager *StacksManager - tunnelOperator agent.TunnelOperator + pollService agent.TunnelOperator key *edgeKey edgeMode bool } @@ -26,12 +26,12 @@ func NewEdgeManager() (*EdgeManager, error) { return &EdgeManager{}, nil } -// Enable enables the manager +// Init initializes the manager func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) error { apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) - operatorConfig := &OperatorConfig{ + pollServiceConfig := &pollServiceConfig{ APIServerAddr: apiServerAddr, EdgeID: options.EdgeID, PollFrequency: agent.DefaultEdgePollInterval, @@ -39,7 +39,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c InsecurePoll: options.EdgeInsecurePoll, } - log.Printf("[DEBUG] [internal,edge] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", operatorConfig.APIServerAddr, operatorConfig.EdgeID, operatorConfig.PollFrequency, operatorConfig.InactivityTimeout, operatorConfig.InsecurePoll) + log.Printf("[DEBUG] [internal,edge] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", pollServiceConfig.APIServerAddr, pollServiceConfig.EdgeID, pollServiceConfig.PollFrequency, pollServiceConfig.InactivityTimeout, pollServiceConfig.InsecurePoll) dockerStackService, err := exec.NewDockerStackService(agent.DockerBinaryPath) if err != nil { @@ -53,11 +53,11 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } manager.stacksManager = stacksManager - tunnelOperator, err := NewTunnelOperator(stacksManager, operatorConfig) + pollService, err := newPollService(stacksManager, pollServiceConfig) if err != nil { return err } - manager.tunnelOperator = tunnelOperator + manager.pollService = pollService manager.infoService = infoService manager.clusterService = clusterService @@ -87,7 +87,7 @@ func (manager *EdgeManager) IsEdgeModeEnabled() bool { // ResetActivityTimer resets the activity timer func (manager *EdgeManager) ResetActivityTimer() { - manager.tunnelOperator.ResetActivityTimer() + manager.pollService.ResetActivityTimer() } func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { @@ -131,13 +131,13 @@ func (manager *EdgeManager) checkRuntimeConfig() error { log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) if !agentRunsOnSwarm || agentRunsOnLeaderNode { - err = manager.tunnelOperator.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) + err = manager.pollService.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) if err != nil { return err } } else { - err = manager.tunnelOperator.Stop() + err = manager.pollService.Stop() if err != nil { return err } diff --git a/internal/edge/operator.go b/internal/edge/operator.go deleted file mode 100644 index 6ea9b978..00000000 --- a/internal/edge/operator.go +++ /dev/null @@ -1,180 +0,0 @@ -package edge - -import ( - "errors" - "log" - "net/http" - "time" - - "github.com/portainer/agent" - "github.com/portainer/agent/chisel" - "github.com/portainer/agent/filesystem" -) - -const tunnelActivityCheckInterval = 30 * time.Second - -// Operator is used to poll a Portainer instance and to establish a reverse tunnel if needed. -// It also takes care of closing the tunnel after a set period of inactivity. -type Operator struct { - apiServerAddr string - pollIntervalInSeconds float64 - insecurePoll bool - inactivityTimeout time.Duration - edgeID string - httpClient *http.Client - tunnelClient agent.ReverseTunnelClient - scheduleManager agent.Scheduler - lastActivity time.Time - refreshSignal chan struct{} - edgeStacksManager *StacksManager - portainerURL string - endpointID string - tunnelServerAddr string - tunnelServerFingerprint string -} - -// OperatorConfig represents the configuration used to create a new Operator. -type OperatorConfig struct { - APIServerAddr string - EdgeID string - InactivityTimeout string - PollFrequency string - InsecurePoll bool -} - -// NewTunnelOperator creates a new reverse tunnel operator -func NewTunnelOperator(edgeStacksManager *StacksManager, config *OperatorConfig) (*Operator, error) { - pollFrequency, err := time.ParseDuration(config.PollFrequency) - if err != nil { - return nil, err - } - - inactivityTimeout, err := time.ParseDuration(config.InactivityTimeout) - if err != nil { - return nil, err - } - - return &Operator{ - apiServerAddr: config.APIServerAddr, - edgeID: config.EdgeID, - pollIntervalInSeconds: pollFrequency.Seconds(), - insecurePoll: config.InsecurePoll, - inactivityTimeout: inactivityTimeout, - tunnelClient: chisel.NewClient(), - scheduleManager: filesystem.NewCronManager(), - refreshSignal: nil, - edgeStacksManager: edgeStacksManager, - }, nil -} - -// CloseTunnel closes the reverse tunnel managed by the operator -func (operator *Operator) CloseTunnel() error { - return operator.tunnelClient.CloseTunnel() -} - -// ResetActivityTimer will reset the last activity time timer -func (operator *Operator) ResetActivityTimer() { - if operator.tunnelClient.IsTunnelOpen() { - operator.lastActivity = time.Now() - } -} - -// Start will start two loops in go routines -// The first loop will poll the Portainer instance for the status of the associated endpoint and create a reverse tunnel -// if needed as well as manage schedules. -// The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel -// inactivity duration. -func (operator *Operator) Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error { - if portainerURL == "" || endpointID == "" || tunnelServerAddr == "" || tunnelServerFingerprint == "" { - return errors.New("Tunnel operator parameters are invalid") - } - - operator.portainerURL = portainerURL - operator.endpointID = endpointID - operator.tunnelServerAddr = tunnelServerAddr - operator.tunnelServerFingerprint = tunnelServerFingerprint - - if operator.refreshSignal != nil { - return nil - } - - operator.refreshSignal = make(chan struct{}) - operator.startStatusPollLoop() - operator.startActivityMonitoringLoop() - - return nil -} - -// Stop stops the poll loop -func (operator *Operator) Stop() error { - if operator.refreshSignal != nil { - close(operator.refreshSignal) - operator.refreshSignal = nil - } - return nil -} - -func (operator *Operator) restartStatusPollLoop() { - operator.Stop() - operator.startStatusPollLoop() -} - -func (operator *Operator) startStatusPollLoop() error { - log.Printf("[DEBUG] [internal,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", operator.pollIntervalInSeconds, operator.portainerURL) - - ticker := time.NewTicker(time.Duration(operator.pollIntervalInSeconds) * time.Second) - go func() { - for { - select { - case <-ticker.C: - err := operator.poll() - if err != nil { - log.Printf("[ERROR] [internal,edge,poll] [message: an error occured during short poll] [error: %s]", err) - } - - case <-operator.refreshSignal: - log.Println("[DEBUG] [internal,edge,poll] [message: shutting down Portainer short-polling client]") - ticker.Stop() - return - } - } - }() - - return nil -} - -func (operator *Operator) startActivityMonitoringLoop() { - ticker := time.NewTicker(tunnelActivityCheckInterval) - quit := make(chan struct{}) - - log.Printf("[DEBUG] [internal,edge,monitoring] [monitoring_interval_seconds: %f] [inactivity_timeout: %s] [message: starting activity monitoring loop]", tunnelActivityCheckInterval.Seconds(), operator.inactivityTimeout.String()) - - go func() { - for { - select { - case <-ticker.C: - - if operator.lastActivity.IsZero() { - continue - } - - elapsed := time.Since(operator.lastActivity) - log.Printf("[DEBUG] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: tunnel activity monitoring]", elapsed.Seconds()) - - if operator.tunnelClient.IsTunnelOpen() && elapsed.Seconds() > operator.inactivityTimeout.Seconds() { - - log.Printf("[INFO] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: shutting down tunnel after inactivity period]", elapsed.Seconds()) - - err := operator.tunnelClient.CloseTunnel() - if err != nil { - log.Printf("[ERROR] [internal,edge,monitoring] [message: unable to shutdown tunnel] [error: %s]", err) - } - } - - case <-quit: - ticker.Stop() - return - } - } - }() -} diff --git a/internal/edge/operator_poll.go b/internal/edge/operator_poll.go deleted file mode 100644 index dad03735..00000000 --- a/internal/edge/operator_poll.go +++ /dev/null @@ -1,156 +0,0 @@ -package edge - -import ( - "crypto/tls" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "log" - "net/http" - "strconv" - "time" - - "github.com/portainer/agent" - "github.com/portainer/libcrypto" -) - -const clientDefaultPollTimeout = 5 - -type stackStatus struct { - ID int - Version int -} - -type pollStatusResponse struct { - Status string `json:"status"` - Port int `json:"port"` - Schedules []agent.Schedule `json:"schedules"` - CheckinInterval float64 `json:"checkin"` - Credentials string `json:"credentials"` - Stacks []stackStatus `json:"stacks"` -} - -func (operator *Operator) createHTTPClient(timeout float64) { - httpCli := &http.Client{ - Timeout: time.Duration(timeout) * time.Second, - } - - if operator.insecurePoll { - httpCli.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - } - - operator.httpClient = httpCli -} - -func (operator *Operator) poll() error { - - pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", operator.portainerURL, operator.endpointID) - req, err := http.NewRequest("GET", pollURL, nil) - if err != nil { - return err - } - - req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, operator.edgeID) - - if operator.httpClient == nil { - operator.createHTTPClient(clientDefaultPollTimeout) - } - - resp, err := operator.httpClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - log.Printf("[DEBUG] [internal,edge,poll] [response_code: %d] [message: Poll request failure]", resp.StatusCode) - return errors.New("short poll request failed") - } - - var responseData pollStatusResponse - err = json.NewDecoder(resp.Body).Decode(&responseData) - if err != nil { - return err - } - - log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [port: %d] [schedule_count: %d] [checkin_interval_seconds: %f]", responseData.Status, responseData.Port, len(responseData.Schedules), responseData.CheckinInterval) - - if responseData.Status == "IDLE" && operator.tunnelClient.IsTunnelOpen() { - log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [message: Idle status detected, shutting down tunnel]", responseData.Status) - - err := operator.tunnelClient.CloseTunnel() - if err != nil { - log.Printf("[ERROR] [internal,edge,poll] [message: Unable to shutdown tunnel] [error: %s]", err) - } - } - - if responseData.Status == "REQUIRED" && !operator.tunnelClient.IsTunnelOpen() { - log.Println("[DEBUG] [internal,edge,poll] [message: Required status detected, creating reverse tunnel]") - - err := operator.createTunnel(responseData.Credentials, responseData.Port) - if err != nil { - log.Printf("[ERROR] [internal,edge,poll] [message: Unable to create tunnel] [error: %s]", err) - return err - } - } - - err = operator.scheduleManager.Schedule(responseData.Schedules) - if err != nil { - log.Printf("[ERROR] [internal,edge,cron] [message: an error occured during schedule management] [err: %s]", err) - } - - if responseData.CheckinInterval != operator.pollIntervalInSeconds { - log.Printf("[DEBUG] [internal,edge,poll] [old_interval: %f] [new_interval: %f] [message: updating poll interval]", operator.pollIntervalInSeconds, responseData.CheckinInterval) - operator.pollIntervalInSeconds = responseData.CheckinInterval - operator.createHTTPClient(responseData.CheckinInterval) - go operator.restartStatusPollLoop() - } - - if responseData.Stacks != nil { - stacks := map[int]int{} - for _, stack := range responseData.Stacks { - stacks[stack.ID] = stack.Version - } - - err := operator.edgeStacksManager.UpdateStacksStatus(stacks) - if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: an error occured during stack management] [error: %s]", err) - return err - } - } - - return nil -} - -func (operator *Operator) createTunnel(encodedCredentials string, remotePort int) error { - decodedCredentials, err := base64.RawStdEncoding.DecodeString(encodedCredentials) - if err != nil { - return err - } - - credentials, err := libcrypto.Decrypt(decodedCredentials, []byte(operator.edgeID)) - if err != nil { - return err - } - - tunnelConfig := agent.TunnelConfig{ - ServerAddr: operator.tunnelServerAddr, - ServerFingerpint: operator.tunnelServerFingerprint, - Credentials: string(credentials), - RemotePort: strconv.Itoa(remotePort), - LocalAddr: operator.apiServerAddr, - } - - err = operator.tunnelClient.CreateTunnel(tunnelConfig) - if err != nil { - return err - } - - operator.ResetActivityTimer() - return nil -} diff --git a/internal/edge/poll.go b/internal/edge/poll.go new file mode 100644 index 00000000..33465e6b --- /dev/null +++ b/internal/edge/poll.go @@ -0,0 +1,325 @@ +package edge + +import ( + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "strconv" + "time" + + "github.com/portainer/agent" + "github.com/portainer/agent/chisel" + "github.com/portainer/agent/filesystem" + "github.com/portainer/libcrypto" +) + +const tunnelActivityCheckInterval = 30 * time.Second + +// PollService is used to poll a Portainer instance and to establish a reverse tunnel if needed. +// It also takes care of closing the tunnel after a set period of inactivity. +type PollService struct { + apiServerAddr string + pollIntervalInSeconds float64 + insecurePoll bool + inactivityTimeout time.Duration + edgeID string + httpClient *http.Client + tunnelClient agent.ReverseTunnelClient + scheduleManager agent.Scheduler + lastActivity time.Time + refreshSignal chan struct{} + edgeStacksManager *StacksManager + portainerURL string + endpointID string + tunnelServerAddr string + tunnelServerFingerprint string +} + +type pollServiceConfig struct { + APIServerAddr string + EdgeID string + InactivityTimeout string + PollFrequency string + InsecurePoll bool +} + +// newPollService creates a new reverse tunnel service +func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) (*PollService, error) { + pollFrequency, err := time.ParseDuration(config.PollFrequency) + if err != nil { + return nil, err + } + + inactivityTimeout, err := time.ParseDuration(config.InactivityTimeout) + if err != nil { + return nil, err + } + + return &PollService{ + apiServerAddr: config.APIServerAddr, + edgeID: config.EdgeID, + pollIntervalInSeconds: pollFrequency.Seconds(), + insecurePoll: config.InsecurePoll, + inactivityTimeout: inactivityTimeout, + tunnelClient: chisel.NewClient(), + scheduleManager: filesystem.NewCronManager(), + refreshSignal: nil, + edgeStacksManager: edgeStacksManager, + }, nil +} + +// CloseTunnel closes the reverse tunnel managed by the service +func (service *PollService) CloseTunnel() error { + return service.tunnelClient.CloseTunnel() +} + +// ResetActivityTimer will reset the last activity time timer +func (service *PollService) ResetActivityTimer() { + if service.tunnelClient.IsTunnelOpen() { + service.lastActivity = time.Now() + } +} + +// Start will start two loops in go routines +// The first loop will poll the Portainer instance for the status of the associated endpoint and create a reverse tunnel +// if needed as well as manage schedules. +// The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel +// inactivity duration. +func (service *PollService) Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error { + if portainerURL == "" || endpointID == "" || tunnelServerAddr == "" || tunnelServerFingerprint == "" { + return errors.New("Tunnel service parameters are invalid") + } + + service.portainerURL = portainerURL + service.endpointID = endpointID + service.tunnelServerAddr = tunnelServerAddr + service.tunnelServerFingerprint = tunnelServerFingerprint + + if service.refreshSignal != nil { + return nil + } + + service.refreshSignal = make(chan struct{}) + service.startStatusPollLoop() + service.startActivityMonitoringLoop() + + return nil +} + +// Stop stops the poll loop +func (service *PollService) Stop() error { + if service.refreshSignal != nil { + close(service.refreshSignal) + service.refreshSignal = nil + } + return nil +} + +func (service *PollService) restartStatusPollLoop() { + service.Stop() + service.startStatusPollLoop() +} + +func (service *PollService) startStatusPollLoop() error { + log.Printf("[DEBUG] [internal,edge,poll] [poll_interval_seconds: %f] [server_url: %s] [message: starting Portainer short-polling client]", service.pollIntervalInSeconds, service.portainerURL) + + ticker := time.NewTicker(time.Duration(service.pollIntervalInSeconds) * time.Second) + go func() { + for { + select { + case <-ticker.C: + err := service.poll() + if err != nil { + log.Printf("[ERROR] [internal,edge,poll] [message: an error occured during short poll] [error: %s]", err) + } + + case <-service.refreshSignal: + log.Println("[DEBUG] [internal,edge,poll] [message: shutting down Portainer short-polling client]") + ticker.Stop() + return + } + } + }() + + return nil +} + +func (service *PollService) startActivityMonitoringLoop() { + ticker := time.NewTicker(tunnelActivityCheckInterval) + quit := make(chan struct{}) + + log.Printf("[DEBUG] [internal,edge,monitoring] [monitoring_interval_seconds: %f] [inactivity_timeout: %s] [message: starting activity monitoring loop]", tunnelActivityCheckInterval.Seconds(), service.inactivityTimeout.String()) + + go func() { + for { + select { + case <-ticker.C: + + if service.lastActivity.IsZero() { + continue + } + + elapsed := time.Since(service.lastActivity) + log.Printf("[DEBUG] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: tunnel activity monitoring]", elapsed.Seconds()) + + if service.tunnelClient.IsTunnelOpen() && elapsed.Seconds() > service.inactivityTimeout.Seconds() { + + log.Printf("[INFO] [internal,edge,monitoring] [tunnel_last_activity_seconds: %f] [message: shutting down tunnel after inactivity period]", elapsed.Seconds()) + + err := service.tunnelClient.CloseTunnel() + if err != nil { + log.Printf("[ERROR] [internal,edge,monitoring] [message: unable to shutdown tunnel] [error: %s]", err) + } + } + + case <-quit: + ticker.Stop() + return + } + } + }() +} + +const clientDefaultPollTimeout = 5 + +type stackStatus struct { + ID int + Version int +} + +type pollStatusResponse struct { + Status string `json:"status"` + Port int `json:"port"` + Schedules []agent.Schedule `json:"schedules"` + CheckinInterval float64 `json:"checkin"` + Credentials string `json:"credentials"` + Stacks []stackStatus `json:"stacks"` +} + +func (service *PollService) createHTTPClient(timeout float64) { + httpCli := &http.Client{ + Timeout: time.Duration(timeout) * time.Second, + } + + if service.insecurePoll { + httpCli.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + } + + service.httpClient = httpCli +} + +func (service *PollService) poll() error { + + pollURL := fmt.Sprintf("%s/api/endpoints/%s/status", service.portainerURL, service.endpointID) + req, err := http.NewRequest("GET", pollURL, nil) + if err != nil { + return err + } + + req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, service.edgeID) + + if service.httpClient == nil { + service.createHTTPClient(clientDefaultPollTimeout) + } + + resp, err := service.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("[DEBUG] [internal,edge,poll] [response_code: %d] [message: Poll request failure]", resp.StatusCode) + return errors.New("short poll request failed") + } + + var responseData pollStatusResponse + err = json.NewDecoder(resp.Body).Decode(&responseData) + if err != nil { + return err + } + + log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [port: %d] [schedule_count: %d] [checkin_interval_seconds: %f]", responseData.Status, responseData.Port, len(responseData.Schedules), responseData.CheckinInterval) + + if responseData.Status == "IDLE" && service.tunnelClient.IsTunnelOpen() { + log.Printf("[DEBUG] [internal,edge,poll] [status: %s] [message: Idle status detected, shutting down tunnel]", responseData.Status) + + err := service.tunnelClient.CloseTunnel() + if err != nil { + log.Printf("[ERROR] [internal,edge,poll] [message: Unable to shutdown tunnel] [error: %s]", err) + } + } + + if responseData.Status == "REQUIRED" && !service.tunnelClient.IsTunnelOpen() { + log.Println("[DEBUG] [internal,edge,poll] [message: Required status detected, creating reverse tunnel]") + + err := service.createTunnel(responseData.Credentials, responseData.Port) + if err != nil { + log.Printf("[ERROR] [internal,edge,poll] [message: Unable to create tunnel] [error: %s]", err) + return err + } + } + + err = service.scheduleManager.Schedule(responseData.Schedules) + if err != nil { + log.Printf("[ERROR] [internal,edge,cron] [message: an error occured during schedule management] [err: %s]", err) + } + + if responseData.CheckinInterval != service.pollIntervalInSeconds { + log.Printf("[DEBUG] [internal,edge,poll] [old_interval: %f] [new_interval: %f] [message: updating poll interval]", service.pollIntervalInSeconds, responseData.CheckinInterval) + service.pollIntervalInSeconds = responseData.CheckinInterval + service.createHTTPClient(responseData.CheckinInterval) + go service.restartStatusPollLoop() + } + + if responseData.Stacks != nil { + stacks := map[int]int{} + for _, stack := range responseData.Stacks { + stacks[stack.ID] = stack.Version + } + + err := service.edgeStacksManager.UpdateStacksStatus(stacks) + if err != nil { + log.Printf("[ERROR] [internal,edge,stack] [message: an error occured during stack management] [error: %s]", err) + return err + } + } + + return nil +} + +func (service *PollService) createTunnel(encodedCredentials string, remotePort int) error { + decodedCredentials, err := base64.RawStdEncoding.DecodeString(encodedCredentials) + if err != nil { + return err + } + + credentials, err := libcrypto.Decrypt(decodedCredentials, []byte(service.edgeID)) + if err != nil { + return err + } + + tunnelConfig := agent.TunnelConfig{ + ServerAddr: service.tunnelServerAddr, + ServerFingerpint: service.tunnelServerFingerprint, + Credentials: string(credentials), + RemotePort: strconv.Itoa(remotePort), + LocalAddr: service.apiServerAddr, + } + + err = service.tunnelClient.CreateTunnel(tunnelConfig) + if err != nil { + return err + } + + service.ResetActivityTimer() + return nil +} From a52633fad7c7ebc906feca4c911a488979175965 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:12:06 +0300 Subject: [PATCH 61/91] refactor(edge): rename function --- internal/edge/edge.go | 5 +++-- internal/edge/key.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index c8c5b9ad..d0f5992d 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -61,7 +61,6 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c manager.infoService = infoService manager.clusterService = clusterService - manager.edgeMode = true edgeKey, err := manager.retrieveEdgeKey(options.EdgeKey) if err != nil { @@ -77,6 +76,8 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } } + manager.edgeMode = true + return nil } @@ -90,7 +91,7 @@ func (manager *EdgeManager) ResetActivityTimer() { manager.pollService.ResetActivityTimer() } -func (manager *EdgeManager) startRuntimeConfigCheckProcess() error { +func (manager *EdgeManager) startEdgeBackgroundProcess() error { runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { diff --git a/internal/edge/key.go b/internal/edge/key.go index 27d507e3..543867db 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -42,7 +42,7 @@ func (manager *EdgeManager) SetKey(key string) error { } } - return manager.startRuntimeConfigCheckProcess() + return manager.startEdgeBackgroundProcess() } // GetKey returns the key associated to the manager From fe6923c6d135e3765a8cf6b26aba8b9756aa03a1 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:44:28 +0300 Subject: [PATCH 62/91] refactor(edge): run init edge only when key is set --- cmd/agent/main.go | 91 ++++++++++++++++++++++++++++++++-- http/edge.go | 5 ++ http/handler/key/key_create.go | 5 ++ internal/edge/edge.go | 44 +++++++--------- internal/edge/key.go | 72 +-------------------------- 5 files changed, 117 insertions(+), 100 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index b1b92261..05d9703d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,8 +8,10 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/crypto" "github.com/portainer/agent/docker" + "github.com/portainer/agent/filesystem" "github.com/portainer/agent/ghw" "github.com/portainer/agent/http" + "github.com/portainer/agent/http/client" "github.com/portainer/agent/internal/edge" "github.com/portainer/agent/logutils" "github.com/portainer/agent/net" @@ -84,18 +86,31 @@ func main() { defer clusterService.Leave() } - edgeManager, err := edge.NewEdgeManager() + edgeManager, err := edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) if err != nil { log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) } if options.EdgeMode { - err = edgeManager.Init(options, advertiseAddr, clusterService, infoService) + edgeKey, err := retrieveEdgeKey(options.EdgeKey, clusterService) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to init edge manager] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to retrieve edge key] [error: %s]", err) } - if !edgeManager.IsKeySet() { + if edgeKey != "" { + log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") + + err := edgeManager.SetKey(edgeKey) + if err != nil { + log.Fatalf("[ERROR] [main,edge] [message: Unable to set edge key] [error: %s]", err) + } + + err = edgeManager.Init() + if err != nil { + log.Fatalf("[ERROR] [main,edge] [message: Unable to init edge manager] [error: %s]", err) + } + + } else { log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") serveEdgeUI(edgeManager, clusterService, options.EdgeServerAddr, options.EdgeServerPort) @@ -175,3 +190,71 @@ func serveEdgeUI(edgeManager *edge.EdgeManager, clusterService agent.ClusterServ } }() } + +func retrieveEdgeKey(edgeKey string, clusterService agent.ClusterService) (string, error) { + + if edgeKey != "" { + log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") + return edgeKey, nil + } + + var keyRetrievalError error + + edgeKey, keyRetrievalError = retrieveEdgeKeyFromFilesystem() + if keyRetrievalError != nil { + return "", keyRetrievalError + } + + if edgeKey == "" && clusterService != nil { + edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(clusterService) + if keyRetrievalError != nil { + return "", keyRetrievalError + } + } + + return edgeKey, nil +} + +func retrieveEdgeKeyFromFilesystem() (string, error) { + var edgeKey string + + edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) + + keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) + if err != nil { + return "", err + } + + if keyFileExists { + filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) + if err != nil { + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") + edgeKey = string(filesystemKey) + } + + return edgeKey, nil +} + +func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { + var edgeKey string + + member := clusterService.GetMemberWithEdgeKeySet() + if member != nil { + httpCli := client.NewAPIClient() + + memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) + memberKey, err := httpCli.GetEdgeKey(memberAddr) + if err != nil { + log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) + return "", err + } + + log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") + edgeKey = memberKey + } + + return edgeKey, nil +} diff --git a/http/edge.go b/http/edge.go index ed94df64..07f1956c 100644 --- a/http/edge.go +++ b/http/edge.go @@ -66,6 +66,11 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { return } + err = server.edgeManager.Init() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + if server.clusterService != nil { tags := server.clusterService.GetTags() go server.propagateKeyInCluster(tags[agent.MemberTagKeyNodeName], key) diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index cc45f47f..294c0129 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -44,5 +44,10 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to associate Edge key", err} } + err = handler.edgeManager.Init() + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to initialize Edge Manager", err} + } + return response.Empty(w) } diff --git a/internal/edge/edge.go b/internal/edge/edge.go index d0f5992d..7e66b21c 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -16,27 +16,35 @@ type EdgeManager struct { infoService agent.InfoService stacksManager *StacksManager pollService agent.TunnelOperator + pollServiceConfig *pollServiceConfig key *edgeKey edgeMode bool + agentOptions *agent.Options + advertiseAddr string } // NewEdgeManager creates an instance of EdgeManager -func NewEdgeManager() (*EdgeManager, error) { - - return &EdgeManager{}, nil +func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { + + return &EdgeManager{ + clusterService: clusterService, + infoService: infoService, + agentOptions: options, + advertiseAddr: advertiseAddr, + edgeMode: options.EdgeMode, + }, nil } // Init initializes the manager -func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) error { - - apiServerAddr := fmt.Sprintf("%s:%s", advertiseAddr, options.AgentServerPort) +func (manager *EdgeManager) Init() error { + apiServerAddr := fmt.Sprintf("%s:%s", manager.advertiseAddr, manager.agentOptions.AgentServerPort) pollServiceConfig := &pollServiceConfig{ APIServerAddr: apiServerAddr, - EdgeID: options.EdgeID, + EdgeID: manager.agentOptions.EdgeID, PollFrequency: agent.DefaultEdgePollInterval, - InactivityTimeout: options.EdgeInactivityTimeout, - InsecurePoll: options.EdgeInsecurePoll, + InactivityTimeout: manager.agentOptions.EdgeInactivityTimeout, + InsecurePoll: manager.agentOptions.EdgeInsecurePoll, } log.Printf("[DEBUG] [internal,edge] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", pollServiceConfig.APIServerAddr, pollServiceConfig.EdgeID, pollServiceConfig.PollFrequency, pollServiceConfig.InactivityTimeout, pollServiceConfig.InsecurePoll) @@ -47,7 +55,7 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } manager.dockerStackService = dockerStackService - stacksManager, err := NewStacksManager(dockerStackService, options.EdgeID) + stacksManager, err := NewStacksManager(dockerStackService, manager.agentOptions.EdgeID) if err != nil { return err } @@ -59,25 +67,11 @@ func (manager *EdgeManager) Init(options *agent.Options, advertiseAddr string, c } manager.pollService = pollService - manager.infoService = infoService - manager.clusterService = clusterService - - edgeKey, err := manager.retrieveEdgeKey(options.EdgeKey) + err = manager.startEdgeBackgroundProcess() if err != nil { return err } - if edgeKey != "" { - log.Println("[DEBUG] [internal,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") - - err := manager.SetKey(edgeKey) - if err != nil { - return err - } - } - - manager.edgeMode = true - return nil } diff --git a/internal/edge/key.go b/internal/edge/key.go index 543867db..232ca613 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -4,12 +4,10 @@ import ( "encoding/base64" "errors" "fmt" - "log" "strings" "github.com/portainer/agent" "github.com/portainer/agent/filesystem" - "github.com/portainer/agent/http/client" ) type edgeKey struct { @@ -42,7 +40,7 @@ func (manager *EdgeManager) SetKey(key string) error { } } - return manager.startEdgeBackgroundProcess() + return nil } // GetKey returns the key associated to the manager @@ -94,71 +92,3 @@ func encodeKey(edgeKey *edgeKey) string { encodedKey := base64.RawStdEncoding.EncodeToString([]byte(keyInfo)) return encodedKey } - -func (manager *EdgeManager) retrieveEdgeKey(edgeKey string) (string, error) { - - if edgeKey != "" { - log.Println("[INFO] [main,edge] [message: Edge key loaded from options]") - return edgeKey, nil - } - - var keyRetrievalError error - - edgeKey, keyRetrievalError = retrieveEdgeKeyFromFilesystem() - if keyRetrievalError != nil { - return "", keyRetrievalError - } - - if edgeKey == "" && manager.clusterService != nil { - edgeKey, keyRetrievalError = retrieveEdgeKeyFromCluster(manager.clusterService) - if keyRetrievalError != nil { - return "", keyRetrievalError - } - } - - return edgeKey, nil -} - -func retrieveEdgeKeyFromFilesystem() (string, error) { - var edgeKey string - - edgeKeyFilePath := fmt.Sprintf("%s/%s", agent.DataDirectory, agent.EdgeKeyFile) - - keyFileExists, err := filesystem.FileExists(edgeKeyFilePath) - if err != nil { - return "", err - } - - if keyFileExists { - filesystemKey, err := filesystem.ReadFromFile(edgeKeyFilePath) - if err != nil { - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from the filesystem]") - edgeKey = string(filesystemKey) - } - - return edgeKey, nil -} - -func retrieveEdgeKeyFromCluster(clusterService agent.ClusterService) (string, error) { - var edgeKey string - - member := clusterService.GetMemberWithEdgeKeySet() - if member != nil { - httpCli := client.NewAPIClient() - - memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) - memberKey, err := httpCli.GetEdgeKey(memberAddr) - if err != nil { - log.Printf("[ERROR] [main,edge,http,cluster] [message: Unable to retrieve Edge key from cluster member] [error: %s]", err) - return "", err - } - - log.Println("[INFO] [main,edge] [message: Edge key loaded from cluster]") - edgeKey = memberKey - } - - return edgeKey, nil -} From f371d99cb32a64bfb4bcbcceb992c6bb664aec96 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:50:38 +0300 Subject: [PATCH 63/91] refactor(edge-stacks): replace log messages --- internal/edge/stack.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/edge/stack.go b/internal/edge/stack.go index b93da973..d7bf3fa4 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -86,13 +86,13 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { if stack.Version == version { continue } - log.Printf("[DEBUG] [internal,edge,stack] [message: received stack to update %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: marking stack for update]", stackID) stack.Action = actionUpdate stack.Version = version stack.Status = statusPending } else { - log.Printf("[DEBUG] [internal,edge,stack] [message: received new stack %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: marking stack for deployment]", stackID) stack = &edgeStack{ Action: actionDeploy, @@ -134,7 +134,7 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { for stackID, stack := range manager.stacks { if _, ok := stacks[int(stackID)]; !ok { - log.Printf("[DEBUG] [internal,edge,stack] [message: received stack to delete %d]", stackID) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: marking stack for deletion]", stackID) stack.Action = actionDelete stack.Status = statusPending @@ -206,7 +206,7 @@ func (manager *StacksManager) next() *edgeStack { } func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [internal,edge,stack] [message: deploying stack %d]", stack.ID) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: stack deployment]", stack.ID) stack.Status = statusDone stack.Action = actionIdle responseStatus := int(edgeStackStatusOk) @@ -219,7 +219,7 @@ func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFile responseStatus = int(edgeStackStatusError) errorMessage = err.Error() } else { - log.Printf("[DEBUG] [internal,edge,stack] [message: deployed stack id: %v, version: %d]", stack.ID, stack.Version) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: stack deployed]", stack.ID, stack.Version) } manager.stacks[stack.ID] = stack From da9ee3a2fcfa89916c3f315f8524cbb9aae36c51 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:52:35 +0300 Subject: [PATCH 64/91] refactor(http): check if edge mode --- http/handler/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/handler/handler.go b/http/handler/handler.go index 964fa722..47b834cb 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -81,7 +81,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, request *http.Request) { return } - if h.edgeManager != nil { + if h.edgeManager.IsEdgeModeEnabled() { h.edgeManager.ResetActivityTimer() } From f926d0b649ad216eecfba91e782749c6a748177f Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 11:55:01 +0300 Subject: [PATCH 65/91] refactor(exec): rename service --- exec/stack.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/exec/stack.go b/exec/stack.go index b3760bfc..f4601ca6 100644 --- a/exec/stack.go +++ b/exec/stack.go @@ -8,7 +8,7 @@ import ( "runtime" ) -// DockerStackService represents a service for managing stacks. +// DockerStackService represents a service for managing stacks by using the Docker binary. type DockerStackService struct { binaryPath string } @@ -16,30 +16,30 @@ type DockerStackService struct { // NewDockerStackService initializes a new DockerStackService service. // It also updates the configuration of the Docker CLI binary. func NewDockerStackService(binaryPath string) (*DockerStackService, error) { - manager := &DockerStackService{ + service := &DockerStackService{ binaryPath: binaryPath, } - return manager, nil + return service, nil } // Login executes the docker login command against a list of registries (including DockerHub). -func (manager *DockerStackService) Login() error { +func (service *DockerStackService) Login() error { // Not implemented yet. return nil } // Logout executes the docker logout command. -func (manager *DockerStackService) Logout() error { - command := manager.prepareDockerCommand(manager.binaryPath) +func (service *DockerStackService) Logout() error { + command := service.prepareDockerCommand(service.binaryPath) args := []string{"logout"} return runCommandAndCaptureStdErr(command, args, "") } // Deploy executes the docker stack deploy command. -func (manager *DockerStackService) Deploy(name, stackFilePath string, prune bool) error { - command := manager.prepareDockerCommand(manager.binaryPath) +func (service *DockerStackService) Deploy(name, stackFilePath string, prune bool) error { + command := service.prepareDockerCommand(service.binaryPath) args := []string{} if prune { @@ -53,8 +53,8 @@ func (manager *DockerStackService) Deploy(name, stackFilePath string, prune bool } // Remove executes the docker stack rm command. -func (manager *DockerStackService) Remove(name string) error { - command := manager.prepareDockerCommand(manager.binaryPath) +func (service *DockerStackService) Remove(name string) error { + command := service.prepareDockerCommand(service.binaryPath) args := []string{"stack", "rm", name} return runCommandAndCaptureStdErr(command, args, "") } @@ -73,7 +73,7 @@ func runCommandAndCaptureStdErr(command string, args []string, workingDir string return nil } -func (manager *DockerStackService) prepareDockerCommand(binaryPath string) string { +func (service *DockerStackService) prepareDockerCommand(binaryPath string) string { // Assume Linux as a default command := path.Join(binaryPath, "docker") From 78fa6aaba6bef3aee18dbcfac028114d94475f44 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:18:03 +0300 Subject: [PATCH 66/91] refactor(edge-stacks): create portainer client on init --- internal/edge/edge.go | 4 ++-- internal/edge/stack.go | 36 +++++++++--------------------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 7e66b21c..81173b90 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -55,7 +55,7 @@ func (manager *EdgeManager) Init() error { } manager.dockerStackService = dockerStackService - stacksManager, err := NewStacksManager(dockerStackService, manager.agentOptions.EdgeID) + stacksManager, err := NewStacksManager(dockerStackService, manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.agentOptions.EdgeID) if err != nil { return err } @@ -139,7 +139,7 @@ func (manager *EdgeManager) checkRuntimeConfig() error { } if agentRunsOnSwarm && agentRunsOnLeaderNode { - err = manager.stacksManager.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID) + err = manager.stacksManager.Start() if err != nil { return err } diff --git a/internal/edge/stack.go b/internal/edge/stack.go index d7bf3fa4..e4db106d 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -1,7 +1,6 @@ package edge import ( - "errors" "fmt" "log" "time" @@ -58,19 +57,21 @@ type StacksManager struct { stacks map[edgeStackID]*edgeStack stopSignal chan struct{} dockerStackService agent.DockerStackService - edgeID string portainerURL string endpointID string isEnabled bool + httpClient *client.PortainerClient } // NewStacksManager creates a new instance of StacksManager -func NewStacksManager(dockerStackService agent.DockerStackService, edgeID string) (*StacksManager, error) { +func NewStacksManager(dockerStackService agent.DockerStackService, portainerURL, endpointID, edgeID string) (*StacksManager, error) { + cli := client.NewPortainerClient(portainerURL, endpointID, edgeID) + return &StacksManager{ dockerStackService: dockerStackService, - edgeID: edgeID, stacks: map[edgeStackID]*edgeStack{}, stopSignal: nil, + httpClient: cli, }, nil } @@ -102,12 +103,7 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { } } - cli, err := manager.createPortainerClient() - if err != nil { - return err - } - - fileContent, prune, err := cli.GetEdgeStackConfig(int(stack.ID)) + fileContent, prune, err := manager.httpClient.GetEdgeStackConfig(int(stack.ID)) if err != nil { return err } @@ -126,7 +122,7 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { manager.stacks[stack.ID] = stack - err = cli.SetEdgeStackStatus(int(stack.ID), int(edgeStackStatusAcknowledged), "") + err = manager.httpClient.SetEdgeStackStatus(int(stack.ID), int(edgeStackStatusAcknowledged), "") if err != nil { return err } @@ -157,13 +153,11 @@ func (manager *StacksManager) Stop() error { } // Start starts the loop checking for stacks to deploy -func (manager *StacksManager) Start(portainerURL, endpointID string) error { +func (manager *StacksManager) Start() error { if manager.stopSignal != nil { return nil } - manager.portainerURL = portainerURL - manager.endpointID = endpointID manager.isEnabled = true manager.stopSignal = make(chan struct{}) @@ -224,24 +218,12 @@ func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFile manager.stacks[stack.ID] = stack - cli, err := manager.createPortainerClient() - if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: failed creating portainer client] [error: %v]", err) - } - - err = cli.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) + err = manager.httpClient.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) if err != nil { log.Printf("[ERROR] [internal,edge,stack] [message: failed setting edge stack status] [error: %v]", err) } } -func (manager *StacksManager) createPortainerClient() (*client.PortainerClient, error) { - if manager.portainerURL == "" || manager.endpointID == "" || manager.edgeID == "" { - return nil, errors.New("Client parameters are invalid") - } - return client.NewPortainerClient(manager.portainerURL, manager.endpointID, manager.edgeID), nil -} - func (manager *StacksManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { log.Printf("[DEBUG] [internal,edge,stack] [message: removing stack %d]", stack.ID) err := filesystem.RemoveFile(stackFileLocation) From 79bfe8fa7b0d340138fe0414cb1fe3a8bd1fc94c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:25:55 +0300 Subject: [PATCH 67/91] feat(edge): start tunnel with key properties --- agent.go | 2 +- internal/edge/edge.go | 16 +++++++++------ internal/edge/poll.go | 47 +++++++++++++++++++++---------------------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/agent.go b/agent.go index 1b4dcce0..81657b53 100644 --- a/agent.go +++ b/agent.go @@ -122,7 +122,7 @@ type ( // TunnelOperator is a service that is used to communicate with a Portainer instance and to manage // the reverse tunnel. TunnelOperator interface { - Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error + Start() error Stop() error CloseTunnel() error ResetActivityTimer() diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 81173b90..fa94449c 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -40,11 +40,15 @@ func (manager *EdgeManager) Init() error { apiServerAddr := fmt.Sprintf("%s:%s", manager.advertiseAddr, manager.agentOptions.AgentServerPort) pollServiceConfig := &pollServiceConfig{ - APIServerAddr: apiServerAddr, - EdgeID: manager.agentOptions.EdgeID, - PollFrequency: agent.DefaultEdgePollInterval, - InactivityTimeout: manager.agentOptions.EdgeInactivityTimeout, - InsecurePoll: manager.agentOptions.EdgeInsecurePoll, + APIServerAddr: apiServerAddr, + EdgeID: manager.agentOptions.EdgeID, + PollFrequency: agent.DefaultEdgePollInterval, + InactivityTimeout: manager.agentOptions.EdgeInactivityTimeout, + InsecurePoll: manager.agentOptions.EdgeInsecurePoll, + PortainerURL: manager.key.PortainerInstanceURL, + EndpointID: manager.key.EndpointID, + TunnelServerAddr: manager.key.TunnelServerAddr, + TunnelServerFingerprint: manager.key.TunnelServerFingerprint, } log.Printf("[DEBUG] [internal,edge] [api_addr: %s] [edge_id: %s] [poll_frequency: %s] [inactivity_timeout: %s] [insecure_poll: %t]", pollServiceConfig.APIServerAddr, pollServiceConfig.EdgeID, pollServiceConfig.PollFrequency, pollServiceConfig.InactivityTimeout, pollServiceConfig.InsecurePoll) @@ -126,7 +130,7 @@ func (manager *EdgeManager) checkRuntimeConfig() error { log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) if !agentRunsOnSwarm || agentRunsOnLeaderNode { - err = manager.pollService.Start(manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.key.TunnelServerAddr, manager.key.TunnelServerFingerprint) + err = manager.pollService.Start() if err != nil { return err } diff --git a/internal/edge/poll.go b/internal/edge/poll.go index 33465e6b..bc84638f 100644 --- a/internal/edge/poll.go +++ b/internal/edge/poll.go @@ -40,11 +40,15 @@ type PollService struct { } type pollServiceConfig struct { - APIServerAddr string - EdgeID string - InactivityTimeout string - PollFrequency string - InsecurePoll bool + APIServerAddr string + EdgeID string + InactivityTimeout string + PollFrequency string + InsecurePoll bool + PortainerURL string + EndpointID string + TunnelServerAddr string + TunnelServerFingerprint string } // newPollService creates a new reverse tunnel service @@ -60,15 +64,19 @@ func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) } return &PollService{ - apiServerAddr: config.APIServerAddr, - edgeID: config.EdgeID, - pollIntervalInSeconds: pollFrequency.Seconds(), - insecurePoll: config.InsecurePoll, - inactivityTimeout: inactivityTimeout, - tunnelClient: chisel.NewClient(), - scheduleManager: filesystem.NewCronManager(), - refreshSignal: nil, - edgeStacksManager: edgeStacksManager, + apiServerAddr: config.APIServerAddr, + edgeID: config.EdgeID, + pollIntervalInSeconds: pollFrequency.Seconds(), + insecurePoll: config.InsecurePoll, + inactivityTimeout: inactivityTimeout, + tunnelClient: chisel.NewClient(), + scheduleManager: filesystem.NewCronManager(), + refreshSignal: nil, + edgeStacksManager: edgeStacksManager, + portainerURL: config.PortainerURL, + endpointID: config.EndpointID, + tunnelServerAddr: config.TunnelServerAddr, + tunnelServerFingerprint: config.TunnelServerFingerprint, }, nil } @@ -89,16 +97,7 @@ func (service *PollService) ResetActivityTimer() { // if needed as well as manage schedules. // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. -func (service *PollService) Start(portainerURL, endpointID, tunnelServerAddr, tunnelServerFingerprint string) error { - if portainerURL == "" || endpointID == "" || tunnelServerAddr == "" || tunnelServerFingerprint == "" { - return errors.New("Tunnel service parameters are invalid") - } - - service.portainerURL = portainerURL - service.endpointID = endpointID - service.tunnelServerAddr = tunnelServerAddr - service.tunnelServerFingerprint = tunnelServerFingerprint - +func (service *PollService) Start() error { if service.refreshSignal != nil { return nil } From 552119f73200cb93dbe436a54fa83b4fb701087d Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:26:21 +0300 Subject: [PATCH 68/91] feat(edge): prevent startup of edge manager without key --- dev.sh | 2 +- internal/edge/edge.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dev.sh b/dev.sh index 4821b6ab..f6f7fc61 100755 --- a/dev.sh +++ b/dev.sh @@ -92,7 +92,6 @@ function deploy_swarm() { sleep 5 echo "Deployment..." - # -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ docker -H "${DOCKER_MANAGER}:2375" network create --driver overlay portainer-agent-dev-net docker -H "${DOCKER_MANAGER}:2375" service create --name portainer-agent-dev \ @@ -101,6 +100,7 @@ function deploy_swarm() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ + -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ --mode global \ --mount type=bind,src=//var/run/docker.sock,dst=/var/run/docker.sock \ --mount type=bind,src=//var/lib/docker/volumes,dst=/var/lib/docker/volumes \ diff --git a/internal/edge/edge.go b/internal/edge/edge.go index fa94449c..08b2d63f 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -1,6 +1,7 @@ package edge import ( + "errors" "fmt" "log" "time" @@ -37,6 +38,10 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService // Init initializes the manager func (manager *EdgeManager) Init() error { + if !manager.IsKeySet() { + return errors.New("Unable to initalize edge manager without key") + } + apiServerAddr := fmt.Sprintf("%s:%s", manager.advertiseAddr, manager.agentOptions.AgentServerPort) pollServiceConfig := &pollServiceConfig{ From f2d90abe19f2de6684943bf791610b751eed27f8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:41:02 +0300 Subject: [PATCH 69/91] refactor(edge): rename edgeManager to edge.Manager --- cmd/agent/main.go | 4 ++-- http/edge.go | 4 ++-- http/handler/handler.go | 4 ++-- http/handler/key/handler.go | 4 ++-- http/server.go | 4 ++-- internal/edge/edge.go | 20 ++++++++++---------- internal/edge/key.go | 6 +++--- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 05d9703d..36011037 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -86,7 +86,7 @@ func main() { defer clusterService.Leave() } - edgeManager, err := edge.NewEdgeManager(options, advertiseAddr, clusterService, infoService) + edgeManager, err := edge.NewManager(options, advertiseAddr, clusterService, infoService) if err != nil { log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) } @@ -166,7 +166,7 @@ func parseOptions() (*agent.Options, error) { return optionParser.Options() } -func serveEdgeUI(edgeManager *edge.EdgeManager, clusterService agent.ClusterService, serverAddr, serverPort string) { +func serveEdgeUI(edgeManager *edge.Manager, clusterService agent.ClusterService, serverAddr, serverPort string) { edgeServer := http.NewEdgeServer(edgeManager, clusterService) go func() { diff --git a/http/edge.go b/http/edge.go index 07f1956c..379816ce 100644 --- a/http/edge.go +++ b/http/edge.go @@ -17,12 +17,12 @@ import ( // EdgeServer expose an UI to associate an Edge key with the agent. type EdgeServer struct { httpServer *http.Server - edgeManager *edge.EdgeManager + edgeManager *edge.Manager clusterService agent.ClusterService } // NewEdgeServer returns a pointer to a new instance of EdgeServer. -func NewEdgeServer(edgeManager *edge.EdgeManager, clusterService agent.ClusterService) *EdgeServer { +func NewEdgeServer(edgeManager *edge.Manager, clusterService agent.ClusterService) *EdgeServer { return &EdgeServer{ clusterService: clusterService, edgeManager: edgeManager, diff --git a/http/handler/handler.go b/http/handler/handler.go index 47b834cb..87be09ec 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -33,7 +33,7 @@ type Handler struct { hostHandler *host.Handler pingHandler *ping.Handler securedProtocol bool - edgeManager *edge.EdgeManager + edgeManager *edge.Manager } // Config represents a server handler configuration @@ -42,7 +42,7 @@ type Config struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService - EdgeManager *edge.EdgeManager + EdgeManager *edge.Manager TunnelOperator agent.TunnelOperator AgentTags map[string]string AgentOptions *agent.Options diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index 866db48d..7175aa11 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -15,14 +15,14 @@ import ( type Handler struct { *mux.Router tunnelOperator agent.TunnelOperator - edgeManager *edge.EdgeManager + edgeManager *edge.Manager } // NewHandler returns a pointer to an Handler // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.EdgeManager) *Handler { +func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.Manager) *Handler { h := &Handler{ Router: mux.NewRouter(), edgeManager: edgeManager, diff --git a/http/server.go b/http/server.go index 048578eb..8009052a 100644 --- a/http/server.go +++ b/http/server.go @@ -17,7 +17,7 @@ type APIServer struct { systemService agent.SystemService clusterService agent.ClusterService signatureService agent.DigitalSignatureService - edgeManager *edge.EdgeManager + edgeManager *edge.Manager agentTags map[string]string agentOptions *agent.Options } @@ -30,7 +30,7 @@ type APIServerConfig struct { SystemService agent.SystemService ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService - EdgeManager *edge.EdgeManager + EdgeManager *edge.Manager AgentTags map[string]string AgentOptions *agent.Options } diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 08b2d63f..b6a65895 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -10,8 +10,8 @@ import ( "github.com/portainer/agent/exec" ) -// EdgeManager manages Edge functionality -type EdgeManager struct { +// Manager manages Edge functionality +type Manager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService infoService agent.InfoService @@ -24,10 +24,10 @@ type EdgeManager struct { advertiseAddr string } -// NewEdgeManager creates an instance of EdgeManager -func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*EdgeManager, error) { +// NewManager creates an instance of Manager +func NewManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*Manager, error) { - return &EdgeManager{ + return &Manager{ clusterService: clusterService, infoService: infoService, agentOptions: options, @@ -37,7 +37,7 @@ func NewEdgeManager(options *agent.Options, advertiseAddr string, clusterService } // Init initializes the manager -func (manager *EdgeManager) Init() error { +func (manager *Manager) Init() error { if !manager.IsKeySet() { return errors.New("Unable to initalize edge manager without key") } @@ -85,16 +85,16 @@ func (manager *EdgeManager) Init() error { } // IsEdgeModeEnabled returns true if edge mode is enabled -func (manager *EdgeManager) IsEdgeModeEnabled() bool { +func (manager *Manager) IsEdgeModeEnabled() bool { return manager.edgeMode } // ResetActivityTimer resets the activity timer -func (manager *EdgeManager) ResetActivityTimer() { +func (manager *Manager) ResetActivityTimer() { manager.pollService.ResetActivityTimer() } -func (manager *EdgeManager) startEdgeBackgroundProcess() error { +func (manager *Manager) startEdgeBackgroundProcess() error { runtimeCheckFrequency, err := time.ParseDuration(agent.DefaultConfigCheckInterval) if err != nil { @@ -123,7 +123,7 @@ func (manager *EdgeManager) startEdgeBackgroundProcess() error { return nil } -func (manager *EdgeManager) checkRuntimeConfig() error { +func (manager *Manager) checkRuntimeConfig() error { agentTags, err := manager.infoService.GetInformationFromDockerEngine() if err != nil { return err diff --git a/internal/edge/key.go b/internal/edge/key.go index 232ca613..30c7b934 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -18,7 +18,7 @@ type edgeKey struct { } // SetKey parses and associate a key to the manager -func (manager *EdgeManager) SetKey(key string) error { +func (manager *Manager) SetKey(key string) error { edgeKey, err := parseEdgeKey(key) if err != nil { return err @@ -44,7 +44,7 @@ func (manager *EdgeManager) SetKey(key string) error { } // GetKey returns the key associated to the manager -func (manager *EdgeManager) GetKey() string { +func (manager *Manager) GetKey() string { var encodedKey string if manager.key != nil { @@ -55,7 +55,7 @@ func (manager *EdgeManager) GetKey() string { } // IsKeySet checks if a key is associated to the manager -func (manager *EdgeManager) IsKeySet() bool { +func (manager *Manager) IsKeySet() bool { if manager.key == nil { return false } From c743b2d39c291665a0c1739bc43e04bf56df4cd4 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:46:52 +0300 Subject: [PATCH 70/91] refactor(edge): move propogation to edge manager --- cmd/agent/main.go | 6 +++--- http/edge.go | 38 +++++++++----------------------------- internal/edge/key.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 36011037..495e08e1 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -113,7 +113,7 @@ func main() { } else { log.Println("[DEBUG] [main,edge] [message: Edge key not specified. Serving Edge UI]") - serveEdgeUI(edgeManager, clusterService, options.EdgeServerAddr, options.EdgeServerPort) + serveEdgeUI(edgeManager, options.EdgeServerAddr, options.EdgeServerPort) } } @@ -166,8 +166,8 @@ func parseOptions() (*agent.Options, error) { return optionParser.Options() } -func serveEdgeUI(edgeManager *edge.Manager, clusterService agent.ClusterService, serverAddr, serverPort string) { - edgeServer := http.NewEdgeServer(edgeManager, clusterService) +func serveEdgeUI(edgeManager *edge.Manager, serverAddr, serverPort string) { + edgeServer := http.NewEdgeServer(edgeManager) go func() { log.Printf("[INFO] [main,edge,http] [server_address: %s] [server_port: %s] [message: Starting Edge server]", serverAddr, serverPort) diff --git a/http/edge.go b/http/edge.go index 379816ce..c13d9b5d 100644 --- a/http/edge.go +++ b/http/edge.go @@ -2,30 +2,25 @@ package http import ( "context" - "fmt" "log" "net/http" "time" - "github.com/portainer/agent/http/client" "github.com/portainer/agent/internal/edge" "github.com/gorilla/mux" - "github.com/portainer/agent" ) // EdgeServer expose an UI to associate an Edge key with the agent. type EdgeServer struct { - httpServer *http.Server - edgeManager *edge.Manager - clusterService agent.ClusterService + httpServer *http.Server + edgeManager *edge.Manager } // NewEdgeServer returns a pointer to a new instance of EdgeServer. -func NewEdgeServer(edgeManager *edge.Manager, clusterService agent.ClusterService) *EdgeServer { +func NewEdgeServer(edgeManager *edge.Manager) *EdgeServer { return &EdgeServer{ - clusterService: clusterService, - edgeManager: edgeManager, + edgeManager: edgeManager, } } @@ -71,32 +66,17 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { http.Error(w, err.Error(), http.StatusInternalServerError) } - if server.clusterService != nil { - tags := server.clusterService.GetTags() - go server.propagateKeyInCluster(tags[agent.MemberTagKeyNodeName], key) - } + go server.propagateKeyInCluster() w.Write([]byte("Agent setup OK. You can close this page.")) server.Shutdown() } } -func (server *EdgeServer) propagateKeyInCluster(currentNodeName, key string) { - httpCli := client.NewAPIClient() - - members := server.clusterService.Members() - for _, member := range members { - - if member.NodeName == currentNodeName || member.EdgeKeySet { - continue - } - - memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) - - err := httpCli.SetEdgeKey(memberAddr, key) - if err != nil { - log.Printf("[ERROR] [edge,http] [member_address: %s] [message: Unable to propagate key to cluster member] [err: %s]", memberAddr, err) - } +func (server *EdgeServer) propagateKeyInCluster() { + err := server.edgeManager.PropagateKeyInCluster() + if err != nil { + log.Printf("[ERROR] [edge,http] [message: Unable to propagate key to cluster] [err: %s]", err) } } diff --git a/internal/edge/key.go b/internal/edge/key.go index 30c7b934..1815f6ae 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -8,6 +8,7 @@ import ( "github.com/portainer/agent" "github.com/portainer/agent/filesystem" + "github.com/portainer/agent/http/client" ) type edgeKey struct { @@ -62,6 +63,34 @@ func (manager *Manager) IsKeySet() bool { return true } +// PropagateKeyInCluster propogates the edge key in the cluster +func (manager *Manager) PropagateKeyInCluster() error { + if manager.clusterService != nil { + return nil + } + + httpCli := client.NewAPIClient() + tags := manager.clusterService.GetTags() + currentNodeName := tags[agent.MemberTagKeyNodeName] + + members := manager.clusterService.Members() + for _, member := range members { + + if member.NodeName == currentNodeName || member.EdgeKeySet { + continue + } + + memberAddr := fmt.Sprintf("%s:%s", member.IPAddress, member.Port) + + err := httpCli.SetEdgeKey(memberAddr, manager.GetKey()) + if err != nil { + return err + } + } + + return nil +} + // parseEdgeKey decodes a base64 encoded key and extract the decoded information from the following // format: ||| // are expected in the user:password format From 80a786eab1e346ea1cbc54a6dec52ff411885ce4 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Wed, 6 May 2020 12:54:00 +0300 Subject: [PATCH 71/91] fix(build): revert dev script --- dev.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev.sh b/dev.sh index f6f7fc61..aa21c631 100755 --- a/dev.sh +++ b/dev.sh @@ -44,9 +44,6 @@ function deploy_local() { echo "Image build..." docker build --no-cache -t "${IMAGE_NAME}" -f build/linux/Dockerfile . - # docker push "${IMAGE_NAME}" - # -e EDGE_KEY="aHR0cDovLzE3Mi4xNy4wLjE6OTAwMHwxNzIuMTcuMC4xOjgwMDB8ZWU6YTQ6YTg6ZDY6YzQ6Njk6MjY6MGI6N2Y6MDk6YTU6YTk6N2Y6NzQ6YTk6ZDZ8Nw" \ - echo "Deployment..." docker run -d --name portainer-agent-dev \ @@ -100,7 +97,6 @@ function deploy_swarm() { -e CAP_HOST_MANAGEMENT=${CAP_HOST_MANAGEMENT} \ -e EDGE=${EDGE} \ -e EDGE_ID=${EDGE_ID} \ - -e EDGE_KEY=aHR0cDovLzEwLjAuMi4yOjkwMDB8MTAuMC4yLjI6ODAwMHxlZTphNDphODpkNjpjNDo2OToyNjowYjo3ZjowOTphNTphOTo3Zjo3NDphOTpkNnw1 \ --mode global \ --mount type=bind,src=//var/run/docker.sock,dst=/var/run/docker.sock \ --mount type=bind,src=//var/lib/docker/volumes,dst=/var/lib/docker/volumes \ From c8bb24ade214796086e680f727f4bc74007eb0b8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:24:09 +0300 Subject: [PATCH 72/91] refactor(agent): remove tunnel operator interface --- agent.go | 9 --------- http/handler/handler.go | 4 ++-- http/handler/key/handler.go | 14 ++++++-------- internal/edge/edge.go | 2 +- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/agent.go b/agent.go index 81657b53..e99a6288 100644 --- a/agent.go +++ b/agent.go @@ -119,15 +119,6 @@ type ( IsTunnelOpen() bool } - // TunnelOperator is a service that is used to communicate with a Portainer instance and to manage - // the reverse tunnel. - TunnelOperator interface { - Start() error - Stop() error - CloseTunnel() error - ResetActivityTimer() - } - // Scheduler is used to manage schedules Scheduler interface { Schedule(schedules []Schedule) error diff --git a/http/handler/handler.go b/http/handler/handler.go index 87be09ec..ba2ebd42 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -43,7 +43,7 @@ type Config struct { ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService EdgeManager *edge.Manager - TunnelOperator agent.TunnelOperator + PollService *edge.PollService AgentTags map[string]string AgentOptions *agent.Options Secured bool @@ -61,7 +61,7 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.TunnelOperator, notaryService, config.EdgeManager), + keyHandler: key.NewHandler(config.PollService, notaryService, config.EdgeManager), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index 7175aa11..e5a0df36 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -3,8 +3,6 @@ package key import ( "net/http" - "github.com/portainer/agent" - "github.com/gorilla/mux" "github.com/portainer/agent/http/security" "github.com/portainer/agent/internal/edge" @@ -14,19 +12,19 @@ import ( // Handler is the HTTP handler used to handle Edge key operations. type Handler struct { *mux.Router - tunnelOperator agent.TunnelOperator - edgeManager *edge.Manager + pollService *edge.PollService + edgeManager *edge.Manager } // NewHandler returns a pointer to an Handler // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(tunnelOperator agent.TunnelOperator, notaryService *security.NotaryService, edgeManager *edge.Manager) *Handler { +func NewHandler(pollService *edge.PollService, notaryService *security.NotaryService, edgeManager *edge.Manager) *Handler { h := &Handler{ - Router: mux.NewRouter(), - edgeManager: edgeManager, - tunnelOperator: tunnelOperator, + Router: mux.NewRouter(), + edgeManager: edgeManager, + pollService: pollService, } h.Handle("/key", diff --git a/internal/edge/edge.go b/internal/edge/edge.go index b6a65895..601719c1 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -16,7 +16,7 @@ type Manager struct { dockerStackService agent.DockerStackService infoService agent.InfoService stacksManager *StacksManager - pollService agent.TunnelOperator + pollService *PollService pollServiceConfig *pollServiceConfig key *edgeKey edgeMode bool From 26bb7a1cf02136998987e670f400427af10e0378 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:34:10 +0300 Subject: [PATCH 73/91] style(agent): replace comments and logs --- cmd/agent/main.go | 8 ++++---- internal/edge/edge.go | 6 +++--- internal/edge/key.go | 9 +++++---- internal/edge/poll.go | 7 ++++--- internal/edge/stack.go | 10 +++++----- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 495e08e1..9cf9525c 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -94,20 +94,20 @@ func main() { if options.EdgeMode { edgeKey, err := retrieveEdgeKey(options.EdgeKey, clusterService) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to retrieve edge key] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to retrieve Edge key] [error: %s]", err) } if edgeKey != "" { - log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key to cluster.]") + log.Println("[DEBUG] [main,edge] [message: Edge key found in environment. Associating Edge key]") err := edgeManager.SetKey(edgeKey) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to set edge key] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to associate Edge edge key] [error: %s]", err) } err = edgeManager.Init() if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to init edge manager] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: An error occured during Edge initialization] [error: %s]", err) } } else { diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 601719c1..276ba66c 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -10,7 +10,7 @@ import ( "github.com/portainer/agent/exec" ) -// Manager manages Edge functionality +// Manager is used to manage all Edge features through multiple sub-components. It is mainly responsible for running the Edge background process. type Manager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService @@ -24,7 +24,7 @@ type Manager struct { advertiseAddr string } -// NewManager creates an instance of Manager +// NewManager returns a pointer to a new instance of Manager func NewManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*Manager, error) { return &Manager{ @@ -39,7 +39,7 @@ func NewManager(options *agent.Options, advertiseAddr string, clusterService age // Init initializes the manager func (manager *Manager) Init() error { if !manager.IsKeySet() { - return errors.New("Unable to initalize edge manager without key") + return errors.New("Unable to initialize Edge manager without key") } apiServerAddr := fmt.Sprintf("%s:%s", manager.advertiseAddr, manager.agentOptions.AgentServerPort) diff --git a/internal/edge/key.go b/internal/edge/key.go index 1815f6ae..9ce85f37 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -18,7 +18,8 @@ type edgeKey struct { EndpointID string } -// SetKey parses and associate a key to the manager +// SetKey parses and associates an Edge key to the agent. +// If the agent is running inside a Swarm cluster, it will also set the "set" flag to specify that a key is set on this agent in the cluster. func (manager *Manager) SetKey(key string) error { edgeKey, err := parseEdgeKey(key) if err != nil { @@ -44,7 +45,7 @@ func (manager *Manager) SetKey(key string) error { return nil } -// GetKey returns the key associated to the manager +// GetKey returns the Edge key associated to the agent func (manager *Manager) GetKey() string { var encodedKey string @@ -55,7 +56,7 @@ func (manager *Manager) GetKey() string { return encodedKey } -// IsKeySet checks if a key is associated to the manager +// IsKeySet returns true if an Edge key is associated to the agent func (manager *Manager) IsKeySet() bool { if manager.key == nil { return false @@ -63,7 +64,7 @@ func (manager *Manager) IsKeySet() bool { return true } -// PropagateKeyInCluster propogates the edge key in the cluster +// PropagateKeyInCluster propagates the Edge key associated to the agent to all the other agents inside the cluster func (manager *Manager) PropagateKeyInCluster() error { if manager.clusterService != nil { return nil diff --git a/internal/edge/poll.go b/internal/edge/poll.go index bc84638f..a0763300 100644 --- a/internal/edge/poll.go +++ b/internal/edge/poll.go @@ -19,8 +19,9 @@ import ( const tunnelActivityCheckInterval = 30 * time.Second -// PollService is used to poll a Portainer instance and to establish a reverse tunnel if needed. -// It also takes care of closing the tunnel after a set period of inactivity. +// PollService is used to poll a Portainer instance to retrieve the status associated to the Edge endpoint. +// It is responsible for managing the state of the reverse tunnel (open and closing after inactivity). +// It is also responsible for retrieving the data associated to Edge stacks and schedules. type PollService struct { apiServerAddr string pollIntervalInSeconds float64 @@ -51,7 +52,7 @@ type pollServiceConfig struct { TunnelServerFingerprint string } -// newPollService creates a new reverse tunnel service +// newPollService returns a pointer to a new instance of PollService func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) (*PollService, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { diff --git a/internal/edge/stack.go b/internal/edge/stack.go index e4db106d..1220a0e7 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -208,7 +208,7 @@ func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFile err := manager.dockerStackService.Deploy(stackName, stackFileLocation, stack.Prune) if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: failed deploying stack] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: stack deployment failed] [error: %s]", err) stack.Status = statusError responseStatus = int(edgeStackStatusError) errorMessage = err.Error() @@ -220,21 +220,21 @@ func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFile err = manager.httpClient.SetEdgeStackStatus(int(stack.ID), responseStatus, errorMessage) if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: failed setting edge stack status] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: unable to update Edge stack status] [error: %s]", err) } } func (manager *StacksManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { - log.Printf("[DEBUG] [internal,edge,stack] [message: removing stack %d]", stack.ID) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: removing stack]", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: failed deleting edge stack file] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: unable to delete Edge stack file] [error: %s]", err) return } err = manager.dockerStackService.Remove(stackName) if err != nil { - log.Printf("[ERROR] [internal,edge,stack] [message: failed removing stack] [error: %v]", err) + log.Printf("[ERROR] [internal,edge,stack] [message: unable to remove stack] [error: %s]", err) return } From a1e7f3f11e70589540a654ef5debd9ef7d355266 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:37:27 +0300 Subject: [PATCH 74/91] fix(edge): propogate key only in clusters --- internal/edge/key.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/edge/key.go b/internal/edge/key.go index 9ce85f37..d2d9241c 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -66,7 +66,7 @@ func (manager *Manager) IsKeySet() bool { // PropagateKeyInCluster propagates the Edge key associated to the agent to all the other agents inside the cluster func (manager *Manager) PropagateKeyInCluster() error { - if manager.clusterService != nil { + if manager.clusterService == nil { return nil } From 7bcf01ac17e9cbfa6f7d01fd1a7a9e994bb3d50a Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:39:38 +0300 Subject: [PATCH 75/91] fix(client): increase default client timeout --- http/client/portainer_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/client/portainer_client.go b/http/client/portainer_client.go index b1b8beec..569eda16 100644 --- a/http/client/portainer_client.go +++ b/http/client/portainer_client.go @@ -28,7 +28,7 @@ func NewPortainerClient(serverAddress, endpointID, edgeID string) *PortainerClie endpointID: endpointID, edgeID: edgeID, httpClient: &http.Client{ - Timeout: time.Second * 3, + Timeout: 10 * time.Second, }, } } From b148d9bcd1d3c6b5aa164e51da884bb0dd7ded93 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:43:24 +0300 Subject: [PATCH 76/91] refactor(poll): change poll functions to private --- internal/edge/edge.go | 6 +++--- internal/edge/poll.go | 17 +++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 276ba66c..f0c114f3 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -91,7 +91,7 @@ func (manager *Manager) IsEdgeModeEnabled() bool { // ResetActivityTimer resets the activity timer func (manager *Manager) ResetActivityTimer() { - manager.pollService.ResetActivityTimer() + manager.pollService.resetActivityTimer() } func (manager *Manager) startEdgeBackgroundProcess() error { @@ -135,13 +135,13 @@ func (manager *Manager) checkRuntimeConfig() error { log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) if !agentRunsOnSwarm || agentRunsOnLeaderNode { - err = manager.pollService.Start() + err = manager.pollService.start() if err != nil { return err } } else { - err = manager.pollService.Stop() + err = manager.pollService.stop() if err != nil { return err } diff --git a/internal/edge/poll.go b/internal/edge/poll.go index a0763300..48c66b6b 100644 --- a/internal/edge/poll.go +++ b/internal/edge/poll.go @@ -81,24 +81,22 @@ func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) }, nil } -// CloseTunnel closes the reverse tunnel managed by the service -func (service *PollService) CloseTunnel() error { +func (service *PollService) closeTunnel() error { return service.tunnelClient.CloseTunnel() } -// ResetActivityTimer will reset the last activity time timer -func (service *PollService) ResetActivityTimer() { +func (service *PollService) resetActivityTimer() { if service.tunnelClient.IsTunnelOpen() { service.lastActivity = time.Now() } } -// Start will start two loops in go routines +// start will start two loops in go routines // The first loop will poll the Portainer instance for the status of the associated endpoint and create a reverse tunnel // if needed as well as manage schedules. // The second loop will check for the last activity of the reverse tunnel and close the tunnel if it exceeds the tunnel // inactivity duration. -func (service *PollService) Start() error { +func (service *PollService) start() error { if service.refreshSignal != nil { return nil } @@ -110,8 +108,7 @@ func (service *PollService) Start() error { return nil } -// Stop stops the poll loop -func (service *PollService) Stop() error { +func (service *PollService) stop() error { if service.refreshSignal != nil { close(service.refreshSignal) service.refreshSignal = nil @@ -120,7 +117,7 @@ func (service *PollService) Stop() error { } func (service *PollService) restartStatusPollLoop() { - service.Stop() + service.stop() service.startStatusPollLoop() } @@ -320,6 +317,6 @@ func (service *PollService) createTunnel(encodedCredentials string, remotePort i return err } - service.ResetActivityTimer() + service.resetActivityTimer() return nil } From abf4143a510484fecc2ad9b8113dcfdd0465029c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:45:53 +0300 Subject: [PATCH 77/91] refactor(edge-stacks): move file path to agent constants --- agent.go | 2 ++ internal/edge/stack.go | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent.go b/agent.go index e99a6288..74c18050 100644 --- a/agent.go +++ b/agent.go @@ -220,4 +220,6 @@ const ( EdgeKeyFile = "agent_edge_key" // DockerBinaryPath is the path of the docker binary DockerBinaryPath = "/app" + // EdgeStackFilesPath is the path where edge stack files are saved + EdgeStackFilesPath = "/tmp/edge_stacls" ) diff --git a/internal/edge/stack.go b/internal/edge/stack.go index 1220a0e7..4a0dd364 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -10,8 +10,6 @@ import ( "github.com/portainer/agent/http/client" ) -const baseDir = "/tmp/edge_stacks" - type edgeStackID int type edgeStack struct { @@ -110,7 +108,7 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { stack.Prune = prune - folder := fmt.Sprintf("%s/%d", baseDir, stackID) + folder := fmt.Sprintf("%s/%d", agent.EdgeStackFilesPath, stackID) fileName := "docker-compose.yml" err = filesystem.WriteFile(folder, fileName, []byte(fileContent), 644) if err != nil { From 023c789d17dfabc9777af4fa357d7b8cd7d64fe6 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:51:53 +0300 Subject: [PATCH 78/91] refactor(stack): rename StackManager --- internal/edge/edge.go | 15 +++++---------- internal/edge/poll.go | 8 ++++---- internal/edge/stack.go | 27 ++++++++++++--------------- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index f0c114f3..9dea6655 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -15,7 +15,7 @@ type Manager struct { clusterService agent.ClusterService dockerStackService agent.DockerStackService infoService agent.InfoService - stacksManager *StacksManager + stackManager *StackManager pollService *PollService pollServiceConfig *pollServiceConfig key *edgeKey @@ -26,7 +26,6 @@ type Manager struct { // NewManager returns a pointer to a new instance of Manager func NewManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*Manager, error) { - return &Manager{ clusterService: clusterService, infoService: infoService, @@ -64,13 +63,9 @@ func (manager *Manager) Init() error { } manager.dockerStackService = dockerStackService - stacksManager, err := NewStacksManager(dockerStackService, manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.agentOptions.EdgeID) - if err != nil { - return err - } - manager.stacksManager = stacksManager + manager.stackManager = newStackManager(dockerStackService, manager.key.PortainerInstanceURL, manager.key.EndpointID, manager.agentOptions.EdgeID) - pollService, err := newPollService(stacksManager, pollServiceConfig) + pollService, err := newPollService(manager.stackManager, pollServiceConfig) if err != nil { return err } @@ -148,13 +143,13 @@ func (manager *Manager) checkRuntimeConfig() error { } if agentRunsOnSwarm && agentRunsOnLeaderNode { - err = manager.stacksManager.Start() + err = manager.stackManager.start() if err != nil { return err } } else { - err = manager.stacksManager.Stop() + err = manager.stackManager.stop() if err != nil { return err } diff --git a/internal/edge/poll.go b/internal/edge/poll.go index 48c66b6b..dbbec923 100644 --- a/internal/edge/poll.go +++ b/internal/edge/poll.go @@ -33,7 +33,7 @@ type PollService struct { scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} - edgeStacksManager *StacksManager + edgeStackManager *StackManager portainerURL string endpointID string tunnelServerAddr string @@ -53,7 +53,7 @@ type pollServiceConfig struct { } // newPollService returns a pointer to a new instance of PollService -func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) (*PollService, error) { +func newPollService(edgeStackManager *StackManager, config *pollServiceConfig) (*PollService, error) { pollFrequency, err := time.ParseDuration(config.PollFrequency) if err != nil { return nil, err @@ -73,7 +73,7 @@ func newPollService(edgeStacksManager *StacksManager, config *pollServiceConfig) tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, - edgeStacksManager: edgeStacksManager, + edgeStackManager: edgeStackManager, portainerURL: config.PortainerURL, endpointID: config.EndpointID, tunnelServerAddr: config.TunnelServerAddr, @@ -283,7 +283,7 @@ func (service *PollService) poll() error { stacks[stack.ID] = stack.Version } - err := service.edgeStacksManager.UpdateStacksStatus(stacks) + err := service.edgeStackManager.UpdateStacksStatus(stacks) if err != nil { log.Printf("[ERROR] [internal,edge,stack] [message: an error occured during stack management] [error: %s]", err) return err diff --git a/internal/edge/stack.go b/internal/edge/stack.go index 4a0dd364..c66679ed 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -50,8 +50,8 @@ const ( edgeStackStatusAcknowledged ) -// StacksManager represents a service for managing Edge stacks -type StacksManager struct { +// StackManager represents a service for managing Edge stacks +type StackManager struct { stacks map[edgeStackID]*edgeStack stopSignal chan struct{} dockerStackService agent.DockerStackService @@ -61,20 +61,19 @@ type StacksManager struct { httpClient *client.PortainerClient } -// NewStacksManager creates a new instance of StacksManager -func NewStacksManager(dockerStackService agent.DockerStackService, portainerURL, endpointID, edgeID string) (*StacksManager, error) { +// newStackManager returns a pointer to a new instance of StackManager +func newStackManager(dockerStackService agent.DockerStackService, portainerURL, endpointID, edgeID string) *StackManager { cli := client.NewPortainerClient(portainerURL, endpointID, edgeID) - return &StacksManager{ + return &StackManager{ dockerStackService: dockerStackService, stacks: map[edgeStackID]*edgeStack{}, stopSignal: nil, httpClient: cli, - }, nil + } } -// UpdateStacksStatus updates stacks version and status -func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { +func (manager *StackManager) updateStacksStatus(stacks map[int]int) error { if !manager.isEnabled { return nil } @@ -139,8 +138,7 @@ func (manager *StacksManager) UpdateStacksStatus(stacks map[int]int) error { return nil } -// Stop stops the manager -func (manager *StacksManager) Stop() error { +func (manager *StackManager) stop() error { if manager.stopSignal != nil { close(manager.stopSignal) manager.stopSignal = nil @@ -150,8 +148,7 @@ func (manager *StacksManager) Stop() error { return nil } -// Start starts the loop checking for stacks to deploy -func (manager *StacksManager) Start() error { +func (manager *StackManager) start() error { if manager.stopSignal != nil { return nil } @@ -188,7 +185,7 @@ func (manager *StacksManager) Start() error { return nil } -func (manager *StacksManager) next() *edgeStack { +func (manager *StackManager) next() *edgeStack { for _, stack := range manager.stacks { if stack.Status == statusPending { return stack @@ -197,7 +194,7 @@ func (manager *StacksManager) next() *edgeStack { return nil } -func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { +func (manager *StackManager) deployStack(stack *edgeStack, stackName, stackFileLocation string) { log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: stack deployment]", stack.ID) stack.Status = statusDone stack.Action = actionIdle @@ -222,7 +219,7 @@ func (manager *StacksManager) deployStack(stack *edgeStack, stackName, stackFile } } -func (manager *StacksManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { +func (manager *StackManager) deleteStack(stack *edgeStack, stackName, stackFileLocation string) { log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: removing stack]", stack.ID) err := filesystem.RemoveFile(stackFileLocation) if err != nil { From ac13bfe305cc421b3a6c3191da6952ccee2e19fd Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:57:52 +0300 Subject: [PATCH 79/91] refactor(edge): replace init with start --- cmd/agent/main.go | 4 ++-- http/edge.go | 2 +- http/handler/key/key_create.go | 4 ++-- internal/edge/edge.go | 6 +++--- internal/edge/poll.go | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 9cf9525c..c34b855d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -102,10 +102,10 @@ func main() { err := edgeManager.SetKey(edgeKey) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to associate Edge edge key] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to associate Edge key] [error: %s]", err) } - err = edgeManager.Init() + err = edgeManager.Start() if err != nil { log.Fatalf("[ERROR] [main,edge] [message: An error occured during Edge initialization] [error: %s]", err) } diff --git a/http/edge.go b/http/edge.go index c13d9b5d..59200742 100644 --- a/http/edge.go +++ b/http/edge.go @@ -61,7 +61,7 @@ func (server *EdgeServer) handleKeySetup() http.HandlerFunc { return } - err = server.edgeManager.Init() + err = server.edgeManager.Start() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/http/handler/key/key_create.go b/http/handler/key/key_create.go index 294c0129..372a73b9 100644 --- a/http/handler/key/key_create.go +++ b/http/handler/key/key_create.go @@ -44,9 +44,9 @@ func (handler *Handler) keyCreate(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to associate Edge key", err} } - err = handler.edgeManager.Init() + err = handler.edgeManager.Start() if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to initialize Edge Manager", err} + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start Edge Manager", err} } return response.Empty(w) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 9dea6655..8722474c 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -35,10 +35,10 @@ func NewManager(options *agent.Options, advertiseAddr string, clusterService age }, nil } -// Init initializes the manager -func (manager *Manager) Init() error { +// Start starts the manager +func (manager *Manager) Start() error { if !manager.IsKeySet() { - return errors.New("Unable to initialize Edge manager without key") + return errors.New("Unable to start Edge manager without key") } apiServerAddr := fmt.Sprintf("%s:%s", manager.advertiseAddr, manager.agentOptions.AgentServerPort) diff --git a/internal/edge/poll.go b/internal/edge/poll.go index dbbec923..aa275c6f 100644 --- a/internal/edge/poll.go +++ b/internal/edge/poll.go @@ -33,7 +33,7 @@ type PollService struct { scheduleManager agent.Scheduler lastActivity time.Time refreshSignal chan struct{} - edgeStackManager *StackManager + edgeStackManager *StackManager portainerURL string endpointID string tunnelServerAddr string @@ -73,7 +73,7 @@ func newPollService(edgeStackManager *StackManager, config *pollServiceConfig) ( tunnelClient: chisel.NewClient(), scheduleManager: filesystem.NewCronManager(), refreshSignal: nil, - edgeStackManager: edgeStackManager, + edgeStackManager: edgeStackManager, portainerURL: config.PortainerURL, endpointID: config.EndpointID, tunnelServerAddr: config.TunnelServerAddr, @@ -283,7 +283,7 @@ func (service *PollService) poll() error { stacks[stack.ID] = stack.Version } - err := service.edgeStackManager.UpdateStacksStatus(stacks) + err := service.edgeStackManager.updateStacksStatus(stacks) if err != nil { log.Printf("[ERROR] [internal,edge,stack] [message: an error occured during stack management] [error: %s]", err) return err From 7b4e5c725dbf9b6b28507c9e3b5c130d2afbac01 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 09:59:05 +0300 Subject: [PATCH 80/91] refactor(edge): remove error from edge manager --- cmd/agent/main.go | 7 ++----- internal/edge/edge.go | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index c34b855d..4c1f0988 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -86,10 +86,7 @@ func main() { defer clusterService.Leave() } - edgeManager, err := edge.NewManager(options, advertiseAddr, clusterService, infoService) - if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to start edge manger] [error: %s]", err) - } + edgeManager := edge.NewManager(options, advertiseAddr, clusterService, infoService) if options.EdgeMode { edgeKey, err := retrieveEdgeKey(options.EdgeKey, clusterService) @@ -107,7 +104,7 @@ func main() { err = edgeManager.Start() if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: An error occured during Edge initialization] [error: %s]", err) + log.Fatalf("[ERROR] [main,edge] [message: Unable to start Edge manager] [error: %s]", err) } } else { diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 8722474c..07d9d2f8 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -25,14 +25,14 @@ type Manager struct { } // NewManager returns a pointer to a new instance of Manager -func NewManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) (*Manager, error) { +func NewManager(options *agent.Options, advertiseAddr string, clusterService agent.ClusterService, infoService agent.InfoService) *Manager { return &Manager{ clusterService: clusterService, infoService: infoService, agentOptions: options, advertiseAddr: advertiseAddr, edgeMode: options.EdgeMode, - }, nil + } } // Start starts the manager From c5a2a09db45638eeaa0825fbd613cc0ebc1feb8e Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 7 May 2020 12:04:05 +0300 Subject: [PATCH 81/91] refactor(stack): show stack version in debug --- internal/edge/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/edge/stack.go b/internal/edge/stack.go index c66679ed..c3394040 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -208,7 +208,7 @@ func (manager *StackManager) deployStack(stack *edgeStack, stackName, stackFileL responseStatus = int(edgeStackStatusError) errorMessage = err.Error() } else { - log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [message: stack deployed]", stack.ID, stack.Version) + log.Printf("[DEBUG] [internal,edge,stack] [stack_identifier: %d] [stack_version: %d] [message: stack deployed]", stack.ID, stack.Version) } manager.stacks[stack.ID] = stack From d10537903dcdeebfffaf5de5dc37137bda2bfba3 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 8 May 2020 14:50:27 +1200 Subject: [PATCH 82/91] Update agent.go --- agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent.go b/agent.go index 74c18050..040ecd9a 100644 --- a/agent.go +++ b/agent.go @@ -221,5 +221,5 @@ const ( // DockerBinaryPath is the path of the docker binary DockerBinaryPath = "/app" // EdgeStackFilesPath is the path where edge stack files are saved - EdgeStackFilesPath = "/tmp/edge_stacls" + EdgeStackFilesPath = "/tmp/edge_stacks" ) From 2ac9de1bb8ae2e32ba3acb66e4956a1fc981db34 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 11 May 2020 11:45:27 +0300 Subject: [PATCH 83/91] refactor(edge): decrease queue sleep interval --- agent.go | 2 ++ internal/edge/stack.go | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/agent.go b/agent.go index 040ecd9a..f7e86103 100644 --- a/agent.go +++ b/agent.go @@ -222,4 +222,6 @@ const ( DockerBinaryPath = "/app" // EdgeStackFilesPath is the path where edge stack files are saved EdgeStackFilesPath = "/tmp/edge_stacks" + // EdgeStackQueueSleepInterval is the interval used to check if there's an Edge stack to deploy + EdgeStackQueueSleepInterval = "5s" ) diff --git a/internal/edge/stack.go b/internal/edge/stack.go index c3394040..c04f72eb 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -156,6 +156,11 @@ func (manager *StackManager) start() error { manager.isEnabled = true manager.stopSignal = make(chan struct{}) + queueSleepInterval, err := time.ParseDuration(agent.EdgeStackQueueSleepInterval) + if err != nil { + return err + } + go (func() { for { select { @@ -165,7 +170,7 @@ func (manager *StackManager) start() error { default: stack := manager.next() if stack == nil { - timer1 := time.NewTimer(1 * time.Minute) + timer1 := time.NewTimer(queueSleepInterval) <-timer1.C continue } From 722f4fc29ca575cf234395a1a707c564e82ee7a8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Sun, 10 May 2020 15:00:03 +0300 Subject: [PATCH 84/91] feat(edge-stacks): use edge stack name for deployed name --- http/client/portainer_client.go | 13 +++++++------ internal/edge/stack.go | 6 ++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/http/client/portainer_client.go b/http/client/portainer_client.go index 569eda16..043e00d5 100644 --- a/http/client/portainer_client.go +++ b/http/client/portainer_client.go @@ -34,39 +34,40 @@ func NewPortainerClient(serverAddress, endpointID, edgeID string) *PortainerClie } type stackConfigResponse struct { + Name string StackFileContent string Prune bool } // GetEdgeStackConfig retrieves the configuration associated to an Edge stack -func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, bool, error) { +func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, string, bool, error) { requestURL := fmt.Sprintf("%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { - return "", false, err + return "", "", false, err } req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, client.edgeID) resp, err := client.httpClient.Do(req) if err != nil { - return "", false, err + return "", "", false, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: GetEdgeStackConfig operation failed]", resp.StatusCode) - return "", false, errors.New("GetEdgeStackConfig operation failed") + return "", "", false, errors.New("GetEdgeStackConfig operation failed") } var data stackConfigResponse err = json.NewDecoder(resp.Body).Decode(&data) if err != nil { - return "", false, err + return "", "", false, err } - return data.StackFileContent, data.Prune, nil + return data.Name, data.StackFileContent, data.Prune, nil } type setEdgeStackStatusPayload struct { diff --git a/internal/edge/stack.go b/internal/edge/stack.go index c04f72eb..12eb557f 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -14,6 +14,7 @@ type edgeStackID int type edgeStack struct { ID edgeStackID + Name string Version int FileFolder string FileName string @@ -100,7 +101,7 @@ func (manager *StackManager) updateStacksStatus(stacks map[int]int) error { } } - fileContent, prune, err := manager.httpClient.GetEdgeStackConfig(int(stack.ID)) + name, fileContent, prune, err := manager.httpClient.GetEdgeStackConfig(int(stack.ID)) if err != nil { return err } @@ -116,6 +117,7 @@ func (manager *StackManager) updateStacksStatus(stacks map[int]int) error { stack.FileFolder = folder stack.FileName = fileName + stack.Name = name manager.stacks[stack.ID] = stack @@ -175,7 +177,7 @@ func (manager *StackManager) start() error { continue } - stackName := fmt.Sprintf("edge_%d", stack.ID) + stackName := fmt.Sprintf("edge_%s", stack.Name) stackFileLocation := fmt.Sprintf("%s/%s", stack.FileFolder, stack.FileName) if stack.Action == actionDeploy || stack.Action == actionUpdate { From 5289fc74dca370a35f3604d89cc907c4e7bb67ab Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Mon, 11 May 2020 10:46:00 +0300 Subject: [PATCH 85/91] refactor(edge): create stack config object --- agent.go | 7 +++++++ http/client/portainer_client.go | 14 +++++++------- internal/edge/stack.go | 8 ++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/agent.go b/agent.go index f7e86103..bc1fb089 100644 --- a/agent.go +++ b/agent.go @@ -27,6 +27,13 @@ type ( EdgeKeySet bool } + // EdgeStackConfig represnts an Edge stack config + EdgeStackConfig struct { + Name string + FileContent string + Prune bool + } + // AgentMetadata is the representation of the metadata object used to decorate // all the objects in the response of a Docker aggregated resource request. Metadata struct { diff --git a/http/client/portainer_client.go b/http/client/portainer_client.go index 043e00d5..c17d6cef 100644 --- a/http/client/portainer_client.go +++ b/http/client/portainer_client.go @@ -34,40 +34,40 @@ func NewPortainerClient(serverAddress, endpointID, edgeID string) *PortainerClie } type stackConfigResponse struct { - Name string + Name string StackFileContent string Prune bool } // GetEdgeStackConfig retrieves the configuration associated to an Edge stack -func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (string, string, bool, error) { +func (client *PortainerClient) GetEdgeStackConfig(edgeStackID int) (*agent.EdgeStackConfig, error) { requestURL := fmt.Sprintf("%s/api/endpoints/%s/edge/stacks/%d", client.serverAddress, client.endpointID, edgeStackID) req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { - return "", "", false, err + return nil, err } req.Header.Set(agent.HTTPEdgeIdentifierHeaderName, client.edgeID) resp, err := client.httpClient.Do(req) if err != nil { - return "", "", false, err + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { log.Printf("[ERROR] [http,client,portainer] [response_code: %d] [message: GetEdgeStackConfig operation failed]", resp.StatusCode) - return "", "", false, errors.New("GetEdgeStackConfig operation failed") + return nil, errors.New("GetEdgeStackConfig operation failed") } var data stackConfigResponse err = json.NewDecoder(resp.Body).Decode(&data) if err != nil { - return "", "", false, err + return nil, err } - return data.Name, data.StackFileContent, data.Prune, nil + return &agent.EdgeStackConfig{Name: data.Name, FileContent: data.StackFileContent, Prune: data.Prune}, nil } type setEdgeStackStatusPayload struct { diff --git a/internal/edge/stack.go b/internal/edge/stack.go index 12eb557f..3b9b4765 100644 --- a/internal/edge/stack.go +++ b/internal/edge/stack.go @@ -101,23 +101,23 @@ func (manager *StackManager) updateStacksStatus(stacks map[int]int) error { } } - name, fileContent, prune, err := manager.httpClient.GetEdgeStackConfig(int(stack.ID)) + stackConfig, err := manager.httpClient.GetEdgeStackConfig(int(stack.ID)) if err != nil { return err } - stack.Prune = prune + stack.Prune = stackConfig.Prune + stack.Name = stackConfig.Name folder := fmt.Sprintf("%s/%d", agent.EdgeStackFilesPath, stackID) fileName := "docker-compose.yml" - err = filesystem.WriteFile(folder, fileName, []byte(fileContent), 644) + err = filesystem.WriteFile(folder, fileName, []byte(stackConfig.FileContent), 644) if err != nil { return err } stack.FileFolder = folder stack.FileName = fileName - stack.Name = name manager.stacks[stack.ID] = stack From 2d987df7df58e15fc44c0d34e1917a154e83f15c Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 14 May 2020 12:28:37 +0300 Subject: [PATCH 86/91] refactor(agent): replace agenttags with a struct --- agent.go | 56 +++++++++--------- cmd/agent/main.go | 4 +- docker/docker.go | 21 +++---- http/handler/docker/docker_operation.go | 6 +- http/handler/docker/handler.go | 4 +- http/handler/handler.go | 5 +- http/handler/key/handler.go | 4 +- http/handler/websocket/attach.go | 2 +- http/handler/websocket/exec.go | 2 +- http/handler/websocket/handler.go | 4 +- http/proxy/agentproxy.go | 6 +- http/server.go | 4 +- internal/edge/edge.go | 6 +- internal/edge/key.go | 4 +- serf/cluster.go | 77 ++++++++++++++++++++----- 15 files changed, 127 insertions(+), 78 deletions(-) diff --git a/agent.go b/agent.go index bc1fb089..8312fc40 100644 --- a/agent.go +++ b/agent.go @@ -78,6 +78,18 @@ type ( Credentials string } + InfoTags struct { + AgentPort string + EdgeKeySet bool + EngineStatus EngineStatus + Leader bool + NodeName string + NodeRole NodeRole + } + + EngineStatus int + NodeRole int + // OptionParser is used to parse options. OptionParser interface { Options() (*Options, error) @@ -88,11 +100,11 @@ type ( Create(advertiseAddr string, joinAddr []string) error Members() []ClusterMember Leave() - GetMemberByRole(role string) *ClusterMember + GetMemberByRole(role NodeRole) *ClusterMember GetMemberByNodeName(nodeName string) *ClusterMember GetMemberWithEdgeKeySet() *ClusterMember - GetTags() map[string]string - UpdateTags(tags map[string]string) error + GetTags() *InfoTags + UpdateTags(tags *InfoTags) error } // DigitalSignatureService is used to validate digital signatures. @@ -102,7 +114,7 @@ type ( // InfoService is used to retrieve information from a Docker environment. InfoService interface { - GetInformationFromDockerEngine() (map[string]string, error) + GetInformationFromDockerEngine() (*InfoTags, error) GetContainerIpFromDockerEngine(containerName string, ignoreNonSwarmNetworks bool) (string, error) GetServiceNameFromDockerEngine(containerName string) (string, error) } @@ -191,30 +203,6 @@ const ( // ResponseMetadataKey is the JSON field used to store any Portainer related information in // response objects. ResponseMetadataKey = "Portainer" - // MemberTagKeyAgentPort is the name of the label storing information about the port exposed - // by the agent. - MemberTagKeyAgentPort = "AgentPort" - // MemberTagKeyIsLeader is the name of the label storing whether the node is a swarm leader - MemberTagKeyIsLeader = "NodeIsLeader" - // MemberTagKeyNodeName is the name of the label storing information about the name of the - // node where the agent is running. - MemberTagKeyNodeName = "NodeName" - // MemberTagKeyNodeRole is the name of the label storing information about the role of the - // node where the agent is running. - MemberTagKeyNodeRole = "NodeRole" - // MemberTagEngineStatus is the name of the label storing information about the status of the Docker engine where - // the agent is running. Possible values are "standalone" or "swarm". - MemberTagEngineStatus = "EngineStatus" - // MemberTagEdgeKeySet is the name of the label storing information regarding the association of an Edge key. - MemberTagEdgeKeySet = "EdgeKeySet" - // NodeRoleManager represents a manager node. - NodeRoleManager = "manager" - // NodeRoleWorker represents a worker node. - NodeRoleWorker = "worker" - // EngineStatusSwarm represents a swarm docker engine - EngineStatusSwarm = "swarm" - // EngineStatusStandalone represents a standalone docker engine - EngineStatusStandalone = "standalone" // TLSCertPath is the default path to the TLS certificate file. TLSCertPath = "cert.pem" // TLSKeyPath is the default path to the TLS key file. @@ -232,3 +220,15 @@ const ( // EdgeStackQueueSleepInterval is the interval used to check if there's an Edge stack to deploy EdgeStackQueueSleepInterval = "5s" ) + +const ( + _ NodeRole = iota + NodeRoleManager + NodeRoleWorker +) + +const ( + _ EngineStatus = iota + EngineStatusStandalone + EngineStatusSwarm +) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 4c1f0988..252419d9 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -34,11 +34,11 @@ func main() { log.Fatalf("[ERROR] [main,docker] [message: Unable to retrieve information from Docker] [error: %s]", err) } - agentTags[agent.MemberTagKeyAgentPort] = options.AgentServerPort + agentTags.AgentPort = options.AgentServerPort log.Printf("[DEBUG] [main,configuration] [Member tags: %+v]", agentTags) clusterMode := false - if agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm { + if agentTags.EngineStatus == agent.EngineStatusSwarm { clusterMode = true log.Println("[INFO] [main] [message: Agent running on a Swarm cluster node. Running in cluster mode]") } diff --git a/docker/docker.go b/docker/docker.go index 7f444746..763bcf9e 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -25,7 +25,7 @@ func NewInfoService() *InfoService { // GetInformationFromDockerEngine retrieves information from a Docker environment // and returns a map of labels. -func (service *InfoService) GetInformationFromDockerEngine() (map[string]string, error) { +func (service *InfoService) GetInformationFromDockerEngine() (*agent.InfoTags, error) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(agent.SupportedDockerAPIVersion)) if err != nil { return nil, err @@ -37,8 +37,8 @@ func (service *InfoService) GetInformationFromDockerEngine() (map[string]string, return nil, err } - info := make(map[string]string) - info[agent.MemberTagKeyNodeName] = dockerInfo.Name + info := &agent.InfoTags{} + info.NodeName = dockerInfo.Name if dockerInfo.Swarm.NodeID == "" { getStandaloneInfo(info) @@ -111,15 +111,16 @@ func (service *InfoService) GetServiceNameFromDockerEngine(containerName string) return containerInspect.Config.Labels[serviceNameLabel], nil } -func getStandaloneInfo(info map[string]string) { - info[agent.MemberTagEngineStatus] = agent.EngineStatusStandalone +func getStandaloneInfo(info *agent.InfoTags) { + info.EngineStatus = agent.EngineStatusStandalone } -func getSwarmInformation(info map[string]string, dockerInfo types.Info, cli *client.Client) error { - info[agent.MemberTagEngineStatus] = agent.EngineStatusSwarm - info[agent.MemberTagKeyNodeRole] = agent.NodeRoleWorker +func getSwarmInformation(info *agent.InfoTags, dockerInfo types.Info, cli *client.Client) error { + info.EngineStatus = agent.EngineStatusSwarm + info.NodeRole = agent.NodeRoleWorker + if dockerInfo.Swarm.ControlAvailable { - info[agent.MemberTagKeyNodeRole] = agent.NodeRoleManager + info.NodeRole = agent.NodeRoleManager node, _, err := cli.NodeInspectWithRaw(context.Background(), dockerInfo.Swarm.NodeID) if err != nil { @@ -127,7 +128,7 @@ func getSwarmInformation(info map[string]string, dockerInfo types.Info, cli *cli } if node.ManagerStatus.Leader { - info[agent.MemberTagKeyIsLeader] = "1" + info.Leader = true } } diff --git a/http/handler/docker/docker_operation.go b/http/handler/docker/docker_operation.go index 28faaa80..e6665513 100644 --- a/http/handler/docker/docker_operation.go +++ b/http/handler/docker/docker_operation.go @@ -59,7 +59,7 @@ func (handler *Handler) dispatchOperation(rw http.ResponseWriter, request *http. } func (handler *Handler) executeOperationOnManagerNode(rw http.ResponseWriter, request *http.Request) *httperror.HandlerError { - if handler.agentTags[agent.MemberTagKeyNodeRole] == agent.NodeRoleManager { + if handler.agentTags.NodeRole == agent.NodeRoleManager { handler.dockerProxy.ServeHTTP(rw, request) } else { targetMember := handler.clusterService.GetMemberByRole(agent.NodeRoleManager) @@ -75,7 +75,7 @@ func (handler *Handler) executeOperationOnManagerNode(rw http.ResponseWriter, re func (handler *Handler) executeOperationOnNode(rw http.ResponseWriter, request *http.Request) *httperror.HandlerError { agentTargetHeader := request.Header.Get(agent.HTTPTargetHeaderName) - if agentTargetHeader == handler.agentTags[agent.MemberTagKeyNodeName] || agentTargetHeader == "" { + if agentTargetHeader == handler.agentTags.NodeName || agentTargetHeader == "" { handler.dockerProxy.ServeHTTP(rw, request) } else { targetMember := handler.clusterService.GetMemberByNodeName(agentTargetHeader) @@ -92,7 +92,7 @@ func (handler *Handler) executeOperationOnNode(rw http.ResponseWriter, request * func (handler *Handler) executeOperationOnCluster(rw http.ResponseWriter, request *http.Request) *httperror.HandlerError { agentTargetHeader := request.Header.Get(agent.HTTPTargetHeaderName) - if agentTargetHeader == handler.agentTags[agent.MemberTagKeyNodeName] { + if agentTargetHeader == handler.agentTags.NodeName { handler.dockerProxy.ServeHTTP(rw, request) return nil } diff --git a/http/handler/docker/handler.go b/http/handler/docker/handler.go index 683f07d9..f9ab0e03 100644 --- a/http/handler/docker/handler.go +++ b/http/handler/docker/handler.go @@ -14,13 +14,13 @@ type Handler struct { dockerProxy *proxy.LocalProxy clusterProxy *proxy.ClusterProxy clusterService agent.ClusterService - agentTags map[string]string + agentTags *agent.InfoTags useTLS bool } // NewHandler returns a new instance of Handler. // It sets the associated handle functions for all the Docker related HTTP endpoints. -func NewHandler(clusterService agent.ClusterService, agentTags map[string]string, notaryService *security.NotaryService, useTLS bool) *Handler { +func NewHandler(clusterService agent.ClusterService, agentTags *agent.InfoTags, notaryService *security.NotaryService, useTLS bool) *Handler { h := &Handler{ Router: mux.NewRouter(), dockerProxy: proxy.NewLocalProxy(), diff --git a/http/handler/handler.go b/http/handler/handler.go index ba2ebd42..09d91636 100644 --- a/http/handler/handler.go +++ b/http/handler/handler.go @@ -43,8 +43,7 @@ type Config struct { ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService EdgeManager *edge.Manager - PollService *edge.PollService - AgentTags map[string]string + AgentTags *agent.InfoTags AgentOptions *agent.Options Secured bool } @@ -61,7 +60,7 @@ func NewHandler(config *Config) *Handler { browseHandler: browse.NewHandler(agentProxy, notaryService, config.AgentOptions), browseHandlerV1: browse.NewHandlerV1(agentProxy, notaryService), dockerProxyHandler: docker.NewHandler(config.ClusterService, config.AgentTags, notaryService, config.Secured), - keyHandler: key.NewHandler(config.PollService, notaryService, config.EdgeManager), + keyHandler: key.NewHandler(notaryService, config.EdgeManager), webSocketHandler: websocket.NewHandler(config.ClusterService, config.AgentTags, notaryService), hostHandler: host.NewHandler(config.SystemService, agentProxy, notaryService), pingHandler: ping.NewHandler(), diff --git a/http/handler/key/handler.go b/http/handler/key/handler.go index e5a0df36..f0040b7c 100644 --- a/http/handler/key/handler.go +++ b/http/handler/key/handler.go @@ -12,7 +12,6 @@ import ( // Handler is the HTTP handler used to handle Edge key operations. type Handler struct { *mux.Router - pollService *edge.PollService edgeManager *edge.Manager } @@ -20,11 +19,10 @@ type Handler struct { // It sets the associated handle functions for all the Edge key related HTTP endpoints. // This handler is meant to be used when the agent is started in Edge mode, all the API endpoints will return // a HTTP 503 service not available if edge mode is disabled. -func NewHandler(pollService *edge.PollService, notaryService *security.NotaryService, edgeManager *edge.Manager) *Handler { +func NewHandler(notaryService *security.NotaryService, edgeManager *edge.Manager) *Handler { h := &Handler{ Router: mux.NewRouter(), edgeManager: edgeManager, - pollService: pollService, } h.Handle("/key", diff --git a/http/handler/websocket/attach.go b/http/handler/websocket/attach.go index 6c4ba2f2..5e2677b2 100644 --- a/http/handler/websocket/attach.go +++ b/http/handler/websocket/attach.go @@ -23,7 +23,7 @@ func (handler *Handler) websocketAttach(w http.ResponseWriter, r *http.Request) } agentTargetHeader := r.Header.Get(agent.HTTPTargetHeaderName) - if agentTargetHeader == handler.agentTags[agent.MemberTagKeyNodeName] { + if agentTargetHeader == handler.agentTags.NodeName { return handler.handleAttachRequest(w, r) } diff --git a/http/handler/websocket/exec.go b/http/handler/websocket/exec.go index 77db8b02..e3c1b290 100644 --- a/http/handler/websocket/exec.go +++ b/http/handler/websocket/exec.go @@ -24,7 +24,7 @@ func (handler *Handler) websocketExec(w http.ResponseWriter, r *http.Request) *h agentTargetHeader := r.Header.Get(agent.HTTPTargetHeaderName) - if agentTargetHeader == handler.agentTags[agent.MemberTagKeyNodeName] { + if agentTargetHeader == handler.agentTags.NodeName { return handler.handleExecRequest(w, r) } diff --git a/http/handler/websocket/handler.go b/http/handler/websocket/handler.go index 8533f299..d3de7284 100644 --- a/http/handler/websocket/handler.go +++ b/http/handler/websocket/handler.go @@ -14,7 +14,7 @@ type ( *mux.Router clusterService agent.ClusterService connectionUpgrader websocket.Upgrader - agentTags map[string]string + agentTags *agent.InfoTags } execStartOperationPayload struct { @@ -24,7 +24,7 @@ type ( ) // NewHandler returns a new instance of Handler. -func NewHandler(clusterService agent.ClusterService, agentTags map[string]string, notaryService *security.NotaryService) *Handler { +func NewHandler(clusterService agent.ClusterService, agentTags *agent.InfoTags, notaryService *security.NotaryService) *Handler { h := &Handler{ Router: mux.NewRouter(), connectionUpgrader: websocket.Upgrader{}, diff --git a/http/proxy/agentproxy.go b/http/proxy/agentproxy.go index f2045f67..beeb4c16 100644 --- a/http/proxy/agentproxy.go +++ b/http/proxy/agentproxy.go @@ -12,12 +12,12 @@ import ( // AgentProxy enables redirection to different nodes type AgentProxy struct { clusterService agent.ClusterService - agentTags map[string]string + agentTags *agent.InfoTags useTLS bool } // NewAgentProxy returns a pointer to a new AgentProxy object -func NewAgentProxy(clusterService agent.ClusterService, agentTags map[string]string, useTLS bool) *AgentProxy { +func NewAgentProxy(clusterService agent.ClusterService, agentTags *agent.InfoTags, useTLS bool) *AgentProxy { return &AgentProxy{ clusterService: clusterService, agentTags: agentTags, @@ -36,7 +36,7 @@ func (p *AgentProxy) Redirect(next http.Handler) http.Handler { agentTargetHeader := r.Header.Get(agent.HTTPTargetHeaderName) - if agentTargetHeader == p.agentTags[agent.MemberTagKeyNodeName] || agentTargetHeader == "" { + if agentTargetHeader == p.agentTags.NodeName || agentTargetHeader == "" { next.ServeHTTP(rw, r) } else { targetMember := p.clusterService.GetMemberByNodeName(agentTargetHeader) diff --git a/http/server.go b/http/server.go index 8009052a..03d7e510 100644 --- a/http/server.go +++ b/http/server.go @@ -18,7 +18,7 @@ type APIServer struct { clusterService agent.ClusterService signatureService agent.DigitalSignatureService edgeManager *edge.Manager - agentTags map[string]string + agentTags *agent.InfoTags agentOptions *agent.Options } @@ -31,7 +31,7 @@ type APIServerConfig struct { ClusterService agent.ClusterService SignatureService agent.DigitalSignatureService EdgeManager *edge.Manager - AgentTags map[string]string + AgentTags *agent.InfoTags AgentOptions *agent.Options } diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 07d9d2f8..301bb200 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -124,10 +124,10 @@ func (manager *Manager) checkRuntimeConfig() error { return err } - agentRunsOnLeaderNode := agentTags[agent.MemberTagKeyIsLeader] == "1" - agentRunsOnSwarm := agentTags[agent.MemberTagEngineStatus] == agent.EngineStatusSwarm + agentRunsOnLeaderNode := agentTags.Leader + agentRunsOnSwarm := agentTags.EngineStatus == agent.EngineStatusSwarm - log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags[agent.MemberTagEngineStatus], agentRunsOnLeaderNode) + log.Printf("[DEBUG] [internal,edge,docker] [message: Docker runtime configuration check] [engine_status: %s] [leader_node: %t]", agentTags.EngineStatus, agentRunsOnLeaderNode) if !agentRunsOnSwarm || agentRunsOnLeaderNode { err = manager.pollService.start() diff --git a/internal/edge/key.go b/internal/edge/key.go index d2d9241c..14bed01e 100644 --- a/internal/edge/key.go +++ b/internal/edge/key.go @@ -35,7 +35,7 @@ func (manager *Manager) SetKey(key string) error { if manager.clusterService != nil { tags := manager.clusterService.GetTags() - tags[agent.MemberTagEdgeKeySet] = "set" + tags.EdgeKeySet = true err = manager.clusterService.UpdateTags(tags) if err != nil { return err @@ -72,7 +72,7 @@ func (manager *Manager) PropagateKeyInCluster() error { httpCli := client.NewAPIClient() tags := manager.clusterService.GetTags() - currentNodeName := tags[agent.MemberTagKeyNodeName] + currentNodeName := tags.NodeName members := manager.clusterService.Members() for _, member := range members { diff --git a/serf/cluster.go b/serf/cluster.go index 3421cc25..a2c9c4e5 100644 --- a/serf/cluster.go +++ b/serf/cluster.go @@ -11,15 +11,29 @@ import ( "github.com/portainer/agent" ) +const ( + memberTagKeyAgentPort = "AgentPort" + memberTagKeyIsLeader = "NodeIsLeader" + memberTagKeyNodeName = "NodeName" + memberTagKeyNodeRole = "NodeRole" + memberTagKeyEngineStatus = "EngineStatus" + memberTagKeyEdgeKeySet = "EdgeKeySet" + + memberTagValueEngineStatusSwarm = "swarm" + memberTagValueEngineStatusStandalone = "standalone" + memberTagValueNodeRoleManager = "manager" + memberTagValueNodeRoleWorker = "worker" +) + // ClusterService is a service used to manage cluster related actions such as joining // the cluster, retrieving members in the clusters... type ClusterService struct { - tags map[string]string + tags *agent.InfoTags cluster *serf.Serf } // NewClusterService returns a pointer to a ClusterService. -func NewClusterService(tags map[string]string) *ClusterService { +func NewClusterService(tags *agent.InfoTags) *ClusterService { return &ClusterService{ tags: tags, } @@ -42,8 +56,8 @@ func (service *ClusterService) Create(advertiseAddr string, joinAddr []string) e conf := serf.DefaultConfig() conf.Init() - conf.NodeName = fmt.Sprintf("%s-%s", service.tags[agent.MemberTagKeyNodeName], conf.NodeName) - conf.Tags = service.tags + conf.NodeName = fmt.Sprintf("%s-%s", service.tags.NodeName, conf.NodeName) + conf.Tags = convertTagsToMap(service.tags) conf.MemberlistConfig.LogOutput = filter conf.LogOutput = filter conf.MemberlistConfig.AdvertiseAddr = advertiseAddr @@ -80,13 +94,13 @@ func (service *ClusterService) Members() []agent.ClusterMember { if member.Status == serf.StatusAlive { clusterMember := agent.ClusterMember{ IPAddress: member.Addr.String(), - Port: member.Tags[agent.MemberTagKeyAgentPort], - NodeRole: member.Tags[agent.MemberTagKeyNodeRole], - NodeName: member.Tags[agent.MemberTagKeyNodeName], + Port: member.Tags[memberTagKeyAgentPort], + NodeRole: member.Tags[memberTagKeyNodeRole], + NodeName: member.Tags[memberTagKeyNodeName], EdgeKeySet: false, } - _, ok := member.Tags[agent.MemberTagEdgeKeySet] + _, ok := member.Tags[memberTagKeyEdgeKeySet] if ok { clusterMember.EdgeKeySet = true } @@ -99,10 +113,16 @@ func (service *ClusterService) Members() []agent.ClusterMember { } // GetMemberByRole will return the first member with the specified role. -func (service *ClusterService) GetMemberByRole(role string) *agent.ClusterMember { +func (service *ClusterService) GetMemberByRole(role agent.NodeRole) *agent.ClusterMember { members := service.Members() + + roleString := memberTagValueNodeRoleManager + if role == agent.NodeRoleWorker { + roleString = memberTagValueNodeRoleWorker + } + for _, member := range members { - if member.NodeRole == role { + if member.NodeRole == roleString { return &member } } @@ -135,12 +155,43 @@ func (service *ClusterService) GetMemberWithEdgeKeySet() *agent.ClusterMember { } // UpdateTags propagate the new tags to the cluster -func (service *ClusterService) UpdateTags(tags map[string]string) error { +func (service *ClusterService) UpdateTags(tags *agent.InfoTags) error { service.tags = tags - return service.cluster.SetTags(tags) + tagsMap := convertTagsToMap(tags) + return service.cluster.SetTags(tagsMap) } // GetTags returns the tags associated to the service -func (service *ClusterService) GetTags() map[string]string { +func (service *ClusterService) GetTags() *agent.InfoTags { return service.tags } + +func convertTagsToMap(tags *agent.InfoTags) map[string]string { + tagsMap := map[string]string{} + + tagsMap[memberTagKeyEdgeKeySet] = "" + if tags.EdgeKeySet { + tagsMap[memberTagKeyEdgeKeySet] = "set" + } + + tagsMap[memberTagKeyEngineStatus] = memberTagValueEngineStatusStandalone + if tags.EngineStatus == agent.EngineStatusSwarm { + tagsMap[memberTagKeyEngineStatus] = memberTagValueEngineStatusSwarm + } + + tagsMap[memberTagKeyAgentPort] = tags.AgentPort + + tagsMap[memberTagKeyIsLeader] = "" + if tags.Leader { + tagsMap[memberTagKeyIsLeader] = "1" + } + + tagsMap[memberTagKeyNodeName] = tags.NodeName + + tagsMap[memberTagKeyNodeRole] = memberTagValueNodeRoleManager + if tags.NodeRole == agent.NodeRoleWorker { + tagsMap[memberTagKeyNodeRole] = memberTagValueNodeRoleWorker + } + + return tagsMap +} From da3929a6dd3d00328b00839ee54ffae526588235 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 14 May 2020 12:33:47 +0300 Subject: [PATCH 87/91] fix(serf): set tags.EdgeKeySet only if set --- serf/cluster.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/serf/cluster.go b/serf/cluster.go index a2c9c4e5..d0e78126 100644 --- a/serf/cluster.go +++ b/serf/cluster.go @@ -169,7 +169,6 @@ func (service *ClusterService) GetTags() *agent.InfoTags { func convertTagsToMap(tags *agent.InfoTags) map[string]string { tagsMap := map[string]string{} - tagsMap[memberTagKeyEdgeKeySet] = "" if tags.EdgeKeySet { tagsMap[memberTagKeyEdgeKeySet] = "set" } @@ -181,7 +180,6 @@ func convertTagsToMap(tags *agent.InfoTags) map[string]string { tagsMap[memberTagKeyAgentPort] = tags.AgentPort - tagsMap[memberTagKeyIsLeader] = "" if tags.Leader { tagsMap[memberTagKeyIsLeader] = "1" } From dbccd291d59d5d4dfc63b97042f233adab1f5960 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 14 May 2020 12:42:34 +0300 Subject: [PATCH 88/91] feat(main): continue startup even if edge key failed to load from cluster --- cmd/agent/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 4c1f0988..58f96bf1 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -91,7 +91,7 @@ func main() { if options.EdgeMode { edgeKey, err := retrieveEdgeKey(options.EdgeKey, clusterService) if err != nil { - log.Fatalf("[ERROR] [main,edge] [message: Unable to retrieve Edge key] [error: %s]", err) + log.Printf("[ERROR] [main,edge] [message: Unable to retrieve Edge key] [error: %s]", err) } if edgeKey != "" { From d746ab741280d3a9766913738e4fa1fc6524c928 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Mon, 18 May 2020 14:12:40 +1200 Subject: [PATCH 89/91] feat(build): add alpine based Dockerfile --- build/linux/alpine.Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 build/linux/alpine.Dockerfile diff --git a/build/linux/alpine.Dockerfile b/build/linux/alpine.Dockerfile new file mode 100644 index 00000000..ca3eda1b --- /dev/null +++ b/build/linux/alpine.Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:latest + +WORKDIR /app + +COPY dist /app/ +COPY static /app/static + +ENTRYPOINT ["./agent"] From c0b6215d2e26d2a0a0853b56446208e5885bb19f Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Wed, 20 May 2020 13:18:50 +1200 Subject: [PATCH 90/91] feat(agent): update the node config check interval --- agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent.go b/agent.go index 8312fc40..5e8c5614 100644 --- a/agent.go +++ b/agent.go @@ -174,7 +174,7 @@ const ( // DefaultEdgeSleepInterval is the default interval after which the agent will close the tunnel if no activity. DefaultEdgeSleepInterval = "5m" // DefaultConfigCheckInterval is the default interval used to check if node config changed - DefaultConfigCheckInterval = "1m" + DefaultConfigCheckInterval = "5s" // SupportedDockerAPIVersion is the minimum Docker API version supported by the agent. SupportedDockerAPIVersion = "1.24" // HTTPTargetHeaderName is the name of the header used to specify a target node. From ef30e131cd9b546d114196db86585cd15c58e07d Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 29 May 2020 14:40:56 +1200 Subject: [PATCH 91/91] chore(version): bump version number --- agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent.go b/agent.go index 5e8c5614..cec2892b 100644 --- a/agent.go +++ b/agent.go @@ -154,7 +154,7 @@ type ( const ( // Version represents the version of the agent. - Version = "1.5.1" + Version = "1.6.0" // APIVersion represents the version of the agent's API. APIVersion = "2" // DefaultAgentAddr is the default address used by the Agent API server.