Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add support for embedded cluster updates
Browse files Browse the repository at this point in the history
TBD
ricardomaraschini committed Dec 6, 2023

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
1 parent 4db82ed commit d7c0bb4
Showing 11 changed files with 329 additions and 40 deletions.
17 changes: 9 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
@@ -41,15 +41,15 @@ require (
github.com/mholt/archiver/v3 v3.5.1
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
github.com/mitchellh/hashstructure v1.1.0
github.com/onsi/ginkgo/v2 v2.13.1
github.com/onsi/ginkgo/v2 v2.13.2
github.com/onsi/gomega v1.30.0
github.com/open-policy-agent/opa v0.58.0
github.com/ory/dockertest/v3 v3.10.0
github.com/otiai10/copy v1.9.0
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
github.com/replicatedhq/embedded-cluster-operator v0.4.1
github.com/replicatedhq/embedded-cluster-operator v0.5.0
github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82
github.com/replicatedhq/kurlkinds v1.3.6
github.com/replicatedhq/troubleshoot v0.76.4-0.20231102041618-a7bb9ea31e61
@@ -67,7 +67,7 @@ require (
github.com/tj/go-spin v1.1.0
github.com/vmware-tanzu/velero v1.10.1
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.25.0
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.14.0
golang.org/x/oauth2 v0.13.0
golang.org/x/sync v0.5.0
@@ -184,10 +184,11 @@ require (
github.com/go-ldap/ldap/v3 v3.4.4 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/loads v0.21.2 // indirect
github.com/go-openapi/runtime v0.26.0 // indirect
@@ -237,7 +238,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.10.1 // indirect
@@ -389,9 +390,9 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
k8s.io/apiextensions-apiserver v0.28.3 // indirect
k8s.io/apiserver v0.28.3 // indirect
k8s.io/component-base v0.28.3 // indirect
k8s.io/apiextensions-apiserver v0.28.4 // indirect
k8s.io/apiserver v0.28.4 // indirect
k8s.io/component-base v0.28.4 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-aggregator v0.19.12 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
33 changes: 17 additions & 16 deletions go.sum
Original file line number Diff line number Diff line change
@@ -356,8 +356,6 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -647,6 +645,7 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -1032,8 +1031,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -1385,8 +1384,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.13.1 h1:LNGfMbR2OVGBfXjvRZIZ2YCTQdGKtPLvuI1rMCCj3OU=
github.com/onsi/ginkgo/v2 v2.13.1/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=
github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -1530,8 +1529,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/replicatedhq/embedded-cluster-operator v0.4.1 h1:4LMbS5Z8adVe+nO4lFG0oI926RiQgSKOn3h2yjHXShI=
github.com/replicatedhq/embedded-cluster-operator v0.4.1/go.mod h1:Z9hN4T1105PiYVh2UcgkYLSLLQDhQiuP3aDB8KDBGZA=
github.com/replicatedhq/embedded-cluster-operator v0.5.0 h1:EihT/WoUU4uHF5F53Fh1K+jhtjhPTrLy/RdUGlHY4Hc=
github.com/replicatedhq/embedded-cluster-operator v0.5.0/go.mod h1:Ahieg2DIkZ3U4rfSdmR12M3ljpjS/lLnCLR92W7Oicw=
github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82 h1:QniKgIpcXu4wBMM4xIXGz+lkAU+hSIXFuVM+vxkNk0Y=
github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
github.com/replicatedhq/kurlkinds v1.3.6 h1:/dhS32cSSZR4yS4vA8EquBvz+VgJCyTqBO9Xw+6eI4M=
@@ -1856,6 +1855,7 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -1872,8 +1872,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -2659,8 +2660,8 @@ k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY=
k8s.io/apiextensions-apiserver v0.17.0/go.mod h1:XiIFUakZywkUl54fVXa7QTEHcqQz9HG55nHd1DCoHj8=
k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08=
k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc=
k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
k8s.io/apimachinery v0.16.8/go.mod h1:Xk2vD2TRRpuWYLQNM6lT9R7DSFZUYG03SarNkbGrnKE=
k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
@@ -2672,8 +2673,8 @@ k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mg
k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg=
k8s.io/apiserver v0.19.12/go.mod h1:ldZAZTNIKfMMv/UUEhk6UyTXC0/34iRdNFHo+MJOPc4=
k8s.io/apiserver v0.28.3 h1:8Ov47O1cMyeDzTXz0rwcfIIGAP/dP7L8rWbEljRcg5w=
k8s.io/apiserver v0.28.3/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM=
k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk=
k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA=
k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk=
@@ -2695,8 +2696,8 @@ k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3
k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
k8s.io/component-base v0.19.12/go.mod h1:tpwExE0sY3A7CwtlxGL7SnQOdQfUlnFybT6GmAD+z/s=
k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0=
k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI=
k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8=
k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
2 changes: 2 additions & 0 deletions migrations/tables/app_status.yaml
Original file line number Diff line number Diff line change
@@ -21,3 +21,5 @@ spec:
type: integer
- name: sequence
type: integer
- name: embeddedcluster_state
type: text
2 changes: 2 additions & 0 deletions migrations/tables/app_version.yaml
Original file line number Diff line number Diff line change
@@ -71,3 +71,5 @@ spec:
type: text
- name: branding_archive
type: text
- name: embeddedcluster_config
type: text
6 changes: 6 additions & 0 deletions pkg/apiserver/server.go
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import (
"github.com/gorilla/mux"
"github.com/replicatedhq/kots/pkg/automation"
"github.com/replicatedhq/kots/pkg/binaries"
"github.com/replicatedhq/kots/pkg/embeddedcluster"
"github.com/replicatedhq/kots/pkg/handlers"
"github.com/replicatedhq/kots/pkg/helm"
identitymigrate "github.com/replicatedhq/kots/pkg/identity/migrate"
@@ -127,6 +128,11 @@ func Start(params *APIServerParams) {

supportbundle.StartServer()

// start the embedded cluster state monitor. moves on in case of failures but logs them.
if err := embeddedcluster.StartInstallationMonitor(context.Background()); err != nil {
log.Println("failed to start embedded cluster installation monitor:", err)
}

if err := informers.Start(); err != nil {
log.Println("Failed to start informers:", err)
}
138 changes: 138 additions & 0 deletions pkg/embeddedcluster/monitor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package embeddedcluster

import (
"context"
"fmt"
"os"
"time"

"github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
apptypes "github.com/replicatedhq/kots/pkg/app/types"
statetypes "github.com/replicatedhq/kots/pkg/appstate/types"
"github.com/replicatedhq/kots/pkg/k8sutil"
"github.com/replicatedhq/kots/pkg/logger"
"github.com/replicatedhq/kots/pkg/store"
storetypes "github.com/replicatedhq/kots/pkg/store/types"
"k8s.io/client-go/kubernetes"
)

// StartInstallationMonitor starts a goroutine that monitors the embedded cluster installation
// and starts the upgrade process if necessary.
func StartInstallationMonitor(ctx context.Context) error {
clientset, err := k8sutil.GetClientset()
if err != nil {
return fmt.Errorf("failed to get kubeclient: %w", err)
}
if isembedded, err := IsEmbeddedCluster(clientset); err != nil {
return fmt.Errorf("failed to check if embedded: %w", err)
} else if !isembedded {
return nil
}
mon := monitor{
store: store.GetStore(),
logger: logger.NewCLILogger(os.Stdout),
clientset: clientset,
}
go mon.start(ctx)
return nil
}

// monitor is a struct that groups all methods needed to monitor the embedded cluster installation.
type monitor struct {
store store.Store
logger *logger.CLILogger
clientset *kubernetes.Clientset
}

// getApp returns the app deployed on top of the embedded cluster.
func (m *monitor) getApp(ctx context.Context) (*apptypes.App, error) {
apps, err := m.store.ListInstalledApps()
if err != nil {
return nil, fmt.Errorf("failed to list installed apps: %w", err)
} else if len(apps) == 0 {
return nil, nil
}
return apps[0], nil
}

// maybeStartUpgrade checks if the embedded cluster is in a state that requires an upgrade. If so,
// it starts the upgrade process. We only start an upgrade if the following conditions are met:
// - We have an app deployed on top of the embedded cluster.
// - The deployed app version is in ready state.
// - The app has an embedded cluster configuration.
// - The app embedded cluster configuration differs from the current embedded cluster config.
func (m *monitor) maybeStartUpgrade(ctx context.Context) error {
app, err := m.getApp(ctx)
if err != nil {
return fmt.Errorf("failed to get app: %w", err)
} else if app == nil {
return nil
}
cid, err := m.store.GetClusterIDFromSlug(app.Slug)
if err != nil {
return fmt.Errorf("failed to get cluster id: %w", err)
}
version, err := m.store.GetCurrentDownstreamVersion(app.ID, cid)
if err != nil {
return fmt.Errorf("failed to get downstream version: %w", err)
}
kinds := version.KOTSKinds
notDeployed := version.Status != storetypes.VersionDeployed
noClusterConfig := kinds == nil || kinds.EmbeddedClusterConfig == nil
if notDeployed || noClusterConfig {
return nil
}
status, err := m.store.GetAppStatus(app.ID)
if err != nil {
return fmt.Errorf("failed to get app status: %w", err)
}
if statetypes.GetState(status.ResourceStates) != statetypes.StateReady {
return nil
}
spec := kinds.EmbeddedClusterConfig.Spec
if upgrade, err := RequiresUpgrade(ctx, spec); err != nil {
return fmt.Errorf("failed to check if upgrade is required: %w", err)
} else if !upgrade {
return nil
}
if err := StartClusterUpgrade(ctx, spec); err != nil {
return fmt.Errorf("failed to start cluster upgrade: %w", err)
}
return nil
}

// updateClusterState updates the cluster state in the database. Gets the state from the cluster
// by reading the latest embedded cluster installation CRD.
func (m *monitor) updateClusterState(ctx context.Context) error {
installation, err := GetCurrentInstallation(ctx)
if err != nil {
return fmt.Errorf("failed to get current installation: %w", err)
}
state := v1beta1.InstallationStateUnknown
if installation.Status.State != "" {
state = installation.Status.State
}
if err := m.store.SetEmbeddedClusterState(state); err != nil {
return fmt.Errorf("failed to update embedded cluster state: %w", err)
}
return nil
}

// start starts the monitor loop. Only returns when the context is cancelled. We first update
// the cluster state and later maybe start an upgrade. We sleep for 5 seconds between each
// iteration.
func (m *monitor) start(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-time.After(time.Second * 5):
}
if err := m.updateClusterState(ctx); err != nil {
m.logger.Errorf("unable to update cluster state: %v", err)
}
if err := m.maybeStartUpgrade(ctx); err != nil {
m.logger.Errorf("unable to start cluster upgrade: %v", err)
}
}
}
96 changes: 85 additions & 11 deletions pkg/embeddedcluster/util.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package embeddedcluster

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"sort"
"time"

embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
"github.com/replicatedhq/kots/pkg/k8sutil"
@@ -13,11 +17,18 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

const configMapName = "embedded-cluster-config"
const configMapNamespace = "embedded-cluster"

func init() {
k8slogger := zap.New(func(o *zap.Options) { o.DestWriter = io.Discard })
log.SetLogger(k8slogger)
}

// ReadConfigMap will read the Kurl config from a configmap
func ReadConfigMap(client kubernetes.Interface) (*corev1.ConfigMap, error) {
return client.CoreV1().ConfigMaps(configMapNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
@@ -58,34 +69,97 @@ func ClusterID(client kubernetes.Interface) (string, error) {
return configMap.Data["embedded-cluster-id"], nil
}

// ClusterConfig will get the list of installations, find the latest installation, and get that installation's config
func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, error) {
// RequiresUpgrade returns true if the provided configuration differs from the latest active configuration.
func RequiresUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec) (bool, error) {
curcfg, err := ClusterConfig(ctx)
if err != nil {
return false, fmt.Errorf("failed to get current cluster config: %w", err)
}
serializedCur, err := json.Marshal(curcfg)
if err != nil {
return false, err
}
serializedNew, err := json.Marshal(newcfg)
if err != nil {
return false, err
}
return !bytes.Equal(serializedCur, serializedNew), nil
}

// GetCurrentInstallation returns the most recent installation object from the cluster.
func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Installation, error) {
clientConfig, err := k8sutil.GetClusterConfig()
if err != nil {
return nil, fmt.Errorf("failed to get cluster config: %w", err)
}

scheme := runtime.NewScheme()
embeddedclusterv1beta1.AddToScheme(scheme)

kbClient, err := kbclient.New(clientConfig, kbclient.Options{
Scheme: scheme,
})
kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme})
if err != nil {
return nil, fmt.Errorf("failed to get kubebuilder client: %w", err)
}

var installationList embeddedclusterv1beta1.InstallationList
err = kbClient.List(ctx, &installationList, &kbclient.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list installations: %w", err)
}
if len(installationList.Items) == 0 {
return nil, fmt.Errorf("no installations found")
}
items := installationList.Items
sort.SliceStable(items, func(i, j int) bool {
return items[j].CreationTimestamp.Before(&items[i].CreationTimestamp)
})
return &installationList.Items[0], nil
}

// ClusterConfig will extract the current cluster configuration from the latest installation
// object found in the cluster.
func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, error) {
latest, err := GetCurrentInstallation(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get current installation: %w", err)
}
return latest.Spec.Config, nil
}

// determine which of these installations is the latest
// StartClusterUpgrade will create a new installation with the provided config.
func StartClusterUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec) error {
clientConfig, err := k8sutil.GetClusterConfig()
if err != nil {
return fmt.Errorf("failed to get cluster config: %w", err)
}
scheme := runtime.NewScheme()
embeddedclusterv1beta1.AddToScheme(scheme)
kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme})
if err != nil {
return fmt.Errorf("failed to get kubebuilder client: %w", err)
}
var installationList embeddedclusterv1beta1.InstallationList
err = kbClient.List(ctx, &installationList, &kbclient.ListOptions{})
if err != nil {
return fmt.Errorf("failed to list installations: %w", err)
}
sort.Slice(installationList.Items, func(i, j int) bool {
return installationList.Items[i].ObjectMeta.CreationTimestamp.After(installationList.Items[j].ObjectMeta.CreationTimestamp.Time)
})

if len(installationList.Items) == 0 {
return fmt.Errorf("no installations found")
}
latest := installationList.Items[0]
return latest.Spec.Config, nil
newins := embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
ClusterID: latest.Spec.ClusterID,
MetricsBaseURL: latest.Spec.MetricsBaseURL,
AirGap: latest.Spec.AirGap,
Config: &newcfg,
},
}
if err := kbClient.Create(ctx, &newins); err != nil {
return fmt.Errorf("failed to create installation: %w", err)
}
return nil
}
33 changes: 33 additions & 0 deletions pkg/kotsutil/kots.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ import (

"github.com/blang/semver"
"github.com/pkg/errors"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
"github.com/replicatedhq/kots/pkg/archives"
"github.com/replicatedhq/kots/pkg/binaries"
"github.com/replicatedhq/kots/pkg/buildversion"
@@ -51,6 +52,7 @@ func init() {
velerov1.AddToScheme(scheme.Scheme)
kurlscheme.AddToScheme(scheme.Scheme)
applicationv1beta1.AddToScheme(scheme.Scheme)
embeddedclusterv1beta1.AddToScheme(scheme.Scheme)
}

var (
@@ -105,6 +107,8 @@ type KotsKinds struct {
Installer *kurlv1beta1.Installer

LintConfig *kotsv1beta1.LintConfig

EmbeddedClusterConfig *embeddedclusterv1beta1.Config
}

func IsKotsKind(apiVersion string, kind string) bool {
@@ -129,6 +133,10 @@ func IsKotsKind(apiVersion string, kind string) bool {
if apiVersion == "kurl.sh/v1beta1" {
return true
}
// In addition to kotskinds, we exclude the embedded cluster configuration.
if apiVersion == "embeddedcluster.replicated.com/v1beta1" {
return true
}
// In addition to kotskinds, we exclude the application crd for now
if apiVersion == "app.k8s.io/v1beta1" {
return true
@@ -448,6 +456,17 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) {
}
}

if g == "embeddedcluster.replicated.com" && v == "v1beta1" && k == "Config" {
if o.EmbeddedClusterConfig == nil {
return "", nil
}
var b bytes.Buffer
if err := s.Encode(o.EmbeddedClusterConfig, &b); err != nil {
return "", errors.Wrap(err, "failed to encode embedded cluster config")
}
return string(b.Bytes()), nil
}

return "", errors.Errorf("unknown gvk %s/%s, Kind=%s", g, v, k)
}

@@ -528,6 +547,8 @@ func (k *KotsKinds) addKotsKinds(content []byte) error {
k.Installer = decoded.(*kurlv1beta1.Installer)
case "app.k8s.io/v1beta1, Kind=Application":
k.Application = decoded.(*applicationv1beta1.Application)
case "embeddedcluster.replicated.com/v1beta1, Kind=Config":
k.EmbeddedClusterConfig = decoded.(*embeddedclusterv1beta1.Config)
}
}

@@ -913,6 +934,18 @@ func LoadLicenseFromBytes(data []byte) (*kotsv1beta1.License, error) {
return obj.(*kotsv1beta1.License), nil
}

func LoadEmbeddedClusterConfigFromBytes(data []byte) (*embeddedclusterv1beta1.Config, error) {
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, gvk, err := decode([]byte(data), nil, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to decode embedded cluster config data")
}
if gvk.Group != "embeddedcluster.replicated.com" || gvk.Version != "v1beta1" || gvk.Kind != "Config" {
return nil, errors.Errorf("unexpected GVK: %s", gvk.String())
}
return obj.(*embeddedclusterv1beta1.Config), nil
}

func LoadConfigValuesFromFile(configValuesFilePath string) (*kotsv1beta1.ConfigValues, error) {
configValuesData, err := ioutil.ReadFile(configValuesFilePath)
if err != nil {
13 changes: 13 additions & 0 deletions pkg/store/kotsstore/appstatus_store.go
Original file line number Diff line number Diff line change
@@ -86,3 +86,16 @@ func (s *KOTSStore) SetAppStatus(appID string, resourceStates appstatetypes.Reso

return nil
}

func (s *KOTSStore) SetEmbeddedClusterState(state string) error {
db := persistence.MustGetDBSession()
query := "update app_status set embeddedcluster_state = ?"
wr, err := db.WriteOneParameterized(gorqlite.ParameterizedStatement{
Query: query,
Arguments: []interface{}{state},
})
if err != nil {
return fmt.Errorf("failed to write: %v: %v", err, wr.Err)
}
return nil
}
28 changes: 23 additions & 5 deletions pkg/store/kotsstore/version_store.go
Original file line number Diff line number Diff line change
@@ -701,15 +701,20 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6
return nil, errors.Wrap(err, "failed to marshal configvalues spec")
}

embeddedClusterConfig, err := kotsKinds.Marshal("embeddedcluster.replicated.com", "v1beta1", "Config")
if err != nil {
return nil, errors.Wrap(err, "failed to marshal configvalues spec")
}

var releasedAt *int64
if kotsKinds.Installation.Spec.ReleasedAt != nil {
t := kotsKinds.Installation.Spec.ReleasedAt.Time.Unix()
releasedAt = &t
}

query := `insert into app_version (app_id, sequence, created_at, version_label, is_required, release_notes, update_cursor, channel_id, channel_name, upstream_released_at, encryption_key,
supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, identity_spec, branding_archive)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, identity_spec, branding_archive, embeddedcluster_config)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(app_id, sequence) DO UPDATE SET
created_at = EXCLUDED.created_at,
version_label = EXCLUDED.version_label,
@@ -731,7 +736,8 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6
config_values = EXCLUDED.config_values,
backup_spec = EXCLUDED.backup_spec,
identity_spec = EXCLUDED.identity_spec,
branding_archive = EXCLUDED.branding_archive`
branding_archive = EXCLUDED.branding_archive,
embeddedcluster_config = EXCLUDED.embeddedcluster_config`

statements = append(statements, gorqlite.ParameterizedStatement{
Query: query,
@@ -759,6 +765,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6
backupSpec,
identitySpec,
base64.StdEncoding.EncodeToString(brandingArchive),
embeddedClusterConfig,
},
})

@@ -811,7 +818,7 @@ func (s *KOTSStore) upsertAppDownstreamVersionStatements(appID string, clusterID

func (s *KOTSStore) GetAppVersion(appID string, sequence int64) (*versiontypes.AppVersion, error) {
db := persistence.MustGetDBSession()
query := `select app_id, sequence, update_cursor, channel_id, version_label, created_at, status, applied_at, kots_installation_spec, kots_app_spec, kots_license from app_version where app_id = ? and sequence = ?`
query := `select app_id, sequence, update_cursor, channel_id, version_label, created_at, status, applied_at, kots_installation_spec, kots_app_spec, kots_license, embeddedcluster_config from app_version where app_id = ? and sequence = ?`
rows, err := db.QueryOneParameterized(gorqlite.ParameterizedStatement{
Query: query,
Arguments: []interface{}{appID, sequence},
@@ -1086,8 +1093,9 @@ func (s *KOTSStore) appVersionFromRow(row gorqlite.QueryResult) (*versiontypes.A
var updateCursor gorqlite.NullString
var channelID gorqlite.NullString
var versionLabel gorqlite.NullString
var embeddedClusterConfig gorqlite.NullString

if err := row.Scan(&v.AppID, &v.Sequence, &updateCursor, &channelID, &versionLabel, &createdAt, &status, &createdAt, &installationSpec, &kotsAppSpec, &licenseSpec); err != nil {
if err := row.Scan(&v.AppID, &v.Sequence, &updateCursor, &channelID, &versionLabel, &createdAt, &status, &createdAt, &installationSpec, &kotsAppSpec, &licenseSpec, &embeddedClusterConfig); err != nil {
return nil, errors.Wrap(err, "failed to scan")
}

@@ -1127,6 +1135,16 @@ func (s *KOTSStore) appVersionFromRow(row gorqlite.QueryResult) (*versiontypes.A
}
}

if embeddedClusterConfig.Valid && embeddedClusterConfig.String != "" {
config, err := kotsutil.LoadEmbeddedClusterConfigFromBytes([]byte(embeddedClusterConfig.String))
if err != nil {
return nil, errors.Wrap(err, "failed to read embedded cluster config")
}
if config != nil {
v.KOTSKinds.EmbeddedClusterConfig = config
}
}

v.CreatedOn = createdAt.Time
if deployedAt.Valid {
v.DeployedAt = &deployedAt.Time
1 change: 1 addition & 0 deletions pkg/store/store_interface.go
Original file line number Diff line number Diff line change
@@ -116,6 +116,7 @@ type SessionStore interface {
type AppStatusStore interface {
GetAppStatus(appID string) (*appstatetypes.AppStatus, error)
SetAppStatus(appID string, resourceStates appstatetypes.ResourceStates, updatedAt time.Time, sequence int64) error
SetEmbeddedClusterState(state string) error
}

type AppStore interface {

0 comments on commit d7c0bb4

Please sign in to comment.