diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index 51fd05ca..ecb4f598 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -117,7 +117,7 @@ jobs: /usr/local/bin/dnsname functional_test_docker: - name: Functional Test Docker + name: Test Docker runs-on: ubuntu-latest needs: functional_test_build @@ -136,6 +136,7 @@ jobs: './examples/remote_exec', './examples/certificates', './examples/terraform', + './examples/registries', ] steps: @@ -171,7 +172,7 @@ jobs: description: "Functional tests for docker: ${{matrix.folder}}" functional_test_podman: - name: Functional Test Podman + name: Test Podman runs-on: ubuntu-latest needs: functional_test_build @@ -181,11 +182,11 @@ jobs: matrix: folder: [ './examples/container', - # './examples/build', - './examples/docs', - # './examples/single_k3s_cluster', - # './examples/nomad', - # './examples/local_exec', + './examples/build', + './examples/docs', + './examples/nomad', + './examples/single_k3s_cluster', + './examples/multiple_k3s_clusters', ] steps: @@ -260,7 +261,7 @@ jobs: needs: # - test - functional_test_docker - # - functional_test_podman + - functional_test_podman # - e2e_mac - e2e_linux if: ${{ github.ref != 'refs/heads/main' }} @@ -377,7 +378,7 @@ jobs: needs: - test - functional_test_docker - #- functional_test_podman + - functional_test_podman #- e2e_mac - e2e_linux if: ${{ github.ref == 'refs/heads/main' }} diff --git a/ChangeLog.md b/ChangeLog.md index 7d7cfa27..fc2ce31f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,73 @@ # Change Log +## version v0.5.60 +* Add capability to add custom container registries to the image cache + + Nomad and Kuberentes clusters are started in a Docker container that does not save any state to the local disk. + This state includes and Docker Image cache, thefore every time an image is pulled to a new cluster it is downloaded + from the internet. This can be slow and bandwidth intensive. To solve this problem Jumppad implemented a pull through + cache that is used by all clusters. By default this cache supported the following registires: + - k8s.gcr.io + - gcr.io + - asia.gcr.io + - eu.gcr.io + - us.gcr.io + - quay.io + - ghcr.io + - docker.pkg.github.com + + To support custom registries Jumppad has added a new resource type `container_registry`. This resource type can be used + to define either a local or remote registry. When a registry is defined it is added to the pull through cache and + any authnetication details are added to the cache meaning you do not need to authenticate each pull on the Nomad or + Kubernetes cluster. Any defined registry must be configured to use HTTPS, the image cache can not be used to pull + from insecure registries. + +```hcl +# Define a custom registry that does not use authentication +resource "container_registry" "noauth" { + hostname = "noauth-registry.demo.gs" // cache can not resolve local jumppad.dev dns for some reason, + // using external dns mapped to the local ip address +} + +# Define a custom registry that uses authentication +resource "container_registry" "auth" { + hostname = "auth-registry.demo.gs" + auth { + username = "admin" + password = "password" + } +} +``` + +* Add capability to add insecure registries and image cache bypass to Kubernetes and Nomad clusters. + + All images pulled to Nomad and Kubernetes clusters are pulled through the image cache. This cache is a Docker + container that is automatically started by Jumppad. To disable the cache and pull images directly from the internet + you can add the `no_proxy` parameter to the new docker config stanza. This will cause the cache to be bypassed and + the image to be pulled direct from the internet. + + To support insecure registries you can add the `insecure_registries` parameter to the docker config stanza. This + must be used in conjunction with the `no_proxy` parameter as the image cache does not support insecure registries. + +```hcl +resource "nomad_cluster" "dev" { + client_nodes = 1 + + datacenter = "dc1" + + network { + id = variable.network_id + } + + // add configuration to allow cache bypass and insecure registry + config { + docker { + no_proxy = ["insecure.container.jumppad.dev"] + insecure_registries = ["insecure.container.jumppad.dev:5003"] + } + } +} +``` ## version v0.5.47 * Fix isuse where filepath.Walk does not respect symlinks * Add `ignore` parameter to `build` resource to allow ignoring of files and folders diff --git a/Makefile b/Makefile index b564b597..fab7a72d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ git_commit = $(shell git log -1 --pretty=format:"%H") test_unit: - go test -v -race ./pkg/config/resources/container + go test -v -race ./... test_functional: go run main.go purge @@ -21,6 +21,21 @@ test_functional: go run main.go purge go run main.go test ./examples/multiple_k3s_clusters + + go run main.go purge + go run main.go test ./examples/local_exec + + go run main.go purge + go run main.go test ./examples/remote_exec + + go run main.go purge + go run main.go test ./examples/certificates + + go run main.go purge + go run main.go test ./examples/terraform + + go run main.go purge + go run main.go test ./examples/registiries test_e2e_cmd: install_local jumppad up --no-browser ./examples/single_k3s_cluster diff --git a/daemon.json b/daemon.json new file mode 100755 index 00000000..13a85a75 --- /dev/null +++ b/daemon.json @@ -0,0 +1,10 @@ +{ + "proxies": { + "http-proxy": "http://default.image-cache.jumppad.dev:3128", + "https-proxy": "http://default.image-cache.jumppad.dev:3128", + "no-proxy": "insecure.container.jumppad.dev" + }, + "insecure-registries": [ + "insecure.container.jumppad.dev:5003" + ] +} \ No newline at end of file diff --git a/examples/build/build.hcl b/examples/build/build.hcl index 7ba65d53..29208bdf 100644 --- a/examples/build/build.hcl +++ b/examples/build/build.hcl @@ -47,7 +47,7 @@ resource "network" "onprem" { module "container" { disabled = !variable.container_enabled - source = "./container" + source = "${dir()}/container" variables = { image = resource.build.app.image @@ -58,7 +58,7 @@ module "container" { module "nomad" { disabled = !variable.nomad_enabled - source = "./nomad" + source = "${dir()}/nomad" variables = { image = resource.build.app.image @@ -69,7 +69,7 @@ module "nomad" { module "kubernetes" { disabled = !variable.kubernetes_enabled - source = "./kubernetes" + source = "${dir()}/kubernetes" variables = { image = resource.build.app.image diff --git a/examples/container/container.hcl b/examples/container/container.hcl index 9b6e5b1d..6f26ac75 100644 --- a/examples/container/container.hcl +++ b/examples/container/container.hcl @@ -7,7 +7,6 @@ variable "envoy_version" { } resource "template" "consul_config" { - source = <<-EOF data_dir = "{{ data_dir }}" log_level = "DEBUG" @@ -55,9 +54,11 @@ resource "container" "consul_capabilities" { } capabilities { - add = ["NET_ADMIN"] + add = ["NET_ADMIN"] drop = ["ALL"] } + + privileged = true } resource "container" "consul_labels" { diff --git a/examples/container/test/container.feature b/examples/container/test/container.feature index 680b6c75..0e56f0ac 100644 --- a/examples/container/test/container.feature +++ b/examples/container/test/container.feature @@ -3,19 +3,21 @@ Feature: Docker Container I should apply a blueprint which defines a simple container setup and test the resources are created correctly -Scenario: Single Container from Local Blueprint +Scenario: Docker Containers from Local Blueprint Given I have a running blueprint Then the following resources should be running | name | | resource.network.consul | | resource.container.consul | + | resource.container.consul_labels | + | resource.container.consul_capabilities | | resource.sidecar.envoy | And the info "{.NetworkSettings.Ports['8501/tcp']}" for the running container "resource.container.consul" should exist And the info "{.NetworkSettings.Ports['8500/tcp'][0].HostPort}" for the running container "resource.container.consul" should equal "8500" And the info "{.NetworkSettings.Ports['8500/tcp'][0].HostPort}" for the running container "resource.container.consul" should contain "85" And a HTTP call to "http://consul.container.shipyard.run:8500/v1/status/leader" should result in status 200 -Scenario: Single Container from Local Blueprint with multiple runs +Scenario: Docker Containers from Local Blueprint with multiple runs Given the environment variable "CONSUL_VERSION" has a value "" And the environment variable "ENVOY_VERSION" has a value "" And I have a running blueprint @@ -23,6 +25,8 @@ Scenario: Single Container from Local Blueprint with multiple runs | name | | resource.network.consul | | resource.container.consul | + | resource.container.consul_labels | + | resource.container.consul_capabilities | | resource.sidecar.envoy | And a HTTP call to "http://consul.container.shipyard.run:8500/v1/status/leader" should result in status 200 And the response body should contain "10.6.0.200" diff --git a/examples/container/test/vars.feature b/examples/container/test/vars.feature index ff7df136..08d5a964 100644 --- a/examples/container/test/vars.feature +++ b/examples/container/test/vars.feature @@ -15,6 +15,8 @@ Scenario: Single Container with jumppad Variables | name | | resource.network.consul | | resource.container.consul | + | resource.container.consul_capabilities | + | resource.container.consul_labels | | resource.sidecar.envoy | And the info "{.Config.Env}" for the running container "resource.container.consul" should contain "something=set by test" And the info "{.Config.Env}" for the running container "resource.container.consul" should contain "foo=bah" \ No newline at end of file diff --git a/examples/nomad/nomad.hcl b/examples/nomad/nomad.hcl index 2e48d446..5d94e546 100644 --- a/examples/nomad/nomad.hcl +++ b/examples/nomad/nomad.hcl @@ -39,11 +39,6 @@ resource "nomad_cluster" "dev" { copy_image { name = "consul:1.10.1" } - - volume { - source = "/tmp" - destination = "/files" - } } resource "template" "example_2" { diff --git a/examples/registries/build.hcl b/examples/registries/build.hcl new file mode 100644 index 00000000..fd1eb8fe --- /dev/null +++ b/examples/registries/build.hcl @@ -0,0 +1,141 @@ + +resource "certificate_leaf" "registry" { + ca_key = "${jumppad()}/certs/root.key" + ca_cert = "${jumppad()}/certs/root.cert" + + ip_addresses = ["127.0.0.1", variable.auth_ip_address, variable.noauth_ip_address] + + dns_names = [ + "localhost", + "auth-registry.demo.gs", + "noauth-registry.demo.gs", // have to set an external dns name as the registry resolves docker dns to localhost + "noauth.container.jumppad.dev", + "auth.container.jumppad.dev", + ] + + output = data("certs") +} + +resource "container" "noauth" { + image { + name = "registry:2" + } + + network { + id = resource.network.cloud.id + ip_address = variable.noauth_ip_address + } + + port { + local = 443 + host = 5000 + } + + environment = { + DEBUG = "true" + REGISTRY_HTTP_ADDR = "0.0.0.0:443" + REGISTRY_HTTP_TLS_CERTIFICATE = "/certs/registry-leaf.cert" + REGISTRY_HTTP_TLS_KEY = "/certs/registry-leaf.key" + } + + volume { + source = data("certs") + destination = "/certs" + } +} + +resource "container" "auth" { + image { + name = "registry:2" + } + + network { + id = resource.network.cloud.id + ip_address = variable.auth_ip_address + } + + port { + local = 443 + host = 5001 + } + + environment = { + DEBUG = "true" + REGISTRY_HTTP_ADDR = "0.0.0.0:443" + REGISTRY_AUTH = "htpasswd" + REGISTRY_AUTH_HTPASSWD_REALM = "Registry Realm" + REGISTRY_AUTH_HTPASSWD_PATH = "/etc/auth/htpasswd" + REGISTRY_HTTP_TLS_CERTIFICATE = "/certs/registry-leaf.cert" + REGISTRY_HTTP_TLS_KEY = "/certs/registry-leaf.key" + } + + volume { + source = "./files/htpasswd" + destination = "/etc/auth/htpasswd" + } + + volume { + source = data("certs") + destination = "/certs" + } +} + +resource "container" "insecure" { + image { + name = "registry:2" + } + + network { + id = resource.network.cloud.id + ip_address = variable.insecure_ip_address + } + + port { + local = 5003 + host = 5003 + } + + environment = { + DEBUG = "true" + REGISTRY_HTTP_ADDR = "0.0.0.0:5003" + } +} + +resource "build" "app" { + container { + dockerfile = "./Docker/Dockerfile" + context = "./src" + ignore = ["**/.terraform"] + } + + // push to the unauthenticated registry + registry { + name = "${resource.container.noauth.container_name}:5000/mine:v0.1.0" + } + + // push to the authenticated registry + registry { + name = "${resource.container.auth.container_name}:5001/mine:v0.1.0" + username = "admin" + password = "password" + } + + // push to the insecure registry + registry { + name = "${resource.container.insecure.container_name}:5003/mine:v0.1.0" + } +} + +# Define a custom registry that will be added to the image cache +resource "container_registry" "noauth" { + hostname = "noauth-registry.demo.gs" // cache can not resolve local jumppad.dev dns for some reason, + // using external dns mapped to the local ip address +} + +resource "container_registry" "auth" { + hostname = "auth-registry.demo.gs" + auth { + username = "admin" + password = "password" + } +} diff --git a/examples/registries/files/htpasswd b/examples/registries/files/htpasswd new file mode 100644 index 00000000..75fa3875 --- /dev/null +++ b/examples/registries/files/htpasswd @@ -0,0 +1,2 @@ +admin:$2y$05$ggnnaO6F8m9tzLF2IMv96.RBe.CY7rYnjzB1NTpX4tr/RzOa1wSE6 + diff --git a/examples/registries/k8s/files/auth.yaml b/examples/registries/k8s/files/auth.yaml new file mode 100644 index 00000000..7066fc45 --- /dev/null +++ b/examples/registries/k8s/files/auth.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth + labels: + app: auth +spec: + replicas: 1 + selector: + matchLabels: + app: auth + template: + metadata: + labels: + app: auth + spec: + containers: + - name: auth + image: "auth-registry.demo.gs/mine:v0.1.0" + ports: + - containerPort: 19091 + env: + - name: LISTEN_ADDR + value: ":19091" + - name: MESSAGE + value: "Registry With Auth" + +--- +apiVersion: v1 +kind: Service +metadata: + name: auth +spec: + selector: + app: auth + ports: + - protocol: TCP + port: 19091 + targetPort: 19091 \ No newline at end of file diff --git a/examples/registries/k8s/files/insecure.yaml b/examples/registries/k8s/files/insecure.yaml new file mode 100644 index 00000000..958e1ccc --- /dev/null +++ b/examples/registries/k8s/files/insecure.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: insecure + labels: + app: insecure +spec: + replicas: 1 + selector: + matchLabels: + app: insecure + template: + metadata: + labels: + app: insecure + spec: + containers: + - name: insecure + image: "insecure.container.jumppad.dev:5003/mine:v0.1.0" + ports: + - containerPort: 19092 + env: + - name: LISTEN_ADDR + value: ":19092" + - name: MESSAGE + value: "Registry Insecure" + +--- +apiVersion: v1 +kind: Service +metadata: + name: insecure +spec: + selector: + app: insecure + ports: + - protocol: TCP + port: 19092 + targetPort: 19092 \ No newline at end of file diff --git a/examples/registries/k8s/files/noauth.yaml b/examples/registries/k8s/files/noauth.yaml new file mode 100644 index 00000000..5a84a0bf --- /dev/null +++ b/examples/registries/k8s/files/noauth.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: noauth + labels: + app: noauth +spec: + replicas: 1 + selector: + matchLabels: + app: noauth + template: + metadata: + labels: + app: noauth + spec: + containers: + - name: noauth + image: "noauth-registry.demo.gs/mine:v0.1.0" + ports: + - containerPort: 19090 + env: + - name: LISTEN_ADDR + value: ":19090" + - name: MESSAGE + value: "Registry With No Auth" + +--- +apiVersion: v1 +kind: Service +metadata: + name: noauth +spec: + selector: + app: noauth + ports: + - protocol: TCP + port: 19090 + targetPort: 19090 \ No newline at end of file diff --git a/examples/registries/k8s/k8s.hcl b/examples/registries/k8s/k8s.hcl new file mode 100644 index 00000000..d16bba57 --- /dev/null +++ b/examples/registries/k8s/k8s.hcl @@ -0,0 +1,105 @@ +variable "network_id" { + default = "" +} + +resource "k8s_cluster" "k3s" { + network { + id = variable.network_id + } + + // add configuration to allow cache bypass and insecure registry + config { + docker { + no_proxy = ["insecure.container.jumppad.dev"] + insecure_registries = ["insecure.container.jumppad.dev:5003"] + } + } +} + +resource "k8s_config" "noauth" { + cluster = resource.k8s_cluster.k3s + + paths = [ + "./files/noauth.yaml", + ] + + wait_until_ready = true +} + +resource "k8s_config" "auth" { + cluster = resource.k8s_cluster.k3s + + paths = [ + "./files/auth.yaml", + ] + + wait_until_ready = true +} + +resource "k8s_config" "insecure" { + cluster = resource.k8s_cluster.k3s + + paths = [ + "./files/insecure.yaml", + ] + + wait_until_ready = true +} + +resource "ingress" "k8s_noauth" { + port = 29090 + + target { + resource = resource.k8s_cluster.k3s + port = 19090 + + config = { + service = "noauth" + namespace = "default" + } + } +} + +resource "ingress" "k8s_auth" { + port = 29091 + + target { + resource = resource.k8s_cluster.k3s + port = 19091 + + config = { + service = "auth" + namespace = "default" + } + } +} + +resource "ingress" "k8s_insecure" { + port = 29092 + + target { + resource = resource.k8s_cluster.k3s + port = 19092 + + config = { + service = "insecure" + namespace = "default" + } + } +} + +output "k8s_noauth_addr" { + value = resource.ingress.k8s_noauth.local_address +} + +output "k8s_auth_addr" { + value = resource.ingress.k8s_auth.local_address +} + +output "k8s_insecure_addr" { + value = resource.ingress.k8s_insecure.local_address +} + +output "KUBECONFIG" { + value = resource.k8s_cluster.k3s.kubeconfig +} \ No newline at end of file diff --git a/examples/registries/main.hcl b/examples/registries/main.hcl new file mode 100644 index 00000000..7aab6458 --- /dev/null +++ b/examples/registries/main.hcl @@ -0,0 +1,31 @@ +resource "network" "cloud" { + subnet = "10.6.0.0/16" +} + +module "nomad" { + disabled = !variable.nomad_enabled + + depends_on = ["resource.build.app"] + source = "./nomad" + + variables = { + network_id = resource.network.cloud.id + } +} + +module "k8s" { + disabled = !variable.k8s_enabled + + depends_on = ["resource.build.app"] + source = "./k8s" + + variables = { + network_id = resource.network.cloud.id + } +} + +output "KUBECONFIG" { + disabled = !variable.k8s_enabled + + value = module.k8s.output.KUBECONFIG +} \ No newline at end of file diff --git a/examples/registries/nomad/files/auth.nomad b/examples/registries/nomad/files/auth.nomad new file mode 100644 index 00000000..b3ddadda --- /dev/null +++ b/examples/registries/nomad/files/auth.nomad @@ -0,0 +1,46 @@ +job "auth" { + datacenters = ["dc1"] + type = "service" + + group "app" { + count = 1 + + network { + port "http" { + to = 19091 + static = 19091 + } + } + + ephemeral_disk { + size = 30 + } + + task "app" { + # The "driver" parameter specifies the task driver that should be used to + # run the task. + driver = "docker" + + logs { + max_files = 2 + max_file_size = 10 + } + + env { + LISTEN_ADDR = ":19091" + MESSAGE = "Registry With Auth" + } + + config { + image = "auth-registry.demo.gs/mine:v0.1.0" + + ports = ["http"] + } + + resources { + cpu = 500 # 500 MHz + memory = 256 # 256MB + } + } + } +} \ No newline at end of file diff --git a/examples/registries/nomad/files/insecure.nomad b/examples/registries/nomad/files/insecure.nomad new file mode 100644 index 00000000..71c32ed7 --- /dev/null +++ b/examples/registries/nomad/files/insecure.nomad @@ -0,0 +1,46 @@ +job "insecure" { + datacenters = ["dc1"] + type = "service" + + group "app" { + count = 1 + + network { + port "http" { + to = 19092 + static = 19092 + } + } + + ephemeral_disk { + size = 30 + } + + task "app" { + # The "driver" parameter specifies the task driver that should be used to + # run the task. + driver = "docker" + + logs { + max_files = 2 + max_file_size = 10 + } + + env { + LISTEN_ADDR = ":19092" + MESSAGE = "Registry Insecure" + } + + config { + image = "insecure.container.jumppad.dev:5003/mine:v0.1.0" + + ports = ["http"] + } + + resources { + cpu = 500 # 500 MHz + memory = 256 # 256MB + } + } + } +} \ No newline at end of file diff --git a/examples/registries/nomad/files/noauth.nomad b/examples/registries/nomad/files/noauth.nomad new file mode 100644 index 00000000..6ea72352 --- /dev/null +++ b/examples/registries/nomad/files/noauth.nomad @@ -0,0 +1,46 @@ +job "noauth" { + datacenters = ["dc1"] + type = "service" + + group "app" { + count = 1 + + network { + port "http" { + to = 19090 + static = 19090 + } + } + + ephemeral_disk { + size = 30 + } + + task "app" { + # The "driver" parameter specifies the task driver that should be used to + # run the task. + driver = "docker" + + logs { + max_files = 2 + max_file_size = 10 + } + + env { + LISTEN_ADDR = ":19090" + MESSAGE = "Registry No Auth" + } + + config { + image = "noauth-registry.demo.gs/mine:v0.1.0" + + ports = ["http"] + } + + resources { + cpu = 500 # 500 MHz + memory = 256 # 256MB + } + } + } +} \ No newline at end of file diff --git a/examples/registries/nomad/nomad.hcl b/examples/registries/nomad/nomad.hcl new file mode 100644 index 00000000..01899403 --- /dev/null +++ b/examples/registries/nomad/nomad.hcl @@ -0,0 +1,111 @@ +variable "network_id" { + default = "" +} + +resource "nomad_cluster" "dev" { + client_nodes = 1 + + datacenter = "dc1" + + network { + id = variable.network_id + } + + // add configuration to allow cache bypass and insecure registry + config { + docker { + no_proxy = ["insecure.container.jumppad.dev"] + insecure_registries = ["insecure.container.jumppad.dev:5003"] + } + } +} + +resource "nomad_job" "noauth" { + cluster = resource.nomad_cluster.dev + + paths = ["./files/noauth.nomad"] + + health_check { + timeout = "60s" + jobs = ["noauth"] + } +} + +resource "nomad_job" "auth" { + cluster = resource.nomad_cluster.dev + + paths = ["./files/auth.nomad"] + + health_check { + timeout = "60s" + jobs = ["auth"] + } +} + +resource "nomad_job" "insecure" { + cluster = resource.nomad_cluster.dev + + paths = ["./files/insecure.nomad"] + + health_check { + timeout = "60s" + jobs = ["insecure"] + } +} + +resource "ingress" "nomad_noauth" { + port = 19090 + + target { + resource = resource.nomad_cluster.dev + named_port = "http" + + config = { + job = "noauth" + group = "app" + task = "app" + } + } +} + +resource "ingress" "nomad_auth" { + port = 19091 + + target { + resource = resource.nomad_cluster.dev + named_port = "http" + + config = { + job = "auth" + group = "app" + task = "app" + } + } +} + +resource "ingress" "nomad_insecure" { + port = 19092 + + target { + resource = resource.nomad_cluster.dev + named_port = "http" + + config = { + job = "insecure" + group = "app" + task = "app" + } + } +} + +output "nomad_noauth_addr" { + value = resource.ingress.nomad_noauth.local_address +} + +output "nomad_auth_addr" { + value = resource.ingress.nomad_auth.local_address +} + +output "nomad_insecure_addr" { + value = resource.ingress.nomad_insecure.local_address +} \ No newline at end of file diff --git a/examples/registries/src/Docker/Dockerfile b/examples/registries/src/Docker/Dockerfile new file mode 100644 index 00000000..f29ea562 --- /dev/null +++ b/examples/registries/src/Docker/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:latest as build + +WORKDIR /go/src/build + +COPY . /go/src/build + +RUN CGO_ENABLED=0 go build -o /bin/app main.go + +FROM alpine:latest + +COPY --from=build /bin/app /bin/app + +CMD /bin/app \ No newline at end of file diff --git a/examples/registries/src/main.go b/examples/registries/src/main.go new file mode 100644 index 00000000..fcc5a7bc --- /dev/null +++ b/examples/registries/src/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" +) + +func main() { + // get the upstream url if present + upstream_url := os.Getenv("UPSTREAM_URL") + + listen_addr := ":9090" + if os.Getenv("LISTEN_ADDR") != "" { + listen_addr = os.Getenv("LISTEN_ADDR") + } + + message := "Hello world" + if os.Getenv("MESSAGE") != "" { + message = os.Getenv("MESSAGE") + } + + http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { + fmt.Fprintf(rw, "%s\n", message) + + if upstream_url != "" { + resp, err := http.Get(upstream_url) + if err != nil { + http.Error(rw, fmt.Sprintf("unable to contact upstream: %s", err), http.StatusInternalServerError) + return + } + + b, _ := ioutil.ReadAll(resp.Body) + fmt.Fprintf(rw, "Response from upstream: %s", string(b)) + } + }) + + fmt.Println(http.ListenAndServe(listen_addr, nil)) +} diff --git a/examples/registries/test/registries.feature b/examples/registries/test/registries.feature new file mode 100644 index 00000000..993b60ab --- /dev/null +++ b/examples/registries/test/registries.feature @@ -0,0 +1,35 @@ + +Feature: Custom Docker Registries + In order to test custom docker registies with Nomad and Kubernetes + I should apply a blueprint + And test the output + + @nomad + Scenario: Nomad Cluster + Given the following jumppad variables are set + | key | value | + | nomad_enabled | true | + | k8s_enabled | false | + And I have a running blueprint + Then the following resources should be running + | name | + | resource.network.cloud | + | module.nomad.resource.nomad_cluster.dev | + And a HTTP call to "http://localhost:19090" should result in status 200 + And a HTTP call to "http://localhost:19091" should result in status 200 + And a HTTP call to "http://localhost:19092" should result in status 200 + + @k8s + Scenario: Kubernetes Cluster + Given the following jumppad variables are set + | key | value | + | nomad_enabled | false | + | k8s_enabled | true | + And I have a running blueprint + Then the following resources should be running + | name | + | resource.network.cloud | + | module.k8s.resource.k8s_cluster.k3s | + And a HTTP call to "http://localhost:29090" should result in status 200 + And a HTTP call to "http://localhost:29091" should result in status 200 + And a HTTP call to "http://localhost:29092" should result in status 200 \ No newline at end of file diff --git a/examples/registries/variables.hcl b/examples/registries/variables.hcl new file mode 100644 index 00000000..f3d5c818 --- /dev/null +++ b/examples/registries/variables.hcl @@ -0,0 +1,19 @@ +variable "nomad_enabled" { + default = true +} + +variable "k8s_enabled" { + default = true +} + +variable "auth_ip_address" { + default = "10.6.0.183" +} + +variable "noauth_ip_address" { + default = "10.6.0.184" +} + +variable "insecure_ip_address" { + default = "10.6.0.185" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 239ab896..b824c2bd 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/charmbracelet/log v0.2.2 github.com/creack/pty v1.1.18 github.com/cucumber/godog v0.12.4 + github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker v24.0.5+incompatible github.com/docker/go-connections v0.4.0 github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 @@ -23,6 +24,7 @@ require ( github.com/go-chi/cors v1.2.1 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 + github.com/gosuri/uitable v0.0.4 github.com/hashicorp/go-getter v1.7.0 github.com/hashicorp/go-hclog v1.1.0 github.com/hashicorp/go-uuid v1.0.2 @@ -30,7 +32,7 @@ require ( github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f github.com/jumppad-labs/connector v0.3.0 github.com/jumppad-labs/gohup v0.3.0 - github.com/jumppad-labs/hclconfig v0.16.1 + github.com/jumppad-labs/hclconfig v0.16.4 github.com/kennygrant/sanitize v1.2.4 github.com/mailgun/raymond/v2 v2.0.48 github.com/moby/sys/signal v0.7.0 @@ -49,7 +51,8 @@ require ( github.com/zclconf/go-cty v1.13.0 golang.org/x/crypto v0.14.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.3 + gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.8.2 k8s.io/api v0.23.5 k8s.io/apimachinery v0.23.5 @@ -93,7 +96,6 @@ require ( github.com/disintegration/imaging v1.6.2 // indirect github.com/dlclark/regexp2 v1.1.6 // indirect github.com/docker/cli v20.10.11+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -131,7 +133,6 @@ require ( github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/guillermo/go.procstat v0.0.0-20131123175440-34c2813d2e7f // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -217,7 +218,6 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect diff --git a/go.sum b/go.sum index ac04c6f6..82ab262a 100644 --- a/go.sum +++ b/go.sum @@ -996,8 +996,8 @@ github.com/jumppad-labs/go-cty v0.0.0-20230804061424-9e985cb751f6 h1:1ADItCWr5pr github.com/jumppad-labs/go-cty v0.0.0-20230804061424-9e985cb751f6/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/jumppad-labs/gohup v0.3.0 h1:YiHZWUkyfP6mBno5ZyYF0XY6LLhThrya9Fvm/wvNe7s= github.com/jumppad-labs/gohup v0.3.0/go.mod h1:Y3Bwz+BItoRYvEYXjWe1KliMJgZqd7CWvGXbhKNAoYs= -github.com/jumppad-labs/hclconfig v0.16.1 h1:hjxPnsB49cJP64vyJZjzP7+KcI/N1PnV3JKUeJntbXQ= -github.com/jumppad-labs/hclconfig v0.16.1/go.mod h1:9z3D2BUfFOWfw0iD7djsGXxFehnYc/K+rPUmrX092cc= +github.com/jumppad-labs/hclconfig v0.16.4 h1:ez61tqZIk7nfxeax8ysVApIzkoGL09dpb5yzGxsTlVw= +github.com/jumppad-labs/hclconfig v0.16.4/go.mod h1:9z3D2BUfFOWfw0iD7djsGXxFehnYc/K+rPUmrX092cc= github.com/jumppad-labs/log v0.0.0-20230711151418-55bbc87954b7 h1:jxF+Sxei1/7c7tLcV3Juu+rCIAhBv2jdicokriFohYE= github.com/jumppad-labs/log v0.0.0-20230711151418-55bbc87954b7/go.mod h1:ZApwwzDbbETVTIRTk7724yQRJAXIktt98yGVMMaa3y8= github.com/karrick/godirwalk v1.15.8 h1:7+rWAZPn9zuRxaIqqT8Ohs2Q2Ac0msBqwRdxNCr2VVs= @@ -2197,8 +2197,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/pkg/clients/container/container_tasks.go b/pkg/clients/container/container_tasks.go index 402e646d..b64a62de 100644 --- a/pkg/clients/container/container_tasks.go +++ b/pkg/clients/container/container_tasks.go @@ -56,6 +56,8 @@ type ContainerTasks interface { // If the force parameter is set then PullImage will pull regardless of the image already // being cached locally. PullImage(image types.Image, force bool) error + // PushImage pushes an image to the registry + PushImage(image types.Image) error // FindContainerIDs returns the Container IDs for the given container name FindContainerIDs(containerName string) ([]string, error) // RemoveImage removes the image with the given id from the local registry @@ -104,6 +106,9 @@ type ContainerTasks interface { // CreateShell in the running container and attach CreateShell(id string, command []string, stdin io.ReadCloser, stdout io.Writer, stderr io.Writer) error + // TagImage tags an image with the given tag + TagImage(source, destination string) error + // Returns basic information related to the Docker Engine EngineInfo() *types.EngineInfo } diff --git a/pkg/clients/container/docker.go b/pkg/clients/container/docker.go index ca4a2774..e60e7691 100644 --- a/pkg/clients/container/docker.go +++ b/pkg/clients/container/docker.go @@ -60,6 +60,8 @@ type Docker interface { ImageSave(ctx context.Context, imageIDs []string) (io.ReadCloser, error) ImageRemove(ctx context.Context, imageID string, options types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) + ImageTag(ctx context.Context, source, target string) error + ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) ServerVersion(ctx context.Context) (types.Version, error) Info(ctx context.Context) (types.Info, error) diff --git a/pkg/clients/container/docker_tasks.go b/pkg/clients/container/docker_tasks.go index 2739ed0d..6eeaa731 100644 --- a/pkg/clients/container/docker_tasks.go +++ b/pkg/clients/container/docker_tasks.go @@ -17,11 +17,13 @@ import ( "sync" "time" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/go-connections/nat" @@ -294,6 +296,9 @@ func (d *DockerTasks) CreateContainer(c *dtypes.Container) (string, error) { // is this a priviledged container hc.Privileged = c.Privileged + if c.Privileged { + hc.CgroupnsMode = "host" + } // are we attaching the container to a sidecar network? for _, n := range c.Networks { @@ -485,6 +490,52 @@ func (d *DockerTasks) PullImage(image dtypes.Image, force bool) error { return nil } +func (d *DockerTasks) PushImage(image dtypes.Image) error { + ipo := types.ImagePushOptions{} + // if the username and password is not null make an authenticated + // image pull + if image.Username != "" && image.Password != "" { + ipo.RegistryAuth = createRegistryAuth(image.Username, image.Password) + } + + ref, err := reference.ParseNormalizedNamed(image.Name) + if err != nil { + return xerrors.Errorf("error parsing image name: %w", err) + } + + //ipo.PrivilegeFunc = RegistryAuthenticationPrivilegedFunc(domain, image.Username, image.Password) + // if pushing to a registry that is not authenticated you still need to + // set a registry auth otherwise the API complains that there is no bearer token + if ipo.RegistryAuth == "" { + domain := reference.Domain(ref) + ac := registrytypes.AuthConfig{} + ac.ServerAddress = domain + + ra, _ := registrytypes.EncodeAuthConfig(ac) + ipo.RegistryAuth = ra + } + + name := reference.FamiliarString(ref) + + out, err := d.c.ImagePush(context.Background(), name, ipo) + if err != nil { + return xerrors.Errorf("Error pushing image: %w", err) + } + + // write the output to the debug log + io.Copy(d.l.StandardWriter(), out) + return nil +} + +func RegistryAuthenticationPrivilegedFunc(server, username, password string) types.RequestPrivilegeFunc { + return func() (string, error) { + ac := registrytypes.AuthConfig{} + ac.ServerAddress = server + + return registrytypes.EncodeAuthConfig(ac) + } +} + // FindContainerIDs returns the Container IDs for the given identifier func (d *DockerTasks) FindContainerIDs(fqdn string) ([]string, error) { args := filters.NewArgs() @@ -873,7 +924,7 @@ func (d *DockerTasks) CopyFilesToVolume(volumeID string, filenames []string, pat for _, f := range filenames { // get the filename part name := filepath.Base(f) - destFile := filepath.Join(destPath, name) + destFile := fmt.Sprintf("%s/%s", destPath, name) // check if the image exists if we are not doing a forced update if !d.force && !force { @@ -1206,7 +1257,7 @@ func (d *DockerTasks) resizeTTY(id string, out *streams.Out) error { } func (d *DockerTasks) AttachNetwork(net, containerID string, aliases []string, ipAddress string) error { - d.l.Debug("Attaching container to network", "ref", containerID, "network", net) + d.l.Debug("Attaching container to network", "id", containerID, "network", net) es := &network.EndpointSettings{NetworkID: net} // if we have network aliases defined, add them to the network connection @@ -1216,7 +1267,7 @@ func (d *DockerTasks) AttachNetwork(net, containerID string, aliases []string, i // are we binding to a specific ip if ipAddress != "" { - d.l.Debug("Assigning static ip address", "ref", containerID, "network", net, "ip_address", ipAddress) + d.l.Debug("Assigning static ip address", "id", containerID, "network", net, "ip_address", ipAddress) es.IPAMConfig = &network.EndpointIPAMConfig{IPv4Address: ipAddress} } @@ -1285,6 +1336,10 @@ func (d *DockerTasks) FindNetwork(id string) (dtypes.NetworkAttachment, error) { return dtypes.NetworkAttachment{}, fmt.Errorf("a network with the label id: %s, was not found", id) } +func (d *DockerTasks) TagImage(source, destination string) error { + return d.c.ImageTag(context.Background(), source, destination) +} + // publishedPorts defines a Docker published port type publishedPorts struct { ExposedPorts map[nat.Port]struct{} @@ -1370,11 +1425,12 @@ func createPublishedPortRanges(ps []dtypes.PortRange) (publishedPorts, error) { // credentials are a json string and need to be base64 encoded func createRegistryAuth(username, password string) string { - return base64.StdEncoding.EncodeToString( - []byte( - fmt.Sprintf(`{"Username": "%s", "Password": "%s"}`, username, password), - ), - ) + ac := registrytypes.AuthConfig{} + ac.Username = username + ac.Password = password + + ec, _ := registrytypes.EncodeAuthConfig(ac) + return ec } // makeImageCanonical makes sure the image reference uses full canonical name i.e. diff --git a/pkg/clients/container/docker_tasks_image_pull_test.go b/pkg/clients/container/docker_tasks_image_pull_test.go index 7ed98921..068ce693 100644 --- a/pkg/clients/container/docker_tasks_image_pull_test.go +++ b/pkg/clients/container/docker_tasks_image_pull_test.go @@ -98,7 +98,7 @@ func TestPullImageWithValidCredentials(t *testing.T) { d, err := base64.StdEncoding.DecodeString(ipo.RegistryAuth) assert.NoError(t, err) - assert.Equal(t, `{"Username": "nicjackson", "Password": "S3cur1t11"}`, string(d)) + assert.Equal(t, `{"username":"nicjackson","password":"S3cur1t11"}`, string(d)) } func TestDoNotPullImageWhenLocalImage(t *testing.T) { diff --git a/pkg/clients/container/docker_tasks_push_test.go b/pkg/clients/container/docker_tasks_push_test.go new file mode 100644 index 00000000..5e5e44a8 --- /dev/null +++ b/pkg/clients/container/docker_tasks_push_test.go @@ -0,0 +1,77 @@ +package container + +import ( + "bytes" + "encoding/base64" + "io" + "testing" + + "github.com/docker/docker/api/types" + "github.com/jumppad-labs/jumppad/pkg/clients/container/mocks" + dtypes "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + "github.com/jumppad-labs/jumppad/pkg/clients/logger" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestTagImageTagstheImage(t *testing.T) { + md := &mocks.Docker{} + md.On("ServerVersion", mock.Anything).Return(types.Version{}, nil) + md.On("ImageTag", mock.Anything, mock.Anything, mock.Anything).Return(nil) + md.On("Info", mock.Anything).Return(types.Info{Driver: StorageDriverOverlay2}, nil) + + dt, err := NewDockerTasks(md, nil, nil, logger.NewTestLogger(t)) + require.NoError(t, err) + + err = dt.TagImage("abc", "def") + require.NoError(t, err) + + md.AssertCalled(t, "ImageTag", mock.Anything, "abc", "def") +} + +func TestPushPushestheImageToTheRegistryWithoutAuth(t *testing.T) { + md := &mocks.Docker{} + md.On("ServerVersion", mock.Anything).Return(types.Version{}, nil) + md.On("Info", mock.Anything).Return(types.Info{Driver: StorageDriverOverlay2}, nil) + md.On("ImagePush", mock.Anything, mock.Anything, mock.Anything).Return(io.NopCloser(&bytes.Buffer{}), nil) + + dt, err := NewDockerTasks(md, nil, nil, logger.NewTestLogger(t)) + require.NoError(t, err) + + err = dt.PushImage(dtypes.Image{Name: "myimage:latest"}) + require.NoError(t, err) + + md.AssertCalled(t, "ImagePush", mock.Anything, "myimage:latest", mock.Anything) + + // ensure auth is set with a domain only + args := md.Calls[2].Arguments + auth := args.Get(2).(types.ImagePushOptions).RegistryAuth + authString, _ := base64.StdEncoding.DecodeString(auth) + + require.Contains(t, string(authString), `"serveraddress":"docker.io"`) + require.NotContains(t, string(authString), `"username":`) + require.NotContains(t, string(authString), `"password":`) +} + +func TestPushPushestheImageToTheRegistryWithAuth(t *testing.T) { + md := &mocks.Docker{} + md.On("ServerVersion", mock.Anything).Return(types.Version{}, nil) + md.On("Info", mock.Anything).Return(types.Info{Driver: StorageDriverOverlay2}, nil) + md.On("ImagePush", mock.Anything, mock.Anything, mock.Anything).Return(io.NopCloser(&bytes.Buffer{}), nil) + + dt, err := NewDockerTasks(md, nil, nil, logger.NewTestLogger(t)) + require.NoError(t, err) + + err = dt.PushImage(dtypes.Image{Name: "myimage:latest", Username: "user", Password: "pass"}) + require.NoError(t, err) + + md.AssertCalled(t, "ImagePush", mock.Anything, "myimage:latest", mock.Anything) + + // ensure auth is not set + args := md.Calls[2].Arguments + auth := args.Get(2).(types.ImagePushOptions).RegistryAuth + authString, _ := base64.StdEncoding.DecodeString(auth) + + require.Contains(t, string(authString), "user") + require.Contains(t, string(authString), "pass") +} diff --git a/pkg/clients/container/mocks/container_tasks.go b/pkg/clients/container/mocks/container_tasks.go index 0e15d173..59bf8af8 100644 --- a/pkg/clients/container/mocks/container_tasks.go +++ b/pkg/clients/container/mocks/container_tasks.go @@ -338,17 +338,17 @@ func (_m *ContainerTasks) ExecuteScript(id string, contents string, env []string return r0, r1 } -// FindContainerIDs provides a mock function with given fields: fqdn -func (_m *ContainerTasks) FindContainerIDs(fqdn string) ([]string, error) { - ret := _m.Called(fqdn) +// FindContainerIDs provides a mock function with given fields: containerName +func (_m *ContainerTasks) FindContainerIDs(containerName string) ([]string, error) { + ret := _m.Called(containerName) var r0 []string var r1 error if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok { - return rf(fqdn) + return rf(containerName) } if rf, ok := ret.Get(0).(func(string) []string); ok { - r0 = rf(fqdn) + r0 = rf(containerName) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) @@ -356,7 +356,7 @@ func (_m *ContainerTasks) FindContainerIDs(fqdn string) ([]string, error) { } if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(fqdn) + r1 = rf(containerName) } else { r1 = ret.Error(1) } @@ -468,6 +468,20 @@ func (_m *ContainerTasks) PullImage(image types.Image, force bool) error { return r0 } +// PushImage provides a mock function with given fields: image +func (_m *ContainerTasks) PushImage(image types.Image) error { + ret := _m.Called(image) + + var r0 error + if rf, ok := ret.Get(0).(func(types.Image) error); ok { + r0 = rf(image) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // RemoveContainer provides a mock function with given fields: id, force func (_m *ContainerTasks) RemoveContainer(id string, force bool) error { ret := _m.Called(id, force) @@ -515,6 +529,20 @@ func (_m *ContainerTasks) SetForcePull(_a0 bool) { _m.Called(_a0) } +// TagImage provides a mock function with given fields: source, destination +func (_m *ContainerTasks) TagImage(source string, destination string) error { + ret := _m.Called(source, destination) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(source, destination) + } else { + r0 = ret.Error(0) + } + + return r0 +} + type mockConstructorTestingTNewContainerTasks interface { mock.TestingT Cleanup(func()) diff --git a/pkg/clients/container/mocks/docker.go b/pkg/clients/container/mocks/docker.go index 8f023bd2..5088490b 100644 --- a/pkg/clients/container/mocks/docker.go +++ b/pkg/clients/container/mocks/docker.go @@ -429,6 +429,32 @@ func (_m *Docker) ImagePull(ctx context.Context, refStr string, options types.Im return r0, r1 } +// ImagePush provides a mock function with given fields: ctx, image, options +func (_m *Docker) ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) { + ret := _m.Called(ctx, image, options) + + var r0 io.ReadCloser + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, types.ImagePushOptions) (io.ReadCloser, error)); ok { + return rf(ctx, image, options) + } + if rf, ok := ret.Get(0).(func(context.Context, string, types.ImagePushOptions) io.ReadCloser); ok { + r0 = rf(ctx, image, options) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.ReadCloser) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, types.ImagePushOptions) error); ok { + r1 = rf(ctx, image, options) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ImageRemove provides a mock function with given fields: ctx, imageID, options func (_m *Docker) ImageRemove(ctx context.Context, imageID string, options types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) { ret := _m.Called(ctx, imageID, options) @@ -481,6 +507,20 @@ func (_m *Docker) ImageSave(ctx context.Context, imageIDs []string) (io.ReadClos return r0, r1 } +// ImageTag provides a mock function with given fields: ctx, source, target +func (_m *Docker) ImageTag(ctx context.Context, source string, target string) error { + ret := _m.Called(ctx, source, target) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, source, target) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Info provides a mock function with given fields: ctx func (_m *Docker) Info(ctx context.Context) (types.Info, error) { ret := _m.Called(ctx) diff --git a/pkg/config/resources/build/provider.go b/pkg/config/resources/build/provider.go index 52b19b5c..517db0fb 100644 --- a/pkg/config/resources/build/provider.go +++ b/pkg/config/resources/build/provider.go @@ -97,6 +97,23 @@ func (b *Provider) Create() error { } } + // if we have a registry, push the image + for _, r := range b.config.Registries { + // first tag the image + b.log.Debug("Tag image", "ref", b.config.ID, "name", b.config.Image, "tag", r.Name) + err = b.client.TagImage(b.config.Image, r.Name) + if err != nil { + return xerrors.Errorf("unable to tag image: %w", err) + } + + // push the image + b.log.Debug("Push image", "ref", b.config.ID, "tag", r.Name) + err = b.client.PushImage(types.Image{Name: r.Name, Username: r.Username, Password: r.Password}) + if err != nil { + return xerrors.Errorf("unable to push image: %w", err) + } + } + return nil } diff --git a/pkg/config/resources/build/provider_test.go b/pkg/config/resources/build/provider_test.go index 5c93a935..c400368a 100644 --- a/pkg/config/resources/build/provider_test.go +++ b/pkg/config/resources/build/provider_test.go @@ -1 +1,57 @@ package build + +import ( + "fmt" + "testing" + + htypes "github.com/jumppad-labs/hclconfig/types" + "github.com/jumppad-labs/jumppad/pkg/clients/container/mocks" + "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + "github.com/jumppad-labs/jumppad/pkg/clients/logger" + "github.com/jumppad-labs/jumppad/pkg/config/resources/container" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func setupProvider(t *testing.T, b *Build) (*Provider, *mocks.ContainerTasks) { + l := logger.NewTestLogger(t) + + mc := &mocks.ContainerTasks{} + mc.On("BuildContainer", mock.Anything, true).Return("buildimage:abcde", nil) + mc.On("FindImagesInLocalRegistry", fmt.Sprintf("jumppad.dev/localcache/%s", b.Name)).Return([]string{}, nil) + mc.On("TagImage", mock.Anything, mock.Anything).Return(nil) + mc.On("PushImage", mock.Anything).Return(nil) + + p := &Provider{ + config: b, + client: mc, + log: l, + } + + return p, mc +} + +func TestCreatePushesToRegistry(t *testing.T) { + b := &Build{ + ResourceMetadata: htypes.ResourceMetadata{Name: "test"}, + Registries: []container.Image{ + container.Image{ + Name: "nicholasjackson/fake:latest", + }, + container.Image{ + Name: "authed/fake:latest", + Username: "test", + Password: "password", + }, + }, + } + + p, mc := setupProvider(t, b) + err := p.Create() + require.NoError(t, err) + + // ensure the image is tagged + mc.AssertCalled(t, "TagImage", "buildimage:abcde", "nicholasjackson/fake:latest") + mc.AssertCalled(t, "PushImage", types.Image{Name: "nicholasjackson/fake:latest", Username: "", Password: ""}) + mc.AssertCalled(t, "PushImage", types.Image{Name: "authed/fake:latest", Username: "test", Password: "password"}) +} diff --git a/pkg/config/resources/build/resource.go b/pkg/config/resources/build/resource.go index e7e7fbf9..32eb865f 100644 --- a/pkg/config/resources/build/resource.go +++ b/pkg/config/resources/build/resource.go @@ -3,6 +3,7 @@ package build import ( "github.com/jumppad-labs/hclconfig/types" "github.com/jumppad-labs/jumppad/pkg/config" + "github.com/jumppad-labs/jumppad/pkg/config/resources/container" "github.com/jumppad-labs/jumppad/pkg/utils" ) @@ -18,6 +19,8 @@ type Build struct { // Outputs allow files or directories to be copied from the container Outputs []Output `hcl:"output,block" json:"outputs"` + Registries []container.Image `hcl:"registry,block" json:"registries"` // Optional registry to push the image to + // outputs // Image is the full local reference of the built image @@ -34,6 +37,9 @@ type BuildContainer struct { Args map[string]string `hcl:"args,optional" json:"args,omitempty"` // Build args to pass to the container } +type Registry struct { +} + type Output struct { Source string `hcl:"source" json:"source"` // Source file or directory in container Destination string `hcl:"destination" json:"destination"` // Destination for copied file or directory diff --git a/pkg/config/resources/cache/init.go b/pkg/config/resources/cache/init.go deleted file mode 100644 index 79c63ed9..00000000 --- a/pkg/config/resources/cache/init.go +++ /dev/null @@ -1,5 +0,0 @@ -package cache - -// register the types and provider -func init() { -} diff --git a/pkg/config/resources/cache/provider.go b/pkg/config/resources/cache/provider.go index 48bcd0d9..1a5315d2 100644 --- a/pkg/config/resources/cache/provider.go +++ b/pkg/config/resources/cache/provider.go @@ -2,27 +2,28 @@ package cache import ( "fmt" - "math/rand" "path/filepath" + "strings" + + ctypes "github.com/jumppad-labs/jumppad/pkg/config/resources/container" dtypes "github.com/docker/docker/api/types" htypes "github.com/jumppad-labs/hclconfig/types" "github.com/jumppad-labs/jumppad/pkg/clients" "github.com/jumppad-labs/jumppad/pkg/clients/container" "github.com/jumppad-labs/jumppad/pkg/clients/container/types" - "github.com/jumppad-labs/jumppad/pkg/clients/http" "github.com/jumppad-labs/jumppad/pkg/clients/logger" "github.com/jumppad-labs/jumppad/pkg/utils" "golang.org/x/xerrors" ) -const cacheImage = "shipyardrun/docker-registry-proxy:0.6.3" +const cacheImage = "ghcr.io/rpardini/docker-registry-proxy:0.6.4" +const defaultRegistries = "k8s.gcr.io gcr.io asia.gcr.io eu.gcr.io us.gcr.io quay.io ghcr.io docker.pkg.github.com" type Provider struct { - config *ImageCache - client container.ContainerTasks - httpClient http.HTTP - log logger.Logger + config *ImageCache + client container.ContainerTasks + log logger.Logger } func (p *Provider) Init(cfg htypes.Resource, l logger.Logger) error { @@ -38,7 +39,6 @@ func (p *Provider) Init(cfg htypes.Resource, l logger.Logger) error { p.config = c p.client = cli.ContainerTasks - p.httpClient = cli.HTTP p.log = l return nil @@ -53,17 +53,34 @@ func (p *Provider) Create() error { return err } - // get a list of dependent networks for the resource - dependentNetworks := p.findDependentNetworks() + var registries []string + var authRegistries []string + + for _, reg := range p.config.Registries { + registries = append(registries, reg.Hostname) + + if reg.Auth != nil { + host := reg.Hostname + if reg.Auth.Hostname != "" { + host = reg.Auth.Hostname + } + + authRegistries = append(authRegistries, host+":::"+reg.Auth.Username+":::"+reg.Auth.Password) + } + } if len(ids) == 0 { - _, err := p.createImageCache(dependentNetworks) + _, err := p.createImageCache(registries, authRegistries) if err != nil { return err } } - return nil + // get a list of dependent networks for the resource + dependentNetworks := p.findDependentNetworks() + + // add the networks and return + return p.reConfigureNetworks(dependentNetworks) } func (p *Provider) Destroy() error { @@ -76,7 +93,10 @@ func (p *Provider) Destroy() error { if len(ids) > 0 { for _, id := range ids { - p.client.RemoveContainer(id, true) + err = p.client.RemoveContainer(id, true) + if err != nil { + p.log.Error(err.Error()) + } } } @@ -105,7 +125,7 @@ func (p *Provider) Changed() (bool, error) { return false, nil } -func (p *Provider) createImageCache(networks []string) (string, error) { +func (p *Provider) createImageCache(registries []string, authRegistries []string) (string, error) { fqdn := utils.FQDN(p.config.Name, p.config.Module, p.config.Type) // Create the volume to store the cache @@ -144,27 +164,45 @@ func (p *Provider) createImageCache(networks []string) (string, error) { } cc.Environment = map[string]string{ - "CA_KEY_FILE": "/cache/ca/root.key", - "CA_CRT_FILE": "/cache/ca/root.cert", - "DOCKER_MIRROR_CACHE": "/cache/docker", - "ENABLE_MANIFEST_CACHE": "true", - "REGISTRIES": "k8s.gcr.io gcr.io asia.gcr.io eu.gcr.io us.gcr.io quay.io ghcr.io docker.pkg.github.com", - "ALLOW_PUSH": "true", + "CA_KEY_FILE": "/cache/ca/root.key", + "CA_CRT_FILE": "/cache/ca/root.cert", + "DEBUG": "false", + "DEBUG_NGINX": "false", + "DEBUG_HUB": "false", + "DOCKER_MIRROR_CACHE": "/cache/docker", + "ENABLE_MANIFEST_CACHE": "true", + "REGISTRIES": strings.Trim(defaultRegistries+" "+strings.Join(registries, " "), " "), + "AUTH_REGISTRY_DELIMITER": ":::", + "AUTH_REGISTRIES": strings.Trim(strings.Join(authRegistries, " "), " "), + "ALLOW_PUSH": "true", + "VERIFY_SSL": "false", } // expose the docker proxy port on a random port num + p1, err1 := utils.RandomAvailablePort(31000, 34000) + p2, err2 := utils.RandomAvailablePort(31000, 34000) + p3, err3 := utils.RandomAvailablePort(31000, 34000) + + if err1 != nil || err2 != nil || err3 != nil { + return "", err + } + cc.Ports = []types.Port{ { Local: "3128", - Host: fmt.Sprintf("%d", rand.Intn(3000)+31000), + Host: fmt.Sprintf("%d", p1), + Protocol: "tcp", + }, + { + Local: "8081", + Host: fmt.Sprintf("%d", p2), + Protocol: "tcp", + }, + { + Local: "8082", + Host: fmt.Sprintf("%d", p3), Protocol: "tcp", }, - } - - // add the networks - cc.Networks = []types.NetworkAttachment{} - for _, n := range networks { - cc.Networks = append(cc.Networks, types.NetworkAttachment{ID: n}) } return p.client.CreateContainer(cc) @@ -174,6 +212,10 @@ func (p *Provider) findDependentNetworks() []string { nets := []string{} for _, n := range p.config.DependsOn { + if strings.HasSuffix(n, ".id") { + // Ignore explicitly configured network dependencies + continue + } target, err := p.client.FindNetwork(n) if err != nil { // ignore this network @@ -225,7 +267,7 @@ func (p *Provider) reConfigureNetworks(dependentNetworks []string) error { return fmt.Errorf("unable to attach cache to network: %s", err) } - p.config.Networks = append(p.config.Networks, n) + p.config.Networks = append(p.config.Networks, ctypes.NetworkAttachment{ID: n}) } added = append(added, n) diff --git a/pkg/config/resources/cache/provider_test.bak b/pkg/config/resources/cache/provider_test.go similarity index 75% rename from pkg/config/resources/cache/provider_test.bak rename to pkg/config/resources/cache/provider_test.go index 0331dd77..02e6a5fe 100644 --- a/pkg/config/resources/cache/provider_test.bak +++ b/pkg/config/resources/cache/provider_test.go @@ -1,28 +1,28 @@ -package providers +package cache import ( "encoding/json" "path/filepath" "testing" - "github.com/docker/docker/api/types" - "github.com/hashicorp/go-hclog" - "github.com/jumppad-labs/jumppad/pkg/clients" - "github.com/jumppad-labs/jumppad/pkg/clients/mocks" - "github.com/jumppad-labs/jumppad/pkg/config" + dtypes "github.com/docker/docker/api/types" + htypes "github.com/jumppad-labs/hclconfig/types" + cmocks "github.com/jumppad-labs/jumppad/pkg/clients/container/mocks" + "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + ctypes "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + "github.com/jumppad-labs/jumppad/pkg/clients/logger" "github.com/jumppad-labs/jumppad/pkg/utils" + "github.com/jumppad-labs/jumppad/testutils" "github.com/stretchr/testify/mock" - assert "github.com/stretchr/testify/require" + "github.com/stretchr/testify/require" ) -func setupImageCacheTests(t *testing.T) (*config.ImageCache, *clients.MockContainerTasks, *mocks.MockHTTP) { - c := config.New() - cc := config.NewImageCache("tests") - c.AddResource(cc) +func setupImageCacheTests(t *testing.T) (*ImageCache, *cmocks.ContainerTasks) { + cc := &ImageCache{ResourceMetadata: htypes.ResourceMetadata{Name: "test"}} - md := &clients.MockContainerTasks{} - hc := &mocks.MockHTTP{} + md := &cmocks.ContainerTasks{} + md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{}, nil).Once() md.On("CreateContainer", mock.Anything).Once().Return("abc", nil) md.On("PullImage", mock.Anything, mock.Anything).Once().Return(nil) md.On("CreateVolume", "images").Once().Return("images", nil) @@ -32,15 +32,15 @@ func setupImageCacheTests(t *testing.T) (*config.ImageCache, *clients.MockContai md.On("DetachNetwork", mock.Anything, mock.Anything).Return(nil) md.On("AttachNetwork", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - return cc, md, hc + return cc, md } func TestImageCacheCreateDoesNotCreateContainerWhenExists(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Once().Return([]string{"abc"}, nil) @@ -49,70 +49,129 @@ func TestImageCacheCreateDoesNotCreateContainerWhenExists(t *testing.T) { } func TestImageCacheCreateCreatesVolume(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) md.AssertCalled(t, "CreateVolume", "images") } func TestImageCachePullsImage(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) - md.AssertCalled(t, "PullImage", config.Image{Name: cacheImage}, false) + md.AssertCalled(t, "PullImage", ctypes.Image{Name: cacheImage}, false) } func TestImageCacheCreateAddsVolumes(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) md.AssertCalled(t, "CreateContainer", mock.Anything) params := testutils.GetCalls(&md.Mock, "CreateContainer")[0] - conf := params.Arguments[0].(*config.Container) + conf := params.Arguments[0].(*ctypes.Container) // check volumes - assert.Equal(t, utils.FQDNVolumeName("images"), conf.Volumes[0].Source) - assert.Equal(t, "/cache", conf.Volumes[0].Destination) - assert.Equal(t, "volume", conf.Volumes[0].Type) + require.Equal(t, utils.FQDNVolumeName("images"), conf.Volumes[0].Source) + require.Equal(t, "/cache", conf.Volumes[0].Destination) + require.Equal(t, "volume", conf.Volumes[0].Type) } func TestImageCacheCreateAddsEnvironmentVariables(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) md.AssertCalled(t, "CreateContainer", mock.Anything) params := testutils.GetCalls(&md.Mock, "CreateContainer")[0] - conf := params.Arguments[0].(*config.Container) + conf := params.Arguments[0].(*ctypes.Container) // check environment variables - assert.Equal(t, conf.EnvVar["CA_KEY_FILE"], "/cache/ca/root.key") - assert.Equal(t, conf.EnvVar["CA_CRT_FILE"], "/cache/ca/root.cert") - assert.Equal(t, conf.EnvVar["DOCKER_MIRROR_CACHE"], "/cache/docker") - assert.Equal(t, conf.EnvVar["ENABLE_MANIFEST_CACHE"], "true") - assert.Equal(t, conf.EnvVar["REGISTRIES"], "k8s.gcr.io gcr.io asia.gcr.io eu.gcr.io us.gcr.io quay.io ghcr.io docker.pkg.github.com") - assert.Equal(t, conf.EnvVar["ALLOW_PUSH"], "true") + require.Equal(t, conf.Environment["CA_KEY_FILE"], "/cache/ca/root.key") + require.Equal(t, conf.Environment["CA_CRT_FILE"], "/cache/ca/root.cert") + require.Equal(t, conf.Environment["DEBUG"], "false") + require.Equal(t, conf.Environment["DEBUG_NGINX"], "false") + require.Equal(t, conf.Environment["DEBUG_HUB"], "false") + require.Equal(t, conf.Environment["DOCKER_MIRROR_CACHE"], "/cache/docker") + require.Equal(t, conf.Environment["ENABLE_MANIFEST_CACHE"], "true") + require.Equal(t, conf.Environment["REGISTRIES"], defaultRegistries) + require.Equal(t, conf.Environment["AUTH_REGISTRY_DELIMITER"], ":::") + require.Equal(t, conf.Environment["AUTH_REGISTRIES"], "") + require.Equal(t, conf.Environment["ALLOW_PUSH"], "true") + require.Equal(t, conf.Environment["VERIFY_SSL"], "false") +} + +func TestImageCacheCreateAddsUnauthenticatedRegistries(t *testing.T) { + cc, md := setupImageCacheTests(t) + cc.Registries = []Registry{ + Registry{ + Hostname: "my.registry", + }, + Registry{ + Hostname: "my.other.registry", + }, + } + + c := Provider{cc, md, logger.NewTestLogger(t)} + err := c.Create() + require.NoError(t, err) + + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0] + conf := params.Arguments[0].(*ctypes.Container) + + require.Equal(t, conf.Environment["REGISTRIES"], defaultRegistries+" my.registry my.other.registry") + require.Equal(t, conf.Environment["AUTH_REGISTRIES"], "") +} + +func TestImageCacheCreateAddsAuthenticatedRegistries(t *testing.T) { + cc, md := setupImageCacheTests(t) + cc.Registries = []Registry{ + Registry{ + Hostname: "my.registry", + Auth: &RegistryAuth{ + Username: "user1", + Password: "password1", + }, + }, + Registry{ + Hostname: "my.other.registry", + Auth: &RegistryAuth{ + Hostname: "alt.domain.registry", + Username: "user2", + Password: "password2", + }, + }, + } + + c := Provider{cc, md, logger.NewTestLogger(t)} + err := c.Create() + require.NoError(t, err) + + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0] + conf := params.Arguments[0].(*ctypes.Container) + + require.Equal(t, conf.Environment["REGISTRIES"], defaultRegistries+" my.registry my.other.registry") + require.Equal(t, conf.Environment["AUTH_REGISTRIES"], "my.registry:::user1:::password1 alt.domain.registry:::user2:::password2") } func TestImageCacheCreateCopiesCerts(t *testing.T) { - cc, md, hc := setupImageCacheTests(t) + cc, md := setupImageCacheTests(t) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) md.AssertCalled(t, "CreateContainer", mock.Anything) @@ -130,26 +189,25 @@ func TestImageCacheCreateCopiesCerts(t *testing.T) { ) } -func TestImageCacheDetachesNetworksAndAttachesNew(t *testing.T) { - net1 := config.NewNetwork("one") - net2 := config.NewNetwork("two") +func TestImageCacheAttachesAndDetatchesNetworks(t *testing.T) { + cc, md := setupImageCacheTests(t) - cc, md, hc := setupImageCacheTests(t) - cc.DependsOn = []string{"network.one", "network.two"} + cc.DependsOn = []string{"resource.network.one", "resource.network.two"} - cc.Config.AddResource(net1) - cc.Config.AddResource(net2) - - containerJSON := &types.ContainerJSON{} + containerJSON := &dtypes.ContainerJSON{} json.Unmarshal([]byte(cacheContainerInfoWithNetworks), containerJSON) testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Once().Return([]string{"abc"}, nil) md.On("ContainerInfo", "abc").Once().Return(*containerJSON, nil) + md.On("FindContainerIDs", mock.Anything).Once().Return([]string{"abc"}, nil) + + md.On("FindNetwork", "resource.network.one").Once().Return(types.NetworkAttachment{Name: "one"}, nil) + md.On("FindNetwork", "resource.network.two").Once().Return(types.NetworkAttachment{Name: "two"}, nil) - c := NewImageCache(cc, md, hc, clients.NewTestLogger(t)) + c := Provider{cc, md, logger.NewTestLogger(t)} err := c.Create() - assert.NoError(t, err) + require.NoError(t, err) // should detatch existing cloud network md.AssertNumberOfCalls(t, "DetachNetwork", 1) diff --git a/pkg/config/resources/cache/resource.go b/pkg/config/resources/cache/resource.go deleted file mode 100644 index 1f983481..00000000 --- a/pkg/config/resources/cache/resource.go +++ /dev/null @@ -1,14 +0,0 @@ -package cache - -import "github.com/jumppad-labs/hclconfig/types" - -// TypeContainer is the resource string for a Container resource -const TypeImageCache string = "image_cache" - -// Container defines a structure for creating Docker containers -type ImageCache struct { - // embedded type holding name, etc - types.ResourceMetadata `hcl:",remain"` - - Networks []string `json:"networks" state:"true"` // Attach to the correct network // only when Image is specified -} diff --git a/pkg/config/resources/cache/resource_cache.go b/pkg/config/resources/cache/resource_cache.go new file mode 100644 index 00000000..102cd430 --- /dev/null +++ b/pkg/config/resources/cache/resource_cache.go @@ -0,0 +1,19 @@ +package cache + +import ( + "github.com/jumppad-labs/hclconfig/types" + ctypes "github.com/jumppad-labs/jumppad/pkg/config/resources/container" +) + +// TypeImageCache is the resource string for a ImageCache resource +const TypeImageCache string = "image_cache" + +// ImageCache defines a structure for creating ImageCache containers +type ImageCache struct { + // embedded type holding name, etc + types.ResourceMetadata `hcl:",remain"` + + Registries []Registry `hcl:"registry,block" json:"registries,omitempty"` + + Networks ctypes.NetworkAttachments `hcl:"network,block" json:"networks,omitempty"` // Attach to the correct network // only when Image is specified +} diff --git a/pkg/config/resources/cache/resource_registry.go b/pkg/config/resources/cache/resource_registry.go new file mode 100644 index 00000000..3d1e8b30 --- /dev/null +++ b/pkg/config/resources/cache/resource_registry.go @@ -0,0 +1,21 @@ +package cache + +import "github.com/jumppad-labs/hclconfig/types" + +const TypeRegistry string = "container_registry" + +// Registry defines a structure for registering additional registries for the image cache +type Registry struct { + // embedded type holding name, etc + types.ResourceMetadata `hcl:",remain"` + + Hostname string `hcl:"hostname" json:"hostname"` // Hostname of the registry + Auth *RegistryAuth `hcl:"auth,block" json:"auth,omitempty"` // auth to authenticate against registry +} + +// RegistryAuth defines a structure for authenticating against a docker registry +type RegistryAuth struct { + Hostname string `hcl:"hostname,optional" json:"hostname,omitempty"` // Hostname for authentication, can be different from registry hostname + Username string `hcl:"username" json:"username"` // Username for authentication + Password string `hcl:"password" json:"password"` // Password for authentication +} diff --git a/pkg/config/resources/container/resource_sidecar.go b/pkg/config/resources/container/resource_sidecar.go index 573cfc80..f16a6b63 100644 --- a/pkg/config/resources/container/resource_sidecar.go +++ b/pkg/config/resources/container/resource_sidecar.go @@ -47,7 +47,6 @@ func (c *Sidecar) Process() error { // make sure mount paths are absolute when type is bind if v.Type == "" || v.Type == "bind" { c.Volumes[i].Source = utils.EnsureAbsolute(v.Source, c.File) - c.Volumes[i].Destination = utils.EnsureAbsolute(v.Destination, c.File) } } diff --git a/pkg/config/resources/k8s/provider_cluster.go b/pkg/config/resources/k8s/provider_cluster.go index 4ea0e1e6..4a93ea93 100644 --- a/pkg/config/resources/k8s/provider_cluster.go +++ b/pkg/config/resources/k8s/provider_cluster.go @@ -25,6 +25,7 @@ import ( "github.com/jumppad-labs/jumppad/pkg/clients/logger" "github.com/jumppad-labs/jumppad/pkg/utils" "golang.org/x/xerrors" + "gopkg.in/yaml.v2" ) // https://github.com/rancher/k3d/blob/master/cli/commands.go @@ -293,12 +294,25 @@ func (p *ClusterProvider) createK3s() error { }) } + // add the registries volume + rc, err := p.createRegistriesConfig() + if err != nil { + return fmt.Errorf("unable to create registries.yaml: %s", err) + } + + if rc != "" { + cc.Volumes = append(cc.Volumes, ctypes.Volume{ + Source: rc, + Destination: "/etc/rancher/k3s/registries.yaml", + Type: "bind", + }) + } + // Add any custom environment variables cc.Environment = map[string]string{} // set the environment variables for the K3S_KUBECONFIG_OUTPUT and K3S_CLUSTER_SECRET cc.Environment["K3S_KUBECONFIG_OUTPUT"] = "/output/kubeconfig.yaml" - cc.Environment["K3S_CLUSTER_SECRET"] = "mysupersecret" // only add the variables for the cache when the kubernetes version is >= v1.18.16 sv, err := semver.NewConstraint(">= v1.18.16") @@ -321,28 +335,21 @@ func (p *ClusterProvider) createK3s() error { if sv.Check(v) { // load the CA from a file - ca, err := ioutil.ReadFile(filepath.Join(utils.CertsDir(""), "/root.cert")) + ca, err := os.ReadFile(filepath.Join(utils.CertsDir(""), "/root.cert")) if err != nil { return fmt.Errorf("unable to read root CA for proxy: %s", err) } - // add the netmask from the network to the proxy bypass - networkSubmasks := []string{} - for _, n := range p.config.Networks { - net, err := p.client.FindNetwork(n.ID) - if err != nil { - return fmt.Errorf("Network not found: %w", err) - } + cc.Environment["CONTAINERD_HTTP_PROXY"] = utils.ImageCacheAddress() + cc.Environment["CONTAINERD_HTTPS_PROXY"] = utils.ImageCacheAddress() + cc.Environment["PROXY_CA"] = string(ca) - networkSubmasks = append(networkSubmasks, net.Subnet) + // add the no-proxy overrides + if p.config.Config != nil && + p.config.Config.DockerConfig != nil && + len(p.config.Config.DockerConfig.NoProxy) > 0 { + cc.Environment["CONTAINERD_NO_PROXY"] = strings.Join(p.config.Config.DockerConfig.NoProxy, ",") } - - proxyBypass := utils.ProxyBypass + "," + strings.Join(networkSubmasks, ",") - - cc.Environment["HTTP_PROXY"] = utils.HTTPProxyAddress() - cc.Environment["HTTPS_PROXY"] = utils.HTTPSProxyAddress() - cc.Environment["NO_PROXY"] = proxyBypass - cc.Environment["PROXY_CA"] = string(ca) } // add any custom environment variables @@ -721,12 +728,49 @@ func (p *ClusterProvider) destroyK3s() error { } } - _, kubePath, _ := utils.CreateKubeConfigPath(p.config.Name) - os.RemoveAll(kubePath) + configDir, _, _ := utils.CreateKubeConfigPath(p.config.Name) + os.RemoveAll(configDir) return nil } +// createRegistriesConfig creates the k3s mirrors config for the cluster +func (p *ClusterProvider) createRegistriesConfig() (string, error) { + dir, _, _ := utils.CreateKubeConfigPath(p.config.Name) + daemonConfigPath := path.Join(dir, "registries.yaml") + + // remove any existing files, fail silently + os.RemoveAll(daemonConfigPath) + + // create the docker config + dc := dockerConfig{ + Mirrors: map[string]dockerMirror{}, + } + + // if the config is nil, do nothing + if p.config.Config == nil || + p.config.Config.DockerConfig == nil || + len(p.config.Config.DockerConfig.InsecureRegistries) < 1 { + return "", nil + } + + for _, ir := range p.config.Config.DockerConfig.InsecureRegistries { + dc.Mirrors[ir] = dockerMirror{ + Endpoints: []string{fmt.Sprintf("http://%s", ir)}, + } + } + + // write the config to a file + data, err := yaml.Marshal(&dc) + if err != nil { + return "", err + } + + err = os.WriteFile(daemonConfigPath, data, os.ModePerm) + + return daemonConfigPath, err +} + func writeConnectorNamespace(path string) error { return ioutil.WriteFile(path, []byte(connectorNamespace), os.ModePerm) } @@ -772,6 +816,14 @@ func writeConnectorRBAC(path string) error { return ioutil.WriteFile(path, []byte(connectorRBAC), os.ModePerm) } +type dockerConfig struct { + Mirrors map[string]dockerMirror `yaml:"mirrors"` +} + +type dockerMirror struct { + Endpoints []string `yaml:"endpoint"` +} + var connectorDeployment = ` apiVersion: v1 kind: ServiceAccount diff --git a/pkg/config/resources/k8s/provider_cluster_test.bak b/pkg/config/resources/k8s/provider_cluster_test.go similarity index 58% rename from pkg/config/resources/k8s/provider_cluster_test.bak rename to pkg/config/resources/k8s/provider_cluster_test.go index 9d18862c..a9e85865 100644 --- a/pkg/config/resources/k8s/provider_cluster_test.bak +++ b/pkg/config/resources/k8s/provider_cluster_test.go @@ -1,9 +1,10 @@ -package providers +package k8s import ( "bytes" "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -11,10 +12,18 @@ import ( "testing" "time" - "github.com/hashicorp/go-hclog" - "github.com/jumppad-labs/jumppad/pkg/clients" - "github.com/jumppad-labs/jumppad/pkg/config" + htypes "github.com/jumppad-labs/hclconfig/types" + conmocks "github.com/jumppad-labs/jumppad/pkg/clients/connector/mocks" + contypes "github.com/jumppad-labs/jumppad/pkg/clients/connector/types" + cmocks "github.com/jumppad-labs/jumppad/pkg/clients/container/mocks" + "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + ctypes "github.com/jumppad-labs/jumppad/pkg/clients/container/types" + "github.com/jumppad-labs/jumppad/pkg/clients/k8s" + "github.com/jumppad-labs/jumppad/pkg/clients/logger" + + container "github.com/jumppad-labs/jumppad/pkg/config/resources/container" "github.com/jumppad-labs/jumppad/pkg/utils" + "github.com/jumppad-labs/jumppad/testutils" "github.com/mohae/deepcopy" "github.com/stretchr/testify/mock" assert "github.com/stretchr/testify/require" @@ -22,15 +31,17 @@ import ( // setupClusterMocks sets up a happy path for mocks func setupClusterMocks(t *testing.T) ( - *config.K8sCluster, *clients.MockContainerTasks, *clients.MockKubernetes, *clients.ConnectorMock) { + *K8sCluster, *cmocks.ContainerTasks, *k8s.MockKubernetes, *conmocks.Connector) { + + md := &cmocks.ContainerTasks{} + md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{}, nil).Once() + md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{"123"}, nil) // second call should find the cluster - md := &clients.MockContainerTasks{} - md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{}, nil) md.On("PullImage", mock.Anything, mock.Anything).Return(nil) md.On("CreateVolume", mock.Anything, mock.Anything).Return("123", nil) md.On("CreateContainer", mock.Anything).Return("containerid", nil) md.On("ContainerLogs", mock.Anything, true, true).Return( - ioutil.NopCloser(bytes.NewBufferString("Running kubelet")), + io.NopCloser(bytes.NewBufferString("Running kubelet")), nil, ) md.On("CopyFromContainer", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -39,8 +50,9 @@ func setupClusterMocks(t *testing.T) ( md.On("RemoveContainer", mock.Anything, mock.Anything).Return(nil) md.On("RemoveVolume", mock.Anything).Return(nil) md.On("DetachNetwork", mock.Anything, mock.Anything, mock.Anything).Return(nil) + md.On("ListNetworks", mock.Anything).Return([]types.NetworkAttachment{}) - md.On("EngineInfo").Return(&clients.EngineInfo{StorageDriver: "overlay2"}) + md.On("EngineInfo").Return(&types.EngineInfo{StorageDriver: "overlay2"}) // set the home folder to a temp folder tmpDir := t.TempDir() @@ -66,29 +78,24 @@ func setupClusterMocks(t *testing.T) ( kcf.Close() // create the Kubernetes client mock - mk := &clients.MockKubernetes{} + mk := &k8s.MockKubernetes{} mk.Mock.On("SetConfig", mock.Anything).Return(nil) mk.Mock.On("HealthCheckPods", mock.Anything, mock.Anything).Return(nil) mk.Mock.On("Apply", mock.Anything, mock.Anything).Return(nil) mk.Mock.On("GetPodLogs", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) - mc := &clients.ConnectorMock{} - mc.On("GetLocalCertBundle", mock.Anything).Return(&clients.CertBundle{}, nil) + mc := &conmocks.Connector{} + mc.On("GetLocalCertBundle", mock.Anything).Return(&contypes.CertBundle{}, nil) mc.On("GenerateLeafCert", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, - ).Return(&clients.CertBundle{}, nil) + ).Return(&contypes.CertBundle{}, nil) // copy the config - cc := deepcopy.Copy(clusterConfig).(*config.K8sCluster) - cn := deepcopy.Copy(clusterNetwork).(*config.Network) - - c := config.New() - c.AddResource(cc) - c.AddResource(cn) + cc := deepcopy.Copy(clusterConfig).(*K8sCluster) t.Cleanup(func() { os.Setenv(utils.HomeEnvName(), currentHome) @@ -98,11 +105,11 @@ func setupClusterMocks(t *testing.T) ( } func TestClusterK3ErrorsWhenUnableToLookupIDs(t *testing.T) { - md := &clients.MockContainerTasks{} + md := &cmocks.ContainerTasks{} md.On("FindContainerIDs", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("boom")) - mk := &clients.MockKubernetes{} - p := NewK8sCluster(clusterConfig, md, mk, nil, nil, clients.NewTestLogger(t)) + mk := &k8s.MockKubernetes{} + p := ClusterProvider{clusterConfig, md, mk, nil, nil, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -110,74 +117,73 @@ func TestClusterK3ErrorsWhenUnableToLookupIDs(t *testing.T) { func TestClusterK3SetsEnvironment(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - cc.Version = "" + cc.Image = nil - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{clusterConfig, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*config.Container) + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*types.Container) - assert.Equal(t, params.EnvVar["K3S_KUBECONFIG_OUTPUT"], "/output/kubeconfig.yaml") - assert.Equal(t, params.EnvVar["K3S_CLUSTER_SECRET"], "mysupersecret") - assert.Equal(t, params.EnvVar["HTTP_PROXY"], utils.HTTPProxyAddress()) - assert.Equal(t, params.EnvVar["HTTPS_PROXY"], utils.HTTPSProxyAddress()) - assert.Equal(t, params.EnvVar["NO_PROXY"], utils.ProxyBypass) - - assert.Equal(t, params.EnvVar["PROXY_CA"], "CA") + assert.Equal(t, params.Environment["K3S_KUBECONFIG_OUTPUT"], "/output/kubeconfig.yaml") + assert.Equal(t, params.Environment["CONTAINERD_HTTP_PROXY"], utils.ImageCacheAddress()) + assert.Equal(t, params.Environment["CONTAINERD_HTTPS_PROXY"], utils.ImageCacheAddress()) + assert.Equal(t, params.Environment["CONTAINERD_NO_PROXY"], "") + assert.Equal(t, params.Environment["PROXY_CA"], "CA") } func TestClusterK3DoesNotSetProxyEnvironmentWithWrongVersion(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - cc.Version = "v1.12.1" + cc.Image = &container.Image{Name: "jumppad.dev/k3s:v1.12.1"} - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*config.Container) + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*types.Container) - assert.Empty(t, params.EnvVar["HTTP_PROXY"]) + assert.Empty(t, params.Environment["CONTAINERD_HTTP_PROXY"]) } -func TestClusterK3ErrorsWhenClusterExists(t *testing.T) { - md := &clients.MockContainerTasks{} - md.On("FindContainerIDs", "server."+clusterConfig.Name, mock.Anything).Return([]string{"abc"}, nil) +func TestClusterK3NoProxyIsSet(t *testing.T) { + cc, md, mk, mc := setupClusterMocks(t) + cc.Config = &Config{DockerConfig: &DockerConfig{NoProxy: []string{"test.com", "test2.com"}}} - mk := &clients.MockKubernetes{} - p := NewK8sCluster(clusterConfig, md, mk, nil, nil, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() - assert.Error(t, err) + assert.NoError(t, err) + + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*types.Container) + assert.Equal(t, "test.com,test2.com", params.Environment["CONTAINERD_NO_PROXY"]) } -func TestClusterK3PullsImageUsingBase(t *testing.T) { - cc, md, mk, mc := setupClusterMocks(t) - cc.Version = "" +func TestClusterK3ErrorsWhenClusterExists(t *testing.T) { + md := &cmocks.ContainerTasks{} + md.On("FindContainerIDs", utils.FQDN("server."+clusterConfig.Name, "", TypeK8sCluster)).Return([]string{"abc"}, nil) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + mk := &k8s.MockKubernetes{} + p := ClusterProvider{clusterConfig, md, mk, nil, nil, logger.NewTestLogger(t)} err := p.Create() - assert.NoError(t, err) - md.AssertCalled(t, "PullImage", config.Image{Name: "shipyardrun/k3s:" + k3sBaseVersion}, false) + assert.Error(t, err) } -func TestClusterK3PullsImageUsingCustom(t *testing.T) { +func TestClusterK3PullsImage(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - md.AssertCalled(t, "PullImage", config.Image{Name: "shipyardrun/k3s:v1.0.0"}, false) + md.AssertCalled(t, "PullImage", types.Image{Name: "shipyardrun/k3s:v1.27.4"}, false) } -func TestClusterK3CreatesANewVolume(t *testing.T) { +func TestClusterK3CreatesNewVolume(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -190,7 +196,7 @@ func TestClusterK3FailsWhenUnableToCreatesANewVolume(t *testing.T) { testutils.RemoveOn(&md.Mock, "CreateVolume") md.On("CreateVolume", mock.Anything, mock.Anything).Return("", fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -200,17 +206,17 @@ func TestClusterK3FailsWhenUnableToCreatesANewVolume(t *testing.T) { func TestClusterK3CreatesAServer(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*config.Container) + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*types.Container) // validate the basic details for the server container assert.Contains(t, params.Name, "server") assert.Contains(t, params.Image.Name, "shipyardrun") - assert.Equal(t, clusterNetwork.Name, params.Networks[0].Name) + assert.Equal(t, "cloud", params.Networks[0].ID) assert.True(t, params.Privileged) // validate that the volume is correctly set @@ -221,19 +227,21 @@ func TestClusterK3CreatesAServer(t *testing.T) { // validate the API port is set localPort, _ := strconv.Atoi(params.Ports[0].Local) hostPort, _ := strconv.Atoi(params.Ports[0].Host) - assert.GreaterOrEqual(t, localPort, utils.MinRandomPort) - assert.LessOrEqual(t, localPort, utils.MaxRandomPort) + assert.Equal(t, localPort, 443) assert.Equal(t, hostPort, localPort) assert.Equal(t, "tcp", params.Ports[0].Protocol) localPort, _ = strconv.Atoi(params.Ports[1].Local) hostPort, _ = strconv.Atoi(params.Ports[1].Host) - assert.GreaterOrEqual(t, hostPort, 30000) + assert.GreaterOrEqual(t, hostPort, utils.MinRandomPort) + assert.LessOrEqual(t, hostPort, utils.MaxRandomPort) + assert.Equal(t, localPort, hostPort) assert.Equal(t, "tcp", params.Ports[1].Protocol) localPort2, _ := strconv.Atoi(params.Ports[2].Local) hostPort2, _ := strconv.Atoi(params.Ports[2].Host) assert.Equal(t, localPort2, localPort+1) + assert.Equal(t, localPort2, hostPort2) assert.GreaterOrEqual(t, hostPort2, 30000) assert.Equal(t, "tcp", params.Ports[2].Protocol) @@ -241,23 +249,23 @@ func TestClusterK3CreatesAServer(t *testing.T) { assert.Equal(t, "server", params.Command[0]) assert.Contains(t, params.Command[1], params.Ports[0].Local) assert.Contains(t, params.Command[2], "--kube-proxy-arg=conntrack-max-per-core=0") - assert.Contains(t, params.Command[3], "--no-deploy=traefik") + assert.Contains(t, params.Command[3], "--disable=traefik") assert.Contains(t, params.Command[4], "--snapshotter=overlayfs") - assert.Contains(t, params.Command[5], "--tls-san=server.test.k8s-cluster.shipyard.run") + assert.Contains(t, params.Command[5], "--tls-san=server.test.k8s-cluster.jumppad.dev") } func TestClusterK3CreatesAServerWithAdditionalPorts(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - cc.Ports = []config.Port{{Local: "8080", Remote: "8080", Host: "8080"}} - cc.PortRanges = []config.PortRange{{Range: "8000-9000", EnableHost: true}} + cc.Ports = []container.Port{{Local: "8080", Remote: "8080", Host: "8080"}} + cc.PortRanges = []container.PortRange{{Range: "8000-9000", EnableHost: true}} - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*config.Container) + params := testutils.GetCalls(&md.Mock, "CreateContainer")[0].Arguments[0].(*types.Container) localPort, _ := strconv.Atoi(params.Ports[3].Local) hostPort, _ := strconv.Atoi(params.Ports[3].Host) @@ -277,7 +285,7 @@ func TestClusterK3sErrorsIfServerNOTStart(t *testing.T) { nil, ) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} startTimeout = 10 * time.Millisecond // reset the startTimeout, do not want to wait 120s err := p.Create() @@ -288,7 +296,7 @@ func TestClusterK3sDownloadsConfig(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) _, kubePath, _ := utils.CreateKubeConfigPath(cc.Name) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -305,7 +313,7 @@ func TestClusterK3sRaisesErrorWhenUnableToDownloadConfig(t *testing.T) { testutils.RemoveOn(&md.Mock, "CopyFromContainer") md.On("CopyFromContainer", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -321,7 +329,7 @@ func TestClusterK3sSetsServerInConfig(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -340,32 +348,33 @@ func TestClusterK3sSetsServerInConfig(t *testing.T) { assert.Contains(t, string(d), "https://"+utils.GetDockerIP()) } -func TestClusterK3sCreatesDockerConfig(t *testing.T) { - cc, md, mk, mc := setupClusterMocks(t) - - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) - - err := p.Create() - assert.NoError(t, err) - - // check the kubeconfig file for docker uses a network ip not localhost - - // check file has been written - _, _, dockerPath := utils.CreateKubeConfigPath(clusterConfig.Name) - f, err := os.Open(dockerPath) - assert.NoError(t, err) - defer f.Close() - - // check file contains docker ip - d, err := ioutil.ReadAll(f) - assert.NoError(t, err) - assert.Contains(t, string(d), fmt.Sprintf("server.%s", utils.FQDN(clusterConfig.Name, string(clusterConfig.Type)))) -} +// Deprecated functionality +//func TestClusterK3sCreatesDockerConfig(t *testing.T) { +// cc, md, mk, mc := setupClusterMocks(t) +// +// p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} +// +// err := p.Create() +// assert.NoError(t, err) +// +// // check the kubeconfig file for docker uses a network ip not localhost +// +// // check file has been written +// _, _, dockerPath := utils.CreateKubeConfigPath(clusterConfig.Name) +// f, err := os.Open(dockerPath) +// assert.NoError(t, err) +// defer f.Close() +// +// // check file contains docker ip +// d, err := io.ReadAll(f) +// assert.NoError(t, err) +// assert.Contains(t, string(d), fmt.Sprintf("server.%s", utils.FQDN(clusterConfig.Name, "", clusterConfig.Type))) +//} func TestClusterK3sCreatesKubeClient(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -378,7 +387,7 @@ func TestClusterK3sErrorsWhenFailedToCreateKubeClient(t *testing.T) { testutils.RemoveOn(&mk.Mock, "SetConfig") mk.Mock.On("SetConfig", mock.Anything).Return(fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -387,7 +396,7 @@ func TestClusterK3sErrorsWhenFailedToCreateKubeClient(t *testing.T) { func TestClusterK3sWaitsForPods(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -400,7 +409,7 @@ func TestClusterK3sErrorsWhenWaitsForPodsFail(t *testing.T) { testutils.RemoveOn(&mk.Mock, "HealthCheckPods") mk.On("HealthCheckPods", mock.Anything, mock.Anything).Return(fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -410,7 +419,7 @@ func TestClusterK3sStreamsLogsWhenRunning(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) mk.On("GetPodLogs", mock.Anything, mock.Anything).Return(fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -424,45 +433,76 @@ func TestClusterK3sStreamsLogsWhenRunning(t *testing.T) { func TestClusterK3sImportDockerImagesDoesNothingWhenEmpty(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - cc.Images[0].Name = "" + cc.CopyImages = append(cc.CopyImages, container.Image{Name: ""}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) + + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) md.AssertNumberOfCalls(t, "PullImage", 2) - md.AssertNotCalled(t, "PullImage", cc.Images[0], false) - md.AssertCalled(t, "PullImage", cc.Images[1], false) + md.AssertNumberOfCalls(t, "ExecuteCommand", 2) // once for the import, once to prune any build images + + // should not pull for empty image + md.AssertNotCalled(t, "PullImage", ctypes.Image{Name: cc.CopyImages[0].Name}, false) + + // should pull for non-empty image + md.AssertCalled(t, "PullImage", ctypes.Image{Name: cc.CopyImages[1].Name}, false) + + // should update the image id from the registry on the struct + // this enables us to track when the copy image changes so + // we can copy a new version to the cluster + assert.Equal(t, "abc123", cc.CopyImages[1].ID) } func TestClusterK3sImportDockerImagesPullsImages(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:abc"}) + + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - md.AssertNumberOfCalls(t, "PullImage", 3) - md.AssertCalled(t, "PullImage", cc.Images[0], false) - md.AssertCalled(t, "PullImage", cc.Images[1], false) + md.AssertNumberOfCalls(t, "PullImage", 3) //once for main image, once for each copy image + md.AssertCalled(t, "PullImage", ctypes.Image{Name: cc.CopyImages[0].Name}, false) + md.AssertCalled(t, "PullImage", ctypes.Image{Name: cc.CopyImages[1].Name}, false) } func TestClusterK3sImportDockerCopiesImages(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:abc"}) + + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) + + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - md.AssertCalled(t, "CopyLocalDockerImagesToVolume", []string{"consul:1.6.1", "vault:1.6.1"}, utils.FQDNVolumeName(utils.ImageVolumeName), false) + md.AssertCalled(t, "CopyLocalDockerImagesToVolume", []string{"test:123", "test:abc"}, utils.FQDNVolumeName(utils.ImageVolumeName), false) } func TestClusterK3sImportDockerCopyImageFailReturnsError(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:abc"}) + + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) + testutils.RemoveOn(&md.Mock, "CopyLocalDockerImagesToVolume") - md.On("CopyLocalDockerImagesToVolume", mock.Anything, mock.Anything, mock.Anything).Return("", fmt.Errorf("boom")) + md.On("CopyLocalDockerImagesToVolume", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -471,19 +511,29 @@ func TestClusterK3sImportDockerCopyImageFailReturnsError(t *testing.T) { func TestClusterK3sImportDockerRunsExecCommand(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:abc"}) + + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) + + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) - md.AssertCalled(t, "ExecuteCommand", "containerid", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + md.AssertCalled(t, "ExecuteCommand", "123", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) } func TestClusterK3sImportDockerExecFailReturnsError(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - testutils.RemoveOn(&md.Mock, "ExecuteCommand") - md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:123"}) + cc.CopyImages = append(cc.CopyImages, container.Image{Name: "test:abc"}) + + md.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(1, fmt.Errorf("boom")) + md.On("FindImageInLocalRegistry", mock.Anything).Return("abc123", nil) + + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.Error(t, err) @@ -492,7 +542,7 @@ func TestClusterK3sImportDockerExecFailReturnsError(t *testing.T) { func TestClusterK3sGeneratesCertsForConnector(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -511,9 +561,9 @@ func TestClusterK3sGeneratesCertsForConnector(t *testing.T) { func TestClusterK3sGeneratesCertsForDeployment(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - cp := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} - err := cp.Create() + err := p.Create() assert.NoError(t, err) //args := testutils.GetCalls(&mk.Mock, "Apply")[0] @@ -525,7 +575,7 @@ func TestClusterK3sGeneratesCertsForDeployment(t *testing.T) { func TestClusterK3sDeploysConnector(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -550,7 +600,7 @@ func TestClusterK3sDeploysConnector(t *testing.T) { func TestClusterK3sWaitsForConnectorStart(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Create() assert.NoError(t, err) @@ -562,11 +612,11 @@ func TestClusterK3sWaitsForConnectorStart(t *testing.T) { func TestClusterK3sDestroyGetsIDr(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Destroy() assert.NoError(t, err) - md.AssertCalled(t, "FindContainerIDs", "server."+clusterConfig.Name, clusterConfig.Type) + md.AssertCalled(t, "FindContainerIDs", "server.test.k8s-cluster.jumppad.dev") } func TestClusterK3sDestroyWithFindIDErrorReturnsError(t *testing.T) { @@ -574,7 +624,7 @@ func TestClusterK3sDestroyWithFindIDErrorReturnsError(t *testing.T) { testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("boom")) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Destroy() assert.Error(t, err) @@ -585,7 +635,7 @@ func TestClusterK3sDestroyWithNoIDReturns(t *testing.T) { testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Return(nil, nil) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Destroy() assert.NoError(t, err) @@ -597,7 +647,7 @@ func TestClusterK3sDestroyRemovesContainer(t *testing.T) { testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{"found"}, nil) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Destroy() assert.NoError(t, err) @@ -609,9 +659,9 @@ func TestClusterK3sDestroyRemovesConfig(t *testing.T) { testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{"found"}, nil) - _, dir := utils.GetClusterConfig(string(cc.Info().Type) + "." + cc.Info().Name) + dir, _, _ := utils.CreateKubeConfigPath(cc.Name) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} err := p.Destroy() assert.NoError(t, err) @@ -622,27 +672,23 @@ func TestClusterK3sDestroyRemovesConfig(t *testing.T) { func TestLookupReturnsIDs(t *testing.T) { cc, md, mk, mc := setupClusterMocks(t) - p := NewK8sCluster(cc, md, mk, nil, mc, clients.NewTestLogger(t)) + testutils.RemoveOn(&md.Mock, "FindContainerIDs") md.On("FindContainerIDs", mock.Anything, mock.Anything).Return([]string{"found"}, nil) + p := ClusterProvider{cc, md, mk, nil, mc, logger.NewTestLogger(t)} + ids, err := p.Lookup() assert.NoError(t, err) assert.Equal(t, []string{"found"}, ids) } -var clusterNetwork = config.NewNetwork("cloud") - -var clusterConfig = &config.K8sCluster{ - ResourceInfo: config.ResourceInfo{Name: "test", Type: config.TypeK8sCluster}, - Driver: "k3s", - Version: "v1.0.0", - Images: []config.Image{ - config.Image{Name: "consul:1.6.1"}, - config.Image{Name: "vault:1.6.1"}, - }, - Networks: []config.NetworkAttachment{config.NetworkAttachment{Name: "cloud"}}, +var clusterConfig = &K8sCluster{ + ResourceMetadata: htypes.ResourceMetadata{Name: "test", Type: TypeK8sCluster}, + Image: &container.Image{Name: "shipyardrun/k3s:v1.27.4"}, + Networks: []container.NetworkAttachment{container.NetworkAttachment{ID: "cloud"}}, + APIPort: 443, } var kubeconfig = ` diff --git a/pkg/config/resources/k8s/resource_cluster.go b/pkg/config/resources/k8s/resource_cluster.go index bf289e56..72c2f75e 100644 --- a/pkg/config/resources/k8s/resource_cluster.go +++ b/pkg/config/resources/k8s/resource_cluster.go @@ -31,6 +31,8 @@ type K8sCluster struct { Environment map[string]string `hcl:"environment,optional" json:"environment,omitempty"` // environment variables to set when starting the container + Config *Config `hcl:"config,block" json:"config,omitempty"` + // output parameters // Path to the Kubernetes config @@ -51,6 +53,19 @@ type K8sCluster struct { ExternalIP string `hcl:"external_ip,optional" json:"external_ip,omitempty"` } +type Config struct { + // Specifies configuration for the Docker driver. + DockerConfig *DockerConfig `hcl:"docker,block" json:"docker,omitempty"` +} + +type DockerConfig struct { + // NoProxy is a list of docker registires that should be excluded from the image cache + NoProxy []string `hcl:"no_proxy,optional" json:"no-proxy,omitempty"` + + // InsecureRegistries is a list of docker registries that should be treated as insecure + InsecureRegistries []string `hcl:"insecure_registries,optional" json:"insecure-registries,omitempty"` +} + const k3sBaseImage = "shipyardrun/k3s" const k3sBaseVersion = "v1.27.4" diff --git a/pkg/config/resources/nomad/provider_cluster.go b/pkg/config/resources/nomad/provider_cluster.go index f77fef07..321f730f 100644 --- a/pkg/config/resources/nomad/provider_cluster.go +++ b/pkg/config/resources/nomad/provider_cluster.go @@ -3,12 +3,14 @@ package nomad import ( "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "math/rand" "os" "path" "path/filepath" + "runtime" "strings" "sync" "time" @@ -154,6 +156,12 @@ func (p *ClusterProvider) Refresh() error { return nil } + // create the docker config + dockerConfigPath, err := p.createDockerConfig() + if err != nil { + return fmt.Errorf("unable to create docker config: %s", err) + } + // do we need to scale the cluster up if p.config.ClientNodes > len(p.config.ClientContainerName) { // need to scale up @@ -164,7 +172,7 @@ func (p *ClusterProvider) Refresh() error { p.log.Debug("Create client node", "ref", p.config.ID, "client", id) - fqdn, _, err := p.createClientNode(randomID(), p.config.Image.Name, utils.ImageVolumeName, p.config.ServerContainerName) + fqdn, _, err := p.createClientNode(randomID(), p.config.Image.Name, utils.ImageVolumeName, p.config.ServerContainerName, dockerConfigPath) if err != nil { return fmt.Errorf(`unable to recreate client node "%s", %s`, id, err) } @@ -338,10 +346,25 @@ func (p *ClusterProvider) createNomad() error { // set the API server port to a random number p.config.ConnectorPort = rand.Intn(utils.MaxRandomPort-utils.MinRandomPort) + utils.MinRandomPort - p.config.ConfigDir = path.Join(utils.JumppadHome(), p.config.Name, "config") + p.config.ConfigDir = path.Join(utils.JumppadHome(), strings.Replace(p.config.ID, ".", "_", -1), "config") + + // set the external IP to the address where the docker daemon is running p.config.ExternalIP = utils.GetDockerIP() - _, err = p.createServerNode(p.config.Image.ToClientImage(), volID, isClient) + // if we are using podman on windows set the external ip to localhost as podman does not bind to the main nic + if p.client.EngineInfo().EngineType == "podman" && runtime.GOOS == "windows" { + p.config.ExternalIP = "127.0.0.1" + } + + p.log.Debug("External IP for server node", "ref", p.config.ID, "ip", p.config.ExternalIP) + + // create the docker config + dockerConfigPath, err := p.createDockerConfig() + if err != nil { + return fmt.Errorf("unable to create docker config: %s", err) + } + + _, err = p.createServerNode(p.config.Image.ToClientImage(), volID, isClient, dockerConfigPath) if err != nil { return err } @@ -358,7 +381,7 @@ func (p *ClusterProvider) createNomad() error { for i := 0; i < p.config.ClientNodes; i++ { // create client node asynchronously go func(id string, image, volID, name string) { - fqdn, _, err := p.createClientNode(id, image, volID, name) + fqdn, _, err := p.createClientNode(id, image, volID, name, dockerConfigPath) if err != nil { clientError = err } @@ -414,7 +437,7 @@ func (p *ClusterProvider) createNomad() error { return nil } -func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, isClient bool) (string, error) { +func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, isClient bool, dockerConfig string) (string, error) { // set the resources for CPU, if not a client set the resources low // so that we can only deploy the connector to the server cpu := "" @@ -428,7 +451,7 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is // write the nomad config to a file os.MkdirAll(p.config.ConfigDir, os.ModePerm) serverConfigPath := path.Join(p.config.ConfigDir, "server_config.hcl") - ioutil.WriteFile(serverConfigPath, []byte(sc), os.ModePerm) + os.WriteFile(serverConfigPath, []byte(sc), os.ModePerm) // create the server // since the server is just a container create the container config and provider @@ -453,6 +476,11 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is Destination: "/cache", Type: "volume", }, + { + Source: dockerConfig, + Destination: "/etc/docker/daemon.json", + Type: "bind", + }, { Source: serverConfigPath, Destination: "/etc/nomad.d/config.hcl", @@ -460,7 +488,7 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is }, } - // Add any user config if set + // Add any server user config if set if p.config.ServerConfig != "" { vol := ctypes.Volume{ Source: p.config.ServerConfig, @@ -471,7 +499,7 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is cc.Volumes = append(cc.Volumes, vol) } - // Add any user config if set + // Add any client user config if set if p.config.ClientConfig != "" { vol := ctypes.Volume{ Source: p.config.ClientConfig, @@ -498,8 +526,6 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is cc.Volumes = append(cc.Volumes, v.ToClientVolume()) } - cc.Environment = p.config.Environment - // expose the API server port cc.Ports = []ctypes.Port{ { @@ -522,12 +548,14 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is cc.Ports = append(cc.Ports, p.config.Ports.ToClientPorts()...) cc.PortRanges = append(cc.PortRanges, p.config.PortRanges.ToClientPortRanges()...) - cc.Environment = map[string]string{} - err := p.appendProxyEnv(cc) - if err != nil { - return "", err + cc.Environment = p.config.Environment + if cc.Environment == nil { + cc.Environment = map[string]string{} } + // add the ca for the proxy + p.appendProxyEnv(cc) + id, err := p.client.CreateContainer(cc) if err != nil { return "", err @@ -538,13 +566,13 @@ func (p *ClusterProvider) createServerNode(img ctypes.Image, volumeID string, is // createClient node creates a Nomad client node // returns the fqdn, docker id, and an error if unsuccessful -func (p *ClusterProvider) createClientNode(id string, image, volumeID, serverID string) (string, string, error) { +func (p *ClusterProvider) createClientNode(id string, image, volumeID, serverID string, dockerConfig string) (string, string, error) { // generate the client config sc := dataDir + "\n" + fmt.Sprintf(clientConfig, p.config.Datacenter, serverID) // write the default config to a file clientConfigPath := path.Join(p.config.ConfigDir, "client_config.hcl") - ioutil.WriteFile(clientConfigPath, []byte(sc), os.ModePerm) + os.WriteFile(clientConfigPath, []byte(sc), os.ModePerm) // create the server // since the server is just a container create the container config and provider @@ -567,6 +595,11 @@ func (p *ClusterProvider) createClientNode(id string, image, volumeID, serverID Destination: "/cache", Type: "volume", }, + { + Source: dockerConfig, + Destination: "/etc/docker/daemon.json", + Type: "bind", + }, { Source: clientConfigPath, Destination: "/etc/nomad.d/config.hcl", @@ -600,13 +633,13 @@ func (p *ClusterProvider) createClientNode(id string, image, volumeID, serverID cc.Volumes = append(cc.Volumes, p.config.Volumes.ToClientVolumes()...) cc.Environment = p.config.Environment - - cc.Environment = map[string]string{} - err := p.appendProxyEnv(cc) - if err != nil { - return "", "", err + if cc.Environment == nil { + cc.Environment = map[string]string{} } + // add the ca for the proxy + p.appendProxyEnv(cc) + cid, err := p.client.CreateContainer(cc) // add the name of the network, we only have the id @@ -622,29 +655,68 @@ func (p *ClusterProvider) createClientNode(id string, image, volumeID, serverID return fqrn, cid, err } -func (p *ClusterProvider) appendProxyEnv(cc *ctypes.Container) error { - // load the CA from a file - ca, err := ioutil.ReadFile(filepath.Join(utils.CertsDir(""), "/root.cert")) - if err != nil { - return fmt.Errorf("unable to read root CA for proxy: %s", err) +type dockerConfig struct { + Proxies dockerProxies `json:"proxies,omitempty"` + InsecureRegistries []string `json:"insecure-registries,omitempty"` +} + +type dockerProxies struct { + HTTP string `json:"http-proxy,omitempty"` + HTTPS string `json:"https-proxy,omitempty"` + NOPROXY string `json:"no-proxy,omitempty"` +} + +// createDockerConfig creates the docker daemon config for the cluster +func (p *ClusterProvider) createDockerConfig() (string, error) { + daemonConfigPath := path.Join(p.config.ConfigDir, "daemon.json") + + // remove any existing files, fail silently + os.RemoveAll(daemonConfigPath) + + // create the config folder + os.MkdirAll(p.config.ConfigDir, os.ModePerm) + + // create the docker config + dc := dockerConfig{ + Proxies: dockerProxies{}, } - // add the netmask from the network to the proxy bypass - networkSubmasks := []string{} - for _, n := range p.config.Networks { - net, err := p.client.FindNetwork(n.ID) - if err != nil { - return fmt.Errorf("network not found: %w", err) - } + // set the insecure registries + if p.config.Config != nil && + p.config.Config.DockerConfig != nil && + len(p.config.Config.DockerConfig.InsecureRegistries) > 0 { + dc.InsecureRegistries = p.config.Config.DockerConfig.InsecureRegistries + } - networkSubmasks = append(networkSubmasks, net.Subnet) + // set the no proxy + if p.config.Config != nil && + p.config.Config.DockerConfig != nil && + len(p.config.Config.DockerConfig.NoProxy) > 0 { + dc.Proxies.NOPROXY = strings.TrimSuffix(strings.Join(p.config.Config.DockerConfig.NoProxy, ","), ",") } - proxyBypass := utils.ProxyBypass + "," + strings.Join(networkSubmasks, ",") + // set the cache details + dc.Proxies.HTTP = utils.ImageCacheAddress() + dc.Proxies.HTTPS = utils.ImageCacheAddress() + + // write the config to a file + data, err := json.MarshalIndent(dc, "", " ") + if err != nil { + return "", err + } + + err = os.WriteFile(daemonConfigPath, data, os.ModePerm) + + return daemonConfigPath, err +} + +func (p *ClusterProvider) appendProxyEnv(cc *ctypes.Container) error { + // load the CA from a file + ca, err := os.ReadFile(filepath.Join(utils.CertsDir(""), "/root.cert")) + if err != nil { + return fmt.Errorf("unable to read root CA for proxy: %s", err) + } - cc.Environment["HTTP_PROXY"] = utils.HTTPProxyAddress() - cc.Environment["HTTPS_PROXY"] = utils.HTTPSProxyAddress() - cc.Environment["NO_PROXY"] = proxyBypass cc.Environment["PROXY_CA"] = string(ca) return nil diff --git a/pkg/config/resources/nomad/resource_cluster.go b/pkg/config/resources/nomad/resource_cluster.go index 69e74c59..154b3484 100644 --- a/pkg/config/resources/nomad/resource_cluster.go +++ b/pkg/config/resources/nomad/resource_cluster.go @@ -36,6 +36,9 @@ type NomadCluster struct { Ports ctypes.Ports `hcl:"port,block" json:"ports,omitempty"` // ports to expose PortRanges ctypes.PortRanges `hcl:"port_range,block" json:"port_ranges,omitempty"` // range of ports to expose + // Configuration for the drivers + Config *Config `hcl:"config,block" json:"config,omitempty"` + // Output Parameters // The APIPort the server is running on @@ -61,6 +64,19 @@ type NomadCluster struct { const nomadBaseImage = "shipyardrun/nomad" const nomadBaseVersion = "1.6.1" +type Config struct { + // Specifies configuration for the Docker driver. + DockerConfig *DockerConfig `hcl:"docker,block" json:"docker,omitempty"` +} + +type DockerConfig struct { + // NoProxy is a list of docker registires that should be excluded from the image cache + NoProxy []string `hcl:"no_proxy,optional" json:"no-proxy,omitempty"` + + // InsecureRegistries is a list of docker registries that should be treated as insecure + InsecureRegistries []string `hcl:"insecure_registries,optional" json:"insecure-registries,omitempty"` +} + func (n *NomadCluster) Process() error { if n.Image == nil { n.Image = &ctypes.Image{Name: fmt.Sprintf("%s:%s", nomadBaseImage, nomadBaseVersion)} @@ -85,6 +101,11 @@ func (n *NomadCluster) Process() error { // Process volumes // make sure mount paths are absolute for i, v := range n.Volumes { + if v.Type != "" && v.Type != "bind" { + // only change path for bind mounts + continue + } + n.Volumes[i].Source = utils.EnsureAbsolute(v.Source, n.File) } diff --git a/pkg/config/resources/nomad/resource_cluster_test.go b/pkg/config/resources/nomad/resource_cluster_test.go index 7d7cf49f..79340772 100644 --- a/pkg/config/resources/nomad/resource_cluster_test.go +++ b/pkg/config/resources/nomad/resource_cluster_test.go @@ -43,6 +43,24 @@ func TestNomadClusterProcessSetsAbsolute(t *testing.T) { require.Equal(t, wd, c.Volumes[0].Source) } +func TestNomadClusterProcessDoesNotSetAbsoluteForNonBindMounts(t *testing.T) { + c := &NomadCluster{ + ResourceMetadata: types.ResourceMetadata{File: "./"}, + + Volumes: []ctypes.Volume{ + { + Type: "volume", + Source: "./", + Destination: "./", + }, + }, + } + + c.Process() + + require.Equal(t, "./", c.Volumes[0].Source) +} + func TestNomadClusterSetsOutputsFromState(t *testing.T) { testutils.SetupState(t, ` { diff --git a/pkg/jumppad/engine.go b/pkg/jumppad/engine.go index 6f38c53a..398dfe24 100644 --- a/pkg/jumppad/engine.go +++ b/pkg/jumppad/engine.go @@ -236,9 +236,10 @@ func (e *EngineImpl) ApplyWithVariables(path string, vars map[string]string, var e.config = c // check to see we already have an image cache - _, err = e.config.FindResourcesByType(cache.TypeImageCache) + _, err = c.FindResourcesByType(cache.TypeImageCache) if err != nil { - cache := &cache.ImageCache{ + // create a new cache with the correct registries + ca := &cache.ImageCache{ ResourceMetadata: types.ResourceMetadata{ Name: "default", Type: cache.TypeImageCache, @@ -247,27 +248,24 @@ func (e *EngineImpl) ApplyWithVariables(path string, vars map[string]string, var }, } - e.log.Debug("Creating new Image Cache", "id", cache.ID) - - p := e.providers.GetProvider(cache) - if p == nil { - // this should never happen - panic("Unable to find provider for Image Cache, Nic assured me that you should never see this message. Sorry, the monkey has broken something again") - } - - // create the cache + // create the image cache + p := e.providers.GetProvider(ca) err := p.Create() if err != nil { - return nil, fmt.Errorf("unable to create image cache: %s", err) + ca.Metadata().Properties[constants.PropertyStatus] = constants.StatusFailed + } else { + ca.Metadata().Properties[constants.PropertyStatus] = constants.StatusCreated } - cache.Properties[constants.PropertyStatus] = constants.StatusCreated - // add the new cache to the config - e.config.AppendResource(cache) + e.config.AppendResource(ca) // save the state config.SaveState(e.config) + + if err != nil { + return nil, fmt.Errorf("unable to create image cache %s", err) + } } // finally we can process and create resources @@ -547,7 +545,7 @@ func (e *EngineImpl) createCallback(r types.Resource) error { // get the image cache ic, err := e.config.FindResource("resource.image_cache.default") if err == nil { - e.log.Debug("Attaching image cache to network", "network", ic.Metadata().ID) + e.log.Debug("Adding network dependency to image cache", "network", r.Metadata().ID) ic.Metadata().DependsOn = appendIfNotContains(ic.Metadata().DependsOn, r.Metadata().ID) // reload the networks @@ -558,6 +556,41 @@ func (e *EngineImpl) createCallback(r types.Resource) error { } } + if r.Metadata().Type == cache.TypeRegistry && r.Metadata().Properties[constants.PropertyStatus] == constants.StatusCreated { + // get the image cache + ic, err := e.config.FindResource("resource.image_cache.default") + if err == nil { + // append the registry if not all ready present + bfound := false + for _, reg := range ic.(*cache.ImageCache).Registries { + if reg.Hostname == r.(*cache.Registry).Hostname { + bfound = true + break + } + } + + if !bfound { + ic.(*cache.ImageCache).Registries = append(ic.(*cache.ImageCache).Registries, *r.(*cache.Registry)) + e.log.Debug("Adding registy to image cache", "registry", r.(*cache.Registry).Hostname) + + // we now need to stop and restart the container to pick up the new registry changes + np := e.providers.GetProvider(ic) + + err := np.Destroy() + if err != nil { + e.log.Error("Unable to destroy Image Cache", "error", err) + } + + err = np.Create() + if err != nil { + e.log.Error("Unable to create Image Cache", "error", err) + } + } + } else { + e.log.Error("Unable to find Image Cache", "error", err) + } + } + return providerError } diff --git a/pkg/jumppad/engine_test.go b/pkg/jumppad/engine_test.go index 9fcb5854..34f6055d 100644 --- a/pkg/jumppad/engine_test.go +++ b/pkg/jumppad/engine_test.go @@ -107,6 +107,39 @@ func TestApplyAddsImageCache(t *testing.T) { require.Equal(t, 1, dc) } +func TestApplyAddsNetworksToImageCache(t *testing.T) { + e, _ := setupTests(t, nil) + + _, err := e.Apply("../../examples/single_file/container.hcl") + require.NoError(t, err) + + dc := e.ResourceCountForType(cache.TypeImageCache) + require.Equal(t, 1, dc) + + r, err := e.config.FindResource("resource.image_cache.default") + require.NoError(t, err) + + // network should be added as a dependency + require.Equal(t, "resource.network.onprem", r.Metadata().DependsOn[0]) +} + +func TestApplyAddsCustomRegistriesToImageCache(t *testing.T) { + e, _ := setupTests(t, nil) + + _, err := e.Apply("../../examples/registries") + require.NoError(t, err) + + dc := e.ResourceCountForType(cache.TypeImageCache) + require.Equal(t, 1, dc) + + dc = e.ResourceCountForType(cache.TypeRegistry) + require.Equal(t, 2, dc) + + r, err := e.config.FindResource("resource.image_cache.default") + require.NoError(t, err) + require.Len(t, r.(*cache.ImageCache).Registries, 2) +} + func TestApplyWithSingleFileAndVariables(t *testing.T) { e, mp := setupTests(t, nil) diff --git a/pkg/jumppad/init.go b/pkg/jumppad/init.go index 2d3bf18e..1bc0c332 100644 --- a/pkg/jumppad/init.go +++ b/pkg/jumppad/init.go @@ -50,9 +50,11 @@ func init() { config.RegisterResource(random.TypeRandomUUID, &random.RandomUUID{}, &random.RandomUUIDProvider{}) config.RegisterResource(random.TypeRandomPassword, &random.RandomPassword{}, &random.RandomPasswordProvider{}) config.RegisterResource(random.TypeRandomCreature, &random.RandomCreature{}, &random.RandomCreatureProvider{}) + config.RegisterResource(cache.TypeRegistry, &cache.Registry{}, &null.Provider{}) config.RegisterResource(template.TypeTemplate, &template.Template{}, &template.TemplateProvider{}) config.RegisterResource(terraform.TypeTerraform, &terraform.Terraform{}, &terraform.TerraformProvider{}) + // register providers for the default types config.RegisterResource(types.TypeModule, &types.Module{}, &null.Provider{}) config.RegisterResource(types.TypeOutput, &types.Output{}, &null.Provider{}) config.RegisterResource(types.TypeVariable, &types.Variable{}, &null.Provider{}) diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index 24ea9774..4693b6bd 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -256,34 +256,20 @@ func TestGetLocalIPAndHostnameReturnsCorrectly(t *testing.T) { assert.NotEqual(t, host, "localhost") } -func TestHTTPProxyAddressReturnsDefaultWhenEnvNotSet(t *testing.T) { - proxy := HTTPProxyAddress() +func TestImageCacheAddressReturnsDefaultWhenEnvNotSet(t *testing.T) { + proxy := ImageCacheAddress() assert.Equal(t, jumppadProxyAddress, proxy) } -func TestHTTPSProxyAddressReturnsDefaultWhenEnvNotSet(t *testing.T) { - proxy := HTTPSProxyAddress() - - assert.Equal(t, jumppadProxyAddress, proxy) -} - -func TestHTTPProxyAddressReturnsEnvWhenEnvSet(t *testing.T) { +func TestImageCacheAddressReturnsEnvWhenEnvSet(t *testing.T) { httpProxy := "http://myproxy.com" - os.Setenv("HTTP_PROXY", httpProxy) - proxy := HTTPProxyAddress() + os.Setenv("IMAGE_CACHE_ADDR", httpProxy) + proxy := ImageCacheAddress() assert.Equal(t, httpProxy, proxy) } -func TestHTTPSProxyAddressReturnsEnvWhenEnvSet(t *testing.T) { - httpsProxy := "https://myproxy.com" - os.Setenv("HTTPS_PROXY", httpsProxy) - proxy := HTTPSProxyAddress() - - assert.Equal(t, httpsProxy, proxy) -} - var testData = ` { "checks": "test", diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 0fb29678..485b2c8d 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/ioutil" + "math/rand" "net" "net/url" "os" @@ -481,22 +482,11 @@ func GetLocalIPAndHostname() (string, string) { return "127.0.0.1", "localhost" } -// HTTPProxyAddress returns the default HTTPProxy used by +// ImageCacheADDR returns the default Image cache used by // Nomad and Kubernetes clusters unless the environment variable -// HTTP_PROXY is set when it returns this value -func HTTPProxyAddress() string { - if p := os.Getenv("HTTP_PROXY"); p != "" { - return p - } - - return jumppadProxyAddress -} - -// HTTPSProxyAddress returns the default HTTPProxy used by -// Nomad and Kubernetes clusters unless the environment variable -// HTTPS_PROXY is set when it returns this value -func HTTPSProxyAddress() string { - if p := os.Getenv("HTTPS_PROXY"); p != "" { +// IMAGE_CACHE_ADDR is set when it returns this value +func ImageCacheAddress() string { + if p := os.Getenv("IMAGE_CACHE_ADDR"); p != "" { return p } @@ -567,6 +557,25 @@ func ChecksumFromInterface(i interface{}) (string, error) { return HashString(string(json)) } +// RandomAvailablePort returns a random free port in the given range +func RandomAvailablePort(from, to int) (int, error) { + + // checks 10 times for a free port + for i := 0; i < 10; i++ { + port := rand.Intn(to-from) + from + + // check if the port is available + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err == nil { + ln.Close() + + return port, nil + } + } + + return 0, fmt.Errorf("unable to find a free port in the range %d-%d", from, to) +} + func incIP(ip net.IP) net.IP { // allocate a new IP newIp := make(net.IP, len(ip))