kubernetes deployment of Vaquero (CiscoCloud#37)

juledwar · Feb 17, 2017 · d91ba48 · d91ba48
1 parent b1d3bda
commit d91ba48
Show file tree

Hide file tree

Showing 10 changed files with 278 additions and 0 deletions.
diff --git a/kube/README.md b/kube/README.md
@@ -0,0 +1,28 @@
+#Kubernetes in the VirtualEnv
+
+It is highly recommended to stand up VS boxes with V_DEV=1, to get more memory and compute allocated to the VM. Performance will drag if not used.
+
+
+*Note: This is for sake fo example and testing, should only be used for testing, we do not set up external routable IPs for the vs-service. Since no external IP is set up for the vaquero-server(s), the vaquero-agent MUST be deployed in the kubernetes cluster, and use the service `clusterIP` to reach the vaquero-server(s)*
+
+
+`V_DEV=1 vagrant up` : VS-1 will be the K8s master, VS-1 and all subsequent VS-<num> machines will be K8s minions.
+
+##For all VS machines you stand up (starting kubernetes services)
+1. `vagrant ssh vs-<num>`
+2. `sudo ./kube-start.sh`
+
+##On any of the VS machines
+3. `/vagrant/kube/vs.sh` - sets up a configmap, secret, reploys the `vs-rc` and `vs-svc`
+4. `kubectl get svc` - Pull the "clusterIP" off of the `vs-svc`
+5. Paste the  "clusterIP" from step 4 into the agent config that lives at `/vagrant/kube/va-config.yaml`
+6. `/vagrant/kube/va.sh` - creates a configmap and runs the `va-pod`
+
+##On your physical host
+7. `./create_cluster/cluster.sh -d core-cloud` - Booting machines like you would normally.
+
+
+If configuration changes are required, you must `kubectl delete configmap <configmap-name>` and then `kubectl create configmap <configmap-name> --from-file=<path-to-file>`. See the `va.sh` and `vs.sh` for reference. Renaming the configs will cause the key names to change, and force you to update the `*Pod.yaml` files.
+
+
+If you wish to use your own container, replace the `image` in the RC or Pod files. Below it add `imagePullPolocy: IfNotPresent` to ensure kubernetes will look at your local docker images before checking the web. *To use local containers the version CANNOT be `latest`, if you use latest it will always pull from the internet*
diff --git a/kube/va-config.yaml b/kube/va-config.yaml
@@ -0,0 +1,9 @@
+ServerClient:
+  Addr: "<INSERT vs-service CLUSTERIP here>"
+  Port: 24601
+  InsecureSkipVerify: true
+SavePath: "/var/vaquero"
+Log:
+  Level: info
+  Location: stdout
+  LogType: text
diff --git a/kube/va.sh b/kube/va.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+kubectl create configmap va-config --from-file=/vagrant/kube/va-config.yaml
+kubectl create -f /vagrant/kube/vaPod.yaml
diff --git a/kube/vaPod.yaml b/kube/vaPod.yaml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: va-pod
+  labels:
+    app: va-pod
+spec:
+  hostNetwork: true
+  containers:
+    - name: va-pod
+      image: shippedrepos-docker-vaquero.bintray.io/vaquero/vaquero:latest
+      args:
+        - "agent"
+        - "--config"
+        - "/config/config.yaml"
+      env:
+        - name: VAQUERO_SHARED_SECRET
+          value: "shared"
+        - name: VAQUERO_SERVER_SECRET
+          value: "server"
+        - name: VAQUERO_SITE_ID
+          value: "test-site"
+        - name: VAQUERO_AGENT_ID
+          value: "test-agent"
+      ports:
+        - containerPort: 24602
+      volumeMounts:
+        - mountPath: /config
+          name: config
+        - mountPath: /var/vaquero/files
+          name: assets
+  volumes:
+    - name: config
+      configMap:
+        name: va-config
+        items:
+        - key: va-config.yaml
+          path: config.yaml
+    - name: assets
+      hostPath:
+        path: /var/vaquero/files
diff --git a/kube/vs-config.yaml b/kube/vs-config.yaml
@@ -0,0 +1,32 @@
+UserApi:
+  Addr: "0.0.0.0"
+  Port: 24604
+ServerAPI:
+  Addr: "0.0.0.0"
+  Port: 24601
+  PrivateKey: "/secret/server.key"
+  PublicKey: "/secret/server.pem"
+SavePath: "/var/vaquero"
+Gitter:
+  Endpoint: "/postreceive"
+  Timeout: 2
+  Addr: "0.0.0.0"
+  Port: 24603
+Etcd:
+  Endpoints:
+  - "http://10.10.10.5:2379"
+  - "http://10.10.10.6:2379"
+  - "http://10.10.10.7:2379"
+  Timeout: 5
+  Retry: 3
+  HA: true
+LocalDir:
+  PollInterval: 2
+SoT:
+- Local:
+    ID: "vaquero-local"
+    Root: /dm
+Log:
+  Level: info
+  Location: stdout
+  LogType: text
diff --git a/kube/vs.sh b/kube/vs.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+kubectl create secret generic ssh-key --from-file=/vagrant/provision_files/secret/server.pem --from-file=/vagrant/provision_files/secret/server.key
+kubectl create configmap vs-config --from-file=/vagrant/kube/vs-config.yaml
+kubectl create -f /vagrant/kube/vsRc.yaml
+kubectl create -f /vagrant/kube/vsSvc.yaml
diff --git a/kube/vsDep.yaml b/kube/vsDep.yaml
@@ -0,0 +1,63 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: vs-dep
+spec:
+  replicas: 3
+  template:
+    metadata:
+      name: vs-pod
+      labels:
+        app: vs-pod
+    spec:
+      containers:
+        - name: vs-pod
+          image: shippedrepos-docker-vaquero.bintray.io/vaquero/vaquero
+          args:
+            - "server"
+            - "--config"
+            - "/config/config.yaml"
+          env:
+            - name: VAQUERO_SHARED_SECRET
+              value: "shared"
+            - name: VAQUERO_SERVER_SECRET
+              value: "server"
+            - name: VAQUERO_SITE_ID
+              value: "test-site"
+            - name: VAQUERO_AGENT_ID
+              value: "test-agent"
+          ports:
+            - containerPort: 24601
+            - containerPort: 24603
+            - containerPort: 24604
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 24604
+            initialDelaySeconds: 5
+            periodSeconds: 1
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /secret
+              name: ssh
+            - mountPath: /dm
+              name: dm
+      volumes:
+        - name: dm
+          hostPath:
+            path: /vagrant/local
+        - name: config
+          configMap:
+            name: vs-config
+            items:
+            - key: vs-config.yaml
+              path: config.yaml
+        - name: ssh
+          secret:
+            secretName: ssh-key
+            items:
+            - key: server.pem
+              path: server.pem
+            - key: server.key
+              path: server.key
diff --git a/kube/vsRc.yaml b/kube/vsRc.yaml
@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: vs-rc
+spec:
+  replicas: 3
+  selector:
+    app: vs-pod
+  template:
+    metadata:
+      name: vs-pod
+      labels:
+        app: vs-pod
+    spec:
+      containers:
+        - name: vs-pod
+          image: shippedrepos-docker-vaquero.bintray.io/vaquero/vaquero:latest
+          args:
+            - "server"
+            - "--config"
+            - "/config/config.yaml"
+          env:
+            - name: VAQUERO_SHARED_SECRET
+              value: "shared"
+            - name: VAQUERO_SERVER_SECRET
+              value: "server"
+            - name: VAQUERO_SITE_ID
+              value: "test-site"
+            - name: VAQUERO_AGENT_ID
+              value: "test-agent"
+          ports:
+            - containerPort: 24601
+            - containerPort: 24603
+            - containerPort: 24604
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 24604
+            initialDelaySeconds: 5
+            periodSeconds: 1
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /secret
+              name: ssh
+            - mountPath: /dm
+              name: dm
+      volumes:
+        - name: dm
+          hostPath:
+            path: /vagrant/local
+        - name: config
+          configMap:
+            name: vs-config
+            items:
+            - key: vs-config.yaml
+              path: config.yaml
+        - name: ssh
+          secret:
+            secretName: ssh-key
+            items:
+            - key: server.pem
+              path: server.pem
+            - key: server.key
+              path: server.key
diff --git a/kube/vsSvc.yaml b/kube/vsSvc.yaml
@@ -0,0 +1,22 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: vs-svc
+spec:
+  type: NodePort
+  selector:
+    app: vs-pod
+  ports:
+  - name: v1
+    protocol: TCP
+    port: 24601
+    nodePort: 30601
+  - name: v3
+    protocol: TCP
+    port: 24603
+    nodePort: 30603
+  - name: v4
+    protocol: TCP
+    port: 24604
+    nodePort: 30604
diff --git a/provision_scripts/kube-config.sh b/provision_scripts/kube-config.sh
@@ -4,6 +4,7 @@ KUBE_CONF=/etc/kubernetes/config
 API_CONF=/etc/kubernetes/apiserver
 FLA_CONF=/etc/sysconfig/flanneld
 LET_CONF=/etc/kubernetes/kubelet
+PXY_CONF=/etc/kubernetes/proxy
 
 if [ $1 = $2 ]; then
 	echo "Loading Kube-Master"
@@ -19,6 +20,7 @@ if [ $1 = $2 ]; then
 
 	sed -i "s#FLANNEL_ETCD_ENDPOINTS=\"http://127.0.0.1:2379\"#FLANNEL_ETCD_ENDPOINTS=\"$3\"#" $FLA_CONF
 	sed -i "s#/atomic.io/network#kube-centos/network#" $FLA_CONF
+	sed -i "s/#FLANNEL_OPTIONS=\"\"/FLANNEL_OPTIONS=\"--iface=enp0s8\"/" $FLA_CONF
 
 	setenforce 0
 	systemctl disable firewalld
@@ -29,13 +31,19 @@ fi
 
 echo "Loading Kube-Minion"
 
+sed -i "s#KUBE_MASTER=\"--master=http://127.0.0.1:8080\"#KUBE_MASTER=\"--master=http://$2:8080\"#" $KUBE_CONF
+
 sed -i "s#KUBELET_ADDRESS=\"--address=127.0.0.1\"#KUBELET_ADDRESS=\"--address=$1\"#" $LET_CONF
 sed -i "s#KUBELET_API_SERVER=\"--api-servers=http://127.0.0.1:8080\"#KUBELET_API_SERVER=\"--api-servers=http://$2:8080\"#" $LET_CONF
 sed -i "s#KUBELET_HOSTNAME=\"--hostname-override=127.0.0.1\"#KUBELET_HOSTNAME=\"--hostname-override=$1\"#" $LET_CONF
 sed -i "s/# KUBELET_PORT/KUBELET_PORT/" $LET_CONF
+sed -i "s#KUBELET_ARGS=\"\"#KUBELET_ARGS=\"--cluster-dns='10.0.2.3'\"#" $LET_CONF
 
 sed -i "s#FLANNEL_ETCD_ENDPOINTS=\"http://127.0.0.1:2379\"#FLANNEL_ETCD_ENDPOINTS=\"$3\"#" $FLA_CONF
 sed -i "s#/atomic.io/network#kube-centos/network#" $FLA_CONF
+sed -i "s/#FLANNEL_OPTIONS=\"\"/FLANNEL_OPTIONS=\"--iface=enp0s8\"/" $FLA_CONF
+
+sed -i "s#KUBE_PROXY_ARGS=\"\"#KUBE_PROXY_ARGS=\"--hostname-override=$1, --cluster-cidr=10.10.10.0/28\"#" $PXY_CONF
 
 echo "Done configuring Kube-Minion"