Merge pull request #318 from TrekkieCoder/main

PR - Enhancements to k3s cicd
loxilb-io · Jun 14, 2023 · eb9eada · eb9eada
2 parents 13ce61e + 93bb5a6
commit eb9eada
Show file tree

Hide file tree

Showing 7 changed files with 204 additions and 95 deletions.
diff --git a/cicd/k0s-weave/config.sh b/cicd/k0s-weave/config.sh
@@ -2,6 +2,41 @@
 
 source ../common.sh
 
+
+function wait_k0s_cluster_ready {
+    Res=$(sudo k0s kubectl get pods -A |
+    while IFS= read -r line; do
+        if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then
+            echo "not ready"
+            return
+        fi
+    done)
+    if [[ $Res == *"not ready"* ]]; then
+        return 1
+    fi
+    return 0
+}
+
+function wait_k0s_cluster_ready_full {
+  i=1
+  nr=0
+  for ((;;)) do
+    wait_k0s_cluster_ready
+    nr=$?
+    if [[ $nr == 0 ]]; then
+        echo "Cluster is ready"
+        break
+    fi
+    i=$(( $i + 1 ))
+    if [[ $i -ge 40 ]]; then
+        echo "Cluster is not ready.Giving up"
+        exit 1
+    fi
+    echo "Cluster is not ready...."
+    sleep 10
+  done
+}
+
 echo "#########################################"
 echo "Spawning all hosts"
 echo "#########################################"
@@ -161,6 +196,8 @@ fi
 sudo chown bird:bird /var/log/bird.log
 sudo systemctl restart bird
 
+wait_k0s_cluster_ready_full
+
 sleep 30
 
 # Start nginx pods and services for test(using kube-loxilb)
@@ -180,5 +217,7 @@ sleep 30
 # External LB service must be created by now
 sudo k0s kubectl get svc
 
+wait_k0s_cluster_ready_full
+
 # Route back to user
 sudo ip route add 1.1.1.1/32 via 12.12.12.1
diff --git a/cicd/k0s-weave/rmweave.sh b/cicd/k0s-weave/rmweave.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+sudo curl -L git.io/weave -o /usr/local/bin/weave
+sudo chmod a+x /usr/local/bin/weave
+weave reset --force
+sudo rm -fr /usr/local/bin/weave
+
+ip a | grep 'vethwepl.*\@' -oP | while read -r line ; do
+    veth=${line::-1}
+    if [[ $veth =~ [0-9] ]]; then
+      echo check $veth
+      pid=$(echo $veth | tr -dc '0-9')
+      if ! ps -p $pid > /dev/null; then
+        echo deleting $veth
+        ip link delete $veth >&2
+      else
+        echo $veth still running
+      fi
+    else
+      echo $veth veth has no number in it and will not be deleted
+    fi
+done
diff --git a/cicd/k0s-weave/validation.sh b/cicd/k0s-weave/validation.sh
@@ -30,65 +30,81 @@ done
 ## Any routing updates  ??
 sleep 30
 
-echo $extIP
-echo "End Points List"
-sudo k0s kubectl get endpoints -A
-echo "SVC List"
-sudo k0s kubectl get svc
-echo "Pod List"
-sudo k0s kubectl get pods -A
-echo "LB List"
-$dexec llb1 loxicmd get lb -o wide
-echo "EP List"
-$dexec llb1 loxicmd get ep -o wide
+echo "ExternalIP $extIP"
 
-out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "cluster-k0s (tcp)  [OK]"
-else
-  echo "cluster-k0s (tcp)  [FAILED]"
+print_debug_info() {
   ## Dump some debug info
-  echo "llb1 lb-info"
-  $dexec llb1 loxicmd get lb
-  echo "llb1 route-info"
+  echo "**** k0s svc info ****"
+  sudo k0s kubectl get svc
+  echo "**** k0s pods info ****"
+  sudo k0s kubectl get pods -A
+  echo "**** k0s endpoints info ****"
+  sudo k0s kubectl get endpoints
+
+  echo "**** llb1 lb-info ****"
+  $dexec llb1 loxicmd get lb -o wide
+  echo "**** loxilb ep-info ****"
+  $dexec llb1 loxicmd get ep -o wide
+  echo "**** llb1 route-info ****"
   $dexec llb1 ip route
-  echo "llb2 lb-info"
-  $dexec llb2 loxicmd get lb
-  echo "llb2 route-info"
+
+  echo "**** llb2 lb-info ****"
+  $dexec llb2 loxicmd get lb -o wide
+  echo "**** loxilb ep-info ****"
+  $dexec llb1 loxicmd get ep -o wide
+  echo "**** llb2 route-info ****"
   $dexec llb2 ip route
-  echo "r1 route-info"
+
+  echo "**** r1 route-info ****"
   $dexec r1 ip route
-  exit 1
+}
+
+code=0
+print_debug_info
+
+out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
+if [[ ${out} == *"Welcome to nginx"* ]]; then
+  echo "cluster-k0s (tcp)  [OK]"
+else
+  code=1
 fi
 
 out=$($hexec user ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
   echo "cluster-k0s (udp)  [OK]"
 else
   echo "cluster-k0s (udp)  [FAILED]"
-  exit 1
+  code=1
 fi
 
 out=$($hexec user ../common/sctp_client 1.1.1.1 34951 $extIP 55004)
 if [[ ${out} == *"server1"* ]]; then
   echo "cluster-k0s (sctp) [OK]"
 else
   echo "cluster-k0s (sctp) [FAILED]"
-  exit 1
+  code=1
 fi
 
 out=$($hexec user ../common/sctp_client 1.1.1.1 0 $extIP 55005)
 if [[ ${out} == *"server1"* ]]; then
   echo "cluster-k0s (sctp2) [OK]"
 else
   echo "cluster-k0s (sctp2) [FAILED]"
-  exit 1
+  code=1
 fi
 
 out=$($hexec user ../common/udp_client $extIP 55006)
 if [[ ${out} == *"Client"* ]]; then
   echo "cluster-k0s (udp2)  [OK]"
 else
   echo "cluster-k0s (udp2)  [FAILED]"
+  code=1  
+fi
+
+if [[ $code -eq 1 ]]; then
+  echo "cluster-k0s failed"
   exit 1
 fi
+
+exit
+
diff --git a/cicd/k3s-calico/config.sh b/cicd/k3s-calico/config.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 source ../common.sh
+source ../k3s_common.sh
 
 echo "#########################################"
 echo "Spawning all hosts"
@@ -120,7 +121,7 @@ else
 
   # Install k3s without external cloud-manager and disabled servicelb
   #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
-  curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh -
+  curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh -
 
   sleep 10
 
@@ -154,6 +155,10 @@ sudo chown bird:bird /var/log/bird.log
 sudo systemctl restart bird
 
 sleep 10
+
+# Wait for cluster to be ready
+wait_cluster_ready_full
+
 # Start nginx pods and services for test
 kubectl $KUBECONFIG apply -f nginx.yml
 kubectl $KUBECONFIG apply -f nginx-svc-lb.yml
@@ -175,3 +180,6 @@ kubectl $KUBECONFIG get svc
 
 # Route back to user
 sudo ip route add 1.1.1.1/32 via 12.12.12.1
+
+# Wait for cluster to be ready
+wait_cluster_ready_full
diff --git a/cicd/k3s-calico/validation.sh b/cicd/k3s-calico/validation.sh
@@ -30,62 +30,71 @@ done
 ## Any routing updates  ??
 sleep 30
 
-echo $extIP
-$dexec llb1 loxicmd get lb -o wide
-$dexec llb1 loxicmd get ep -o wide
+echo "ExternalIP $extIP"
 
-out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) 
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo calico-k3s [OK]
-else
-  echo calico-k3s [FAILED]
+print_debug_info() {
   ## Dump some debug info
-  echo "llb1 lb-info"
-  $dexec llb1 loxicmd get lb
-  echo "llb1 route-info"
+  echo "**** k3s svc info ****"
+  sudo kubectl $KUBECONFIG get svc
+  echo "**** k3s pods info ****"
+  sudo kubectl $KUBECONFIG get pods -A
+
+  echo "**** llb1 lb-info ****"
+  $dexec llb1 loxicmd get lb -o wide
+  echo "**** loxilb ep-info ****"
+  $dexec llb1 loxicmd get ep -o wide
+  echo "**** llb1 route-info ****"
   $dexec llb1 ip route
-  echo "llb2 lb-info"
-  $dexec llb2 loxicmd get lb
-  echo "llb2 route-info"
+
+  echo "**** llb2 lb-info ****"
+  $dexec llb2 loxicmd get lb -o wide
+  echo "**** loxilb ep-info ****"
+  $dexec llb1 loxicmd get ep -o wide
+  echo "**** llb2 route-info ****"
   $dexec llb2 ip route
-  echo "r1 route-info"
+
+  echo "**** r1 route-info ****"
   $dexec r1 ip route
-  exit 1
+}
+
+code=0
+print_debug_info
+
+out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) 
+if [[ ${out} == *"Welcome to nginx"* ]]; then
+  echo "calico-k3s (ccm) [OK]"
+else
+  echo "calico-k3s (ccm) [FAILED]"
+  code=1
 fi
 
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "calico-k3s (kube-loxilb) [OK]"
+  echo "calico-k3s (kube-loxilb) tcp [OK]"
 else
-  echo "calico-k3s (kube-loxilb) [FAILED]"
-  ## Dump some debug info
-  echo "llb1 lb-info"
-  $dexec llb1 loxicmd get lb
-  echo "llb1 route-info"
-  $dexec llb1 ip route
-  echo "llb2 lb-info"
-  $dexec llb2 loxicmd get lb
-  echo "llb2 route-info"
-  $dexec llb2 ip route
-  echo "r1 route-info"
-  $dexec r1 ip route
-  exit 1
+  echo "calico-k3s (kube-loxilb) tcp [FAILED]"
+  code=1
 fi
 
 out=$($hexec user timeout 30 ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
-  echo calico-k3s udp [OK]
+  echo "calico-k3s (kube-loxillb) udp [OK]"
 else
-  echo calico-k3s udp [FAILED]
-  exit 1
+  echo "calico-k3s (kube-loxillb) udp [FAILED]"
+  code=1
 fi
 
 out=$($hexec user timeout 30 ../common/sctp_client 1.1.1.1 41291 $extIP 55004)
 if [[ ${out} == *"server1"* ]]; then
-  echo calico-k3s sctp [OK]
+  echo "calico-k3s (kube-loxillb) sctp [OK]"
 else
-  echo calico-k3s sctp [FAILED]
+  echo "calico-k3s (kube-loxillb) sctp [FAILED]"
+  code=1
+fi
+
+if [[ $code -eq 1 ]]; then
+  echo "calico-k3s failed"
   exit 1
 fi
 

diff --git a/cicd/k3s-flannel/config.sh b/cicd/k3s-flannel/config.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 source ../common.sh
+source ../k3s_common.sh
 
 echo "#########################################"
 echo "Spawning all hosts"
@@ -119,8 +120,8 @@ else
   echo "Start K3s installation"
 
   # Install k3s without external cloud-manager and disabled servicelb
-  curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
-  #curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
+  #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
+  curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
 
   sleep 10
 
@@ -148,6 +149,9 @@ fi
 sudo chown bird:bird /var/log/bird.log
 sudo systemctl restart bird
 
+# Wait for cluster readiness
+wait_cluster_ready_full
+
 sleep 10
 # Start nginx pods and services for test
 kubectl $KUBECONFIG apply -f nginx.yml
@@ -165,3 +169,6 @@ sleep 30
 
 # External LB service must be created by now
 kubectl $KUBECONFIG get svc
+
+# Wait for cluster readiness
+wait_cluster_ready_full