Bash scripts refactoring into Golang

.github/workflows/build_setup.yml

@@ -0,0 +1,38 @@
+name: vHive setup scripts build tests
+
+on:
+    push:
+      branches: [ main ]
+      paths-ignore:
+      - 'docs/**'
+      - '**.md'
+    pull_request:
+      branches: [ main ]
+      paths-ignore:
+      - 'docs/**'
+      - '**.md'
+    workflow_dispatch:
+
+env:
+    GOOS: linux
+    GO111MODULE: on
+
+jobs:
+    build_setup:
+        name: Build setup scripts
+        runs-on: ubuntu-20.04
+        steps:
+        - name: Set up Go 1.19
+          uses: actions/setup-go@v4
+          with:
+            go-version: '1.19'
+
+        - name: Check out the code
+          uses: actions/checkout@v3
+
+        - name: Build scripts
+          run: pushd scripts && go build -o setup_tool && popd
+
+        - name: Show scripts help info
+          run: ./scripts/setup_tool -h
+

.github/workflows/cri_stock_containerd_test.yml

@@ -22,16 +22,24 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
 
+    - name: Set up golang
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.19'
+
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
+
     - name: Start Containerd
       run: sudo containerd &
 
     - name: Create k8s cluster
-      run: ./scripts/cluster/create_one_node_cluster.sh stock-only && sleep 2m
+      run: ./scripts/setup_tool create_one_node_cluster stock-only && sleep 2m
 
     - name: Deploy zipkin
       env:
         KUBECONFIG: /etc/kubernetes/admin.conf
-      run: ./scripts/setup_zipkin.sh && sleep 5s
+      run: ./scripts/setup_tool setup_zipkin && sleep 5s
 
     - name: Create helloworld container
       run: KUBECONFIG=/etc/kubernetes/admin.conf kn service create helloworld-go --image gcr.io/knative-samples/helloworld-go --env TARGET="vHive CRI test"

.github/workflows/cri_tests.yml

@@ -28,6 +28,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Set up golang
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.19'
+
+      - name: Build setup scripts
+        run: pushd scripts && go build -o setup_tool && popd
+
       - name: Setup vHive CRI test environment
         run: ./scripts/github_runner/setup_cri_test_env.sh ${{ inputs.sandbox }}
 
@@ -41,6 +49,8 @@ jobs:
           name: ctrd-logs
           path: |
             /tmp/ctrd-logs/${{ github.run_id }}
+            ${{ github.workspace }}/*.log
+            ${{ github.workspace }}/scripts/github_runner/*.log
 
       - name: Cleaning
         if: ${{ always() }}

.github/workflows/integration_tests.yml

@@ -50,8 +50,11 @@ jobs:
       with:
         lfs: true
 
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
+
     - name: Pull binaries
-      run: ./scripts/setup_firecracker_containerd.sh
+      run: ./scripts/setup_tool setup_firecracker_containerd
 
     - name: Build
       run: go build -race -v -a ./...
@@ -66,7 +69,9 @@ jobs:
         name: ctrd-logs
         path: |
           /tmp/ctrd-logs/
+          ${{ github.workspace }}/*.log
+          ${{ github.workspace }}/scripts/github_runner/*.log
 
     - name: Cleaning
       if: ${{ always() }}
-      run: ./scripts/clean_fcctr.sh
+      run: ./scripts/setup_tool clean_fcctr

.github/workflows/nightly_tests.yml

@@ -36,8 +36,11 @@ jobs:
       with:
         lfs: true
 
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
+
     - name: Pull binaries
-      run: ./scripts/setup_firecracker_containerd.sh
+      run: ./scripts/setup_tool setup_firecracker_containerd
 
     - name: Build
       run: go build -race -v -a ./...
@@ -47,7 +50,7 @@ jobs:
 
     - name: Cleaning
       if: ${{ always() }}
-      run: ./scripts/clean_fcctr.sh
+      run: ./scripts/setup_tool clean_fcctr
 
   cri-tests:
     name: CRI tests (nightly)
@@ -80,8 +83,11 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
 
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
+
     - name: Setup firecracker-containerd
-      run: ./scripts/setup_firecracker_containerd.sh
+      run: ./scripts/setup_tool setup_firecracker_containerd
 
     - name: Build
       run: go build

.github/workflows/stargz_tests.yml

@@ -47,6 +47,9 @@ jobs:
         with:
           go-version: '1.19'
 
+      - name: Build setup scripts
+        run: pushd scripts && go build -o setup_tool && popd
+
       - name: Create k8s Kind Cluster
         run: bash ./scripts/stargz/01-kind.sh
 
@@ -64,7 +67,7 @@ jobs:
           kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}"
 
       - name: Setup stock-only node
-        run: ./scripts/cloudlab/setup_node.sh stock-only use-stargz
+        run: ./scripts/setup_tool setup_node stock-only use-stargz
 
       - name: Check containerd service is running
         run: sudo screen -list | grep "containerd"

.github/workflows/unit_tests.yml

@@ -42,11 +42,14 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
 
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
+
     - name: Setup System
-      run: ./scripts/setup_system.sh
+      run: ./scripts/setup_tool setup_system
 
     - name: Install PMU tools
-      run: ./scripts/install_pmutools.sh
+      run: ./scripts/setup_tool install_pmutools
 
     - name: Build
       run: go build -race -v -a ./...
@@ -80,9 +83,12 @@ jobs:
       uses: actions/checkout@v3
       with:
         lfs: true
+
+    - name: Build setup scripts
+      run: pushd scripts && go build -o setup_tool && popd
 
     - name: Pull binaries
-      run: ./scripts/setup_firecracker_containerd.sh
+      run: ./scripts/setup_tool setup_firecracker_containerd
 
     - name: Build
       run: go build -race -v -a ./...
@@ -94,4 +100,4 @@ jobs:
 
     - name: Cleaning
       if: ${{ always() }}
-      run: ./scripts/clean_fcctr.sh
+      run: ./scripts/setup_tool clean_fcctr

.gitignore

@@ -25,3 +25,4 @@
 
 # Binaries
 vhive
+setup_tool

configs/.wordlist.txt

@@ -314,6 +314,8 @@ NEURALWALKER
 neuroevolution
 NewServer
 Nicopoulos
+Nvidia
+nvidia
 nmi
 NoC
 NOC

configs/setup/knative.json

@@ -0,0 +1,17 @@
+{
+	"KnativeVersion": "1.9.2",
+	"KnativeCLIBranch": "release-1.9",
+	"KnativeCLIRepoUrl": "https://github.com/knative/client.git",
+	"NotStockOnlyKnativeServingYamlUrlPrefix": "https://raw.githubusercontent.com/vhive-serverless/vHive/main/configs/knative_yamls",
+	"IstioVersion": "1.16.3",
+	"IstioOperatorConfigPath": "configs/istio/istio-minimal-operator.yaml",
+	"IstioDownloadUrlTemplate": "https://github.com/istio/istio/releases/download/%s/istio-%s-linux-%s.tar.gz",
+	"IstioZipkinVersion": "1.16",
+	"IstioZipkinDownloadUrlTemplate": "https://raw.githubusercontent.com/istio/istio/release-%s/samples/addons/extras/zipkin.yaml",
+	"MetalLBVersion": "0.13.9",
+	"LocalRegistryRepoVolumeSize": "5Gi",
+	"LocalRegistryVolumeConfigPath": "configs/registry/repository-volume.yaml",
+	"LocalRegistryDockerRegistryConfigPath": "configs/registry/docker-registry.yaml",
+	"LocalRegistryHostUpdateConfigPath": "configs/registry/repository-update-hosts.yaml",
+	"MagicDNSConfigPath": "configs/knative_yamls/serving-default-domain.yaml"
+}

configs/setup/kube.json

@@ -0,0 +1,10 @@
+{
+	"K8sVersion": "1.25.9",
+	"AlternativeImageRepo": "",
+	"ApiserverAdvertiseAddress": "",
+	"PodNetworkCidr": "192.168.0.0/16",
+	"PodNetworkAddonConfigURL": "https://raw.githubusercontent.com/vhive-serverless/vHive/main/configs/calico/canal.yaml",
+	"ApiserverPort": "6443",
+	"ApiserverToken": "",
+	"ApiserverTokenHash": ""
+}

configs/setup/system.json

@@ -0,0 +1,17 @@
+{
+	"GoVersion": "1.19.10",
+	"GoDownloadUrlTemplate": "https://go.dev/dl/go%s.linux-%s.tar.gz",
+	"ContainerdVersion": "1.6.18",
+	"ContainerdDownloadUrlTemplate": "https://github.com/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz",
+	"ContainerdSystemdProfileDownloadUrl": "https://raw.githubusercontent.com/containerd/containerd/main/containerd.service",
+	"RuncVersion": "1.1.4",
+	"RuncDownloadUrlTemplate": "https://github.com/opencontainers/runc/releases/download/v%s/runc.%s",
+	"RunscVersion": "20210622",
+	"RunscDownloadUrlTemplate": "https://storage.googleapis.com/gvisor/releases/release/%s/%s/runsc",
+	"CniPluginsVersion": "1.2.0",
+	"CniPluginsDownloadUrlTemplate": "https://github.com/containernetworking/plugins/releases/download/v%s/cni-plugins-linux-%s-v%s.tgz",
+	"KubeVersion": "1.25.9-00",
+	"PmuToolsRepoUrl": "https://github.com/vhive-serverless/pmu-tools",
+	"ProtocVersion": "3.19.4",
+	"ProtocDownloadUrlTemplate": "https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-linux-x86_64.zip"
+}

configs/setup/vhive.json

@@ -0,0 +1,4 @@
+{
+	"FirecrackerKernelImgDownloadUrl": "https://s3.amazonaws.com/spec.ccfc.min/img/hello/kernel/hello-vmlinux.bin",
+	"StargzVersion": "0.13.0"
+}

scripts/.gitignore

@@ -0,0 +1 @@
+*.log

scripts/README.md

@@ -0,0 +1,107 @@
+# vHive Setup Scripts
+- [vHive Setup Scripts](#vhive-setup-scripts)
+  - [1. Get Setup Tool](#1-get-setup-tool)
+    - [1.1 Download the Binary Executable Directly](#11-download-the-binary-executable-directly)
+    - [1.2 Build from Source](#12-build-from-source)
+  - [2. Config the Setup Tool](#2-config-the-setup-tool)
+  - [3. Use of Setup Tool](#3-use-of-setup-tool)
+    - [3.1 General Usage](#31-general-usage)
+    - [3.2 Specify Config Files](#32-specify-config-files)
+    - [3.3 Use with Local vHive Repo](#33-use-with-local-vhive-repo)
+    - [3.4 Use with Remote vHive Repo (Standalone Use)](#34-use-with-remote-vhive-repo-standalone-use)
+    - [3.5 Migrate from Legacy Shell Scripts](#35-migrate-from-legacy-shell-scripts)
+  - [4. Logs](#4-logs)
+  - [5. Supported Platform](#5-supported-platform)
+## 1. Get Setup Tool
+There are basically two ways to get the setup tool
+
+### 1.1 Download the Binary Executable Directly
+Check [vHive GitHub Repo](https://github.com/vhive-serverless/vHive/releases) for more details and choose the appropriate version to download.
+
+### 1.2 Build from Source
+**Building from source requires Go (version 1.19 at least) installed on your system.**
+```bash
+git clone --depth 1 https://github.com/vhive-serverless/vHive
+cd vHive
+pushd scripts && go build -o setup_tool && popd
+```
+Compiled executable file will be in the `scripts` directory.
+
+## 2. Config the Setup Tool
+**Normally, just skip this section and use the default config files** which are located in the `configs/setup` directory inside the [vHive repo](https://github.com/vhive-serverless/vHive).
+
+- `configs/setup/knative.json`: knative related configs (all the path in the config file should be relative path inside the vHive repo)
+- `configs/setup/kube.json`: Kubernetes related configs
+- `configs/setup/system.json`: system related configs
+- `configs/setup/vhive.json`: vHive related configs
+
+You can modify the config files on your demand and then place all of them in one directory for the later use.
+
+## 3. Use of Setup Tool
+### 3.1 General Usage
+```bash
+./setup_tool [options] <subcommand> [parameters]
+```
+use the `-h` or `--help` option to look for the help
+### 3.2 Specify Config Files
+By default, the setup_tool will use the config files in `configs/setup` directory inside the vHive repo. 
+
+To change the path of config files, use the `--setup-configs-dir` option to specify it.
+```bash
+./setup_tool --setup-configs-dir <CONFIG PATH> ...
+```
+
+### 3.3 Use with Local vHive Repo
+By default, the setup_tool will check the current directory to ensure it is a vHive repo and then use it during the setup process.
+
+To use other vHive repos locally, provide the `--vhive-repo-dir` option to specify it.
+```bash
+./setup_tool --vhive-repo-dir <VHIVE REPO PATH> ...
+```
+
+If the current directory or the provided path is not a valid vHive repo, the setup_tool will [automatically clone the remote vHive repo and use it](#34-use-with-remote-vhive-repo).
+
+### 3.4 Use with Remote vHive Repo (Standalone Use)
+When the setup_tool is directly downloaded or targeted for  standalone use, the setup_tool will automatically clone the remote vHive repo to the temporary directory and then use it during the setup process.
+
+To change the URL and branch of the [default remote vHive repo](https://github.com/vhive-serverless/vHive), use `--vhive-repo-url` and `--vhive-repo-branch` options to specify them.
+```bash
+./setup_tool --vhive-repo-url <URL> --vhive-repo-branch <BRANCH> ...
+``` 
+
+Besides, when the current directory is a vHive repo or the `--vhive-repo-dir` option is valid, **the local repo will be prioritized for use**. **To force the setup_tool to clone and use the remote vHive repo**, provide `--force-remote` option to the setup_tool.
+```bash
+./setup_tool --force-remote ...
+```
+
+
+### 3.5 Migrate from Legacy Shell Scripts
+Just type the name of the original shell script and append corresponding parameters behind. For example:
+```bash
+# Legacy ==>
+scripts/cloudlab/setup_node.sh stock-only
+# ==> Current
+./setup_tool [options] setup_node stock-only
+
+# Legacy ==>
+scripts/create_devmapper.sh
+# ==> Current
+./setup_tool [options] create_devmapper
+
+# Legacy ==>
+scripts/gpu/setup_nvidia_gpu.sh
+# ==> Current
+./setup_tool [options] setup_nvidia_gpu
+```
+
+**NOTICE**: Shell scripts in `scripts/stargz`, `scripts/self-hosted-kind`, and `scripts/github_runner` **are not supported to be invoked in this way at present**.
+
+
+## 4. Logs
+The log files will be named as `<subcommand>_common.log` and `<subcommand>_error.log`. All log files will be stored in the directory where the setup_tool is executed.
+
+- `<subcommand>_common.log`: all output originally writes to `stdout` will be redirected to this log file.
+- `<subcommand>_error.log`: all output originally writes to `stderr` will be redirected to this log file.
+
+## 5. Supported Platform
+At present, only `Ubuntu 20.04 (amd64)` is officially tested. Other versions of `Ubuntu` may also work, but not guaranteed. 

scripts/cloudlab/go.mod

@@ -0,0 +1,3 @@
+module github.com/vhive-serverless/vHive/scripts/cloudlab
+
+go 1.19