Baremetal e2e scripts

workloads/kube-burner/common.sh

@@ -27,8 +27,6 @@ collect_pprof() {
 
 
 deploy_operator() {
-  log "Removing benchmark-operator namespace, if it already exists"
-  oc delete namespace benchmark-operator --ignore-not-found
   log "Cloning benchmark-operator from branch ${OPERATOR_BRANCH} of ${OPERATOR_REPO}"
   rm -rf benchmark-operator
   git clone --single-branch --branch ${OPERATOR_BRANCH} ${OPERATOR_REPO} --depth 1
@@ -183,4 +181,3 @@ snappy_backup() {
  ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path
  store_on_elastic
 }
-

workloads/logging/common.sh

@@ -66,8 +66,6 @@ export TEST_TIMEOUT=${TEST_TIMEOUT:-7200}
 export TEST_CLEANUP=${TEST_CLEANUP:-"false"}
 
 deploy_operator() {
-  log "Removing benchmark-operator namespace, if it already exists"
-  oc delete namespace benchmark-operator --ignore-not-found
   log "Cloning benchmark-operator from branch ${operator_branch} of ${operator_repo}"
   rm -rf benchmark-operator
   git clone --single-branch --branch ${operator_branch} ${operator_repo} --depth 1

workloads/network-perf/common.sh

@@ -30,6 +30,7 @@ export_defaults() {
   operator_repo=${OPERATOR_REPO:=https://github.com/cloud-bulldozer/benchmark-operator.git}
   operator_branch=${OPERATOR_BRANCH:=master}
   CRD=${CRD:-ripsaw-uperf-crd.yaml}
+  export cr_name=${BENCHMARK:=benchmark}
   export _es=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443}
   _es_baseline=${ES_SERVER_BASELINE:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443}
   export _metadata_collection=${METADATA_COLLECTION:=true}
@@ -43,11 +44,41 @@ export_defaults() {
   export pin=true
   export networkpolicy=${NETWORK_POLICY:=false}
   export multi_az=${MULTI_AZ:=true}
+  export baremetalCheck=$(oc get infrastructure cluster -o json | jq .spec.platformSpec.type)
   zones=($(oc get nodes -l node-role.kubernetes.io/workload!=,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/worker -o go-template='{{ range .items }}{{ index .metadata.labels "topology.kubernetes.io/zone" }}{{ "\n" }}{{ end }}' | uniq))
   platform=$(oc get infrastructure cluster -o jsonpath='{.status.platformStatus.type}' | tr '[:upper:]' '[:lower:]')
   log "Platform is found to be : ${platform} "
-  # If multi_az we use one node from the two first AZs
-  if [[ ${platform} == "vsphere" ]]; then
+
+  #Check to see if the infrastructure type is baremetal to adjust script as necessary 
+  if [[ "${baremetalCheck}" == '"BareMetal"' ]]; then
+    log "BareMetal infastructure: setting isBareMetal accordingly"
+    export isBareMetal=true
+  else
+    export isBareMetal=false
+  fi
+
+  #If using baremetal we use different query to find worker nodes
+  if [[ "${isBareMetal}" == "true" ]]; then
+    nodeCount=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | wc -l)
+    if [[ ${nodeCount} -ge 2 ]]; then
+      serverNumber=$(( $RANDOM %${nodeCount} + 1 ))
+      clientNumber=$(( $RANDOM %${nodeCount} + 1 ))
+      while (( $serverNumber == $clientNumber ))
+        do
+          clientNumber=$(( $RANDOM %${nodeCount} + 1 ))
+        done
+      export server=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='${serverNumber}'{print $1}')
+      export client=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='${clientNumber}'{print $1}')
+    else
+      log "Colocating uperf pods for baremetal, since only one worker node available"
+      export server=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='1'{print $1}')
+      export client=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='1'{print $1}')
+    fi  
+    log "Finished assigning server and client nodes"
+    log "Server to be scheduled on node: $server"
+    log "Client to be scheduled on node: $client"
+    # If multi_az we use one node from the two first AZs
+  elif [[ ${platform} == "vsphere" ]]; then
     nodes=($(oc get nodes -l node-role.kubernetes.io/worker,node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="" -o jsonpath='{range .items[*]}{ .metadata.labels.kubernetes\.io/hostname}{"\n"}{end}'))
     if [[ ${#nodes[@]} -lt 2 ]]; then
       log "At least 2 worker nodes placed are required"
@@ -83,9 +114,13 @@ export_defaults() {
     export serviceip=false
   elif [ ${WORKLOAD} == "service" ]
   then
-    export _metadata_targeted=false
     export hostnetwork=false
     export serviceip=true
+    if [[ "${isBareMetal}" == "true" ]]; then
+      export _metadata_targeted=true
+    else  
+      export _metadata_targeted=false
+    fi
   else
     export hostnetwork=false
     export serviceip=false
@@ -132,8 +167,6 @@ export_defaults() {
 }
 
 deploy_operator() {
-  log "Removing benchmark-operator namespace, if it already exists"
-  oc delete namespace benchmark-operator --ignore-not-found
   log "Cloning benchmark-operator from branch ${operator_branch} of ${operator_repo}"
   rm -rf benchmark-operator
   git clone --single-branch --branch ${operator_branch} ${operator_repo} --depth 1  
@@ -153,7 +186,8 @@ deploy_workload() {
 }
 
 check_logs_for_errors() {
-client_pod=$(oc get pods -n benchmark-operator --no-headers | awk '{print $1}' | grep uperf-client | awk 'NR==1{print $1}')
+uuid=$(oc describe -n benchmark-operator benchmarks/uperf-${cr_name}-${WORKLOAD}-network-${pairs} | grep  Suuid | awk  '{print $2}')
+client_pod=$(oc get pods -n benchmark-operator --no-headers | awk '{print $1}' | grep $uuid | grep uperf-client | awk 'NR==1{print $1}')
 if [ ! -z "$client_pod" ]; then
   num_critical=$(oc logs ${client_pod} -n benchmark-operator | grep CRITICAL | wc -l)
   if [ $num_critical -gt 3 ] ; then
@@ -174,7 +208,11 @@ wait_for_benchmark() {
       log "Cerberus status is False, Cluster is unhealthy"
       exit 1
     fi
-    oc describe -n benchmark-operator benchmarks/uperf-benchmark-${WORKLOAD}-network-${pairs} | grep State | grep Complete
+    if [ "${benchmark_state}" == "Failed" ]; then
+      log "Benchmark state is Failed, exiting"
+      exit 1
+    fi
+    oc describe -n benchmark-operator benchmarks/uperf-${cr_name}-${WORKLOAD}-network-${pairs} | grep State | grep Complete
     if [ $? -eq 0 ]; then
       log "uperf workload done!"
       uperf_state=$?
@@ -224,31 +262,26 @@ assign_uuid() {
 }
 
 run_benchmark_comparison() {
+  log "Begining benchamrk comparison"
   ../../utils/touchstone-compare/run_compare.sh uperf ${baseline_uperf_uuid} ${compare_uperf_uuid} ${pairs}
   pairs_array=( "${pairs_array[@]}" "compare_output_${pairs}.yaml" )
+  log "Finished benchmark comparison"
 }
 
 generate_csv() {
+  log "Generating CSV"
   python3 csv_gen.py --files $(echo "${pairs_array[@]}") --latency_tolerance=$latency_tolerance --throughput_tolerance=$throughput_tolerance  
-}
-
-init_cleanup() {
-  log "Cloning benchmark-operator from branch ${operator_branch} of ${operator_repo}"
-  rm -rf /tmp/benchmark-operator
-  git clone --single-branch --branch ${operator_branch} ${operator_repo} /tmp/benchmark-operator --depth 1
-  oc delete -f /tmp/benchmark-operator/deploy
-  oc delete -f /tmp/benchmark-operator/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml
-  oc delete -f /tmp/benchmark-operator/resources/operator.yaml  
+  log "Finished generating CSV"
 }
 
 delete_benchmark() {
-  oc delete benchmarks.ripsaw.cloudbulldozer.io/uperf-benchmark-${WORKLOAD}-network-${pairs} -n benchmark-operator
+    oc delete benchmarks.ripsaw.cloudbulldozer.io/uperf-${cr_name}-${WORKLOAD}-network-${pairs} -n benchmark-operator
 }
 
 update() {
-  benchmark_state=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-benchmark-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.status.state}')
-  benchmark_uuid=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-benchmark-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.status.uuid}')
-  benchmark_current_pair=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-benchmark-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.spec.workload.args.pair}')
+  benchmark_state=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-${cr_name}-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.status.state}')
+  benchmark_uuid=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-${cr_name}-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.status.uuid}')
+  benchmark_current_pair=$(oc get benchmarks.ripsaw.cloudbulldozer.io/uperf-${cr_name}-${WORKLOAD}-network-${pairs} -n benchmark-operator -o jsonpath='{.spec.workload.args.pair}')
 }
 
 get_gold_ocp_version(){
@@ -257,6 +290,7 @@ get_gold_ocp_version(){
 }
 
 print_uuid() {
+  log "Logging uuid.txt"
   cat uuid.txt
 }
 
@@ -267,6 +301,6 @@ normal=$(tput sgr0)
 python3 -m pip install -r requirements.txt | grep -v 'already satisfied'
 check_cluster_present
 export_defaults
-init_cleanup
 check_cluster_health
 deploy_operator
+

workloads/network-perf/requirements.txt

@@ -1,4 +1,5 @@
 gspread
 gspread-formatting
 oauth2client
-pyyaml
+PyYAML>=5.4.1
+make

workloads/network-perf/ripsaw-uperf-crd.yaml

@@ -3,7 +3,7 @@
 apiVersion: ripsaw.cloudbulldozer.io/v1alpha1
 kind: Benchmark
 metadata:
-  name: uperf-benchmark-${WORKLOAD}-network-${pairs}
+  name: uperf-${cr_name}-${WORKLOAD}-network-${pairs}
   namespace: benchmark-operator
 spec:
   uuid: ${UUID}

workloads/network-perf/run_hostnetwork_network_test_fromgit.sh

@@ -27,4 +27,5 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then
  ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path
  store_on_elastic
  rm -rf files_list
-fi
+fi
+echo -e "${bold}Finished workload run_hostnetwork_network_test_gromgit.sh"

workloads/network-perf/run_multus_network_tests_fromgit.sh

@@ -114,4 +114,5 @@ fi
 # Cleanup
 rm -rf /tmp/benchmark-operator
 rm -f compare_output_*.yaml
+echo -e "${bold}Finished workload run_multus_network_tests_fromgit.sh"
 exit 0

workloads/network-perf/run_pod_network_policy_test_fromgit.sh

@@ -15,3 +15,4 @@ delete_benchmark
 done
 print_uuid
 generate_csv
+echo -e "${bold}Finished workload run_pod_network_policy_test_fromgit.sh"

workloads/network-perf/run_pod_network_test_fromgit.sh

@@ -31,4 +31,5 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then
  ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path
  store_on_elastic
  rm -rf files_list
-fi
+fi
+echo -e "${bold}Finished workload run_pod_network_test_fromgit.sh"

workloads/network-perf/run_serviceip_network_policy_test_fromgit.sh

@@ -15,3 +15,4 @@ delete_benchmark
 done
 print_uuid
 generate_csv
+echo -e "${bold}Finished workload run_serviceip_network_policy_test_fromgit.sh"

workloads/network-perf/run_serviceip_network_test_fromgit.sh

@@ -31,4 +31,5 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then
  ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path
  store_on_elastic
  rm -rf files_list
-fi
+fi
+echo -e "${bold}Finished workload run_serviceip_network_test_fromgit.sh"

workloads/network-perf/smoke_test.sh

@@ -31,4 +31,5 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then
  ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path
  store_on_elastic
  rm -rf files_list
-fi
+fi
+echo -e "${bold}Finished workload smoke_test.sh"