diff --git a/utils/benchmark-operator.sh b/utils/benchmark-operator.sh index f72f619c..e10a9c67 100644 --- a/utils/benchmark-operator.sh +++ b/utils/benchmark-operator.sh @@ -4,7 +4,11 @@ install_cli() { ripsaw_tmp=/tmp/ripsaw-cli mkdir -p ${ripsaw_tmp} if [[ ! -f ${ripsaw_tmp}/bin/activate ]]; then - python -m venv ${ripsaw_tmp} + if [[ "${isBareMetal}" == "true" ]]; then + python3.8 -m venv ${ripsaw_tmp} + else + python -m venv ${ripsaw_tmp} + fi fi source ${ripsaw_tmp}/bin/activate pip3 install -U "git+https://github.com/cloud-bulldozer/benchmark-operator.git/#egg=ripsaw-cli&subdirectory=cli" diff --git a/workloads/kube-burner/common.sh b/workloads/kube-burner/common.sh index 5c5d0246..aa39ccfb 100644 --- a/workloads/kube-burner/common.sh +++ b/workloads/kube-burner/common.sh @@ -154,4 +154,3 @@ snappy_backup() { ../../utils/snappy-move-results/run_snappy.sh metadata.json $snappy_path store_on_elastic } - diff --git a/workloads/network-perf/common.sh b/workloads/network-perf/common.sh index 237ffae4..2d19f307 100755 --- a/workloads/network-perf/common.sh +++ b/workloads/network-perf/common.sh @@ -16,13 +16,47 @@ check_cluster_health() { export_defaults() { - network_type=$(oc get network cluster -o jsonpath='{.status.networkType}' | tr '[:upper:]' '[:lower:]') + network_type=$(oc get network cluster -o jsonpath='{.status.networkType}' | tr '[:upper:]' '[:lower:]') export client_server_pairs=(1 2 4) + export CR_NAME=${BENCHMARK:=benchmark} + export baremetalCheck=$(oc get infrastructure cluster -o json | jq .spec.platformSpec.type) zones=($(oc get nodes -l node-role.kubernetes.io/workload!=,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/worker -o go-template='{{ range .items }}{{ index .metadata.labels "topology.kubernetes.io/zone" }}{{ "\n" }}{{ end }}' | uniq)) platform=$(oc get infrastructure cluster -o jsonpath='{.status.platformStatus.type}' | tr '[:upper:]' '[:lower:]') log "Platform is found to be: ${platform} " - # If MULTI_AZ we use one node from the two first AZs - if [[ ${platform} == "vsphere" ]]; then + + #Check to see if the infrastructure type is baremetal to adjust script as necessary + if [[ "${baremetalCheck}" == '"BareMetal"' ]]; then + log "BareMetal infastructure: setting isBareMetal accordingly" + export isBareMetal=true + else + export isBareMetal=false + fi + + #If using baremetal we use different query to find worker nodes + if [[ "${isBareMetal}" == "true" ]]; then + #Installing python3.8 + sudo yum -y install python3.8 + + nodeCount=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | wc -l) + if [[ ${nodeCount} -ge 2 ]]; then + serverNumber=$(( $RANDOM %${nodeCount} + 1 )) + clientNumber=$(( $RANDOM %${nodeCount} + 1 )) + while (( $serverNumber == $clientNumber )) + do + clientNumber=$(( $RANDOM %${nodeCount} + 1 )) + done + export server=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='${serverNumber}'{print $1}') + export client=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='${clientNumber}'{print $1}') + else + log "Colocating uperf pods for baremetal, since only one worker node available" + export server=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='1'{print $1}') + export client=$(oc get nodes --no-headers -l node-role.kubernetes.io/worker | awk 'NR=='1'{print $1}') + fi + log "Finished assigning server and client nodes" + log "Server to be scheduled on node: $server" + log "Client to be scheduled on node: $client" + # If multi_az we use one node from the two first AZs + elif [[ ${platform} == "vsphere" ]]; then nodes=($(oc get nodes -l node-role.kubernetes.io/worker,node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="" -o jsonpath='{range .items[*]}{ .metadata.labels.kubernetes\.io/hostname}{"\n"}{end}')) if [[ ${#nodes[@]} -lt 2 ]]; then log "At least 2 worker nodes placed are required" @@ -52,20 +86,6 @@ export_defaults() { export client=${nodes[1]} fi - if [ ${WORKLOAD} == "hostnet" ] - then - export hostnetwork=true - export serviceip=false - elif [ ${WORKLOAD} == "service" ] - then - export _metadata_targeted=false - export hostnetwork=false - export serviceip=true - else - export hostnetwork=false - export serviceip=false - fi - if [[ -z "$GSHEET_KEY_LOCATION" ]]; then export GSHEET_KEY_LOCATION=$HOME/.secrets/gsheet_key.json fi @@ -154,15 +174,18 @@ assign_uuid() { } run_benchmark_comparison() { + log "Begining benchamrk comparison" ../../utils/touchstone-compare/run_compare.sh uperf ${baseline_uperf_uuid} ${compare_uperf_uuid} ${pairs} pairs_array=( "${pairs_array[@]}" "compare_output_${pairs}.yaml" ) + log "Finished benchmark comparison" } generate_csv() { + log "Generating CSV" python3 csv_gen.py --files $(echo "${pairs_array[@]}") --latency_tolerance=$latency_tolerance --throughput_tolerance=$throughput_tolerance + log "Finished generating CSV" } - get_gold_ocp_version(){ current_version=`oc get clusterversion | grep -o [0-9.]* | head -1 | cut -c 1-3` export GOLD_OCP_VERSION=$( bc <<< "$current_version - 0.1" ) @@ -188,3 +211,4 @@ python3 -m pip install -r requirements.txt | grep -v 'already satisfied' export_defaults check_cluster_health deploy_operator + diff --git a/workloads/network-perf/requirements.txt b/workloads/network-perf/requirements.txt index 7838021a..40444262 100644 --- a/workloads/network-perf/requirements.txt +++ b/workloads/network-perf/requirements.txt @@ -1,4 +1,5 @@ gspread gspread-formatting oauth2client -pyyaml +PyYAML>=5.4.1 +make \ No newline at end of file diff --git a/workloads/network-perf/ripsaw-uperf-crd.yaml b/workloads/network-perf/ripsaw-uperf-crd.yaml index 3a15e20b..45012f2f 100644 --- a/workloads/network-perf/ripsaw-uperf-crd.yaml +++ b/workloads/network-perf/ripsaw-uperf-crd.yaml @@ -3,7 +3,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: uperf-benchmark-${WORKLOAD}-network-${pairs} + name: uperf-${CR_NAME}-${WORKLOAD}-network-${pairs} namespace: benchmark-operator spec: uuid: ${UUID} diff --git a/workloads/network-perf/run_hostnetwork_network_test_fromgit.sh b/workloads/network-perf/run_hostnetwork_network_test_fromgit.sh index 40cbe995..e73911eb 100755 --- a/workloads/network-perf/run_hostnetwork_network_test_fromgit.sh +++ b/workloads/network-perf/run_hostnetwork_network_test_fromgit.sh @@ -16,3 +16,4 @@ generate_csv if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then snappy_backup network_perf_hostnetwork_test fi +log "Finished workload ${0}" diff --git a/workloads/network-perf/run_multus_network_tests_fromgit.sh b/workloads/network-perf/run_multus_network_tests_fromgit.sh index 5892be3c..d0eafde6 100755 --- a/workloads/network-perf/run_multus_network_tests_fromgit.sh +++ b/workloads/network-perf/run_multus_network_tests_fromgit.sh @@ -114,4 +114,4 @@ fi # Cleanup rm -rf /tmp/benchmark-operator rm -f compare_output_*.yaml -exit 0 +log "Finished workload ${0}" diff --git a/workloads/network-perf/run_pod_network_policy_test_fromgit.sh b/workloads/network-perf/run_pod_network_policy_test_fromgit.sh index 6bd67b39..00d405c5 100755 --- a/workloads/network-perf/run_pod_network_policy_test_fromgit.sh +++ b/workloads/network-perf/run_pod_network_policy_test_fromgit.sh @@ -14,3 +14,4 @@ for pairs in 1 2 4; do run_benchmark_comparison done generate_csv +log "Finished workload ${0}" diff --git a/workloads/network-perf/run_pod_network_test_fromgit.sh b/workloads/network-perf/run_pod_network_test_fromgit.sh index 9ff20bff..08d9f91e 100755 --- a/workloads/network-perf/run_pod_network_test_fromgit.sh +++ b/workloads/network-perf/run_pod_network_test_fromgit.sh @@ -17,3 +17,4 @@ generate_csv if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then snappy_backup network_perf_pod_network_test fi +log "Finished workload ${0}" diff --git a/workloads/network-perf/run_serviceip_network_policy_test_fromgit.sh b/workloads/network-perf/run_serviceip_network_policy_test_fromgit.sh index 89578e90..d2b16c36 100755 --- a/workloads/network-perf/run_serviceip_network_policy_test_fromgit.sh +++ b/workloads/network-perf/run_serviceip_network_policy_test_fromgit.sh @@ -4,6 +4,9 @@ export NETWORK_POLICY=true source ./common.sh export SERVICEIP=true +if [[ "${isBareMetal}" == "true" ]]; then + export METADATA_TARGETED=true +fi for pairs in 1 2 4; do export pairs=${pairs} @@ -15,3 +18,4 @@ for pairs in 1 2 4; do run_benchmark_comparison done generate_csv +log "Finished workload ${0}" diff --git a/workloads/network-perf/run_serviceip_network_test_fromgit.sh b/workloads/network-perf/run_serviceip_network_test_fromgit.sh index e1691a5f..374ed5a3 100755 --- a/workloads/network-perf/run_serviceip_network_test_fromgit.sh +++ b/workloads/network-perf/run_serviceip_network_test_fromgit.sh @@ -18,3 +18,4 @@ generate_csv if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then snappy_backup network_perf_serviceip_network_test fi +log "Finished workload ${0}" diff --git a/workloads/network-perf/smoke_test.sh b/workloads/network-perf/smoke_test.sh index ffdf5458..6a59a404 100755 --- a/workloads/network-perf/smoke_test.sh +++ b/workloads/network-perf/smoke_test.sh @@ -22,7 +22,6 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then tar -zcvf snappy_files.tar.gz ./files_list export workload=network_perf_smoke_test - export snappy_path="$SNAPPY_USER_FOLDER/$runid$platform-$cluster_version-$network_type/$workload/$folder_date_time/" generate_metadata > metadata.json ../../utils/snappy-move-results/run_snappy.sh snappy_files.tar.gz $snappy_path @@ -30,3 +29,4 @@ if [[ ${ENABLE_SNAPPY_BACKUP} == "true" ]] ; then store_on_elastic rm -rf files_list fi +log "Finished workload ${0}" diff --git a/workloads/upgrade-perf/baremetal_func.sh b/workloads/upgrade-perf/baremetal_func.sh new file mode 100755 index 00000000..d2396322 --- /dev/null +++ b/workloads/upgrade-perf/baremetal_func.sh @@ -0,0 +1,399 @@ +#!/usr/bin/env bash + +log() { + echo ${bold}$(date -u): ${@}${normal} +} + +baremetal_upgrade_auxiliary() { + log "---------------------Checking Env Vars---------------------" + total_mcps=$TOTAL_MCPS + mcp_node_count=$MCP_NODE_COUNT + create_mcps_bool=${CREATE_MCPS_BOOL:=false} + if [ ! -z $total_mcps ] && [ $total_mcps -eq 0 ]; then + echo "TOTAL_MCPS env var was set to 0, exiting" + exit 1 + fi + if [ ! -z $mcp_node_count ] && [ $mcp_node_count -eq 0 ]; then + echo "MCP_NODE_COUNT env var was set to 0, exiting" + exit 1 + fi + if [[ ! -z "$total_mcps" ]] && [[ ! $total_mcps =~ ^[0-9]+$ ]]; then + echo "total_mcps input is not empty or not an integer, exiting" + exit 1 + fi + if [[ ! -z "$mcp_node_count" ]] && [[ ! $mcp_node_count =~ ^[0-9]+$ ]]; then + echo "mcp_node_count input is not empty or not an integer, exiting" + exit 1 + fi + if [ "${create_mcps_bool}" == "true" ]; then + mcp_bool="true" + echo "CREATE_MCPS_BOOL was set. Creating new MCPs" + else + mcp_bool="false" + echo "CREATE_MCPS_BOOL set to skip. Skipping MCP Creation" + fi + + log "---------------------Retrieving Available Nodes---------------------" + # Retrieve nodes in cluster + echo "Retrieving all nodes in the cluster that do not have the role 'master' or 'worker-lb'" + node_list=($(oc get nodes --no-headers | grep -v master | grep -v worker-lb | grep -v worker-rt | awk '{print $1}')) + node_count=${#node_list[@]} + + # Check for 1 or more nodes to be used + if [ $node_count -eq 0 ]; then + echo "Did not find any nodes that were not 'master' or 'worker-lb' nodes, exiting" + exit 1 + else + echo "$node_count node(s) found" + fi + + if [ $mcp_bool == "true" ]; then + log "---------------------Calculating MCP count---------------------" + # Use Defaults if TOTAL_MCPS and MCP_NODE_COUNT not set + if [ -z $total_mcps ] && [ -z $mcp_node_count ]; then + echo "TOTAL_MCPS and MCP_NODE_COUNT not set, calculating defaults!" + if [ $node_count -le 10 ]; then + total_mcps=$node_count + mcp_node_count=1 + echo "10 or less nodes found, defaulting new MCP(s) to node count: $total_mcps new MCP(s) required with 1 node in each" + defaults="true" + else + echo "Calculating number of MCP's required with a default of 10 nodes per MCP" + mcp_node_count=10 + total_mcps=$((($node_count / 10) + ($node_count % 10 > 0))) + echo "$total_mcps new MCP(s) required" + fi + else + + # Calculate total MCP's needed if only MCP_NODE_COUNT are provided + if [ -z $total_mcps ] && [ ! -z $mcp_node_count ]; then + if [ $mcp_node_count -gt $node_count ]; then + echo "Supplied MCP_NODE_COUNT is greater than available nodes, exiting" + exit 1 + else + echo "Found MCP_NODE_COUNT value, but not TOTAL_MCPS, attempting to calculate TOTAL_MCPS with supplied MCP_NODE_COUNT" + total_mcps=$((($node_count / $mcp_node_count) + ($node_count % $mcp_node_count > 0))) + echo "$total_mcps new MCP(s) required for supplied MCP_NODE_COUNT of $mcp_node_count with $node_count node(s) available" + fi + + + # Calculate total nodes per MCP if only total MCP's are provided + elif [ ! -z $total_mcps ] && [ -z $mcp_node_count ]; then + if [ $total_mcps -gt $node_count ]; then + echo "Supplied TOTAL_MCPS is greater than available nodes, exiting" + exit 1 + else + echo "Found TOTAL_MCPS value, but not MCP_NODE_COUNT, attempting to calculate MCP_NODE_COUNT with supplied TOTAL_MCPS" + mcp_node_count=$((($node_count / $total_mcps) + ($node_count % $total_mcps > 0))) + echo "$mcp_node_count node(s) required per MCP for supplied TOTAL_MCPS of $total_mcps with $node_count node(s) available" + fi + + + # Verify that TOTAL_MCPS and MCP_NODE_COUNT set equal available nodes + elif [ ! -z $total_mcps ] && [ ! -z $mcp_node_count ]; then + if [ $((($total_mcps * $mcp_node_count))) -ne $node_count ]; then + echo "The product of TOTAL_MCPS and MCP_NODE_COUNT supplied values does not equal available nodes, unless node count is already known, please set one or the other" + exit 1 + fi + fi + fi + + mcp_list=() + mcp_deployment=1 + + # Deploy MCP's required + log "---------------------Creating new MCPs---------------------" + while [ $mcp_deployment -le $total_mcps ]; do + export_label="upgrade$mcp_deployment" + export CUSTOM_NAME=${export_label} + export CUSTOM_VALUE=${export_label} + export CUSTOM_LABEL=${export_label} + echo "Removing MCP ${export_label} if it exists" + oc delete mcp ${export_label} --ignore-not-found + echo "Deploying new MCP ${export_label}" + envsubst < mcp.yaml | oc apply -f - + mcp_list+=(${export_label}) + ((mcp_deployment++)) + done + + # Label nodes in each new MCP + log "---------------------Applying custom labels to nodes in each new MCP---------------------" + nodes_labeled_mcp=1 + element=0 + node_element=0 + i=0 + + while [ $nodes_labeled_mcp -le $total_mcps ]; do + temp_node_list=() + if [ "$defaults" = "true" ]; then + temp_node_list+=(${node_list[$element]}) + ((element++)) + else + temp_node_list+=(${node_list[@]:$element:$mcp_node_count}) + element=$(($element + $mcp_node_count)) + fi + + while [ $node_element -lt ${#temp_node_list[@]} ]; do + oc label nodes ${temp_node_list[$node_element]} node-role.kubernetes.io/custom=${mcp_list[$i]} --overwrite=true + ((node_element++)) + done + node_element=0 + temp_node_list=() + ((i++)) + ((nodes_labeled_mcp++)) + done + + echo "Completed applying custom labels to nodes in new MCP's" + else + log "---------------------Applying Custom Label to Availalbe Nodes---------------------" + nodes_labeled=1 + element=0 + + while [ $nodes_labeled -le $node_count ]; do + oc label nodes ${node_list[$element]} node-role.kubernetes.io/custom=upgrade --overwrite=true + ((nodes_labeled++)) + ((element++)) + done + fi + + if [ $mcp_bool == "true" ]; then + # Calculate allocatable CPU per MCP + log "---------------------Calculating Replica Count and Deploy---------------------" + mcp_count_var=1 + mcp_counter=0 + temp_node_list_counter=0 + temp_node_list=() + node_element=0 + #allocation_calc_amounts=() + + # Store information to be used when creating json file for mb + project_list=() + replica_count=() + + while [ $mcp_count_var -le $total_mcps ]; do + echo "Begining deployment of project and sample-app for MCP ${mcp_list[${mcp_counter}]}" + nodes=($(oc get nodes --no-headers --selector=node-role.kubernetes.io/custom=${mcp_list[${mcp_counter}]} | awk '{print $1 }')) + temp_node_list+=(${nodes[@]}) + node_count=${#temp_node_list[@]} + allocation_amounts=() + node_count_var=1 + while [ $node_count_var -le $node_count ]; do + node_alloc_cpu=$(oc get node ${temp_node_list[${temp_node_list_counter}]} -o json | jq .status.allocatable.cpu) + node_alloc_cpu_int=${node_alloc_cpu//[a-z]/} + node_alloc_cpu_int_p1="${node_alloc_cpu_int%\"}" + node_alloc_cpu_int_p2="${node_alloc_cpu_int_p1#\"}" + allocation_amounts+=($node_alloc_cpu_int_p2) + ((temp_node_list_counter++)) + ((node_count_var++)) + ((node_element++)) + done + node_count_var=1 + temp_node_list_counter=0 + sum_alloc_cpu=$(IFS=+; echo "$((${allocation_amounts[*]}))") + echo "Total allocatable cpu ${mcp_list[$mcp_counter]} is $sum_alloc_cpu" + calc=$(( sum_alloc_cpu / 2 )) + new_allocatable="${calc}m" + echo "New calculated allocatable cpu to use in MCP ${mcp_list[$mcp_counter]} is $new_allocatable" + #allocation_calc_amounts+=($new_allocatable) + calc_replica=$(( $calc / 1000 )) + echo "$calc_replica replica(s) will be deployed in MCP ${mcp_list[$mcp_counter]} nodes" + + # Create new project per MCP + new_project="${mcp_list[$mcp_counter]}" + echo "Removing project ${new_project} if it exists" + exist=($(oc get project ${new_project} --ignore-not-found)) + if [ ! -z $exist ]; then + echo "Project ${new_project} was found, attempting to delete" + oc delete project ${new_project} --force --grace-period=0 --ignore-not-found + phase=$(oc get project ${new_project} -o json --ignore-not-found | jq .status.phase) + while [[ ${phase} == '"Terminating"' ]]; do + check=($(oc get project ${new_project} --ignore-not-found)) + if [ ! -z $check ]; then + phase=$(oc get project ${new_project} -o json --ignore-not-found | jq .status.phase) + echo "Waiting for project ${new_project} to terminate" + oc delete pods --all -n ${new_project} --force > /dev/null 2>&1 + sleep 5 + else + echo "Project ${new_project} deleted" + phase="complete" + fi + done + else + echo "Project ${new_project} not found" + fi + echo "Creating new project ${new_project}" + oc new-project $new_project + + # Append information arrays to be used for mb json file + project_list+=($new_project) + replica_count+=($calc_replica) + + # Export vars and deploy sample app + export PROJECT=$new_project + export APP_NAME="sampleapp-${new_project}" + export REPLICAS=$calc_replica + #export NODE_SELECTOR_KEY="node-role.kubernetes.io/custom" + export NODE_SELECTOR_VALUE=${mcp_list[${mcp_counter}]} + echo "Deploying $calc_replica replica(s) of the sample-app for MCP ${mcp_list[${mcp_counter}]}" + envsubst < deployment-sampleapp.yml | oc apply -f - + oc expose service samplesvc -n $new_project + + # Verify all pods are running + echo "Waiting for all pods to spin up successfully in project ${mcp_list[${mcp_counter}]}" + sleep 10 + running_pods=$(oc get pods -n ${mcp_list[${mcp_counter}]} --field-selector=status.phase==Running | wc -l) + while [ $running_pods -le 1 ]; do + sleep 5 + running_pods=$(oc get pods -n ${mcp_list[${mcp_counter}]} --field-selector=status.phase==Running | wc -l) + done + while [ $running_pods -lt $calc_replica ]; do + echo "Waiting for all pods to spin up successfully in project ${mcp_list[${mcp_counter}]}" + sleep 5 + running_pods=$(oc get pods -n ${mcp_list[${mcp_counter}]} --field-selector=status.phase==Running | wc -l) + done + + # Unset vars and begin to loop to next MCP + echo "Completed ${APP_NAME} deployment in ${mcp_list[${mcp_counter}]}" + unset temp_node_list + unset PROJECT + unset REPLICAS + unset NODE_SELECTOR_KEY + unset NODE_SELECTOR_VALUE + ((mcp_count_var++)) + ((mcp_counter++)) + done + else + log "---------------------Calculating Replica Count and Deploying---------------------" + # Retrieve allocatable CPU across all available nodes + nodes=($(oc get nodes --no-headers --selector=node-role.kubernetes.io/custom=upgrade| awk '{print $1 }')) + node_count_var=1 + node_element=0 + allocation_amounts=() + while [ $node_count_var -le $node_count ]; do + node_alloc_cpu=$(oc get node ${node_list[${node_element}]} -o json | jq .status.allocatable.cpu) + node_alloc_cpu_int=${node_alloc_cpu//[a-z]/} + node_alloc_cpu_int_p1="${node_alloc_cpu_int%\"}" + node_alloc_cpu_int_p2="${node_alloc_cpu_int_p1#\"}" + allocation_amounts+=($node_alloc_cpu_int_p2) + ((node_count_var++)) + ((node_element++)) + done + + # Calculate replica count + sum_alloc_cpu=$(IFS=+; echo "$((${allocation_amounts[*]}))") + echo "Total allocatable cpu is $sum_alloc_cpu" + calc=$(( sum_alloc_cpu / 2 )) + new_allocatable="${calc}m" + echo "New calculated allocatable cpu to use is $new_allocatable" + #allocation_calc_amounts+=($new_allocatable) + calc_replica=$(( $calc / 1000 )) + echo "$calc_replica replica(s) will be deployed" + + new_project="upgrade" + echo "Removing project upgrade if it exists" + exist=($(oc get project upgrade --ignore-not-found)) + if [ ! -z $exist ]; then + echo "Project upgrade was found, attempting to delete" + oc delete project upgrade --force --grace-period=0 --ignore-not-found + phase=$(oc get project upgrade -o json --ignore-not-found | jq .status.phase) + while [[ ${phase} == '"Terminating"' ]]; do + check=($(oc get project upgrade --ignore-not-found)) + if [ ! -z $check ]; then + phase=$(oc get project upgrade -o json --ignore-not-found | jq .status.phase) + echo "Waiting for project upgrade to terminate" + oc delete pods --all -n upgrade --force > /dev/null 2>&1 + sleep 5 + else + echo "Project upgrade deleted" + phase="complete" + fi + done + else + echo "Project upgrade not found" + fi + echo "Creating new project upgrade" + oc new-project upgrade + + # Append information arrays to be used for mb json file + project_list=("upgrade") + replica_count=($calc_replica) + + # Export vars and deploy sample app + export PROJECT="upgrade" + export APP_NAME="sampleapp-upgrade" + export REPLICAS=$calc_replica + #export NODE_SELECTOR_KEY="node-role.kubernetes.io/custom" + export NODE_SELECTOR_VALUE="upgrade" + echo "Deploying $calc_replica replica(s) of the sample-app in project upgrade" + envsubst < deployment-sampleapp.yml | oc apply -f - + oc expose service samplesvc -n upgrade + + # Verify all pods are running + echo "Waiting for all pods to spin up successfully in project upgrade" + sleep 10 + running_pods=$(oc get pods -n upgrade --field-selector=status.phase==Running | wc -l) + while [ $running_pods -le 1 ]; do + sleep 5 + running_pods=$(oc get pods -n upgrade --field-selector=status.phase==Running | wc -l) + done + while [ $running_pods -lt $calc_replica ]; do + echo "Waiting for all pods to spin up successfully in project upgrade" + sleep 5 + running_pods=$(oc get pods -n upgrade --field-selector=status.phase==Running | wc -l) + done + + # Unset vars and begin to loop to next MCP + echo "Completed ${APP_NAME} deployment in upgrade" + + fi + + + log "---------------------Creating request.json---------------------" + rm configmap.yml --force + requests=${#project_list[@]} + iterations=1 + element=0 + + cat >> configmap.yml << EOF + apiVersion: v1 + kind: ConfigMap + metadata: + name: request-configmap + namespace: default + labels: + app: mb-pod + data: + requests.json: | + [ +EOF + + while [ $iterations -le $requests ]; do + host=($(oc get route samplesvc -n ${project_list[$element]} -ojson | jq .spec.host)) + clients=${replica_count[$element]} + + cat >> configmap.yml << EOF + { + "scheme": "http", + "host": ${host}, + "port": 80, + "method": "GET", + "path": "/", + "keep-alive-requests": 1000, + "clients": 1000 # ${clients} + }, +EOF + ((iterations++)) + ((element++)) + done + + cat >> configmap.yml << EOF + ] +EOF + + cat configmap.yml + oc apply -f configmap.yml + log "---------------------Deploy mb-pod---------------------" + oc apply -f mb_pod.yml + sleep 30 +# oc logs -n default mb-pod -f + } \ No newline at end of file diff --git a/workloads/upgrade-perf/common.sh b/workloads/upgrade-perf/common.sh index 49b9918e..8a695049 100755 --- a/workloads/upgrade-perf/common.sh +++ b/workloads/upgrade-perf/common.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -x +export TERM=screen-256color +bold=$(tput bold) +uline=$(tput smul) +normal=$(tput sgr0) + # Check cluster's health if [[ ${CERBERUS_URL} ]]; then response=$(curl ${CERBERUS_URL}) @@ -15,6 +20,7 @@ _es_baseline=${ES_SERVER_BASELINE:=https://search-perfscale-dev-chmf5l4sh66lvxbn _poll_interval=${POLL_INTERVAL:=5} COMPARE=${COMPARE:=false} _timeout=${TIMEOUT:=240} +export baremetalCheck=$(oc get infrastructure cluster -o json | jq .spec.platformSpec.type) if [[ -n $UUID ]]; then _uuid=${UUID} @@ -31,6 +37,13 @@ if [ "$cloud_name" == "" ]; then cloud_name="test_cloud" fi +#Check to see if the infrastructure type is baremetal to adjust script as necessary +if [[ "${baremetalCheck}" == '"BareMetal"' ]]; then + echo "BareMetal infastructure: setting isBareMetal accordingly" + export isBareMetal=true +else + export isBareMetal=false +fi # check if cluster is up date @@ -55,7 +68,14 @@ rm -rf /tmp/snafu upgrade git clone https://github.com/cloud-bulldozer/benchmark-wrapper.git /tmp/snafu -python3 -m venv upgrade +if [[ "${isBareMetal}" == "true" ]]; then + sudo yum -y install python3.8 + python3.8 -m venv upgrade + +else + python3 -m venv upgrade +fi + source upgrade/bin/activate pip3 install -e /tmp/snafu diff --git a/workloads/upgrade-perf/deployment-sampleapp.yml b/workloads/upgrade-perf/deployment-sampleapp.yml new file mode 100644 index 00000000..1df7e176 --- /dev/null +++ b/workloads/upgrade-perf/deployment-sampleapp.yml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${APP_NAME} + namespace: ${PROJECT} +spec: + replicas: ${REPLICAS} + selector: + matchLabels: + app: sample + template: + metadata: + labels: + app: sample + spec: + containers: + - name: app + image: quay.io/smalleni/sampleapp:latest + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 3 + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "1" + limits: + cpu: "1" + nodeSelector: + node-role.kubernetes.io/custom: ${NODE_SELECTOR_VALUE} + + +--- +apiVersion: v1 +kind: Service +metadata: + name: samplesvc + namespace: ${PROJECT} +spec: + selector: + app: sample + ports: + - port: 80 + targetPort: 8080 + +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: except + namespace: ${PROJECT} +spec: + podSelector: + matchLabels: + app: sample + ingress: + - from: + - ipBlock: + cidr: 10.128.0.0/14 + except: + - "10.130.36.0/23" + - "10.130.12.0/23" + - "10.128.18.0/23" + - "10.131.10.0/23" + - "10.131.22.0/23" + - "10.128.24.0/23" + - "10.128.14.0/23" diff --git a/workloads/upgrade-perf/mb_pod.yml b/workloads/upgrade-perf/mb_pod.yml new file mode 100644 index 00000000..0dbb3e7f --- /dev/null +++ b/workloads/upgrade-perf/mb_pod.yml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mb-pod + namespace: default +spec: + containers: + - name: mb-container + image: "quay.io/mukrishn/snomb:2" + command: [ "/bin/sh", "-c", "mb -i /etc/config/requests.json -d 86400" ] # -d in seconds + volumeMounts: + - name: config-volume + mountPath: /etc/config + resources: + limits: + memory: '16Gi' + cpu: 16 + requests: + memory: '16Gi' + cpu: 16 + volumes: + - name: config-volume + configMap: + name: request-configmap + restartPolicy: Never \ No newline at end of file diff --git a/workloads/upgrade-perf/mcp.yaml b/workloads/upgrade-perf/mcp.yaml new file mode 100644 index 00000000..90b9f495 --- /dev/null +++ b/workloads/upgrade-perf/mcp.yaml @@ -0,0 +1,13 @@ +--- + +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + name: ${CUSTOM_NAME} +spec: + machineConfigSelector: + matchExpressions: + - {key: machineconfiguration.openshift.io/role, operator: In, values: [worker, "${CUSTOM_VALUE}"]} + nodeSelector: + matchLabels: + node-role.kubernetes.io/custom: ${CUSTOM_LABEL} \ No newline at end of file diff --git a/workloads/upgrade-perf/run_upgrade_fromgit.sh b/workloads/upgrade-perf/run_upgrade_fromgit.sh index 86268170..294c73a8 100755 --- a/workloads/upgrade-perf/run_upgrade_fromgit.sh +++ b/workloads/upgrade-perf/run_upgrade_fromgit.sh @@ -3,6 +3,23 @@ set -x source ./common.sh +# Check if we're on bareMetal +export baremetalCheck=$(oc get infrastructure cluster -o json | jq .spec.platformSpec.type) + +#Check to see if the infrastructure type is baremetal to adjust script as necessary +if [[ "${baremetalCheck}" == '"BareMetal"' ]]; then + log "BareMetal infastructure Upgrade" + source ./baremetal_func.sh + + export TOTAL_MCPS=${TOTAL_MCPS:-} # will skip if CREATE_MCPS_BOOL is set to false! + export MCP_NODE_COUNT=${MCP_NODE_COUNT:-} # will skip if CREATE_MCPS_BOOL is set to false! + export CREATE_MCPS_BOOL=true # true or false + + baremetal_upgrade_auxiliary + +fi + + # set the channel to find the builds to upgrade to if [[ -n $CHANNEL ]]; then echo "Setting the upgrade channel to $CHANNEL"