Skip to content

Commit

Permalink
more fixes for one-tool
Browse files Browse the repository at this point in the history
  • Loading branch information
@atheurer
atheurer committed Dec 8, 2023
1 parent 8e7b718 commit 63e5610
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 61 deletions.
34 changes: 15 additions & 19 deletions endpoints/base
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ function set_osruntime_numanode_cpupart() {
# Always force tools to podman because they tend to clean
# themselves up reliably
if echo $this_cs_label | grep -P '^profiler-\w+-\d+-\w+-\d+$'; then
os_runtime="pod"
os_runtime="podman"
elif [ ! -z "${osruntime[$this_cs_label]}" ]; then
os_runtime=${osruntime[$this_cs_label]}
else
Expand Down Expand Up @@ -1258,27 +1258,23 @@ function get_opt_field() {

function get_image() {
local label=$1; shift
local var value type id
local var image type id bench_or_tool

typeset -n var=$1; shift
type=`echo $label | awk -F- '{print $1}'`
id=`echo $label | sed -e s/^$type-//`

if [ "$type" != "client" -a "$type" != "server" ]; then
# For any engine which is not running a benchmark client or server:
# We don't care what benchmark is in the image. We only care that
# an image exists for the ID. Since the ID for a non-client/server
# can exceed the ID range of a benchmark client[/server], (for example
# there could be a "worker-2" when there is only client-1 and server-1)
# always force the ID to 1 for purposes of looking up benchmark->image.
# When we build separate images for tools and benchmarks, this will
# change (significantly) and a new method to find the correct image for
# en engine will be implemented.
id="1"

if [ "$type" == "profiler" ]; then
# example label: profiler-remotehost-2-sysstat-1
# we need only 'sysstat'
bench_or_tool=`echo $label | awk -F- '{print $4}'`
else
id=`echo $label | sed -e s/^$type-//`
bench_or_tool=${id_to_bench[$id]}
fi
bench=${id_to_bench[$id]}
value=${bench_to_image[$bench]}
echo "image for $label is [$value]"
var=$value

image=${bench_to_image[$bench_or_tool]}
echo "image for $label is [$image]"
var=$image
}


10 changes: 7 additions & 3 deletions endpoints/remotehost/remotehost
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@ function cleanup_osruntime() {
elif [ "$os_runtime" == "podman" ]; then
container_name="${run_id}_${engine_label}"
do_ssh $user@$host podman rm $container_name
else
echo "WARNING: os_runtime $os_runtime not supported"
fi
done

Expand Down Expand Up @@ -575,8 +577,10 @@ function launch_osruntimes() {
echo "About to exec $engine_label via $os_runtime"
if [ "$os_runtime" == "chroot" ]; then
exec_chroot $engine_label $this_image
else
elif [ "$os_runtime" == "podman" ]; then
exec_pod $engine_label $this_image
else
echo "WARNING: os_runtime $os_runtime not supported"
fi
done
}
Expand All @@ -594,8 +598,6 @@ function move_remotehost_logs() {
for engine_label in ${clients[@]} ${servers[@]} ${profilers[@]}; do
set_osruntime_numanode_cpupart $engine_label
log_file=${engine_label}.txt
echo "engine: $engine_label"
echo "osruntime: $os_runtime"

if [ "${os_runtime}" == "chroot" ]; then
delete_remote_dir=1
Expand All @@ -606,6 +608,8 @@ function move_remotehost_logs() {
container_name="${run_id}_${engine_label}"
do_ssh $user@$host podman logs $container_name > "$engine_logs_dir/$log_file"
(( total_rc += $? ))
else
echo "WARNING: os_runtime $os_runtime not supported"
fi
done

Expand Down
2 changes: 1 addition & 1 deletion engine/engine-script-library
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ function validate_core_env() {
# client-1
regex2='^\w+-\d+$'
if echo $cs_label | grep -q -P $regex1; then
tool_name=`echo $cs_label | awk -F- '{print $4}'`
export tool_name=`echo $cs_label | awk -F- '{print $4}'`
echo "engine-label \"$cs_label\" is valid and runs this tool: $tool_name"
elif echo $cs_label | grep -q -P $regex2; then
echo "engine-label \"$cs_label\" is valid"
Expand Down
10 changes: 5 additions & 5 deletions rickshaw-post-process-bench
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,6 @@ for (my $i = 1; $i <= scalar @{ $run{'iterations'} }; $i++) {
}
if (opendir($iter_dh, $run_dir . "/" . $this_iter_dir)) {
my @samp_dirs = grep(/^sample-\d+$/, readdir($iter_dh));
if (scalar @samp_dirs < $run{'num-samples'}) {
printf "Iteration %d, [%s] is considered failed because there were not enough passing samples (%d)\n",
$i, $this_iter_dir, $run{'num-samples'};
system("/bin/mv " . $run_dir . "/" . $this_iter_dir . " " . $run_dir . "/" . $this_iter_dir . "-fail");
}
my $iter_array_idx = $i -1;
my @samples;
my $primary_metric;
Expand Down Expand Up @@ -280,5 +275,10 @@ for (my $i = 1; $i <= scalar @{ $run{'iterations'} }; $i++) {
while (wait() > -1) {}
print "Post-processing complete\n";
}
if (scalar @samp_dirs < $run{'num-samples'}) {
printf "Iteration %d, [%s] is considered failed because there were not enough passing samples (%d)\n",
$i, $this_iter_dir, $run{'num-samples'};
system("/bin/mv " . $run_dir . "/" . $this_iter_dir . " " . $run_dir . "/" . $this_iter_dir . "-fail");
}
}
}
17 changes: 9 additions & 8 deletions rickshaw-post-process-tools
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,14 +171,15 @@ my $tool_dir = "tool-data";
if (opendir(TOOLDIR, $run_dir . "/" . $tool_dir)) {
my @collectors = grep(/\w+/, readdir(TOOLDIR));
for my $collector (@collectors) {
my $collector_dir = $tool_dir . "/" . $collector; # $run_dir/tool-data/[client|server|worker|master]
my $collector_dir = $tool_dir . "/" . $collector; # $run_dir/tool-data/[client|server|worker|master|profiler]
if (opendir(COLLECTORDIR, $run_dir . "/" . $collector_dir)) {
my @numbers = grep (/\d+/, readdir(COLLECTORDIR));
for my $num (@numbers) {
my $num_dir = $collector_dir . "/" . $num; # $run_dir/tool-data/[client|server|worker|master]/[0-N]
if (opendir(NUMDIR, $run_dir . "/" . $num_dir)) {
my @tools = grep(/\w+/, readdir(NUMDIR));
printf "Working on tool dir %s\n", $num_dir;
# sample directory entry: remotehost-1-kernel-1
my @engines = grep (/\w+-\d+-\w+-\d+/, readdir(COLLECTORDIR));
for my $engine (@engines) {
my $engine_dir = $collector_dir . "/" . $engine; # $run_dir/tool-data/[client|server|worker|master]/[0-N]
if (opendir(ENGDIR, $run_dir . "/" . $engine_dir)) {
my @tools = grep(/\w+/, readdir(ENGDIR));
printf "Working on tool dir %s\n", $engine_dir;
for my $tool (@tools) {
if (! exists($tools_config{$tool})) {
# Load a tool configuration for every tool the user is asking for
Expand All @@ -201,7 +202,7 @@ if (opendir(TOOLDIR, $run_dir . "/" . $tool_dir)) {
if (my $pid = fork) {
push(@pids, $pid);
} else {
my $pushd_dir = pushd($run_dir . "/" . $num_dir . "/" . $tool);
my $pushd_dir = pushd($run_dir . "/" . $engine_dir . "/" . $tool);
my $pp_cmd = $tools_config{$tool}{'controller'}{'post-script'};
$pp_cmd =~ s/\%tool-dir\%/$run{'tools-dir'}\/$tool\//g;
$pp_cmd =~ s/\%run-dir\%/$run_dir\//g;
Expand Down
54 changes: 29 additions & 25 deletions rickshaw-run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use toolbox::logging;
use toolbox::run;
use toolbox::jsonsettings;

$toolbox::logging::debug = 1;
$toolbox::logging::debug = 0;

my $ug = Data::UUID->new;
my %defaults = ( "num-samples" => 1, "tool-group" => "default", "test-order" => "s",
Expand All @@ -51,7 +51,7 @@ my %benchmark_to_ids;
my %ids_to_benchmark;
my @endpoints;
#my %userenvs;
my %image_ids; # {$benchmark-type}{$userenv}
my %image_ids; # {$benchmark-or-tool}{$userenv}
my %run; # A multi-dimensional, nested hash, schema TBD
# This hash documents what was run.
my $redis_passwd = "flubber"; # TODO: make this cmdline setting
Expand Down Expand Up @@ -789,9 +789,6 @@ sub build_reqs {
}
}
}
foreach my $tool_entry (@tools_params) {
push(@$req_ref, "--requirement " . $run{'tools-dir'} . "/" . $$tool_entry{'tool'} . "/workshop.json ");
}
push (@$req_ref, "--requirement " . $bench_dirs{$benchmark} . "/workshop.json");
}

Expand Down Expand Up @@ -874,7 +871,7 @@ sub source_container_image {
" 2>&1";

get_image_urls;
printf "Sourcing container image for userenv '%s' and benchmark '%s'; this may take a few minutes\n", $userenv, $benchmark;
printf "Sourcing container image for userenv '%s' and benchmark/tool '%s'; this may take a few minutes\n", $userenv, $benchmark;

my @requirements;
build_reqs(\@requirements, $userenv, $benchmark);
Expand Down Expand Up @@ -972,10 +969,6 @@ sub source_container_image {
push(@local_images, $userenv_image);
}
my $x = 0;
while ($x < $i) {
printf "Processing stage %d (%s)...\n\tReady\n", $x + 1, $workshop_args[$x]{'tag'};
$x++;
}
while ($i <= $num_images - 1) {
printf "Processing stage %d (%s)...\n", $i + 1, $workshop_args[$i]{'tag'};
my $begin = time();
Expand Down Expand Up @@ -1396,7 +1389,8 @@ sub load_tool_params() {
# Load a tool configuration for every tool the user is asking for
foreach my $tool_entry (@tools_params) {
my $tool_name = $$tool_entry{'tool'};
my $this_tool_config = $run{'tools-dir'} . "/" . $tool_name . "/rickshaw.json";
my $this_tool_dir = $run{'tools-dir'} . "/" . $tool_name;
my $this_tool_config = $this_tool_dir . "/rickshaw.json";
my ($rc, $json_ref) = get_json_file($this_tool_config, $tool_schema_file);
if ($rc > 0 or ! defined $json_ref) {
printf "Could not open the tool config file: %s\n", $this_tool_config;
Expand All @@ -1411,6 +1405,11 @@ sub load_tool_params() {
exit 1;
}
$tools_configs{$$json_ref{'tool'}} = $json_ref;
$bench_dirs{$tool_name} = $this_tool_dir;

# Populate the image_ids with tools
my %userenv_info = ( 'image' => '' );
$image_ids{$tool_name}{'fedora38'} = \%userenv_info;
}
}

Expand Down Expand Up @@ -2107,8 +2106,8 @@ sub deploy_endpoints() {
my $endpoint_image_opt = "";
my $bench_ids_opt = "";
if (defined $endpoints[$i]{'userenv'}) {
foreach my $bench (keys %bench_configs) {
$endpoint_image_opt .= "," . $bench . "::" . $image_ids{$bench}{$endpoints[$i]{'userenv'}}{'image'};
foreach my $bench_or_tool (keys %image_ids) {
$endpoint_image_opt .= "," . $bench_or_tool . "::" . $image_ids{$bench_or_tool}{$endpoints[$i]{'userenv'}}{'image'};
}
$endpoint_image_opt =~ s/^,/ --image=/;
}
Expand Down Expand Up @@ -2273,6 +2272,7 @@ sub organize_run_data() {
my $archive_full_path = $engine_archives_dir . "/" . $archive;
my $cs_type = $1;
my $cs_id = $2;
printf "cs_type: %s, cs_id: %s\n", $cs_type, $cs_id;
my $tar_cmd = "tar zmxf " . $archive_full_path;
($tar_cmd, my $tar_output, my $tar_cmd_rc) = run_cmd($tar_cmd);
if ($1 =~ /^(client|server)$/) {
Expand All @@ -2295,12 +2295,16 @@ sub organize_run_data() {
}
}
}
if (-e "tool-data" and scalar dir_entries("tool-data", qr/\w+/) > 0) {
my $tool_dir = pushd("tool-data");
my $cs_tool_dest_path = $run_dir . "/tool-data/" . $cs_type . "/" . $cs_id;
make_path($cs_tool_dest_path);
my $mv_cmd = "/bin/mv * " . $cs_tool_dest_path;
($mv_cmd, my $mv_output, my $mv_cmd_rc) = run_cmd($mv_cmd);
if (-e "tool-data") {
if (scalar dir_entries("tool-data", qr/\w+/) > 0) {
my $tool_dir = pushd("tool-data");
my $cs_tool_dest_path = $run_dir . "/tool-data/" . $cs_type . "/" . $cs_id;
make_path($cs_tool_dest_path);
my $mv_cmd = "/bin/mv * " . $cs_tool_dest_path;
($mv_cmd, my $mv_output, my $mv_cmd_rc) = run_cmd($mv_cmd);
} else {
printf "WARNING: did not find expected sub-directories in %s\n", $pushd_dir . "/" . "tool-data";
}
}
if (scalar dir_entries("sysinfo", qr/\w+/) > 0) {
my $sysinfo_dir = pushd("sysinfo");
Expand Down Expand Up @@ -2447,16 +2451,16 @@ build_test_order();
prepare_bench_tool_engines();
print "Preparing userenvs:\n";
debug_log (sprintf "image_ids (before):\n" . Dumper \%image_ids);
foreach my $bench (keys %bench_configs) {
printf "Working on %s benchmark\n", $bench;
foreach my $userenv (keys %{ $image_ids{$bench} }) {
foreach my $bench_or_tool (keys %image_ids) {
printf "Working on %s benchmark or tool\n", $bench_or_tool;
foreach my $userenv (keys %{ $image_ids{$bench_or_tool} }) {
printf "Working on %s userenv\n", $userenv;
my $image = source_container_image($userenv, $bench, $arch);
my $image = source_container_image($userenv, $bench_or_tool, $arch);
if (!defined $image) {
die "Could not get valid image [" . $image_ids{$bench}{$userenv}{'image'} . "] for container image build for userenv [" . $userenv . "] and benchmark [" . $bench . "]\n";
die "Could not get valid image [" . $image_ids{$bench_or_tool}{$userenv}{'image'} . "] for container image build for userenv [" . $userenv . "] and benchmark [" . $bench_or_tool . "]\n";
}
printf "Image is: %s\n", $image;
$image_ids{$bench}{$userenv}{'image'} = $image;
$image_ids{$bench_or_tool}{$userenv}{'image'} = $image;
}
}
debug_log(sprintf "image_ids(after):\n" . Dumper \%image_ids);
Expand Down

0 comments on commit 63e5610

Please sign in to comment.