From e18ad0a724df83688a78a8555ba3fcc90326e630 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Wed, 1 May 2024 21:43:38 +0900 Subject: [PATCH 1/4] fmtutil: run rebuild in parallel - code by Tiago de Paula --- texlive-scripts/fmtutil.pl | 55 +++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/texlive-scripts/fmtutil.pl b/texlive-scripts/fmtutil.pl index 63b0d66..f99646e 100755 --- a/texlive-scripts/fmtutil.pl +++ b/texlive-scripts/fmtutil.pl @@ -38,6 +38,7 @@ BEGIN use File::Basename; use File::Spec; use Cwd; +use Parallel::ForkManager; # don't import anything automatically, this requires us to explicitly # call functions with TeXLive::TLUtils prefix, and makes it easier to @@ -469,38 +470,48 @@ sub callback_build_formats { my $nobuild = 0; my $notavail = 0; my $total = 0; + my $nproc = 24; # TODO should be dynamically tweaked to actual CPU + my $pm = Parallel::ForkManager->new($nproc); + $pm->run_on_finish(sub { + my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_; + my ($fmt, $eng) = @{$data_structure_reference}; + if ($exit_code == $FMT_DISABLED) { + log_to_status("DISABLED", $fmt, $eng, $what, $whatarg); + $disabled++; + } elsif ($exit_code == $FMT_NOTSELECTED) { + log_to_status("NOTSELECTED", $fmt, $eng, $what, $whatarg); + $nobuild++; + } elsif ($exit_code == $FMT_FAILURE) { + log_to_status("FAILURE", $fmt, $eng, $what, $whatarg); + $err++; + push (@err, "$eng/$fmt"); + } elsif ($exit_code == $FMT_SUCCESS) { + log_to_status("SUCCESS", $fmt, $eng, $what, $whatarg); + $suc++; + } elsif ($exit_code == $FMT_NOTAVAIL) { + log_to_status("NOTAVAIL", $fmt, $eng, $what, $whatarg); + $notavail++; + } + else { + log_to_status("UNKNOWN", $fmt, $eng, $what, $whatarg); + print_error("callback_build_format (round 1): unknown return " + . "from select_and_rebuild.\n"); + } + }); for my $swi (qw/format=engine format!=engine/) { for my $fmt (keys %{$alldata->{'merged'}}) { for my $eng (keys %{$alldata->{'merged'}{$fmt}}) { next if ($swi eq "format=engine" && $fmt ne $eng); next if ($swi eq "format!=engine" && $fmt eq $eng); $total++; + $pm->start("select_and_rebuild_format($fmt, $eng, $what, $whatarg)") and next; my $val = select_and_rebuild_format($fmt, $eng, $what, $whatarg); - if ($val == $FMT_DISABLED) { - log_to_status("DISABLED", $fmt, $eng, $what, $whatarg); - $disabled++; - } elsif ($val == $FMT_NOTSELECTED) { - log_to_status("NOTSELECTED", $fmt, $eng, $what, $whatarg); - $nobuild++; - } elsif ($val == $FMT_FAILURE) { - log_to_status("FAILURE", $fmt, $eng, $what, $whatarg); - $err++; - push (@err, "$eng/$fmt"); - } elsif ($val == $FMT_SUCCESS) { - log_to_status("SUCCESS", $fmt, $eng, $what, $whatarg); - $suc++; - } elsif ($val == $FMT_NOTAVAIL) { - log_to_status("NOTAVAIL", $fmt, $eng, $what, $whatarg); - $notavail++; - } - else { - log_to_status("UNKNOWN", $fmt, $eng, $what, $whatarg); - print_error("callback_build_format (round 1): unknown return " - . "from select_and_rebuild.\n"); - } + my @array = ($fmt, $eng); + $pm->finish($val, \@array); } } } + $pm->wait_all_children; # if the user asked to rebuild something, but we did nothing, report # unless we tried to rebuild only missing formats. From d3fd6e36c82059a0c706b60439368bb9eb1639a5 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Wed, 8 May 2024 11:19:53 +0900 Subject: [PATCH 2/4] fmtutil - usability fixes - make forking optional - test for availability of ForkManager - add command line option to disable forking - when running with forkmanager, dump output of compile logs to $nul - if --quiet is given, also hide the compile logs --- texlive-scripts/fmtutil.pl | 63 +++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/texlive-scripts/fmtutil.pl b/texlive-scripts/fmtutil.pl index f99646e..9ffa02c 100755 --- a/texlive-scripts/fmtutil.pl +++ b/texlive-scripts/fmtutil.pl @@ -3,7 +3,7 @@ # fmtutil - utility to maintain format files. # (Maintained in TeX Live:Master/texmf-dist/scripts/texlive.) # -# Copyright 2014-2023 Norbert Preining +# Copyright 2014-2024 Norbert Preining # This file is licensed under the GNU General Public License version 2 # or any later version. # @@ -38,7 +38,6 @@ BEGIN use File::Basename; use File::Spec; use Cwd; -use Parallel::ForkManager; # don't import anything automatically, this requires us to explicitly # call functions with TeXLive::TLUtils prefix, and makes it easier to @@ -47,6 +46,8 @@ BEGIN require TeXLive::TLWinGoo if wndws(); +my $USE_FORKMANAGER = 0; + # numerical constants my $FMT_NOTSELECTED = 0; my $FMT_DISABLED = 1; @@ -129,6 +130,7 @@ BEGIN "no-engine-subdir", "no-error-if-no-engine=s", "no-error-if-no-format", + "no-fork", "nohash", "recorder", "refresh", @@ -204,6 +206,17 @@ sub main { die "$0: Unexpected non-option argument(s): @ARGV\n" . "Try \"$prg --help\" for more information.\n"; } + # for full fmtutil, let us try to use ForkManager if available + if ($opts{"no-fork"}) { + $USE_FORKMANAGER = 0; + } else { + eval { require Parallel::ForkManager; }; + if ($@) { + $USE_FORKMANAGER = 0; + } else { + $USE_FORKMANAGER = 1; + } + } } help() if $opts{'help'}; @@ -470,11 +483,8 @@ sub callback_build_formats { my $nobuild = 0; my $notavail = 0; my $total = 0; - my $nproc = 24; # TODO should be dynamically tweaked to actual CPU - my $pm = Parallel::ForkManager->new($nproc); - $pm->run_on_finish(sub { - my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_; - my ($fmt, $eng) = @{$data_structure_reference}; + my $finish_sub = sub { + my ($exit_code, $fmt, $eng) = @_; if ($exit_code == $FMT_DISABLED) { log_to_status("DISABLED", $fmt, $eng, $what, $whatarg); $disabled++; @@ -494,24 +504,42 @@ sub callback_build_formats { } else { log_to_status("UNKNOWN", $fmt, $eng, $what, $whatarg); - print_error("callback_build_format (round 1): unknown return " + print_error("callback_build_format: unknown return " . "from select_and_rebuild.\n"); } - }); + }; + my $pm; + if ($USE_FORKMANAGER) { + my $nproc = 24; # TODO should be dynamically tweaked to actual CPU + $pm = Parallel::ForkManager->new($nproc); + $pm->run_on_finish(sub { + my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_; + my ($fmt, $eng) = @{$data_structure_reference}; + $finish_sub->($exit_code, $fmt, $eng); + }); + } for my $swi (qw/format=engine format!=engine/) { for my $fmt (keys %{$alldata->{'merged'}}) { for my $eng (keys %{$alldata->{'merged'}{$fmt}}) { next if ($swi eq "format=engine" && $fmt ne $eng); next if ($swi eq "format!=engine" && $fmt eq $eng); $total++; - $pm->start("select_and_rebuild_format($fmt, $eng, $what, $whatarg)") and next; + if ($USE_FORKMANAGER) { + $pm->start("select_and_rebuild_format($fmt, $eng, $what, $whatarg)") and next; + } my $val = select_and_rebuild_format($fmt, $eng, $what, $whatarg); - my @array = ($fmt, $eng); - $pm->finish($val, \@array); + if ($USE_FORKMANAGER) { + my @array = ($fmt, $eng); + $pm->finish($val, \@array); + } else { + $finish_sub->($val, $fmt, $eng); + } } } + # We need to wait for format=engine round to be finished before + # we can run the format!=engine round + $pm->wait_all_children if ($USE_FORKMANAGER); } - $pm->wait_all_children; # if the user asked to rebuild something, but we did nothing, report # unless we tried to rebuild only missing formats. @@ -815,8 +843,12 @@ sub rebuild_one_format { $ENV{'TEXPOOL'} = cwd() . $sep . ($texpool ? $texpool : ""); } - # in mktexfmtMode we must redirect *all* output to stderr - $cmdline .= " >&2" if $mktexfmtMode; + if ($opts{'quiet'} || $USE_FORKMANAGER) { + $cmdline .= " >$nul 2>$nul"; + } else { + # in mktexfmtMode we must redirect *all* output to stderr + $cmdline .= " >&2" if $mktexfmtMode; + } $cmdline .= " <$nul"; my $retval = system("$DRYRUN$cmdline"); @@ -1483,6 +1515,7 @@ sub help { --no-error-if-no-engine=ENGINE1,ENGINE2,... exit successfully even if a required ENGINE is missing, if it is included in the list. + --no-fork do not try to use forking format creation --no-strict exit successfully even if a format fails to build --nohash don't update ls-R files --recorder pass the -recorder option and save .fls files From a87d745488a27604dfd0fc67357eecef69c9cfa1 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 10 May 2024 09:17:52 +0900 Subject: [PATCH 3/4] Disable fork on windows, get nproc for others --- texlive-scripts/fmtutil.pl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/texlive-scripts/fmtutil.pl b/texlive-scripts/fmtutil.pl index 9ffa02c..194563c 100755 --- a/texlive-scripts/fmtutil.pl +++ b/texlive-scripts/fmtutil.pl @@ -207,7 +207,8 @@ sub main { . "Try \"$prg --help\" for more information.\n"; } # for full fmtutil, let us try to use ForkManager if available - if ($opts{"no-fork"}) { + # do not try this on Windows + if (wndws() || $opts{"no-fork"}) { $USE_FORKMANAGER = 0; } else { eval { require Parallel::ForkManager; }; @@ -510,7 +511,13 @@ sub callback_build_formats { }; my $pm; if ($USE_FORKMANAGER) { - my $nproc = 24; # TODO should be dynamically tweaked to actual CPU + # get number of cores/cpus according to https://stackoverflow.com/questions/45181115 + # checked on Linux, MacOS + # We have forking disabled for Windows + my $nproc = `getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu`; + # if we get something not numerical here, reset it to 0 + # which means don't do any forking + $nproc = 0 if ($nproc !~ s/[0-9]+//); $pm = Parallel::ForkManager->new($nproc); $pm->run_on_finish(sub { my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_; From bd413fd14848b1031211053bcb7580cd97ff64a6 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 10 May 2024 10:07:13 +0900 Subject: [PATCH 4/4] Fix , catch stdout, deal with deferred printing --- texlive-scripts/fmtutil.pl | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/texlive-scripts/fmtutil.pl b/texlive-scripts/fmtutil.pl index 194563c..16d9c6b 100755 --- a/texlive-scripts/fmtutil.pl +++ b/texlive-scripts/fmtutil.pl @@ -517,11 +517,13 @@ sub callback_build_formats { my $nproc = `getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu`; # if we get something not numerical here, reset it to 0 # which means don't do any forking - $nproc = 0 if ($nproc !~ s/[0-9]+//); + $nproc = 0 if ($nproc !~ m/[0-9]+/); $pm = Parallel::ForkManager->new($nproc); $pm->run_on_finish(sub { my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_; - my ($fmt, $eng) = @{$data_structure_reference}; + my ($fmt, $eng, $ref_deferred_stdout, $ref_deferred_stderr) = @{$data_structure_reference}; + push @deferred_stdout, @$ref_deferred_stdout; + push @deferred_stderr, @$ref_deferred_stderr; $finish_sub->($exit_code, $fmt, $eng); }); } @@ -536,7 +538,7 @@ sub callback_build_formats { } my $val = select_and_rebuild_format($fmt, $eng, $what, $whatarg); if ($USE_FORKMANAGER) { - my @array = ($fmt, $eng); + my @array = ($fmt, $eng, \@deferred_stdout, \@deferred_stderr); $pm->finish($val, \@array); } else { $finish_sub->($val, $fmt, $eng); @@ -850,17 +852,24 @@ sub rebuild_one_format { $ENV{'TEXPOOL'} = cwd() . $sep . ($texpool ? $texpool : ""); } - if ($opts{'quiet'} || $USE_FORKMANAGER) { - $cmdline .= " >$nul 2>$nul"; - } else { + $cmdline .= " <$nul"; + + my ($out, $retval); + + if ($mktexfmtMode) { # in mktexfmtMode we must redirect *all* output to stderr - $cmdline .= " >&2" if $mktexfmtMode; + $out = ""; + $retval = system("$DRYRUN$cmdline >&2"); + } else { + # we want to catch stdout and stderr into $out + ($out, $retval) = TeXLive::TLUtils::run_cmd("$DRYRUN$cmdline 2>&1"); + $out =~ s/\n+$//; # trailing newlines don't seem interesting } - $cmdline .= " <$nul"; - my $retval = system("$DRYRUN$cmdline"); # report error if it failed. if ($retval != 0) { + # print out all the output for debugging + print($out); $retval /= 256 if ($retval > 0); print_deferred_error("running \`$cmdline' return status: $retval\n"); }