feat: Add remote-workers configuration support to run remoteinventory…

… concurrently
glpi-project · Aug 16, 2022 · 7ceacb6 · 7ceacb6
1 parent f38f145
commit 7ceacb6
Show file tree

Hide file tree

Showing 12 changed files with 278 additions and 43 deletions.
diff --git a/Changes b/Changes
@@ -20,6 +20,8 @@ remoteinventory:
 * Fix remoteGlob function for ssh remote inventory as it was preventing storage
   inventory to work properly when accessing remote via ssh command
 * Don't try to register/update remotes when provided via --remote glpi-agent option
+* Support 'remote-workers' configuration to define how many remoteinventory we can
+  run in parallel
 
 netdiscovery/netinventory:
 * Avoid to record invalid MAC Address from Netbios during netdiscovery task

diff --git a/bin/glpi-agent b/bin/glpi-agent
@@ -59,6 +59,7 @@ GetOptions(
     'list-categories',
     'listen',
     'remote=s',
+    'remote-workers=i',
     'set-forcerun',
     'scan-homedirs',
     'scan-profiles',
@@ -262,8 +263,9 @@ glpi-agent [options] [--server server|--local path]
     --credentials                  set credentials to support database inventory
 
   RemoteInventory task specific options:
-    --remote                       specify a list of remotes to process in place
+    --remote=REMOTE[,REMOTE]...    specify a list of remotes to process in place
                                    of remotes managed via glpi-remote command
+    --remote-workers=COUNT         maximum number of workers for remoteinventory task
 
   Package deployment task specific options:
     --no-p2p                       do not use peer to peer to download

diff --git a/contrib/unix/glpi-agent-appimage-hook b/contrib/unix/glpi-agent-appimage-hook
@@ -96,6 +96,7 @@ GetOptions(
     'httpd-port=s',
     'httpd-trust=s',
     'remote=s',
+    'remote-workers=i',
     'scan-homedirs',
     'scan-profiles',
     'server|s=s',
@@ -575,7 +576,8 @@ glpi-agent-linux-installer.AppImage [options]
 
 =head2 Remote inventory task specific options
 
-    --remote=REMOTE                setup remote for which request remote inventory
+    --remote=REMOTE[,REMOTE]...    list of remotes for remoteinventory task
+    --remote-workers=COUNT         maximum number of workers for remoteinventory task
 
 =head2 Deploy task specific options
 

diff --git a/contrib/unix/installer/Getopt.pm b/contrib/unix/installer/Getopt.pm
@@ -35,6 +35,8 @@ my @options = (
     'httpd-port=s',
     'httpd-trust=s',
     'reinstall',
+    'remote=s',
+    'remote-workers=i',
     'runnow',
     'scan-homedirs',
     'scan-profiles',
@@ -122,6 +124,10 @@ glpi-agent-linux-installer [options]
     --backend-collect-timeout=TIME set timeout for inventory modules execution (30)
     -t --tag=TAG                   configure tag to define in inventories
 
+  RemoteInventory specific options:
+    --remote=REMOTE[,REMOTE]...    list of remotes for remoteinventory task
+    --remote-workers=COUNT         maximum number of workers for remoteinventory task
+
   Package deployment task specific options:
     --no-p2p                       set to not use peer to peer to download
                                    deploy task packages

diff --git a/contrib/windows/glpi-agent-packaging.pl b/contrib/windows/glpi-agent-packaging.pl
@@ -482,6 +482,7 @@ sub _tree2xml {
                 $result .= $ident ."  ". qq[    <RegistryValue Name="vardir" Type="string" Value="[VARDIR]" />\n];
                 $result .= $ident ."  ". qq[    <RegistryValue Name="listen" Type="string" Value="[LISTEN]" />\n];
                 $result .= $ident ."  ". qq[    <RegistryValue Name="remote" Type="string" Value="[REMOTE]" />\n];
+                $result .= $ident ."  ". qq[    <RegistryValue Name="remote-workers" Type="string" Value="[REMOTE_WORKERS]" />\n];
                 $result .= $ident ."  ". qq[  </RegistryKey>\n];
                 $result .= $ident ."  ". qq[  <RegistryKey Root="HKLM" Key="$regpath\\Installer">\n];
                 $result .= $ident ."  ". qq[    <RegistryValue Name="InstallDir" Type="string" Value="[INSTALLDIR]" />\n];

diff --git a/contrib/windows/packaging/MSI_main-v2.wxs.tt b/contrib/windows/packaging/MSI_main-v2.wxs.tt
@@ -256,6 +256,12 @@
     <SetProperty Id="CMDLINE_REMOTE" Before="AppSearch" Value="[REMOTE]" />
     <SetProperty Id="REMOTE"         After="AppSearch"  Value="[CMDLINE_REMOTE]"><![CDATA[CMDLINE_REMOTE<>""]]></SetProperty>
 
+    <Property Id="REMOTE_WORKERS" Secure="yes">
+      <RegistrySearch Id="RemoteWorkers" Root="HKLM" Key="[%agent_regpath%]" Name="remote-workers" Type="raw"/>
+    </Property>
+    <SetProperty Id="CMDLINE_REMOTE_WORKERS" Before="AppSearch" Value="[REMOTE_WORKERS]" />
+    <SetProperty Id="REMOTE_WORKERS"         After="AppSearch"  Value="[CMDLINE_REMOTE_WORKERS]"><![CDATA[CMDLINE_REMOTE_WORKERS<>""]]></SetProperty>
+
     <Property Id="HTML" Secure="yes">
       <RegistrySearch Id="Html" Root="HKLM" Key="[%agent_regpath%]" Name="html" Type="raw"/>
     </Property>

diff --git a/lib/GLPI/Agent/Config.pm b/lib/GLPI/Agent/Config.pm
@@ -22,6 +22,8 @@ my $default = {
     'conf-reload-interval'    => 0,
     'debug'                   => undef,
     'delaytime'               => 3600,
+    'remote-scheduling'       => 0,
+    'remote-workers'          => 1,
     'force'                   => undef,
     'html'                    => undef,
     'json'                    => undef,

diff --git a/lib/GLPI/Agent/Target.pm b/lib/GLPI/Agent/Target.pm
@@ -52,7 +52,7 @@ sub _init {
     # handle persistent state
     $self->_loadState();
 
-    $self->{nextRunDate} = $self->_computeNextRunDate()
+    $self->{nextRunDate} = $self->computeNextRunDate()
         if (!$self->{nextRunDate} || $self->{nextRunDate} < time-$self->getMaxDelay());
 
     $self->_saveState();
@@ -117,7 +117,7 @@ sub resetNextRunDate {
     return if delete $self->{_expiration};
 
     $self->{_nextrundelay} = 0;
-    $self->{nextRunDate} = $self->_computeNextRunDate();
+    $self->{nextRunDate} = $self->computeNextRunDate();
     $self->_saveState();
 }
 
@@ -274,7 +274,7 @@ sub isGlpiServer {
 
 # compute a run date, as current date and a random delay
 # between maxDelay / 2 and maxDelay
-sub _computeNextRunDate {
+sub computeNextRunDate {
     my ($self) = @_;
 
     my $ret;

diff --git a/lib/GLPI/Agent/Task/RemoteInventory.pm b/lib/GLPI/Agent/Task/RemoteInventory.pm
@@ -5,6 +5,8 @@ use warnings;
 
 use parent 'GLPI::Agent::Task::Inventory';
 
+use Parallel::ForkManager;
+
 use GLPI::Agent::Tools;
 use GLPI::Agent::Task::RemoteInventory::Remotes;
 
@@ -17,23 +19,26 @@ sub isEnabled {
         return 0;
     }
 
+    # Always enable remoteinventory task if remote option is set
+    return 1 if $self->{config}->{remote};
+
     my $remotes = GLPI::Agent::Task::RemoteInventory::Remotes->new(
         config  => $self->{config},
         storage => $self->{target}->getStorage(),
         logger  => $self->{logger},
     );
 
     my $errors = 0;
-    foreach my $remote ($remotes->getall()) {
+    while (my $remote = $remotes->next()) {
         my $error = $remote->checking_error();
         last unless $error;
         my $deviceid = $remote->deviceid
             or next;
         $self->{logger}->debug("Skipping remote inventory task execution for $deviceid: $error");
-        # We want to retry in a hour
-        $self->{target}->setNextRunDateFromNow(3600);
-        $remote->expiration($self->{target}->getNextRunDate());
-        $remotes->store();
+        if ($self->{config}->{"remote-scheduling"}) {
+            $remotes->retry($remote, $self->{target}->getMaxDelay());
+            $remotes->store();
+        }
         $errors++;
     }
 
@@ -43,8 +48,10 @@ sub isEnabled {
         return 1;
     }
 
-    $self->{logger}->debug("Remote inventory task execution disabled: ".(!$count ?
-        "no remote to inventory" : "all $count remotes are failing"));
+    $self->{logger}->debug(
+        "Remote inventory task execution disabled: ".
+        ($count ? "all $count remotes are failing" : "no remote to inventory")
+    );
 
     return 0;
 }
@@ -58,43 +65,76 @@ sub run {
         logger  => $self->{logger},
     );
 
-    # Handle only one reachable remote at a time
-    my $remote;
-    while ($remote = $remotes->next()) {
-        my $error = $remote->checking_error();
-        last unless $error;
-        my $deviceid = $remote->deviceid
-            or next;
-        $self->{logger}->debug("Skipping remote inventory task execution for $deviceid: $error");
-        # We want to retry in a hour
-        $self->{target}->setNextRunDateFromNow(3600);
-        $remote->expiration($self->{target}->getNextRunDate());
-        $remotes->store();
-    }
-    return unless $remote;
+    my $worker_count = $remotes->count() > 1 ? $self->{config}->{'remote-workers'} : 0;
 
     my $start = time;
 
-    $self->{deviceid} = $remote->deviceid();
+    my $manager = Parallel::ForkManager->new($worker_count);
+    $manager->set_waitpid_blocking_sleep(0);
 
-    # Set now we are remote
-    $self->setRemote($remote->protocol());
+    if ($worker_count) {
+        $manager->run_on_start(
+            sub {
+                my ($pid, $remote) = @_;
+                $self->{logger}->debug("Starting remoteinventory worker[$pid] to handle ".$remote->safe_url());
+            }
+        );
+    }
 
-    setRemoteForTools($remote);
+    $manager->run_on_finish(
+        sub {
+            my ($pid, $ret, $remote) = @_;
+            my $remoteid = $remote->safe_url();
+            if ($ret) {
+                $self->{logger}->debug("Remoteinventory worker[$pid] failed to handle $remoteid") if $worker_count;
+                # We want to schedule a retry but limited by target max delay
+                $remotes->retry($remote, $self->{target}->getMaxDelay());
+            } else {
+                $self->{logger}->debug("Remoteinventory worker[$pid] finished to handle $remoteid") if $worker_count;
+                $remote->expiration($self->{target}->computeNextRunDate());
+            }
+            # Store new remotes scheduling if required
+            $remotes->store();
+        }
+    );
 
-    $self->SUPER::run(%params);
+    while (my $remote = $remotes->next()) {
+        $manager->start($remote) and next;
 
-    $remote->disconnect();
+        my $error = $remote->checking_error();
+        my $deviceid = $remote->deviceid;
 
-    resetRemoteForTools();
+        my $remoteid = $deviceid // $remote->safe_url();
+        $self->{logger}->{prefix} = "[worker $$] $remoteid, " if $worker_count;
+        if ($error || !$deviceid) {
+            $self->{logger}->debug("Skipping remote inventory task execution for $remoteid: $error");
+            $manager->finish(1);
+            # In the case we have only one remote, finish won't leave the loop, so always last here
+            last;
+        }
 
-    my $timing = time - $start;
-    $self->{logger}->debug("Remote inventory run in $timing seconds");
+        $self->{deviceid} = $deviceid;
+
+        # Set now we are remote
+        $self->setRemote($remote->protocol());
+
+        setRemoteForTools($remote);
+
+        $self->SUPER::run(%params);
+
+        $remote->disconnect();
+
+        resetRemoteForTools();
 
-    # Set expiration from target for the remote before storing remotes
-    $self->{target}->resetNextRunDate();
-    $remote->expiration($self->{target}->getNextRunDate());
-    $remotes->store();
+        delete $self->{logger}->{prefix} if $worker_count;
+
+        $manager->finish();
+    }
+
+    $manager->wait_all_children();
+
+    my $timing = time - $start;
+    $self->{logger}->debug("Remote inventory task run in $timing seconds");
 }
 
 1;
diff --git a/lib/GLPI/Agent/Task/RemoteInventory/Remote.pm b/lib/GLPI/Agent/Task/RemoteInventory/Remote.pm
@@ -116,6 +116,17 @@ sub mode {
     return $self->{_mode} // '';
 }
 
+sub retry {
+    my ($self, $delay) = @_;
+
+    if (defined($delay)) {
+        $self->{_retry} = $delay;
+        $self->expiration(time+$delay) if $delay;
+    }
+
+    return $self->{_retry} ? $self : 0;
+}
+
 sub resetmode {
     my ($self) = @_;
     delete $self->{_mode};

diff --git a/lib/GLPI/Agent/Task/RemoteInventory/Remotes.pm b/lib/GLPI/Agent/Task/RemoteInventory/Remotes.pm
@@ -76,19 +76,59 @@ sub getall {
     return values %{$self->{_remotes}};
 }
 
+sub sort {
+    my ($self) = @_;
+
+    return unless $self->{_list} && @{$self->{_list}} > 1;
+
+    $self->{_list} = [
+        sort { $a->expiration() <=> $b->expiration() } @{$self->{_list}}
+    ];
+}
+
 sub next {
     my ($self) = @_;
 
-    my @remotes = $self->getall()
-        or return;
+    # next API now always initialize an internal list
+    unless ($self->{_list}) {
+        my @remotes = $self->getall()
+            or return;
+
+        $self->{_list} = \@remotes;
 
-    my ($remote) = sort { $a->expiration() <=> $b->expiration() } @remotes;
+        $self->sort();
+    }
 
-    return unless $remote->expiration() <= time || $self->{_config}->{force};
+    my $remote = shift @{$self->{_list}}
+        or return;
+
+    # Skip scheduling check if forcing and not re-trying a failed remote
+    unless ($self->{_config}->{force} && !$remote->retry()) {
+        return if ($self->{_config}->{'remote-scheduling'} || $remote->retry()) && $remote->expiration() > time;
+    }
 
     return $remote;
 }
 
+sub retry {
+    my ($self, $remote, $maxdelay) = @_;
+
+    return unless $self->{_list};
+
+    # Add one hour to last retry time
+    my $timeout = $remote->retry() // 0;
+    $timeout += 3600;
+    # But don't retry if the delay is overdue
+    return if $timeout > $maxdelay;
+
+    push @{$self->{_list}}, $remote->retry($timeout);
+
+    # Always sort in case of a not empty big list running for a bigger time than
+    # the retry expiration but not if forcing so retry always occurs after the
+    # last pending remote
+    $self->sort() unless $self->{_config}->{force};
+}
+
 sub store {
     my ($self) = @_;