diff --git a/README.md b/README.md index d114e63..9a15257 100644 --- a/README.md +++ b/README.md @@ -73,15 +73,16 @@ You can pass hosts via CLI arguments: hostpatrol compile --host my-host-1 --host my-host-2 > /tmp/hostpatrol-report.json ``` -This command connects to hosts sequentially and ignores problematic -hosts in the output. +This command connects to hosts in parallel and ignores all failed +hosts by reporting errors in the output. -To use parallel mode, use `--parallel` flag. In this case, if any of -the hosts cause an error, entire operation will fail: - -```sh -hostpatrol compile --parallel --host my-host-1 --host my-host-2 > /tmp/hostpatrol-report.json -``` +> If you want to change the number of concurrent tasks, do so with +> `--parallel` option: +> +> +> ```sh +> hostpatrol compile --parallel 10 --host my-host-1 --host my-host-2 ... > /tmp/hostpatrol-report.json +> ``` Alternatively, you can use a configuration file which has additional benefit of attaching static information to your hosts such as external diff --git a/package.yaml b/package.yaml index a677643..f4b0b12 100644 --- a/package.yaml +++ b/package.yaml @@ -20,12 +20,12 @@ library: dependencies: - aeson - aeson-combinators + - async-pool - autodocodec - autodocodec-schema - bytestring - file-embed - githash - - monad-parallel - mtl - optparse-applicative - path diff --git a/src/HostPatrol/Cli.hs b/src/HostPatrol/Cli.hs index ff2c570..911c689 100644 --- a/src/HostPatrol/Cli.hs +++ b/src/HostPatrol/Cli.hs @@ -64,11 +64,11 @@ commandCompile = OA.hsubparser (OA.command "compile" (OA.info parser infomod) <> doCompile <$> OA.optional (OA.strOption (OA.short 'c' <> OA.long "config" <> OA.action "file" <> OA.help "Path to the configuration file.")) <*> OA.many (OA.strOption (OA.short 'h' <> OA.long "host" <> OA.help "Remote host (in SSH destination format).")) - <*> OA.switch (OA.short 'p' <> OA.long "parallel" <> OA.help "Hit remote hosts in parallel.") + <*> OA.option OA.auto (OA.short 'p' <> OA.long "parallel" <> OA.value 4 <> OA.showDefault <> OA.help "Number of hosts to hit concurrently.") -- | @compile@ CLI command program. -doCompile :: Maybe FilePath -> [T.Text] -> Bool -> IO ExitCode +doCompile :: Maybe FilePath -> [T.Text] -> Int -> IO ExitCode doCompile cpath dests par = do baseConfig <- maybe (pure (Config.Config [] [])) Config.readConfigFile cpath let config = diff --git a/src/HostPatrol/Remote.hs b/src/HostPatrol/Remote.hs index 9c3b4f1..bf36600 100644 --- a/src/HostPatrol/Remote.hs +++ b/src/HostPatrol/Remote.hs @@ -7,23 +7,24 @@ -- host information and produce host report. module HostPatrol.Remote where +import qualified Control.Concurrent.Async.Pool as Async.Pool import Control.Monad.Except (ExceptT, MonadError (..), runExceptT) import Control.Monad.IO.Class (MonadIO (liftIO)) -import qualified Control.Monad.Parallel as MP import qualified Data.Aeson as Aeson import qualified Data.Aeson.Combinators.Decode as ACD -import Data.Bool (bool) import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Char8 as BLC +import Data.Either (partitionEithers) import Data.FileEmbed (embedStringFile) import qualified Data.List as List -import Data.Maybe (catMaybes, fromMaybe) +import Data.Maybe (fromMaybe) import qualified Data.Scientific as S import qualified Data.Text as T +import qualified Data.Time as Time import qualified HostPatrol.Config as Config +import qualified HostPatrol.Meta as Meta import qualified HostPatrol.Types as Types +import qualified Path as P import System.Exit (ExitCode (..)) -import System.IO (hPutStrLn, stderr) import qualified System.Process.Typed as TP import Text.Read (readEither) import qualified Zamazingo.Ssh as Z.Ssh @@ -36,23 +37,37 @@ import qualified Zamazingo.Text as Z.Text -- | Attempts to compile host patrol report for a given configuration. compileReport :: MonadError HostPatrolError m - => MP.MonadParallel m => MonadIO m - => Bool + => Int -> Config.Config -> m Types.Report compileReport par Config.Config {..} = do - _reportHosts <- reporter _configHosts + now <- liftIO Time.getCurrentTime + (errs, _reportHosts) <- liftIO (compileHostReportsIO par _configHosts) _reportKnownSshKeys <- concat <$> mapM parseSshPublicKeys _configKnownSshKeys + let _reportMeta = + Types.ReportMeta + { _reportMetaVersion = Meta._buildInfoVersion Meta.buildInfo + , _reportMetaBuildTag = Meta._buildInfoGitTag Meta.buildInfo + , _reportMetaBuildHash = Meta._buildInfoGitHash Meta.buildInfo + , _reportMetaTimestamp = now + } + _reportErrors = fmap toReportError errs pure Types.Report {..} - where - reporter = bool (fmap catMaybes . mapM go) (MP.mapM compileHostReport) par - go h@Config.HostSpec {..} = do - liftIO (hPutStrLn stderr ("Patrolling " <> T.unpack _hostSpecName)) - res <- runExceptT (compileHostReport h) - case res of - Left err -> liftIO (BLC.hPutStrLn stderr (Aeson.encode err) >> pure Nothing) - Right sr -> pure (Just sr) + + +-- | Attempts to compile host reports for a given list of host +-- specifications and parallelism. +-- +-- If any host report fails to compile, it will be returned in the +-- error list, otherwise in the host report list. +compileHostReportsIO + :: Int + -> [Config.HostSpec] + -> IO ([HostPatrolError], [Types.HostReport]) +compileHostReportsIO par hs = + Async.Pool.withTaskGroup par $ + \tg -> partitionEithers <$> Async.Pool.mapConcurrently tg (runExceptT . compileHostReport) hs -- | Attempts to retrieve remote host information and produce a host @@ -116,6 +131,18 @@ instance Aeson.ToJSON HostPatrolError where toJSON (HostPatrolErrorUnknown err) = Aeson.object [("type", "unknown"), "error" Aeson..= err] +-- | Converts a 'HostPatrolError' to 'Types.ReportError'. +toReportError :: HostPatrolError -> Types.ReportError +toReportError (HostPatrolErrorSsh h ssherror) = Types.ReportError (Just h) $ case ssherror of + Z.Ssh.SshErrorConnection _ err -> "SSH connection error: " <> err + Z.Ssh.SshErrorCommandTimeout _ cmd -> "SSH command timeout: " <> T.unwords cmd + Z.Ssh.SshErrorCommand _ cmd -> "SSH command error: " <> T.unwords cmd + Z.Ssh.SshErrorFileRead _ p -> "SSH file read error: " <> T.pack (P.toFilePath p) + Z.Ssh.SshErrorMissingFile _ p -> "SSH missing file error: " <> T.pack (P.toFilePath p) +toReportError (HostPatrolErrorParse h err) = Types.ReportError (Just h) err +toReportError (HostPatrolErrorUnknown err) = Types.ReportError Nothing err + + -- * Internal diff --git a/src/HostPatrol/Types.hs b/src/HostPatrol/Types.hs index 07a406c..095f7b2 100644 --- a/src/HostPatrol/Types.hs +++ b/src/HostPatrol/Types.hs @@ -23,6 +23,8 @@ import Zamazingo.Ssh (SshConfig) data Report = Report { _reportHosts :: ![HostReport] , _reportKnownSshKeys :: ![SshPublicKey] + , _reportMeta :: !ReportMeta + , _reportErrors :: ![ReportError] } deriving (Eq, Generic, Show) deriving (Aeson.FromJSON, Aeson.ToJSON) via (ADC.Autodocodec Report) @@ -37,6 +39,58 @@ instance ADC.HasCodec Report where Report <$> ADC.requiredField "hosts" "List of host reports." ADC..= _reportHosts <*> ADC.requiredField "knownSshKeys" "List of known SSH public keys." ADC..= _reportKnownSshKeys + <*> ADC.requiredField "meta" "Meta information of the report." ADC..= _reportMeta + <*> ADC.requiredField "errors" "List of errors encountered during the report generation." ADC..= _reportErrors + + +-- * Meta Information + + +-- | Data definition for the meta-information of the report. +data ReportMeta = ReportMeta + { _reportMetaVersion :: !T.Text + , _reportMetaBuildTag :: !(Maybe T.Text) + , _reportMetaBuildHash :: !(Maybe T.Text) + , _reportMetaTimestamp :: !Time.UTCTime + } + deriving (Eq, Generic, Show) + deriving (Aeson.FromJSON, Aeson.ToJSON) via (ADC.Autodocodec ReportMeta) + + +instance ADC.HasCodec ReportMeta where + codec = + _codec ADC. "Report Meta Information" + where + _codec = + ADC.object "ReportMeta" $ + ReportMeta + <$> ADC.requiredField "version" "Version of the application." ADC..= _reportMetaVersion + <*> ADC.optionalField "buildTag" "Build tag of the application." ADC..= _reportMetaBuildTag + <*> ADC.optionalField "buildHash" "Build hash of the application." ADC..= _reportMetaBuildHash + <*> ADC.requiredField "timestamp" "Timestamp of the report." ADC..= _reportMetaTimestamp + + +-- * Meta Information + + +-- | Data definition for errors. +data ReportError = ReportError + { _reportErrorHost :: !(Maybe T.Text) + , _reportErrorMessage :: !T.Text + } + deriving (Eq, Generic, Show) + deriving (Aeson.FromJSON, Aeson.ToJSON) via (ADC.Autodocodec ReportError) + + +instance ADC.HasCodec ReportError where + codec = + _codec ADC. "Report Error" + where + _codec = + ADC.object "ReportError" $ + ReportError + <$> ADC.optionalField "host" "Host of the error if applicable." ADC..= _reportErrorHost + <*> ADC.requiredField "message" "Error message." ADC..= _reportErrorMessage -- * Host diff --git a/website/package-lock.json b/website/package-lock.json index ac3f271..119ee2c 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -10,12 +10,14 @@ "dependencies": { "@nextui-org/react": "^2.2.10", "ajv": "^8.12.0", + "dayjs": "^1.11.10", "framer-motion": "^11.0.22", "json-schema-to-ts": "^3.0.1", "next": "14.1.4", "purify-ts": "^2.0.3", "react": "^18", "react-dom": "^18", + "react-icons": "^5.1.0", "react-toastify": "^10.0.5", "recharts": "^2.12.3" }, @@ -3601,6 +3603,11 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/dayjs": { + "version": "1.11.10", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", + "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" + }, "node_modules/debug": { "version": "4.3.4", "dev": true, @@ -6125,6 +6132,14 @@ "react": "^18.2.0" } }, + "node_modules/react-icons": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/react-icons/-/react-icons-5.1.0.tgz", + "integrity": "sha512-D3zug1270S4hbSlIRJ0CUS97QE1yNNKDjzQe3HqY0aefp2CBn9VgzgES27sRR2gOvFK+0CNx/BW0ggOESp6fqQ==", + "peerDependencies": { + "react": "*" + } + }, "node_modules/react-is": { "version": "16.13.1", "license": "MIT" diff --git a/website/package.json b/website/package.json index bd7ac05..2862f8f 100644 --- a/website/package.json +++ b/website/package.json @@ -14,12 +14,14 @@ "dependencies": { "@nextui-org/react": "^2.2.10", "ajv": "^8.12.0", + "dayjs": "^1.11.10", "framer-motion": "^11.0.22", "json-schema-to-ts": "^3.0.1", "next": "14.1.4", "purify-ts": "^2.0.3", "react": "^18", "react-dom": "^18", + "react-icons": "^5.1.0", "react-toastify": "^10.0.5", "recharts": "^2.12.3" }, diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx index e13652f..38380bb 100644 --- a/website/src/app/page.tsx +++ b/website/src/app/page.tsx @@ -86,15 +86,14 @@ hosts: 
{`hostpatrol compile --host my-host-1 --host my-host-2 > /tmp/hostpatrol-report.json`}
-

This command connects to hosts sequentially and ignores problematic hosts in the output.

-

- To use parallel mode, use `--parallel` flag. In this case, if any of the hosts cause an error, entire - operation will fail: + This command connects to hosts in parallel and ignores all failed hosts by reporting errors in the output.

+

If you want to change the number of concurrent tasks, do so with `--parallel` option:

+ - 
{`hostpatrol compile --parallel --host my-host-1 --host my-host-2 > /tmp/hostpatrol-report.json`}
+ 
{`hostpatrol compile --parallel 10 --host my-host-1 --host my-host-2 ... > /tmp/hostpatrol-report.json`}

diff --git a/website/src/components/report/TabOverview.tsx b/website/src/components/report/TabOverview.tsx index b4734b0..a087800 100644 --- a/website/src/components/report/TabOverview.tsx +++ b/website/src/components/report/TabOverview.tsx @@ -1,40 +1,78 @@ import { HostPatrolReport, buildSshKeysTable } from '@/lib/data'; +import dayjs from 'dayjs'; +import LocalizedFormat from 'dayjs/plugin/localizedFormat'; +import { FaCircleInfo } from 'react-icons/fa6'; import { SimpleBarChart, histogram } from '../helpers'; +dayjs.extend(LocalizedFormat); export function TabOverview({ data }: { data: HostPatrolReport }) { const sshkeys = Object.values(buildSshKeysTable(data)); return ( -

-
- Showing summary of {data.hosts.length} host{data.hosts.length === 1 ? '' : 's'}, {data.knownSshKeys.length} SSH - public key{sshkeys.length === 1 ? '' : 's'} known, and total of {sshkeys.length} SSH public key - {sshkeys.length === 1 ? '' : 's'} seen. -
+
+
+
+ (x.isKnown ? 'Known SSH Keys' : 'Unknown SSH Keys'), sshkeys, { + 'Known SSH Keys': 0, + 'Unknown SSH Keys': 0, + })} + size={[480, 320]} + /> +
-
- (x.isKnown ? 'Known SSH Keys' : 'Unknown SSH Keys'), sshkeys, { - 'Known SSH Keys': 0, - 'Unknown SSH Keys': 0, - })} - size={[480, 320]} - /> -
+
+ x.timezone.split(' ', 1)[0] || 'UNKNOWN', data.hosts)} + size={[480, 320]} + /> +
-
- x.timezone.split(' ', 1)[0] || 'UNKNOWN', data.hosts)} - size={[480, 320]} - /> -
+
+ x.cloud.name, data.hosts)} size={[480, 320]} /> +
-
- x.cloud.name, data.hosts)} size={[480, 320]} /> +
+ x.distribution.name, data.hosts)} size={[480, 320]} /> +
-
- x.distribution.name, data.hosts)} size={[480, 320]} /> +
+
+

+ {data.errors.length > 0 ? ( + + + {data.errors.length} error{data.errors.length === 1 ? '' : 's'} + {' '} + were encountered during patrol. + + ) : ( + No errors were encountered during the patrol. + )} +

+ + {data.errors.map((error, i) => ( +
+ {error.host} {error.message} +
+ ))} + +
+ This report is generated by{' '} + + hostpatrol v{data.meta.version} + {' '} + on{' '} + + {dayjs(data.meta.timestamp).format('LLLL')} + + . +
+
); diff --git a/website/src/lib/data.ts b/website/src/lib/data.ts index 018d5fb..0d03392 100644 --- a/website/src/lib/data.ts +++ b/website/src/lib/data.ts @@ -7,6 +7,19 @@ import { Just, Maybe, Nothing } from 'purify-ts/Maybe'; export const HOSTPATROL_REPORT_SCHEMA = { $comment: 'Host Patrol Report\nReport', properties: { + errors: { + $comment: 'List of errors encountered during the report generation.', + items: { + $comment: 'Report Error\nReportError', + properties: { + host: { $comment: 'Host of the error if applicable.', type: 'string' }, + message: { $comment: 'Error message.', type: 'string' }, + }, + required: ['message'], + type: 'object', + }, + type: 'array', + }, hosts: { $comment: 'List of host reports.', items: { @@ -257,8 +270,19 @@ export const HOSTPATROL_REPORT_SCHEMA = { }, type: 'array', }, + meta: { + $comment: 'Meta information of the report.\nReport Meta Information\nReportMeta', + properties: { + buildHash: { $comment: 'Build hash of the application.', type: 'string' }, + buildTag: { $comment: 'Build tag of the application.', type: 'string' }, + timestamp: { $comment: 'Timestamp of the report.\nLocalTime', type: 'string' }, + version: { $comment: 'Version of the application.', type: 'string' }, + }, + required: ['timestamp', 'version'], + type: 'object', + }, }, - required: ['knownSshKeys', 'hosts'], + required: ['errors', 'meta', 'knownSshKeys', 'hosts'], type: 'object', } as const satisfies JSONSchema;