From 636f42099fa898a42d4b1893dc1b0f31c4af69c2 Mon Sep 17 00:00:00 2001 From: Ramanan Ravikumar <38394463+ramanan-ravi@users.noreply.github.com> Date: Mon, 30 Jan 2023 21:32:28 +0530 Subject: [PATCH] Reduce memory usage (#85) --- config.yaml | 974 ++++++++++++++++++++-------------------- core/config.go | 5 +- go.mod | 16 +- go.sum | 44 +- output/output.go | 11 +- scan/process_image.go | 68 ++- server/grpc.go | 16 +- server/http.go | 4 +- signature/signatures.go | 5 +- 9 files changed, 579 insertions(+), 564 deletions(-) diff --git a/config.yaml b/config.yaml index 2d0f892..472d285 100644 --- a/config.yaml +++ b/config.yaml @@ -1,494 +1,494 @@ # Secret Scanner Configuration File -blacklisted_strings: [] # skip matches containing any of these strings (case sensitive) -blacklisted_extensions: [".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar.gz", ".ttf", ".lock", ".pem"] -blacklisted_paths: ["{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes"] -exclude_paths: ["{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd"] # use {sep} for the OS' path seperator and {name_sep} for - (i.e. / or \) +blacklisted_strings: [ ] # skip matches containing any of these strings (case sensitive) +blacklisted_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] +blacklisted_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] +exclude_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd" ] # use {sep} for the OS' path seperator and {name_sep} for - (i.e. / or \) signatures: - - part: 'extension' - match: '.pem' - name: 'Potential cryptographic private key' - - part: 'extension' - match: '.log' - name: 'Log file' - - part: 'extension' - match: '.pkcs12' - name: 'Potential cryptographic key bundle' - - part: 'extension' - match: '.p12' - name: 'Potential cryptographic key bundle' - - part: 'extension' - match: '.pfx' - name: 'Potential cryptographic key bundle' - - part: 'extension' - match: '.asc' - name: 'Potential cryptographic key bundle' - - part: 'filename' - match: 'otr.private_key' - name: 'Pidgin OTR private key' - - part: 'extension' - match: '.ovpn' - name: 'OpenVPN client configuration file' - - part: 'extension' - match: '.cscfg' - name: 'Azure service configuration schema file' - - part: 'extension' - match: '.rdp' - name: 'Remote Desktop connection file' - - part: 'extension' - match: '.mdf' - name: 'Microsoft SQL database file' - - part: 'extension' - match: '.sdf' - name: 'Microsoft SQL server compact database file' - - part: 'extension' - match: '.sqlite' - name: 'SQLite database file' - - part: 'extension' - match: '.sqlite3' - name: 'SQLite3 database file' - - part: 'extension' - match: '.bek' - name: 'Microsoft BitLocker recovery key file' - - part: 'extension' - match: '.tpm' - name: 'Microsoft BitLocker Trusted Platform Module password file' - - part: 'extension' - match: '.fve' - name: 'Windows BitLocker full volume encrypted data file' - - part: 'extension' - match: '.jks' - name: 'Java keystore file' - - part: 'extension' - match: '.psafe3' - name: 'Password Safe database file' - - part: 'filename' - match: 'secret_token.rb' - name: 'Ruby On Rails secret token configuration file' - - part: 'filename' - match: 'carrierwave.rb' - name: 'Carrierwave configuration file' - - part: 'filename' - match: 'database.yml' - name: 'Potential Ruby On Rails database configuration file' - - part: 'filename' - match: 'omniauth.rb' - name: 'OmniAuth configuration file' - - part: 'filename' - match: 'settings.py' - name: 'Django configuration file' - - part: 'extension' - match: '.agilekeychain' - name: '1Password password manager database file' - - part: 'extension' - match: '.keychain' - name: 'Apple Keychain database file' - - part: 'extension' - match: '.pcap' - name: 'Network traffic capture file' - - part: 'extension' - match: '.gnucash' - name: 'GnuCash database file' - - part: 'filename' - match: 'jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml' - name: 'Jenkins publish over SSH plugin file' - - part: 'filename' - match: 'credentials.xml' - name: 'Potential Jenkins credentials file' - - part: 'extension' - match: '.kwallet' - name: 'KDE Wallet Manager database file' - - part: 'filename' - match: 'LocalSettings.php' - name: 'Potential MediaWiki configuration file' - - part: 'extension' - match: '.tblk' - name: 'Tunnelblick VPN configuration file' - - part: 'filename' - match: 'Favorites.plist' - name: 'Sequel Pro MySQL database manager bookmark file' - - part: 'filename' - match: 'configuration.user.xpl' - name: 'Little Snitch firewall configuration file' - - part: 'extension' - match: '.dayone' - name: 'Day One journal file' - - part: 'filename' - match: 'journal.txt' - name: 'Potential jrnl journal file' - - part: 'filename' - match: 'knife.rb' - name: 'Chef Knife configuration file' - - part: 'filename' - match: 'proftpdpasswd' - name: 'cPanel backup ProFTPd credentials file' - - part: 'filename' - match: 'robomongo.json' - name: 'Robomongo MongoDB manager configuration file' - - part: 'filename' - match: 'filezilla.xml' - name: 'FileZilla FTP configuration file' - - part: 'filename' - match: 'recentservers.xml' - name: 'FileZilla FTP recent servers file' - - part: 'filename' - match: 'ventrilo_srv.ini' - name: 'Ventrilo server configuration file' - - part: 'filename' - match: 'terraform.tfvars' - name: 'Terraform variable config file' - - part: 'filename' - match: '.exports' - name: 'Shell configuration file' - - part: 'filename' - match: '.functions' - name: 'Shell configuration file' - - part: 'filename' - match: '.extra' - name: 'Shell configuration file' - - - part: 'filename' - regex: '^.*_rsa$' - name: 'Private SSH key' - - part: 'filename' - regex: '^.*_dsa$' - name: 'Private SSH key' - - part: 'filename' - regex: '^.*_ed25519$' - name: 'Private SSH key' - - part: 'filename' - regex: '^.*_ecdsa$' - name: 'Private SSH key' - - part: 'path' - regex: '\.?ssh/config$' - name: 'SSH configuration file' - - part: 'extension' - regex: '^key(pair)?$' - name: 'Potential cryptographic private key' - - part: 'filename' - regex: '^\.?(bash_|zsh_|sh_|z)?history$' - name: 'Shell command history file' - - part: 'filename' - regex: '^\.?mysql_history$' - name: 'MySQL client command history file' - - part: 'filename' - regex: '^\.?psql_history$' - name: 'PostgreSQL client command history file' - - part: 'filename' - regex: '^\.?pgpass$' - name: 'PostgreSQL password file' - - part: 'filename' - regex: '^\.?irb_history$' - name: 'Ruby IRB console history file' - - part: 'path' - regex: '\.?purple/accounts\.xml$' - name: 'Pidgin chat client account configuration file' - - part: 'path' - regex: '\.?xchat2?/servlist_?\.conf$' - name: 'Hexchat/XChat IRC client server list configuration file' - - part: 'path' - regex: '\.?irssi/config$' - name: 'Irssi IRC client configuration file' - - part: 'path' - regex: '\.?recon-ng/keys\.db$' - name: 'Recon-ng web reconnaissance framework API key database' - - part: 'filename' - regex: '^\.?dbeaver-data-sources.xml$' - name: 'DBeaver SQL database manager configuration file' - - part: 'filename' - regex: '^\.?muttrc$' - name: 'Mutt e-mail client configuration file' - - part: 'filename' - regex: '^\.?s3cfg$' - name: 'S3cmd configuration file' - - part: 'path' - regex: '\.?aws/credentials$' - name: 'AWS CLI credentials file' - - part: 'filename' - regex: '^sftp-config(\.json)?$' - name: 'SFTP connection configuration file' - - part: 'filename' - regex: '^\.?trc$' - name: 'T command-line Twitter client configuration file' - - part: 'filename' - regex: '^\.?(bash|zsh|csh)rc$' - name: 'Shell configuration file' - - part: 'filename' - regex: '^\.?(bash_|zsh_)?profile$' - name: 'Shell profile configuration file' - - part: 'filename' - regex: '^\.?(bash_|zsh_)?aliases$' - name: 'Shell command alias configuration file' - - part: 'filename' - regex: 'config(\.inc)?\.php$' - name: 'PHP configuration file' - - part: 'extension' - regex: '^key(store|ring)$' - name: 'GNOME Keyring database file' - - part: 'extension' - regex: '^kdbx?$' - name: 'KeePass password manager database file' - - part: 'extension' - regex: '^sql(dump)?$' - name: 'SQL dump file' - - part: 'filename' - regex: '^\.?htpasswd$' - name: 'Apache htpasswd file' - - part: 'filename' - regex: '^(\.|_)?netrc$' - name: 'Configuration file for auto-login process' - - part: 'path' - regex: '\.?gem/credentials$' - name: 'Rubygems credentials file' - - part: 'filename' - regex: '^\.?tugboat$' - name: 'Tugboat DigitalOcean management tool configuration' - - part: 'path' - regex: 'doctl/config.yaml$' - name: 'DigitalOcean doctl command-line client configuration file' - - part: 'filename' - regex: '^\.?git-credentials$' - name: 'git-credential-store helper credentials file' - - part: 'path' - regex: 'config/hub$' - name: 'GitHub Hub command-line client configuration file' - - part: 'filename' - regex: '^\.?gitconfig$' - name: 'Git configuration file' - - part: 'path' - regex: '\.?chef/(.*)\.pem$' - name: 'Chef private key' - - part: 'path' - regex: 'etc/shadow$' - name: 'Potential Linux shadow file' - - part: 'path' - regex: 'etc/passwd$' - name: 'Potential Linux passwd file' - comment: 'Contains system user information' - - part: 'filename' - regex: '^\.?dockercfg$' - name: 'Docker configuration file' - - part: 'filename' - regex: '^\.?npmrc$' - name: 'NPM configuration file' - - part: 'filename' - regex: '^\.?env$' - name: 'Environment configuration file' +- part: 'extension' + match: '.pem' + name: 'Potential cryptographic private key' +- part: 'extension' + match: '.log' + name: 'Log file' +- part: 'extension' + match: '.pkcs12' + name: 'Potential cryptographic key bundle' +- part: 'extension' + match: '.p12' + name: 'Potential cryptographic key bundle' +- part: 'extension' + match: '.pfx' + name: 'Potential cryptographic key bundle' +- part: 'extension' + match: '.asc' + name: 'Potential cryptographic key bundle' +- part: 'filename' + match: 'otr.private_key' + name: 'Pidgin OTR private key' +- part: 'extension' + match: '.ovpn' + name: 'OpenVPN client configuration file' +- part: 'extension' + match: '.cscfg' + name: 'Azure service configuration schema file' +- part: 'extension' + match: '.rdp' + name: 'Remote Desktop connection file' +- part: 'extension' + match: '.mdf' + name: 'Microsoft SQL database file' +- part: 'extension' + match: '.sdf' + name: 'Microsoft SQL server compact database file' +- part: 'extension' + match: '.sqlite' + name: 'SQLite database file' +- part: 'extension' + match: '.sqlite3' + name: 'SQLite3 database file' +- part: 'extension' + match: '.bek' + name: 'Microsoft BitLocker recovery key file' +- part: 'extension' + match: '.tpm' + name: 'Microsoft BitLocker Trusted Platform Module password file' +- part: 'extension' + match: '.fve' + name: 'Windows BitLocker full volume encrypted data file' +- part: 'extension' + match: '.jks' + name: 'Java keystore file' +- part: 'extension' + match: '.psafe3' + name: 'Password Safe database file' +- part: 'filename' + match: 'secret_token.rb' + name: 'Ruby On Rails secret token configuration file' +- part: 'filename' + match: 'carrierwave.rb' + name: 'Carrierwave configuration file' +- part: 'filename' + match: 'database.yml' + name: 'Potential Ruby On Rails database configuration file' +- part: 'filename' + match: 'omniauth.rb' + name: 'OmniAuth configuration file' +- part: 'filename' + match: 'settings.py' + name: 'Django configuration file' +- part: 'extension' + match: '.agilekeychain' + name: '1Password password manager database file' +- part: 'extension' + match: '.keychain' + name: 'Apple Keychain database file' +- part: 'extension' + match: '.pcap' + name: 'Network traffic capture file' +- part: 'extension' + match: '.gnucash' + name: 'GnuCash database file' +- part: 'filename' + match: 'jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml' + name: 'Jenkins publish over SSH plugin file' +- part: 'filename' + match: 'credentials.xml' + name: 'Potential Jenkins credentials file' +- part: 'extension' + match: '.kwallet' + name: 'KDE Wallet Manager database file' +- part: 'filename' + match: 'LocalSettings.php' + name: 'Potential MediaWiki configuration file' +- part: 'extension' + match: '.tblk' + name: 'Tunnelblick VPN configuration file' +- part: 'filename' + match: 'Favorites.plist' + name: 'Sequel Pro MySQL database manager bookmark file' +- part: 'filename' + match: 'configuration.user.xpl' + name: 'Little Snitch firewall configuration file' +- part: 'extension' + match: '.dayone' + name: 'Day One journal file' +- part: 'filename' + match: 'journal.txt' + name: 'Potential jrnl journal file' +- part: 'filename' + match: 'knife.rb' + name: 'Chef Knife configuration file' +- part: 'filename' + match: 'proftpdpasswd' + name: 'cPanel backup ProFTPd credentials file' +- part: 'filename' + match: 'robomongo.json' + name: 'Robomongo MongoDB manager configuration file' +- part: 'filename' + match: 'filezilla.xml' + name: 'FileZilla FTP configuration file' +- part: 'filename' + match: 'recentservers.xml' + name: 'FileZilla FTP recent servers file' +- part: 'filename' + match: 'ventrilo_srv.ini' + name: 'Ventrilo server configuration file' +- part: 'filename' + match: 'terraform.tfvars' + name: 'Terraform variable config file' +- part: 'filename' + match: '.exports' + name: 'Shell configuration file' +- part: 'filename' + match: '.functions' + name: 'Shell configuration file' +- part: 'filename' + match: '.extra' + name: 'Shell configuration file' - - part: 'contents' - regex: '(A3T[A-Z0-9]|AKIA|AGPA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' - name: 'AWS Access Key ID Value' - - part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)access)_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Access Key ID' - - part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)account)_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[0-9]{4}-?[0-9]{4}-?[0-9]{4}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Account ID' - - part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)secret)_?((?i)access)?_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{40}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Secret Access Key' - - part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)session)?_?((?i)token)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{100,400}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Session Token' - - part: 'contents' - regex: "(?i)artifactory.{0,50}(\\\"|'|`)?[a-zA-Z0-9=]{112}(\\\"|'|`)?" - regextype: 'large' - name: 'Artifactory' - - part: 'contents' - regex: "(?i)codeclima.{0,50}(\\\"|'|`)?[0-9a-f]{64}(\\\"|'|`)?" - regextype: 'large' - name: 'CodeClimate' - - part: 'contents' - regex: 'EAACEdEose0cBA[0-9A-Za-z]+' - name: 'Facebook access token' - - part: 'contents' - regex: "((\\\"|'|`)?type(\\\"|'|`)?\\\\s{0,50}(:|=>|=)\\\\s{0,50}(\\\"|'|`)?service_account(\\\"|'|`)?,?)" - regextype: 'large' - name: 'Google (GCM) Service account' - - part: 'contents' - regex: '(?:r|s)k_(live|test)_[0-9a-zA-Z]{24}' - name: 'Stripe API key' - - part: 'contents' - regex: '[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - name: 'Google OAuth Key' - - part: 'contents' - regex: 'AIza[0-9A-Za-z\\-_]{35}' - name: 'Google Cloud API Key' - - part: 'contents' - regex: 'ya29\\.[0-9A-Za-z\\-_]+' - name: 'Google OAuth Access Token' - - part: 'contents' - regex: 'sk_[live|test]_[0-9a-z]{32}' - name: 'Picatic API key' - - part: 'contents' - regex: 'sq0atp-[0-9A-Za-z\-_]{22}' - name: 'Square Access Token' - - part: 'contents' - regex: 'sq0csp-[0-9A-Za-z\-_]{43}' - name: 'Square OAuth Secret' - - part: 'contents' - regex: 'access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}' - name: 'PayPal/Braintree Access Token' - - part: 'contents' - regex: 'amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' - name: 'Amazon MWS Auth Token' - - part: 'contents' - regex: 'SK[0-9a-fA-F]{32}' - name: 'Twilo API Key' - - part: 'contents' - regex: 'SG\.[0-9A-Za-z\-_]{22}\.[0-9A-Za-z\-_]{43}' - name: 'SendGrid API Key' - - part: 'contents' - regex: 'key-[0-9a-zA-Z]{32}' - name: 'MailGun API Key' - - part: 'contents' - regex: '[0-9a-f]{32}-us[0-9]{12}' - name: 'MailChimp API Key' - - part: 'contents' - regex: "sshpass -p.*['|\\\"]" - regextype: 'large' - name: 'SSH Password' - - part: 'contents' - regex: '(https\\://outlook\\.office.com/webhook/[0-9a-f-]{36}\\@)' - name: 'Outlook team' - - part: 'contents' - regex: "(?i)sauce.{0,50}(\\\"|'|`)?[0-9a-f-]{36}(\\\"|'|`)?" - name: 'Sauce Token' - - part: 'contents' - regex: '(xox[pboa]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})' - name: 'Slack Token' - - part: 'contents' - regex: 'https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}' - name: 'Slack Webhook' - - part: 'contents' - regex: "(?i)sonar.{0,50}(\\\"|'|`)?[0-9a-f]{40}(\\\"|'|`)?" - name: 'SonarQube Docs API Key' - - part: 'contents' - regex: "(?i)hockey.{0,50}(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" - name: 'HockeyApp' - - part: 'contents' - regex: '([\w+]{1,24})(://)([^$<]{1})([^\s";]{1,}):([^$<]{1})([^\s";/]{1,})@[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,24}([^\s]+)' - regextype: 'large' - name: 'Username and password in URI' - - part: 'contents' - regex: '(username|user)=?([^$<]{1})([^\s\\\";]{1,})(;|,|:)?(password|pwd|passwd)=' - regextype: 'large' - name: 'Username and password in file' - - part: 'contents' - regex: 'oy2[a-z0-9]{43}' - name: 'NuGet API Key' - - part: 'contents' - regex: "(?i)appid=(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" - name: 'OpenWeather API Key' - - part: 'contents' - regex: 'hawk\.[0-9A-Za-z\-_]{20}\.[0-9A-Za-z\-_]{20}' - regextype: 'large' - name: 'StackHawk API Key' - - - part: 'extension' - match: '.ppk' - name: 'Potential PuTTYgen private key' - - part: 'filename' - match: 'heroku.json' - name: 'Heroku config file' - - part: 'extension' - match: '.sqldump' - name: 'SQL Data dump file' - - part: 'filename' - match: 'dump.sql' - name: 'MySQL dump w/ bcrypt hashes' - - part: 'filename' - match: 'id_rsa_pub' - name: 'Public ssh key' - - part: 'filename' - match: 'mongoid.yml' - name: 'Mongoid config file' - - part: 'filename' - match: 'salesforce.js' - name: 'Salesforce credentials in a nodejs project' - - part: 'extension' - match: '.netrc' - name: 'netrc with SMTP credentials' - - - part: 'filename' - regex: '.remote-sync.json$' - name: 'Created by remote-sync for Atom, contains FTP and/or SCP/SFTP/SSH server details and credentials' - - part: 'filename' - regex: '.esmtprc$' - name: 'esmtp configuration' - - part: 'filename' - regex: '^deployment-config.json?$' - name: 'Created by sftp-deployment for Atom, contains server details and credentials' - - part: 'filename' - regex: '.ftpconfig$' - name: 'Created by sftp-deployment for Atom, contains server details and credentials' +- part: 'filename' + regex: '^.*_rsa$' + name: 'Private SSH key' +- part: 'filename' + regex: '^.*_dsa$' + name: 'Private SSH key' +- part: 'filename' + regex: '^.*_ed25519$' + name: 'Private SSH key' +- part: 'filename' + regex: '^.*_ecdsa$' + name: 'Private SSH key' +- part: 'path' + regex: '\.?ssh/config$' + name: 'SSH configuration file' +- part: 'extension' + regex: '^key(pair)?$' + name: 'Potential cryptographic private key' +- part: 'filename' + regex: '^\.?(bash_|zsh_|sh_|z)?history$' + name: 'Shell command history file' +- part: 'filename' + regex: '^\.?mysql_history$' + name: 'MySQL client command history file' +- part: 'filename' + regex: '^\.?psql_history$' + name: 'PostgreSQL client command history file' +- part: 'filename' + regex: '^\.?pgpass$' + name: 'PostgreSQL password file' +- part: 'filename' + regex: '^\.?irb_history$' + name: 'Ruby IRB console history file' +- part: 'path' + regex: '\.?purple/accounts\.xml$' + name: 'Pidgin chat client account configuration file' +- part: 'path' + regex: '\.?xchat2?/servlist_?\.conf$' + name: 'Hexchat/XChat IRC client server list configuration file' +- part: 'path' + regex: '\.?irssi/config$' + name: 'Irssi IRC client configuration file' +- part: 'path' + regex: '\.?recon-ng/keys\.db$' + name: 'Recon-ng web reconnaissance framework API key database' +- part: 'filename' + regex: '^\.?dbeaver-data-sources.xml$' + name: 'DBeaver SQL database manager configuration file' +- part: 'filename' + regex: '^\.?muttrc$' + name: 'Mutt e-mail client configuration file' +- part: 'filename' + regex: '^\.?s3cfg$' + name: 'S3cmd configuration file' +- part: 'path' + regex: '\.?aws/credentials$' + name: 'AWS CLI credentials file' +- part: 'filename' + regex: '^sftp-config(\.json)?$' + name: 'SFTP connection configuration file' +- part: 'filename' + regex: '^\.?trc$' + name: 'T command-line Twitter client configuration file' +- part: 'filename' + regex: '^\.?(bash|zsh|csh)rc$' + name: 'Shell configuration file' +- part: 'filename' + regex: '^\.?(bash_|zsh_)?profile$' + name: 'Shell profile configuration file' +- part: 'filename' + regex: '^\.?(bash_|zsh_)?aliases$' + name: 'Shell command alias configuration file' +- part: 'filename' + regex: 'config(\.inc)?\.php$' + name: 'PHP configuration file' +- part: 'extension' + regex: '^key(store|ring)$' + name: 'GNOME Keyring database file' +- part: 'extension' + regex: '^kdbx?$' + name: 'KeePass password manager database file' +- part: 'extension' + regex: '^sql(dump)?$' + name: 'SQL dump file' +- part: 'filename' + regex: '^\.?htpasswd$' + name: 'Apache htpasswd file' +- part: 'filename' + regex: '^(\.|_)?netrc$' + name: 'Configuration file for auto-login process' +- part: 'path' + regex: '\.?gem/credentials$' + name: 'Rubygems credentials file' +- part: 'filename' + regex: '^\.?tugboat$' + name: 'Tugboat DigitalOcean management tool configuration' +- part: 'path' + regex: 'doctl/config.yaml$' + name: 'DigitalOcean doctl command-line client configuration file' +- part: 'filename' + regex: '^\.?git-credentials$' + name: 'git-credential-store helper credentials file' +- part: 'path' + regex: 'config/hub$' + name: 'GitHub Hub command-line client configuration file' +- part: 'filename' + regex: '^\.?gitconfig$' + name: 'Git configuration file' +- part: 'path' + regex: '\.?chef/(.*)\.pem$' + name: 'Chef private key' +- part: 'path' + regex: 'etc/shadow$' + name: 'Potential Linux shadow file' +- part: 'path' + regex: 'etc/passwd$' + name: 'Potential Linux passwd file' + comment: 'Contains system user information' +- part: 'filename' + regex: '^\.?dockercfg$' + name: 'Docker configuration file' +- part: 'filename' + regex: '^\.?npmrc$' + name: 'NPM configuration file' +- part: 'filename' + regex: '^\.?env$' + name: 'Environment configuration file' - - part: 'contents' - regex: '-----BEGIN (EC|RSA|DSA|OPENSSH|PGP) PRIVATE KEY' - name: 'Contains a private key' - - part: 'contents' - regex: 'define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?[''|"].{10,120}[''|"]' - regextype: 'large' - name: 'WP-Config' - - part: 'contents' - regex: '(?i)(aws_access_key_id|aws_secret_access_key)(.{0,20})?=.[0-9a-zA-Z\/+]{20,40}' - name: 'AWS cred file info' - - part: 'contents' - regex: '(?i)(facebook|fb)(.{0,20})?(?-i)[''\"][0-9a-f]{32}[''\"]' - name: 'Facebook Secret Key' - - part: 'contents' - regex: '(?i)(facebook|fb)(.{0,20})?[''\"][0-9]{13,17}[''\"]' - name: 'Facebook Client ID' - - part: 'contents' - regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{35,44}[''\"]' - name: 'Twitter Secret Key' - - part: 'contents' - regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{18,25}[''\"]' - name: 'Twitter Client ID' - - part: 'contents' - regex: '(?i)github(.{0,20})?(?-i)[''\"][0-9a-zA-Z]{35,40}[''\"]' - name: 'Github Key' - - part: 'contents' - regex: '(?i)heroku(.{0,20})?[''"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}[''"]' - name: 'Heroku API key' - - part: 'contents' - regex: '(?i)linkedin(.{0,20})?(?-i)[''\"][0-9a-z]{12}[''\"]' - name: 'Linkedin Client ID' - - part: 'contents' - regex: '(?i)linkedin(.{0,20})?[''\"][0-9a-z]{16}[''\"]' - name: 'LinkedIn Secret Key' - - - part: 'path' - regex: '\.?idea[\\\/]WebServers.xml$' - name: 'Created by Jetbrains IDEs, contains webserver credentials with encoded passwords (not encrypted!)' - - part: 'path' - regex: '\.?vscode[\\\/]sftp.json$' - name: 'Created by vscode-sftp for VSCode, contains SFTP/SSH server details and credentials' - - part: 'path' - regex: 'web[\\\/]ruby[\\\/]secrets.yml' - name: 'Ruby on rails secrets.yml file (contains passwords)' - - part: 'path' - regex: '\.?docker[\\\/]config.json$' - name: 'Docker registry authentication file' - - part: 'path' - regex: 'ruby[\\\/]config[\\\/]master.key$' - name: 'Rails master key (used for decrypting credentials.yml.enc for Rails 5.2+)' - - part: 'path' - regex: '\.?mozilla[\\\/]firefox[\\\/]logins.json$' - name: 'Firefox saved password collection (can be decrypted using keys4.db)' - - - part: 'filename' - match: 'wallet.dat' - name: 'Bitcoin Core wallet' - - part: 'filename' - match: 'onion_v3_private_key' - name: 'Private key for Bitcoin Core onion service' - - part: 'filename' - match: 'bitcoin.conf' - name: 'Bitcoin Core config' +- part: 'contents' + regex: '(A3T[A-Z0-9]|AKIA|AGPA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' + name: 'AWS Access Key ID Value' +- part: 'contents' + regex: "((\\\"|'|`)?((?i)aws)?_?((?i)access)_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}(\\\"|'|`)?)" + regextype: 'large' + name: 'AWS Access Key ID' +- part: 'contents' + regex: "((\\\"|'|`)?((?i)aws)?_?((?i)account)_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[0-9]{4}-?[0-9]{4}-?[0-9]{4}(\\\"|'|`)?)" + regextype: 'large' + name: 'AWS Account ID' +- part: 'contents' + regex: "((\\\"|'|`)?((?i)aws)?_?((?i)secret)_?((?i)access)?_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{40}(\\\"|'|`)?)" + regextype: 'large' + name: 'AWS Secret Access Key' +- part: 'contents' + regex: "((\\\"|'|`)?((?i)aws)?_?((?i)session)?_?((?i)token)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{100,400}(\\\"|'|`)?)" + regextype: 'large' + name: 'AWS Session Token' +- part: 'contents' + regex: "(?i)artifactory.{0,50}(\\\"|'|`)?[a-zA-Z0-9=]{112}(\\\"|'|`)?" + regextype: 'large' + name: 'Artifactory' +- part: 'contents' + regex: "(?i)codeclima.{0,50}(\\\"|'|`)?[0-9a-f]{64}(\\\"|'|`)?" + regextype: 'large' + name: 'CodeClimate' +- part: 'contents' + regex: 'EAACEdEose0cBA[0-9A-Za-z]+' + name: 'Facebook access token' +- part: 'contents' + regex: "((\\\"|'|`)?type(\\\"|'|`)?\\\\s{0,50}(:|=>|=)\\\\s{0,50}(\\\"|'|`)?service_account(\\\"|'|`)?,?)" + regextype: 'large' + name: 'Google (GCM) Service account' +- part: 'contents' + regex: '(?:r|s)k_(live|test)_[0-9a-zA-Z]{24}' + name: 'Stripe API key' +- part: 'contents' + regex: '[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' + name: 'Google OAuth Key' +- part: 'contents' + regex: 'AIza[0-9A-Za-z\\-_]{35}' + name: 'Google Cloud API Key' +- part: 'contents' + regex: 'ya29\\.[0-9A-Za-z\\-_]+' + name: 'Google OAuth Access Token' +- part: 'contents' + regex: 'sk_[live|test]_[0-9a-z]{32}' + name: 'Picatic API key' +- part: 'contents' + regex: 'sq0atp-[0-9A-Za-z\-_]{22}' + name: 'Square Access Token' +- part: 'contents' + regex: 'sq0csp-[0-9A-Za-z\-_]{43}' + name: 'Square OAuth Secret' +- part: 'contents' + regex: 'access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}' + name: 'PayPal/Braintree Access Token' +- part: 'contents' + regex: 'amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + name: 'Amazon MWS Auth Token' +- part: 'contents' + regex: 'SK[0-9a-fA-F]{32}' + name: 'Twilo API Key' +- part: 'contents' + regex: 'SG\.[0-9A-Za-z\-_]{22}\.[0-9A-Za-z\-_]{43}' + name: 'SendGrid API Key' +- part: 'contents' + regex: 'key-[0-9a-zA-Z]{32}' + name: 'MailGun API Key' +- part: 'contents' + regex: '[0-9a-f]{32}-us[0-9]{12}' + name: 'MailChimp API Key' +- part: 'contents' + regex: "sshpass -p.*['|\\\"]" + regextype: 'large' + name: 'SSH Password' +- part: 'contents' + regex: '(https\\://outlook\\.office.com/webhook/[0-9a-f-]{36}\\@)' + name: 'Outlook team' +- part: 'contents' + regex: "(?i)sauce.{0,50}(\\\"|'|`)?[0-9a-f-]{36}(\\\"|'|`)?" + name: 'Sauce Token' +- part: 'contents' + regex: '(xox[pboa]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})' + name: 'Slack Token' +- part: 'contents' + regex: 'https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}' + name: 'Slack Webhook' +- part: 'contents' + regex: "(?i)sonar.{0,50}(\\\"|'|`)?[0-9a-f]{40}(\\\"|'|`)?" + name: 'SonarQube Docs API Key' +- part: 'contents' + regex: "(?i)hockey.{0,50}(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" + name: 'HockeyApp' +- part: 'contents' + regex: '([\w+]{1,24})(://)([^$<]{1})([^\s";]{1,}):([^$<]{1})([^\s";/]{1,})@[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,24}([^\s]+)' + regextype: 'large' + name: 'Username and password in URI' +- part: 'contents' + regex: '(username|user)=?([^$<]{1})([^\s\\\";]{1,})(;|,|:)?(password|pwd|passwd)=' + regextype: 'large' + name: 'Username and password in file' +- part: 'contents' + regex: 'oy2[a-z0-9]{43}' + name: 'NuGet API Key' +- part: 'contents' + regex: "(?i)appid=(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" + name: 'OpenWeather API Key' +- part: 'contents' + regex: 'hawk\.[0-9A-Za-z\-_]{20}\.[0-9A-Za-z\-_]{20}' + regextype: 'large' + name: 'StackHawk API Key' + +- part: 'extension' + match: '.ppk' + name: 'Potential PuTTYgen private key' +- part: 'filename' + match: 'heroku.json' + name: 'Heroku config file' +- part: 'extension' + match: '.sqldump' + name: 'SQL Data dump file' +- part: 'filename' + match: 'dump.sql' + name: 'MySQL dump w/ bcrypt hashes' +- part: 'filename' + match: 'id_rsa_pub' + name: 'Public ssh key' +- part: 'filename' + match: 'mongoid.yml' + name: 'Mongoid config file' +- part: 'filename' + match: 'salesforce.js' + name: 'Salesforce credentials in a nodejs project' +- part: 'extension' + match: '.netrc' + name: 'netrc with SMTP credentials' + +- part: 'filename' + regex: '.remote-sync.json$' + name: 'Created by remote-sync for Atom, contains FTP and/or SCP/SFTP/SSH server details and credentials' +- part: 'filename' + regex: '.esmtprc$' + name: 'esmtp configuration' +- part: 'filename' + regex: '^deployment-config.json?$' + name: 'Created by sftp-deployment for Atom, contains server details and credentials' +- part: 'filename' + regex: '.ftpconfig$' + name: 'Created by sftp-deployment for Atom, contains server details and credentials' + +- part: 'contents' + regex: '-----BEGIN (EC|RSA|DSA|OPENSSH|PGP) PRIVATE KEY' + name: 'Contains a private key' +- part: 'contents' + regex: 'define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?[''|"].{10,120}[''|"]' + regextype: 'large' + name: 'WP-Config' +- part: 'contents' + regex: '(?i)(aws_access_key_id|aws_secret_access_key)(.{0,20})?=.[0-9a-zA-Z\/+]{20,40}' + name: 'AWS cred file info' +- part: 'contents' + regex: '(?i)(facebook|fb)(.{0,20})?(?-i)[''\"][0-9a-f]{32}[''\"]' + name: 'Facebook Secret Key' +- part: 'contents' + regex: '(?i)(facebook|fb)(.{0,20})?[''\"][0-9]{13,17}[''\"]' + name: 'Facebook Client ID' +- part: 'contents' + regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{35,44}[''\"]' + name: 'Twitter Secret Key' +- part: 'contents' + regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{18,25}[''\"]' + name: 'Twitter Client ID' +- part: 'contents' + regex: '(?i)github(.{0,20})?(?-i)[''\"][0-9a-zA-Z]{35,40}[''\"]' + name: 'Github Key' +- part: 'contents' + regex: '(?i)heroku(.{0,20})?[''"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}[''"]' + name: 'Heroku API key' +- part: 'contents' + regex: '(?i)linkedin(.{0,20})?(?-i)[''\"][0-9a-z]{12}[''\"]' + name: 'Linkedin Client ID' +- part: 'contents' + regex: '(?i)linkedin(.{0,20})?[''\"][0-9a-z]{16}[''\"]' + name: 'LinkedIn Secret Key' + +- part: 'path' + regex: '\.?idea[\\\/]WebServers.xml$' + name: 'Created by Jetbrains IDEs, contains webserver credentials with encoded passwords (not encrypted!)' +- part: 'path' + regex: '\.?vscode[\\\/]sftp.json$' + name: 'Created by vscode-sftp for VSCode, contains SFTP/SSH server details and credentials' +- part: 'path' + regex: 'web[\\\/]ruby[\\\/]secrets.yml' + name: 'Ruby on rails secrets.yml file (contains passwords)' +- part: 'path' + regex: '\.?docker[\\\/]config.json$' + name: 'Docker registry authentication file' +- part: 'path' + regex: 'ruby[\\\/]config[\\\/]master.key$' + name: 'Rails master key (used for decrypting credentials.yml.enc for Rails 5.2+)' +- part: 'path' + regex: '\.?mozilla[\\\/]firefox[\\\/]logins.json$' + name: 'Firefox saved password collection (can be decrypted using keys4.db)' + +- part: 'filename' + match: 'wallet.dat' + name: 'Bitcoin Core wallet' +- part: 'filename' + match: 'onion_v3_private_key' + name: 'Private key for Bitcoin Core onion service' +- part: 'filename' + match: 'bitcoin.conf' + name: 'Bitcoin Core config' diff --git a/core/config.go b/core/config.go index 94b0366..4976ae9 100644 --- a/core/config.go +++ b/core/config.go @@ -2,7 +2,6 @@ package core import ( "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -132,9 +131,9 @@ func loadConfigFile(configPath string) (*Config, error) { } if fstat.IsDir() { - data, err = ioutil.ReadFile(path.Join(configPath, "config.yaml")) + data, err = os.ReadFile(path.Join(configPath, "config.yaml")) } else { - data, err = ioutil.ReadFile(configPath) + data, err = os.ReadFile(configPath) } if err != nil { return nil, err diff --git a/go.mod b/go.mod index 90f6aee..127ba7c 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc require ( github.com/Jeffail/tunny v0.1.4 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 - github.com/deepfence/vessel v0.9.0 - github.com/fatih/color v1.13.0 + github.com/deepfence/vessel v0.9.1 + github.com/fatih/color v1.14.1 github.com/flier/gohs v1.2.1 - google.golang.org/grpc v1.51.0 + google.golang.org/grpc v1.52.3 gopkg.in/yaml.v3 v3.0.1 ) @@ -32,10 +32,10 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.11.13 // indirect - github.com/mattn/go-colorable v0.1.9 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.5.0 // indirect github.com/moby/sys/signal v0.6.0 // indirect @@ -51,6 +51,6 @@ require ( golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 2df05b9..d2f196c 100644 --- a/go.sum +++ b/go.sum @@ -132,11 +132,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -260,8 +256,8 @@ github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjI github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deepfence/vessel v0.9.0 h1:WkkqnYt/VvxuT+EY/ExxsLGfmSi0QB7DDE02/9fJ7Y8= -github.com/deepfence/vessel v0.9.0/go.mod h1:cN+m37ETZPBZvsZGZWxuMGagSC2dllUam5ypvC23IbI= +github.com/deepfence/vessel v0.9.1 h1:43IuWdrRqHRW8O065qtporepudEGQWhMSqpxiI0/lSE= +github.com/deepfence/vessel v0.9.1/go.mod h1:cN+m37ETZPBZvsZGZWxuMGagSC2dllUam5ypvC23IbI= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -300,12 +296,11 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flier/gohs v1.2.1 h1:nTtedK6JDwFtLIIOQWJL0DsSmdDuX8BawbPWhUL0koI= github.com/flier/gohs v1.2.1/go.mod h1:/r9eNSHbvhLiHC7szrfFljZXhSEdmimYhcjefOXgGXM= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -402,8 +397,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -426,8 +421,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -521,13 +517,13 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -963,7 +959,6 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1016,6 +1011,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1105,7 +1101,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1183,8 +1178,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1210,9 +1205,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1226,8 +1220,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/output/output.go b/output/output.go index 4b630af..654b8fa 100644 --- a/output/output.go +++ b/output/output.go @@ -3,7 +3,6 @@ package output import ( "encoding/json" "fmt" - "io/ioutil" "os" "time" // "strings" @@ -43,11 +42,11 @@ type JsonDirSecretsOutput struct { } type JsonImageSecretsOutput struct { - Timestamp time.Time - ImageName string `json:"Image Name"` - ImageId string `json:"Image ID"` + Timestamp time.Time + ImageName string `json:"Image Name"` + ImageId string `json:"Image ID"` ContainerId string `json:"Container ID"` - Secrets []SecretFound + Secrets []SecretFound } func (imageOutput *JsonImageSecretsOutput) SetImageName(imageName string) { @@ -95,7 +94,7 @@ func printSecretsToJsonFile(secretsJson interface{}, outputFilename string) erro return err } - err = ioutil.WriteFile(outputFilename, file, os.ModePerm) + err = os.WriteFile(outputFilename, file, os.ModePerm) if err != nil { core.GetSession().Log.Error("printSecretsToJsonFile: Couldn't write json output to file: %s", err) return err diff --git a/scan/process_image.go b/scan/process_image.go index a26b9a7..690363d 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -2,12 +2,13 @@ package scan import ( "archive/tar" + "bufio" "bytes" "compress/gzip" "encoding/json" "errors" + "fmt" "io" - "io/ioutil" "log" "os" "os/exec" @@ -16,8 +17,6 @@ import ( "strings" "syscall" - "fmt" - "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/signature" @@ -102,6 +101,38 @@ func (imageScan *ImageScan) scan() ([]output.SecretFound, error) { return tempSecretsFound, nil } +func readFile(path string) ([]byte, error) { + var content string + file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + content += scanner.Text() + "\n" + } + return []byte(content), nil +} + +func scanFile(filePath, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { + contents, err := readFile(filePath) + if err != nil { + return nil, err + } + // fmt.Println(relPath, file.Filename, file.Extension, layer) + secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) + if err != nil { + return nil, err + } + return secrets, nil +} + // ScanSecretsInDir Scans a given directory recursively to find all secrets inside any file in the dir // @parameters // layer - layer ID, if we are scanning directory inside container image @@ -126,8 +157,6 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre maxFileSize := *session.Options.MaximumFileSize * 1024 var file core.MatchFile var relPath string - var contents []byte - var secrets []output.SecretFound walkErr := filepath.Walk(fullDir, func(path string, f os.FileInfo, err error) error { if err != nil { @@ -136,7 +165,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre var scanDirPath string if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir + "/" + layer) + scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) if scanDirPath == "" { scanDirPath = "/" } @@ -176,24 +205,17 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre } } - contents, err = ioutil.ReadFile(file.Path) + secrets, err := scanFile(file.Path, relPath, file.Filename, file.Extension, layer, numSecrets, matchedRuleSet) if err != nil { - session.Log.Error("scanSecretsInDir reading file: %s", err) - // return tempSecretsFound, err + session.Log.Info("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) + session.Log.Error("scanSecretsInDir: %s", err) } else { - // fmt.Println(relPath, file.Filename, file.Extension, layer) - secrets, err = signature.MatchPatternSignatures(contents, relPath, file.Filename, file.Extension, - layer, numSecrets, matchedRuleSet) - if err != nil { - session.Log.Info("relPath: %s, Filename: %s, Extension: %s, layer: %s", - relPath, file.Filename, file.Extension, layer) - session.Log.Error("scanSecretsInDir: %s", err) - // return tempSecretsFound, err - } - if *session.Options.Quiet { - output.PrintColoredSecrets(secrets, isFirstSecret) + if len(secrets) > 0 { + if *session.Options.Quiet { + output.PrintColoredSecrets(secrets, isFirstSecret) + } + tempSecretsFound = append(tempSecretsFound, secrets...) } - tempSecretsFound = append(tempSecretsFound, secrets...) } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, numSecrets) @@ -378,7 +400,7 @@ func untar(tarName string, xpath string) (err error) { relPath := strings.Split(fileName, "/") var absDirPath string if len(relPath) > 1 { - dirs := relPath[0: len(relPath) - 1] + dirs := relPath[0 : len(relPath)-1] absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) } if err := os.MkdirAll(absDirPath, 0755); err != nil { @@ -402,7 +424,7 @@ func untar(tarName string, xpath string) (err error) { // fmt.Printf("x %s\n", absFileName) n, cpErr := io.Copy(file, tr) if closeErr := file.Close(); closeErr != nil { // close file immediately - fmt.Println("clserr:"+closeErr.Error()) + fmt.Println("closeErr:" + closeErr.Error()) return err } if cpErr != nil { diff --git a/server/grpc.go b/server/grpc.go index 4566d74..84dcd2c 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -24,15 +24,15 @@ type gRPCServer struct { } func (s *gRPCServer) GetName(context.Context, *pb.Empty) (*pb.Name, error) { - return &pb.Name { Str: s.plugin_name }, nil + return &pb.Name{Str: s.plugin_name}, nil } func (s *gRPCServer) GetUID(context.Context, *pb.Empty) (*pb.Uid, error) { - return &pb.Uid { Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path) }, nil + return &pb.Uid{Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path)}, nil } func (s *gRPCServer) FindSecretInfo(_ context.Context, r *pb.FindRequest) (*pb.FindResult, error) { - if r.GetPath() != "" { + if r.GetPath() != "" { var isFirstSecret bool = true var numSecrets uint = 0 @@ -42,7 +42,7 @@ func (s *gRPCServer) FindSecretInfo(_ context.Context, r *pb.FindRequest) (*pb.F } return &pb.FindResult{ Timestamp: time.Now().String(), - Secrets: output.SecretsToSecretInfos(secrets), + Secrets: output.SecretsToSecretInfos(secrets), Input: &pb.FindResult_Path{ Path: r.GetPath(), }, @@ -55,11 +55,11 @@ func (s *gRPCServer) FindSecretInfo(_ context.Context, r *pb.FindRequest) (*pb.F return &pb.FindResult{ Timestamp: time.Now().String(), - Secrets: output.SecretsToSecretInfos(res.Secrets), + Secrets: output.SecretsToSecretInfos(res.Secrets), Input: &pb.FindResult_Image{ Image: &pb.DockerImage{ Name: r.GetImage().Name, - Id: res.ImageId, + Id: res.ImageId, }, }, }, nil @@ -71,11 +71,11 @@ func (s *gRPCServer) FindSecretInfo(_ context.Context, r *pb.FindRequest) (*pb.F return &pb.FindResult{ Timestamp: time.Now().String(), - Secrets: output.SecretsToSecretInfos(res.Secrets), + Secrets: output.SecretsToSecretInfos(res.Secrets), Input: &pb.FindResult_Container{ Container: &pb.Container{ Namespace: r.GetContainer().Namespace, - Id: res.ContainerId, + Id: res.ContainerId, }, }, }, nil diff --git a/server/http.go b/server/http.go index bc98282..ecd5144 100644 --- a/server/http.go +++ b/server/http.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httputil" "net/url" @@ -74,7 +74,7 @@ func runSecretScanStandalone(writer http.ResponseWriter, request *http.Request) } fmt.Println(string(requestDump)) - b, err := ioutil.ReadAll(request.Body) + b, err := io.ReadAll(request.Body) defer request.Body.Close() if err != nil { http.Error(writer, err.Error(), 500) diff --git a/signature/signatures.go b/signature/signatures.go index ceb5cdf..50c3c0a 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -7,12 +7,13 @@ import ( "bytes" "errors" "fmt" + "math" + "regexp" + "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/fatih/color" "github.com/flier/gohs/hyperscan" - "math" - "regexp" ) // Constants representing different parts to be matched