From 265d517ae836ef2932b6f1874efddd38795e5f99 Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sun, 1 Sep 2024 22:56:31 +0200 Subject: [PATCH 1/9] chore(tests): Add description and resource for manual test with an authenticating proxy --- src/test/manual-test-proxy-auth/Readme.md | 42 +++++++++++++++++++ .../debian.conf.override | 11 +++++ .../squid-auth-for-localnet.conf | 7 ++++ .../start-docker-squidproxy-with-auth | 2 + .../stop-docker-squidproxy-with-auth | 2 + .../user-proxy-passwd-insecure | 1 + 6 files changed, 65 insertions(+) create mode 100644 src/test/manual-test-proxy-auth/Readme.md create mode 100644 src/test/manual-test-proxy-auth/debian.conf.override create mode 100644 src/test/manual-test-proxy-auth/squid-auth-for-localnet.conf create mode 100755 src/test/manual-test-proxy-auth/start-docker-squidproxy-with-auth create mode 100755 src/test/manual-test-proxy-auth/stop-docker-squidproxy-with-auth create mode 100644 src/test/manual-test-proxy-auth/user-proxy-passwd-insecure diff --git a/src/test/manual-test-proxy-auth/Readme.md b/src/test/manual-test-proxy-auth/Readme.md new file mode 100644 index 00000000000..df8de14ad0f --- /dev/null +++ b/src/test/manual-test-proxy-auth/Readme.md @@ -0,0 +1,42 @@ +# Utilities for manual test scenario with an authenticating proxy + +# Prerequisites + +* A docker environment is available on the local machine +* A compatible unix/linux shell (at least `bash` and `zsh` should work) +* **NOTE:** The testing should also be possible on windows, but then the start command for the container would have to + be adapted to properly map the configuration files into the squid-proxy container which is left as an exercise to the + tester that wants to run it on windows. +* A working connection to internet from your docker runtime environment (squid-proxy would need to be able to reach the various internet resources ODC uses) + +# Preparation + +* Start the docker container running squid-proxy exposing the proxy to port 53128 by running the shellsceript + ```shell + ./start-docker-squid-proxy-with-auth + ``` +* Set JAVA_TOOL_OPTIONS to reflect the proxy just started + ```shell + export JAVA_TOOL_OPTIONS="-Dhttps.proxyHost=localhost -Dhttps.proxyPort=53128 -Dhttps.proxyUser=proxy -Dhttps.proxyPassword=insecure" + ``` + +# Manual test execution + +Run whichever integration of DependencyCheck to validate its proper working across an authenticating proxy from the same +shell (or make sure in a new shell that the same `JAVA_TOOL_OPTIONS` environment variable is active) + +# Cleanup + +* Stop the docker container running squid-proxy (due to start with --rm the container will be deleted upon termination) + ```shell + ./start-docker-squid-proxy-with-auth + ``` +* Unset JAVA_TOOL_OPTIONS or set it back to your regular value + ```shell + export JAVA_TOOL_OPTIONS= + ``` + or + ```shell + export JAVA_TOOL_OPTIONS=...your regular options... + ``` + diff --git a/src/test/manual-test-proxy-auth/debian.conf.override b/src/test/manual-test-proxy-auth/debian.conf.override new file mode 100644 index 00000000000..5c145d6de4b --- /dev/null +++ b/src/test/manual-test-proxy-auth/debian.conf.override @@ -0,0 +1,11 @@ +# +# Squid configuration settings for Debian +# + +# Logs are managed by logrotate on Debian +logfile_rotate 0 + +# For extra security Debian packages only allow +# localhost to use the proxy on new installs +# +# http_access allow localnet diff --git a/src/test/manual-test-proxy-auth/squid-auth-for-localnet.conf b/src/test/manual-test-proxy-auth/squid-auth-for-localnet.conf new file mode 100644 index 00000000000..f83a85cc48b --- /dev/null +++ b/src/test/manual-test-proxy-auth/squid-auth-for-localnet.conf @@ -0,0 +1,7 @@ +# configure basic auth ACL with our passwordfile +auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwords +auth_param basic realm proxy +acl authenticated proxy_auth REQUIRED +# now grant authenticaed HTTP access using the ACL above from +# localnet IP addresses (an ACL already defined in container's squid.conf) +http_access allow authenticated localnet \ No newline at end of file diff --git a/src/test/manual-test-proxy-auth/start-docker-squidproxy-with-auth b/src/test/manual-test-proxy-auth/start-docker-squidproxy-with-auth new file mode 100755 index 00000000000..863fa8cae98 --- /dev/null +++ b/src/test/manual-test-proxy-auth/start-docker-squidproxy-with-auth @@ -0,0 +1,2 @@ +docker run --rm -d --name odc-squid-testproxy -e TZ=UTC -v $(pwd)/debian.conf.override:/etc/squid/conf.d/debian.conf:ro -v $(pwd)/squid-auth-for-localnet.conf:/etc/squid/conf.d/squid-auth-for-localnet.conf:ro -v $(pwd)/user-proxy-passwd-insecure:/etc/squid/passwords:ro -p 127.0.0.1:53128:3128 -p ::1:53128:3128 ubuntu/squid:5.2-22.04_beta + diff --git a/src/test/manual-test-proxy-auth/stop-docker-squidproxy-with-auth b/src/test/manual-test-proxy-auth/stop-docker-squidproxy-with-auth new file mode 100755 index 00000000000..05710b538df --- /dev/null +++ b/src/test/manual-test-proxy-auth/stop-docker-squidproxy-with-auth @@ -0,0 +1,2 @@ +docker stop odc-squid-testproxy + diff --git a/src/test/manual-test-proxy-auth/user-proxy-passwd-insecure b/src/test/manual-test-proxy-auth/user-proxy-passwd-insecure new file mode 100644 index 00000000000..bb59e24d217 --- /dev/null +++ b/src/test/manual-test-proxy-auth/user-proxy-passwd-insecure @@ -0,0 +1 @@ +proxy:$apr1$n1cgKVJ4$.3IE1rLwMCAPnOLZMadZ5/ From 9688ad748f11b6ab4f258888e9ede051e52c3aa7 Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sat, 7 Sep 2024 13:56:41 +0200 Subject: [PATCH 2/9] feat: Replace old Downloader by an Apache HTTPClient based downloader --- .../owasp/dependencycheck/taskdefs/Check.java | 7 + .../owasp/dependencycheck/taskdefs/Purge.java | 7 + .../dependencycheck/taskdefs/Update.java | 7 + .../java/org/owasp/dependencycheck/App.java | 4 + .../analyzer/AbstractSuppressionAnalyzer.java | 5 +- .../analyzer/ArtifactoryAnalyzer.java | 3 +- .../analyzer/CentralAnalyzer.java | 3 +- .../analyzer/HintAnalyzer.java | 5 +- .../analyzer/NexusAnalyzer.java | 3 +- .../update/HostedSuppressionsDataSource.java | 3 +- .../data/update/NvdApiDataSource.java | 8 +- .../data/update/RetireJSDataSource.java | 3 +- .../data/update/nvd/api/DownloadTask.java | 3 +- .../maven/BaseDependencyCheckMojo.java | 13 +- .../dependencycheck/maven/PurgeMojo.java | 11 + pom.xml | 6 + utils/pom.xml | 4 + .../dependencycheck/utils/Downloader.java | 404 ++++++++++++------ ...plicitEncodingToStringResponseHandler.java | 47 ++ .../utils/SaveToFileResponseHandler.java | 59 +++ .../owasp/dependencycheck/utils/Settings.java | 25 ++ .../dependencycheck/utils/DownloaderIT.java | 4 +- 22 files changed, 490 insertions(+), 144 deletions(-) mode change 100755 => 100644 utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java create mode 100644 utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java create mode 100644 utils/src/main/java/org/owasp/dependencycheck/utils/SaveToFileResponseHandler.java diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java index bc99a42f832..b08e2d55c51 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java @@ -38,6 +38,8 @@ import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.ReportException; import org.owasp.dependencycheck.reporting.ReportGenerator.Format; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.utils.SeverityUtil; import org.slf4j.impl.StaticLoggerBinder; @@ -2050,6 +2052,11 @@ protected void executeWithContextClassloader() throws BuildException { dealWithReferences(); validateConfiguration(); populateSettings(); + try { + Downloader.getInstance().configure(getSettings()); + } catch (InvalidSettingException e) { + throw new BuildException(e); + } try (Engine engine = new Engine(Check.class.getClassLoader(), getSettings())) { for (Resource resource : getPath()) { final FileProvider provider = resource.as(FileProvider.class); diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java index 8b215d06f55..1599a6372b4 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java @@ -26,6 +26,8 @@ import org.apache.tools.ant.Project; import org.apache.tools.ant.Task; import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.impl.StaticLoggerBinder; @@ -179,6 +181,11 @@ private void muteNoisyLoggers() { @SuppressWarnings("squid:RedundantThrowsDeclarationCheck") protected void executeWithContextClassloader() throws BuildException { populateSettings(); + try { + Downloader.getInstance().configure(settings); + } catch (InvalidSettingException e) { + throw new BuildException(e); + } try (Engine engine = new Engine(Engine.Mode.EVIDENCE_PROCESSING, getSettings())) { engine.purge(); } finally { diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java index 54a87fcbbfa..eb2555fd685 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java @@ -22,6 +22,8 @@ import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.impl.StaticLoggerBinder; @@ -594,6 +596,11 @@ public void setHostedSuppressionsEnabled(Boolean hostedSuppressionsEnabled) { @Override protected void executeWithContextClassloader() throws BuildException { populateSettings(); + try { + Downloader.getInstance().configure(getSettings()); + } catch (InvalidSettingException e) { + throw new BuildException(e); + } try (Engine engine = new Engine(Update.class.getClassLoader(), getSettings())) { engine.doUpdates(); } catch (UpdateException ex) { diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java index b2058552b3f..a1271cd4691 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/App.java +++ b/cli/src/main/java/org/owasp/dependencycheck/App.java @@ -33,6 +33,7 @@ import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.ReportException; +import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -141,6 +142,7 @@ public int run(String[] args) { } else { try { populateSettings(cli); + Downloader.getInstance().configure(settings); } catch (InvalidSettingException ex) { LOGGER.error(ex.getMessage()); LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex); @@ -162,6 +164,7 @@ public int run(String[] args) { try { populateSettings(cli); settings.setBoolean(Settings.KEYS.AUTO_UPDATE, true); + Downloader.getInstance().configure(settings); } catch (InvalidSettingException ex) { LOGGER.error(ex.getMessage()); LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex); @@ -182,6 +185,7 @@ public int run(String[] args) { } else if (cli.isRunScan()) { try { populateSettings(cli); + Downloader.getInstance().configure(settings); } catch (InvalidSettingException ex) { LOGGER.error(ex.getMessage(), ex); LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex); diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java index 5e890dbe28f..33c78c5243e 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java @@ -345,14 +345,13 @@ private List loadSuppressionFile(final SuppressionParser parser deleteTempFile = true; file = getSettings().getTempFile("suppression", "xml"); final URL url = new URL(suppressionFilePath); - final Downloader downloader = new Downloader(getSettings()); try { - downloader.fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD); + Downloader.getInstance().fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD); } catch (DownloadFailedException ex) { LOGGER.trace("Failed download suppression file - first attempt", ex); try { Thread.sleep(500); - downloader.fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD); + Downloader.getInstance().fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD); } catch (TooManyRequestsException ex1) { throw new SuppressionParseException("Unable to download supression file `" + file + "`; received 429 - too many requests", ex1); diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java index 59f6c9adc93..2b1c349a068 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java @@ -234,8 +234,7 @@ private void processPom(Dependency dependency, MavenArtifact ma) throws IOExcept Files.delete(pomFile.toPath()); LOGGER.debug("Downloading {}", ma.getPomUrl()); //TODO add caching - final Downloader downloader = new Downloader(getSettings()); - downloader.fetchFile(new URL(ma.getPomUrl()), pomFile, + Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile, true, Settings.KEYS.ANALYZER_ARTIFACTORY_API_USERNAME, Settings.KEYS.ANALYZER_ARTIFACTORY_API_TOKEN); PomUtils.analyzePOM(dependency, pomFile); diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java index ce86e76cb14..098585c22dd 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java @@ -241,7 +241,6 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy + "this could result in undetected CPE/CVEs.", dependency.getFileName()); LOGGER.debug("Unable to delete temp file"); } - final Downloader downloader = new Downloader(getSettings()); final int maxAttempts = this.getSettings().getInt(Settings.KEYS.ANALYZER_CENTRAL_RETRY_COUNT, 3); int retryCount = 0; long sleepingTimeBetweenRetriesInMillis = BASE_RETRY_WAIT; @@ -258,7 +257,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy do { //CSOFF: NestedTryDepth try { - downloader.fetchFile(new URL(ma.getPomUrl()), pomFile); + Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile); success = true; } catch (DownloadFailedException ex) { try { diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java index 51b4374f1ed..2a4cd757dc5 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java @@ -267,13 +267,12 @@ private void loadHintRules() throws HintParseException { deleteTempFile = true; file = getSettings().getTempFile("hint", "xml"); final URL url = new URL(filePath); - final Downloader downloader = new Downloader(getSettings()); try { - downloader.fetchFile(url, file, false); + Downloader.getInstance().fetchFile(url, file, false); } catch (DownloadFailedException ex) { try { Thread.sleep(500); - downloader.fetchFile(url, file, true); + Downloader.getInstance().fetchFile(url, file, true); } catch (TooManyRequestsException ex1) { throw new HintParseException("Unable to download hint file `" + file + "`; received 429 - too many requests", ex1); } catch (ResourceNotFoundException ex1) { diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index 9506b9de2a5..ce3e5313db3 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -282,8 +282,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy LOGGER.debug("Unable to delete temp file"); } LOGGER.debug("Downloading {}", ma.getPomUrl()); - final Downloader downloader = new Downloader(getSettings()); - downloader.fetchFile(new URL(ma.getPomUrl()), pomFile); + Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile); PomUtils.analyzePOM(dependency, pomFile); } catch (DownloadFailedException ex) { LOGGER.warn("Unable to download pom.xml for {} from Nexus repository; " diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java index c00cd217afb..fd498dbbcb1 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java @@ -133,8 +133,7 @@ private void fetchHostedSuppressions(Settings settings, URL repoUrl, File repoFi if (LOGGER.isDebugEnabled()) { LOGGER.debug("Hosted Suppressions URL: {}", repoUrl.toExternalForm()); } - final Downloader downloader = new Downloader(settings); - downloader.fetchFile(repoUrl, repoFile); + Downloader.getInstance().fetchFile(repoUrl, repoFile); } catch (IOException | TooManyRequestsException | ResourceNotFoundException | WriteLockException ex) { throw new UpdateException("Failed to update the hosted suppressions file", ex); } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index f58a771422e..4111afa3759 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -30,6 +30,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.time.Duration; import java.time.ZoneId; @@ -596,11 +597,10 @@ protected final Map getUpdatesNeeded(String url, String filePatt * downloaded */ protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException { - final Downloader d = new Downloader(settings); final Properties properties = new Properties(); try { final URL u = new URI(url + "cache.properties").toURL(); - final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + final String content = Downloader.getInstance().fetchContent(u, StandardCharsets.UTF_8); properties.load(new StringReader(content)); } catch (URISyntaxException ex) { @@ -614,7 +614,7 @@ protected final Properties getRemoteCacheProperties(String url, String pattern) } try { URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL(); - String content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + String content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8); Properties props = new Properties(); props.load(new StringReader(content)); ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); @@ -625,7 +625,7 @@ protected final Properties getRemoteCacheProperties(String url, String pattern) final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear(); for (int y = startYear; y <= endYear; y++) { metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL(); - content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8); props.clear(); props.load(new StringReader(content)); lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java index e5daa65d75b..b2d04273c14 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java @@ -137,8 +137,7 @@ protected boolean shouldUpdate(File repo) throws NumberFormatException { private void initializeRetireJsRepo(Settings settings, URL repoUrl, File repoFile) throws UpdateException { try (WriteLock lock = new WriteLock(settings, true, repoFile.getName() + ".lock")) { LOGGER.debug("RetireJS Repo URL: {}", repoUrl.toExternalForm()); - final Downloader downloader = new Downloader(settings); - downloader.fetchFile(repoUrl, repoFile, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD); + Downloader.getInstance().fetchFile(repoUrl, repoFile); } catch (IOException | TooManyRequestsException | ResourceNotFoundException | WriteLockException ex) { throw new UpdateException("Failed to initialize the RetireJS repo", ex); } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java index 03b26634ecf..0c6ea41fdbf 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -84,9 +84,8 @@ public Future call() throws Exception { final URL u = new URL(url); LOGGER.info("Download Started for NVD Cache - {}", url); final long startDownload = System.currentTimeMillis(); - final Downloader d = new Downloader(settings); final File outputFile = settings.getTempFile("nvd-datafeed-", "json.gz"); - d.fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + Downloader.getInstance().fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); if (this.processorService == null) { return null; } diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java index 2e2b9b92283..1f73fd47dda 100644 --- a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java +++ b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java @@ -68,6 +68,8 @@ import org.owasp.dependencycheck.exception.ReportException; import org.owasp.dependencycheck.utils.Checksum; import org.owasp.dependencycheck.utils.Filter; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.sonatype.plexus.components.sec.dispatcher.DefaultSecDispatcher; import org.sonatype.plexus.components.sec.dispatcher.SecDispatcher; @@ -2158,8 +2160,17 @@ public String getCategoryName() { * @return a newly instantiated Engine * @throws DatabaseException thrown if there is a database exception */ - protected Engine initializeEngine() throws DatabaseException { + protected Engine initializeEngine() throws DatabaseException, MojoExecutionException, MojoFailureException { populateSettings(); + try { + Downloader.getInstance().configure(settings); + } catch (InvalidSettingException e) { + if (this.failOnError) { + throw new MojoFailureException(e.getMessage(), e); + } else { + throw new MojoExecutionException(e.getMessage(), e); + } + } return new Engine(settings); } diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java index 5444262207d..c57f17b1f86 100644 --- a/maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java +++ b/maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java @@ -25,6 +25,8 @@ import org.apache.maven.plugins.annotations.ResolutionScope; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.exception.ExceptionCollection; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; /** * Maven Plugin that purges the local copy of the NVD data. @@ -63,6 +65,15 @@ public boolean canGenerateReport() { @Override protected void runCheck() throws MojoExecutionException, MojoFailureException { populateSettings(); + try { + Downloader.getInstance().configure(getSettings()); + } catch (InvalidSettingException e) { + if (isFailOnError()) { + throw new MojoFailureException(e.getMessage(), e); + } else { + throw new MojoExecutionException(e.getMessage(), e); + } + } try (Engine engine = new Engine(Engine.Mode.EVIDENCE_PROCESSING, getSettings())) { engine.purge(); } finally { diff --git a/pom.xml b/pom.xml index bd2b6c7e6f3..5b2519c8e4e 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,7 @@ Copyright (c) 2012 - Jeremy Long 2.16.1 3.14.0 1.12.0 + 5.3.1 @@ -972,6 +973,11 @@ Copyright (c) 2012 - Jeremy Long commons-jcs3-core ${commons-jcs-core.version} + + org.apache.httpcomponents.client5 + httpclient5 + ${httpcomponents.version} + io.github.jeremylong jcs3-slf4j diff --git a/utils/pom.xml b/utils/pom.xml index d20ddf2bf4b..7103cfe1a64 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -49,6 +49,10 @@ Copyright (c) 2014 - Jeremy Long. All Rights Reserved. org.apache.commons commons-lang3 + + org.apache.httpcomponents.client5 + httpclient5 + com.fasterxml.jackson.core jackson-databind diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java old mode 100755 new mode 100644 index 90c0ab3fc76..8088f9c4c93 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -13,203 +13,369 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + * Copyright (c) 2024 Hans Aikema. All Rights Reserved. */ package org.owasp.dependencycheck.utils; -import java.io.ByteArrayOutputStream; +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.Credentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.Method; +import org.apache.hc.core5.http.message.BasicClassicHttpRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import static java.lang.String.format; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; -import java.nio.charset.StandardCharsets; -import java.util.zip.GZIPInputStream; +public final class Downloader { -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + /** + * The builder to use for a HTTP Client that uses the configured proxy-settings + */ + private final HttpClientBuilder httpClientBuilder; + + /** + * The builder to use for a HTTP Client that explicitly opts out of proxy-usage + */ + private final HttpClientBuilder httpClientBuilderExplicitNoproxy; -/** - * A utility to download files from the Internet. - * - * @author Jeremy Long - * @version $Id: $Id - */ -public final class Downloader { /** - * UTF-8 character set name. + * The settings */ - private static final String UTF8 = StandardCharsets.UTF_8.name(); + private Settings settings; + /** * The Logger for use throughout the class. */ private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class); + /** - * The configured settings. + * The singleton instance of the downloader */ - private final Settings settings; + private static final Downloader INSTANCE = new Downloader(); + + private Downloader() { + // Singleton class + final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + //TODO: ensure proper closure and eviction policy + httpClientBuilder = HttpClientBuilder.create().useSystemProperties().setConnectionManager(connectionManager).setConnectionManagerShared(true); + httpClientBuilderExplicitNoproxy = HttpClientBuilder.create().setConnectionManager(connectionManager).setConnectionManagerShared(true); + } /** - * Constructs a new Downloader object. + * The singleton instance for downloading file resources. * - * @param settings the configured settings + * @return The singleton instance managing download credentials and proxy configuration */ - public Downloader(Settings settings) { - this.settings = settings; + public static Downloader getInstance() { + return INSTANCE; } /** - * Retrieves a file from a given URL and saves it to the outputPath. + * Initialize the Downloader from the settings. + * Extracts the configured proxy- and credential information from the settings and system properties and + * caches those for future use by the Downloader. * - * @param url the URL of the file to download - * @param outputPath the path to the save the file to - * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown - * if there is an error downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received + * @param settings The settings to configure from + * @throws InvalidSettingException When improper configurations are found. */ - public void fetchFile(URL url, File outputPath) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - fetchFile(url, outputPath, true, null, null); + public void configure(Settings settings) throws InvalidSettingException { + this.settings = settings; + final CredentialsProviderBuilder credentialsProviderBuilder = CredentialsProviderBuilder.create(); + if (settings.getString(Settings.KEYS.PROXY_SERVER) != null) { + // Legacy proxy configuration present + // So don't rely on the system properties for proxy; use the legacy settings configuration + final String proxyHost = settings.getString(Settings.KEYS.PROXY_SERVER); + final int proxyPort = settings.getInt(Settings.KEYS.PROXY_PORT, -1); + httpClientBuilder.setProxy(new HttpHost(proxyHost, proxyPort)); + if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) { + final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME); + final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray(); + credentialsProviderBuilder.add(new AuthScope(null, proxyHost, proxyPort, null, null), proxyuser, proxypass); + } + } + tryAddRetireJSCredentials(settings, credentialsProviderBuilder); + tryAddHostedSuppressionCredentials(settings, credentialsProviderBuilder); + tryAddKEVCredentials(settings, credentialsProviderBuilder); + tryAddNexusAnalyzerCredentials(settings, credentialsProviderBuilder); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProviderBuilder.build()); + httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProviderBuilder.build()); + } + + private void tryAddRetireJSCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder) throws InvalidSettingException { + if (settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD) != null) { + validateAndAddUsernamePasswordCredentials(settings, credentialsProviderBuilder, + Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, + Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, + Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD, + "RetireJS repo.js"); + } + } + + private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder) throws InvalidSettingException { + if (settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD) != null) { + validateAndAddUsernamePasswordCredentials(settings, credentialsProviderBuilder, + Settings.KEYS.HOSTED_SUPPRESSIONS_USER, + Settings.KEYS.HOSTED_SUPPRESSIONS_URL, + Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD, + "Hosted suppressions"); + } + } + + private void tryAddKEVCredentials(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + if (settings.getString(Settings.KEYS.KEV_PASSWORD) != null) { + validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + Settings.KEYS.KEV_USER, + Settings.KEYS.KEV_URL, + Settings.KEYS.KEV_PASSWORD, + "Known Exploited Vulnerabilities"); + } + } + + private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + if (settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD) != null) { + validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + Settings.KEYS.ANALYZER_NEXUS_URL, + Settings.KEYS.ANALYZER_NEXUS_USER, + Settings.KEYS.ANALYZER_NEXUS_PASSWORD, + "Nexus Analyzer"); + } + } + + private void tryAddNVDApiDatafeed(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + if (settings.getString(Settings.KEYS.NVD_API_DATAFEED_PASSWORD) != null) { + validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + Settings.KEYS.NVD_API_DATAFEED_URL, + Settings.KEYS.NVD_API_DATAFEED_USER, + Settings.KEYS.NVD_API_DATAFEED_PASSWORD, + "NVD API Datafeed"); + } + } + + private void validateAndAddUsernamePasswordCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder, String userKey, String urlKey, String passwordKey, String messageScope) throws InvalidSettingException { + final String theUser = settings.getString(userKey); + final String theURL = settings.getString(urlKey); + final char[] thePass = settings.getString(passwordKey, "").toCharArray(); + if (theUser == null || theURL == null || thePass.length == 0) { + throw new InvalidSettingException(messageScope + " URL and username are required when setting " + messageScope + " password"); + } + try { + final URL parsedURL = new URL(theURL); + addCredentials(credentialsProviderBuilder, messageScope, parsedURL, theUser, thePass); + } catch (MalformedURLException e) { + throw new InvalidSettingException(messageScope + " URL must be a valid URL", e); + } + } + + private static void addCredentials(CredentialsProviderBuilder credentialsProviderBuilder, String messageScope, URL parsedURL, String theUser, char[] thePass) throws InvalidSettingException { + final String theProtocol = parsedURL.getProtocol(); + if ("file".equals(theProtocol)) { + LOGGER.warn("Credentials are not supported for file-protocol, double-check your configuration options for {}.", messageScope); + return; + } else if ("http".equals(theProtocol)) { + LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. Consider migrating to https to guard the credentials.", messageScope); + } else if (!"https".equals(theProtocol)) { + throw new InvalidSettingException("Unsupported protocol in the " + messageScope + " URL; only file://, http:// and https:// are supported"); + } + final String theHost = parsedURL.getHost(); + final int thePort = parsedURL.getPort(); + final Credentials creds = new UsernamePasswordCredentials(theUser, thePass); + final AuthScope scope = new AuthScope(theProtocol, theHost, thePort, null, null); + credentialsProviderBuilder.add(scope, creds); } /** * Retrieves a file from a given URL and saves it to the outputPath. * - * @param url the URL of the file to download + * @param url the URL of the file to download * @param outputPath the path to the save the file to - * @param userKey the settings key for the username to be used - * @param passwordKey the settings key for the password to be used * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown - * if there is an error downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received + * if there is an error downloading the file + * @throws TooManyRequestsException thrown when a 429 is received + * @throws ResourceNotFoundException thrown when a 404 is received */ - public void fetchFile(URL url, File outputPath, String userKey, String passwordKey) - throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - fetchFile(url, outputPath, true, userKey, passwordKey); + public void fetchFile(URL url, File outputPath) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { + fetchFile(url, outputPath, true); } /** * Retrieves a file from a given URL and saves it to the outputPath. * - * @param url the URL of the file to download + * @param url the URL of the file to download * @param outputPath the path to the save the file to - * @param useProxy whether to use the configured proxy when downloading - * files + * @param useProxy whether to use the configured proxy when downloading + * files * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown - * if there is an error downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received + * if there is an error downloading the file + * @throws TooManyRequestsException thrown when a 429 is received + * @throws ResourceNotFoundException thrown when a 404 is received */ public void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - fetchFile(url, outputPath, useProxy, null, null); + try { + if ("file".equals(url.getProtocol())) { + final Path p = Paths.get(url.toURI()); + Files.copy(p, outputPath.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + final BasicClassicHttpRequest req; + req = new BasicClassicHttpRequest(Method.GET, url.toURI()); + try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { + final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath); + hc.execute(req, responseHandler); + } + } + } catch (HttpResponseException hre) { + final String messageFormat = "%s - Server status: %d - Server reason: %s"; + switch (hre.getStatusCode()) { + case 404: + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + case 429: + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + default: + throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + } + } catch (RuntimeException | URISyntaxException | IOException ex) { + final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); + throw new DownloadFailedException(msg, ex); + } } /** - * Retrieves a file from a given URL and saves it to the outputPath. + * Retrieves a file from a given URL using an ad-hoc created CredentialsProvider if needed + * and saves it to the outputPath. * - * @param url the URL of the file to download - * @param outputPath the path to the save the file to - * @param useProxy whether to use the configured proxy when downloading - * files - * @param userKey the settings key for the username to be used + * @param url the URL of the file to download + * @param outputPath the path to the save the file to + * @param useProxy whether to use the configured proxy when downloading + * files + * @param userKey the settings key for the username to be used * @param passwordKey the settings key for the password to be used * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown - * if there is an error downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received + * if there is an error downloading the file + * @throws TooManyRequestsException thrown when a 429 is received + * @throws ResourceNotFoundException thrown when a 404 is received + * @implNote This method should only be used in cases where the URL cannot be + * determined beforehand from settings, so that ad-hoc Credentials needs to + * be constructed for the target URL when the user/password keys point to configured credentials. + * The method delegates to {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file. */ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) { - in = conn.fetch(url); - try (ReadableByteChannel sourceChannel = Channels.newChannel(in); - FileChannel destChannel = new FileOutputStream(outputPath).getChannel()) { - ByteBuffer buffer = ByteBuffer.allocateDirect(8192); - while (sourceChannel.read(buffer) != -1) { - // cast is a workaround, see https://github.com/plasma-umass/doppio/issues/497#issuecomment-334740243 - ((Buffer)buffer).flip(); - destChannel.write(buffer); - buffer.compact(); - } + if ("file".equals(url.getProtocol()) + || userKey == null || settings.getString(userKey) == null + || passwordKey == null || settings.getString(passwordKey) == null + ) { + // no credentials configured, so use the default fetchFile + fetchFile(url, outputPath, useProxy); + return; + } + final String theProtocol = url.getProtocol(); + if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { + throw new DownloadFailedException("Unsupported protocol in the URL; only file://, http:// and https:// are supported"); + } + try { + final HttpClientContext context = HttpClientContext.create(); + final CredentialsProviderBuilder credBuilder = new CredentialsProviderBuilder(); + addCredentials(credBuilder, url.toExternalForm(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); + context.setCredentialsProvider(credBuilder.build()); + try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { + final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); + final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath); + hc.execute(req, context, responseHandler); } - } catch (IOException ex) { - final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", - url.toString(), outputPath.getAbsolutePath(), ex.getMessage()); - throw new DownloadFailedException(msg, ex); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException ex) { - LOGGER.trace("Ignorable error", ex); - } + } catch (HttpResponseException hre) { + final String messageFormat = "%s - Server status: %d - Server reason: %s"; + switch (hre.getStatusCode()) { + case 404: + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + case 429: + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + default: + throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); } + } catch (RuntimeException | URISyntaxException | IOException ex) { + final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); + throw new DownloadFailedException(msg, ex); } } /** * Retrieves a file from a given URL and returns the contents. * - * @param url the URL of the file to download - * @param useProxy whether to use the configured proxy when downloading - * files + * @param url the URL of the file to download + * @param charset The characterset to use to interpret the binary content of the file * @return the content of the file - * @throws DownloadFailedException is thrown if there is an error - * downloading the file - * @throws TooManyRequestsException thrown when a 429 is received + * @throws DownloadFailedException is thrown if there is an error + * downloading the file + * @throws TooManyRequestsException thrown when a 429 is received * @throws ResourceNotFoundException thrown when a 404 is received */ - public String fetchContent(URL url, boolean useProxy) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - return fetchContent(url, useProxy, null, null); + public String fetchContent(URL url, Charset charset) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { + return fetchContent(url, true, charset); } /** * Retrieves a file from a given URL and returns the contents. * - * @param url the URL of the file to download - * @param useProxy whether to use the configured proxy when downloading - * files - * @param userKey the settings key for the username to be used - * @param passwordKey the settings key for the password to be used + * @param url the URL of the file to download + * @param useProxy whether to use the configured proxy when downloading + * files + * @param charset The characterset to use to interpret the binary content of the file * @return the content of the file - * @throws DownloadFailedException is thrown if there is an error - * downloading the file - * @throws TooManyRequestsException thrown when a 429 is received + * @throws DownloadFailedException is thrown if there is an error + * downloading the file + * @throws TooManyRequestsException thrown when a 429 is received * @throws ResourceNotFoundException thrown when a 404 is received */ - public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey) + public String fetchContent(URL url, boolean useProxy, Charset charset) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); - ByteArrayOutputStream out = new ByteArrayOutputStream()) { - in = conn.fetch(url); - IOUtils.copy(in, out); - return out.toString(UTF8); - } catch (IOException ex) { - final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage()); - throw new DownloadFailedException(msg, ex); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException ex) { - LOGGER.trace("Ignorable error", ex); + String result = ""; + try { + if ("file".equals(url.getProtocol())) { + final Path p = Paths.get(url.toURI()); + result = new String(Files.readAllBytes(p), charset); + } else { + final BasicClassicHttpRequest req; + req = new BasicClassicHttpRequest(Method.GET, url.toURI()); + try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { + final ExplicitEncodingToStringResponseHandler responseHandler = new ExplicitEncodingToStringResponseHandler(charset); + result = hc.execute(req, responseHandler); } } + } catch (HttpResponseException hre) { + final String messageFormat = "%s - Server status: %d - Server reason: %s"; + switch (hre.getStatusCode()) { + case 404: + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + case 429: + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + default: + throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + } + } catch (RuntimeException | URISyntaxException | IOException ex) { + final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage()); + throw new DownloadFailedException(msg, ex); } + return result; } + } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java b/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java new file mode 100644 index 00000000000..f102628d893 --- /dev/null +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java @@ -0,0 +1,47 @@ +/* + * This file is part of dependency-check-utils. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2024 Hans Aikema. All Rights Reserved. + */ +package org.owasp.dependencycheck.utils; + +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; + +import java.io.IOException; +import java.nio.charset.Charset; + +public class ExplicitEncodingToStringResponseHandler extends AbstractHttpClientResponseHandler { + + /** + * The explicit Charset used for interpreting the bytes of the HTTP response entity. + */ + private final Charset charset; + + /** + * Constructs a repsonse handler to transfor the binary contents received using the given Charset. + * + * @param charset The Charset to be used to transform a downloaded file into a String. + */ + public ExplicitEncodingToStringResponseHandler(Charset charset) { + this.charset = charset; + } + + @Override + public String handleEntity(HttpEntity entity) throws IOException { + return new String(EntityUtils.toByteArray(entity), charset); + } +} diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/SaveToFileResponseHandler.java b/utils/src/main/java/org/owasp/dependencycheck/utils/SaveToFileResponseHandler.java new file mode 100644 index 00000000000..a404b1c4ff5 --- /dev/null +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/SaveToFileResponseHandler.java @@ -0,0 +1,59 @@ +/* + * This file is part of dependency-check-utils. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2024 Hans Aikema. All Rights Reserved. + */ +package org.owasp.dependencycheck.utils; + +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; + +class SaveToFileResponseHandler extends AbstractHttpClientResponseHandler { + + /** + * The output path where the response content should be stored as a file + */ + private final File outputPath; + + SaveToFileResponseHandler(File outputPath) { + this.outputPath = outputPath; + } + + @Override + public Void handleEntity(HttpEntity responseEntity) throws IOException { + try (InputStream in = responseEntity.getContent(); + ReadableByteChannel sourceChannel = Channels.newChannel(in); + FileOutputStream fos = new FileOutputStream(outputPath); + FileChannel destChannel = fos.getChannel()) { + final ByteBuffer buffer = ByteBuffer.allocateDirect(8192); + while (sourceChannel.read(buffer) != -1) { + buffer.flip(); + destChannel.write(buffer); + buffer.compact(); + } + } + return null; + } + +} diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index d4a719de12c..7e262147448 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -228,6 +228,19 @@ public static final class KEYS { * Vulnerabilities.. */ public static final String KEV_URL = "kev.url"; + + /** + * The properties key for the hosted suppressions username. + * For use when hosted suppressions are mirrored locally on a site requiring authentication + */ + public static final String KEV_USER = "kev.user"; + + /** + * The properties key for the hosted suppressions password. + * For use when hosted suppressions are mirrored locally on a site requiring authentication + */ + public static final String KEV_PASSWORD = "kev.password"; + /** * The properties key to control the skipping of the check for Known * Exploited Vulnerabilities updates. @@ -302,6 +315,18 @@ public static final class KEYS { */ public static final String HOSTED_SUPPRESSIONS_URL = "hosted.suppressions.url"; + /** + * The properties key for the hosted suppressions username. + * For use when hosted suppressions are mirrored locally on a site requiring authentication + */ + public static final String HOSTED_SUPPRESSIONS_USER = "hosted.suppressions.user"; + + /** + * The properties key for the hosted suppressions password. + * For use when hosted suppressions are mirrored locally on a site requiring authentication + */ + public static final String HOSTED_SUPPRESSIONS_PASSWORD = "hosted.suppressions.password"; + /** * The properties key for defining whether the hosted suppressions file * will be updated regardless of the autoupdate settings. diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java index 7c905809cbb..6d008a01020 100644 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java +++ b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java @@ -49,8 +49,8 @@ public void testFetchFile() throws Exception { final String str = getSettings().getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL, "https://jeremylong.github.io/DependencyCheck/current.txt"); URL url = new URL(str); File outputPath = new File("target/current.txt"); - Downloader downloader = new Downloader(getSettings()); - downloader.fetchFile(url, outputPath); + Downloader.getInstance().configure(getSettings()); + Downloader.getInstance().fetchFile(url, outputPath); assertTrue(outputPath.isFile()); } From f61418960418aa96c0eaf8d07814a059f7bc448b Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sat, 7 Sep 2024 18:18:43 +0200 Subject: [PATCH 3/9] fix: Fixup broken proxy authentication in first attempt; extend to include KEV downloads --- .../data/update/KnownExploitedDataSource.java | 34 +++-- .../dependencycheck/utils/Downloader.java | 140 ++++++++++++++---- 2 files changed, 137 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java index f56b7dcb040..1a3bc87014c 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java @@ -21,6 +21,10 @@ import java.io.InputStream; import java.net.URL; import java.sql.SQLException; + +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.knownexploited.json.KnownExploitedVulnerabilitiesSchema; import org.owasp.dependencycheck.data.nvdcve.CveDB; @@ -29,7 +33,7 @@ import org.owasp.dependencycheck.data.update.cisa.KnownExploitedVulnerabilityParser; import org.owasp.dependencycheck.data.update.exception.CorruptedDatastreamException; import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.HttpResourceConnection; +import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.utils.TooManyRequestsException; @@ -74,11 +78,24 @@ public boolean update(Engine engine) throws UpdateException { try { final URL url = new URL(settings.getString(Settings.KEYS.KEV_URL, DEFAULT_URL)); LOGGER.info("Updating CISA Known Exploited Vulnerability list: " + url.toString()); - //TODO - add all the proxy config, likely use the same as configured for NVD - final HttpResourceConnection conn = new HttpResourceConnection(settings); - try (InputStream in = conn.fetch(url)) { - final KnownExploitedVulnerabilityParser parser = new KnownExploitedVulnerabilityParser(); - final KnownExploitedVulnerabilitiesSchema data = parser.parse(in); + + final HttpClientResponseHandler kevParsingResponseHandler + = new AbstractHttpClientResponseHandler<>() { + @Override + public KnownExploitedVulnerabilitiesSchema handleEntity(HttpEntity entity) throws IOException { + try (InputStream in = entity.getContent()) { + final KnownExploitedVulnerabilityParser parser = new KnownExploitedVulnerabilityParser(); + final KnownExploitedVulnerabilitiesSchema data = parser.parse(in); + return data; + } catch (CorruptedDatastreamException | UpdateException e) { + throw new IOException("Error processing response", e); + } + } + }; + +// final HttpResourceConnection conn = new HttpResourceConnection(settings); +// try { + final KnownExploitedVulnerabilitiesSchema data = Downloader.getInstance().fetchAndHandleContent(url, kevParsingResponseHandler); final String currentVersion = dbProperties.getProperty(DatabaseProperties.KEV_VERSION, ""); if (!currentVersion.equals(data.getCatalogVersion())) { cveDB.updateKnownExploitedVulnerabilities(data.getVulnerabilities()); @@ -86,9 +103,8 @@ public boolean update(Engine engine) throws UpdateException { //all dates in the db are now stored in seconds as opposed to previously milliseconds. dbProperties.save(DatabaseProperties.KEV_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000)); return true; - } - } catch (TooManyRequestsException | ResourceNotFoundException | IOException - | CorruptedDatastreamException | DatabaseException | SQLException ex) { +// } + } catch (TooManyRequestsException | ResourceNotFoundException | IOException | DatabaseException | SQLException ex) { throw new UpdateException(ex); } } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 8088f9c4c93..a891311d0a3 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -20,21 +20,35 @@ import org.apache.hc.client5.http.HttpResponseException; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.Credentials; +import org.apache.hc.client5.http.auth.CredentialsStore; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.auth.SystemDefaultCredentialsProvider; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.Method; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; +import org.apache.hc.core5.http.io.entity.BasicHttpEntity; import org.apache.hc.core5.http.message.BasicClassicHttpRequest; +import org.apache.hc.core5.http.message.BasicClassicHttpResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.SocketAddress; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.Charset; @@ -42,6 +56,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.Collections; +import java.util.List; import static java.lang.String.format; @@ -77,8 +93,25 @@ private Downloader() { // Singleton class final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); //TODO: ensure proper closure and eviction policy - httpClientBuilder = HttpClientBuilder.create().useSystemProperties().setConnectionManager(connectionManager).setConnectionManagerShared(true); - httpClientBuilderExplicitNoproxy = HttpClientBuilder.create().setConnectionManager(connectionManager).setConnectionManagerShared(true); + httpClientBuilder = HttpClientBuilder.create() + .useSystemProperties() + .setConnectionManager(connectionManager) + .setConnectionManagerShared(true); + httpClientBuilderExplicitNoproxy = HttpClientBuilder.create() + .useSystemProperties() + .setConnectionManager(connectionManager) + .setConnectionManagerShared(true) + .setProxySelector(new ProxySelector() { + @Override + public List select(URI uri) { + return Collections.singletonList(Proxy.NO_PROXY); + } + + @Override + public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { + + } + }); } /** @@ -100,7 +133,8 @@ public static Downloader getInstance() { */ public void configure(Settings settings) throws InvalidSettingException { this.settings = settings; - final CredentialsProviderBuilder credentialsProviderBuilder = CredentialsProviderBuilder.create(); + + final SystemDefaultCredentialsProvider credentialsProvider = new SystemDefaultCredentialsProvider(); if (settings.getString(Settings.KEYS.PROXY_SERVER) != null) { // Legacy proxy configuration present // So don't rely on the system properties for proxy; use the legacy settings configuration @@ -110,20 +144,20 @@ public void configure(Settings settings) throws InvalidSettingException { if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) { final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME); final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray(); - credentialsProviderBuilder.add(new AuthScope(null, proxyHost, proxyPort, null, null), proxyuser, proxypass); + credentialsProvider.setCredentials(new AuthScope(null, proxyHost, proxyPort, null, null), new UsernamePasswordCredentials(proxyuser, proxypass)); } } - tryAddRetireJSCredentials(settings, credentialsProviderBuilder); - tryAddHostedSuppressionCredentials(settings, credentialsProviderBuilder); - tryAddKEVCredentials(settings, credentialsProviderBuilder); - tryAddNexusAnalyzerCredentials(settings, credentialsProviderBuilder); - httpClientBuilder.setDefaultCredentialsProvider(credentialsProviderBuilder.build()); - httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProviderBuilder.build()); + tryAddRetireJSCredentials(settings, credentialsProvider); + tryAddHostedSuppressionCredentials(settings, credentialsProvider); + tryAddKEVCredentials(settings, credentialsProvider); + tryAddNexusAnalyzerCredentials(settings, credentialsProvider); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProvider); } - private void tryAddRetireJSCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder) throws InvalidSettingException { + private void tryAddRetireJSCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsProviderBuilder, + validateAndAddUsernamePasswordCredentials(settings, credentialsStore, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD, @@ -131,9 +165,9 @@ private void tryAddRetireJSCredentials(Settings settings, CredentialsProviderBui } } - private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder) throws InvalidSettingException { + private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsProviderBuilder, + validateAndAddUsernamePasswordCredentials(settings, credentialsStore, Settings.KEYS.HOSTED_SUPPRESSIONS_USER, Settings.KEYS.HOSTED_SUPPRESSIONS_URL, Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD, @@ -141,9 +175,9 @@ private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsPr } } - private void tryAddKEVCredentials(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + private void tryAddKEVCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.KEV_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + validateAndAddUsernamePasswordCredentials(settings, credentialsStore, Settings.KEYS.KEV_USER, Settings.KEYS.KEV_URL, Settings.KEYS.KEV_PASSWORD, @@ -151,9 +185,9 @@ private void tryAddKEVCredentials(Settings settings, CredentialsProviderBuilder } } - private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + validateAndAddUsernamePasswordCredentials(settings, credentialsStore, Settings.KEYS.ANALYZER_NEXUS_URL, Settings.KEYS.ANALYZER_NEXUS_USER, Settings.KEYS.ANALYZER_NEXUS_PASSWORD, @@ -161,9 +195,9 @@ private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsProvid } } - private void tryAddNVDApiDatafeed(Settings settings, CredentialsProviderBuilder credProviderBuilder) throws InvalidSettingException { + private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.NVD_API_DATAFEED_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credProviderBuilder, + validateAndAddUsernamePasswordCredentials(settings, credentialsStore, Settings.KEYS.NVD_API_DATAFEED_URL, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD, @@ -171,7 +205,7 @@ private void tryAddNVDApiDatafeed(Settings settings, CredentialsProviderBuilder } } - private void validateAndAddUsernamePasswordCredentials(Settings settings, CredentialsProviderBuilder credentialsProviderBuilder, String userKey, String urlKey, String passwordKey, String messageScope) throws InvalidSettingException { + private void validateAndAddUsernamePasswordCredentials(Settings settings, CredentialsStore credentialsStore, String userKey, String urlKey, String passwordKey, String messageScope) throws InvalidSettingException { final String theUser = settings.getString(userKey); final String theURL = settings.getString(urlKey); final char[] thePass = settings.getString(passwordKey, "").toCharArray(); @@ -180,13 +214,13 @@ private void validateAndAddUsernamePasswordCredentials(Settings settings, Creden } try { final URL parsedURL = new URL(theURL); - addCredentials(credentialsProviderBuilder, messageScope, parsedURL, theUser, thePass); + addCredentials(credentialsStore, messageScope, parsedURL, theUser, thePass); } catch (MalformedURLException e) { throw new InvalidSettingException(messageScope + " URL must be a valid URL", e); } } - private static void addCredentials(CredentialsProviderBuilder credentialsProviderBuilder, String messageScope, URL parsedURL, String theUser, char[] thePass) throws InvalidSettingException { + private static void addCredentials(CredentialsStore credentialsStore, String messageScope, URL parsedURL, String theUser, char[] thePass) throws InvalidSettingException { final String theProtocol = parsedURL.getProtocol(); if ("file".equals(theProtocol)) { LOGGER.warn("Credentials are not supported for file-protocol, double-check your configuration options for {}.", messageScope); @@ -200,7 +234,7 @@ private static void addCredentials(CredentialsProviderBuilder credentialsProvide final int thePort = parsedURL.getPort(); final Credentials creds = new UsernamePasswordCredentials(theUser, thePass); final AuthScope scope = new AuthScope(theProtocol, theHost, thePort, null, null); - credentialsProviderBuilder.add(scope, creds); + credentialsStore.setCredentials(scope, creds); } /** @@ -294,9 +328,9 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey } try { final HttpClientContext context = HttpClientContext.create(); - final CredentialsProviderBuilder credBuilder = new CredentialsProviderBuilder(); - addCredentials(credBuilder, url.toExternalForm(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); - context.setCredentialsProvider(credBuilder.build()); + final BasicCredentialsProvider localCredentials = new BasicCredentialsProvider(); + addCredentials(localCredentials, url.toExternalForm(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); + context.setCredentialsProvider(localCredentials); try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath); @@ -378,4 +412,54 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) return result; } + /** + * Download a resource from the given URL and have its content handled by the given ResponseHandler. + * + * @param url The url of the resource + * @param responseHandler The responsehandler that handles the response's inputstream + * @return The response handler result + * @param The return-type for the responseHandler + * @throws IOException on I/O Exceptions + * @throws TooManyRequestsException When HTTP status 429 is encountered + * @throws ResourceNotFoundException When HTTP status 404 is encountered + */ + public T fetchAndHandleContent(URL url, HttpClientResponseHandler responseHandler) throws IOException, TooManyRequestsException, ResourceNotFoundException { + try { + T data = null; + if ("file".equals(url.getProtocol())) { + final Path p = Paths.get(url.toURI()); + try (InputStream is = Files.newInputStream(p)) { + final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON); + final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200); + dummyResponse.setEntity(dummyEntity); + data = responseHandler.handleResponse(dummyResponse); + } catch (HttpException e) { + throw new IllegalStateException("HttpException encountered without HTTP traffic", e); + } + } else { + final String theProtocol = url.getProtocol(); + if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { + throw new DownloadFailedException("Unsupported protocol in the URL; only file://, http:// and https:// are supported"); + } + try (CloseableHttpClient hc = httpClientBuilder.build()) { + final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); + data = hc.execute(req, responseHandler); + } + } + return data; + } catch (HttpResponseException hre) { + final String messageFormat = "%s - Server status: %d - Server reason: %s"; + switch (hre.getStatusCode()) { + case 404: + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + case 429: + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + default: + throw new IOException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + } + } catch (RuntimeException | URISyntaxException ex) { + final String msg = format("Download failed, unable to retrieve and parse '%s'; %s", url, ex.getMessage()); + throw new IOException(msg, ex); + } + } } From 82839dd2a2f6ea58926ad255e1f419e1065d9f22 Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sat, 7 Sep 2024 18:53:27 +0200 Subject: [PATCH 4/9] chore: cleanup of stale code; migrating a still valuable test-scenario to the DownloaderIT --- .../utils/HttpResourceConnection.java | 348 ------------------ .../dependencycheck/utils/DownloaderIT.java | 30 ++ .../utils/HttpResourceConnectionTest.java | 78 ---- 3 files changed, 30 insertions(+), 426 deletions(-) delete mode 100644 utils/src/main/java/org/owasp/dependencycheck/utils/HttpResourceConnection.java delete mode 100644 utils/src/test/java/org/owasp/dependencycheck/utils/HttpResourceConnectionTest.java diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/HttpResourceConnection.java b/utils/src/main/java/org/owasp/dependencycheck/utils/HttpResourceConnection.java deleted file mode 100644 index 379032afbc9..00000000000 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/HttpResourceConnection.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * This file is part of dependency-check-utils. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.utils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import static java.lang.String.format; - -import java.net.HttpURLConnection; -import java.net.URISyntaxException; -import java.net.URL; -import java.security.InvalidAlgorithmParameterException; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; - -/** - * A utility to download files from the Internet. - * - * @author Jeremy Long - * @version $Id: $Id - */ -public class HttpResourceConnection implements AutoCloseable { - - /** - * The logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(HttpResourceConnection.class); - /** - * The maximum number of redirects that will be followed when attempting to - * download a file. - */ - private static final int MAX_REDIRECT_ATTEMPTS = 5; - /** - * The default HTTP request method for query timestamp - */ - private static final String HEAD = "HEAD"; - - /** - * The HTTP request method which can be used by query timestamp - */ - private static final String GET = "GET"; - /** - * The configured settings. - */ - private final Settings settings; - /** - * The URL conn factory. - */ - private final URLConnectionFactory connFactory; - /** - * The current conn. - */ - private HttpURLConnection connection = null; - /** - * Whether or not the conn will use the defined proxy. - */ - private final boolean usesProxy; - - /** - * The settings key for the username to be used. - */ - private String userKey = null; - /** - * The settings key for the password to be used. - */ - private String passwordKey = null; - - /** - * Constructs a new HttpResourceConnection object. - * - * @param settings the configured settings - */ - public HttpResourceConnection(Settings settings) { - this(settings, true); - } - - /** - * Constructs a new HttpResourceConnection object. - * - * @param settings the configured settings - * @param usesProxy control whether this conn will use the defined proxy. - */ - public HttpResourceConnection(Settings settings, boolean usesProxy) { - this.settings = settings; - this.connFactory = new URLConnectionFactory(settings); - this.usesProxy = usesProxy; - } - - /** - * Constructs a new HttpResourceConnection object. - * - * @param settings the configured settings - * @param usesProxy control whether this conn will use the defined proxy - * @param userKey the settings key for the username to be used - * @param passwordKey the settings key for the password to be used - */ - public HttpResourceConnection(Settings settings, boolean usesProxy, String userKey, String passwordKey) { - this.settings = settings; - this.connFactory = new URLConnectionFactory(settings); - this.usesProxy = usesProxy; - this.userKey = userKey; - this.passwordKey = passwordKey; - } - - /** - * Retrieves the resource identified by the given URL and returns the - * InputStream. - * - * @param url the URL of the resource to download - * @return the stream to read the retrieved content from - * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown - * if there is an error downloading the resource - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received - */ - public InputStream fetch(URL url) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - if ("file".equalsIgnoreCase(url.getProtocol())) { - final File file; - try { - file = new File(url.toURI()); - } catch (URISyntaxException ex) { - final String msg = format("Download failed, unable to locate '%s'", url); - throw new DownloadFailedException(msg); - } - if (file.exists()) { - try { - return new FileInputStream(file); - } catch (IOException ex) { - final String msg = format("Download failed, unable to rerieve '%s'", url); - throw new DownloadFailedException(msg, ex); - } - } else { - final String msg = format("Download failed, file ('%s') does not exist", url); - throw new DownloadFailedException(msg); - } - } else { - if (connection != null) { - LOGGER.warn("HTTP URL Connection was not properly closed"); - connection.disconnect(); - connection = null; - } - connection = obtainConnection(url); - - final String encoding = connection.getContentEncoding(); - try { - if ("gzip".equalsIgnoreCase(encoding)) { - return new GZIPInputStream(connection.getInputStream()); - } else if ("deflate".equalsIgnoreCase(encoding)) { - return new InflaterInputStream(connection.getInputStream()); - } else { - return connection.getInputStream(); - } - } catch (IOException ex) { - checkForCommonExceptionTypes(ex); - final String msg = format("Error retrieving '%s'%nConnection Timeout: %d%nEncoding: %s%n", - url, connection.getConnectTimeout(), encoding); - throw new DownloadFailedException(msg, ex); - } catch (Exception ex) { - final String msg = format("Unexpected exception retrieving '%s'%nConnection Timeout: %d%nEncoding: %s%n", - url, connection.getConnectTimeout(), encoding); - throw new DownloadFailedException(msg, ex); - } - } - } - - /** - * Obtains the HTTP URL Connection. - * - * @param url the URL - * @return the HTTP URL Connection - * @throws DownloadFailedException thrown if there is an error creating the - * HTTP URL Connection - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received - */ - private HttpURLConnection obtainConnection(URL url) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - HttpURLConnection conn = null; - try { - LOGGER.debug("Attempting retrieval of {}", url.toString()); - conn = connFactory.createHttpURLConnection(url, this.usesProxy); - if (userKey != null && passwordKey != null) { - connFactory.addBasicAuthentication(conn, userKey, passwordKey); - } - conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); - conn.connect(); - int status = conn.getResponseCode(); - final String message = conn.getResponseMessage(); - int redirectCount = 0; - // TODO - should this get replaced by using the conn.setInstanceFollowRedirects(true); - while ((status == HttpURLConnection.HTTP_MOVED_TEMP - || status == HttpURLConnection.HTTP_MOVED_PERM - || status == HttpURLConnection.HTTP_SEE_OTHER) - && MAX_REDIRECT_ATTEMPTS > redirectCount++) { - final String location = conn.getHeaderField("Location"); - try { - conn.disconnect(); - } finally { - conn = null; - } - LOGGER.debug("Download is being redirected from {} to {}", url, location); - conn = connFactory.createHttpURLConnection(new URL(location), this.usesProxy); - conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); - conn.connect(); - status = conn.getResponseCode(); - } - if (status == 404) { - try { - conn.disconnect(); - } finally { - conn = null; - } - throw new ResourceNotFoundException("Requested resource does not exist - received a 404"); - } else if (status == 429) { - try { - conn.disconnect(); - } finally { - conn = null; - } - throw new TooManyRequestsException("Download failed - too many connection requests"); - } else if (status != 200) { - try { - conn.disconnect(); - } finally { - conn = null; - } - final String msg = format("Error retrieving %s; received response code %s; %s", url, status, message); - LOGGER.error(msg); - throw new DownloadFailedException(msg); - } - } catch (IOException ex) { - try { - if (conn != null) { - conn.disconnect(); - } - } finally { - conn = null; - } - if ("Connection reset".equalsIgnoreCase(ex.getMessage())) { - final String msg = format("TLS Connection Reset%nPlease see " - + "http://jeremylong.github.io/DependencyCheck/data/tlsfailure.html " - + "for more information regarding how to resolve the issue."); - LOGGER.error(msg); - throw new DownloadFailedException(msg, ex); - } - final String msg = format("Error downloading file %s; unable to connect.", url); - throw new DownloadFailedException(msg, ex); - } - return conn; - } - - /** - * {@inheritDoc} - *

- * Releases the underlying HTTP URL Connection. - */ - @Override - public void close() { - if (connection != null) { - try { - connection.disconnect(); - } finally { - connection = null; - } - } - } - - /** - * Returns whether or not the connection has been closed. - * - * @return whether or not the connection has been closed - */ - public boolean isClosed() { - return connection == null; - } - - /** - * Returns the HEAD or GET HTTP method. HEAD is the default. - * - * @return the HTTP method to use - */ - private String determineHttpMethod() { - return isQuickQuery() ? HEAD : GET; - } - - /** - * Determines if the HTTP method GET or HEAD should be used to check the - * timestamp on external resources. - * - * @return true if configured to use HEAD requests - */ - private boolean isQuickQuery() { - return settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true); - } - - /** - * Analyzes the IOException, logs the appropriate information for debugging - * purposes, and then throws a DownloadFailedException that wraps the IO - * Exception for common IO Exceptions. This is to provide additional details - * to assist in resolution of the exception. - * - * @param ex the original exception - * @throws org.owasp.dependencycheck.utils.DownloadFailedException a wrapper - * exception that contains the original exception as the cause - */ - public void checkForCommonExceptionTypes(IOException ex) throws DownloadFailedException { - Throwable cause = ex; - while (cause != null) { - if (cause instanceof java.net.UnknownHostException) { - final String msg = format("Unable to resolve domain '%s'", cause.getMessage()); - LOGGER.error(msg); - throw new DownloadFailedException(msg); - } - if (cause instanceof InvalidAlgorithmParameterException) { - final String keystore = System.getProperty("javax.net.ssl.keyStore"); - final String version = System.getProperty("java.version"); - final String vendor = System.getProperty("java.vendor"); - LOGGER.info("Error making HTTPS request - InvalidAlgorithmParameterException"); - LOGGER.info("There appears to be an issue with the installation of Java and the cacerts." - + "See closed issue #177 here: https://github.com/jeremylong/DependencyCheck/issues/177"); - LOGGER.info("Java Info:\njavax.net.ssl.keyStore='{}'\njava.version='{}'\njava.vendor='{}'", - keystore, version, vendor); - throw new DownloadFailedException("Error making HTTPS request. Please see the log for more details."); - } - cause = cause.getCause(); - } - } -} diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java index 6d008a01020..f74319f35fc 100644 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java +++ b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java @@ -18,8 +18,15 @@ package org.owasp.dependencycheck.utils; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URL; + +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertTrue; import org.junit.Before; @@ -54,6 +61,29 @@ public void testFetchFile() throws Exception { assertTrue(outputPath.isFile()); } + /** + * Test of fetchAndHandleContent method. + * + * @throws Exception thrown when an exception occurs. + */ + @Test + public void testfetchAndHandleContent() throws Exception { + URL url = new URL(getSettings().getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL)); + AbstractHttpClientResponseHandler versionHandler = new AbstractHttpClientResponseHandler() { + @Override + public String handleEntity(HttpEntity entity) throws IOException { + try (InputStream in = entity.getContent()) { + byte[] read = new byte[90]; + in.read(read); + String text = new String(read, UTF_8); + assertTrue(text.matches("^\\d+\\.\\d+\\.\\d+.*")); + } + return ""; + } + }; + Downloader.getInstance().fetchAndHandleContent(url, versionHandler); + } + /** * Upgrading to org.mock-server:mockserver-netty:5.8.0 caused this test case * to fail as netty does not allow TLSv1.3 to be "used" in Java 1.8. Under diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/HttpResourceConnectionTest.java b/utils/src/test/java/org/owasp/dependencycheck/utils/HttpResourceConnectionTest.java deleted file mode 100644 index 3ecaabd3f93..00000000000 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/HttpResourceConnectionTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.utils; - -import java.io.InputStream; -import java.net.URL; -import static java.nio.charset.StandardCharsets.UTF_8; -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * - * @author Jeremy Long - */ -public class HttpResourceConnectionTest extends BaseTest { - - /** - * Test of fetch method, of class HttpResourceConnection. - * - * @throws Exception thrown when an exception occurs. - */ - @Test - public void testFetch() throws Exception { - URL url = new URL(getSettings().getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL)); - try (HttpResourceConnection resource = new HttpResourceConnection(getSettings())) { - InputStream in = resource.fetch(url); - byte[] read = new byte[90]; - in.read(read); - String text = new String(read, UTF_8); - assertTrue(text.matches("^\\d+\\.\\d+\\.\\d+.*")); - assertFalse(resource.isClosed()); - } - } - - /** - * Test of close method, of class HttpResourceConnection. - */ - @Test - public void testClose() { - HttpResourceConnection instance = new HttpResourceConnection(getSettings()); - instance.close(); - assertTrue(instance.isClosed()); - } - - /** - * Test of isClosed method, of class HttpResourceConnection. - */ - @Test - public void testIsClosed() throws Exception { - HttpResourceConnection resource = null; - try { - URL url = new URL(getSettings().getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL)); - resource = new HttpResourceConnection(getSettings()); - resource.fetch(url); - assertFalse(resource.isClosed()); - } finally { - if (resource != null) { - resource.close(); - assertTrue(resource.isClosed()); - } - } - } -} From b78ab24120df2d2017c57081ab6288d6833c8ecf Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sat, 7 Sep 2024 19:37:49 +0200 Subject: [PATCH 5/9] feat: Extend apache HTTP-client usage to EngineVersionCheck --- .../data/update/EngineVersionCheck.java | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java index 4fffe1a5373..a000e38d8bc 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java @@ -18,13 +18,10 @@ package org.owasp.dependencycheck.data.update; import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; import javax.annotation.concurrent.ThreadSafe; -import org.apache.commons.io.IOUtils; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; @@ -32,9 +29,10 @@ import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.utils.DateUtil; import org.owasp.dependencycheck.utils.DependencyVersion; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; -import org.owasp.dependencycheck.utils.URLConnectionFailureException; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -208,30 +206,20 @@ protected boolean shouldUpdate(final long lastChecked, final long now, final Dat * @return the current released version number */ protected String getCurrentReleaseVersion() { - HttpURLConnection conn = null; try { final String str = settings.getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL, "https://jeremylong.github.io/DependencyCheck/current.txt"); final URL url = new URL(str); - final URLConnectionFactory factory = new URLConnectionFactory(settings); - conn = factory.createHttpURLConnection(url); - conn.connect(); - if (conn.getResponseCode() != 200) { - return null; - } - try (InputStream is = conn.getInputStream()) { - final String releaseVersion = new String(IOUtils.toByteArray(is), StandardCharsets.UTF_8); - return releaseVersion.trim(); - } + String releaseVersion = null; + releaseVersion = Downloader.getInstance().fetchContent(url, StandardCharsets.UTF_8); + return releaseVersion.trim(); + } catch (TooManyRequestsException ex) { + LOGGER.debug("Unable to retrieve current release version of dependency-check - downloader failed on HTTP 429 Too many requests"); + } catch (ResourceNotFoundException ex) { + LOGGER.debug("Unable to retrieve current release version of dependency-check - downloader failed on HTTP 404 ResourceNotFound"); } catch (MalformedURLException ex) { LOGGER.debug("Unable to retrieve current release version of dependency-check - malformed url?"); - } catch (URLConnectionFailureException ex) { - LOGGER.debug("Unable to retrieve current release version of dependency-check - connection failed"); } catch (IOException ex) { LOGGER.debug("Unable to retrieve current release version of dependency-check - i/o exception"); - } finally { - if (conn != null) { - conn.disconnect(); - } } return null; } From 45a8825585be7c77aa57ce5525e42b0316b7473e Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sun, 8 Sep 2024 12:59:55 +0200 Subject: [PATCH 6/9] feat: Migrate CentralSearch to use Apache HTTP-client via Downloader --- .../data/central/CentralSearch.java | 148 ++++++++++-------- .../data/update/KnownExploitedDataSource.java | 19 +-- .../dependencycheck/utils/Downloader.java | 78 ++++++--- .../dependencycheck/utils/DownloaderIT.java | 2 +- 4 files changed, 149 insertions(+), 98 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java b/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java index e0068a3b188..99b7491378d 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java @@ -17,10 +17,16 @@ */ package org.owasp.dependencycheck.data.central; +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.message.BasicHeader; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.TooManyRequestsException; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.HttpURLConnection; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -38,7 +44,6 @@ import org.owasp.dependencycheck.data.cache.DataCacheFactory; import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; import org.owasp.dependencycheck.utils.XmlUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +110,7 @@ public CentralSearch(Settings settings) throws MalformedURLException { } this.query = queryStr; LOGGER.debug("Central Search Full URL: {}", String.format(query, rootURL, "[SHA1]")); - if (null != settings.getString(Settings.KEYS.PROXY_SERVER)) { + if (null != settings.getString(Settings.KEYS.PROXY_SERVER) || null != System.getProperty("https.proxyHost")) { useProxy = true; LOGGER.debug("Using proxy"); } else { @@ -154,69 +159,25 @@ public List searchSha1(String sha1) throws IOException, TooManyRe LOGGER.trace("Searching Central url {}", url); - // Determine if we need to use a proxy. The rules: - // 1) If the proxy is set, AND the setting is set to true, use the proxy - // 2) Otherwise, don't use the proxy (either the proxy isn't configured, - // or proxy is specifically set to false) - final URLConnectionFactory factory = new URLConnectionFactory(settings); - final HttpURLConnection conn = factory.createHttpURLConnection(url, useProxy); - - conn.setDoOutput(true); - // JSON would be more elegant, but there's not currently a dependency // on JSON, so don't want to add one just for this - conn.addRequestProperty("Accept", "application/xml"); - conn.connect(); - - if (conn.getResponseCode() == 200) { - boolean missing = false; - try { - final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); - final Document doc = builder.parse(conn.getInputStream()); - final XPath xpath = XPathFactory.newInstance().newXPath(); - final String numFound = xpath.evaluate("/response/result/@numFound", doc); - if ("0".equals(numFound)) { - missing = true; - } else { - final NodeList docs = (NodeList) xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET); - for (int i = 0; i < docs.getLength(); i++) { - final String g = xpath.evaluate("./str[@name='g']", docs.item(i)); - LOGGER.trace("GroupId: {}", g); - final String a = xpath.evaluate("./str[@name='a']", docs.item(i)); - LOGGER.trace("ArtifactId: {}", a); - final String v = xpath.evaluate("./str[@name='v']", docs.item(i)); - final NodeList attributes = (NodeList) xpath.evaluate("./arr[@name='ec']/str", docs.item(i), XPathConstants.NODESET); - boolean pomAvailable = false; - boolean jarAvailable = false; - for (int x = 0; x < attributes.getLength(); x++) { - final String tmp = xpath.evaluate(".", attributes.item(x)); - if (".pom".equals(tmp)) { - pomAvailable = true; - } else if (".jar".equals(tmp)) { - jarAvailable = true; - } - } - final String centralContentUrl = settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL); - String artifactUrl = null; - String pomUrl = null; - if (jarAvailable) { - //org/springframework/spring-core/3.2.0.RELEASE/spring-core-3.2.0.RELEASE.pom - artifactUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/' - + v + '/' + a + '-' + v + ".jar"; - } - if (pomAvailable) { - //org/springframework/spring-core/3.2.0.RELEASE/spring-core-3.2.0.RELEASE.pom - pomUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/' - + v + '/' + a + '-' + v + ".pom"; - } - result.add(new MavenArtifact(g, a, v, artifactUrl, pomUrl)); - } + final BasicHeader acceptHeader = new BasicHeader("Accept", "application/xml"); + final AbstractHttpClientResponseHandler handler = new AbstractHttpClientResponseHandler<>() { + @Override + public Document handleEntity(HttpEntity entity) throws IOException { + try (InputStream in = entity.getContent()) { + final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); + return builder.parse(in); + } catch (ParserConfigurationException | SAXException | IOException e) { + // Anything else is jacked up XML stuff that we really can't recover from well + final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage(); + throw new IOException(errorMessage, e); } - } catch (ParserConfigurationException | IOException | SAXException | XPathExpressionException e) { - // Anything else is jacked up XML stuff that we really can't recover from well - final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage(); - throw new IOException(errorMessage, e); } + }; + try { + final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(acceptHeader), useProxy); + final boolean missing = addMavenArtifacts(doc, result); if (missing) { if (cache != null) { @@ -224,12 +185,15 @@ public List searchSha1(String sha1) throws IOException, TooManyRe } throw new FileNotFoundException("Artifact not found in Central"); } - } else if (conn.getResponseCode() == 429) { + } catch (XPathExpressionException e) { + final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage(); + throw new IOException(errorMessage, e); + } catch (TooManyRequestsException e) { final String errorMessage = "Too many requests sent to MavenCentral; additional requests are being rejected."; - throw new TooManyRequestsException(errorMessage); - } else { - final String errorMessage = "Could not connect to MavenCentral (" + conn.getResponseCode() + "): " + conn.getResponseMessage(); - throw new IOException(errorMessage); + throw new TooManyRequestsException(errorMessage, e); + } catch (ResourceNotFoundException | DownloadFailedException e) { + final String errorMessage = "Could not connect to MavenCentral " + e.getMessage(); + throw new IOException(errorMessage, e); } if (cache != null) { cache.put(sha1, result); @@ -237,6 +201,56 @@ public List searchSha1(String sha1) throws IOException, TooManyRe return result; } + /** + * Collect the artifacts from a MavenCentral search result and add them to the list. + * @param doc The Document received in response to the SHA1 search-request + * @param result The list of MavenArtifacts to which found artifacts will be added + * @return Whether the given document holds no search results + */ + private boolean addMavenArtifacts(Document doc, List result) throws XPathExpressionException { + boolean missing = false; + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String numFound = xpath.evaluate("/response/result/@numFound", doc); + if ("0".equals(numFound)) { + missing = true; + } else { + final NodeList docs = (NodeList) xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET); + for (int i = 0; i < docs.getLength(); i++) { + final String g = xpath.evaluate("./str[@name='g']", docs.item(i)); + LOGGER.trace("GroupId: {}", g); + final String a = xpath.evaluate("./str[@name='a']", docs.item(i)); + LOGGER.trace("ArtifactId: {}", a); + final String v = xpath.evaluate("./str[@name='v']", docs.item(i)); + final NodeList attributes = (NodeList) xpath.evaluate("./arr[@name='ec']/str", docs.item(i), XPathConstants.NODESET); + boolean pomAvailable = false; + boolean jarAvailable = false; + for (int x = 0; x < attributes.getLength(); x++) { + final String tmp = xpath.evaluate(".", attributes.item(x)); + if (".pom".equals(tmp)) { + pomAvailable = true; + } else if (".jar".equals(tmp)) { + jarAvailable = true; + } + } + final String centralContentUrl = settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL); + String artifactUrl = null; + String pomUrl = null; + if (jarAvailable) { + //org/springframework/spring-core/3.2.0.RELEASE/spring-core-3.2.0.RELEASE.pom + artifactUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/' + + v + '/' + a + '-' + v + ".jar"; + } + if (pomAvailable) { + //org/springframework/spring-core/3.2.0.RELEASE/spring-core-3.2.0.RELEASE.pom + pomUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/' + + v + '/' + a + '-' + v + ".pom"; + } + result.add(new MavenArtifact(g, a, v, artifactUrl, pomUrl)); + } + } + return missing; + } + /** * Tests to determine if the given URL is invalid. * diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java index 1a3bc87014c..5d2fee2f1bb 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java @@ -93,17 +93,14 @@ public KnownExploitedVulnerabilitiesSchema handleEntity(HttpEntity entity) throw } }; -// final HttpResourceConnection conn = new HttpResourceConnection(settings); -// try { - final KnownExploitedVulnerabilitiesSchema data = Downloader.getInstance().fetchAndHandleContent(url, kevParsingResponseHandler); - final String currentVersion = dbProperties.getProperty(DatabaseProperties.KEV_VERSION, ""); - if (!currentVersion.equals(data.getCatalogVersion())) { - cveDB.updateKnownExploitedVulnerabilities(data.getVulnerabilities()); - } - //all dates in the db are now stored in seconds as opposed to previously milliseconds. - dbProperties.save(DatabaseProperties.KEV_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000)); - return true; -// } + final KnownExploitedVulnerabilitiesSchema data = Downloader.getInstance().fetchAndHandle(url, kevParsingResponseHandler); + final String currentVersion = dbProperties.getProperty(DatabaseProperties.KEV_VERSION, ""); + if (!currentVersion.equals(data.getCatalogVersion())) { + cveDB.updateKnownExploitedVulnerabilities(data.getVulnerabilities()); + } + //all dates in the db are now stored in seconds as opposed to previously milliseconds. + dbProperties.save(DatabaseProperties.KEV_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000)); + return true; } catch (TooManyRequestsException | ResourceNotFoundException | IOException | DatabaseException | SQLException ex) { throw new UpdateException(ex); } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index a891311d0a3..11760ffe44c 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -30,6 +30,7 @@ import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpHost; @@ -38,6 +39,7 @@ import org.apache.hc.core5.http.io.entity.BasicHttpEntity; import org.apache.hc.core5.http.message.BasicClassicHttpRequest; import org.apache.hc.core5.http.message.BasicClassicHttpResponse; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -307,10 +309,9 @@ public void fetchFile(URL url, File outputPath, boolean useProxy) throws Downloa * if there is an error downloading the file * @throws TooManyRequestsException thrown when a 429 is received * @throws ResourceNotFoundException thrown when a 404 is received - * @implNote This method should only be used in cases where the URL cannot be - * determined beforehand from settings, so that ad-hoc Credentials needs to - * be constructed for the target URL when the user/password keys point to configured credentials. - * The method delegates to {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file. + * @implNote This method should only be used in cases where the target host cannot be determined beforehand from settings, so that ad-hoc + * Credentials needs to be constructed for the target URL when the user/password keys point to configured credentials. The method delegates to + * {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file. */ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { @@ -355,8 +356,8 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey /** * Retrieves a file from a given URL and returns the contents. * - * @param url the URL of the file to download - * @param charset The characterset to use to interpret the binary content of the file + * @param url the URL of the file to download + * @param charset The characterset to use to interpret the binary content of the file * @return the content of the file * @throws DownloadFailedException is thrown if there is an error * downloading the file @@ -370,10 +371,10 @@ public String fetchContent(URL url, Charset charset) throws DownloadFailedExcept /** * Retrieves a file from a given URL and returns the contents. * - * @param url the URL of the file to download - * @param useProxy whether to use the configured proxy when downloading - * files - * @param charset The characterset to use to interpret the binary content of the file + * @param url the URL of the file to download + * @param useProxy whether to use the configured proxy when downloading + * files + * @param charset The characterset to use to interpret the binary content of the file * @return the content of the file * @throws DownloadFailedException is thrown if there is an error * downloading the file @@ -415,15 +416,51 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) /** * Download a resource from the given URL and have its content handled by the given ResponseHandler. * - * @param url The url of the resource - * @param responseHandler The responsehandler that handles the response's inputstream + * @param url The url of the resource + * @param responseHandler The responsehandler to handle the response + * @param The return-type for the responseHandler + * @return The response handler result + * @throws IOException on I/O Exceptions + * @throws TooManyRequestsException When HTTP status 429 is encountered + * @throws ResourceNotFoundException When HTTP status 404 is encountered + */ + public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler responseHandler) + throws IOException, TooManyRequestsException, ResourceNotFoundException { + return fetchAndHandle(url, responseHandler, Collections.emptyList(), true); + } + + /** + * Download a resource from the given URL and have its content handled by the given ResponseHandler. + * + * @param url The url of the resource + * @param handler The responsehandler to handle the response + * @param hdr Additional headers to add to the HTTP request + * @param The return-type for the responseHandler * @return The response handler result - * @param The return-type for the responseHandler - * @throws IOException on I/O Exceptions - * @throws TooManyRequestsException When HTTP status 429 is encountered + * @throws IOException on I/O Exceptions + * @throws TooManyRequestsException When HTTP status 429 is encountered * @throws ResourceNotFoundException When HTTP status 404 is encountered */ - public T fetchAndHandleContent(URL url, HttpClientResponseHandler responseHandler) throws IOException, TooManyRequestsException, ResourceNotFoundException { + public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List

hdr) + throws IOException, TooManyRequestsException, ResourceNotFoundException { + return fetchAndHandle(url, handler, hdr, true); + } + + /** + * Download a resource from the given URL and have its content handled by the given ResponseHandler. + * + * @param url The url of the resource + * @param handler The responsehandler to handle the response + * @param hdr Additional headers to add to the HTTP request + * @param useProxy Whether to use the configured proxy for the connection + * @param The return-type for the responseHandler + * @return The response handler result + * @throws IOException on I/O Exceptions + * @throws TooManyRequestsException When HTTP status 429 is encountered + * @throws ResourceNotFoundException When HTTP status 404 is encountered + */ + public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List
hdr, boolean useProxy) + throws IOException, TooManyRequestsException, ResourceNotFoundException { try { T data = null; if ("file".equals(url.getProtocol())) { @@ -432,7 +469,7 @@ public T fetchAndHandleContent(URL url, HttpClientResponseHandler respons final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON); final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200); dummyResponse.setEntity(dummyEntity); - data = responseHandler.handleResponse(dummyResponse); + data = handler.handleResponse(dummyResponse); } catch (HttpException e) { throw new IllegalStateException("HttpException encountered without HTTP traffic", e); } @@ -441,9 +478,12 @@ public T fetchAndHandleContent(URL url, HttpClientResponseHandler respons if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { throw new DownloadFailedException("Unsupported protocol in the URL; only file://, http:// and https:// are supported"); } - try (CloseableHttpClient hc = httpClientBuilder.build()) { + try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); - data = hc.execute(req, responseHandler); + for (Header h : hdr) { + req.addHeader(h); + } + data = hc.execute(req, handler); } } return data; diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java index f74319f35fc..abceef209ec 100644 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java +++ b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java @@ -81,7 +81,7 @@ public String handleEntity(HttpEntity entity) throws IOException { return ""; } }; - Downloader.getInstance().fetchAndHandleContent(url, versionHandler); + Downloader.getInstance().fetchAndHandle(url, versionHandler); } /** From 2d4e487fe0175e975fef423419cd1784bb6bce9b Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sun, 8 Sep 2024 13:02:42 +0200 Subject: [PATCH 7/9] fix: Fixup the missing addition of NVD API Datafeed credentials (if configured) --- .../main/java/org/owasp/dependencycheck/utils/Downloader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 11760ffe44c..97ecb7b5b33 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -153,6 +153,7 @@ public void configure(Settings settings) throws InvalidSettingException { tryAddHostedSuppressionCredentials(settings, credentialsProvider); tryAddKEVCredentials(settings, credentialsProvider); tryAddNexusAnalyzerCredentials(settings, credentialsProvider); + tryAddNVDApiDatafeed(settings, credentialsProvider); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProvider); } From a90c89c0a0bbdf0ad17ec52f65e41439e14be712 Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sun, 8 Sep 2024 17:09:26 +0200 Subject: [PATCH 8/9] feat: Also make OSSIndexAnalyzer use our HTTPClient based connections --- .../data/ossindex/ODCConnectionTransport.java | 61 +++++--- .../data/ossindex/OssindexClientFactory.java | 10 +- .../dependencycheck/utils/Downloader.java | 142 ++++++++++++------ 3 files changed, 135 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java index 64701c8ef34..9ca9ae22ccf 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java @@ -18,13 +18,19 @@ package org.owasp.dependencycheck.data.ossindex; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.message.BasicHeader; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.sonatype.ossindex.service.client.OssindexClientConfiguration; import org.sonatype.ossindex.service.client.transport.BasicAuthHelper; -import org.sonatype.ossindex.service.client.transport.HttpUrlConnectionTransport; +import org.sonatype.ossindex.service.client.transport.Transport; import org.sonatype.ossindex.service.client.transport.UserAgentSupplier; /** @@ -33,20 +39,12 @@ * * @author Jeremy Long */ -public class ODCConnectionTransport extends HttpUrlConnectionTransport { +public class ODCConnectionTransport implements Transport { - /** - * The authorization header. - */ - private static final String AUTHORIZATION = "Authorization"; /** * The OSS Index client configuration. */ private final OssindexClientConfiguration configuration; - /** - * The URL Connection factory. - */ - private final URLConnectionFactory connectionFactory; /** * The user agent to send in the HTTP connection. */ @@ -55,27 +53,40 @@ public class ODCConnectionTransport extends HttpUrlConnectionTransport { /** * Constructs a new transport object to connect to the OSS Index. * - * @param settings the ODC settings * @param config the OSS client configuration * @param userAgent the user agent to send to OSS Index */ - public ODCConnectionTransport(Settings settings, OssindexClientConfiguration config, UserAgentSupplier userAgent) { - super(userAgent); + public ODCConnectionTransport(OssindexClientConfiguration config, UserAgentSupplier userAgent) { this.userAgent = userAgent; this.configuration = config; - connectionFactory = new URLConnectionFactory(settings); } @Override - protected HttpURLConnection connect(final URL url) throws IOException { - final HttpURLConnection connection = connectionFactory.createHttpURLConnection(url); - connection.setRequestProperty("User-Agent", userAgent.get()); + public void init(OssindexClientConfiguration configuration) { + // no initialisation needed + } - final String authorization = BasicAuthHelper.authorizationHeader(configuration.getAuthConfiguration()); - if (authorization != null) { - connection.setRequestProperty(AUTHORIZATION, authorization); + @Override + public String post(URI url, String payloadType, String payload, String acceptType) throws TransportException, IOException { + try { + final List
headers = new ArrayList<>(3); + headers.add(new BasicHeader(HttpHeaders.ACCEPT, acceptType)); + headers.add(new BasicHeader(HttpHeaders.USER_AGENT, userAgent.get())); + // TODO consider to promote pre-emptive authentication by default to the Downloader and also load the OSSIndex credentials there. + final String authorization = BasicAuthHelper.authorizationHeader(configuration.getAuthConfiguration()); + if (authorization != null) { + headers.add(new BasicHeader(HttpHeaders.AUTHORIZATION, authorization)); + } + return Downloader.getInstance().postBasedFetchContent(url, payload, payloadType, headers); + } catch (TooManyRequestsException e) { + throw new TransportException("Too many requests for " + url.toString() + " HTTP status 429", e); + } catch (ResourceNotFoundException e) { + throw new TransportException("Not found for " + url.toString() + "HTTP status 404", e); } - return connection; } + @Override + public void close() throws Exception { + // no resource closure needed; fully delegated to HTTPClient + } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/OssindexClientFactory.java b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/OssindexClientFactory.java index 78026f58ebb..6502be6e9dd 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/OssindexClientFactory.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/OssindexClientFactory.java @@ -85,12 +85,6 @@ public static OssindexClient create(final Settings settings) { final int batchSize = settings.getInt(Settings.KEYS.ANALYZER_OSSINDEX_BATCH_SIZE, OssindexClientConfiguration.DEFAULT_BATCH_SIZE); config.setBatchSize(batchSize); - // proxy likely does not need to be configured here as we are using the - // URLConnectionFactory#createHttpURLConnection() which automatically configures - // the proxy on the connection. -// ProxyConfiguration proxy = new ProxyConfiguration(); -// settings.getString(Settings.KEYS.PROXY_PASSWORD); -// config.setProxyConfiguration(proxy); if (settings.getBoolean(Settings.KEYS.ANALYZER_OSSINDEX_USE_CACHE, true)) { final DirectoryCache.Configuration cache = new DirectoryCache.Configuration(); final File data; @@ -101,7 +95,7 @@ public static OssindexClient create(final Settings settings) { cache.setBaseDir(cacheDir.toPath()); cache.setExpireAfter(Duration.standardHours(24)); config.setCacheConfiguration(cache); - LOGGER.debug("OSS Index Cache: " + cache); + LOGGER.debug("OSS Index Cache: {}", cache); } else { LOGGER.warn("Unable to use a cache for the OSS Index"); } @@ -115,7 +109,7 @@ public static OssindexClient create(final Settings settings) { settings.getString(Settings.KEYS.APPLICATION_VERSION, "unknown") ); - final Transport transport = new ODCConnectionTransport(settings, config, userAgent); + final Transport transport = new ODCConnectionTransport(config, userAgent); final Marshaller marshaller = new GsonMarshaller(); diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 97ecb7b5b33..e6b8777f09b 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -24,6 +24,7 @@ import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.auth.SystemDefaultCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; @@ -37,6 +38,7 @@ import org.apache.hc.core5.http.Method; import org.apache.hc.core5.http.io.HttpClientResponseHandler; import org.apache.hc.core5.http.io.entity.BasicHttpEntity; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicClassicHttpRequest; import org.apache.hc.core5.http.message.BasicClassicHttpResponse; import org.jetbrains.annotations.NotNull; @@ -54,12 +56,14 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Collections; import java.util.List; +import java.util.Locale; import static java.lang.String.format; @@ -146,7 +150,10 @@ public void configure(Settings settings) throws InvalidSettingException { if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) { final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME); final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray(); - credentialsProvider.setCredentials(new AuthScope(null, proxyHost, proxyPort, null, null), new UsernamePasswordCredentials(proxyuser, proxypass)); + credentialsProvider.setCredentials( + new AuthScope(null, proxyHost, proxyPort, null, null), + new UsernamePasswordCredentials(proxyuser, proxypass) + ); } } tryAddRetireJSCredentials(settings, credentialsProvider); @@ -160,7 +167,7 @@ public void configure(Settings settings) throws InvalidSettingException { private void tryAddRetireJSCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD, @@ -170,7 +177,7 @@ private void tryAddRetireJSCredentials(Settings settings, CredentialsStore crede private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, Settings.KEYS.HOSTED_SUPPRESSIONS_USER, Settings.KEYS.HOSTED_SUPPRESSIONS_URL, Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD, @@ -180,7 +187,7 @@ private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsSt private void tryAddKEVCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.KEV_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, Settings.KEYS.KEV_USER, Settings.KEYS.KEV_URL, Settings.KEYS.KEV_PASSWORD, @@ -190,7 +197,7 @@ private void tryAddKEVCredentials(Settings settings, CredentialsStore credential private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, Settings.KEYS.ANALYZER_NEXUS_URL, Settings.KEYS.ANALYZER_NEXUS_USER, Settings.KEYS.ANALYZER_NEXUS_PASSWORD, @@ -200,7 +207,7 @@ private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsStore private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { if (settings.getString(Settings.KEYS.NVD_API_DATAFEED_PASSWORD) != null) { - validateAndAddUsernamePasswordCredentials(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, Settings.KEYS.NVD_API_DATAFEED_URL, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD, @@ -208,30 +215,45 @@ private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credential } } - private void validateAndAddUsernamePasswordCredentials(Settings settings, CredentialsStore credentialsStore, String userKey, String urlKey, String passwordKey, String messageScope) throws InvalidSettingException { + /** + * Add user/password credentials for the host/port of the URL, all configured in the settings, to the credential-store. + * + * @param settings The settings to retrieve the values from + * @param store The credentialStore + * @param userKey The key for a configured username credential part + * @param passwordKey The key for a configured password credential part + * @param urlKey The key for a configured url for which the credentials hold + * @param desc A descriptive text for use in error messages for this credential + * @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings. + */ + private void addUserPasswordCreds(Settings settings, CredentialsStore store, String userKey, String urlKey, String passwordKey, String desc) + throws InvalidSettingException { final String theUser = settings.getString(userKey); final String theURL = settings.getString(urlKey); final char[] thePass = settings.getString(passwordKey, "").toCharArray(); if (theUser == null || theURL == null || thePass.length == 0) { - throw new InvalidSettingException(messageScope + " URL and username are required when setting " + messageScope + " password"); + throw new InvalidSettingException(desc + " URL and username are required when setting " + desc + " password"); } try { final URL parsedURL = new URL(theURL); - addCredentials(credentialsStore, messageScope, parsedURL, theUser, thePass); + addCredentials(store, desc, parsedURL, theUser, thePass); } catch (MalformedURLException e) { - throw new InvalidSettingException(messageScope + " URL must be a valid URL", e); + throw new InvalidSettingException(desc + " URL must be a valid URL", e); } } - private static void addCredentials(CredentialsStore credentialsStore, String messageScope, URL parsedURL, String theUser, char[] thePass) throws InvalidSettingException { + private static void addCredentials(CredentialsStore credentialsStore, String messageScope, URL parsedURL, String theUser, char[] thePass) + throws InvalidSettingException { final String theProtocol = parsedURL.getProtocol(); if ("file".equals(theProtocol)) { LOGGER.warn("Credentials are not supported for file-protocol, double-check your configuration options for {}.", messageScope); return; } else if ("http".equals(theProtocol)) { - LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. Consider migrating to https to guard the credentials.", messageScope); + LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. " + + "Consider migrating to https to guard the credentials.", messageScope); } else if (!"https".equals(theProtocol)) { - throw new InvalidSettingException("Unsupported protocol in the " + messageScope + " URL; only file://, http:// and https:// are supported"); + throw new InvalidSettingException("Unsupported protocol in the " + messageScope + + " URL; only file://, http:// and https:// are supported"); } final String theHost = parsedURL.getHost(); final int thePort = parsedURL.getPort(); @@ -281,21 +303,26 @@ public void fetchFile(URL url, File outputPath, boolean useProxy) throws Downloa } } } catch (HttpResponseException hre) { - final String messageFormat = "%s - Server status: %d - Server reason: %s"; - switch (hre.getStatusCode()) { - case 404: - throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - case 429: - throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - default: - throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - } + wrapAndThrowHttpResponseException(url.toString(), hre); } catch (RuntimeException | URISyntaxException | IOException ex) { final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); throw new DownloadFailedException(msg, ex); } } + private static void wrapAndThrowHttpResponseException(String url, HttpResponseException hre) + throws ResourceNotFoundException, TooManyRequestsException, DownloadFailedException { + final String messageFormat = "%s - Server status: %d - Server reason: %s"; + switch (hre.getStatusCode()) { + case 404: + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + case 429: + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + default: + throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + } + } + /** * Retrieves a file from a given URL using an ad-hoc created CredentialsProvider if needed * and saves it to the outputPath. @@ -331,7 +358,7 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey try { final HttpClientContext context = HttpClientContext.create(); final BasicCredentialsProvider localCredentials = new BasicCredentialsProvider(); - addCredentials(localCredentials, url.toExternalForm(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); + addCredentials(localCredentials, url.toString(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); context.setCredentialsProvider(localCredentials); try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); @@ -339,21 +366,53 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey hc.execute(req, context, responseHandler); } } catch (HttpResponseException hre) { - final String messageFormat = "%s - Server status: %d - Server reason: %s"; - switch (hre.getStatusCode()) { - case 404: - throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - case 429: - throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - default: - throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - } + wrapAndThrowHttpResponseException(url.toString(), hre); } catch (RuntimeException | URISyntaxException | IOException ex) { final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); throw new DownloadFailedException(msg, ex); } } + /** + * Posts a payload to the URL and returns the response as a string. + * + * @param url the URL to POST to + * @param payload the Payload to post + * @param payloadType the string describing the payload's mime-type + * @param hdr Additional headers to add to the HTTP request + * @return the content of the response + * @throws DownloadFailedException is thrown if there is an error + * downloading the file + * @throws TooManyRequestsException thrown when a 429 is received + * @throws ResourceNotFoundException thrown when a 404 is received + */ + public String postBasedFetchContent(URI url, String payload, String payloadType, List
hdr) + throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { + try { + if (url.getScheme() == null || !url.getScheme().toLowerCase(Locale.ROOT).matches("^https?")) { + throw new IllegalArgumentException("Unsupported protocol in the URL; only http and https are supported"); + } else { + final BasicClassicHttpRequest req; + req = new BasicClassicHttpRequest(Method.POST, url); + req.setEntity(new StringEntity(payload, ContentType.create(payloadType, StandardCharsets.UTF_8))); + for (Header h : hdr) { + req.addHeader(h); + } + final String result; + try (CloseableHttpClient hc = httpClientBuilder.build()) { + result = hc.execute(req, new BasicHttpClientResponseHandler()); + } + return result; + } + } catch (HttpResponseException hre) { + wrapAndThrowHttpResponseException(url.toString(), hre); + throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it"); + } catch (RuntimeException | IOException ex) { + final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage()); + throw new DownloadFailedException(msg, ex); + } + } + /** * Retrieves a file from a given URL and returns the contents. * @@ -384,11 +443,11 @@ public String fetchContent(URL url, Charset charset) throws DownloadFailedExcept */ public String fetchContent(URL url, boolean useProxy, Charset charset) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - String result = ""; try { + final String result; if ("file".equals(url.getProtocol())) { final Path p = Paths.get(url.toURI()); - result = new String(Files.readAllBytes(p), charset); + result = Files.readString(p, charset); } else { final BasicClassicHttpRequest req; req = new BasicClassicHttpRequest(Method.GET, url.toURI()); @@ -397,21 +456,14 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) result = hc.execute(req, responseHandler); } } + return result; } catch (HttpResponseException hre) { - final String messageFormat = "%s - Server status: %d - Server reason: %s"; - switch (hre.getStatusCode()) { - case 404: - throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - case 429: - throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - default: - throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); - } + wrapAndThrowHttpResponseException(url.toString(), hre); + throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it"); } catch (RuntimeException | URISyntaxException | IOException ex) { final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage()); throw new DownloadFailedException(msg, ex); } - return result; } /** @@ -463,7 +515,7 @@ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List
hdr, boolean useProxy) throws IOException, TooManyRequestsException, ResourceNotFoundException { try { - T data = null; + final T data; if ("file".equals(url.getProtocol())) { final Path p = Paths.get(url.toURI()); try (InputStream is = Files.newInputStream(p)) { From 11ae195b1507151ed9f05220d21e51de6d6b4e73 Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Thu, 12 Sep 2024 21:05:47 +0200 Subject: [PATCH 9/9] feat: Also make NodeAuditSearch usr our HTTPClient based connections --- .../data/nodeaudit/NodeAuditSearch.java | 146 ++++++++---------- .../data/ossindex/ODCConnectionTransport.java | 4 +- .../dependencycheck/utils/Downloader.java | 59 +++++-- 3 files changed, 111 insertions(+), 98 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NodeAuditSearch.java b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NodeAuditSearch.java index 3cbe7955023..e3032ab1a67 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NodeAuditSearch.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NodeAuditSearch.java @@ -17,28 +17,31 @@ */ package org.owasp.dependencycheck.data.nodeaudit; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.security.SecureRandom; +import java.util.ArrayList; import java.util.List; import javax.annotation.concurrent.ThreadSafe; +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.message.BasicHeader; +import org.json.JSONException; import org.json.JSONObject; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.json.Json; import javax.json.JsonObject; -import javax.json.JsonReader; import org.apache.commons.jcs3.access.exception.CacheException; import static org.owasp.dependencycheck.analyzer.NodeAuditAnalyzer.DEFAULT_URL; @@ -146,80 +149,63 @@ public List submitPackage(JsonObject packageJson) throws SearchExcepti * @throws IOException if it's unable to connect to Node Audit API */ private List submitPackage(JsonObject packageJson, String key, int count) throws SearchException, IOException { - try { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("----------------------------------------"); - LOGGER.trace("Node Audit Payload:"); - LOGGER.trace(packageJson.toString()); - LOGGER.trace("----------------------------------------"); - LOGGER.trace("----------------------------------------"); - } - final byte[] packageDatabytes = packageJson.toString().getBytes(StandardCharsets.UTF_8); - final URLConnectionFactory factory = new URLConnectionFactory(settings); - final HttpURLConnection conn = factory.createHttpURLConnection(nodeAuditUrl, useProxy); - conn.setDoOutput(true); - conn.setDoInput(true); - conn.setRequestMethod("POST"); - conn.setRequestProperty("user-agent", "npm/6.1.0 node/v10.5.0 linux x64"); - conn.setRequestProperty("npm-in-ci", "false"); - conn.setRequestProperty("npm-scope", ""); - conn.setRequestProperty("npm-session", generateRandomSession()); - conn.setRequestProperty("content-type", "application/json"); - conn.setRequestProperty("Content-Length", Integer.toString(packageDatabytes.length)); - conn.connect(); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("----------------------------------------"); + LOGGER.trace("Node Audit Payload:"); + LOGGER.trace(packageJson.toString()); + LOGGER.trace("----------------------------------------"); + LOGGER.trace("----------------------------------------"); + } + final List
additionalHeaders = new ArrayList<>(); + additionalHeaders.add(new BasicHeader(HttpHeaders.USER_AGENT, "npm/6.1.0 node/v10.5.0 linux x64")); + additionalHeaders.add(new BasicHeader("npm-in-ci", "false")); + additionalHeaders.add(new BasicHeader("npm-scope", "")); + additionalHeaders.add(new BasicHeader("npm-session", generateRandomSession())); - try (OutputStream os = new BufferedOutputStream(conn.getOutputStream())) { - os.write(packageDatabytes); - os.flush(); + try { + final String response = Downloader.getInstance().postBasedFetchContent(nodeAuditUrl.toURI(), packageJson.toString(), ContentType.APPLICATION_JSON, additionalHeaders); + final JSONObject jsonResponse = new JSONObject(response); + final NpmAuditParser parser = new NpmAuditParser(); + final List advisories = parser.parse(jsonResponse); + if (cache != null) { + cache.put(key, advisories); } - - switch (conn.getResponseCode()) { - case 200: - try (InputStream in = new BufferedInputStream(conn.getInputStream()); - JsonReader jsonReader = Json.createReader(in)) { - final JSONObject jsonResponse = new JSONObject(jsonReader.readObject().toString()); - final NpmAuditParser parser = new NpmAuditParser(); - final List advisories = parser.parse(jsonResponse); - if (cache != null) { - cache.put(key, advisories); - } - return advisories; - } catch (Exception ex) { - LOGGER.debug("Error connecting to Node Audit API. Error: {}", - ex.getMessage()); - throw new SearchException("Could not connect to Node Audit API: " + ex.getMessage(), ex); - } - case 503: - LOGGER.debug("Node Audit API returned `{} {}` - retrying request.", - conn.getResponseCode(), conn.getResponseMessage()); - if (count < 5) { - final int next = count + 1; - try { - Thread.sleep(1500L * next); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - throw new UnexpectedAnalysisException(ex); + return advisories; + } catch (RuntimeException | URISyntaxException | JSONException | TooManyRequestsException | ResourceNotFoundException ex) { + LOGGER.debug("Error connecting to Node Audit API. Error: {}", + ex.getMessage()); + throw new SearchException("Could not connect to Node Audit API: " + ex.getMessage(), ex); + } catch (DownloadFailedException e) { + if (e.getCause() instanceof HttpResponseException) { + final HttpResponseException hre = (HttpResponseException) e.getCause(); + switch (hre.getStatusCode()) { + case 503: + LOGGER.debug("Node Audit API returned `{} {}` - retrying request.", + hre.getStatusCode(), hre.getReasonPhrase()); + if (count < 5) { + final int next = count + 1; + try { + Thread.sleep(1500L * next); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new UnexpectedAnalysisException(ex); + } + return submitPackage(packageJson, key, next); } - return submitPackage(packageJson, key, next); - } - throw new SearchException("Could not perform Node Audit analysis - service returned a 503."); - case 400: - LOGGER.debug("Invalid payload submitted to Node Audit API. Received response code: {} {}", - conn.getResponseCode(), conn.getResponseMessage()); - throw new SearchException("Could not perform Node Audit analysis. Invalid payload submitted to Node Audit API."); - default: - LOGGER.debug("Could not connect to Node Audit API. Received response code: {} {}", - conn.getResponseCode(), conn.getResponseMessage()); - throw new IOException("Could not connect to Node Audit API"); - } - } catch (IOException ex) { - if (ex instanceof javax.net.ssl.SSLHandshakeException - && ex.getMessage().contains("unable to find valid certification path to requested target")) { - final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. " - + " Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", nodeAuditUrl); - throw new URLConnectionFailureException(msg, ex); + throw new SearchException("Could not perform Node Audit analysis - service returned a 503.", e); + case 400: + LOGGER.debug("Invalid payload submitted to Node Audit API. Received response code: {} {}", + hre.getStatusCode(), hre.getReasonPhrase()); + throw new SearchException("Could not perform Node Audit analysis. Invalid payload submitted to Node Audit API.", e); + default: + LOGGER.debug("Could not connect to Node Audit API. Received response code: {} {}", + hre.getStatusCode(), hre.getReasonPhrase()); + throw new IOException("Could not connect to Node Audit API", e); + } + } else { + LOGGER.debug("Could not connect to Node Audit API. Received generic DownloadException", e); + throw new IOException("Could not connect to Node Audit API", e); } - throw ex; } } @@ -235,6 +221,6 @@ private String generateRandomSession() { while (sb.length() < length) { sb.append(Integer.toHexString(r.nextInt())); } - return sb.toString().substring(0, length); + return sb.substring(0, length); } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java index 9ca9ae22ccf..0e766b78625 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/ossindex/ODCConnectionTransport.java @@ -19,9 +19,11 @@ import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.message.BasicHeader; @@ -77,7 +79,7 @@ public String post(URI url, String payloadType, String payload, String acceptTyp if (authorization != null) { headers.add(new BasicHeader(HttpHeaders.AUTHORIZATION, authorization)); } - return Downloader.getInstance().postBasedFetchContent(url, payload, payloadType, headers); + return Downloader.getInstance().postBasedFetchContent(url, payload, ContentType.create(payloadType, StandardCharsets.UTF_8), headers); } catch (TooManyRequestsException e) { throw new TransportException("Too many requests for " + url.toString() + " HTTP status 429", e); } catch (ResourceNotFoundException e) { diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index e6b8777f09b..ad1dfba6f4f 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.net.ssl.SSLHandshakeException; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -56,7 +57,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -253,7 +253,7 @@ private static void addCredentials(CredentialsStore credentialsStore, String mes + "Consider migrating to https to guard the credentials.", messageScope); } else if (!"https".equals(theProtocol)) { throw new InvalidSettingException("Unsupported protocol in the " + messageScope - + " URL; only file://, http:// and https:// are supported"); + + " URL; only file, http and https are supported"); } final String theHost = parsedURL.getHost(); final int thePort = parsedURL.getPort(); @@ -272,7 +272,8 @@ private static void addCredentials(CredentialsStore credentialsStore, String mes * @throws TooManyRequestsException thrown when a 429 is received * @throws ResourceNotFoundException thrown when a 404 is received */ - public void fetchFile(URL url, File outputPath) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { + public void fetchFile(URL url, File outputPath) + throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException { fetchFile(url, outputPath, true); } @@ -289,7 +290,7 @@ public void fetchFile(URL url, File outputPath) throws DownloadFailedException, * @throws ResourceNotFoundException thrown when a 404 is received */ public void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException, - TooManyRequestsException, ResourceNotFoundException { + TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException { try { if ("file".equals(url.getProtocol())) { final Path p = Paths.get(url.toURI()); @@ -304,6 +305,14 @@ public void fetchFile(URL url, File outputPath, boolean useProxy) throws Downloa } } catch (HttpResponseException hre) { wrapAndThrowHttpResponseException(url.toString(), hre); + } catch (SSLHandshakeException ex) { + if (ex.getMessage().contains("unable to find valid certification path to requested target")) { + final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. " + + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url); + throw new URLConnectionFailureException(msg, ex); + } + final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); + throw new DownloadFailedException(msg, ex); } catch (RuntimeException | URISyntaxException | IOException ex) { final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); throw new DownloadFailedException(msg, ex); @@ -315,11 +324,11 @@ private static void wrapAndThrowHttpResponseException(String url, HttpResponseEx final String messageFormat = "%s - Server status: %d - Server reason: %s"; switch (hre.getStatusCode()) { case 404: - throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre); case 429: - throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre); default: - throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase())); + throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre); } } @@ -342,7 +351,7 @@ private static void wrapAndThrowHttpResponseException(String url, HttpResponseEx * {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file. */ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException, - TooManyRequestsException, ResourceNotFoundException { + TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException { if ("file".equals(url.getProtocol()) || userKey == null || settings.getString(userKey) == null || passwordKey == null || settings.getString(passwordKey) == null @@ -353,7 +362,7 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey } final String theProtocol = url.getProtocol(); if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { - throw new DownloadFailedException("Unsupported protocol in the URL; only file://, http:// and https:// are supported"); + throw new DownloadFailedException("Unsupported protocol in the URL; only file, http and https are supported"); } try { final HttpClientContext context = HttpClientContext.create(); @@ -367,6 +376,14 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey } } catch (HttpResponseException hre) { wrapAndThrowHttpResponseException(url.toString(), hre); + } catch (SSLHandshakeException ex) { + if (ex.getMessage().contains("unable to find valid certification path to requested target")) { + final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. " + + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url); + throw new URLConnectionFailureException(msg, ex); + } + final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); + throw new DownloadFailedException(msg, ex); } catch (RuntimeException | URISyntaxException | IOException ex) { final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage()); throw new DownloadFailedException(msg, ex); @@ -386,15 +403,15 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey * @throws TooManyRequestsException thrown when a 429 is received * @throws ResourceNotFoundException thrown when a 404 is received */ - public String postBasedFetchContent(URI url, String payload, String payloadType, List
hdr) - throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { + public String postBasedFetchContent(URI url, String payload, ContentType payloadType, List
hdr) + throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException { try { if (url.getScheme() == null || !url.getScheme().toLowerCase(Locale.ROOT).matches("^https?")) { throw new IllegalArgumentException("Unsupported protocol in the URL; only http and https are supported"); } else { final BasicClassicHttpRequest req; req = new BasicClassicHttpRequest(Method.POST, url); - req.setEntity(new StringEntity(payload, ContentType.create(payloadType, StandardCharsets.UTF_8))); + req.setEntity(new StringEntity(payload, payloadType)); for (Header h : hdr) { req.addHeader(h); } @@ -407,7 +424,15 @@ public String postBasedFetchContent(URI url, String payload, String payloadType, } catch (HttpResponseException hre) { wrapAndThrowHttpResponseException(url.toString(), hre); throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it"); - } catch (RuntimeException | IOException ex) { + } catch (SSLHandshakeException ex) { + if (ex.getMessage().contains("unable to find valid certification path to requested target")) { + final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. " + + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url); + throw new URLConnectionFailureException(msg, ex); + } + final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage()); + throw new DownloadFailedException(msg, ex); + } catch (IOException | RuntimeException ex) { final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage()); throw new DownloadFailedException(msg, ex); } @@ -470,16 +495,16 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) * Download a resource from the given URL and have its content handled by the given ResponseHandler. * * @param url The url of the resource - * @param responseHandler The responsehandler to handle the response + * @param handler The responsehandler to handle the response * @param The return-type for the responseHandler * @return The response handler result * @throws IOException on I/O Exceptions * @throws TooManyRequestsException When HTTP status 429 is encountered * @throws ResourceNotFoundException When HTTP status 404 is encountered */ - public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler responseHandler) + public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler) throws IOException, TooManyRequestsException, ResourceNotFoundException { - return fetchAndHandle(url, responseHandler, Collections.emptyList(), true); + return fetchAndHandle(url, handler, Collections.emptyList(), true); } /** @@ -529,7 +554,7 @@ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler } else { final String theProtocol = url.getProtocol(); if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { - throw new DownloadFailedException("Unsupported protocol in the URL; only file://, http:// and https:// are supported"); + throw new DownloadFailedException("Unsupported protocol in the URL; only file, http and https are supported"); } try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI());