Skip to content

Commit

Permalink
fix: use temporary files to reduce memory usage (#6270)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeremylong
jeremylong authored Dec 10, 2023
1 parent cb3bcb9 commit 6f972cc
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 67 deletions.
15 changes: 13 additions & 2 deletions core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
*/
package org.owasp.dependencycheck.data.update;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.net.MalformedURLException;
Expand All @@ -41,6 +44,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.zip.GZIPOutputStream;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
Expand Down Expand Up @@ -328,14 +332,21 @@ private boolean processApi() throws UpdateException {
int ctr = 0;
try (NvdCveClient api = builder.build()) {
while (api.hasNext()) {
final Collection<DefCveItem> items = api.next();
Collection<DefCveItem> items = api.next();
max = api.getTotalAvailable();
if (ctr == 0) {
LOGGER.info(String.format("NVD API has %,d records in this update", max));
}
if (items != null && !items.isEmpty()) {
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, items));
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
try (FileOutputStream fos = new FileOutputStream(outputFile);
GZIPOutputStream out = new GZIPOutputStream(fos);) {
objectMapper.writeValue(out, items);
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
submitted.add(f);
}
ctr += 1;
if ((ctr % 5) == 0) {
final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100;
Expand Down
15 changes: 4 additions & 11 deletions core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,14 @@
*/
package org.owasp.dependencycheck.data.update.nvd.api;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20;
import java.io.File;
import java.net.URL;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.commons.lang3.StringUtils;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
Expand Down Expand Up @@ -72,7 +69,6 @@ public class DownloadTask implements Callable<Future<NvdApiProcessor>> {
* @param settings a reference to the global settings object; this is
* necessary so that when the thread is started the dependencies have a
* correct reference to the global settings.
* @throws UpdateException thrown if temporary files could not be created
*/
public DownloadTask(String url, ExecutorService processor, CveDB cveDB, Settings settings) {
this.url = url;
Expand All @@ -89,15 +85,12 @@ public Future<NvdApiProcessor> call() throws Exception {
LOGGER.info("Download Started for NVD Cache - {}", url);
final long startDownload = System.currentTimeMillis();
final Downloader d = new Downloader(settings);
final String content = d.fetchGzContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
final CveApiJson20 data = objectMapper.readValue(content, CveApiJson20.class);

final File outputFile = settings.getTempFile("nvd-datafeed-", "json.gz");
d.fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
if (this.processorService == null) {
return null;
}
final NvdApiProcessor task = new NvdApiProcessor(cveDB, data.getVulnerabilities(), startDownload);
final NvdApiProcessor task = new NvdApiProcessor(cveDB, outputFile, startDownload);
final Future<NvdApiProcessor> val = this.processorService.submit(task);
return val;
} catch (Throwable ex) {
Expand Down
64 changes: 49 additions & 15 deletions core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,20 @@
*/
package org.owasp.dependencycheck.data.update.nvd.api;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20;
import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.Callable;
import java.util.zip.GZIPInputStream;
import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -41,9 +50,9 @@ public class NvdApiProcessor implements Callable<NvdApiProcessor> {
*/
private final CveDB cveDB;
/**
* The collection of NVD API data to add to the database.
* The file containing the data to inject.
*/
private Collection<DefCveItem> data;
private File jsonFile;
/**
* Reference to the CVE Ecosystem Mapper object.
*/
Expand All @@ -60,37 +69,62 @@ public class NvdApiProcessor implements Callable<NvdApiProcessor> {
/**
* Create a new processor to put the NVD data into the database.
*
* @param cveDB a reference to the database
* @param data the data to add to the database
* @param cveDB a reference to the database.
* @param jsonFile the JSON data file to inject.
* @param startTime the start time of the update process.
*/
public NvdApiProcessor(final CveDB cveDB, Collection<DefCveItem> data, long startTime) {
public NvdApiProcessor(final CveDB cveDB, File jsonFile, long startTime) {
this.cveDB = cveDB;
this.data = data;
this.jsonFile = jsonFile;
this.startTime = startTime;
}

/**
* Create a new processor to put the NVD data into the database.
*
* @param cveDB a reference to the database
* @param data the data to add to the database
* @param jsonFile the JSON data file to inject.
*/
public NvdApiProcessor(final CveDB cveDB, Collection<DefCveItem> data) {
this(cveDB, data, System.currentTimeMillis());
public NvdApiProcessor(final CveDB cveDB, File jsonFile) {
this(cveDB, jsonFile, System.currentTimeMillis());
}

@Override
public NvdApiProcessor call() throws Exception {
for (DefCveItem entry : data) {
try {
cveDB.updateVulnerability(entry, mapper.getEcosystem(entry));
} catch (Exception ex) {
LOGGER.error("Failed to process " + entry.getCve().getId(), ex);
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
Collection<DefCveItem> data = null;

if (jsonFile.getName().endsWith(".jsonarray.gz")) {
try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
data = objectMapper.readValue(gzipInputStream, new TypeReference<Collection<DefCveItem>>(){});
} catch (IOException exception) {
throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
}
} else if (jsonFile.getName().endsWith(".gz")) {
try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class);
if (cveData != null) {
data = cveData.getVulnerabilities();
}
} catch (IOException exception) {
throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
}
} else {
data = objectMapper.readValue(jsonFile, new TypeReference<Collection<DefCveItem>>(){});
}
if (data != null ) {
for (DefCveItem entry : data) {
try {
cveDB.updateVulnerability(entry, mapper.getEcosystem(entry));
} catch (Exception ex) {
LOGGER.error("Failed to process " + entry.getCve().getId(), ex);
}
}
}
endTime = System.currentTimeMillis();
data = null;
return this;
}

Expand Down
40 changes: 1 addition & 39 deletions utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ public String fetchContent(URL url, boolean useProxy) throws DownloadFailedExcep
public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
InputStream in = null;
try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey);
try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey);
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
in = conn.fetch(url);
IOUtils.copy(in, out);
Expand All @@ -196,42 +196,4 @@ public String fetchContent(URL url, boolean useProxy, String userKey, String pas
}
}
}

/**
* Retrieves a gzip file from a given URL and returns the uncompressed contents.
*
* @param url the URL of the file to download
* @param useProxy whether to use the configured proxy when downloading
* files
* @return the content of the file
* @param userKey the settings key for the username to be used
* @param passwordKey the settings key for the password to be used
* @throws DownloadFailedException is thrown if there is an error
* downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public String fetchGzContent(URL url, boolean useProxy, String userKey, String passwordKey)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
InputStream in = null;
try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) {
in = conn.fetch(url);
try (GZIPInputStream gzipIn = new GZIPInputStream(in);
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
IOUtils.copy(gzipIn, out);
return out.toString(UTF8);
}
} catch (IOException ex) {
final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage());
throw new DownloadFailedException(msg, ex);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException ex) {
LOGGER.trace("Ignorable error", ex);
}
}
}
}
}

0 comments on commit 6f972cc

Please sign in to comment.