From f49028c42cd120c1b505eb2a1da34c66eda1b1f4 Mon Sep 17 00:00:00 2001 From: dernasherbrezon Date: Sun, 13 Oct 2024 11:31:14 +0100 Subject: [PATCH] use gzip to download sigmf data + do not re-encode if the source file is already gzipped --- .../api/observation/ObservationSigMfData.java | 8 ++-- src/test/java/ru/r2cloud/TestUtil.java | 48 ++++++++++++++++++- .../ru/r2cloud/it/ObservationLoadTest.java | 8 +++- .../java/ru/r2cloud/it/util/RestClient.java | 31 +++++++++++- 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java b/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java index af673aae..bb81cdf7 100644 --- a/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java +++ b/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java @@ -8,7 +8,6 @@ import java.util.Date; import java.util.Locale; import java.util.TimeZone; -import java.util.zip.GZIPInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +19,6 @@ import ru.r2cloud.satellite.IObservationDao; import ru.r2cloud.util.Configuration; import ru.r2cloud.util.SignedURL; -import ru.r2cloud.util.Util; import ru.r2cloud.web.AbstractHttpController; import ru.r2cloud.web.BadRequest; import ru.r2cloud.web.ModelAndView; @@ -78,12 +76,12 @@ public ModelAndView doGet(IHTTPSession session) { if (ifModifiedSince != null && ifModifiedSince >= entity.getRawPath().lastModified() / 1000) { response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.NOT_MODIFIED, "application/octet-stream", null); } else { - Long totalBytes = Util.readTotalBytes(entity.getRawPath().toPath()); InputStream is = new BufferedInputStream(new FileInputStream(entity.getRawPath())); + response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.OK, "application/octet-stream", is, entity.getRawPath().length()); if (entity.getRawPath().toString().endsWith(".gz")) { - is = new GZIPInputStream(is); + // do not re-encode in NanoHTTPD + response.addHeader("Content-Encoding", "gzip"); } - response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.OK, "application/octet-stream", is, totalBytes); // convert to seconds response.addHeader("Cache-Control", "private, max-age=" + ((int) (config.getLong("server.static.signed.validMillis") / 1000))); } diff --git a/src/test/java/ru/r2cloud/TestUtil.java b/src/test/java/ru/r2cloud/TestUtil.java index a5d88374..4c902eed 100644 --- a/src/test/java/ru/r2cloud/TestUtil.java +++ b/src/test/java/ru/r2cloud/TestUtil.java @@ -1,5 +1,6 @@ package ru.r2cloud; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -21,13 +22,14 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.UUID; +import java.util.zip.GZIPInputStream; import javax.imageio.ImageIO; -import com.eclipsesource.json.JsonArray; import org.junit.rules.TemporaryFolder; import com.eclipsesource.json.Json; +import com.eclipsesource.json.JsonArray; import com.eclipsesource.json.JsonObject; import com.eclipsesource.json.JsonValue; @@ -209,4 +211,48 @@ private TestUtil() { // do nothing } + public static void assertFile(File actual, File expected) { + InputStream actualIs = null; + InputStream expectedIs = null; + try { + actualIs = new BufferedInputStream(new FileInputStream(actual)); + if (actual.getName().endsWith("gz")) { + actualIs = new GZIPInputStream(actualIs); + } + expectedIs = new BufferedInputStream(new FileInputStream(expected)); + if (expected.getName().endsWith("gz")) { + expectedIs = new GZIPInputStream(expectedIs); + } + byte[] actualBuf = new byte[1024]; + byte[] expectedBuf = new byte[1024]; + while (!Thread.currentThread().isInterrupted()) { + int actualBytes = actualIs.read(actualBuf); + int expectedBytes; + if (actualBytes == -1) { + expectedBytes = expectedIs.read(expectedBuf); + } else { + expectedBytes = expectedIs.readNBytes(expectedBuf, 0, actualBytes); + } + assertEquals(expectedBytes, actualBytes); + if (actualBytes == -1) { + break; + } + assertArrayEquals(expectedBuf, actualBuf); + } + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + Util.closeQuietly(actualIs); + Util.closeQuietly(expectedIs); + } + +// try (InputStream actualIs = new FileInputStream(actual); InputStream expectedIs = new FileInputStream(expected)) { +// if (actual.getName().endsWith("gz")) { +// actualIs = new GZIPInputStream(actualIs); +// } +// } catch (IOException e) { +// throw new RuntimeException(e); +// } + } + } diff --git a/src/test/java/ru/r2cloud/it/ObservationLoadTest.java b/src/test/java/ru/r2cloud/it/ObservationLoadTest.java index 40a674cc..2e040cea 100644 --- a/src/test/java/ru/r2cloud/it/ObservationLoadTest.java +++ b/src/test/java/ru/r2cloud/it/ObservationLoadTest.java @@ -20,11 +20,15 @@ public void testLoadAausat4() throws Exception { File basepath = new File(config.getProperty("satellites.basepath.location") + File.separator + "41460" + File.separator + "data" + File.separator + "1559942730784"); TestUtil.copy("aausat4Observation/1559942730784.json", new File(basepath, "meta.json")); TestUtil.copy("aausat4Observation/data.bin", new File(basepath, "data.bin")); - TestUtil.copy("data/aausat.raw.gz", new File(basepath, "output.raw.gz")); + File expectedIqFile = new File(basepath, "output.raw.gz"); + TestUtil.copy("data/aausat.raw.gz", expectedIqFile); JsonObject observation = client.getObservation("41460", "1559942730784"); TestUtil.assertJson("aausat4Observation/expected.json", observation); - JsonObject sigmf = client.getSigMf(observation.getString("sigmfMetaURL", null)); + JsonObject sigmf = client.getSigMfMeta(observation.getString("sigmfMetaURL", null)); TestUtil.assertJson("aausat4Observation/1559942730784.sigmf-meta.json", sigmf); + File actual = new File(tempFolder.getRoot(), "actual.raw"); + client.downloadSigMfData(observation.getString("sigmfDataURL", null), actual); + TestUtil.assertFile(actual, expectedIqFile); } @Test diff --git a/src/test/java/ru/r2cloud/it/util/RestClient.java b/src/test/java/ru/r2cloud/it/util/RestClient.java index 9223c2ca..bddfd940 100644 --- a/src/test/java/ru/r2cloud/it/util/RestClient.java +++ b/src/test/java/ru/r2cloud/it/util/RestClient.java @@ -1,6 +1,11 @@ package ru.r2cloud.it.util; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.Socket; import java.net.URI; import java.net.URLEncoder; @@ -24,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; +import java.util.zip.GZIPInputStream; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; @@ -41,6 +48,7 @@ import ru.r2cloud.model.GeneralConfiguration; import ru.r2cloud.model.IntegrationConfiguration; import ru.r2cloud.model.Page; +import ru.r2cloud.util.Util; public class RestClient { @@ -651,7 +659,7 @@ public JsonObject awaitObservation(String satelliteId, String observationId, boo return null; } - public JsonObject getSigMf(String url) { + public JsonObject getSigMfMeta(String url) { if (url == null) { return null; } @@ -671,6 +679,27 @@ public JsonObject getSigMf(String url) { } } + public void downloadSigMfData(String url, File actual) { + HttpRequest request = createAuthRequest(url).GET().build(); + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(actual))) { + HttpResponse response = httpclient.send(request, BodyHandlers.ofInputStream()); + if (response.statusCode() != 200) { + throw new RuntimeException("invalid status code: " + response.statusCode()); + } + Optional val = response.headers().firstValue("content-encoding"); + InputStream is = response.body(); + if (val.isPresent() && val.get().equals("gzip")) { + is = new GZIPInputStream(is); + } + Util.copy(is, os); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("interrupted"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public JsonObject getObservationPresentation(String satelliteId, String observationId) { HttpResponse response = getObservationResponse("/api/v1/observation/load", satelliteId, observationId); if (response.statusCode() == 404) {