use gzip to download sigmf data

+ do not re-encode if the source file is already gzipped
dernasherbrezon · Oct 13, 2024 · f49028c · f49028c
1 parent 9e4428d
commit f49028c
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 9 deletions.
diff --git a/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java b/src/main/java/ru/r2cloud/web/api/observation/ObservationSigMfData.java
@@ -8,7 +8,6 @@
 import java.util.Date;
 import java.util.Locale;
 import java.util.TimeZone;
-import java.util.zip.GZIPInputStream;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -20,7 +19,6 @@
 import ru.r2cloud.satellite.IObservationDao;
 import ru.r2cloud.util.Configuration;
 import ru.r2cloud.util.SignedURL;
-import ru.r2cloud.util.Util;
 import ru.r2cloud.web.AbstractHttpController;
 import ru.r2cloud.web.BadRequest;
 import ru.r2cloud.web.ModelAndView;
@@ -78,12 +76,12 @@ public ModelAndView doGet(IHTTPSession session) {
 			if (ifModifiedSince != null && ifModifiedSince >= entity.getRawPath().lastModified() / 1000) {
 				response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.NOT_MODIFIED, "application/octet-stream", null);
 			} else {
-				Long totalBytes = Util.readTotalBytes(entity.getRawPath().toPath());
 				InputStream is = new BufferedInputStream(new FileInputStream(entity.getRawPath()));
+				response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.OK, "application/octet-stream", is, entity.getRawPath().length());
 				if (entity.getRawPath().toString().endsWith(".gz")) {
-					is = new GZIPInputStream(is);
+					// do not re-encode in NanoHTTPD
+					response.addHeader("Content-Encoding", "gzip");
 				}
-				response = NanoHTTPD.newFixedLengthResponse(fi.iki.elonen.NanoHTTPD.Response.Status.OK, "application/octet-stream", is, totalBytes);
 				// convert to seconds
 				response.addHeader("Cache-Control", "private, max-age=" + ((int) (config.getLong("server.static.signed.validMillis") / 1000)));
 			}

diff --git a/src/test/java/ru/r2cloud/TestUtil.java b/src/test/java/ru/r2cloud/TestUtil.java
@@ -1,5 +1,6 @@
 package ru.r2cloud;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -21,13 +22,14 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.UUID;
+import java.util.zip.GZIPInputStream;
 
 import javax.imageio.ImageIO;
 
-import com.eclipsesource.json.JsonArray;
 import org.junit.rules.TemporaryFolder;
 
 import com.eclipsesource.json.Json;
+import com.eclipsesource.json.JsonArray;
 import com.eclipsesource.json.JsonObject;
 import com.eclipsesource.json.JsonValue;
 
@@ -209,4 +211,48 @@ private TestUtil() {
 		// do nothing
 	}
 
+	public static void assertFile(File actual, File expected) {
+		InputStream actualIs = null;
+		InputStream expectedIs = null;
+		try {
+			actualIs = new BufferedInputStream(new FileInputStream(actual));
+			if (actual.getName().endsWith("gz")) {
+				actualIs = new GZIPInputStream(actualIs);
+			}
+			expectedIs = new BufferedInputStream(new FileInputStream(expected));
+			if (expected.getName().endsWith("gz")) {
+				expectedIs = new GZIPInputStream(expectedIs);
+			}
+			byte[] actualBuf = new byte[1024];
+			byte[] expectedBuf = new byte[1024];
+			while (!Thread.currentThread().isInterrupted()) {
+				int actualBytes = actualIs.read(actualBuf);
+				int expectedBytes;
+				if (actualBytes == -1) {
+					expectedBytes = expectedIs.read(expectedBuf);
+				} else {
+					expectedBytes = expectedIs.readNBytes(expectedBuf, 0, actualBytes);
+				}
+				assertEquals(expectedBytes, actualBytes);
+				if (actualBytes == -1) {
+					break;
+				}
+				assertArrayEquals(expectedBuf, actualBuf);
+			}
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		} finally {
+			Util.closeQuietly(actualIs);
+			Util.closeQuietly(expectedIs);
+		}
+
+//		try (InputStream actualIs = new FileInputStream(actual); InputStream expectedIs = new FileInputStream(expected)) {
+//			if (actual.getName().endsWith("gz")) {
+//				actualIs = new GZIPInputStream(actualIs);
+//			}
+//		} catch (IOException e) {
+//			throw new RuntimeException(e);
+//		}
+	}
+
 }
diff --git a/src/test/java/ru/r2cloud/it/ObservationLoadTest.java b/src/test/java/ru/r2cloud/it/ObservationLoadTest.java
@@ -20,11 +20,15 @@ public void testLoadAausat4() throws Exception {
 		File basepath = new File(config.getProperty("satellites.basepath.location") + File.separator + "41460" + File.separator + "data" + File.separator + "1559942730784");
 		TestUtil.copy("aausat4Observation/1559942730784.json", new File(basepath, "meta.json"));
 		TestUtil.copy("aausat4Observation/data.bin", new File(basepath, "data.bin"));
-		TestUtil.copy("data/aausat.raw.gz", new File(basepath, "output.raw.gz"));
+		File expectedIqFile = new File(basepath, "output.raw.gz");
+		TestUtil.copy("data/aausat.raw.gz", expectedIqFile);
 		JsonObject observation = client.getObservation("41460", "1559942730784");
 		TestUtil.assertJson("aausat4Observation/expected.json", observation);
-		JsonObject sigmf = client.getSigMf(observation.getString("sigmfMetaURL", null));
+		JsonObject sigmf = client.getSigMfMeta(observation.getString("sigmfMetaURL", null));
 		TestUtil.assertJson("aausat4Observation/1559942730784.sigmf-meta.json", sigmf);
+		File actual = new File(tempFolder.getRoot(), "actual.raw");
+		client.downloadSigMfData(observation.getString("sigmfDataURL", null), actual);
+		TestUtil.assertFile(actual, expectedIqFile);
 	}
 
 	@Test

diff --git a/src/test/java/ru/r2cloud/it/util/RestClient.java b/src/test/java/ru/r2cloud/it/util/RestClient.java
@@ -1,6 +1,11 @@
 package ru.r2cloud.it.util;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.Socket;
 import java.net.URI;
 import java.net.URLEncoder;
@@ -24,6 +29,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.zip.GZIPInputStream;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
@@ -41,6 +48,7 @@
 import ru.r2cloud.model.GeneralConfiguration;
 import ru.r2cloud.model.IntegrationConfiguration;
 import ru.r2cloud.model.Page;
+import ru.r2cloud.util.Util;
 
 public class RestClient {
 
@@ -651,7 +659,7 @@ public JsonObject awaitObservation(String satelliteId, String observationId, boo
 		return null;
 	}
 
-	public JsonObject getSigMf(String url) {
+	public JsonObject getSigMfMeta(String url) {
 		if (url == null) {
 			return null;
 		}
@@ -671,6 +679,27 @@ public JsonObject getSigMf(String url) {
 		}
 	}
 
+	public void downloadSigMfData(String url, File actual) {
+		HttpRequest request = createAuthRequest(url).GET().build();
+		try (OutputStream os = new BufferedOutputStream(new FileOutputStream(actual))) {
+			HttpResponse<InputStream> response = httpclient.send(request, BodyHandlers.ofInputStream());
+			if (response.statusCode() != 200) {
+				throw new RuntimeException("invalid status code: " + response.statusCode());
+			}
+			Optional<String> val = response.headers().firstValue("content-encoding");
+			InputStream is = response.body();
+			if (val.isPresent() && val.get().equals("gzip")) {
+				is = new GZIPInputStream(is);
+			}
+			Util.copy(is, os);
+		} catch (InterruptedException e) {
+			Thread.currentThread().interrupt();
+			throw new RuntimeException("interrupted");
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
 	public JsonObject getObservationPresentation(String satelliteId, String observationId) {
 		HttpResponse<String> response = getObservationResponse("/api/v1/observation/load", satelliteId, observationId);
 		if (response.statusCode() == 404) {