store current device configuration and transmitter in the observation

This will allow research how different device settings affect the result

+ add custom sigmf tags to be able to re-run decoding/demodulation

src/main/java/ru/r2cloud/R2Cloud.java

@@ -164,7 +164,7 @@ public R2Cloud(Configuration props, Clock clock) {
 		decoders = new Decoders(predict, props, processFactory);
 		decoderService = new DecoderService(props, decoders, resultDao, leoSatDataService, threadFactory, metrics, satelliteDao);
 		priorityService = new PriorityService(props, clock);
-		houseKeeping = new Housekeeping(props, satelliteDao, threadFactory, new CelestrakClient(props), tleDao, satnogsClient, leoSatDataClient, decoderService, priorityService);
+		houseKeeping = new Housekeeping(props, satelliteDao, threadFactory, new CelestrakClient(props, clock), tleDao, satnogsClient, leoSatDataClient, decoderService, priorityService);
 
 		observationFactory = new ObservationFactory(predict);
 

src/main/java/ru/r2cloud/SpectogramService.java

@@ -77,6 +77,10 @@ private File createFromIq(Observation req) {
 			LOG.error("corrupted raw file: {}", req.getRawPath().getAbsolutePath());
 			return null;
 		}
+		if (req.getDataFormat() == null) {
+			LOG.error("data format is missing");
+			return null;
+		}
 		FloatInput source = null;
 		try {
 			InputStream is = new BufferedInputStream(new FileInputStream(req.getRawPath()));

src/main/java/ru/r2cloud/device/Device.java

@@ -117,11 +117,7 @@ private void schedule(ObservationRequest req, Transmitter transmitter) {
 			public void safeRun() {
 				IQData data;
 				Observation observation = new Observation(req);
-				observation.setGain(String.valueOf(deviceConfiguration.getGain()));
-				// write some device-specific parameters
-				observation.setBiast(deviceConfiguration.isBiast());
-				observation.setRtlDeviceId(deviceConfiguration.getRtlDeviceId());
-				observation.setPpm(deviceConfiguration.getPpm());
+				observation.setDevice(deviceConfiguration);
 				observation.setStatus(ObservationStatus.RECEIVING_DATA);
 				// do not use lock for multiple concurrent observations
 				if (numberOfConcurrentObservations > 1) {

src/main/java/ru/r2cloud/model/AntennaConfiguration.java

@@ -72,4 +72,15 @@ public JsonObject toJson() {
 		return json;
 	}
 
+	public static AntennaConfiguration fromJson(JsonObject meta) {
+		AntennaConfiguration result = new AntennaConfiguration();
+		result.setType(AntennaType.valueOf(meta.getString("antennaType", "OMNIDIRECTIONAL")));
+		result.setAzimuth(meta.getDouble("azimuth", 0));
+		result.setElevation(meta.getDouble("elevation", 0));
+		result.setBeamwidth(meta.getDouble("beamwidth", 0));
+		result.setMinElevation(meta.getDouble("minElevation", 0));
+		result.setGuaranteedElevation(meta.getDouble("guaranteedElevation", 0));
+		return result;
+	}
+
 }

src/main/java/ru/r2cloud/model/DeviceConfiguration.java

@@ -1,6 +1,7 @@
 package ru.r2cloud.model;
 
 import com.eclipsesource.json.JsonObject;
+import com.eclipsesource.json.JsonValue;
 
 public class DeviceConfiguration {
 
@@ -193,7 +194,9 @@ public JsonObject toJson() {
 		if (id != null) {
 			json.add("id", id);
 		}
-		json.add("name", name);
+		if (name != null) {
+			json.add("name", name);
+		}
 		if (deviceType != null) {
 			json.add("deviceType", deviceType.name());
 		}
@@ -238,4 +241,40 @@ public JsonObject toJson() {
 		}
 		return json;
 	}
+
+	public static DeviceConfiguration fromJson(JsonObject meta) {
+		DeviceConfiguration result = new DeviceConfiguration();
+		result.setId(meta.getString("id", null));
+		result.setName(meta.getString("name", null));
+		result.setDeviceType(DeviceType.valueOf(meta.getString("deviceType", "RTLSDR")));
+		result.setMinimumFrequency(meta.getLong("minimumFrequency", 0));
+		result.setMaximumFrequency(meta.getLong("maximumFrequency", 0));
+		result.setHost(meta.getString("host", null));
+		result.setPort(meta.getInt("port", 0));
+		result.setUsername(meta.getString("username", null));
+		result.setGain(meta.getFloat("gain", 0));
+		result.setRtlDeviceId(meta.getInt("rtlDeviceId", 0));
+		result.setBiast(meta.getBoolean("biast", false));
+		result.setPpm(meta.getInt("ppm", 0));
+		result.setMaximumBatteryVoltage(meta.getDouble("maximumBatteryVoltage", 0));
+		result.setMinimumBatteryVoltage(meta.getDouble("minimumBatteryVoltage", 0));
+		JsonValue bandwidth = meta.get("bandwidth");
+		if (bandwidth != null) {
+			SdrServerConfiguration sdrConfig = new SdrServerConfiguration();
+			sdrConfig.setBandwidth(bandwidth.asLong());
+			sdrConfig.setBandwidthCrop(meta.getLong("bandwidthCrop", 0));
+			sdrConfig.setBasepath(meta.getString("basepath", null));
+			sdrConfig.setUseGzip(meta.getBoolean("usegzip", false));
+			result.setSdrServerConfiguration(sdrConfig);
+		}
+		JsonValue rotator = meta.get("rotator");
+		if (rotator != null) {
+			result.setRotatorConfiguration(RotatorConfiguration.fromJson(rotator.asObject()));
+		}
+		JsonValue antenna = meta.get("antenna");
+		if (antenna != null) {
+			result.setAntennaConfiguration(AntennaConfiguration.fromJson(antenna.asObject()));
+		}
+		return result;
+	}
 }

src/main/java/ru/r2cloud/model/Observation.java

@@ -22,14 +22,9 @@ public class Observation {
 	private GeodeticPoint groundStation;
 
 	private DataFormat dataFormat;
-	private SdrType sdrType;
 	private long sampleRate;
 	private long frequency;
-	private String gain;
-	private boolean biast;
-	private long centerBandFrequency;
-	private int rtlDeviceId;
-	private int ppm;
+	private DeviceConfiguration device;
 
 	// observation status
 	private String channelA;
@@ -80,6 +75,14 @@ public ObservationRequest getReq() {
 		return result;
 	}
 
+	public DeviceConfiguration getDevice() {
+		return device;
+	}
+
+	public void setDevice(DeviceConfiguration device) {
+		this.device = device;
+	}
+
 	public String getSigmfDataURL() {
 		return sigmfDataURL;
 	}
@@ -104,32 +107,6 @@ public void setDataFormat(DataFormat dataFormat) {
 		this.dataFormat = dataFormat;
 	}
 
-	public long getCenterBandFrequency() {
-		return centerBandFrequency;
-	}
-
-	public void setCenterBandFrequency(long centerBandFrequency) {
-		this.centerBandFrequency = centerBandFrequency;
-	}
-
-	@Deprecated
-	public SdrType getSdrType() {
-		return sdrType;
-	}
-
-	@Deprecated
-	public void setSdrType(SdrType sdrType) {
-		this.sdrType = sdrType;
-	}
-
-	public boolean isBiast() {
-		return biast;
-	}
-
-	public void setBiast(boolean biast) {
-		this.biast = biast;
-	}
-
 	public ObservationStatus getStatus() {
 		return status;
 	}
@@ -298,30 +275,6 @@ public void setSampleRate(long sampleRate) {
 		this.sampleRate = sampleRate;
 	}
 
-	public String getGain() {
-		return gain;
-	}
-
-	public void setGain(String gain) {
-		this.gain = gain;
-	}
-
-	public int getRtlDeviceId() {
-		return rtlDeviceId;
-	}
-
-	public void setRtlDeviceId(int rtlDeviceId) {
-		this.rtlDeviceId = rtlDeviceId;
-	}
-
-	public int getPpm() {
-		return ppm;
-	}
-
-	public void setPpm(int ppm) {
-		this.ppm = ppm;
-	}
-
 	public static Observation fromJson(JsonObject meta) {
 		Observation result = new Observation();
 		result.setId(meta.getString("id", null));
@@ -337,31 +290,8 @@ public static Observation fromJson(JsonObject meta) {
 		if (groundStation != null && groundStation.isObject()) {
 			result.setGroundStation(groundStationFromJson(groundStation.asObject()));
 		}
-		String sdrTypeStr = meta.getString("sdrType", null);
-		SdrType sdrType;
-		if (sdrTypeStr != null) {
-			sdrType = SdrType.valueOf(sdrTypeStr);
-		} else {
-			sdrType = SdrType.RTLSDR;
-		}
-		result.setSdrType(sdrType);
 		String dataFormatStr = meta.getString("dataFormat", null);
-		if (dataFormatStr == null) {
-			// backward compatible
-			switch (sdrType) {
-			case PLUTOSDR:
-				result.setDataFormat(DataFormat.COMPLEX_SIGNED_SHORT);
-				break;
-			case RTLSDR:
-				result.setDataFormat(DataFormat.COMPLEX_UNSIGNED_BYTE);
-				break;
-			case SDRSERVER:
-				result.setDataFormat(DataFormat.COMPLEX_FLOAT);
-				break;
-			default:
-				result.setDataFormat(DataFormat.UNKNOWN);
-			}
-		} else {
+		if (dataFormatStr != null) {
 			result.setDataFormat(DataFormat.valueOf(dataFormatStr));
 		}
 		int legacyInputRate = meta.getInt("inputSampleRate", 0);
@@ -371,12 +301,6 @@ public static Observation fromJson(JsonObject meta) {
 			result.setSampleRate(meta.getLong("sampleRate", -1));
 		}
 		result.setFrequency(meta.getLong("actualFrequency", -1));
-		result.setGain(meta.getString("gain", null));
-		result.setBiast(meta.getBoolean("biast", false));
-		result.setCenterBandFrequency(meta.getLong("centerBandFrequency", 0));
-		result.setRtlDeviceId(meta.getInt("rtlDeviceId", 0));
-		result.setPpm(meta.getInt("ppm", 0));
-
 		result.setChannelA(meta.getString("channelA", null));
 		result.setChannelB(meta.getString("channelB", null));
 		JsonValue numberOfDecodedPackets = meta.get("numberOfDecodedPackets");
@@ -389,14 +313,14 @@ public static Observation fromJson(JsonObject meta) {
 		result.setDataURL(meta.getString("data", null));
 		String statusStr = meta.getString("status", null);
 		if (statusStr != null) {
-			ObservationStatus status = ObservationStatus.valueOf(statusStr);
-			if (status.equals(ObservationStatus.NEW)) {
-				status = ObservationStatus.RECEIVED;
-			}
-			result.setStatus(status);
+			result.setStatus(ObservationStatus.valueOf(statusStr));
 		} else {
 			result.setStatus(ObservationStatus.UPLOADED);
 		}
+		JsonValue deviceConfig = meta.get("device");
+		if (deviceConfig != null) {
+			result.setDevice(DeviceConfiguration.fromJson(deviceConfig.asObject()));
+		}
 		return result;
 	}
 
@@ -413,19 +337,11 @@ public JsonObject toJson(SignedURL signed) {
 		if (getGroundStation() != null) {
 			json.add("groundStation", toJson(getGroundStation()));
 		}
-		if (sdrType != null) {
-			json.add("sdrType", sdrType.name());
-		}
 		if (dataFormat != null) {
 			json.add("dataFormat", dataFormat.name());
 		}
 		json.add("sampleRate", getSampleRate());
 		json.add("actualFrequency", getFrequency());
-		json.add("gain", getGain());
-		json.add("biast", isBiast());
-		json.add("centerBandFrequency", centerBandFrequency);
-		json.add("rtlDeviceId", getRtlDeviceId());
-		json.add("ppm", getPpm());
 
 		if (getChannelA() != null) {
 			json.add("channelA", getChannelA());
@@ -448,7 +364,9 @@ public JsonObject toJson(SignedURL signed) {
 			statusToSave = ObservationStatus.UPLOADED;
 		}
 		json.add("status", statusToSave.name());
-
+		if (device != null) {
+			json.add("device", device.toJson());
+		}
 		return json;
 	}
 

src/main/java/ru/r2cloud/model/ObservationStatus.java

@@ -2,9 +2,6 @@
 
 public enum ObservationStatus {
 
-	@Deprecated
-	NEW,
-
 	RECEIVING_DATA, RECEIVED, DECODED, UPLOADED, FAILED
 
 }

src/main/java/ru/r2cloud/model/RotatorConfiguration.java

@@ -68,4 +68,13 @@ public JsonObject toJson() {
 		return json;
 	}
 
+	public static RotatorConfiguration fromJson(JsonObject meta) {
+		RotatorConfiguration result = new RotatorConfiguration();
+		result.setHostname(meta.getString("rotctrldHostname", null));
+		result.setPort(meta.getInt("rotctrldPort", 0));
+		result.setTolerance(meta.getDouble("rotatorTolerance", 0));
+		result.setCycleMillis(meta.getInt("rotatorCycle", 0));
+		return result;
+	}
+
 }

src/main/java/ru/r2cloud/model/Transmitter.java

@@ -529,8 +529,10 @@ public JsonObject toJson() {
 		if (loraLdro != 0) {
 			result.add("loraLdro", loraLdro);
 		}
-		result.add("loraExplicitHeader", loraExplicitHeader);
-		result.add("loraCrc", loraCrc);
+		if (modulation != null && modulation.equals(Modulation.LORA)) {
+			result.add("loraExplicitHeader", loraExplicitHeader);
+			result.add("loraCrc", loraCrc);
+		}
 		if (deviation != 5000) {
 			result.add("deviation", deviation);
 		}

src/main/java/ru/r2cloud/tle/CelestrakClient.java

@@ -13,6 +13,7 @@
 
 import ru.r2cloud.R2Cloud;
 import ru.r2cloud.model.Tle;
+import ru.r2cloud.util.Clock;
 import ru.r2cloud.util.Configuration;
 import ru.r2cloud.util.Util;
 
@@ -21,9 +22,11 @@ public class CelestrakClient {
 	private static final Logger LOG = LoggerFactory.getLogger(CelestrakClient.class);
 
 	private final List<String> urls;
+	private final Clock clock;
 	private final int timeout;
 
-	public CelestrakClient(Configuration props) {
+	public CelestrakClient(Configuration props, Clock clock) {
+		this.clock = clock;
 		this.urls = props.getProperties("tle.urls");
 		this.timeout = props.getInteger("tle.timeout");
 	}
@@ -66,7 +69,7 @@ private Map<String, Tle> downloadTle(String location) {
 						}
 						String noradId = line2.substring(2, 2 + 5).trim();
 						Tle value = new Tle(new String[] { curLine.trim(), line1, line2 });
-						value.setLastUpdateTime(System.currentTimeMillis());
+						value.setLastUpdateTime(clock.millis());
 						value.setSource(obj.getHost());
 						result.put(noradId, value);
 					}