From 2d3a94aec9065f18c7e4aac671ff953d1cd70562 Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 19 Mar 2024 22:00:58 +0700 Subject: [PATCH] night-coding 2 --- .../dynamicpack/IDValidator.java | 4 +- .../com/adamcalculator/dynamicpack/Mod.java | 4 +- .../adamcalculator/dynamicpack/PackUtil.java | 18 ++-- .../dynamicpack/pack/DynamicRepoRemote.java | 14 ++- .../pack/DynamicRepoSyncProcessV1.java | 102 ++++++++++-------- .../dynamicpack/pack/ModrinthRemote.java | 24 ++--- .../dynamicpack/sync/SyncingTask.java | 10 +- .../dynamicpack/util/AFiles.java | 80 +++++++++----- .../dynamicpack/util/Hashes.java | 7 +- .../adamcalculator/dynamicpack/util/Out.java | 14 +-- .../adamcalculator/dynamicpack/util/Urls.java | 56 +++++++--- ...erifier.java => GPGSignatureVerifier.java} | 4 +- .../java/tests/SecurityTrustedUrlsTest.java | 16 +-- common/src/test/java/tests/UrlsTest.java | 4 + .../src/test/java/tests/enc/EncodingTest.java | 4 +- 15 files changed, 214 insertions(+), 147 deletions(-) rename common/src/main/java/com/adamcalculator/dynamicpack/util/enc/{GPGDetachedSignatureVerifier.java => GPGSignatureVerifier.java} (97%) create mode 100644 common/src/test/java/tests/UrlsTest.java diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/IDValidator.java b/common/src/main/java/com/adamcalculator/dynamicpack/IDValidator.java index 408498f..2e7e31f 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/IDValidator.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/IDValidator.java @@ -4,10 +4,10 @@ import java.util.regex.Pattern; public class IDValidator { - private static final Pattern pattern = Pattern.compile("^[a-z0-9A-Z]{0,100}$"); + private static final Pattern PATTERN_ID = Pattern.compile("^[a-z0-9A-Z]{0,100}$"); public static boolean isValid(String input) { - Matcher matcher = pattern.matcher(input); + Matcher matcher = PATTERN_ID.matcher(input); return matcher.matches(); } } diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/Mod.java b/common/src/main/java/com/adamcalculator/dynamicpack/Mod.java index b494197..09d28e3 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/Mod.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/Mod.java @@ -66,12 +66,12 @@ private static boolean isLocalHostAllowed() { return false; } - // file_debug_only:// allowed + // file_debug_only:// allowed RELEASE=false public static boolean isFileDebugSchemeAllowed() { return false; } - // http:// allowed + // http:// allowed RELEASE=false public static boolean isHTTPTrafficAllowed() { return false; } diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/PackUtil.java b/common/src/main/java/com/adamcalculator/dynamicpack/PackUtil.java index 9a97155..fa7b5b5 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/PackUtil.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/PackUtil.java @@ -1,12 +1,17 @@ package com.adamcalculator.dynamicpack; -import com.adamcalculator.dynamicpack.util.AFiles; import org.json.JSONObject; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.nio.file.*; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -32,13 +37,6 @@ public static String readString(Path path) throws IOException { return Files.readString(path, StandardCharsets.UTF_8); } - - public static void addFileToZip(File zipFile, String name, String text) throws IOException { - openPackFileSystem(zipFile, path -> { - AFiles.nioWriteText(path.resolve(name), text); - }); - } - public static void openPackFileSystem(File pack, Consumer consumer) throws IOException { if (!pack.exists()) { throw new FileNotFoundException(pack.getCanonicalPath()); diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoRemote.java b/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoRemote.java index 859192e..7b81b83 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoRemote.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoRemote.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; +import java.util.function.LongConsumer; public class DynamicRepoRemote extends Remote { public static final String REPO_JSON = "dynamicmcpack.repo.json"; @@ -68,17 +69,22 @@ public boolean sync(PackSyncProgress progress) throws IOException, NoSuchAlgorit public void sync0(PackSyncProgress progress, Path path) throws IOException, NoSuchAlgorithmException { String packUrlContent; - progress.downloading(REPO_JSON, 0); + + LongConsumer parseProgress = new FileDownloadConsumer() { + @Override + public void onUpdate(FileDownloadConsumer it) { + progress.downloading(REPO_JSON, it.getPercentage()); + } + }; if (skipSign) { - packUrlContent = Urls.parseContent(packUrl, Mod.MOD_FILES_LIMIT); + packUrlContent = Urls.parseContent(packUrl, Mod.MOD_FILES_LIMIT, parseProgress); Out.warn("Dynamic pack " + parent.getName() + " is skipping signing."); progress.textLog("File parsed, verify skipped."); } else { - packUrlContent = Urls.parseContentAndVerify(packSigUrl, packUrl, publicKey, Mod.MOD_FILES_LIMIT); + packUrlContent = Urls.parseContentAndVerify(packSigUrl, packUrl, publicKey, Mod.MOD_FILES_LIMIT, parseProgress); progress.textLog("Success parse and verify file."); } - progress.downloading(REPO_JSON, 100); JSONObject repoJson = new JSONObject(packUrlContent); long formatVersion; diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoSyncProcessV1.java b/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoSyncProcessV1.java index 2b4f23b..e6a2572 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoSyncProcessV1.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoSyncProcessV1.java @@ -8,7 +8,6 @@ import com.adamcalculator.dynamicpack.sync.state.StateFileDeleted; import com.adamcalculator.dynamicpack.util.AFiles; import com.adamcalculator.dynamicpack.util.Hashes; -import com.adamcalculator.dynamicpack.util.Out; import com.adamcalculator.dynamicpack.util.Urls; import org.json.JSONArray; import org.json.JSONObject; @@ -17,7 +16,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -27,27 +25,26 @@ public class DynamicRepoSyncProcessV1 { private final Pack parent; private final DynamicRepoRemote remote; private final PackSyncProgress progress; - private final JSONObject j; + private final JSONObject repoJson; private final Set oldestFilesList = new HashSet<>(); - private Path packRootPath; + private final Path packRootPath; - public DynamicRepoSyncProcessV1(Pack pack, DynamicRepoRemote dynamicRepoRemote, PackSyncProgress progress, JSONObject j, Path path) throws IOException { + public DynamicRepoSyncProcessV1(Pack pack, DynamicRepoRemote dynamicRepoRemote, PackSyncProgress progress, JSONObject repoJson, Path path) { this.parent = pack; this.remote = dynamicRepoRemote; this.progress = progress; - this.j = j; + this.repoJson = repoJson; this.packRootPath = path; } - public void run() throws IOException, NoSuchAlgorithmException { + public void run() throws IOException { PackUtil.walkScan(oldestFilesList, packRootPath); - List jsonObjects = calcActiveContents(); + List activeContents = calcActiveContents(); - for (JSONObject jsonObject : jsonObjects) { - Out.println("process activeContent: " + jsonObject); - processContent(jsonObject); + for (JSONObject jsonContent : activeContents) { + processContent(jsonContent); } for (String s : oldestFilesList) { @@ -56,21 +53,22 @@ public void run() throws IOException, NoSuchAlgorithmException { progress.stateChanged(new StateFileDeleted(path)); progress.textLog("File deleted from resource-pack: " + s); - AFiles.noEmptyDirDelete(path); + AFiles.nioSmartDelete(path); } } public void close() throws IOException { } - private void processContent(JSONObject object) throws IOException, NoSuchAlgorithmException { - String id = object.getString("id"); - String contentRemoteHash = object.getString("hash"); + private void processContent(JSONObject jsonContent) throws IOException { + String id = jsonContent.getString("id"); if (!IDValidator.isValid(id)) { throw new RuntimeException("Id of content is not valid."); } - String url = object.getString("url"); - String urlCompressed = object.optString("url_compressed", null); + + String contentRemoteHash = jsonContent.getString("hash"); + String url = jsonContent.getString("url"); + String urlCompressed = jsonContent.optString("url_compressed", null); boolean compressSupported = urlCompressed != null; checkPathSafety(url); @@ -82,51 +80,62 @@ private void processContent(JSONObject object) throws IOException, NoSuchAlgorit } progress.textLog("process content id:" + id); - progress.downloading(".json", 0); - String content = compressSupported ? Urls.parseGZipContent(urlCompressed, Mod.GZIP_LIMIT) : Urls.parseContent(url, Mod.MOD_FILES_LIMIT); + var contentDownloadProgress = new FileDownloadConsumer() { + @Override + public void onUpdate(FileDownloadConsumer it) { + progress.downloading(".json", it.getPercentage()); + } + }; + String content = compressSupported ? Urls.parseGZipContent(urlCompressed, Mod.GZIP_LIMIT, contentDownloadProgress) : Urls.parseContent(url, Mod.MOD_FILES_LIMIT, contentDownloadProgress); String receivedHash = Hashes.calcHashForBytes(content.getBytes(StandardCharsets.UTF_8)); if (!contentRemoteHash.equals(receivedHash)) { throw new SecurityException("Hash of content at " + url + " not verified. remote: " + contentRemoteHash + "; received: " + receivedHash); } - progress.downloading(".json", 100); processContentParsed(new JSONObject(content)); } - private void processContentParsed(JSONObject j) throws IOException { - if (j.getLong("formatVersion") != 1) { - throw new RuntimeException("Incompatible formatVersion"); + private void processContentParsed(JSONObject jsonContent) throws IOException { + long formatVersion; + if ((formatVersion = jsonContent.getLong("formatVersion")) != 1) { + throw new RuntimeException("Incompatible formatVersion: " + formatVersion); } - JSONObject c = j.getJSONObject("content"); + JSONObject c = jsonContent.getJSONObject("content"); String par = c.optString("parent", ""); + String rem = c.optString("remote_parent", ""); JSONObject files = c.getJSONObject("files"); + int processedFiles = 0; - for (String localPath : files.keySet()) { - String path = getPath(par, localPath); - String realUrl = getUrlFromPath(path); - JSONObject fileExtra = files.getJSONObject(localPath); - String hash = fileExtra.getString("hash"); + for (final String _relativePath : files.keySet()) { + var path = getAndCheckPath(par, _relativePath); // parent / path. assets/minecraft + var filePath = packRootPath.resolve(path); + var fileRemoteUrl = getUrlFromPath(rem, path); + JSONObject fileExtra = files.getJSONObject(_relativePath); + String hash = fileExtra.getString("hash"); - Path filePath = packRootPath.resolve(path); + // remove from unused list oldestFilesList.remove(filePath.toString()); boolean isOverwrite = false; - if (!Files.notExists(filePath)) { - String localHash = Hashes.calcHashForInputStream(Files.newInputStream(filePath)); + if (Files.exists(filePath)) { + String localHash = Hashes.nioCalcHashForPath(filePath); if (!localHash.equals(hash)) { isOverwrite = true; - this.progress.textLog("hash not equal: local:" + localHash+ " remote:"+hash); + this.progress.textLog(filePath + ": overwrite! hash not equal: local:" + localHash+ " remote:"+hash); } } else { - this.progress.textLog("Not exists: " + filePath); + this.progress.textLog("Overwrite! Not exists: " + filePath); isOverwrite = true; } if (isOverwrite) { - if (filePath.getFileName().toString().contains(DynamicPackModBase.CLIENT_FILE)) continue; - this.progress.textLog("(over)write file: " + filePath); - Urls.downloadDynamicFile(realUrl, filePath, hash, new FileDownloadConsumer() { + if (filePath.getFileName().toString().contains(DynamicPackModBase.CLIENT_FILE)) { + continue; + } + + this.progress.textLog("Overwriting: " + filePath); + Urls.downloadDynamicFile(fileRemoteUrl, filePath, hash, new FileDownloadConsumer() { @Override public void onUpdate(FileDownloadConsumer it) { progress.downloading(filePath.getFileName().toString(), it.getPercentage()); @@ -139,12 +148,17 @@ public void onUpdate(FileDownloadConsumer it) { this.progress.textLog("Files processed in this content: " + processedFiles); } - private String getUrlFromPath(String path) { - checkPathSafety(path); - return remote.getUrl() + "/" + path; + private String getUrlFromPath(String remoteParent, String path) { + checkPathSafety(remoteParent); + + if (remoteParent.isEmpty()) { + return remote.getUrl() + "/" + path; + } + + return remote.getUrl() + "/" + remoteParent + "/" + path; } - public static String getPath(String parent, String path) { + public static String getAndCheckPath(String parent, String path) { checkPathSafety(path); checkPathSafety(parent); @@ -155,14 +169,14 @@ public static String getPath(String parent, String path) { } public static void checkPathSafety(String s) { - if (s.contains("://") || s.contains("..") || s.contains(" ") || s.contains(".exe")) { - throw new SecurityException("This url not supported redirects to other servers or jump-up!"); + if (s.contains("://") || s.contains("..") || s.contains(" ") || s.contains(".exe") || s.contains(":") || s.contains(".jar")) { + throw new SecurityException("This url not safe: " + s); } } private List calcActiveContents() { List activeContents = new ArrayList<>(); - JSONArray contents = j.getJSONArray("contents"); + JSONArray contents = repoJson.getJSONArray("contents"); int i = 0; while (i < contents.length()) { JSONObject content = contents.getJSONObject(i); diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/pack/ModrinthRemote.java b/common/src/main/java/com/adamcalculator/dynamicpack/pack/ModrinthRemote.java index 98e2d84..d4642ca 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/pack/ModrinthRemote.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/pack/ModrinthRemote.java @@ -13,7 +13,6 @@ import java.io.File; import java.io.IOException; -import java.util.zip.ZipFile; public class ModrinthRemote extends Remote { private Pack parent; @@ -28,7 +27,7 @@ public ModrinthRemote() { public void init(Pack parent, JSONObject json) { this.parent = parent; this.projectId = json.getString("modrinth_project_id"); - var ver = json.getString("game_version"); + var ver = json.optString("game_version", "current"); this.gameVersion = ver.equalsIgnoreCase("current") ? getCurrentGameVersion() : ver; } @@ -79,10 +78,10 @@ public boolean sync(PackSyncProgress progress) throws IOException { ModrinthRemote.LatestModrinthVersion latest = getLatest(); progress.textLog("downloading..."); - File file = null; + File tempFile = null; int attempts = 3; while (attempts > 0) { - file = Urls.downloadFileToTemp(latest.url, "dynamicpack_download", ".zip", Mod.MODRINTH_HTTPS_FILE_SIZE_LIMIT, new FileDownloadConsumer(){ + tempFile = Urls.downloadFileToTemp(latest.url, "dynamicpack_download", ".zip", Mod.MODRINTH_HTTPS_FILE_SIZE_LIMIT, new FileDownloadConsumer(){ @Override public void onUpdate(FileDownloadConsumer it) { float percentage = it.getPercentage(); @@ -90,7 +89,7 @@ public void onUpdate(FileDownloadConsumer it) { } }); - if (Hashes.calcHashForFile(file).equals(latest.fileHash)) { + if (Hashes.calcHashForFile(tempFile).equals(latest.fileHash)) { progress.textLog("Download done! Hashes is equals."); break; } @@ -100,25 +99,20 @@ public void onUpdate(FileDownloadConsumer it) { throw new RuntimeException("Failed to download correct file from modrinth."); } - ZipFile zipFile = new ZipFile(file); - boolean isDynamicPack = zipFile.getEntry(DynamicPackModBase.CLIENT_FILE) != null; - parent.cachedJson.getJSONObject("current").put("version", latest.latestId); parent.cachedJson.getJSONObject("current").remove("version_number"); - if (!isDynamicPack) { - PackUtil.addFileToZip(file, DynamicPackModBase.CLIENT_FILE, parent.cachedJson.toString(2)); - } + PackUtil.openPackFileSystem(tempFile, path -> AFiles.nioWriteText(path.resolve(DynamicPackModBase.CLIENT_FILE), parent.cachedJson.toString(2))); + progress.textLog("dynamicmcpack.json is updated."); if (parent.isZip()) { - AFiles.moveFile(file, parent.getLocation()); + AFiles.moveFile(tempFile, parent.getLocation()); } else { - AFiles.deleteDirectory(parent.getLocation()); - AFiles.unzip(file, parent.getLocation()); + AFiles.recursiveDeleteDirectory(parent.getLocation()); + AFiles.unzip(tempFile, parent.getLocation()); } - progress.textLog("dynamicmcpack.json is updated."); progress.textLog("done!"); return true; diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/sync/SyncingTask.java b/common/src/main/java/com/adamcalculator/dynamicpack/sync/SyncingTask.java index d661ba7..3b885aa 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/sync/SyncingTask.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/sync/SyncingTask.java @@ -64,21 +64,21 @@ public PackSyncProgress createSyncProgressForPack(Pack pack) { @Override public void start() { - Out.println(pack.getLocation().getName() + ": Sync started."); + Out.println(pack.getName() + ": Sync started."); } @Override public void done(boolean reloadRequired) { - Out.println(pack.getLocation().getName() + ": Sync done. pack reloadRequired=" + reloadRequired); + Out.println(pack.getName() + ": Sync done. pack reloadRequired=" + reloadRequired); if (reloadRequired && !SyncingTask.this.reloadRequired) { try { if (DynamicPackModBase.INSTANCE.isResourcePackActive(pack)) { SyncingTask.this.reloadRequired = true; - Out.println(pack.getLocation().getName() + ": SyncTask.reloadRequired now true!"); + Out.println(pack.getName() + ": SyncTask.reloadRequired now true!"); } } catch (Exception e) { - Out.error(pack.getLocation().getName() + ": SyncTask.reloadRequired now true!", e); + Out.error(pack.getName() + ": SyncTask.reloadRequired now true!", e); SyncingTask.this.reloadRequired = true; } } @@ -86,7 +86,7 @@ public void done(boolean reloadRequired) { @Override public void textLog(String s) { - Out.println(pack.getLocation().getName() + ": [textLog] " + s); + Out.println(pack.getName() + ": [textLog] " + s); } @Override diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/util/AFiles.java b/common/src/main/java/com/adamcalculator/dynamicpack/util/AFiles.java index 2e85e9c..d54b65c 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/util/AFiles.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/util/AFiles.java @@ -2,15 +2,15 @@ import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.model.UnzipParameters; -import org.apache.commons.io.IOUtils; +import org.apache.commons.io.FileUtils; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.Writer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.stream.Stream; @@ -19,18 +19,20 @@ public static File[] lists(File file) { return file.listFiles(); } - public static boolean exists(File dynamic) { - return dynamic.exists(); - } - - public static String read(File file) throws IOException { - return java.nio.file.Files.readString(file.toPath()); - } - + /** + * Move a file source to dest place + * @param source file from + * @param dest file to + */ public static void moveFile(File source, File dest) throws IOException { - Files.move(source.toPath(), dest.toPath()); + Files.move(source.toPath(), dest.toPath(), StandardCopyOption.REPLACE_EXISTING); } + /** + * Extract a zipFilePath to dir + * @param zipFilePath file.zip + * @param dir for example /resourcepacks/Pack1/ + */ public static void unzip(File zipFilePath, File dir) throws IOException { UnzipParameters unzipParameters = new UnzipParameters(); unzipParameters.setExtractSymbolicLinks(false); @@ -39,39 +41,48 @@ public static void unzip(File zipFilePath, File dir) throws IOException { zip.close(); } - public static boolean deleteDirectory(File directoryToBeDeleted) { - File[] allContents = directoryToBeDeleted.listFiles(); - if (allContents != null) { - for (File file : allContents) { - deleteDirectory(file); + + /** + * Force delete a directory + * @param file directory only! + */ + public static void recursiveDeleteDirectory(File file) { + try { + if (!file.isDirectory()) { + throw new RuntimeException("File not a directory."); } + FileUtils.deleteDirectory(file); + + } catch (IOException e) { + throw new RuntimeException("Exception while recursive delete dir " + file, e); } - return directoryToBeDeleted.delete(); } - public static boolean isEmpty(Path path) throws IOException { + private static boolean _nioIsDirExistsAndEmpty(Path path) throws IOException { if (Files.isDirectory(path)) { try (Stream entries = Files.list(path)) { - return !entries.findFirst().isPresent(); + return entries.findFirst().isEmpty(); } } return false; } - public static void noEmptyDirDelete(Path toDel) throws IOException { + + /** + * Delete path and remove empty parent dirs + */ + public static void nioSmartDelete(Path toDel) throws IOException { Path toDelParent = toDel.getParent(); Files.deleteIfExists(toDel); - if (toDelParent != null && isEmpty(toDelParent)) { - noEmptyDirDelete(toDelParent); + if (toDelParent != null && _nioIsDirExistsAndEmpty(toDelParent)) { + nioSmartDelete(toDelParent); } } - public static void write(File file, String string) throws IOException { - FileOutputStream close; - IOUtils.write(string, (close = new FileOutputStream(file)), StandardCharsets.UTF_8); - close.close(); - } + /** + * Write a text to path + */ public static void nioWriteText(Path path, String text) { try (Writer writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) { writer.write(text); @@ -80,4 +91,19 @@ public static void nioWriteText(Path path, String text) { throw new RuntimeException("nioWriteText exception!", e); } } + + /** + * Read text from file + */ + public static String nioReadText(Path path) { + try { + if (!Files.exists(path) || Files.isDirectory(path)) { + throw new RuntimeException("This is not a file. Not found or directory. Cannot be read as text."); + } + return Files.readString(path); + + } catch (IOException e) { + throw new RuntimeException("nioReadText exception!", e); + } + } } diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/util/Hashes.java b/common/src/main/java/com/adamcalculator/dynamicpack/util/Hashes.java index 4014fc9..e43eb6a 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/util/Hashes.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/util/Hashes.java @@ -6,10 +6,15 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; public class Hashes { public static String calcHashForFile(File file) throws IOException { - return DigestUtils.sha1Hex(Files.newInputStream(file.toPath())); + return nioCalcHashForPath(file.toPath()); + } + + public static String nioCalcHashForPath(Path path) throws IOException { + return DigestUtils.sha1Hex(Files.newInputStream(path)); } public static String calcHashForInputStream(InputStream inputStream) throws IOException { diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/util/Out.java b/common/src/main/java/com/adamcalculator/dynamicpack/util/Out.java index c1244cf..f242f0b 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/util/Out.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/util/Out.java @@ -7,9 +7,11 @@ public class Out { public static final Logger LOGGER = LoggerFactory.getLogger("dynamicpack"); + public static boolean ENABLE = true; public static boolean USE_SOUT = false; public static void println(Object o) { + if (!ENABLE) return; if (USE_SOUT) { System.out.println(o); return; @@ -17,16 +19,8 @@ public static void println(Object o) { LOGGER.warn(o + ""); } - public static void e(Exception e) { - if (USE_SOUT) { - System.out.println(e); - e.printStackTrace(); - return; - } - LOGGER.error("Out", e); - } - public static void error(String s, Exception e) { + if (!ENABLE) return; if (USE_SOUT) { System.err.println(s); e.printStackTrace(); @@ -36,6 +30,7 @@ public static void error(String s, Exception e) { } public static void downloading(String url, File file) { + if (!ENABLE) return; if (USE_SOUT) { System.out.println(file.getName() + " downloading from " + url); return; @@ -44,6 +39,7 @@ public static void downloading(String url, File file) { } public static void warn(String s) { + if (!ENABLE) return; if (USE_SOUT) { System.out.println("WARN: " + s); return; diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/util/Urls.java b/common/src/main/java/com/adamcalculator/dynamicpack/util/Urls.java index d424a15..2093e87 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/util/Urls.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/util/Urls.java @@ -2,7 +2,7 @@ import com.adamcalculator.dynamicpack.DynamicPackModBase; import com.adamcalculator.dynamicpack.Mod; -import com.adamcalculator.dynamicpack.util.enc.GPGDetachedSignatureVerifier; +import com.adamcalculator.dynamicpack.util.enc.GPGSignatureVerifier; import java.io.*; import java.net.URL; @@ -22,24 +22,49 @@ public static boolean isHTTPTrafficAllowed() { return Mod.isHTTPTrafficAllowed(); } - public static String parseContentAndVerify(String signatureUrl, String url, String publicKeyBase64, long maxLimit) throws IOException { - boolean isVerified = GPGDetachedSignatureVerifier - .verify(_getInputStreamOfUrl(url, maxLimit), - _getInputStreamOfUrl(signatureUrl, maxLimit), + /** + * parse content and verify it + * @param signatureUrl signature binary url + * @param url url of source file + * @param publicKeyBase64 base64 public key + * @param maxLimit max limit of download + * @param progress progress + * @return parsed content or throw error + * @throws IOException IOException + * @throws SecurityException is signature not valid + */ + public static String parseContentAndVerify(String signatureUrl, String url, String publicKeyBase64, long maxLimit, LongConsumer progress) throws IOException { + InputStream inputStream = _getInputStreamOfUrl(url, maxLimit, progress); + + ByteArrayOutputStream temp = new ByteArrayOutputStream(); + _transferStreams(inputStream, temp, progress); + + boolean isVerified = GPGSignatureVerifier + .verify(new ByteArrayInputStream(temp.toByteArray()), + _getInputStreamOfUrl(signatureUrl, maxLimit, null), publicKeyBase64); if (!isVerified) { throw new SecurityException("Failed to verify " + url + " using signature at " + signatureUrl + " and publicKey: " + publicKeyBase64); } - return _parseContentFromStream(_getInputStreamOfUrl(url, maxLimit), maxLimit); + return _parseContentFromStream(new ByteArrayInputStream(temp.toByteArray()), maxLimit, null); } /** - * Parse text content from url + * Parse text content from url with no progress * @param url url */ public static String parseContent(String url, long limit) throws IOException { - return _parseContentFromStream(_getInputStreamOfUrl(url, limit), limit); + return parseContent(url, limit, null); + } + + /** + * Parse text content from url + * @param url url + * @param progress progress + */ + public static String parseContent(String url, long limit, LongConsumer progress) throws IOException { + return _parseContentFromStream(_getInputStreamOfUrl(url, limit, progress), limit, progress); } @@ -47,8 +72,8 @@ public static String parseContent(String url, long limit) throws IOException { * Parse GZip compressed content from url * @param url url */ - public static String parseGZipContent(String url, long limit) throws IOException { - return _parseContentFromStream(new GZIPInputStream(_getInputStreamOfUrl(url, limit)), limit); + public static String parseGZipContent(String url, long limit, LongConsumer progress) throws IOException { + return _parseContentFromStream(new GZIPInputStream(_getInputStreamOfUrl(url, limit, progress)), limit, progress); } @@ -97,11 +122,6 @@ public static void downloadDynamicFile(String url, Path path, String hash, LongC } - - private static InputStream _getInputStreamOfUrl(String url, long sizeLimit) throws IOException { - return _getInputStreamOfUrl(url, sizeLimit, null); - } - private static InputStream _getInputStreamOfUrl(String url, long sizeLimit, /*@Nullable*/ LongConsumer progress) throws IOException { if (url.contains(" ")) { throw new IOException("URL can't contains spaces!"); @@ -167,7 +187,7 @@ private static InputStream _getInputStreamOfUrl(String url, long sizeLimit, /*@N } } - private static String _parseContentFromStream(InputStream stream, long maxLimit) throws IOException { + private static String _parseContentFromStream(InputStream stream, long maxLimit, /* Nullable */ LongConsumer progress) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); byte[] dataBuffer = new byte[1024]; int bytesRead; @@ -180,6 +200,10 @@ private static String _parseContentFromStream(InputStream stream, long maxLimit) throw new SecurityException("Download limit! " + total + " > " + maxLimit); } + if (progress != null) { + progress.accept(total); + } + Mod.debugNetwork(); } String s = byteArrayOutputStream.toString(StandardCharsets.UTF_8); diff --git a/common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGDetachedSignatureVerifier.java b/common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGSignatureVerifier.java similarity index 97% rename from common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGDetachedSignatureVerifier.java rename to common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGSignatureVerifier.java index 277795b..b390970 100644 --- a/common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGDetachedSignatureVerifier.java +++ b/common/src/main/java/com/adamcalculator/dynamicpack/util/enc/GPGSignatureVerifier.java @@ -47,9 +47,9 @@ import java.nio.charset.StandardCharsets; import java.util.Iterator; -public class GPGDetachedSignatureVerifier { +public class GPGSignatureVerifier { - private GPGDetachedSignatureVerifier() { + private GPGSignatureVerifier() { } public static boolean verify(InputStream signedFileInputStream, InputStream signatureIs, String base64publicKey) throws IOException { diff --git a/common/src/test/java/tests/SecurityTrustedUrlsTest.java b/common/src/test/java/tests/SecurityTrustedUrlsTest.java index b774329..de64342 100644 --- a/common/src/test/java/tests/SecurityTrustedUrlsTest.java +++ b/common/src/test/java/tests/SecurityTrustedUrlsTest.java @@ -21,25 +21,25 @@ public void d() { Assertions.assertDoesNotThrow(() -> { - DynamicRepoSyncProcessV1.getPath("assets", "minecraft/lang/en_us.json"); - DynamicRepoSyncProcessV1.getPath("assets/", "minecraft/lang/en_us.json"); - DynamicRepoSyncProcessV1.getPath("assets", "/minecraft/lang/en_us.json"); - DynamicRepoSyncProcessV1.getPath("assets", "/minecraft/lang/en_us.json"); - DynamicRepoSyncProcessV1.getPath("/assets/", "///minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets", "minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets/", "minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets", "/minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets", "/minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("/assets/", "///minecraft/lang/en_us.json"); }); Assertions.assertThrows(Exception.class, () -> { - DynamicRepoSyncProcessV1.getPath("assets/../../../../", "minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets/../../../../", "minecraft/lang/en_us.json"); }); Assertions.assertThrows(Exception.class, () -> { - DynamicRepoSyncProcessV1.getPath("assets/../../../../", "minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets/../../../../", "minecraft/lang/en_us.json"); }); Assertions.assertThrows(Exception.class, () -> { - DynamicRepoSyncProcessV1.getPath("assets/../../../../", "minecraft/lang/en_us.json"); + DynamicRepoSyncProcessV1.getAndCheckPath("assets/../../../../", "minecraft/lang/en_us.json"); }); } diff --git a/common/src/test/java/tests/UrlsTest.java b/common/src/test/java/tests/UrlsTest.java new file mode 100644 index 0000000..0da8079 --- /dev/null +++ b/common/src/test/java/tests/UrlsTest.java @@ -0,0 +1,4 @@ +package tests; + +public class UrlsTest { +} diff --git a/common/src/test/java/tests/enc/EncodingTest.java b/common/src/test/java/tests/enc/EncodingTest.java index 894ead8..69cdcb7 100644 --- a/common/src/test/java/tests/enc/EncodingTest.java +++ b/common/src/test/java/tests/enc/EncodingTest.java @@ -1,7 +1,7 @@ package tests.enc; import com.adamcalculator.dynamicpack.util.Out; -import com.adamcalculator.dynamicpack.util.enc.GPGDetachedSignatureVerifier; +import com.adamcalculator.dynamicpack.util.enc.GPGSignatureVerifier; import org.apache.commons.codec.binary.Base64InputStream; import org.junit.jupiter.api.Test; @@ -41,7 +41,7 @@ public void d() throws IOException { =08ZH """.getBytes(StandardCharsets.UTF_8))); - var verified = GPGDetachedSignatureVerifier.verify(i1, i2, """ + var verified = GPGSignatureVerifier.verify(i1, i2, """ mQINBGX1YNIBEAC3JgBxOPjy1BaQeHFU9fKaS6pCA647RUryfE7A/9mmhDZV6RyJ miGAw6QKuVCzez8cbQXC6FgEp/2Iq07OzajHsSZW4ibC6GHp1pe7IqiM1Ad4huIX 1fLBXkz3SkIt2Ef/L85rKTCx3SJxGrOPY33+dKfv7cOjwD38DiGKiNVundY+CKox