Skip to content

Commit

Permalink
night-coding 2
Browse files Browse the repository at this point in the history
  • Loading branch information
adam committed Mar 19, 2024
1 parent e054a5e commit 2d3a94a
Show file tree
Hide file tree
Showing 15 changed files with 214 additions and 147 deletions.
4 changes: 2 additions & 2 deletions common/src/main/java/com/adamcalculator/dynamicpack/IDValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import java.util.regex.Pattern;

public class IDValidator {
private static final Pattern pattern = Pattern.compile("^[a-z0-9A-Z]{0,100}$");
private static final Pattern PATTERN_ID = Pattern.compile("^[a-z0-9A-Z]{0,100}$");

public static boolean isValid(String input) {
Matcher matcher = pattern.matcher(input);
Matcher matcher = PATTERN_ID.matcher(input);
return matcher.matches();
}
}
4 changes: 2 additions & 2 deletions common/src/main/java/com/adamcalculator/dynamicpack/Mod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,12 @@ private static boolean isLocalHostAllowed() {
return false;
}

// file_debug_only:// allowed
// file_debug_only:// allowed RELEASE=false
public static boolean isFileDebugSchemeAllowed() {
return false;
}

// http:// allowed
// http:// allowed RELEASE=false
public static boolean isHTTPTrafficAllowed() {
return false;
}
Expand Down
18 changes: 8 additions & 10 deletions common/src/main/java/com/adamcalculator/dynamicpack/PackUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package com.adamcalculator.dynamicpack;

import com.adamcalculator.dynamicpack.util.AFiles;
import org.json.JSONObject;

import java.io.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
Expand All @@ -32,13 +37,6 @@ public static String readString(Path path) throws IOException {
return Files.readString(path, StandardCharsets.UTF_8);
}


public static void addFileToZip(File zipFile, String name, String text) throws IOException {
openPackFileSystem(zipFile, path -> {
AFiles.nioWriteText(path.resolve(name), text);
});
}

public static void openPackFileSystem(File pack, Consumer<Path> consumer) throws IOException {
if (!pack.exists()) {
throw new FileNotFoundException(pack.getCanonicalPath());
Expand Down
14 changes: 10 additions & 4 deletions common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoRemote.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.io.IOException;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.util.function.LongConsumer;

public class DynamicRepoRemote extends Remote {
public static final String REPO_JSON = "dynamicmcpack.repo.json";
Expand Down Expand Up @@ -68,17 +69,22 @@ public boolean sync(PackSyncProgress progress) throws IOException, NoSuchAlgorit

public void sync0(PackSyncProgress progress, Path path) throws IOException, NoSuchAlgorithmException {
String packUrlContent;
progress.downloading(REPO_JSON, 0);

LongConsumer parseProgress = new FileDownloadConsumer() {
@Override
public void onUpdate(FileDownloadConsumer it) {
progress.downloading(REPO_JSON, it.getPercentage());
}
};
if (skipSign) {
packUrlContent = Urls.parseContent(packUrl, Mod.MOD_FILES_LIMIT);
packUrlContent = Urls.parseContent(packUrl, Mod.MOD_FILES_LIMIT, parseProgress);
Out.warn("Dynamic pack " + parent.getName() + " is skipping signing.");
progress.textLog("File parsed, verify skipped.");

} else {
packUrlContent = Urls.parseContentAndVerify(packSigUrl, packUrl, publicKey, Mod.MOD_FILES_LIMIT);
packUrlContent = Urls.parseContentAndVerify(packSigUrl, packUrl, publicKey, Mod.MOD_FILES_LIMIT, parseProgress);
progress.textLog("Success parse and verify file.");
}
progress.downloading(REPO_JSON, 100);

JSONObject repoJson = new JSONObject(packUrlContent);
long formatVersion;
Expand Down
102 changes: 58 additions & 44 deletions common/src/main/java/com/adamcalculator/dynamicpack/pack/DynamicRepoSyncProcessV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import com.adamcalculator.dynamicpack.sync.state.StateFileDeleted;
import com.adamcalculator.dynamicpack.util.AFiles;
import com.adamcalculator.dynamicpack.util.Hashes;
import com.adamcalculator.dynamicpack.util.Out;
import com.adamcalculator.dynamicpack.util.Urls;
import org.json.JSONArray;
import org.json.JSONObject;
Expand All @@ -17,7 +16,6 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
Expand All @@ -27,27 +25,26 @@ public class DynamicRepoSyncProcessV1 {
private final Pack parent;
private final DynamicRepoRemote remote;
private final PackSyncProgress progress;
private final JSONObject j;
private final JSONObject repoJson;

private final Set<String> oldestFilesList = new HashSet<>();
private Path packRootPath;
private final Path packRootPath;

public DynamicRepoSyncProcessV1(Pack pack, DynamicRepoRemote dynamicRepoRemote, PackSyncProgress progress, JSONObject j, Path path) throws IOException {
public DynamicRepoSyncProcessV1(Pack pack, DynamicRepoRemote dynamicRepoRemote, PackSyncProgress progress, JSONObject repoJson, Path path) {
this.parent = pack;
this.remote = dynamicRepoRemote;
this.progress = progress;
this.j = j;
this.repoJson = repoJson;
this.packRootPath = path;
}

public void run() throws IOException, NoSuchAlgorithmException {
public void run() throws IOException {
PackUtil.walkScan(oldestFilesList, packRootPath);

List<JSONObject> jsonObjects = calcActiveContents();
List<JSONObject> activeContents = calcActiveContents();

for (JSONObject jsonObject : jsonObjects) {
Out.println("process activeContent: " + jsonObject);
processContent(jsonObject);
for (JSONObject jsonContent : activeContents) {
processContent(jsonContent);
}

for (String s : oldestFilesList) {
Expand All @@ -56,21 +53,22 @@ public void run() throws IOException, NoSuchAlgorithmException {
progress.stateChanged(new StateFileDeleted(path));

progress.textLog("File deleted from resource-pack: " + s);
AFiles.noEmptyDirDelete(path);
AFiles.nioSmartDelete(path);
}
}

public void close() throws IOException {
}

private void processContent(JSONObject object) throws IOException, NoSuchAlgorithmException {
String id = object.getString("id");
String contentRemoteHash = object.getString("hash");
private void processContent(JSONObject jsonContent) throws IOException {
String id = jsonContent.getString("id");
if (!IDValidator.isValid(id)) {
throw new RuntimeException("Id of content is not valid.");
}
String url = object.getString("url");
String urlCompressed = object.optString("url_compressed", null);

String contentRemoteHash = jsonContent.getString("hash");
String url = jsonContent.getString("url");
String urlCompressed = jsonContent.optString("url_compressed", null);
boolean compressSupported = urlCompressed != null;

checkPathSafety(url);
Expand All @@ -82,51 +80,62 @@ private void processContent(JSONObject object) throws IOException, NoSuchAlgorit
}

progress.textLog("process content id:" + id);
progress.downloading("<content>.json", 0);
String content = compressSupported ? Urls.parseGZipContent(urlCompressed, Mod.GZIP_LIMIT) : Urls.parseContent(url, Mod.MOD_FILES_LIMIT);
var contentDownloadProgress = new FileDownloadConsumer() {
@Override
public void onUpdate(FileDownloadConsumer it) {
progress.downloading("<content:"+ id +">.json", it.getPercentage());
}
};
String content = compressSupported ? Urls.parseGZipContent(urlCompressed, Mod.GZIP_LIMIT, contentDownloadProgress) : Urls.parseContent(url, Mod.MOD_FILES_LIMIT, contentDownloadProgress);
String receivedHash = Hashes.calcHashForBytes(content.getBytes(StandardCharsets.UTF_8));
if (!contentRemoteHash.equals(receivedHash)) {
throw new SecurityException("Hash of content at " + url + " not verified. remote: " + contentRemoteHash + "; received: " + receivedHash);
}
progress.downloading("<content>.json", 100);
processContentParsed(new JSONObject(content));
}

private void processContentParsed(JSONObject j) throws IOException {
if (j.getLong("formatVersion") != 1) {
throw new RuntimeException("Incompatible formatVersion");
private void processContentParsed(JSONObject jsonContent) throws IOException {
long formatVersion;
if ((formatVersion = jsonContent.getLong("formatVersion")) != 1) {
throw new RuntimeException("Incompatible formatVersion: " + formatVersion);
}

JSONObject c = j.getJSONObject("content");
JSONObject c = jsonContent.getJSONObject("content");
String par = c.optString("parent", "");
String rem = c.optString("remote_parent", "");
JSONObject files = c.getJSONObject("files");

int processedFiles = 0;
for (String localPath : files.keySet()) {
String path = getPath(par, localPath);
String realUrl = getUrlFromPath(path);
JSONObject fileExtra = files.getJSONObject(localPath);
String hash = fileExtra.getString("hash");
for (final String _relativePath : files.keySet()) {
var path = getAndCheckPath(par, _relativePath); // parent / path. assets/minecraft
var filePath = packRootPath.resolve(path);
var fileRemoteUrl = getUrlFromPath(rem, path);

JSONObject fileExtra = files.getJSONObject(_relativePath);
String hash = fileExtra.getString("hash");

Path filePath = packRootPath.resolve(path);
// remove from unused list
oldestFilesList.remove(filePath.toString());

boolean isOverwrite = false;
if (!Files.notExists(filePath)) {
String localHash = Hashes.calcHashForInputStream(Files.newInputStream(filePath));
if (Files.exists(filePath)) {
String localHash = Hashes.nioCalcHashForPath(filePath);
if (!localHash.equals(hash)) {
isOverwrite = true;
this.progress.textLog("hash not equal: local:" + localHash+ " remote:"+hash);
this.progress.textLog(filePath + ": overwrite! hash not equal: local:" + localHash+ " remote:"+hash);
}
} else {
this.progress.textLog("Not exists: " + filePath);
this.progress.textLog("Overwrite! Not exists: " + filePath);
isOverwrite = true;
}

if (isOverwrite) {
if (filePath.getFileName().toString().contains(DynamicPackModBase.CLIENT_FILE)) continue;
this.progress.textLog("(over)write file: " + filePath);
Urls.downloadDynamicFile(realUrl, filePath, hash, new FileDownloadConsumer() {
if (filePath.getFileName().toString().contains(DynamicPackModBase.CLIENT_FILE)) {
continue;
}

this.progress.textLog("Overwriting: " + filePath);
Urls.downloadDynamicFile(fileRemoteUrl, filePath, hash, new FileDownloadConsumer() {
@Override
public void onUpdate(FileDownloadConsumer it) {
progress.downloading(filePath.getFileName().toString(), it.getPercentage());
Expand All @@ -139,12 +148,17 @@ public void onUpdate(FileDownloadConsumer it) {
this.progress.textLog("Files processed in this content: " + processedFiles);
}

private String getUrlFromPath(String path) {
checkPathSafety(path);
return remote.getUrl() + "/" + path;
private String getUrlFromPath(String remoteParent, String path) {
checkPathSafety(remoteParent);

if (remoteParent.isEmpty()) {
return remote.getUrl() + "/" + path;
}

return remote.getUrl() + "/" + remoteParent + "/" + path;
}

public static String getPath(String parent, String path) {
public static String getAndCheckPath(String parent, String path) {
checkPathSafety(path);
checkPathSafety(parent);

Expand All @@ -155,14 +169,14 @@ public static String getPath(String parent, String path) {
}

public static void checkPathSafety(String s) {
if (s.contains("://") || s.contains("..") || s.contains(" ") || s.contains(".exe")) {
throw new SecurityException("This url not supported redirects to other servers or jump-up!");
if (s.contains("://") || s.contains("..") || s.contains(" ") || s.contains(".exe") || s.contains(":") || s.contains(".jar")) {
throw new SecurityException("This url not safe: " + s);
}
}

private List<JSONObject> calcActiveContents() {
List<JSONObject> activeContents = new ArrayList<>();
JSONArray contents = j.getJSONArray("contents");
JSONArray contents = repoJson.getJSONArray("contents");
int i = 0;
while (i < contents.length()) {
JSONObject content = contents.getJSONObject(i);
Expand Down
24 changes: 9 additions & 15 deletions common/src/main/java/com/adamcalculator/dynamicpack/pack/ModrinthRemote.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

import java.io.File;
import java.io.IOException;
import java.util.zip.ZipFile;

public class ModrinthRemote extends Remote {
private Pack parent;
Expand All @@ -28,7 +27,7 @@ public ModrinthRemote() {
public void init(Pack parent, JSONObject json) {
this.parent = parent;
this.projectId = json.getString("modrinth_project_id");
var ver = json.getString("game_version");
var ver = json.optString("game_version", "current");
this.gameVersion = ver.equalsIgnoreCase("current") ? getCurrentGameVersion() : ver;
}

Expand Down Expand Up @@ -79,18 +78,18 @@ public boolean sync(PackSyncProgress progress) throws IOException {
ModrinthRemote.LatestModrinthVersion latest = getLatest();

progress.textLog("downloading...");
File file = null;
File tempFile = null;
int attempts = 3;
while (attempts > 0) {
file = Urls.downloadFileToTemp(latest.url, "dynamicpack_download", ".zip", Mod.MODRINTH_HTTPS_FILE_SIZE_LIMIT, new FileDownloadConsumer(){
tempFile = Urls.downloadFileToTemp(latest.url, "dynamicpack_download", ".zip", Mod.MODRINTH_HTTPS_FILE_SIZE_LIMIT, new FileDownloadConsumer(){
@Override
public void onUpdate(FileDownloadConsumer it) {
float percentage = it.getPercentage();
progress.downloading("Modrinth pack (zip)", percentage);
}
});

if (Hashes.calcHashForFile(file).equals(latest.fileHash)) {
if (Hashes.calcHashForFile(tempFile).equals(latest.fileHash)) {
progress.textLog("Download done! Hashes is equals.");
break;
}
Expand All @@ -100,25 +99,20 @@ public void onUpdate(FileDownloadConsumer it) {
throw new RuntimeException("Failed to download correct file from modrinth.");
}

ZipFile zipFile = new ZipFile(file);
boolean isDynamicPack = zipFile.getEntry(DynamicPackModBase.CLIENT_FILE) != null;

parent.cachedJson.getJSONObject("current").put("version", latest.latestId);
parent.cachedJson.getJSONObject("current").remove("version_number");


if (!isDynamicPack) {
PackUtil.addFileToZip(file, DynamicPackModBase.CLIENT_FILE, parent.cachedJson.toString(2));
}
PackUtil.openPackFileSystem(tempFile, path -> AFiles.nioWriteText(path.resolve(DynamicPackModBase.CLIENT_FILE), parent.cachedJson.toString(2)));
progress.textLog("dynamicmcpack.json is updated.");

if (parent.isZip()) {
AFiles.moveFile(file, parent.getLocation());
AFiles.moveFile(tempFile, parent.getLocation());

} else {
AFiles.deleteDirectory(parent.getLocation());
AFiles.unzip(file, parent.getLocation());
AFiles.recursiveDeleteDirectory(parent.getLocation());
AFiles.unzip(tempFile, parent.getLocation());
}
progress.textLog("dynamicmcpack.json is updated.");

progress.textLog("done!");
return true;
Expand Down
Loading

0 comments on commit 2d3a94a

Please sign in to comment.