From cc36281620d8ab370a3aa931b0e52f379aadaa59 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:23:34 -0500 Subject: [PATCH 001/124] Ignore env directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2da080a4..cc0bed69 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ previous_spacehaven_path.txt quicklaunch_*.jar quicklaunch_*/ extra_mods_path.txt +env/ From 6844c6eb0bd4001a15e83ab4a9325233773c56b1 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:31:16 -0500 Subject: [PATCH 002/124] Add vscode launch configuration --- .vscode/launch.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..c17b327b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Startup file", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/spacehaven-modloader.py", + "console": "integratedTerminal" + } + ] +} \ No newline at end of file From da8f056017d5632d3dfb6a549ab0ee3f1ed557f8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:33:09 -0500 Subject: [PATCH 003/124] Break reused code into function --- loader/assets/merge.py | 119 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index db4e9923..622973eb 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -147,61 +147,7 @@ def _mod_path(filename): if not os.path.exists(mod_file): continue - # Do an element-wise merge (replacing conflicts) - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Randomizer", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/GOAPAction", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/BackPack", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Element", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Product", idAttribute="eid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DataLogFragment", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/RandomShip", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/IsoFX", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Item", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/SubCat", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Monster", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/PersonalitySettings", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Encounter", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CostGroup", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterSet", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DifficultySettings", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Room", idAttribute="rid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/ObjectiveCollection", idAttribute="nid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Notes", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DialogChoice", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Faction", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CelestialObject", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Character", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Craft", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Sector", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DataLog", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Plan", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/BackStory", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DefaultStuff", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/TradingValues", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterTrait", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Effect", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterCondition", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Ship", idAttribute="rid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/IdleAnim", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/MainCat", idAttribute="id") - - mergeDefinitions(coreLibrary, modLibrary, file="library/texts", xpath="/t", idAttribute="id") - - # do that before merging animations and textures because references might have to be remapped! - coreLibrary['_all_modded_textures'].update(_detect_textures(coreLibrary, modLibrary, mod)) - - # this way the last mod loaded will overwrite previous textures - #FIXME reimplement this test - # if region_id in all_modded_textures: - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # continue - - mergeDefinitions(coreLibrary, modLibrary, file="library/animations", xpath="/AllAnimations/animations", idAttribute="n") - mergeDefinitions(coreLibrary, modLibrary, file="library/textures", xpath="/AllTexturesAndRegions/textures", idAttribute="i") - mergeDefinitions(coreLibrary, modLibrary, file="library/textures", xpath="/AllTexturesAndRegions/regions", idAttribute="n") - + doMerges(coreLibrary, modLibrary, mod) ui.log.updateLaunchState("Updating XML") @@ -259,6 +205,69 @@ def _mod_path(filename): return extra_assets +def doMerges(coreLib, modLib, mod): + def mergeShim(file, xpath, idAttribute): + mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + + havenIDLookUpTable = { + "/data/Randomizer": "id", + "/data/GOAPAction": "id", + "/data/BackPack": "mid", + "/data/Element": "mid", + "/data/Product": "eid", + "/data/DataLogFragment": "id", + "/data/RandomShip": "id", + "/data/IsoFX": "id", + "/data/Item": "mid", + "/data/SubCat": "id", + "/data/Monster": "cid", + "/data/PersonalitySettings": "id", + "/data/Encounter": "id", + "/data/CostGroup": "id", + "/data/CharacterSet": "cid", + "/data/DifficultySettings": "id", + "/data/Room": "rid", + "/data/ObjectiveCollection": "nid", + "/data/Notes": "id", + "/data/DialogChoice": "id", + "/data/Faction": "id", + "/data/CelestialObject": "id", + "/data/Character": "cid", + "/data/Craft": "cid", + "/data/Sector": "id", + "/data/DataLog": "id", + "/data/Plan": "id", + "/data/BackStory": "id", + "/data/DefaultStuff": "id", + "/data/TradingValues": "id", + "/data/CharacterTrait": "id", + "/data/Effect": "id", + "/data/CharacterCondition": "id", + "/data/Ship": "rid", + "/data/IdleAnim": "id", + "/data/MainCat": "id", + } + + # Do an element-wise merge (replacing conflicts) + for k,v in havenIDLookUpTable.items(): mergeShim("library/haven", k, v) + mergeShim("library/texts", "/t", idAttribute="id") + + # do that before merging animations and textures because references might have to be remapped! + coreLib['_all_modded_textures'].update(_detect_textures(coreLib, modLib, mod)) + + # this way the last mod loaded will overwrite previous textures + #FIXME reimplement this test + # if region_id in all_modded_textures: + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # continue + + mergeShim("library/animations", "/AllAnimations/animations", "n") + mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") + mergeShim("library/textures", "/AllTexturesAndRegions/regions", "n") + + def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): if not file in modLibrary: ui.log.log(" {}: Not present".format(file)) From 88881d8641bde4458e0db3525e06452f5f819816 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:42:37 -0500 Subject: [PATCH 004/124] Documentation --- loader/assets/merge.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 622973eb..bba72a22 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -205,10 +205,13 @@ def _mod_path(filename): return extra_assets -def doMerges(coreLib, modLib, mod): - def mergeShim(file, xpath, idAttribute): +def doMerges(coreLib, modLib, mod: str): + """Do merge-based modding sequence""" + def mergeShim(file: str, xpath: str, idAttribute: str): + '''Shim to reduce function call complexity''' mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { "/data/Randomizer": "id", "/data/GOAPAction": "id", From ce66c434f4ef9f92cbbfd68a381280b194dd0b32 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:43:50 -0500 Subject: [PATCH 005/124] Prevent indetation error down the line --- loader/assets/merge.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index bba72a22..489bb21b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -260,11 +260,11 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # this way the last mod loaded will overwrite previous textures #FIXME reimplement this test - # if region_id in all_modded_textures: - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # continue + #if region_id in all_modded_textures: + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # continue mergeShim("library/animations", "/AllAnimations/animations", "n") mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") From b3254188fa63a3bf7d56bc5185071ba9eccf3719 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:48:11 -0500 Subject: [PATCH 006/124] Sort LUT to make it easier on coders --- loader/assets/merge.py | 54 +++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 489bb21b..e99a59c1 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -213,42 +213,42 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { - "/data/Randomizer": "id", - "/data/GOAPAction": "id", "/data/BackPack": "mid", - "/data/Element": "mid", - "/data/Product": "eid", - "/data/DataLogFragment": "id", - "/data/RandomShip": "id", - "/data/IsoFX": "id", - "/data/Item": "mid", - "/data/SubCat": "id", - "/data/Monster": "cid", - "/data/PersonalitySettings": "id", - "/data/Encounter": "id", - "/data/CostGroup": "id", - "/data/CharacterSet": "cid", - "/data/DifficultySettings": "id", - "/data/Room": "rid", - "/data/ObjectiveCollection": "nid", - "/data/Notes": "id", - "/data/DialogChoice": "id", - "/data/Faction": "id", + "/data/BackStory": "id", "/data/CelestialObject": "id", "/data/Character": "cid", + "/data/CharacterCondition": "id", + "/data/CharacterSet": "cid", + "/data/CharacterTrait": "id", + "/data/CostGroup": "id", "/data/Craft": "cid", - "/data/Sector": "id", "/data/DataLog": "id", - "/data/Plan": "id", - "/data/BackStory": "id", + "/data/DataLogFragment": "id", "/data/DefaultStuff": "id", - "/data/TradingValues": "id", - "/data/CharacterTrait": "id", + "/data/DialogChoice": "id", + "/data/DifficultySettings": "id", "/data/Effect": "id", - "/data/CharacterCondition": "id", - "/data/Ship": "rid", + "/data/Element": "mid", + "/data/Encounter": "id", + "/data/Faction": "id", + "/data/GOAPAction": "id", "/data/IdleAnim": "id", + "/data/IsoFX": "id", + "/data/Item": "mid", "/data/MainCat": "id", + "/data/Monster": "cid", + "/data/Notes": "id", + "/data/ObjectiveCollection": "nid", + "/data/PersonalitySettings": "id", + "/data/Plan": "id", + "/data/Product": "eid", + "/data/RandomShip": "id", + "/data/Randomizer": "id", + "/data/Room": "rid", + "/data/Sector": "id", + "/data/Ship": "rid", + "/data/SubCat": "id", + "/data/TradingValues": "id", } # Do an element-wise merge (replacing conflicts) From 752700f389cc8bf377e85d4ea2937ebe1a357b35 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:50:53 -0500 Subject: [PATCH 007/124] Remove blank whitespace --- loader/assets/merge.py | 76 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e99a59c1..4a6a6711 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -13,23 +13,23 @@ def _detect_textures(coreLibrary, modLibrary, mod): textures_path = os.path.join(mod, 'textures') if not os.path.isdir(textures_path): return {} - + mapping_n_region = {} modded_textures = {} seen_textures = set() - + def _add_texture(region_id): filename = region_id + '.png' if filename in seen_textures: return - + path = os.path.join(textures_path, filename) if not os.path.isfile(path): ui.log.log(" ERROR MISSING {}...".format(filename)) ui.log.log(" ERROR MISSING {}...".format(filename)) ui.log.log(" ERROR MISSING {}...".format(filename)) return - + ui.log.log(" Found {}...".format(filename)) if int(region_id) > coreLibrary['_last_core_region_id']: # adding a new texture, this gets tricky as they have to have consecutive numbers. @@ -38,14 +38,14 @@ def _add_texture(region_id): coreLibrary['_next_region_id'] += 1 else: core_region_id = region_id - + seen_textures.add(filename) modded_textures[core_region_id] = { 'mapped_from_id' : region_id, 'filename' : filename, 'path' : path, } - + for filename in os.listdir(textures_path): # also scan the directory for overwriting existing core textures if not filename.endswith('.png'): @@ -56,26 +56,26 @@ def _add_texture(region_id): # wrong format continue _add_texture(filename.split('.')[0]) - + if 'library/textures' not in modLibrary: # no textures.xml file, we're done return modded_textures - + #FIXME verify that there's only one file textures_mod = modLibrary['library/textures'][0] - + for texture_pack in textures_mod.xpath("//t[@i]"): cim_id = texture_pack.get('i') coreLibrary['_custom_textures_cim'][cim_id] = texture_pack.attrib - + for region in textures_mod.xpath("//re[@n]"): region_id = region.get('n') _add_texture(region_id) - + if not mapping_n_region: # no custom mod textures, no need to remap ids return modded_textures - + for animation_chunk in modLibrary['library/animations']: for asset in animation_chunk.xpath("//assetPos[@a]"): mod_local_id = asset.get('a') @@ -84,7 +84,7 @@ def _add_texture(region_id): new_id = mapping_n_region[mod_local_id] ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) asset.set('a', new_id) - + for asset in textures_mod.xpath("//re[@n]"): mod_local_id = asset.get('n') if mod_local_id not in mapping_n_region: @@ -92,7 +92,7 @@ def _add_texture(region_id): new_id = mapping_n_region[mod_local_id] ui.log.log(" Mapping texture 're' {} to {}...".format(mod_local_id, new_id)) asset.set('n', new_id) - + return modded_textures @@ -101,33 +101,33 @@ def mods(corePath, modPaths): coreLibrary = {} def _core_path(filename): return os.path.join(corePath, filename.replace('/', os.sep)) - + for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), 'rb') as f: coreLibrary[filename] = lxml.etree.parse(f, parser=lxml.etree.XMLParser(recover=True)) - + # find the last region in the texture file and remember its index # we will need this to add mod textures with consecutive indexes... coreLibrary['_last_core_region_id'] = int(coreLibrary['library/textures'].find("//re[@n][last()]").get('n')) coreLibrary['_next_region_id'] = coreLibrary['_last_core_region_id'] + 1 coreLibrary['_all_modded_textures'] = {} coreLibrary['_custom_textures_cim'] = {} - + # Merge in modded files for mod in modPaths: ui.log.updateLaunchState("Installing {}".format(os.path.basename(mod))) - + ui.log.log(" Loading mod {}...".format(mod)) - + # Load the mod's library modLibrary = {} def _mod_path(filename): return os.path.join(mod, filename.replace('/', os.sep)) - + mod_files = [] for mod_file in os.listdir(_mod_path('library')): mod_files.append('library/' + mod_file) - + # we allow breaking down mod xml files into smaller pieces for readability for target in PATCHABLE_XML_FILES: for mod_file in mod_files: @@ -138,8 +138,8 @@ def _mod_path(filename): ui.log.log("{} => {}".format(mod_file, target)) with open(_mod_path(mod_file)) as f: modLibrary[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) - - + + mod_file = _mod_path(target) if not os.path.exists(mod_file): # try again with the extension ? @@ -150,26 +150,26 @@ def _mod_path(filename): doMerges(coreLibrary, modLibrary, mod) ui.log.updateLaunchState("Updating XML") - + # Write out the new base library for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), "wb") as f: f.write(lxml.etree.tostring(coreLibrary[filename], pretty_print=True, encoding="UTF-8")) - + ui.log.updateLaunchState("Packing textures") # add or overwrite textures from mods. This is done after all the XML has been merged into the core "textures" file cims = {} reexport_cims = {} extra_assets = [] - + for region in coreLibrary['library/textures'].xpath("//re[@n]"): name = region.get("n") - + if name not in coreLibrary['_all_modded_textures']: continue - + png_file = coreLibrary['_all_modded_textures'][name]['path'] - + page = region.get("t") if not page in cims: cim_name = '{}.cim'.format(page) @@ -181,27 +181,27 @@ def _mod_path(filename): kwargs['height'] = coreLibrary['_custom_textures_cim'][page]['h'] extra_assets.append('library/' + cim_name) cims[page] = Texture(os.path.join(corePath, 'library', cim_name), **kwargs) - + reexport_cims[page] = set() - + # write back the cim file as png for debugging reexport_cims[page].add(os.path.dirname(png_file)) - + x = int(region.get("x")) y = int(region.get("y")) w = int(region.get("w")) h = int(region.get("h")) - + ui.log.log(" Patching {}.cim...".format(page)) cims[page].pack_png(png_file, x, y, w, h) - + # cims contains only the textures files that have actually been modified for page in cims: ui.log.log(" Writing {}.cim...".format(page)) cims[page].export_cim(os.path.join(corePath, 'library', '{}.cim'.format(page))) for path in reexport_cims[page]: cims[page].export_png(os.path.join(path, 'modded_cim_{}.png'.format(page))) - + return extra_assets @@ -282,13 +282,13 @@ def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): #that's a big error if we can't find it in the core! ui.log.log(" {}: ERROR CORE NOTHING AT {}".format(file, xpath)) return - + for mod_xml in modLibrary[file]: try: modRoot = mod_xml.xpath(xpath)[0] except: continue - + merged = 0 for element in list(modRoot): conflicts = baseRoot.xpath("*[@{}='{}']".format(idAttribute, element.get(idAttribute))) @@ -298,7 +298,7 @@ def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): baseRoot.append(copy.deepcopy(element)) merged += 1 - + if merged: # TODO add source filename ui.log.log(" {}: Merged {} elements into {}".format(file, merged, xpath)) From 8a187103b30d68f212f6ad903c876b56dc4910cb Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:32:10 -0500 Subject: [PATCH 008/124] Ignore rope metafiles --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cc0bed69..bdf9139e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ quicklaunch_*.jar quicklaunch_*/ extra_mods_path.txt env/ +.vscode/.ropeproject/ From 73c24328d2ac71f53d9b0d7d74caf2459e049e32 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:33:49 -0500 Subject: [PATCH 009/124] Put modLibrary code into a function --- loader/assets/merge.py | 56 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 4a6a6711..e3fdfbb4 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -102,6 +102,35 @@ def mods(corePath, modPaths): def _core_path(filename): return os.path.join(corePath, filename.replace('/', os.sep)) + def buildLibrary(location: str): + """Build up a library dict of files in `location`""" + def _mod_path(filename): + return os.path.join(mod, filename.replace('/', os.sep)) + location_library = {} + try: + location_files = [location + '/' + mod_file for mod_file in os.listdir(_mod_path(location))] + except FileNotFoundError: + location_files = [] + + # we allow breaking down mod xml files into smaller pieces for readability + for target in PATCHABLE_XML_FILES: + targetInLocation = target.replace('library', location) + for mod_file in location_files: + if not mod_file.startswith(targetInLocation): continue + if target not in location_library: location_library[target] = [] + + ui.log.log(" {} <= {}".format(target, mod_file)) + with open(_mod_path(mod_file)) as f: + location_library[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) + + mod_file = _mod_path(target) + # try again with the extension ? + if not os.path.exists(mod_file): + mod_file += '.xml' + if not os.path.exists(mod_file): + continue + return location_library + for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), 'rb') as f: coreLibrary[filename] = lxml.etree.parse(f, parser=lxml.etree.XMLParser(recover=True)) @@ -120,32 +149,7 @@ def _core_path(filename): ui.log.log(" Loading mod {}...".format(mod)) # Load the mod's library - modLibrary = {} - def _mod_path(filename): - return os.path.join(mod, filename.replace('/', os.sep)) - - mod_files = [] - for mod_file in os.listdir(_mod_path('library')): - mod_files.append('library/' + mod_file) - - # we allow breaking down mod xml files into smaller pieces for readability - for target in PATCHABLE_XML_FILES: - for mod_file in mod_files: - if not mod_file.startswith(target): - continue - if target not in modLibrary: - modLibrary[target] = [] - ui.log.log("{} => {}".format(mod_file, target)) - with open(_mod_path(mod_file)) as f: - modLibrary[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) - - - mod_file = _mod_path(target) - if not os.path.exists(mod_file): - # try again with the extension ? - mod_file += '.xml' - if not os.path.exists(mod_file): - continue + modLibrary = buildLibrary('library') doMerges(coreLibrary, modLibrary, mod) From 7a6796a39dc81fb894b81c149d5e511ead1779a5 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:53:32 -0500 Subject: [PATCH 010/124] Reduce log spam --- loader/assets/merge.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e3fdfbb4..88a2c0f6 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -215,6 +215,10 @@ def mergeShim(file: str, xpath: str, idAttribute: str): '''Shim to reduce function call complexity''' mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + def mergeAbortMessage(filename: str): + """Shim to standardize error message""" + ui.log.log(f" No merges needed: {filename}") + # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { "/data/BackPack": "mid", @@ -256,8 +260,15 @@ def mergeShim(file: str, xpath: str, idAttribute: str): } # Do an element-wise merge (replacing conflicts) - for k,v in havenIDLookUpTable.items(): mergeShim("library/haven", k, v) - mergeShim("library/texts", "/t", idAttribute="id") + currentFile = "library/haven" + if currentFile in modLib: + for path, idText in havenIDLookUpTable.items(): mergeShim(currentFile, path, idText) + else: mergeAbortMessage(currentFile) + + currentFile = "library/texts" + if currentFile in modLib: + mergeShim(currentFile, "/t", idAttribute="id") + else: mergeAbortMessage(currentFile) # do that before merging animations and textures because references might have to be remapped! coreLib['_all_modded_textures'].update(_detect_textures(coreLib, modLib, mod)) @@ -270,9 +281,16 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # ui.log.log(" ERROR CONFLICT {}...".format(filename)) # continue - mergeShim("library/animations", "/AllAnimations/animations", "n") - mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") - mergeShim("library/textures", "/AllTexturesAndRegions/regions", "n") + + currentFile = "library/animations" + if currentFile in modLib: + mergeShim(currentFile, "/AllAnimations/animations", "n") + else: mergeAbortMessage(currentFile) + + currentFile = "library/textures" + if currentFile in modLib: + mergeShim(currentFile, "/AllTexturesAndRegions/regions", "n") + else: mergeAbortMessage(currentFile) def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): From 76e679335ac8ad1bc70e25f172d63b3ed8a3e832 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:59:09 -0500 Subject: [PATCH 011/124] Create patch loop --- loader/assets/merge.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 88a2c0f6..5bbdec1e 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -153,6 +153,12 @@ def _mod_path(filename): doMerges(coreLibrary, modLibrary, mod) + # Do patches after merges to avoid clobbers + for mod in modPaths: + ui.log.updateLaunchState(f"Patching {os.path.basename(mod)}") + ui.log.log(f" Loading patches {mod}...") + modPatchesLibrary = buildLibrary('patches') + ui.log.updateLaunchState("Updating XML") # Write out the new base library From 58220f441ddacc69253c9ee78b84a58d2b18642c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 19:43:29 -0500 Subject: [PATCH 012/124] First framework for XPath-patching --- loader/assets/merge.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 5bbdec1e..bf3bc94b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -158,6 +158,7 @@ def _mod_path(filename): ui.log.updateLaunchState(f"Patching {os.path.basename(mod)}") ui.log.log(f" Loading patches {mod}...") modPatchesLibrary = buildLibrary('patches') + doPatches(coreLibrary, modPatchesLibrary, mod) ui.log.updateLaunchState("Updating XML") @@ -214,6 +215,58 @@ def _mod_path(filename): return extra_assets +def doPatches(coreLib, modLib: dict, mod: str): + # Pretyping + patchList : lxml.etree._ElementTree + patchOperation : lxml.etree._Element + + def doPatchType(patch: lxml.etree._Element, location: str): + """Execute a single patch. Provided to reduce indentation level""" + # Pretyping + currentCoreLib : lxml.etree._ElementTree + + pType = patch.attrib["Class"] + xpath = patch.find('xpath').text + value = patch.find('value') + if "Attribute" in pType: + attribute = patch.find("attribute").text + + currentCoreLib = coreLib[location] + currentCoreLibElems = currentCoreLib.xpath(xpath) + + def AttributeSet(): + ui.log.log(f" Set attr {attribute} on node {xpath}") + for elem in currentCoreLibElems: + elem.set(attribute, value.text) + def AttributeAdd(): pass + def AttributeRemove(): pass + + def NodeSet(): pass + def NodeAdd(): pass + def NodeRemove(): pass + def NodeReplace(): + ui.log.log(f" Replacing node {xpath}") + def BadOp(): pass + + patchDispatcher = { + "AttributeSet" : AttributeSet, + "AttributeAdd" : AttributeAdd, + "AttributeRemove" : AttributeRemove, + "Set": NodeSet, + "Add": NodeAdd, + "Remove": NodeRemove, + "Replace": NodeReplace, + } + + patchDispatcher.get(pType,BadOp)() + + + # Execution + for location in modLib: + for patchList in modLib[location]: + for patchOperation in patchList.getroot(): + doPatchType(patchOperation, location) + def doMerges(coreLib, modLib, mod: str): """Do merge-based modding sequence""" From 5ec1b07c2021db97ddfd23ec07aa76e6cc4d1edf Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:08:59 -0500 Subject: [PATCH 013/124] Add the remaining Attribute path operations --- loader/assets/merge.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index bf3bc94b..56d94c30 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -225,6 +225,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Pretyping currentCoreLib : lxml.etree._ElementTree + logIndent = " " * 4 pType = patch.attrib["Class"] xpath = patch.find('xpath').text value = patch.find('value') @@ -233,13 +234,23 @@ def doPatchType(patch: lxml.etree._Element, location: str): currentCoreLib = coreLib[location] currentCoreLibElems = currentCoreLib.xpath(xpath) + if len(currentCoreLibElems) == 0: + ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") + return + # AttributeAdd and AttributeSet are able to use the same code due to + # lxml backing library implementation def AttributeSet(): - ui.log.log(f" Set attr {attribute} on node {xpath}") - for elem in currentCoreLibElems: - elem.set(attribute, value.text) - def AttributeAdd(): pass - def AttributeRemove(): pass + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + def AttributeAdd(): + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + def AttributeRemove(): + ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + ui.log.log(f"{logIndent}Remove '{attribute}' from nodes: {xpath}") + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.attrib.pop(attribute) def NodeSet(): pass def NodeAdd(): pass From da51aa683495f5c5457d482dc547a1c4d24b7297 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:26:34 -0500 Subject: [PATCH 014/124] Cleanly handle failing patch --- loader/assets/merge.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 56d94c30..2bceba92 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -276,7 +276,14 @@ def BadOp(): pass for location in modLib: for patchList in modLib[location]: for patchOperation in patchList.getroot(): - doPatchType(patchOperation, location) + try: + doPatchType(patchOperation, location) + except Exception as e: + uri = patchOperation.base + line = patchOperation.sourceline + ui.log.log(f" Failed to apply patch operation {uri}:{line}") + ui.log.log(f" Reason: {repr(e)}") + raise SyntaxError("Issue in patch operation. Check logs for info.") from None def doMerges(coreLib, modLib, mod: str): From e0b9b26e5666d291b5501ac1927e7dba4794c625 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:28:29 -0500 Subject: [PATCH 015/124] Add editor typing --- loader/assets/merge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2bceba92..ea79bb11 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -224,6 +224,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): """Execute a single patch. Provided to reduce indentation level""" # Pretyping currentCoreLib : lxml.etree._ElementTree + patchOperation : lxml.etree._Element logIndent = " " * 4 pType = patch.attrib["Class"] From 2f28a1549b05e807f8012f0f1e5337ed9bc3d64f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:47:09 -0500 Subject: [PATCH 016/124] Add per-patch logging --- loader/assets/merge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index ea79bb11..87433ba7 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -233,6 +233,8 @@ def doPatchType(patch: lxml.etree._Element, location: str): if "Attribute" in pType: attribute = patch.find("attribute").text + ui.log.log(f"{logIndent}XPATH => {location:>15}: {pType:18}{xpath}") + currentCoreLib = coreLib[location] currentCoreLibElems = currentCoreLib.xpath(xpath) if len(currentCoreLibElems) == 0: @@ -249,7 +251,6 @@ def AttributeAdd(): for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeRemove(): ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") - ui.log.log(f"{logIndent}Remove '{attribute}' from nodes: {xpath}") elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) From dd85011c41328230b3dc02358439edc177b702a8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:13:48 -0500 Subject: [PATCH 017/124] Implement XPath node replace --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 87433ba7..47d8c306 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -258,7 +258,11 @@ def NodeSet(): pass def NodeAdd(): pass def NodeRemove(): pass def NodeReplace(): - ui.log.log(f" Replacing node {xpath}") + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.replace(elem, copy.deepcopy(value[0])) def BadOp(): pass patchDispatcher = { From 1aad8d5b3a9ef18b7cc964ea9feab75a1bde2117 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:33:26 -0500 Subject: [PATCH 018/124] Change Set operation to Insert, Change pass to NotImplemented --- loader/assets/merge.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 47d8c306..4f4188e3 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -254,30 +254,30 @@ def AttributeRemove(): elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) - def NodeSet(): pass - def NodeAdd(): pass - def NodeRemove(): pass + def NodeAdd(): raise NotImplementedError() + def NodeInsert(): raise NotImplementedError() + def NodeRemove(): raise NotImplementedError() def NodeReplace(): elem : lxml.etree._Element parent: lxml.etree._Element for elem in currentCoreLibElems: parent = elem.find('./..') parent.replace(elem, copy.deepcopy(value[0])) - def BadOp(): pass + def BadOp(): + raise SyntaxError(f"BAD PATCH OPERATION") patchDispatcher = { "AttributeSet" : AttributeSet, "AttributeAdd" : AttributeAdd, "AttributeRemove" : AttributeRemove, - "Set": NodeSet, "Add": NodeAdd, + "Insert": NodeInsert, "Remove": NodeRemove, "Replace": NodeReplace, } patchDispatcher.get(pType,BadOp)() - # Execution for location in modLib: for patchList in modLib[location]: From 61ad411576340f375b82e1849ea00cab91fb2cb0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:34:14 -0500 Subject: [PATCH 019/124] Add documentation --- loader/assets/merge.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 4f4188e3..0b83e0fa 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -220,11 +220,14 @@ def doPatches(coreLib, modLib: dict, mod: str): patchList : lxml.etree._ElementTree patchOperation : lxml.etree._Element + # Helper functions def doPatchType(patch: lxml.etree._Element, location: str): """Execute a single patch. Provided to reduce indentation level""" # Pretyping currentCoreLib : lxml.etree._ElementTree patchOperation : lxml.etree._Element + xpath: str + value: lxml.etree._Element logIndent = " " * 4 pType = patch.attrib["Class"] From 16dac596f634d810b8912055dced74a3791d9cc6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:40:37 -0500 Subject: [PATCH 020/124] Update documentation --- loader/assets/merge.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 0b83e0fa..2e75ca75 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -244,23 +244,31 @@ def doPatchType(patch: lxml.etree._Element, location: str): ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") return - # AttributeAdd and AttributeSet are able to use the same code due to - # lxml backing library implementation def AttributeSet(): + """Set the attribute on the node, adding if not present""" elem : lxml.etree._Element for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeAdd(): + """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeRemove(): + """Remove the attribute from the node""" ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) - def NodeAdd(): raise NotImplementedError() - def NodeInsert(): raise NotImplementedError() - def NodeRemove(): raise NotImplementedError() + def NodeAdd(): + """Adds a provided child node to the selected node""" + raise NotImplementedError() + def NodeInsert(): + """Adds a provided sibling node to the selected node""" + raise NotImplementedError() + def NodeRemove(): + """Deletes the selected node""" + raise NotImplementedError() def NodeReplace(): + """Replaces the selected node with the provided node""" elem : lxml.etree._Element parent: lxml.etree._Element for elem in currentCoreLibElems: From 27459d07205b2292c69cb10d1c9432053b0daa9e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:54:30 -0500 Subject: [PATCH 021/124] Update AttributeAdd to match PatchOperation spec - Spec requires PatchOperationAttributeAdd to fail if attribute alread exists --- loader/assets/merge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2e75ca75..01689838 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -251,7 +251,10 @@ def AttributeSet(): def AttributeAdd(): """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.set(attribute, value.text) + for elem in currentCoreLibElems: + if elem.get(attribute, None) is not None: + raise KeyError(f"Attribute '{attribute}' already exists") + elem.set(attribute, value.text) def AttributeRemove(): """Remove the attribute from the node""" ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") From 0bcb097b1c7b0bd8df11c9abcfcf99adddc49fb2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:55:11 -0500 Subject: [PATCH 022/124] Increase indent level --- loader/assets/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 01689838..05f29495 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -302,7 +302,7 @@ def BadOp(): uri = patchOperation.base line = patchOperation.sourceline ui.log.log(f" Failed to apply patch operation {uri}:{line}") - ui.log.log(f" Reason: {repr(e)}") + ui.log.log(f" Reason: {repr(e)}") raise SyntaxError("Issue in patch operation. Check logs for info.") from None From 0fe9dac8c9f4121e7b978bfa4593e36fa16ca6fd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:01:26 -0500 Subject: [PATCH 023/124] Implement PatchOperationRemove --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 05f29495..3d86dc80 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -269,7 +269,11 @@ def NodeInsert(): raise NotImplementedError() def NodeRemove(): """Deletes the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.remove(elem) def NodeReplace(): """Replaces the selected node with the provided node""" elem : lxml.etree._Element From 177d4c1b55e55a42f1f9dbf32a6ae03e686268a2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:08:24 -0500 Subject: [PATCH 024/124] Implement PatchOperationInsert --- loader/assets/merge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 3d86dc80..c70c9db6 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -266,7 +266,12 @@ def NodeAdd(): raise NotImplementedError() def NodeInsert(): """Adds a provided sibling node to the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + elemIDX = parent.index(elem) + parent.insert(elemIDX + 1, copy.deepcopy(value[0])) def NodeRemove(): """Deletes the selected node""" elem : lxml.etree._Element From f8cb321b2373264a8c08c72c5e951de660c4060d Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:18:14 -0500 Subject: [PATCH 025/124] Implement PatchOperationAdd Insertion action isn't specified in the PatchOperation spec This implementation inserts it at the end of the list. --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index c70c9db6..32ad4c3c 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -263,7 +263,11 @@ def AttributeRemove(): def NodeAdd(): """Adds a provided child node to the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + lastelemIDX = len(elem.getchildren()) + elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) def NodeInsert(): """Adds a provided sibling node to the selected node""" elem : lxml.etree._Element From 50c82eeede51c00827e29f66db4f114d9f0c951e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:18:35 -0500 Subject: [PATCH 026/124] Bump version number --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 9d2c0024..613ac843 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.0.7" +version = "0.0.8" From d670ea187d509293168db5e5a5b60de5467e8188 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 03:59:29 -0500 Subject: [PATCH 027/124] Move PatchOperation functions and switchDict into main scope This should improve performance, though it does mean arguments have to be explicitly passed into each PatchOperation now. Oh well. --- loader/assets/merge.py | 128 ++++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 59 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 32ad4c3c..7b08e07f 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -215,6 +215,71 @@ def _mod_path(filename): return extra_assets + + +def AttributeSet(currentCoreLibElems, value, attribute): + """Set the attribute on the node, adding if not present""" + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + +def AttributeAdd(currentCoreLibElems, value, attribute): + """Adds the attribute to the node IFF the attribute name is not already present""" + elem : lxml.etree._Element + for elem in currentCoreLibElems: + if elem.get(attribute, None) is not None: + raise KeyError(f"Attribute '{attribute}' already exists") + elem.set(attribute, value.text) + +def AttributeRemove(currentCoreLibElems, value, attribute): + """Remove the attribute from the node""" + ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.attrib.pop(attribute) + +def NodeAdd(currentCoreLibElems, value, attribute): + """Adds a provided child node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + lastelemIDX = len(elem.getchildren()) + elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) + +def NodeInsert(currentCoreLibElems, value, attribute): + """Adds a provided sibling node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + elemIDX = parent.index(elem) + parent.insert(elemIDX + 1, copy.deepcopy(value[0])) + +def NodeRemove(currentCoreLibElems, value, attribute): + """Deletes the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.remove(elem) + +def NodeReplace(currentCoreLibElems, value, attribute): + """Replaces the selected node with the provided node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.replace(elem, copy.deepcopy(value[0])) + +patchDispatcher = { + "AttributeSet" : AttributeSet, + "AttributeAdd" : AttributeAdd, + "AttributeRemove" : AttributeRemove, + "Add": NodeAdd, + "Insert": NodeInsert, + "Remove": NodeRemove, + "Replace": NodeReplace, +} + + def doPatches(coreLib, modLib: dict, mod: str): # Pretyping patchList : lxml.etree._ElementTree @@ -243,67 +308,12 @@ def doPatchType(patch: lxml.etree._Element, location: str): if len(currentCoreLibElems) == 0: ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") return - - def AttributeSet(): - """Set the attribute on the node, adding if not present""" - elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.set(attribute, value.text) - def AttributeAdd(): - """Adds the attribute to the node IFF the attribute name is not already present""" - elem : lxml.etree._Element - for elem in currentCoreLibElems: - if elem.get(attribute, None) is not None: - raise KeyError(f"Attribute '{attribute}' already exists") - elem.set(attribute, value.text) - def AttributeRemove(): - """Remove the attribute from the node""" - ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") - elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.attrib.pop(attribute) - - def NodeAdd(): - """Adds a provided child node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - lastelemIDX = len(elem.getchildren()) - elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) - def NodeInsert(): - """Adds a provided sibling node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - elemIDX = parent.index(elem) - parent.insert(elemIDX + 1, copy.deepcopy(value[0])) - def NodeRemove(): - """Deletes the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.remove(elem) - def NodeReplace(): - """Replaces the selected node with the provided node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.replace(elem, copy.deepcopy(value[0])) - def BadOp(): + def BadOp(currentCoreLibElems, value, attribute): raise SyntaxError(f"BAD PATCH OPERATION") - patchDispatcher = { - "AttributeSet" : AttributeSet, - "AttributeAdd" : AttributeAdd, - "AttributeRemove" : AttributeRemove, - "Add": NodeAdd, - "Insert": NodeInsert, - "Remove": NodeRemove, - "Replace": NodeReplace, - } - - patchDispatcher.get(pType,BadOp)() + + + patchDispatcher.get(pType,BadOp)(currentCoreLibElems, value, attribute) # Execution for location in modLib: From f33a0eace767af131eaf59674866a5a26f3837e7 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 04:35:05 -0500 Subject: [PATCH 028/124] Simplify PatchOperation function call at cost of extended setup in function --- loader/assets/merge.py | 68 ++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 7b08e07f..dcc4d1da 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -217,58 +217,85 @@ def _mod_path(filename): -def AttributeSet(currentCoreLibElems, value, attribute): +def AttributeSet(patchArgs): """Set the attribute on the node, adding if not present""" elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + value = patchArgs["value"] for elem in currentCoreLibElems: elem.set(attribute, value.text) -def AttributeAdd(currentCoreLibElems, value, attribute): + +def AttributeAdd(patchArgs): """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + for elem in currentCoreLibElems: if elem.get(attribute, None) is not None: raise KeyError(f"Attribute '{attribute}' already exists") elem.set(attribute, value.text) -def AttributeRemove(currentCoreLibElems, value, attribute): + +def AttributeRemove(patchArgs): """Remove the attribute from the node""" - ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + ui.log.log(f" WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text for elem in currentCoreLibElems: elem.attrib.pop(attribute) -def NodeAdd(currentCoreLibElems, value, attribute): + +def NodeAdd(patchArgs): """Adds a provided child node to the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: lastelemIDX = len(elem.getchildren()) elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) -def NodeInsert(currentCoreLibElems, value, attribute): + +def NodeInsert(patchArgs): """Adds a provided sibling node to the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: parent = elem.find('./..') elemIDX = parent.index(elem) parent.insert(elemIDX + 1, copy.deepcopy(value[0])) -def NodeRemove(currentCoreLibElems, value, attribute): + +def NodeRemove(patchArgs): """Deletes the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] for elem in currentCoreLibElems: parent = elem.find('./..') parent.remove(elem) -def NodeReplace(currentCoreLibElems, value, attribute): + +def NodeReplace(patchArgs): """Replaces the selected node with the provided node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: parent = elem.find('./..') parent.replace(elem, copy.deepcopy(value[0])) + +# Default case function +def BadOp(patchArgs): + raise SyntaxError(f"BAD PATCH OPERATION") + + patchDispatcher = { "AttributeSet" : AttributeSet, "AttributeAdd" : AttributeAdd, @@ -291,29 +318,24 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Pretyping currentCoreLib : lxml.etree._ElementTree patchOperation : lxml.etree._Element - xpath: str value: lxml.etree._Element - logIndent = " " * 4 - pType = patch.attrib["Class"] + pType = patch.attrib["Class"] xpath = patch.find('xpath').text - value = patch.find('value') - if "Attribute" in pType: - attribute = patch.find("attribute").text - - ui.log.log(f"{logIndent}XPATH => {location:>15}: {pType:18}{xpath}") + currentCoreLibElems = coreLib[location].xpath(xpath) - currentCoreLib = coreLib[location] - currentCoreLibElems = currentCoreLib.xpath(xpath) + ui.log.log(f" XPATH => {location:>15}: {pType:18}{xpath}") if len(currentCoreLibElems) == 0: - ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") + ui.log.log(f" Unable to perform patch. XPath found no results {xpath}") return - def BadOp(currentCoreLibElems, value, attribute): - raise SyntaxError(f"BAD PATCH OPERATION") - + patchArgs = { + "value": patch.find('value'), + "attribute": patch.find("attribute"), # Defer exception throw to later. + "coreLibElems": currentCoreLibElems, + } - patchDispatcher.get(pType,BadOp)(currentCoreLibElems, value, attribute) + patchDispatcher.get(pType,BadOp)(patchArgs) # Execution for location in modLib: From 6ae90ad01ed26d02153eec59d9c1da12d6d05b44 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 05:02:33 -0500 Subject: [PATCH 029/124] Add PatchOperationAtrributeMath - Not in spec This extension allows for the easy mass changing of values. IT IS NOT GAURDED. Bad math can and WILL cause errors. --- loader/assets/merge.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index dcc4d1da..e8a68e7f 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -247,6 +247,30 @@ def AttributeRemove(patchArgs): for elem in currentCoreLibElems: elem.attrib.pop(attribute) +def AttributeMath(patchArgs): + """Set the attribute on the node, via math""" + elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + value = patchArgs["value"] + opType = value.get("opType", None) + valueFloat = float(value.text) + for elem in currentCoreLibElems: + startVal = float(elem.get(attribute, 0)) + if opType == "add": + newVal = startVal + valueFloat + elif opType == "subtract": + newVal = startVal - valueFloat + elif opType == "multiply": + newVal = startVal * valueFloat + elif opType == "divide": + newVal = startVal / valueFloat + else: + raise AttributeError("Unknown opType") + + elem.set(attribute, f"{newVal:.3}") + + def NodeAdd(patchArgs): """Adds a provided child node to the selected node""" elem : lxml.etree._Element @@ -300,6 +324,7 @@ def BadOp(patchArgs): "AttributeSet" : AttributeSet, "AttributeAdd" : AttributeAdd, "AttributeRemove" : AttributeRemove, + "AttributeMath" : AttributeMath, "Add": NodeAdd, "Insert": NodeInsert, "Remove": NodeRemove, From 1385feef5182c61c8f32f00a300e18c9bedfd079 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 05:11:28 -0500 Subject: [PATCH 030/124] Bump version drastically Something about four-segment versions breaks the window code As a result, let's pretend the revision number was the minor version this whole time. Bump from 0.0.8 -> 0.8.1 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 613ac843..d2fccaa3 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.0.8" +version = "0.8.1" From f89281f5aa5e871c207222d04b46a284db7ef188 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 06:31:43 -0500 Subject: [PATCH 031/124] Add method to disable patches --- loader/assets/merge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e8a68e7f..8246f0c3 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -365,6 +365,8 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Execution for location in modLib: for patchList in modLib[location]: + if patchList.find("Noload") is not None: + continue for patchOperation in patchList.getroot(): try: doPatchType(patchOperation, location) From 4593b69362ffa6cbd33e9cfa669b9f1f7dd6b9fd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 06:37:17 -0500 Subject: [PATCH 032/124] Bugfix: Assume output of any math operation is an int unless specified --- loader/assets/merge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 8246f0c3..c8dbce1f 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -254,6 +254,7 @@ def AttributeMath(patchArgs): attribute = patchArgs["attribute"].text value = patchArgs["value"] opType = value.get("opType", None) + isFloat = value.attrib.has_key("float") valueFloat = float(value.text) for elem in currentCoreLibElems: startVal = float(elem.get(attribute, 0)) @@ -268,7 +269,11 @@ def AttributeMath(patchArgs): else: raise AttributeError("Unknown opType") - elem.set(attribute, f"{newVal:.3}") + if isFloat: + elem.set(attribute, f"{newVal:.1f}") + else: + newVal = int(newVal) + elem.set(attribute, f"{newVal}") def NodeAdd(patchArgs): From 22246b80aa4d9659ca9bbf02590d6d1573082210 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 13:50:42 -0500 Subject: [PATCH 033/124] Switch to automatic float determination --- loader/assets/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index c8dbce1f..ed04759a 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -254,10 +254,10 @@ def AttributeMath(patchArgs): attribute = patchArgs["attribute"].text value = patchArgs["value"] opType = value.get("opType", None) - isFloat = value.attrib.has_key("float") valueFloat = float(value.text) for elem in currentCoreLibElems: startVal = float(elem.get(attribute, 0)) + isFloat = "." in elem.get(attribute, 0) if opType == "add": newVal = startVal + valueFloat elif opType == "subtract": From 0203b5d523a95519c35e148e133af8273cc32680 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 13:59:24 -0500 Subject: [PATCH 034/124] Log skipped files This should mitigate issues with skipped files since the log will track it --- loader/assets/merge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index ed04759a..dd81fc65 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -371,6 +371,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): for location in modLib: for patchList in modLib[location]: if patchList.find("Noload") is not None: + ui.log.log(f" Skipping file {patchList.getroot().base} (Noload tag)") continue for patchOperation in patchList.getroot(): try: From 65dcfb4cdd012133dd3e8778adfe38047d0062cd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 22:02:51 -0500 Subject: [PATCH 035/124] Move function out into global scope --- loader/assets/merge.py | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index dd81fc65..2b232144 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -96,41 +96,42 @@ def _add_texture(region_id): return modded_textures +def buildLibrary(location: str): + """Build up a library dict of files in `location`""" + def _mod_path(filename): + return os.path.join(mod, filename.replace('/', os.sep)) + location_library = {} + try: + location_files = [location + '/' + mod_file for mod_file in os.listdir(_mod_path(location))] + except FileNotFoundError: + location_files = [] + + # we allow breaking down mod xml files into smaller pieces for readability + for target in PATCHABLE_XML_FILES: + targetInLocation = target.replace('library', location) + for mod_file in location_files: + if not mod_file.startswith(targetInLocation): continue + if target not in location_library: location_library[target] = [] + + ui.log.log(" {} <= {}".format(target, mod_file)) + with open(_mod_path(mod_file)) as f: + location_library[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) + + mod_file = _mod_path(target) + # try again with the extension ? + if not os.path.exists(mod_file): + mod_file += '.xml' + if not os.path.exists(mod_file): + continue + return location_library + + def mods(corePath, modPaths): # Load the core library files coreLibrary = {} def _core_path(filename): return os.path.join(corePath, filename.replace('/', os.sep)) - def buildLibrary(location: str): - """Build up a library dict of files in `location`""" - def _mod_path(filename): - return os.path.join(mod, filename.replace('/', os.sep)) - location_library = {} - try: - location_files = [location + '/' + mod_file for mod_file in os.listdir(_mod_path(location))] - except FileNotFoundError: - location_files = [] - - # we allow breaking down mod xml files into smaller pieces for readability - for target in PATCHABLE_XML_FILES: - targetInLocation = target.replace('library', location) - for mod_file in location_files: - if not mod_file.startswith(targetInLocation): continue - if target not in location_library: location_library[target] = [] - - ui.log.log(" {} <= {}".format(target, mod_file)) - with open(_mod_path(mod_file)) as f: - location_library[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) - - mod_file = _mod_path(target) - # try again with the extension ? - if not os.path.exists(mod_file): - mod_file += '.xml' - if not os.path.exists(mod_file): - continue - return location_library - for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), 'rb') as f: coreLibrary[filename] = lxml.etree.parse(f, parser=lxml.etree.XMLParser(recover=True)) @@ -150,7 +151,6 @@ def _mod_path(filename): # Load the mod's library modLibrary = buildLibrary('library') - doMerges(coreLibrary, modLibrary, mod) # Do patches after merges to avoid clobbers From 7bf7464d0f17cfaef60db9bc4873cbb22a608489 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 22:04:10 -0500 Subject: [PATCH 036/124] Clean up editor typing --- loader/assets/merge.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2b232144..d4a9e680 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -336,20 +336,10 @@ def BadOp(patchArgs): "Replace": NodeReplace, } - -def doPatches(coreLib, modLib: dict, mod: str): - # Pretyping - patchList : lxml.etree._ElementTree - patchOperation : lxml.etree._Element - - # Helper functions +def doPatches(coreLib, modLib, mod: str): + # Helper function def doPatchType(patch: lxml.etree._Element, location: str): """Execute a single patch. Provided to reduce indentation level""" - # Pretyping - currentCoreLib : lxml.etree._ElementTree - patchOperation : lxml.etree._Element - value: lxml.etree._Element - pType = patch.attrib["Class"] xpath = patch.find('xpath').text currentCoreLibElems = coreLib[location].xpath(xpath) @@ -370,10 +360,12 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Execution for location in modLib: for patchList in modLib[location]: + patchList : lxml.etree._ElementTree if patchList.find("Noload") is not None: ui.log.log(f" Skipping file {patchList.getroot().base} (Noload tag)") continue for patchOperation in patchList.getroot(): + patchOperation : lxml.etree._Element try: doPatchType(patchOperation, location) except Exception as e: From 875e03cdd31e377a81b7ec83c2515ebbdc7f879b Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 22:05:00 -0500 Subject: [PATCH 037/124] Centralize patch dispatching --- loader/assets/merge.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index d4a9e680..0320ff38 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -335,6 +335,9 @@ def BadOp(patchArgs): "Remove": NodeRemove, "Replace": NodeReplace, } +def PatchDispatch(ptype): + """Return the correct PatchOperation function""" + return patchDispatcher.get(pType,BadOp) def doPatches(coreLib, modLib, mod: str): # Helper function @@ -354,8 +357,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): "attribute": patch.find("attribute"), # Defer exception throw to later. "coreLibElems": currentCoreLibElems, } - - patchDispatcher.get(pType,BadOp)(patchArgs) + PatchDispatch(ptype)(patchArgs) # Execution for location in modLib: From 7291db09e314f07cfa381e95f49afad6b5219ded Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 1 Dec 2020 22:05:59 -0500 Subject: [PATCH 038/124] Clean whitespace and add comments --- loader/assets/merge.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 0320ff38..81184eb7 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -32,7 +32,7 @@ def _add_texture(region_id): ui.log.log(" Found {}...".format(filename)) if int(region_id) > coreLibrary['_last_core_region_id']: - # adding a new texture, this gets tricky as they have to have consecutive numbers. + # adding a new texture, this gets tricky as they have to have consecutive numbers. core_region_id = str(coreLibrary['_next_region_id']) mapping_n_region[region_id] = core_region_id coreLibrary['_next_region_id'] += 1 @@ -43,7 +43,7 @@ def _add_texture(region_id): modded_textures[core_region_id] = { 'mapped_from_id' : region_id, 'filename' : filename, - 'path' : path, + 'path' : path, } for filename in os.listdir(textures_path): @@ -73,7 +73,7 @@ def _add_texture(region_id): _add_texture(region_id) if not mapping_n_region: - # no custom mod textures, no need to remap ids + # no custom mod textures, no need to remap ids return modded_textures for animation_chunk in modLibrary['library/animations']: @@ -163,7 +163,7 @@ def _core_path(filename): ui.log.updateLaunchState("Updating XML") # Write out the new base library - for filename in PATCHABLE_XML_FILES: + for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), "wb") as f: f.write(lxml.etree.tostring(coreLibrary[filename], pretty_print=True, encoding="UTF-8")) @@ -216,7 +216,6 @@ def _core_path(filename): return extra_assets - def AttributeSet(patchArgs): """Set the attribute on the node, adding if not present""" elem : lxml.etree._Element @@ -350,7 +349,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): ui.log.log(f" XPATH => {location:>15}: {pType:18}{xpath}") if len(currentCoreLibElems) == 0: ui.log.log(f" Unable to perform patch. XPath found no results {xpath}") - return + return # Don't perform patch if no matches patchArgs = { "value": patch.find('value'), From 2c1a36eedcccc2673cf5cc604d3341b58335e183 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 2 Dec 2020 01:35:57 -0500 Subject: [PATCH 039/124] Activate enviroment before building --- tools/build-windows.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-windows.sh b/tools/build-windows.sh index 6c98558b..c88a782b 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -3,7 +3,7 @@ VERSION=`python -c 'import version; print(version.version)'` rm -rf build/* dist/* - +source env/Scripts/activate rm -rf dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows pyinstaller spacehaven-modloader.py --noconsole From 91eeaa3842d68d3571d26a18acf73cd7b28bb493 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 02:22:37 -0500 Subject: [PATCH 040/124] BUGFIX: Fix incorrect capitalization in variable name --- loader/assets/merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 81184eb7..c4a1917b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -334,7 +334,7 @@ def BadOp(patchArgs): "Remove": NodeRemove, "Replace": NodeReplace, } -def PatchDispatch(ptype): +def PatchDispatch(pType): """Return the correct PatchOperation function""" return patchDispatcher.get(pType,BadOp) @@ -356,7 +356,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): "attribute": patch.find("attribute"), # Defer exception throw to later. "coreLibElems": currentCoreLibElems, } - PatchDispatch(ptype)(patchArgs) + PatchDispatch(pType)(patchArgs) # Execution for location in modLib: From 3d63a02214d91c1a477198b76139c064388e765b Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 02:26:52 -0500 Subject: [PATCH 041/124] BUGFIX: Add variable that was previously caught by encapsulation --- loader/assets/merge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index c4a1917b..af1ceb54 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -96,7 +96,7 @@ def _add_texture(region_id): return modded_textures -def buildLibrary(location: str): +def buildLibrary(location: str, mod: str): """Build up a library dict of files in `location`""" def _mod_path(filename): return os.path.join(mod, filename.replace('/', os.sep)) @@ -150,14 +150,14 @@ def _core_path(filename): ui.log.log(" Loading mod {}...".format(mod)) # Load the mod's library - modLibrary = buildLibrary('library') + modLibrary = buildLibrary('library', mod) doMerges(coreLibrary, modLibrary, mod) # Do patches after merges to avoid clobbers for mod in modPaths: ui.log.updateLaunchState(f"Patching {os.path.basename(mod)}") ui.log.log(f" Loading patches {mod}...") - modPatchesLibrary = buildLibrary('patches') + modPatchesLibrary = buildLibrary('patches', mod) doPatches(coreLibrary, modPatchesLibrary, mod) ui.log.updateLaunchState("Updating XML") From 4511e9d945ceba17d13cdffa75913a1d96381479 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 03:14:45 -0500 Subject: [PATCH 042/124] Build from spec file instead of autogenerating it. (Only for windows) --- spacehaven-modloader.spec | 39 +++++++++++++++++++++++++++++++++++++++ tools/build-windows.sh | 5 ++++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 spacehaven-modloader.spec diff --git a/spacehaven-modloader.spec b/spacehaven-modloader.spec new file mode 100644 index 00000000..8b868b8b --- /dev/null +++ b/spacehaven-modloader.spec @@ -0,0 +1,39 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['spacehaven-modloader.py'], + pathex=['G:\\SteamLibrary\\steamapps\\common\\spacehaven\\modloader-dev'], + binaries=[], + datas=[ + ('textures_annotations.xml','.') + ], + hiddenimports=['click', 'lxml', 'pypng'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='spacehaven-modloader', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False ) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='spacehaven-modloader') diff --git a/tools/build-windows.sh b/tools/build-windows.sh index c88a782b..d231caa2 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -3,10 +3,13 @@ VERSION=`python -c 'import version; print(version.version)'` rm -rf build/* dist/* + source env/Scripts/activate rm -rf dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows -pyinstaller spacehaven-modloader.py --noconsole +python -m PyInstaller --noconsole spacehaven-modloader.spec mv dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows +echo "-- Press enter key to continue --" +read $null start dist From af8f87153449a3fdce0930db4544074bbdec4588 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 03:18:02 -0500 Subject: [PATCH 043/124] Be more clear about annotation process --- loader/assets/annotate.py | 5 +++-- spacehaven-modloader.py | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/loader/assets/annotate.py b/loader/assets/annotate.py index d6dea366..e7a155c8 100644 --- a/loader/assets/annotate.py +++ b/loader/assets/annotate.py @@ -21,10 +21,11 @@ def annotate(corePath): if not asset_id in texture_names: continue assetPos.set('_annotation', texture_names[asset_id]) + annotatedPath = os.path.join(corePath, "library", "animations_annotated.xml") animations.write(annotatedPath) - ui.log.log(" Wrote annotated annimations to {}".format(annotatedPath)) + ui.log.log(" Wrote annotated annimations to {}".format(annotatedPath)) haven = ElementTree.parse(os.path.join(corePath, "library", "haven"), parser=XMLParser(recover=True)) texts = ElementTree.parse(os.path.join(corePath, "library", "texts"), parser=XMLParser(recover=True)) @@ -147,4 +148,4 @@ def _annotate_elt(element, attr = None): annotatedHavenPath = os.path.join(corePath, "library", "haven_annotated.xml") haven.write(annotatedHavenPath) - ui.log.log(" Wrote annotated spacehaven library to {}".format(annotatedHavenPath)) + ui.log.log(" Wrote annotated spacehaven library to {}".format(annotatedHavenPath)) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index bbc193ff..89a59b05 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -415,8 +415,12 @@ def extract_assets(self): def annotate(self): corePath = self._core_extract_path() - - loader.assets.annotate.annotate(corePath) + ui.log.log(f"Annotating and putting files in {corePath}") + try: + loader.assets.annotate.annotate(corePath) + except Exception as e: + ui.log.log(" Error during annotation!") + ui.log.log(repr(e)) ui.launcher.open(os.path.join(corePath, 'library')) From 9fe2e69ef952d6682b7a78adab93d11cd3669ac0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 12:25:08 -0500 Subject: [PATCH 044/124] Freeze pip requirements --- requirements.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a5715ca1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +altgraph==0.17 +astroid==2.4.2 +click==7.1.2 +colorama==0.4.4 +future==0.18.2 +isort==5.6.4 +lazy-object-proxy==1.4.3 +lxml==4.6.2 +mccabe==0.6.1 +pefile==2019.4.18 +pyinstaller==4.1 +pyinstaller-hooks-contrib==2020.10 +pypng==0.0.20 +pywin32-ctypes==0.2.0 +six==1.15.0 +toml==0.10.2 +typed-ast==1.4.1 +wrapt==1.12.1 From 035f14ebeba416cf9890468631d4fe1e489a65cb Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Thu, 3 Dec 2020 12:53:47 -0500 Subject: [PATCH 045/124] Update build script to fix PATH clobbering from env - Also prevents previous builds from being erased if versions are different --- tools/build-windows.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/build-windows.sh b/tools/build-windows.sh index d231caa2..9b42213b 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -1,13 +1,16 @@ #!/bin/bash +OLDPATH=$PATH +source env/Scripts/activate VERSION=`python -c 'import version; print(version.version)'` -rm -rf build/* dist/* +PATH=$OLDPATH +rm -rf build/* dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows -source env/Scripts/activate -rm -rf dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows python -m PyInstaller --noconsole spacehaven-modloader.spec +deactivate +PATH=$OLDPATH mv dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows echo "-- Press enter key to continue --" From e6adb8abc1e369a1df46d664005e8b1a86f127c6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 8 Dec 2020 17:38:42 -0500 Subject: [PATCH 046/124] Change name of spec file, prevent clobbers --- spacehaven-modloader.spec => modloader.spec | 0 tools/build-windows.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename spacehaven-modloader.spec => modloader.spec (100%) diff --git a/spacehaven-modloader.spec b/modloader.spec similarity index 100% rename from spacehaven-modloader.spec rename to modloader.spec diff --git a/tools/build-windows.sh b/tools/build-windows.sh index 9b42213b..73d0fae2 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -7,7 +7,7 @@ VERSION=`python -c 'import version; print(version.version)'` PATH=$OLDPATH rm -rf build/* dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows -python -m PyInstaller --noconsole spacehaven-modloader.spec +python -m PyInstaller --noconsole modloader.spec deactivate PATH=$OLDPATH From 61ffd50db6bf013f08a50689270059fb97ca6b84 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 8 Dec 2020 17:57:43 -0500 Subject: [PATCH 047/124] Fix build script to properly control virtual environment --- tools/build-windows.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/build-windows.sh b/tools/build-windows.sh index 73d0fae2..97450432 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -4,8 +4,9 @@ OLDPATH=$PATH source env/Scripts/activate VERSION=`python -c 'import version; print(version.version)'` -PATH=$OLDPATH +deactivate rm -rf build/* dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows +source env/Scripts/activate python -m PyInstaller --noconsole modloader.spec deactivate From e2bf1b4bbfb33be056bd4a3ec39a0ba93ebe544c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 8 Dec 2020 18:06:38 -0500 Subject: [PATCH 048/124] Reorder build operations Don't delete build directory Get version after build Remove version-specific folder after build --- tools/build-windows.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/build-windows.sh b/tools/build-windows.sh index 97450432..33b2c5bc 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -1,17 +1,12 @@ #!/bin/bash +rm -rf dist/spacehaven-modloader -OLDPATH=$PATH source env/Scripts/activate -VERSION=`python -c 'import version; print(version.version)'` - -deactivate -rm -rf build/* dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows -source env/Scripts/activate - python -m PyInstaller --noconsole modloader.spec +VERSION=`python -c 'import version; print(version.version)'` deactivate -PATH=$OLDPATH +rm -rf dist/spacehaven-modloader-$VERSION.windows mv dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows echo "-- Press enter key to continue --" From 8252f810b54f72014fd971653b04e331369829d8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 8 Dec 2020 18:08:08 -0500 Subject: [PATCH 049/124] Remove hard path from spec file --- modloader.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modloader.spec b/modloader.spec index 8b868b8b..0a90ee37 100644 --- a/modloader.spec +++ b/modloader.spec @@ -4,7 +4,7 @@ block_cipher = None a = Analysis(['spacehaven-modloader.py'], - pathex=['G:\\SteamLibrary\\steamapps\\common\\spacehaven\\modloader-dev'], + pathex=[], binaries=[], datas=[ ('textures_annotations.xml','.') From 1f4042abd29b3d3645eca5a53a4d2b8e85f7008c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 8 Dec 2020 18:11:32 -0500 Subject: [PATCH 050/124] Include README and CHANGELOG in build --- modloader.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modloader.spec b/modloader.spec index 0a90ee37..51c72d42 100644 --- a/modloader.spec +++ b/modloader.spec @@ -7,7 +7,9 @@ a = Analysis(['spacehaven-modloader.py'], pathex=[], binaries=[], datas=[ - ('textures_annotations.xml','.') + ('textures_annotations.xml','.'), + ('README.md','.'), + ('CHANGELOG.md','.'), ], hiddenimports=['click', 'lxml', 'pypng'], hookspath=[], From bfb4fce5d1f63b4a82878c1ef704488ecf7aa23d Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 9 Dec 2020 04:49:39 -0500 Subject: [PATCH 051/124] BUGFIX: textures were not merged in Fixes bug introduced in 7a6796a39dc81fb894b81c149d5e511ead1779a5 --- loader/assets/merge.py | 1 + version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index af1ceb54..e74f671d 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -457,6 +457,7 @@ def mergeAbortMessage(filename: str): currentFile = "library/textures" if currentFile in modLib: + mergeShim(currentFile, "/AllTexturesAndRegions/textures", "i") mergeShim(currentFile, "/AllTexturesAndRegions/regions", "n") else: mergeAbortMessage(currentFile) diff --git a/version.py b/version.py index d2fccaa3..057d865f 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.8.1" +version = "0.8.2" From 44c4b1e5d5b304da3a02a182d1b1574ccc582326 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 10:36:49 -0500 Subject: [PATCH 052/124] Break patch operations into their own module --- loader/assets/merge.py | 162 +--------------------------------------- loader/assets/patch.py | 164 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 161 deletions(-) create mode 100644 loader/assets/patch.py diff --git a/loader/assets/merge.py b/loader/assets/merge.py index af1ceb54..719b93e9 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -6,6 +6,7 @@ import loader.assets.library from .library import PATCHABLE_XML_FILES, PATCHABLE_CIM_FILES from .explode import Texture +from .patch import doPatches import ui.log @@ -216,167 +217,6 @@ def _core_path(filename): return extra_assets -def AttributeSet(patchArgs): - """Set the attribute on the node, adding if not present""" - elem : lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - attribute = patchArgs["attribute"].text - value = patchArgs["value"] - for elem in currentCoreLibElems: elem.set(attribute, value.text) - - -def AttributeAdd(patchArgs): - """Adds the attribute to the node IFF the attribute name is not already present""" - elem : lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - attribute = patchArgs["attribute"].text - - for elem in currentCoreLibElems: - if elem.get(attribute, None) is not None: - raise KeyError(f"Attribute '{attribute}' already exists") - elem.set(attribute, value.text) - - -def AttributeRemove(patchArgs): - """Remove the attribute from the node""" - ui.log.log(f" WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") - elem : lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - attribute = patchArgs["attribute"].text - for elem in currentCoreLibElems: elem.attrib.pop(attribute) - - -def AttributeMath(patchArgs): - """Set the attribute on the node, via math""" - elem : lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - attribute = patchArgs["attribute"].text - value = patchArgs["value"] - opType = value.get("opType", None) - valueFloat = float(value.text) - for elem in currentCoreLibElems: - startVal = float(elem.get(attribute, 0)) - isFloat = "." in elem.get(attribute, 0) - if opType == "add": - newVal = startVal + valueFloat - elif opType == "subtract": - newVal = startVal - valueFloat - elif opType == "multiply": - newVal = startVal * valueFloat - elif opType == "divide": - newVal = startVal / valueFloat - else: - raise AttributeError("Unknown opType") - - if isFloat: - elem.set(attribute, f"{newVal:.1f}") - else: - newVal = int(newVal) - elem.set(attribute, f"{newVal}") - - -def NodeAdd(patchArgs): - """Adds a provided child node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - value = patchArgs["value"] - for elem in currentCoreLibElems: - lastelemIDX = len(elem.getchildren()) - elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) - - -def NodeInsert(patchArgs): - """Adds a provided sibling node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - value = patchArgs["value"] - for elem in currentCoreLibElems: - parent = elem.find('./..') - elemIDX = parent.index(elem) - parent.insert(elemIDX + 1, copy.deepcopy(value[0])) - - -def NodeRemove(patchArgs): - """Deletes the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.remove(elem) - - -def NodeReplace(patchArgs): - """Replaces the selected node with the provided node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - currentCoreLibElems = patchArgs["coreLibElems"] - value = patchArgs["value"] - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.replace(elem, copy.deepcopy(value[0])) - - -# Default case function -def BadOp(patchArgs): - raise SyntaxError(f"BAD PATCH OPERATION") - - -patchDispatcher = { - "AttributeSet" : AttributeSet, - "AttributeAdd" : AttributeAdd, - "AttributeRemove" : AttributeRemove, - "AttributeMath" : AttributeMath, - "Add": NodeAdd, - "Insert": NodeInsert, - "Remove": NodeRemove, - "Replace": NodeReplace, -} -def PatchDispatch(pType): - """Return the correct PatchOperation function""" - return patchDispatcher.get(pType,BadOp) - -def doPatches(coreLib, modLib, mod: str): - # Helper function - def doPatchType(patch: lxml.etree._Element, location: str): - """Execute a single patch. Provided to reduce indentation level""" - pType = patch.attrib["Class"] - xpath = patch.find('xpath').text - currentCoreLibElems = coreLib[location].xpath(xpath) - - ui.log.log(f" XPATH => {location:>15}: {pType:18}{xpath}") - if len(currentCoreLibElems) == 0: - ui.log.log(f" Unable to perform patch. XPath found no results {xpath}") - return # Don't perform patch if no matches - - patchArgs = { - "value": patch.find('value'), - "attribute": patch.find("attribute"), # Defer exception throw to later. - "coreLibElems": currentCoreLibElems, - } - PatchDispatch(pType)(patchArgs) - - # Execution - for location in modLib: - for patchList in modLib[location]: - patchList : lxml.etree._ElementTree - if patchList.find("Noload") is not None: - ui.log.log(f" Skipping file {patchList.getroot().base} (Noload tag)") - continue - for patchOperation in patchList.getroot(): - patchOperation : lxml.etree._Element - try: - doPatchType(patchOperation, location) - except Exception as e: - uri = patchOperation.base - line = patchOperation.sourceline - ui.log.log(f" Failed to apply patch operation {uri}:{line}") - ui.log.log(f" Reason: {repr(e)}") - raise SyntaxError("Issue in patch operation. Check logs for info.") from None - - def doMerges(coreLib, modLib, mod: str): """Do merge-based modding sequence""" def mergeShim(file: str, xpath: str, idAttribute: str): diff --git a/loader/assets/patch.py b/loader/assets/patch.py new file mode 100644 index 00000000..cb449f47 --- /dev/null +++ b/loader/assets/patch.py @@ -0,0 +1,164 @@ +import lxml.etree +import ui.log +import copy + + +def AttributeSet(patchArgs): + """Set the attribute on the node, adding if not present""" + elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + value = patchArgs["value"] + for elem in currentCoreLibElems: elem.set(attribute, value.text) + + +def AttributeAdd(patchArgs): + """Adds the attribute to the node IFF the attribute name is not already present""" + elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + + for elem in currentCoreLibElems: + if elem.get(attribute, None) is not None: + raise KeyError(f"Attribute '{attribute}' already exists") + elem.set(attribute, value.text) + + +def AttributeRemove(patchArgs): + """Remove the attribute from the node""" + ui.log.log(f" WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + for elem in currentCoreLibElems: elem.attrib.pop(attribute) + + +def AttributeMath(patchArgs): + """Set the attribute on the node, via math""" + elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + value = patchArgs["value"] + opType = value.get("opType", None) + valueFloat = float(value.text) + for elem in currentCoreLibElems: + startVal = float(elem.get(attribute, 0)) + isFloat = "." in elem.get(attribute, 0) + if opType == "add": + newVal = startVal + valueFloat + elif opType == "subtract": + newVal = startVal - valueFloat + elif opType == "multiply": + newVal = startVal * valueFloat + elif opType == "divide": + newVal = startVal / valueFloat + else: + raise AttributeError("Unknown opType") + + if isFloat: + elem.set(attribute, f"{newVal:.1f}") + else: + newVal = int(newVal) + elem.set(attribute, f"{newVal}") + + +def NodeAdd(patchArgs): + """Adds a provided child node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] + for elem in currentCoreLibElems: + lastelemIDX = len(elem.getchildren()) + elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) + + +def NodeInsert(patchArgs): + """Adds a provided sibling node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] + for elem in currentCoreLibElems: + parent = elem.find('./..') + elemIDX = parent.index(elem) + parent.insert(elemIDX + 1, copy.deepcopy(value[0])) + + +def NodeRemove(patchArgs): + """Deletes the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.remove(elem) + + +def NodeReplace(patchArgs): + """Replaces the selected node with the provided node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.replace(elem, copy.deepcopy(value[0])) + + +# Default case function +def BadOp(patchArgs): + raise SyntaxError(f"BAD PATCH OPERATION") + + +patchDispatcher = { + "AttributeSet" : AttributeSet, + "AttributeAdd" : AttributeAdd, + "AttributeRemove" : AttributeRemove, + "AttributeMath" : AttributeMath, + "Add": NodeAdd, + "Insert": NodeInsert, + "Remove": NodeRemove, + "Replace": NodeReplace, +} +def PatchDispatch(pType): + """Return the correct PatchOperation function""" + return patchDispatcher.get(pType,BadOp) + +def doPatches(coreLib, modLib, mod: str): + # Helper function + def doPatchType(patch: lxml.etree._Element, location: str): + """Execute a single patch. Provided to reduce indentation level""" + pType = patch.attrib["Class"] + xpath = patch.find('xpath').text + currentCoreLibElems = coreLib[location].xpath(xpath) + + ui.log.log(f" XPATH => {location:>15}: {pType:18}{xpath}") + if len(currentCoreLibElems) == 0: + ui.log.log(f" Unable to perform patch. XPath found no results {xpath}") + return # Don't perform patch if no matches + + patchArgs = { + "value": patch.find('value'), + "attribute": patch.find("attribute"), # Defer exception throw to later. + "coreLibElems": currentCoreLibElems, + } + PatchDispatch(pType)(patchArgs) + + # Execution + for location in modLib: + for patchList in modLib[location]: + patchList : lxml.etree._ElementTree + if patchList.find("Noload") is not None: + ui.log.log(f" Skipping file {patchList.getroot().base} (Noload tag)") + continue + for patchOperation in patchList.getroot(): + patchOperation : lxml.etree._Element + try: + doPatchType(patchOperation, location) + except Exception as e: + uri = patchOperation.base + line = patchOperation.sourceline + ui.log.log(f" Failed to apply patch operation {uri}:{line}") + ui.log.log(f" Reason: {repr(e)}") + raise SyntaxError("Issue in patch operation. Check logs for info.") from None From 28fd8c8f91abace2a6210224faee8ec2892bed76 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 12:34:37 -0500 Subject: [PATCH 053/124] Todo for automatic IDs --- loader/assets/merge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e74f671d..f377f3fc 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -482,6 +482,8 @@ def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): merged = 0 for element in list(modRoot): + # TODO auto-id algo: if element.get(idAttribute + "_auto") then + # id = prefix * idSpaceSize + id conflicts = baseRoot.xpath("*[@{}='{}']".format(idAttribute, element.get(idAttribute))) for conflict in conflicts: From ae9095a037e16c844e07a60117b5729069cc5fd6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 14:00:06 -0500 Subject: [PATCH 054/124] Prepare for codestyle --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..d2f8d7de --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length=160 +exclude=env,header.py From 3afec50ee00579cd90386d4101fbad5c716ce67e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 14:37:44 -0500 Subject: [PATCH 055/124] Sort imports --- loader/assets/merge.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index f377f3fc..574efa63 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -1,13 +1,14 @@ -import os import copy -import lxml.etree +import os import loader.assets.library -from .library import PATCHABLE_XML_FILES, PATCHABLE_CIM_FILES +import lxml.etree +import ui.log + from .explode import Texture +from .library import PATCHABLE_CIM_FILES, PATCHABLE_XML_FILES -import ui.log def _detect_textures(coreLibrary, modLibrary, mod): textures_path = os.path.join(mod, 'textures') From e6306e67af3efffbdee5f1abd55b5d384b6f2786 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 14:38:14 -0500 Subject: [PATCH 056/124] Make ModDatabase a naive Singleton --- ui/database.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ui/database.py b/ui/database.py index 5dd12260..298528aa 100644 --- a/ui/database.py +++ b/ui/database.py @@ -11,11 +11,15 @@ class ModDatabase: """Information about a collection of mods""" + __instance = None def __init__(self, path_list, gameInfo): + if ModDatabase.__instance is not None: + raise Exception("Cannot insantiate more than one ModDatabase") self.path_list = path_list self.gameInfo = gameInfo self.locateMods() + ModDatabase.__instance = self def locateMods(self): self.mods = [] @@ -43,6 +47,11 @@ def _get_mods_from_dir(path): self.mods.sort(key=lambda mod: mod.name) + def getInstance(): + if ModDatabase is None: + raise Exception("Mod Database not ready.") + return ModDatabase.__instance + DISABLED_MARKER = "disabled.txt" class Mod: """Details about a specific mod (name, description)""" From a34448ff00a7c30b6a5cf0d976fb7efd381ea63f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 15:28:52 -0500 Subject: [PATCH 057/124] BUGFIX: ? Fix skipping of extracted files --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index 298528aa..713dabfa 100644 --- a/ui/database.py +++ b/ui/database.py @@ -27,7 +27,7 @@ def locateMods(self): ui.log.log("Locating mods...") def _get_mods_from_dir(path): for modFolder in os.listdir(path): - if modFolder == 'spacehaven': + if 'spacehaven' in modFolder: continue # don't need to load core game definitions modPath = os.path.join(path, modFolder) if os.path.isfile(modPath): From 22d6b3d49c4e3ce1582470bd7745c19e69059964 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 15:28:52 -0500 Subject: [PATCH 058/124] BUGFIX: ? Fix skipping of extracted files --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index 5dd12260..313bc90f 100644 --- a/ui/database.py +++ b/ui/database.py @@ -23,7 +23,7 @@ def locateMods(self): ui.log.log("Locating mods...") def _get_mods_from_dir(path): for modFolder in os.listdir(path): - if modFolder == 'spacehaven': + if 'spacehaven' in modFolder: continue # don't need to load core game definitions modPath = os.path.join(path, modFolder) if os.path.isfile(modPath): From fd3a9b2c83e8bb11b97696a3a3f9b9183150a071 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 15:41:17 -0500 Subject: [PATCH 059/124] Break away from Singleton paradigm The important thing is that we get the most recent database, not that there only ever be one database. With this paradigm we are just asking what the last generated database was. --- ui/database.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/database.py b/ui/database.py index 713dabfa..c44de864 100644 --- a/ui/database.py +++ b/ui/database.py @@ -11,15 +11,13 @@ class ModDatabase: """Information about a collection of mods""" - __instance = None + __lastInstance = None def __init__(self, path_list, gameInfo): - if ModDatabase.__instance is not None: - raise Exception("Cannot insantiate more than one ModDatabase") self.path_list = path_list self.gameInfo = gameInfo self.locateMods() - ModDatabase.__instance = self + ModDatabase.__lastInstance = self def locateMods(self): self.mods = [] @@ -48,9 +46,10 @@ def _get_mods_from_dir(path): self.mods.sort(key=lambda mod: mod.name) def getInstance(): + """Return the last generated instance of a mod database.""" if ModDatabase is None: raise Exception("Mod Database not ready.") - return ModDatabase.__instance + return ModDatabase.__lastInstance DISABLED_MARKER = "disabled.txt" class Mod: From 811baae17df6d1e87cc3f16ab7cd887505e01141 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 15:55:36 -0500 Subject: [PATCH 060/124] Refactor, remove useless inline function This inline function did nothing to reduce indent level. In addition, the if-checks on the info filename were able to be made less nested. --- ui/database.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/database.py b/ui/database.py index c44de864..e3acbe2c 100644 --- a/ui/database.py +++ b/ui/database.py @@ -23,7 +23,7 @@ def locateMods(self): self.mods = [] ui.log.log("Locating mods...") - def _get_mods_from_dir(path): + for path in self.path_list: for modFolder in os.listdir(path): if 'spacehaven' in modFolder: continue # don't need to load core game definitions @@ -32,16 +32,16 @@ def _get_mods_from_dir(path): # TODO add support for zip files ? unzip them on the fly ? continue # don't load logs, prefs, etc + # TODO Pass the mod path to Mod() instead of the info_file and let it handle + # the info file check. It already does this! Let it do its job! info_file = os.path.join(modPath, "info") - if os.path.isfile(info_file): - self.mods.append(Mod(info_file, self.gameInfo)) - else: + if not os.path.isfile(info_file): info_file += '.xml' - if os.path.isfile(info_file): - self.mods.append(Mod(info_file, self.gameInfo)) - - for path in self.path_list: - _get_mods_from_dir(path) + if not os.path.isfile(info_file): + # no info file, don't create a mod. + continue + + self.mods.append(Mod(info_file, self.gameInfo)) self.mods.sort(key=lambda mod: mod.name) From 994cdb1701ad93fda5d26842be02b2ba2760a3ba Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 15:57:50 -0500 Subject: [PATCH 061/124] Remove weird whitespace --- ui/database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ui/database.py b/ui/database.py index e3acbe2c..849c0aec 100644 --- a/ui/database.py +++ b/ui/database.py @@ -60,13 +60,9 @@ def __init__(self, info_file, gameInfo): ui.log.log(" Loading mod at {}...".format(self.path)) # TODO add a flag to warn users about savegame compatibility ? - self.name = os.path.basename(self.path) - self.gameInfo = gameInfo - self.enabled = not os.path.isfile(os.path.join(self.path, DISABLED_MARKER)) - self.loadInfo(info_file) def loadInfo(self, infoFile): @@ -160,8 +156,6 @@ def verifyGameVersion(self, mod, gameInfo): ", ".join(self.gameVersions) )) - - def warn(self, message): ui.log.log(" Warning: {}".format(message)) self.name += " [!]" From 7d69e9806b986a57268636d0e56a6058694e6f74 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 16:10:28 -0500 Subject: [PATCH 062/124] Let the Mod class build its description --- spacehaven-modloader.py | 11 +---------- ui/database.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index 89a59b05..c99570c3 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -328,16 +328,7 @@ def showMod(self, mod): self.modDetailsName.config(text = title) self.modEnableDisable.config(text = command_label) - description = mod.description - if mod.known_issues: - description += "\n\n" + "KNOWN ISSUES: " + mod.known_issues - if mod.author: - description += "\n\n" + "AUTHOR: " + mod.author - if mod.website: - # FIXME make it a separate textfield, can't select from this one - description += "\n\n" + "URL: " + mod.website - - self.update_description(description) + self.update_description(mod.getDescription()) def showModError(self, title, error): self.modDetailsName.config(text = title) diff --git a/ui/database.py b/ui/database.py index 849c0aec..42e8b5af 100644 --- a/ui/database.py +++ b/ui/database.py @@ -124,6 +124,18 @@ def title(self): title += " (%s)" % self.version return title + def getDescription(self): + """Build a description from the mod data""" + description = self.description + if self.known_issues: + description += "\n\n" + "KNOWN ISSUES: " + self.known_issues + if self.author: + description += "\n\n" + "AUTHOR: " + self.author + if self.website: + # FIXME make it a separate textfield, can't select from this one + description += "\n\n" + "URL: " + self.website + return description + def verifyLoaderVersion(self, mod): self.minimumLoaderVersion = mod.find("minimumLoaderVersion").text if distutils.version.StrictVersion(self.minimumLoaderVersion) > distutils.version.StrictVersion(version.version): From dd1b5a5260f780c566b1cdcb7fb6d926b5c25138 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 17:16:27 -0500 Subject: [PATCH 063/124] Add new metadata: Mod ID prefix --- ui/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/database.py b/ui/database.py index 42e8b5af..affb035f 100644 --- a/ui/database.py +++ b/ui/database.py @@ -94,6 +94,7 @@ def _optional(tag): self.author = _optional("author") self.website = _optional("website") self.updates = _optional("updates") + self.prefix = int(_optional("modPrefix") or "0") self.verifyLoaderVersion(mod) self.verifyGameVersion(mod, self.gameInfo) From a62f626298753fff1921a9c8375e86c65e3d7ff6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 17:17:49 -0500 Subject: [PATCH 064/124] Update mod description Author first Use F-strings, reads better include modid prefix --- ui/database.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ui/database.py b/ui/database.py index affb035f..b6aed4ac 100644 --- a/ui/database.py +++ b/ui/database.py @@ -127,14 +127,17 @@ def title(self): def getDescription(self): """Build a description from the mod data""" - description = self.description - if self.known_issues: - description += "\n\n" + "KNOWN ISSUES: " + self.known_issues + description = "" if self.author: - description += "\n\n" + "AUTHOR: " + self.author + description += f"AUTHOR: {self.author}\n" + description += self.description + "\n" + if self.known_issues: + description += "\n" + "KNOWN ISSUES: " + self.known_issues + if self.prefix: + description += f"\nPREFIX: {self.prefix}" if self.website: # FIXME make it a separate textfield, can't select from this one - description += "\n\n" + "URL: " + self.website + description += f"\nURL: {self.website}" return description def verifyLoaderVersion(self, mod): From 63eb0831a90c62fc266a960e5771201c5ba7ee15 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 17:21:25 -0500 Subject: [PATCH 065/124] Change mod ID tag to something more intuitive --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index b6aed4ac..c9b9181a 100644 --- a/ui/database.py +++ b/ui/database.py @@ -94,7 +94,7 @@ def _optional(tag): self.author = _optional("author") self.website = _optional("website") self.updates = _optional("updates") - self.prefix = int(_optional("modPrefix") or "0") + self.prefix = int(_optional("modid") or "0") self.verifyLoaderVersion(mod) self.verifyGameVersion(mod, self.gameInfo) From 0f8ebe06ec061e90bee9b0e439c3ef37f30b2e8f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 17:27:42 -0500 Subject: [PATCH 066/124] Sort usings --- ui/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/database.py b/ui/database.py index c9b9181a..7566739f 100644 --- a/ui/database.py +++ b/ui/database.py @@ -1,12 +1,12 @@ -import os import distutils.version - +import os from xml.etree import ElementTree import version -import ui.log + import ui.gameinfo +import ui.log class ModDatabase: From 312c92eaa6ba4b323f9640f525c4e67e1b3a5220 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Fri, 11 Dec 2020 17:52:23 -0500 Subject: [PATCH 067/124] ModDatabase tracks active mods --- ui/database.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index 7566739f..c62218fc 100644 --- a/ui/database.py +++ b/ui/database.py @@ -21,6 +21,8 @@ def __init__(self, path_list, gameInfo): def locateMods(self): self.mods = [] + self.modsActive = [] + self.modsInactive = [] ui.log.log("Locating mods...") for path in self.path_list: @@ -41,7 +43,12 @@ def locateMods(self): # no info file, don't create a mod. continue - self.mods.append(Mod(info_file, self.gameInfo)) + newMod = Mod(info_file, self.gameInfo) + self.mods.append(newMod) + if newMod.enabled: + self.modsActive.append(newMod) + else: + self.modsInactive.append(newMod) self.mods.sort(key=lambda mod: mod.name) From b1eca07f4bb9c9fa94705753288f2b68469b27f7 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 13:22:40 -0500 Subject: [PATCH 068/124] Add API to get active and inactive mods --- ui/database.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ui/database.py b/ui/database.py index c62218fc..312601f1 100644 --- a/ui/database.py +++ b/ui/database.py @@ -21,8 +21,6 @@ def __init__(self, path_list, gameInfo): def locateMods(self): self.mods = [] - self.modsActive = [] - self.modsInactive = [] ui.log.log("Locating mods...") for path in self.path_list: @@ -45,14 +43,26 @@ def locateMods(self): newMod = Mod(info_file, self.gameInfo) self.mods.append(newMod) - if newMod.enabled: - self.modsActive.append(newMod) - else: - self.modsInactive.append(newMod) self.mods.sort(key=lambda mod: mod.name) def getInstance(): + def getActiveMods(): + mod: Mod + return [ + mod + for mod in ModDatabase.getInstance().mods + if mod.enabled + ] + + def getInactiveMods(): + mod: Mod + return [ + mod + for mod in ModDatabase.getInstance().mods + if not mod.enabled + ] + """Return the last generated instance of a mod database.""" if ModDatabase is None: raise Exception("Mod Database not ready.") From b2f9d124f654852a1ee0c1f884a7f62d1eea8246 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 13:45:26 -0500 Subject: [PATCH 069/124] Add API to get whether or not mod database is empty --- ui/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/database.py b/ui/database.py index 312601f1..2dec9c77 100644 --- a/ui/database.py +++ b/ui/database.py @@ -63,6 +63,9 @@ def getInactiveMods(): if not mod.enabled ] + def isEmpty(self): + return not len(self.mods) + """Return the last generated instance of a mod database.""" if ModDatabase is None: raise Exception("Mod Database not ready.") From 9f6bc471d0edc16ce96dc2706ecf42c1c2783196 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 14:22:28 -0500 Subject: [PATCH 070/124] Use API instead of doing work in Window --- spacehaven-modloader.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index c99570c3..a4451112 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -282,7 +282,7 @@ def update_list_style(self, mod): self.modList.itemconfig(mod.display_idx, foreground = 'grey', selectforeground = 'lightgrey') def selected_mod(self): - if not len(self.modDatabase.mods): + if self.modDatabase.isEmpty(): return None if len(self.modList.curselection()) == 0: self.modList.selection_set(0) @@ -416,7 +416,7 @@ def annotate(self): ui.launcher.open(os.path.join(corePath, 'library')) def mods_enabled(self): - return [mod for mod in self.modDatabase.mods if mod.enabled] + return ui.database.ModDatabase.getActiveMods() def current_mods_signature(self): import hashlib @@ -482,11 +482,7 @@ def quick_launch(self): messagebox.showerror("Error during quick launch", traceback.format_exc(3)) def patchAndLaunch(self): - activeModPaths = [] - for mod in self.modDatabase.mods: - if not mod.enabled: - continue - activeModPaths.append(mod.path) + activeModPaths = [mod.path for mod in ui.database.ModDatabase.getActiveMods()] try: loader.load.load(self.jarPath, activeModPaths, self.current_mods_signature()) From 8c40724c3aa4d48754445c3faf52f1d7bbb6e500 Mon Sep 17 00:00:00 2001 From: surface level speck Date: Sat, 12 Dec 2020 13:36:47 -0600 Subject: [PATCH 071/124] auto-detects Steam install directory for Windows 32-bit & 64-bit systems --- .gitignore | 1 + requirements.txt | 1 + spacehaven-modloader.py | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/.gitignore b/.gitignore index bdf9139e..09749292 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ quicklaunch_*/ extra_mods_path.txt env/ .vscode/.ropeproject/ +.vscode/settings.json diff --git a/requirements.txt b/requirements.txt index a5715ca1..8e1195c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ six==1.15.0 toml==0.10.2 typed-ast==1.4.1 wrapt==1.12.1 +steamfiles==0.1.4 \ No newline at end of file diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index 89a59b05..ef3ff045 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -5,6 +5,8 @@ import threading import traceback +from collections import OrderedDict +from steamfiles import acf from tkinter import filedialog from tkinter import messagebox from tkinter import * @@ -18,6 +20,8 @@ import loader.load import version +import winreg +import platform POSSIBLE_SPACEHAVEN_LOCATIONS = [ # MacOS @@ -158,6 +162,22 @@ def autolocateSpacehaven(self): traceback.print_exc() pass + try: + registry_path = "SOFTWARE\\WOW6432Node\\Valve\\Steam" if (platform.architecture()[0] == "64bit") else "SOFTWARE\\Valve\\Steam" + steam_path = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_path), "InstallPath")[0] + library_folders = acf.load(open(steam_path + "\\steamapps\\libraryfolders.vdf"), wrapper=OrderedDict) + locations = [steam_path + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe"] + for key, value in library_folders["LibraryFolders"].items(): + if str.isnumeric(key): locations.append(value + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe") + for location in locations: + if os.path.exists(location): + self.locateSpacehaven(location) + return + except: + import traceback + traceback.print_exc() + pass + for location in POSSIBLE_SPACEHAVEN_LOCATIONS: try: location = os.path.abspath(location) From 83176cb8c01b65d08e693118a968e2d922a55646 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 14:55:39 -0500 Subject: [PATCH 072/124] Return type documentation --- ui/database.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ui/database.py b/ui/database.py index 2dec9c77..b62d8e18 100644 --- a/ui/database.py +++ b/ui/database.py @@ -1,4 +1,5 @@ +from __future__ import annotations # Required to annotate ModDatabase.getInstance() with own type import distutils.version import os from xml.etree import ElementTree @@ -46,8 +47,7 @@ def locateMods(self): self.mods.sort(key=lambda mod: mod.name) - def getInstance(): - def getActiveMods(): + def getActiveMods() -> list[Mod]: mod: Mod return [ mod @@ -55,7 +55,7 @@ def getActiveMods(): if mod.enabled ] - def getInactiveMods(): + def getInactiveMods() -> list[Mod]: mod: Mod return [ mod @@ -63,9 +63,10 @@ def getInactiveMods(): if not mod.enabled ] - def isEmpty(self): + def isEmpty(self) -> bool: return not len(self.mods) + def getInstance() -> ModDatabase: """Return the last generated instance of a mod database.""" if ModDatabase is None: raise Exception("Mod Database not ready.") From a67af684a897bebfd35aae06fed0bea4f37e753e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 14:58:00 -0500 Subject: [PATCH 073/124] Ensure self.mods is always defined --- ui/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/database.py b/ui/database.py index b62d8e18..2002a77c 100644 --- a/ui/database.py +++ b/ui/database.py @@ -17,6 +17,7 @@ class ModDatabase: def __init__(self, path_list, gameInfo): self.path_list = path_list self.gameInfo = gameInfo + self.mods = [] self.locateMods() ModDatabase.__lastInstance = self From 611a3618536969a4ac68f7b52da6ff848de83eb5 Mon Sep 17 00:00:00 2001 From: surface level speck Date: Sat, 12 Dec 2020 14:03:48 -0600 Subject: [PATCH 074/124] made fixes per conversation --- spacehaven-modloader.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index ef3ff045..e46b6349 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -162,21 +162,16 @@ def autolocateSpacehaven(self): traceback.print_exc() pass - try: - registry_path = "SOFTWARE\\WOW6432Node\\Valve\\Steam" if (platform.architecture()[0] == "64bit") else "SOFTWARE\\Valve\\Steam" - steam_path = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_path), "InstallPath")[0] - library_folders = acf.load(open(steam_path + "\\steamapps\\libraryfolders.vdf"), wrapper=OrderedDict) - locations = [steam_path + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe"] - for key, value in library_folders["LibraryFolders"].items(): - if str.isnumeric(key): locations.append(value + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe") - for location in locations: - if os.path.exists(location): - self.locateSpacehaven(location) - return - except: - import traceback - traceback.print_exc() - pass + registry_path = "SOFTWARE\\WOW6432Node\\Valve\\Steam" if (platform.architecture()[0] == "64bit") else "SOFTWARE\\Valve\\Steam" + steam_path = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_path), "InstallPath")[0] + library_folders = acf.load(open(steam_path + "\\steamapps\\libraryfolders.vdf"), wrapper=OrderedDict) + locations = [steam_path + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe"] + for key, value in library_folders["LibraryFolders"].items(): + if str.isnumeric(key): locations.append(value + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe") + for location in locations: + if os.path.exists(location): + self.locateSpacehaven(location) + return for location in POSSIBLE_SPACEHAVEN_LOCATIONS: try: @@ -186,7 +181,7 @@ def autolocateSpacehaven(self): return except: pass - + def locateSpacehaven(self, path): if path is None: return From 49c002718fe373e216404db745b0e383659d72f4 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 18:25:41 -0500 Subject: [PATCH 075/124] Allocate core region IDs to arbitrary texture filenames --- loader/assets/merge.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 574efa63..a85489a9 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -19,26 +19,26 @@ def _detect_textures(coreLibrary, modLibrary, mod): modded_textures = {} seen_textures = set() - def _add_texture(region_id): - filename = region_id + '.png' - if filename in seen_textures: + def _add_texture(filename): + region_id = str.join(".", filename.split('.')[:-1]) + # Early exit if this texture exists + if (region_id in modded_textures) or (region_id in mapping_n_region): return path = os.path.join(textures_path, filename) - if not os.path.isfile(path): - ui.log.log(" ERROR MISSING {}...".format(filename)) - ui.log.log(" ERROR MISSING {}...".format(filename)) - ui.log.log(" ERROR MISSING {}...".format(filename)) - return + # Removed file existence check - file should already exist given how this function is being called + # If the file no longer exists, let the program thrown an error later (plus the file might be + # deleted by later anyway) - ui.log.log(" Found {}...".format(filename)) - if int(region_id) > coreLibrary['_last_core_region_id']: + if not region_id.isdecimal() or int(region_id) > coreLibrary['_last_core_region_id']: # adding a new texture, this gets tricky as they have to have consecutive numbers. core_region_id = str(coreLibrary['_next_region_id']) mapping_n_region[region_id] = core_region_id coreLibrary['_next_region_id'] += 1 + ui.log.log(f" Allocated new core region idx {core_region_id:>5} to file {filename}") else: core_region_id = region_id + ui.log.log(f" Mod updated texture region {core_region_id}") seen_textures.add(filename) modded_textures[core_region_id] = { @@ -51,12 +51,7 @@ def _add_texture(region_id): # also scan the directory for overwriting existing core textures if not filename.endswith('.png'): continue - try: - int(filename.split('.')[0]) - except: - # wrong format - continue - _add_texture(filename.split('.')[0]) + _add_texture(filename) if 'library/textures' not in modLibrary: # no textures.xml file, we're done @@ -83,7 +78,7 @@ def _add_texture(region_id): if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] - ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) + #ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) asset.set('a', new_id) for asset in textures_mod.xpath("//re[@n]"): From 8c5b7b2e41c26b1139cc4b1bef23107cb52486fc Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 12 Dec 2020 23:36:50 -0500 Subject: [PATCH 076/124] BUGFIX: don't split non-filenames on dot arguments passed into _add_texture without a dot were being split on nothing, then joined with everything but the last element, resulting in a blank string. --- loader/assets/merge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index a85489a9..4eee366e 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -20,7 +20,10 @@ def _detect_textures(coreLibrary, modLibrary, mod): seen_textures = set() def _add_texture(filename): - region_id = str.join(".", filename.split('.')[:-1]) + if '.' in filename: + region_id = str.join(".", filename.split('.')[:-1]) + else: + region_id = filename # Early exit if this texture exists if (region_id in modded_textures) or (region_id in mapping_n_region): return From 3d24efdc7c2b8934cbc0a104ea3c28842a4d21dd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 00:17:29 -0500 Subject: [PATCH 077/124] Walk subdirectories in moddir/textures for texture files --- loader/assets/merge.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 4eee366e..79064715 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -11,7 +11,7 @@ def _detect_textures(coreLibrary, modLibrary, mod): - textures_path = os.path.join(mod, 'textures') + textures_path = os.path.join(mod, 'textures').replace("\\","/") if not os.path.isdir(textures_path): return {} @@ -50,7 +50,14 @@ def _add_texture(filename): 'path' : path, } - for filename in os.listdir(textures_path): + texture_files = [ + f"{cur}/{file}".replace("\\","/") + for cur, dirs, files in os.walk(textures_path) + for file in files + if ".png" in file and "modded_cim" not in file + ] + texture_files = [x.replace(textures_path + "/", "") for x in texture_files] + for filename in texture_files: # also scan the directory for overwriting existing core textures if not filename.endswith('.png'): continue From 279063ac6afe5f7978511fc6f4879039f23c4128 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 01:47:39 -0500 Subject: [PATCH 078/124] Automatically generate texture and region metadata for custom assets --- loader/assets/merge.py | 49 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 79064715..54f8d4b8 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -1,6 +1,9 @@ import copy import os +import png +import rectpack +import math import loader.assets.library import lxml.etree @@ -82,15 +85,59 @@ def _add_texture(filename): # no custom mod textures, no need to remap ids return modded_textures + needs_autogeneration = [] for animation_chunk in modLibrary['library/animations']: for asset in animation_chunk.xpath("//assetPos[@a]"): - mod_local_id = asset.get('a') + mod_local_id = asset.get("filename") + if mod_local_id is None: + mod_local_id = asset.get('a') + elif mod_local_id not in needs_autogeneration: + needs_autogeneration.append(mod_local_id) if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] #ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) asset.set('a', new_id) + if len(needs_autogeneration): + import random + random.seed() + regionsNode = textures_mod.find("//regions") + texturesNode = textures_mod.find("//textures") + textureID = random.randrange(len(PATCHABLE_CIM_FILES), 500) + packer = rectpack.newPacker(rotation=False) + sum = 0 + minRequiredDimension = 0 + # First get all the files and pack them into a new texture square + for regionName in needs_autogeneration: + (w, h, rows, info) = png.Reader(textures_path + "/" + regionName + ".png").asRGBA() + packer.add_rect(w, h, regionName) + minRequiredDimension = max(minRequiredDimension, w, h) + sum += (w * h) + + size = max(int(math.sqrt(sum) * 1.2), minRequiredDimension) + + newTex = lxml.etree.SubElement(texturesNode, "t") + newTex.set("i", str(textureID)) + newTex.set("w", str(size)) + newTex.set("h", str(size)) + coreLibrary['_custom_textures_cim'][str(textureID)] = newTex.attrib + + packer.add_bin(size, size) + packer.pack() + + for rect in packer.rect_list(): + b, x, y, w, h, rid = rect + remappedID = mapping_n_region[rid] + remapData = modded_textures[remappedID] + newNode = lxml.etree.SubElement(regionsNode, "re") + newNode.set("n", remappedID) + newNode.set("t", str(textureID)) + newNode.set("x", str(x)) + newNode.set("y", str(y)) + newNode.set("w", str(w)) + newNode.set("h", str(h)) + for asset in textures_mod.xpath("//re[@n]"): mod_local_id = asset.get('n') if mod_local_id not in mapping_n_region: From f2985a583246183e10005e732c8d67ac0cee776e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 02:19:17 -0500 Subject: [PATCH 079/124] Change way texture regions are added to the list of remaps This fixes a bug - previously, if a file was remapped but no region was created for it then Space Haven would crash on load because the highest region index would be higher than the actual number of regions. --- loader/assets/merge.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 54f8d4b8..705ae5b0 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -23,20 +23,22 @@ def _detect_textures(coreLibrary, modLibrary, mod): seen_textures = set() def _add_texture(filename): - if '.' in filename: - region_id = str.join(".", filename.split('.')[:-1]) - else: - region_id = filename - # Early exit if this texture exists + filename += ".png" + region_id = str.join(".", filename.split('.')[:-1]) + isCoreRegion = region_id.isdecimal() and int(region_id) <= coreLibrary['_last_core_region_id'] if (region_id in modded_textures) or (region_id in mapping_n_region): + # Early exit if this texture exists return path = os.path.join(textures_path, filename) + if isCoreRegion and not os.path.exists(path): + #core region file without an associated file, return early + return # Removed file existence check - file should already exist given how this function is being called # If the file no longer exists, let the program thrown an error later (plus the file might be # deleted by later anyway) - if not region_id.isdecimal() or int(region_id) > coreLibrary['_last_core_region_id']: + if not isCoreRegion: # adding a new texture, this gets tricky as they have to have consecutive numbers. core_region_id = str(coreLibrary['_next_region_id']) mapping_n_region[region_id] = core_region_id @@ -53,19 +55,6 @@ def _add_texture(filename): 'path' : path, } - texture_files = [ - f"{cur}/{file}".replace("\\","/") - for cur, dirs, files in os.walk(textures_path) - for file in files - if ".png" in file and "modded_cim" not in file - ] - texture_files = [x.replace(textures_path + "/", "") for x in texture_files] - for filename in texture_files: - # also scan the directory for overwriting existing core textures - if not filename.endswith('.png'): - continue - _add_texture(filename) - if 'library/textures' not in modLibrary: # no textures.xml file, we're done return modded_textures @@ -93,6 +82,7 @@ def _add_texture(filename): mod_local_id = asset.get('a') elif mod_local_id not in needs_autogeneration: needs_autogeneration.append(mod_local_id) + _add_texture(mod_local_id) if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] From 67ecea38c2e8fb25604d1aa77132a6cfdc98c912 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 02:39:10 -0500 Subject: [PATCH 080/124] Update requirements --- requirements.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/requirements.txt b/requirements.txt index 8e1195c5..0946b1ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,15 +5,24 @@ colorama==0.4.4 future==0.18.2 isort==5.6.4 lazy-object-proxy==1.4.3 +linecache2==1.0.0 lxml==4.6.2 mccabe==0.6.1 +nose==1.3.7 pefile==2019.4.18 +protobuf==4.0.0rc2 +protobuf3-to-dict==0.1.5 pyinstaller==4.1 pyinstaller-hooks-contrib==2020.10 pypng==0.0.20 pywin32-ctypes==0.2.0 +rectpack==0.2.1 +rope==0.18.0 six==1.15.0 +steamfiles==0.1.4 toml==0.10.2 +traceback2==1.4.0 typed-ast==1.4.1 +unittest2==1.1.0 wrapt==1.12.1 steamfiles==0.1.4 \ No newline at end of file From c4fe910db5df4e047c6cb500f5e8ae69d8650d91 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 18:46:51 -0500 Subject: [PATCH 081/124] Database tracks prefixes --- ui/database.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ui/database.py b/ui/database.py index 2002a77c..688939a6 100644 --- a/ui/database.py +++ b/ui/database.py @@ -13,6 +13,7 @@ class ModDatabase: """Information about a collection of mods""" __lastInstance = None + Prefixes = {} def __init__(self, path_list, gameInfo): self.path_list = path_list @@ -23,6 +24,7 @@ def __init__(self, path_list, gameInfo): def locateMods(self): self.mods = [] + ModDatabase.Prefixes = {} ui.log.log("Locating mods...") for path in self.path_list: @@ -44,6 +46,11 @@ def locateMods(self): continue newMod = Mod(info_file, self.gameInfo) + if newMod.prefix: + if newMod.prefix in ModDatabase.Prefixes and ModDatabase.Prefixes[newMod.prefix]: + ui.log.log(f" Warning: Mod prefix {newMod.prefix} for mod {newMod.title()} is already in use.") + else: + ModDatabase.Prefixes[newMod.prefix] = newMod.enabled self.mods.append(newMod) self.mods.sort(key=lambda mod: mod.name) From 546fb885d23acb01605f311a1934a2f7e2e118e5 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 18:48:02 -0500 Subject: [PATCH 082/124] Add API to get a mod from the database when path is known --- ui/database.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui/database.py b/ui/database.py index 688939a6..75007f70 100644 --- a/ui/database.py +++ b/ui/database.py @@ -71,6 +71,12 @@ def getInactiveMods() -> list[Mod]: if not mod.enabled ] + def getMod(modPath): + """Get a specific mod from its installation path.""" + for mod in ModDatabase.getInstance().mods: + if mod.path == modPath: + return mod + def isEmpty(self) -> bool: return not len(self.mods) From b2d4c22baa171bcb130aace6f8c25ab2acb499e0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 18:49:18 -0500 Subject: [PATCH 083/124] Switch to class level type documentation --- ui/database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/database.py b/ui/database.py index 75007f70..41a99400 100644 --- a/ui/database.py +++ b/ui/database.py @@ -14,6 +14,7 @@ class ModDatabase: """Information about a collection of mods""" __lastInstance = None Prefixes = {} + mods: list[Mod] def __init__(self, path_list, gameInfo): self.path_list = path_list @@ -56,7 +57,6 @@ def locateMods(self): self.mods.sort(key=lambda mod: mod.name) def getActiveMods() -> list[Mod]: - mod: Mod return [ mod for mod in ModDatabase.getInstance().mods @@ -64,7 +64,6 @@ def getActiveMods() -> list[Mod]: ] def getInactiveMods() -> list[Mod]: - mod: Mod return [ mod for mod in ModDatabase.getInstance().mods From 781538814c8cff7b5191c422553bae2deffb2e6f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 19:13:20 -0500 Subject: [PATCH 084/124] Declare static methods correctly --- ui/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/database.py b/ui/database.py index 41a99400..11bb53ec 100644 --- a/ui/database.py +++ b/ui/database.py @@ -56,6 +56,7 @@ def locateMods(self): self.mods.sort(key=lambda mod: mod.name) + @staticmethod def getActiveMods() -> list[Mod]: return [ mod @@ -63,6 +64,7 @@ def getActiveMods() -> list[Mod]: if mod.enabled ] + @staticmethod def getInactiveMods() -> list[Mod]: return [ mod @@ -70,6 +72,7 @@ def getInactiveMods() -> list[Mod]: if not mod.enabled ] + @staticmethod def getMod(modPath): """Get a specific mod from its installation path.""" for mod in ModDatabase.getInstance().mods: @@ -79,6 +82,7 @@ def getMod(modPath): def isEmpty(self) -> bool: return not len(self.mods) + @staticmethod def getInstance() -> ModDatabase: """Return the last generated instance of a mod database.""" if ModDatabase is None: From cf36d89ba91bd47d333e351455cec31d1b358241 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 13 Dec 2020 19:16:10 -0500 Subject: [PATCH 085/124] Group static methods --- ui/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/database.py b/ui/database.py index 11bb53ec..cad1ff2b 100644 --- a/ui/database.py +++ b/ui/database.py @@ -56,6 +56,9 @@ def locateMods(self): self.mods.sort(key=lambda mod: mod.name) + def isEmpty(self) -> bool: + return not len(self.mods) + @staticmethod def getActiveMods() -> list[Mod]: return [ @@ -79,9 +82,6 @@ def getMod(modPath): if mod.path == modPath: return mod - def isEmpty(self) -> bool: - return not len(self.mods) - @staticmethod def getInstance() -> ModDatabase: """Return the last generated instance of a mod database.""" From 2c393a65f6cac491e90ccea43fd9404fde2c5114 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 01:20:54 -0500 Subject: [PATCH 086/124] Change to class methods --- ui/database.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ui/database.py b/ui/database.py index cad1ff2b..7aadd899 100644 --- a/ui/database.py +++ b/ui/database.py @@ -59,35 +59,35 @@ def locateMods(self): def isEmpty(self) -> bool: return not len(self.mods) - @staticmethod - def getActiveMods() -> list[Mod]: + @classmethod + def getActiveMods(cls) -> list[Mod]: return [ mod - for mod in ModDatabase.getInstance().mods + for mod in cls.getInstance().mods if mod.enabled ] - @staticmethod - def getInactiveMods() -> list[Mod]: + @classmethod + def getInactiveMods(cls) -> list[Mod]: return [ mod - for mod in ModDatabase.getInstance().mods + for mod in cls.getInstance().mods if not mod.enabled ] - @staticmethod - def getMod(modPath): + @classmethod + def getMod(cls, modPath): """Get a specific mod from its installation path.""" - for mod in ModDatabase.getInstance().mods: + for mod in cls.getInstance().mods: if mod.path == modPath: return mod - @staticmethod - def getInstance() -> ModDatabase: + @classmethod + def getInstance(cls) -> ModDatabase: """Return the last generated instance of a mod database.""" - if ModDatabase is None: + if cls.__lastInstance is None: raise Exception("Mod Database not ready.") - return ModDatabase.__lastInstance + return cls.__lastInstance DISABLED_MARKER = "disabled.txt" class Mod: From 355f17031be815891bb5a39f35a3c3d30a04c9df Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 01:47:30 -0500 Subject: [PATCH 087/124] Add class method API to get registered mods --- ui/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/database.py b/ui/database.py index 7aadd899..84280e9c 100644 --- a/ui/database.py +++ b/ui/database.py @@ -75,6 +75,10 @@ def getInactiveMods(cls) -> list[Mod]: if not mod.enabled ] + @classmethod + def getRegisteredMods(cls) -> list[Mod]: + return cls.getInstance().mods + @classmethod def getMod(cls, modPath): """Get a specific mod from its installation path.""" From 34e97e78cdd652cf41af539b30a60fa8b30259a2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 02:31:29 -0500 Subject: [PATCH 088/124] Burn out hard link to ModDatabase and allow for database handler swapping --- spacehaven-modloader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index a4451112..57fcaa35 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -41,6 +41,7 @@ "~/Games/SpaceHaven/spacehaven", ".local/share/Steam/steamapps/common/SpaceHaven/spacehaven", ] +DatabaseHandler = ui.database.ModDatabase class Window(Frame): def __init__(self, master=None): @@ -215,6 +216,7 @@ def locateSpacehaven(self, path): except: pass + DatabaseHandler(self.modPath, self.gameInfo) self.refreshModList() def checkForLoadedMods(self): @@ -260,10 +262,10 @@ def refreshModList(self): self.showModError("Spacehaven not found", "Please use the 'Find game' button below to locate Spacehaven.") return - self.modDatabase = ui.database.ModDatabase(self.modPath, self.gameInfo) + DatabaseHandler.getInstance().locateMods() mod_idx = 0 - for mod in self.modDatabase.mods: + for mod in DatabaseHandler.getRegisteredMods(): self.modList.insert(END, mod.name) mod.display_idx = mod_idx @@ -282,7 +284,7 @@ def update_list_style(self, mod): self.modList.itemconfig(mod.display_idx, foreground = 'grey', selectforeground = 'lightgrey') def selected_mod(self): - if self.modDatabase.isEmpty(): + if DatabaseHandler.getInstance().isEmpty(): return None if len(self.modList.curselection()) == 0: self.modList.selection_set(0) @@ -290,7 +292,7 @@ def selected_mod(self): else: selected = self.modList.curselection()[0] - return self.modDatabase.mods[self.modList.curselection()[0]] + return DatabaseHandler.getRegisteredMods()[self.modList.curselection()[0]] def showCurrentMod(self, _arg=None): self.showMod(self.selected_mod()) @@ -416,7 +418,7 @@ def annotate(self): ui.launcher.open(os.path.join(corePath, 'library')) def mods_enabled(self): - return ui.database.ModDatabase.getActiveMods() + return DatabaseHandler.getActiveMods() def current_mods_signature(self): import hashlib @@ -482,7 +484,7 @@ def quick_launch(self): messagebox.showerror("Error during quick launch", traceback.format_exc(3)) def patchAndLaunch(self): - activeModPaths = [mod.path for mod in ui.database.ModDatabase.getActiveMods()] + activeModPaths = [mod.path for mod in DatabaseHandler.getActiveMods()] try: loader.load.load(self.jarPath, activeModPaths, self.current_mods_signature()) From 177eb46467fade6648bb61f714f8bad8d442b4c8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 21:28:44 -0500 Subject: [PATCH 089/124] Remove un-needed code --- ui/database.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index 84280e9c..f7a1c6d9 100644 --- a/ui/database.py +++ b/ui/database.py @@ -19,7 +19,6 @@ class ModDatabase: def __init__(self, path_list, gameInfo): self.path_list = path_list self.gameInfo = gameInfo - self.mods = [] self.locateMods() ModDatabase.__lastInstance = self From 18901ee63f9fa49fc771d43818de53f71cd4cbe5 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 22:01:17 -0500 Subject: [PATCH 090/124] Pull Mod ID from mod database for CIM ID --- loader/assets/merge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 705ae5b0..3024ecd9 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -8,6 +8,7 @@ import loader.assets.library import lxml.etree import ui.log +import ui.database from .explode import Texture from .library import PATCHABLE_CIM_FILES, PATCHABLE_XML_FILES @@ -90,11 +91,9 @@ def _add_texture(filename): asset.set('a', new_id) if len(needs_autogeneration): - import random - random.seed() regionsNode = textures_mod.find("//regions") texturesNode = textures_mod.find("//textures") - textureID = random.randrange(len(PATCHABLE_CIM_FILES), 500) + textureID = ui.database.ModDatabase.getMod(mod).prefix packer = rectpack.newPacker(rotation=False) sum = 0 minRequiredDimension = 0 From 4bd4e29915adb3bca608dd88a1c4c35626dc70e7 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 22:01:29 -0500 Subject: [PATCH 091/124] Sort imports --- loader/assets/merge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 3024ecd9..79fa9aad 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -1,12 +1,12 @@ import copy -import os -import png -import rectpack import math +import os import loader.assets.library import lxml.etree +import png +import rectpack import ui.log import ui.database From 3f0cab674d529c5498e12ef38b4b9d19571c7078 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 14 Dec 2020 22:02:02 -0500 Subject: [PATCH 092/124] Add return value typing --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index f7a1c6d9..665acbff 100644 --- a/ui/database.py +++ b/ui/database.py @@ -79,7 +79,7 @@ def getRegisteredMods(cls) -> list[Mod]: return cls.getInstance().mods @classmethod - def getMod(cls, modPath): + def getMod(cls, modPath) -> Mod: """Get a specific mod from its installation path.""" for mod in cls.getInstance().mods: if mod.path == modPath: From 06ea2a157fe8778c1b2187af17b11a89a49630e3 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 00:52:32 -0500 Subject: [PATCH 093/124] Sort textures by remap before creating nodes --- loader/assets/merge.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 79fa9aad..2ce35c2c 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -115,17 +115,26 @@ def _add_texture(filename): packer.add_bin(size, size) packer.pack() + packedRectsSorted = {} for rect in packer.rect_list(): b, x, y, w, h, rid = rect remappedID = mapping_n_region[rid] + packedRectsSorted[remappedID] = (str(x), str(y), str(w), str(h), str(rid)) + # NOT YET SORTED + packedRectsSorted = {k: v for k,v in sorted(packedRectsSorted.items())} + # NOW SORTED: We need this to make sure the IDs are added to the textures file in the correct order + + for remappedID, data in packedRectsSorted.items(): + x, y, w, h, regionFileName = data remapData = modded_textures[remappedID] newNode = lxml.etree.SubElement(regionsNode, "re") newNode.set("n", remappedID) newNode.set("t", str(textureID)) - newNode.set("x", str(x)) - newNode.set("y", str(y)) - newNode.set("w", str(w)) - newNode.set("h", str(h)) + newNode.set("x", x) + newNode.set("y", y) + newNode.set("w", w) + newNode.set("h", h) + newNode.set("file", regionFileName) for asset in textures_mod.xpath("//re[@n]"): mod_local_id = asset.get('n') From db0527690b5e62b229d9e444bf7c817639eb64d6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 01:01:30 -0500 Subject: [PATCH 094/124] Allow for fully automatic texture files --- loader/assets/merge.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2ce35c2c..9aa4220b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -56,9 +56,20 @@ def _add_texture(filename): 'path' : path, } - if 'library/textures' not in modLibrary: + autoAnimations = False + for animation_chunk in modLibrary['library/animations']: + filenameAssetPos = animation_chunk.find("//assetPos[@filename]") + if filenameAssetPos is not None: + autoAnimations = True + + if 'library/textures' not in modLibrary and not autoAnimations: # no textures.xml file, we're done return modded_textures + if 'library/textures' not in modLibrary and autoAnimations: + texRoot = lxml.etree.Element("AllTexturesAndRegions") + lxml.etree.SubElement(texRoot, "textures") + lxml.etree.SubElement(texRoot, "regions") + modLibrary['library/textures'] = [lxml.etree.ElementTree(texRoot)] #FIXME verify that there's only one file textures_mod = modLibrary['library/textures'][0] @@ -71,7 +82,7 @@ def _add_texture(filename): region_id = region.get('n') _add_texture(region_id) - if not mapping_n_region: + if not mapping_n_region and not autoAnimations: # no custom mod textures, no need to remap ids return modded_textures From a6a4a273a93739723086a75bf2ee5663e94185e2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 01:15:45 -0500 Subject: [PATCH 095/124] Bump version number New feature: installation autolocation via steam New feature: automatic texture mapping --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 057d865f..fc2091ff 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.8.2" +version = "0.9.0a1" From 76c97adfda34fb77a9294adfca6ac9ba8225d18f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 03:01:56 -0500 Subject: [PATCH 096/124] Try again to get proper deactivation --- tools/build-windows.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/build-windows.sh b/tools/build-windows.sh index 33b2c5bc..b77b099c 100644 --- a/tools/build-windows.sh +++ b/tools/build-windows.sh @@ -1,10 +1,12 @@ #!/bin/bash rm -rf dist/spacehaven-modloader +OLDPATH=$PATH source env/Scripts/activate python -m PyInstaller --noconsole modloader.spec VERSION=`python -c 'import version; print(version.version)'` deactivate +PATH=$OLDPATH rm -rf dist/spacehaven-modloader-$VERSION.windows mv dist/spacehaven-modloader dist/spacehaven-modloader-$VERSION.windows From d638c5d21b9c209d7dd96a58a89efae43e4ec300 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 03:07:27 -0500 Subject: [PATCH 097/124] Add todo list --- TODO.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..b09a622b --- /dev/null +++ b/TODO.md @@ -0,0 +1,6 @@ +# Codebase +- PEP 8 compliance +- Better file structure + +# Repository +- Create a powershell build script for Windows From 4525c1e38c681c2b51df08c68cad0b29f8afcc49 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 04:08:29 -0500 Subject: [PATCH 098/124] Update changelog --- CHANGELOG.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 300b9600..c90aedd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,32 @@ # Change Log +## v0.9.0 +- On Windows, the game will be autolocated via Steam if possible. +- `` tag in info.xml: Defines a prefix that can be used in various places during mod creation. +- Automatic texture packing: instead of defining a `textures` file, texture regions can be defined as needed in `animations`. Add a `filename=""` attribute to the `` tag, and it will be packed automatically into `modid.cim`. This will fail if a mod ID is not specified. Textures must still be located in `moddir/textures` and paths are relative to this directory. + +## v0.8.2 +Bugfix: textures were not being merged in due to missing file during the build process + +## v0.8.1 +- New PatchOperation: AttributeMath (not in PatchOperation specification) + - Requires an addtional attribute on the `` tag: opType + - Supported operations: `add`, `subtract`, `multiply`, `divide` +- `` tag - Prevents a patch from loading (good for optional patches, or patches in development) +- Version bump to 0.8.1 to allow for better versioning in the future. +- General code refactoring + +## v0.0.8 +Support for [PatchOperation](https://rimworldwiki.com/wiki/Modding_Tutorials/PatchOperations) modding +- AttributeSet -> PatchOperationAttributeSet +- AttributeAdd -> PatchOperationAttributeAdd +- AttributeRemove -> PatchOperationAttributeRemove +- Add -> PatchOperationAdd +- Insert -> PatchOperationInsert +- Remove -> PatchOperationRemove +- Replace -> PatchOperationReplace + +Patches are loaded from `moddir/patches`, and are loaded after merge-by-id to allow modding other mods and to prevent clobbering. +Patch failure is logged to logs.txt ## v0.0.2 - Adds support for patching all definitions in `library/haven`, not just `Elements` and `Products`. @@ -7,4 +35,4 @@ - Adds game version checking/warnings ## v0.0.1 -Initial Release \ No newline at end of file +Initial Release From d1c36eec7555a55d00723a37d2852419b554f48c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 04:09:06 -0500 Subject: [PATCH 099/124] Update TODO --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index b09a622b..8a876754 100644 --- a/TODO.md +++ b/TODO.md @@ -4,3 +4,4 @@ # Repository - Create a powershell build script for Windows +- Remove need for textures_annotations.xml? From adb3aa0ad97686690326585e930bfd95256e00f9 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 04:20:41 -0500 Subject: [PATCH 100/124] Update TODO --- TODO.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TODO.md b/TODO.md index 8a876754..bf776d70 100644 --- a/TODO.md +++ b/TODO.md @@ -5,3 +5,8 @@ # Repository - Create a powershell build script for Windows - Remove need for textures_annotations.xml? +- Update README.md + - At least update the modloader image + - Then the Modding Guide + - Then other parts +- Look into github wikis From 5117cb2098a8ea68ec7454a5b6e8d7c561b7e2aa Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 04:26:37 -0500 Subject: [PATCH 101/124] Change to reference-style links --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c90aedd0..919b3721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Bugfix: textures were not being merged in due to missing file during the build p - General code refactoring ## v0.0.8 -Support for [PatchOperation](https://rimworldwiki.com/wiki/Modding_Tutorials/PatchOperations) modding +Support for [PatchOperation][1] modding - AttributeSet -> PatchOperationAttributeSet - AttributeAdd -> PatchOperationAttributeAdd - AttributeRemove -> PatchOperationAttributeRemove @@ -36,3 +36,6 @@ Patch failure is logged to logs.txt ## v0.0.1 Initial Release + + +[1]: From 09d616c7169cb6a95652db84ec3b23cc459f2b5c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 12:03:18 -0500 Subject: [PATCH 102/124] Add API to map internal IDs to global IDs Allows modders to define a prefix once and only need to worry about keeping internal IDs unique. --- ui/database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ui/database.py b/ui/database.py index 665acbff..aff8af81 100644 --- a/ui/database.py +++ b/ui/database.py @@ -181,6 +181,14 @@ def getDescription(self): description += f"\nURL: {self.website}" return description + def getAutomaticID(self, internalID): + """Returns a new ID prefixed by the mod prefix.""" + autoIDAllocatedSize = 1000 + id = self.prefix * autoIDAllocatedSize + internalID + if internalID > autoIDAllocatedSize: + raise RuntimeError(f"{self.title()} requested an ID outside of the auto-ID allocation limit ({internalID} limit {autoIDAllocatedSize}). File a bug report.") + return str(id) + def verifyLoaderVersion(self, mod): self.minimumLoaderVersion = mod.find("minimumLoaderVersion").text if distutils.version.StrictVersion(self.minimumLoaderVersion) > distutils.version.StrictVersion(version.version): From 5de37dde961d15b19f536bf10343374e3c6eb482 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Tue, 15 Dec 2020 12:36:54 -0500 Subject: [PATCH 103/124] Prevent double-allocation of internal IDs --- ui/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/database.py b/ui/database.py index aff8af81..5faa6b27 100644 --- a/ui/database.py +++ b/ui/database.py @@ -103,6 +103,7 @@ def __init__(self, info_file, gameInfo): # TODO add a flag to warn users about savegame compatibility ? self.name = os.path.basename(self.path) self.gameInfo = gameInfo + self._mappedIDs = [] self.enabled = not os.path.isfile(os.path.join(self.path, DISABLED_MARKER)) self.loadInfo(info_file) @@ -184,6 +185,9 @@ def getDescription(self): def getAutomaticID(self, internalID): """Returns a new ID prefixed by the mod prefix.""" autoIDAllocatedSize = 1000 + if internalID in self._mappedIDs: + raise ValueError(f"{self.title()} tried to double-allocate internal ID {internalID}") + self._mappedIDs.append(internalID) id = self.prefix * autoIDAllocatedSize + internalID if internalID > autoIDAllocatedSize: raise RuntimeError(f"{self.title()} requested an ID outside of the auto-ID allocation limit ({internalID} limit {autoIDAllocatedSize}). File a bug report.") From 04eb84cc02571f29342f28b5ac2afac5b0cf1018 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 10:57:28 -0500 Subject: [PATCH 104/124] BUGFIX: Grab nodes with a OR filename attributes --- loader/assets/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 9aa4220b..53ee0a58 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -88,7 +88,7 @@ def _add_texture(filename): needs_autogeneration = [] for animation_chunk in modLibrary['library/animations']: - for asset in animation_chunk.xpath("//assetPos[@a]"): + for asset in animation_chunk.xpath("//assetPos[@a | @filename]"): mod_local_id = asset.get("filename") if mod_local_id is None: mod_local_id = asset.get('a') From f69306b897d7b6f0e9fa541c640639228fee06fd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 10:59:08 -0500 Subject: [PATCH 105/124] Bump version for bugfix --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index fc2091ff..c58582aa 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.9.0a1" +version = "0.9.0a2" From 3782bd94bdef06595587c1284d0883518aa471ee Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 11:16:56 -0500 Subject: [PATCH 106/124] Update TODO with low priority tasks for autopacking --- TODO.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO.md b/TODO.md index bf776d70..c3fe36c9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,9 @@ # Codebase - PEP 8 compliance - Better file structure +- Prevent autopacking from making multiple copies of the same CIM file +- Strip leading slashes on autopacked region images +- Strip trailing extensions from autopacked region images # Repository - Create a powershell build script for Windows From 8967773d6e8d42adf1c35c59331837c04b3031c0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 11:39:41 -0500 Subject: [PATCH 107/124] Export to moddir/textures, not any and all subdirs --- TODO.md | 1 - loader/assets/merge.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index c3fe36c9..892b1a55 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,6 @@ # Codebase - PEP 8 compliance - Better file structure -- Prevent autopacking from making multiple copies of the same CIM file - Strip leading slashes on autopacked region images - Strip trailing extensions from autopacked region images diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 53ee0a58..373de7a2 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -258,7 +258,7 @@ def _core_path(filename): reexport_cims[page] = set() # write back the cim file as png for debugging - reexport_cims[page].add(os.path.dirname(png_file)) + reexport_cims[page].add(os.path.normpath(mod + "/textures")) x = int(region.get("x")) y = int(region.get("y")) From 33586f71418b862945d8b380b77be43614290e68 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 11:42:36 -0500 Subject: [PATCH 108/124] Normalize mod directory --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index 5faa6b27..d4a7390d 100644 --- a/ui/database.py +++ b/ui/database.py @@ -97,7 +97,7 @@ class Mod: """Details about a specific mod (name, description)""" def __init__(self, info_file, gameInfo): - self.path = os.path.dirname(info_file) + self.path = os.path.normpath(os.path.dirname(info_file)) ui.log.log(" Loading mod at {}...".format(self.path)) # TODO add a flag to warn users about savegame compatibility ? From 4d957a13bbaf3aef5162097581f46baf24e6cfc0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 16:30:08 -0500 Subject: [PATCH 109/124] Throw error if packer failed to pack all assets --- loader/assets/merge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 373de7a2..ffd21ac4 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -115,7 +115,14 @@ def _add_texture(filename): minRequiredDimension = max(minRequiredDimension, w, h) sum += (w * h) - size = max(int(math.sqrt(sum) * 1.2), minRequiredDimension) + sizeEstimate = 1.2 + size = max(int(math.sqrt(sum) * sizeEstimate), minRequiredDimension) + + packer.add_bin(size, size) + packer.pack() + if len(needs_autogeneration) != len(packer.rect_list()): + raise IndexError( f"Unable to pack all {len(needs_autogeneration)} regions with size estimate {sizeEstimate}" + + f", was able to pack {len(packer.rect_list())} rectangles. Please file a bug report.") newTex = lxml.etree.SubElement(texturesNode, "t") newTex.set("i", str(textureID)) @@ -123,9 +130,6 @@ def _add_texture(filename): newTex.set("h", str(size)) coreLibrary['_custom_textures_cim'][str(textureID)] = newTex.attrib - packer.add_bin(size, size) - packer.pack() - packedRectsSorted = {} for rect in packer.rect_list(): b, x, y, w, h, rid = rect From 55a9ecfba636058f6a485369fbe607424f1bbcb4 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 16 Dec 2020 17:01:00 -0500 Subject: [PATCH 110/124] Throw error if non-numerical a attribute --- loader/assets/merge.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index ffd21ac4..e4f2c9a7 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -92,6 +92,9 @@ def _add_texture(filename): mod_local_id = asset.get("filename") if mod_local_id is None: mod_local_id = asset.get('a') + if not str.isdecimal(mod_local_id): + raise ValueError(f"Cannot specify a non-numerical 'a' attribute {mod_local_id}. " + + "Specify in 'filename' attribute instead.") elif mod_local_id not in needs_autogeneration: needs_autogeneration.append(mod_local_id) _add_texture(mod_local_id) From 13400c63c0a86090e6333f8430c7929b3fbde2df Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 10:09:21 -0500 Subject: [PATCH 111/124] Improve auto-location logic Don't throw error on missing previous_spacehave_path.txt Don't throw error on missing steam keys Write to log if autolocation failed. --- spacehaven-modloader.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/spacehaven-modloader.py b/spacehaven-modloader.py index cd7b7244..239c2368 100755 --- a/spacehaven-modloader.py +++ b/spacehaven-modloader.py @@ -152,27 +152,30 @@ def autolocateSpacehaven(self): self.jarPath = None self.modPath = None + # Open previous location if known try: with open("previous_spacehaven_path.txt", 'r') as f: location = f.read() if os.path.exists(location): self.locateSpacehaven(location) return - except: - import traceback - traceback.print_exc() - pass + except FileNotFoundError: + ui.log.log("Unable to get last space haven location. Autolocating again.") - registry_path = "SOFTWARE\\WOW6432Node\\Valve\\Steam" if (platform.architecture()[0] == "64bit") else "SOFTWARE\\Valve\\Steam" - steam_path = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_path), "InstallPath")[0] - library_folders = acf.load(open(steam_path + "\\steamapps\\libraryfolders.vdf"), wrapper=OrderedDict) - locations = [steam_path + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe"] - for key, value in library_folders["LibraryFolders"].items(): - if str.isnumeric(key): locations.append(value + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe") - for location in locations: - if os.path.exists(location): - self.locateSpacehaven(location) - return + # Steam based locator (Windows) + try: + registry_path = "SOFTWARE\\WOW6432Node\\Valve\\Steam" if (platform.architecture()[0] == "64bit") else "SOFTWARE\\Valve\\Steam" + steam_path = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_path), "InstallPath")[0] + library_folders = acf.load(open(steam_path + "\\steamapps\\libraryfolders.vdf"), wrapper=OrderedDict) + locations = [steam_path + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe"] + for key, value in library_folders["LibraryFolders"].items(): + if str.isnumeric(key): locations.append(value + "\\steamapps\\common\\SpaceHaven\\spacehaven.exe") + for location in locations: + if os.path.exists(location): + self.locateSpacehaven(location) + return + except FileNotFoundError: + ui.log.log("Unable to locate Steam registry keys, aborting Steam autolocator") for location in POSSIBLE_SPACEHAVEN_LOCATIONS: try: @@ -182,6 +185,7 @@ def autolocateSpacehaven(self): return except: pass + ui.log.log("Unable to autolocate installation. User will need to pick manually.") def locateSpacehaven(self, path): if path is None: From e1af9a6cf7d34a13531e690d37a88a26f3b235e3 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 11:17:50 -0500 Subject: [PATCH 112/124] Cleanup + Add comment documentation --- loader/assets/merge.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e4f2c9a7..cdc269f3 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -62,9 +62,10 @@ def _add_texture(filename): if filenameAssetPos is not None: autoAnimations = True + # no textures.xml file and no autoAnimations, we're done if 'library/textures' not in modLibrary and not autoAnimations: - # no textures.xml file, we're done return modded_textures + # Create a textures xml tree if there was no manually-defined file if 'library/textures' not in modLibrary and autoAnimations: texRoot = lxml.etree.Element("AllTexturesAndRegions") lxml.etree.SubElement(texRoot, "textures") @@ -72,18 +73,21 @@ def _add_texture(filename): modLibrary['library/textures'] = [lxml.etree.ElementTree(texRoot)] #FIXME verify that there's only one file + # TODO Maybe don't require only a single file? textures_mod = modLibrary['library/textures'][0] + # Allocate any manually defined texture regions into the CTC lib for texture_pack in textures_mod.xpath("//t[@i]"): cim_id = texture_pack.get('i') coreLibrary['_custom_textures_cim'][cim_id] = texture_pack.attrib + # Map manually defined regions in textures file to autoIDs for region in textures_mod.xpath("//re[@n]"): region_id = region.get('n') _add_texture(region_id) + # no custom mod textures, no need to remap ids if not mapping_n_region and not autoAnimations: - # no custom mod textures, no need to remap ids return modded_textures needs_autogeneration = [] @@ -101,7 +105,6 @@ def _add_texture(filename): if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] - #ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) asset.set('a', new_id) if len(needs_autogeneration): From 9c7b53c3e350af3347fbc807a6e849e30e8d03d2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 11:27:41 -0500 Subject: [PATCH 113/124] Stop caring about backslashes vs forward slashes --- loader/assets/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index cdc269f3..28e2bb33 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -15,7 +15,7 @@ def _detect_textures(coreLibrary, modLibrary, mod): - textures_path = os.path.join(mod, 'textures').replace("\\","/") + textures_path = os.path.join(mod, 'textures') if not os.path.isdir(textures_path): return {} From 388b1ce25a321313bf93806ffe9815ebf8c30cff Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 11:29:01 -0500 Subject: [PATCH 114/124] Remove confusing comments --- loader/assets/merge.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 28e2bb33..a05ebd27 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -35,9 +35,6 @@ def _add_texture(filename): if isCoreRegion and not os.path.exists(path): #core region file without an associated file, return early return - # Removed file existence check - file should already exist given how this function is being called - # If the file no longer exists, let the program thrown an error later (plus the file might be - # deleted by later anyway) if not isCoreRegion: # adding a new texture, this gets tricky as they have to have consecutive numbers. From faff0479e227c9810df96d9524dedabcbc8d5b11 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 11:33:30 -0500 Subject: [PATCH 115/124] More comment cleanup --- loader/assets/merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index a05ebd27..338ec534 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -27,13 +27,13 @@ def _add_texture(filename): filename += ".png" region_id = str.join(".", filename.split('.')[:-1]) isCoreRegion = region_id.isdecimal() and int(region_id) <= coreLibrary['_last_core_region_id'] + # Early exit if this texture exists if (region_id in modded_textures) or (region_id in mapping_n_region): - # Early exit if this texture exists return path = os.path.join(textures_path, filename) + #core region file without an associated file, return early if isCoreRegion and not os.path.exists(path): - #core region file without an associated file, return early return if not isCoreRegion: From 513e548e041aa0f943a61a77022e81cb7b022eff Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 11:33:52 -0500 Subject: [PATCH 116/124] Only append .png if not already present --- loader/assets/merge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 338ec534..ee380950 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -24,7 +24,8 @@ def _detect_textures(coreLibrary, modLibrary, mod): seen_textures = set() def _add_texture(filename): - filename += ".png" + if ".png" not in filename: + filename += ".png" region_id = str.join(".", filename.split('.')[:-1]) isCoreRegion = region_id.isdecimal() and int(region_id) <= coreLibrary['_last_core_region_id'] # Early exit if this texture exists From c0f9abdb60f120db40ac072768e7876653a9eaf3 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 12:14:40 -0500 Subject: [PATCH 117/124] Improve asset loading code This introduces some code duplication but simplifies the loops otherwise Also ensures stripping of leading slashes and normalization of regionNames when loading into the packer later --- loader/assets/merge.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index ee380950..a16f770f 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -24,8 +24,6 @@ def _detect_textures(coreLibrary, modLibrary, mod): seen_textures = set() def _add_texture(filename): - if ".png" not in filename: - filename += ".png" region_id = str.join(".", filename.split('.')[:-1]) isCoreRegion = region_id.isdecimal() and int(region_id) <= coreLibrary['_last_core_region_id'] # Early exit if this texture exists @@ -88,23 +86,33 @@ def _add_texture(filename): if not mapping_n_region and not autoAnimations: return modded_textures - needs_autogeneration = [] + needs_autogeneration = set() for animation_chunk in modLibrary['library/animations']: - for asset in animation_chunk.xpath("//assetPos[@a | @filename]"): - mod_local_id = asset.get("filename") - if mod_local_id is None: - mod_local_id = asset.get('a') - if not str.isdecimal(mod_local_id): - raise ValueError(f"Cannot specify a non-numerical 'a' attribute {mod_local_id}. " + - "Specify in 'filename' attribute instead.") - elif mod_local_id not in needs_autogeneration: - needs_autogeneration.append(mod_local_id) + # iterate on autogeneration nodes + for asset in animation_chunk.xpath("//assetPos[@filename]"): + # asset.get will never return null here + mod_local_id = asset.get("filename").lstrip("/") + if ".png" not in mod_local_id: + mod_local_id += ".png" + needs_autogeneration.add(mod_local_id) _add_texture(mod_local_id) if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] asset.set('a', new_id) + # iterate on manually defined nodes + for asset in animation_chunk.xpath("//assetPos[@a & not(@filename)]"): + mod_local_id = asset.get('a') + if not str.isdecimal(mod_local_id): + raise ValueError(f"Cannot specify a non-numerical 'a' attribute {mod_local_id}. " + + "Specify in 'filename' attribute instead.") + _add_texture(mod_local_id + ".png") + if mod_local_id not in mapping_n_region: + continue + new_id = mapping_n_region[mod_local_id] + asset.set('a', new_id) + if len(needs_autogeneration): regionsNode = textures_mod.find("//regions") texturesNode = textures_mod.find("//textures") @@ -114,7 +122,7 @@ def _add_texture(filename): minRequiredDimension = 0 # First get all the files and pack them into a new texture square for regionName in needs_autogeneration: - (w, h, rows, info) = png.Reader(textures_path + "/" + regionName + ".png").asRGBA() + (w, h, rows, info) = png.Reader(textures_path + "/" + regionName).asRGBA() packer.add_rect(w, h, regionName) minRequiredDimension = max(minRequiredDimension, w, h) sum += (w * h) From bdbf29d54b04db2c67d35fa6e840702448a7bd4f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 13:08:23 -0500 Subject: [PATCH 118/124] Correct xpath and change (unify) mapping_n_region key --- loader/assets/merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index a16f770f..37309521 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -38,7 +38,7 @@ def _add_texture(filename): if not isCoreRegion: # adding a new texture, this gets tricky as they have to have consecutive numbers. core_region_id = str(coreLibrary['_next_region_id']) - mapping_n_region[region_id] = core_region_id + mapping_n_region[filename] = core_region_id coreLibrary['_next_region_id'] += 1 ui.log.log(f" Allocated new core region idx {core_region_id:>5} to file {filename}") else: @@ -102,7 +102,7 @@ def _add_texture(filename): asset.set('a', new_id) # iterate on manually defined nodes - for asset in animation_chunk.xpath("//assetPos[@a & not(@filename)]"): + for asset in animation_chunk.xpath("//assetPos[@a and not(@filename)]"): mod_local_id = asset.get('a') if not str.isdecimal(mod_local_id): raise ValueError(f"Cannot specify a non-numerical 'a' attribute {mod_local_id}. " + From b57579dcbc58707957ffd34f67ea60bb9ecbd0b8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 13:20:55 -0500 Subject: [PATCH 119/124] Write debug textures xml if changed --- loader/assets/merge.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 37309521..26834ff1 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -171,6 +171,13 @@ def _add_texture(filename): ui.log.log(" Mapping texture 're' {} to {}...".format(mod_local_id, new_id)) asset.set('n', new_id) + # write the new textures XML if changed. + if autoAnimations: + modLibrary['library/textures'][0].write( + os.path.join(mod, "library", "generated_textures.xml"), + pretty_print=True + ) + return modded_textures From 330216b6203fb876d67de4821b10c56aa6a7a76f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 14:07:18 -0500 Subject: [PATCH 120/124] Do not load mods until requested --- ui/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/database.py b/ui/database.py index d4a7390d..68a5fb4f 100644 --- a/ui/database.py +++ b/ui/database.py @@ -19,7 +19,7 @@ class ModDatabase: def __init__(self, path_list, gameInfo): self.path_list = path_list self.gameInfo = gameInfo - self.locateMods() + self.mods = [] ModDatabase.__lastInstance = self def locateMods(self): From e6e475de7c9b871a31554d7e992cc32221659b7f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 14:51:08 -0500 Subject: [PATCH 121/124] Bump version number in preparation for release --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index c58582aa..fd069b94 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.9.0a2" +version = "0.9.0b1" From 9a221eb582242a18f5a26063e7f4218c04e33159 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sat, 19 Dec 2020 15:08:02 -0500 Subject: [PATCH 122/124] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 919b3721..a16b7bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ - On Windows, the game will be autolocated via Steam if possible. - `` tag in info.xml: Defines a prefix that can be used in various places during mod creation. - Automatic texture packing: instead of defining a `textures` file, texture regions can be defined as needed in `animations`. Add a `filename=""` attribute to the `` tag, and it will be packed automatically into `modid.cim`. This will fail if a mod ID is not specified. Textures must still be located in `moddir/textures` and paths are relative to this directory. +- Automatic texture patching writes the resulting textures XML to `moddir/library/generated_textures.xml` for debugging +- Attempt to normalize file paths in a bunch of places +- More instances of log cleanup - less errors, more error messages ## v0.8.2 Bugfix: textures were not being merged in due to missing file during the build process From 90ca2f91daff264fe7772b4f6f3563a21e9512c9 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 20 Dec 2020 20:03:51 -0500 Subject: [PATCH 123/124] BUGFIX: autopacked IDs were doing duplicate IDs again --- loader/assets/merge.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index a6566a90..58ba1013 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -95,8 +95,9 @@ def _add_texture(filename): mod_local_id = asset.get("filename").lstrip("/") if ".png" not in mod_local_id: mod_local_id += ".png" - needs_autogeneration.add(mod_local_id) - _add_texture(mod_local_id) + if mod_local_id not in needs_autogeneration: + needs_autogeneration.add(mod_local_id) + _add_texture(mod_local_id) if mod_local_id not in mapping_n_region: continue new_id = mapping_n_region[mod_local_id] From ee9b2e5ed0f3c501ce91f2498a30c1c30e545fc8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 20 Dec 2020 20:04:02 -0500 Subject: [PATCH 124/124] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a16b7bd3..0553ddf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Automatic texture patching writes the resulting textures XML to `moddir/library/generated_textures.xml` for debugging - Attempt to normalize file paths in a bunch of places - More instances of log cleanup - less errors, more error messages +- Decouple mod database from window class +- Decouple mod info from window class ## v0.8.2 Bugfix: textures were not being merged in due to missing file during the build process