diff --git a/README.md b/README.md index b407435..11500b5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Make sure to do a select the "add python to PATH" and "install pip" options. * `python auto.py savename` Generate a snapshot of *savename* and store it to folder *savename*. * `python auto.py outfolder savename` Generate a snapshot of *savename* and store it to folder *outfolder*. * `python auto.py outfolder savename1 savename2 savename3` Generate timeline snapshots of *savename1*, *savename2*, *savename3* in that order, and store it to folder *outfolder*. + * `python auto.py outfolder savename*` Generate timeline snapshots of all savefiles that match the glob pattern `savename*` in natural order, and store it to folder *outfolder*. * `python auto.py --factorio=PATH` Same as `python auto.py`, but will use `factorio.exe` from *PATH* instead of attempting to find it in common locations. * `python auto.py --verbose` Displays factoriomaps related logs. * `python auto.py --verbosegame` Displays *all* game logs. @@ -40,6 +41,7 @@ Heres a list of flags that `auto.py` can accept: | `--hd`*\** | Take screenshots of resolution 64 x 64 pixels per in-game tile instead of 32 x 32 to match the resolution of the newer HD textures. | | `--no-altmode` | Hides entity info (alt mode) | | `--no-tags` | Hides map tags | +| `--default-timestamp=-1` | Snapshot that will be loaded by the webpage by default. Negative values indicate newest snapshots, so -1 indicates the newest map while 0 indicates the oldest map. | | `--build-range=5.2`*\** | The maximum range from buildings around which pictures are saved (in chunks, 32 by 32 in-game tiles). | | `--connect-range=1.2`*\** | The maximum range from connection buildings (rails, electric poles) around which pictures are saved. | | `--tag-range=5.2`*\** | The maximum range from mapview tags around which pictures are saved. | @@ -50,7 +52,7 @@ Heres a list of flags that `auto.py` can accept: | `--date=dd/mm/yy` | Date attached to the snapshot, default is today. | | `--verbose` | Displays factoriomaps script logs. | | `--verbosegame` | Displays *all* game logs. | -| `--noupdate` | Skips the update check. | +| `--no-update` | Skips the update check. | | `--maxthreads=N` | Sets the number of threads used for all steps. By default this is equal to the amount of logical processor cores available. | | `--cropthreads=N` | Sets the number of threads used for the crop step. | | `--refthreads=N` | Sets the number of threads used for the crossreferencing step. | @@ -58,6 +60,7 @@ Heres a list of flags that `auto.py` can accept: | `--screenshotthreads=N` | Set the number of screenshotting threads factorio uses. | | `--delete` | Deletes the output folder specified before running the script. | | `--dry` | Skips starting factorio, making screenshots and doing the main steps, only execute setting up and finishing of script. | +| `--force-lib-update` | Forces an update of the web dependencies. | Image quality settings can be changed in the top of `zoom.py`. @@ -77,7 +80,7 @@ If you wish to host your map for other people to a server, you need to take into All other files, including txt and other non-image files in `Images\`, are not used by the client. Some of them are temporary files, some of them are used as savestate to create additional snapshots on the timeline. # Known mods that make use of the API to improve compability - * **Factorissimo** ⩾2.3.5: Able to render the inside of factory buildings recursively. + * Factorissimo ⩾2.3.5: Able to render the inside of factory buildings recursively. * Your mod? If you want to have a chat, you can always find me on discord: L0laapk3#2010 # Known limitations diff --git a/auto.py b/auto.py index 607c4f1..9cf3f0a 100644 --- a/auto.py +++ b/auto.py @@ -1,82 +1,59 @@ import sys -if sys.maxsize <= 2**32: - raise Exception("64 bit Python is required.") +if sys.maxsize <= 2**32 or sys.hexversion < 0x3060000: + raise Exception("64 bit Python 3.6 or higher is required for this script.") -if sys.hexversion < 0x3060000: - raise Exception("Python 3.6 or higher is required for this script.") - -import traceback, os, pkg_resources +import os +import traceback +import pkg_resources from pkg_resources import DistributionNotFound, VersionConflict +from pathlib import Path try: - with open('packages.txt') as f: + with Path(__file__, "..", "packages.txt").open("r", encoding="utf-8") as f: pkg_resources.require(f.read().splitlines()) except (DistributionNotFound, VersionConflict) as ex: traceback.print_exc() print("\nDependencies not met. Run `pip install -r packages.txt` to install missing dependencies.") sys.exit(1) - - - -import subprocess, signal -import json -import threading, psutil -import time -import re -import random -import math +import argparse import configparser -from subprocess import call import datetime -import urllib.request, urllib.error, urllib.parse +import json +import math +import multiprocessing as mp +import random +import re +import string +import signal +import subprocess +import tempfile +from tempfile import TemporaryDirectory +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from argparse import Namespace +from shutil import copy, copytree +from shutil import get_terminal_size as tsize +from shutil import rmtree from socket import timeout -from shutil import copy, copytree, rmtree, get_terminal_size as tsize +from subprocess import call from zipfile import ZipFile -import tempfile + +import psutil from PIL import Image, ImageChops -import multiprocessing as mp +from orderedset import OrderedSet from crop import crop from ref import ref -from zoom import zoom, zoomRenderboxes from updateLib import update as updateLib +from zoom import zoom, zoomRenderboxes - - -kwargs = { - 'dayonly': False, - 'nightonly': False, - 'hd': False, - 'no-altmode': False, - 'no-tags': False, - 'tag-range': 5.2, - 'build-range': 5.2, - 'connect-range': 1.2, - 'factorio': None, - 'modpath': "../../mods", - 'basepath': "FactorioMaps", - 'date': datetime.date.today().strftime("%d/%m/%y"), - 'verbosegame': False, - 'verbose': False, - 'noupdate': False, - 'reverseupdatetest': False, - 'maxthreads': mp.cpu_count(), - 'cropthreads': None, - 'refthreads': None, - 'zoomthreads': None, - 'screenshotthreads': None, - 'delete': False, - 'dry': False, - 'surface': [] -} -changedKwargs = [] - - - - +userFolder = Path(__file__, "..", "..", "..").resolve() def printErase(arg): try: @@ -87,7 +64,7 @@ def printErase(arg): pass -def startGameAndReadGameLogs(results, condition, popenArgs, tmpDir, pidBlacklist, rawTags, **kwargs): +def startGameAndReadGameLogs(results, condition, popenArgs, usedSteamLaunchHack, tmpDir, pidBlacklist, rawTags, args): pipeOut, pipeIn = os.pipe() p = subprocess.Popen(popenArgs, stdout=pipeIn) @@ -102,9 +79,9 @@ def handleGameLine(line): if prevPrinted: printErase(line) return - + prevPrinted = False - + m = re.match(r'^\ *\d+(?:\.\d+)? *Script *@__L0laapk3_FactorioMaps__\/data-final-fixes\.lua:\d+: FactorioMaps_Output_RawTagPaths:([^:]+):(.*)$', line, re.IGNORECASE) if m is not None: rawTags[m.group(1)] = m.group(2) @@ -119,29 +96,31 @@ def handleGameLine(line): if m is not None and m.group(2) is not None: printErase(m.group(3)) prevPrinted = True - elif m is not None and kwargs["verbose"]: + elif m is not None and args.verbose: printErase(m.group(1)) prevPrinted = True - elif line.lower() in ("error", "warn", "exception", "fail", "invalid") or (kwargs["verbosegame"] and len(line) > 0): + elif line.lower() in ("error", "warn", "exception", "fail", "invalid") or (args.verbosegame and len(line) > 0): printErase("[GAME] %s" % line) prevPrinted = True return False with os.fdopen(pipeOut, 'r') as pipef: - + line = pipef.readline().rstrip("\n") printingStackTraceback = handleGameLine(line) isSteam = False - if line.endswith("Initializing Steam API."): + if line.endswith("Initializing Steam API.") or usedSteamLaunchHack: isSteam = True elif not re.match(r'^ *\d+\.\d{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d; Factorio (\d+\.\d+\.\d+) \(build (\d+), [^)]+\)$', line): raise Exception("Unrecognised output from factorio (maybe your version is outdated?)\n\nOutput from factorio:\n" + line) if isSteam: - # note: possibility to avoid this: https://www.reddit.com/r/Steam/comments/4rgrxj/where_are_launch_options_saved_for_games/ - # requirements for this approach: root?, need to figure out steam userid, parse the file format, ensure no conflicts between instances. overall probably not worth it. - print("WARNING: Running in limited support mode trough steam. Consider using standalone factorio instead.\n\t If you have any default arguments set in steam for factorio, delete them and restart the script.\n\t Please alt tab to steam and confirm the steam 'start game with arguments' popup.\n\t (Yes, you'll have to click this every time the game starts for the steam version)") + if usedSteamLaunchHack: + printErase("using steam launch hack.") + else: + printErase("WARNING: Could not find steam exe. Falling back to old steam method.\n\t If you have any default arguments set in steam for factorio, delete them and restart the script.\n\t Please alt tab to steam and confirm the steam 'start game with arguments' popup.\n\t To avoid this in the future, use the --steam-exe argument.") + attrs = ('pid', 'name', 'create_time') # on some devices, the previous check wasn't enough apparently, so explicitely wait until the log file is created. @@ -170,7 +149,7 @@ def handleGameLine(line): if isSteam: pipef.close() - with open(os.path.join(tmpDir, "factorio-current.log"), "r") as f: + with Path(tmpDir, "factorio-current.log").open("r", encoding="utf-8") as f: while psutil.pid_exists(pid): where = f.tell() line = f.readline() @@ -185,9 +164,192 @@ def handleGameLine(line): line = pipef.readline() printingStackTraceback = handleGameLine(line) - +def checkUpdate(reverseUpdateTest:bool = False): + try: + print("checking for updates") + latestUpdates = json.loads(urllib.request.urlopen('https://cdn.jsdelivr.net/gh/L0laapk3/FactorioMaps@latest/updates.json', timeout=30).read()) + with Path(__file__, "..", "updates.json").open("r", encoding="utf-8") as f: + currentUpdates = json.load(f) + if reverseUpdateTest: + latestUpdates, currentUpdates = currentUpdates, latestUpdates + + updates = [] + majorUpdate = False + currentVersion = (0, 0, 0) + for verStr, changes in currentUpdates.items(): + ver = tuple(map(int, verStr.split("."))) + if currentVersion[0] < ver[0] or (currentVersion[0] == ver[0] and currentVersion[1] < ver[1]): + currentVersion = ver + for verStr, changes in latestUpdates.items(): + if verStr not in currentUpdates: + ver = tuple(map(int, verStr.split("."))) + updates.append((verStr, changes)) + updates.sort(key = lambda u: u[0]) + if len(updates) > 0: + + padding = max(map(lambda u: len(u[0]), updates)) + changelogLines = [] + for update in updates: + if isinstance(update[1], str): + updateText = update[1] + else: + updateText = str(("\r\n " + " "*padding).join(update[1])) + if updateText[0] == "!": + majorUpdate = True + updateText = updateText[1:] + changelogLines.append(" %s: %s" % (update[0].rjust(padding), updateText)) + print("") + print("") + print("================================================================================") + print("") + print((" An " + ("important" if majorUpdate else "incremental") + " update has been found!")) + print("") + print(" Here's what changed:") + for line in changelogLines: + print(line) + print("") + print("") + print(" Download: https://git.io/factoriomaps") + if majorUpdate: + print("") + print(" You can dismiss this by using --no-update (not recommended)") + print("") + print("================================================================================") + print("") + print("") + if majorUpdate or reverseUpdateTest: + exit(1) + except (urllib.error.URLError, timeout) as e: + print("Failed to check for updates. %s: %s" % (type(e).__name__, e)) + + +def linkDir(src: Path, dest:Path): + if os.name == 'nt': + subprocess.check_call(("MKLINK", "/J", src.resolve(), dest.resolve()), stdout=subprocess.DEVNULL, shell=True) + else: + os.symlink(dest.resolve(), src.resolve()) + + +def linkCustomModFolder(modpath: Path): + print(f"Verifying mod version in custom mod folder ({modpath})") + modPattern = re.compile(r'^L0laapk3_FactorioMaps_', flags=re.IGNORECASE) + for entry in [entry for entry in modpath.iterdir() if modPattern.match(entry.name)]: + print("Found other factoriomaps mod in custom mod folder, deleting.") + path = Path(modpath, entry) + if path.is_file() or path.is_symlink(): + path.unlink() + elif path.is_dir(): + rmtree(path) + else: + raise Exception(f"Unable to remove {path} unknown type") + + linkDir(Path(modpath, Path('.').resolve().name), Path(".")) + + +def changeModlist(modpath: Path,newState: bool): + print(f"{'Enabling' if newState else 'Disabling'} FactorioMaps mod") + done = False + modlistPath = Path(modpath, "mod-list.json") + with modlistPath.open("r", encoding="utf-8") as f: + modlist = json.load(f) + for mod in modlist["mods"]: + if mod["name"] == "L0laapk3_FactorioMaps": + mod["enabled"] = newState + done = True + break + if not done: + modlist["mods"].append({"name": "L0laapk3_FactorioMaps", "enabled": newState}) + with modlistPath.open("w", encoding="utf-8") as f: + json.dump(modlist, f, indent=2) + + +def buildAutorun(args: Namespace, workFolder: Path, outFolder: Path, isFirstSnapshot: bool, daytime: str): + printErase("Building autorun.lua") + mapInfoPath = Path(workFolder, "mapInfo.json") + if mapInfoPath.is_file(): + with mapInfoPath.open("r", encoding='utf-8') as f: + mapInfoLua = re.sub(r'"([^"]+)" *:', lambda m: '["'+m.group(1)+'"] = ', f.read().replace("[", "{").replace("]", "}")) + # TODO: Update for new argument parsing +# if isFirstSnapshot: +# f.seek(0) +# mapInfo = json.load(f) +# if "options" in mapInfo: +# for kwarg in changedKwargs: +# if kwarg in ("hd", "dayonly", "nightonly", "build-range", "connect-range", "tag-range"): +# printErase("Warning: flag '" + kwarg + "' is overriden by previous setting found in existing timeline.") + else: + mapInfoLua = "{}" + + isFirstSnapshot = False + + chunkCachePath = Path(workFolder, "chunkCache.json") + if chunkCachePath.is_file(): + with chunkCachePath.open("r", encoding="utf-8") as f: + chunkCache = re.sub(r'"([^"]+)" *:', lambda m: '["'+m.group(1)+'"] = ', f.read().replace("[", "{").replace("]", "}")) + else: + chunkCache = "{}" + + def lowerBool(value: bool): + return str(value).lower() + + with open("autorun.lua", "w", encoding="utf-8") as f: + surfaceString = '{"' + '", "'.join(args.surface) + '"}' if args.surface else "nil" + autorunString = \ + f'''fm.autorun = {{ + HD = {lowerBool(args.hd)}, + daytime = "{daytime}", + alt_mode = {lowerBool(args.altmode)}, + tags = {lowerBool(args.tags)}, + around_tag_range = {args.tag_range}, + around_build_range = {args.build_range}, + around_connect_range = {args.connect_range}, + connect_types = {{"lamp", "electric-pole", "radar", "straight-rail", "curved-rail", "rail-signal", "rail-chain-signal", "locomotive", "cargo-wagon", "fluid-wagon", "car"}}, + date = "{datetime.datetime.strptime(args.date, "%d/%m/%y").strftime("%d/%m/%y")}", + surfaces = {surfaceString}, + name = "{str(outFolder) + "/"}", + mapInfo = {mapInfoLua.encode("utf-8").decode("unicode-escape")}, + chunkCache = {chunkCache} + }}''' + f.write(autorunString) + if args.verbose: + printErase(autorunString) + + +def buildConfig(args: Namespace, tmpDir, basepath): + printErase("Building config.ini") + if args.verbose > 2: + print(f"Using temporary directory '{tmpDir}'") + configPath = Path(tmpDir, "config","config.ini") + configPath.parent.mkdir(parents=True) + + config = configparser.ConfigParser() + config.read(Path(args.config_path, "config.ini")) + + if "interface" not in config: + config["interface"] = {} + config["interface"]["show-tips-and-tricks"] = "false" + + if "path" not in config: + config["path"] = {} + config["path"]["write-data"] = tmpDir + + if "graphics" not in config: + config["graphics"] = {} + config["graphics"]["screenshots-threads-count"] = str(args.screenshotthreads if args.screenshotthreads else args.maxthreads) + config["graphics"]["max-threads"] = config["graphics"]["screenshots-threads-count"] + + with configPath.open("w+", encoding="utf-8") as configFile: + configFile.writelines(("; version=3\n", )) + config.write(configFile, space_around_delimiters=False) + + # TODO: change this when https://forums.factorio.com/viewtopic.php?f=28&t=81221 is implemented + linkDir(Path(tmpDir, "script-output"), basepath) + + copy(Path(userFolder, 'player-data.json'), tmpDir) + + return configPath def auto(*args): @@ -209,47 +371,74 @@ def kill(pid, onlyStall=False): printErase("killed factorio") #time.sleep(0.1) - - - - - - def parseArg(arg): - if arg[0:2] != "--": - return True - key = arg[2:].split("=",2)[0].lower() - if key in kwargs: - changedKwargs.append(key) - if isinstance(kwargs[key], list): - kwargs[key].append(arg[2:].split("=",2)[1]) - else: - kwargs[key] = arg[2:].split("=",2)[1].lower() if len(arg[2:].split("=",2)) > 1 else True - if kwargs[key] == "true": - kwargs[key] = True - if kwargs[key] == "false": - kwargs[key] = False - else: - print(f'Bad flag: "{key}"') - raise ValueError(f'Bad flag: "{key}"') - return False + parser = argparse.ArgumentParser(description="FactorioMaps") + daytime = parser.add_mutually_exclusive_group() + daytime.add_argument("--dayonly", dest="night", action="store_false", help="Only take daytime screenshots.") + daytime.add_argument("--nightonly", dest="day", action="store_false", help="Only take nighttime screenshots.") + parser.add_argument("--hd", action="store_true", help="Take screenshots of resolution 64 x 64 pixels per in-game tile.") + parser.add_argument("--no-altmode", dest="altmode", action="store_false", help="Hides entity info (alt mode).") + parser.add_argument("--no-tags", dest="tags", action="store_false", help="Hides map tags") + parser.add_argument("--default-timestamp", type=int, default=None, dest="default_timestamp", help="Snapshot that will be loaded by the webpage by default. Negative values indicate newest snapshots, so -1 indicates the newest map while 0 indicates the oldest map.") + parser.add_argument("--build-range", type=float, default=5.2, help="The maximum range from buildings around which pictures are saved (in chunks, 32 by 32 in-game tiles).") + parser.add_argument("--connect-range", type=float, default=1.2, help="The maximum range from connection buildings (rails, electric poles) around which pictures are saved.") + parser.add_argument("--tag-range", type=float, default=5.2, help="The maximum range from mapview tags around which pictures are saved.") + parser.add_argument("--surface", action="append", default=[], help="Used to capture other surfaces. If left empty, the surface the player is standing on will be used. To capture multiple surfaces, use the argument multiple times: --surface nauvis --surface 'Factory floor 1'") + parser.add_argument("--factorio", type=lambda p: Path(p).resolve(), help="Use factorio.exe from PATH instead of attempting to find it in common locations.") + parser.add_argument("--output-path", dest="basepath", type=lambda p: Path(p).resolve(), default=Path(userFolder, "script-output", "FactorioMaps"), help="path to the output folder (default is '..\\..\\script-output\\FactorioMaps')") + parser.add_argument("--mod-path", "--modpath", type=lambda p: Path(p).resolve(), default=Path(userFolder, 'mods'), help="Use PATH as the mod folder. (default is '..\\..\\mods')") + parser.add_argument("--config-path", type=lambda p: Path(p).resolve(), default=Path(userFolder, 'config'), help="Use PATH as the mod folder. (default is '..\\..\\config')") + parser.add_argument("--date", default=datetime.date.today().strftime("%d/%m/%y"), help="Date attached to the snapshot, default is today. [dd/mm/yy]") + parser.add_argument('--verbose', '-v', action='count', default=0, help="Displays factoriomaps script logs.") + parser.add_argument('--verbosegame', action='count', default=0, help="Displays all game logs.") + parser.add_argument("--no-update", "--noupdate", dest="update", action="store_false", help="Skips the update check.") + parser.add_argument("--reverseupdatetest", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--maxthreads", type=int, default=mp.cpu_count(), help="Sets the number of threads used for all steps. By default this is equal to the amount of logical processor cores available.") + parser.add_argument("--cropthreads", type=int, default=None, help="Sets the number of threads used for the crop step.") + parser.add_argument("--refthreads", type=int, default=None, help="Sets the number of threads used for the crossreferencing step.") + parser.add_argument("--zoomthreads", type=int, default=None, help="Sets the number of threads used for the zoom step.") + parser.add_argument("--screenshotthreads", type=int, default=None, help="Set the number of screenshotting threads factorio uses.") + parser.add_argument("--delete", action="store_true", help="Deletes the output folder specified before running the script.") + parser.add_argument("--dry", action="store_true", help="Skips starting factorio, making screenshots and doing the main steps, only execute setting up and finishing of script.") + parser.add_argument("targetname", nargs="?", help="output folder name for the generated snapshots.") + parser.add_argument("savename", nargs="*", help="Names of the savegames to generate snapshots from. If no savegames are provided the latest save or the save matching outfolder will be gerated. Glob patterns are supported.") + parser.add_argument("--force-lib-update", action="store_true", help="Forces an update of the web dependencies.") + + args = parser.parse_args() + if args.verbose > 0: + print(args) + if args.update: + checkUpdate(args.reverseupdatetest) - newArgs = list(filter(parseArg, args)) - if kwargs["verbose"]: - print(args) - if len(newArgs) > 0: - foldername = newArgs[0] + saves = Path(userFolder, "saves") + if args.targetname: + foldername = args.targetname else: - foldername = os.path.splitext(os.path.basename(max([os.path.join("../../saves", basename) for basename in os.listdir("../../saves") if basename not in { "_autosave1.zip", "_autosave2.zip", "_autosave3.zip" }], key=os.path.getmtime)))[0] + timestamp, filePath = max( + (save.stat().st_mtime, save) + for save in saves.iterdir() + if save.stem not in {"_autosave1", "_autosave2", "_autosave3"} + ) + foldername = filePath.stem print("No save name passed. Using most recent save: %s" % foldername) - savenames = newArgs[1:] or [ foldername ] + saveNames = args.savename or [foldername] + foldername = foldername.replace('*', '').replace('?', '') + + saveGames = OrderedSet() + for saveName in saveNames: + globResults = list(saves.glob(saveName)) + globResults += list(saves.glob(f"{saveName}.zip")) - for saveName in savenames: - savePath = os.path.join("../../saves", saveName) - if not (os.path.isdir(savePath) or os.path.isfile(savePath) or os.path.isfile(savePath + ".zip")): + if not globResults: print(f'Cannot find savefile: "{saveName}"') raise ValueError(f'Cannot find savefile: "{saveName}"') + results = [save for save in globResults if save.is_file()] + for result in results: + saveGames.add(result.stem) + + if args.verbose > 0: + print(f"Will generate snapshots for : {list(saveGames)}") windowsPaths = [ "Program Files/Factorio/bin/x64/factorio.exe", @@ -257,395 +446,256 @@ def parseArg(arg): "Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe", "Steam/steamapps/common/Factorio/bin/x64/factorio.exe", ] - possiblePaths = [driveletter + ":/" + path for driveletter in 'CDEFGHIJKL' for path in windowsPaths] + [ - "../../bin/x64/factorio.exe", - "../../bin/x64/factorio", + + availableDrives = [ + "%s:/" % d for d in string.ascii_uppercase if Path(f"{d}:/").exists() ] + possiblePaths = [ + drive + path for drive in availableDrives for path in windowsPaths + ] + ["../../bin/x64/factorio.exe", "../../bin/x64/factorio",] try: - factorioPath = next(x for x in map(os.path.abspath, [kwargs["factorio"]] if kwargs["factorio"] else possiblePaths) if os.path.isfile(x)) + factorioPath = next( + x + for x in map(Path, [args.factorio] if args.factorio else possiblePaths) + if x.is_file() + ) except StopIteration: - raise Exception("Can't find factorio.exe. Please pass --factorio=PATH as an argument.") + raise Exception( + "Can't find factorio.exe. Please pass --factorio=PATH as an argument." + ) print("factorio path: {}".format(factorioPath)) psutil.Process(os.getpid()).nice(psutil.ABOVE_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 5) - basepath = os.path.join("../../script-output", kwargs["basepath"]) workthread = None - workfolder = os.path.join(basepath, foldername) - print("output folder: {}".format(os.path.relpath(workfolder, "../.."))) + workfolder = Path(args.basepath, foldername).resolve() + try: + print("output folder: {}".format(workfolder.relative_to(Path(userFolder)))) + except ValueError: + print("output folder: {}".format(workfolder.resolve())) - try: - os.makedirs(workfolder) + workfolder.mkdir(parents=True, exist_ok=True) except FileExistsError: - pass - - - if not kwargs["noupdate"]: - try: - print("checking for updates") - latestUpdates = json.loads(urllib.request.urlopen('https://cdn.jsdelivr.net/gh/L0laapk3/FactorioMaps@latest/updates.json', timeout=30).read()) - with open("updates.json", "r") as f: - currentUpdates = json.load(f) - if kwargs["reverseupdatetest"]: - latestUpdates, currentUpdates = currentUpdates, latestUpdates - - updates = [] - majorUpdate = False - currentVersion = (0, 0, 0) - for verStr, changes in currentUpdates.items(): - ver = tuple(map(int, verStr.split("."))) - if currentVersion[0] < ver[0] or (currentVersion[0] == ver[0] and currentVersion[1] < ver[1]): - currentVersion = ver - for verStr, changes in latestUpdates.items(): - if verStr not in currentUpdates: - ver = tuple(map(int, verStr.split("."))) - updates.append((verStr, changes)) - updates.sort(key = lambda u: u[0]) - if len(updates) > 0: - - padding = max(map(lambda u: len(u[0]), updates)) - changelogLines = [] - for update in updates: - if isinstance(update[1], str): - updateText = update[1] - else: - updateText = str(("\r\n " + " "*padding).join(update[1])) - if updateText[0] == "!": - majorUpdate = True - updateText = updateText[1:] - changelogLines.append(" %s: %s" % (update[0].rjust(padding), updateText)) - print("") - print("") - print("================================================================================") - print("") - print((" an " + ("important" if majorUpdate else "incremental") + " update has been found!")) - print("") - print(" heres what changed:") - for line in changelogLines: - print(line) - print("") - print("") - print(" Download: https://git.io/factoriomaps") - if majorUpdate: - print("") - print("You can dismiss this by using --noupdate (not recommended)") - print("") - print("================================================================================") - print("") - print("") - if majorUpdate or kwargs["reverseupdatetest"]: - sys.exit(1)(1) - - - except (urllib.error.URLError, timeout) as e: - print("Failed to check for updates. %s: %s" % (type(e).__name__, e)) - + raise Exception(f"{workfolder} exists and is not a directory!") + updateLib(args.force_lib_update) + #TODO: integrity check, if done files aren't there or there are any bmps left, complain. - updateLib(False) - - - - #TODO: integrity check, if done files arent there or there are any bmp's left, complain. - - - def linkDir(src, dest): - if os.name == 'nt': - subprocess.check_call(("MKLINK", "/J", os.path.abspath(src), os.path.abspath(dest)), stdout=subprocess.DEVNULL, shell=True) - else: - os.symlink(os.path.abspath(dest), os.path.abspath(src)) - - - print("enabling FactorioMaps mod") - modListPath = os.path.join(kwargs["modpath"], "mod-list.json") - - if not os.path.samefile(kwargs["modpath"], "../../mods"): - for f in os.listdir(kwargs["modpath"]): - if re.match(r'^L0laapk3_FactorioMaps_', f, flags=re.IGNORECASE): - print("Found other factoriomaps mod in custom mod folder, deleting.") - path = os.path.join(kwargs["modpath"], f) - if os.path.islink(path): - os.unlink(path) - else: - os.remove(path) - - linkDir(os.path.join(kwargs["modpath"], os.path.basename(os.path.abspath("."))), ".") - - - - def changeModlist(newState): - done = False - with open(modListPath, "r") as f: - modlist = json.load(f) - for mod in modlist["mods"]: - if mod["name"] == "L0laapk3_FactorioMaps": - mod["enabled"] = newState - done = True - if not done: - modlist["mods"].append({"name": "L0laapk3_FactorioMaps", "enabled": newState}) - with open(modListPath, "w") as f: - json.dump(modlist, f, indent=2) - - changeModlist(True) + if args.mod_path.resolve() != Path(userFolder,"mods").resolve(): + linkCustomModFolder(args.mod_path) + changeModlist(args.mod_path, True) manager = mp.Manager() rawTags = manager.dict() rawTags["__used"] = False - - - - if kwargs["delete"]: - print("deleting output folder") + if args.delete: + print(f"Deleting output folder ({workfolder})") try: rmtree(workfolder) except (FileNotFoundError, NotADirectoryError): pass + ########################################### + # # + # Start of Work # + # # + ########################################### - - - - datapath = os.path.join(workfolder, "latest.txt") - allTmpDirs = [] + datapath = Path(workfolder, "latest.txt") isFirstSnapshot = True try: - for index, savename in () if kwargs["dry"] else enumerate(savenames): - - + daytimes = [] + if args.day: + daytimes.append("day") + if args.night: + daytimes.append("night") - printErase("cleaning up") - if os.path.isfile(datapath): - os.remove(datapath) + for index, savename in () if args.dry else enumerate(saveGames): + for daytimeIndex, setDaytime in enumerate(daytimes): + printErase("cleaning up") + if datapath.is_file(): + datapath.unlink() - - printErase("building autorun.lua") - if (os.path.isfile(os.path.join(workfolder, "mapInfo.json"))): - with open(os.path.join(workfolder, "mapInfo.json"), "r", encoding='utf-8') as f: - mapInfoLua = re.sub(r'"([^"]+)" *:', lambda m: '["'+m.group(1)+'"] = ', f.read().replace("[", "{").replace("]", "}")) - if isFirstSnapshot: - f.seek(0) - mapInfo = json.load(f) - if "options" in mapInfo: - for kwarg in changedKwargs: - if kwarg in ("hd", "dayonly", "nightonly", "build-range", "connect-range", "tag-range"): - printErase("Warning: flag '" + kwarg + "' is overriden by previous setting found in existing timeline.") - isFirstSnapshot = False - - else: - mapInfoLua = "{}" + buildAutorun(args, workfolder, foldername, isFirstSnapshot, setDaytime) isFirstSnapshot = False - if (os.path.isfile(os.path.join(workfolder, "chunkCache.json"))): - with open(os.path.join(workfolder, "chunkCache.json"), "r") as f: - chunkCache = re.sub(r'"([^"]+)" *:', lambda m: '["'+m.group(1)+'"] = ', f.read().replace("[", "{").replace("]", "}")) - else: - chunkCache = "{}" - - with open("autorun.lua", "w", encoding="utf-8") as f: - surfaceString = '{"' + '", "'.join(kwargs["surface"]) + '"}' if len(kwargs["surface"]) > 0 else "nil" - autorunString = (f'fm.autorun = {{\n' - f'HD = {str(kwargs["hd"] == True).lower()},\n' - f'day = {str(kwargs["nightonly"] != True).lower()},\n' - f'night = {str(kwargs["dayonly"] != True).lower()},\n' - f'alt_mode = {str(kwargs["no-altmode"] != True).lower()},\n' - f'tags = {str(kwargs["no-tags"] != True).lower()},\n' - f'around_tag_range = {float(kwargs["tag-range"])},\n' - f'around_build_range = {float(kwargs["build-range"])},\n' - f'around_connect_range = {float(kwargs["connect-range"])},\n' - f'connect_types = {{"lamp", "electric-pole", "radar", "straight-rail", "curved-rail", "rail-signal", "rail-chain-signal", "locomotive", "cargo-wagon", "fluid-wagon", "car"}},\n' - f'date = "{datetime.datetime.strptime(kwargs["date"], "%d/%m/%y").strftime("%d/%m/%y")}",\n' - f'surfaces = {surfaceString},\n' - f'name = "{foldername + "/"}",\n' - f'mapInfo = {mapInfoLua.encode("utf-8").decode("unicode-escape")},\n' - f'chunkCache = {chunkCache},\n' - f'}}') - f.write(autorunString) - if kwargs["verbose"]: - printErase(autorunString) - - - printErase("building config.ini") - tmpDir = os.path.join(tempfile.gettempdir(), "FactorioMaps-%s" % random.randint(1, 999999999)) - allTmpDirs.append(tmpDir) - try: - rmtree(tmpDir) - except (FileNotFoundError, NotADirectoryError): - pass - os.makedirs(os.path.join(tmpDir, "config")) - - configPath = os.path.join(tmpDir, "config/config.ini") - config = configparser.ConfigParser() - config.read("../../config/config.ini") - - config["interface"]["show-tips-and-tricks"] = "false" - - config["path"]["write-data"] = tmpDir - config["graphics"]["screenshots-threads-count"] = str(int(kwargs["screenshotthreads" if kwargs["screenshotthreads"] else "maxthreads"])) - config["graphics"]["max-threads"] = config["graphics"]["screenshots-threads-count"] - - with open(configPath, 'w+') as outf: - outf.writelines(("; version=3\n", )) - config.write(outf, space_around_delimiters=False) - - - linkDir(os.path.join(tmpDir, "script-output"), "../../script-output") - copy("../../player-data.json", os.path.join(tmpDir, "player-data.json")) - - pid = None - isSteam = None - pidBlacklist = [p.info["pid"] for p in psutil.process_iter(attrs=['pid', 'name']) if p.info['name'] == "factorio.exe"] - - popenArgs = (factorioPath, '--load-game', os.path.abspath(os.path.join("../../saves", savename)), '--disable-audio', '--config', configPath, "--mod-directory", os.path.abspath(kwargs["modpath"]), "--disable-migration-window") - if kwargs["verbose"]: - printErase(popenArgs) - - - condition = mp.Condition() - - - results = manager.list() + with TemporaryDirectory(prefix="FactorioMaps-") as tmpDir: + configPath = buildConfig(args, tmpDir, args.basepath) - printErase("starting factorio") - startLogProcess = mp.Process(target=startGameAndReadGameLogs, args=(results, condition, popenArgs, tmpDir, pidBlacklist, rawTags), kwargs=kwargs) - startLogProcess.daemon = True - startLogProcess.start() - - - with condition: - condition.wait() - isSteam, pid = results[:] - - - if isSteam is None: - raise Exception("isSteam error") - if pid is None: - raise Exception("pid error") - - - - - while not os.path.exists(datapath): - time.sleep(0.4) - + pid = None + isSteam = None + pidBlacklist = [p.info["pid"] for p in psutil.process_iter(attrs=['pid', 'name']) if p.info['name'] == "factorio.exe"] - open("autorun.lua", 'w').close() - + launchArgs = [ + '--load-game', + str(Path(userFolder, 'saves', savename).absolute()), + '--disable-audio', + '--config', + str(configPath), + "--mod-directory",str(args.mod_path.absolute()), + "--disable-migration-window" + ] - latest = [] - with open(datapath, 'r') as f: - for line in f: - latest.append(line.rstrip("\n")) - if kwargs["verbose"]: - printErase(latest) + usedSteamLaunchHack = False - - firstOtherInputs = latest[-1].split(" ") - firstOutFolder = firstOtherInputs.pop(0).replace("/", " ") - waitfilename = os.path.join(basepath, firstOutFolder, "Images", firstOtherInputs[0], firstOtherInputs[1], firstOtherInputs[2], "done.txt") - - - isKilled = [False] - def waitKill(isKilled, pid): - while not isKilled[0]: - #print(f"Can I kill yet? {os.path.isfile(waitfilename)} {waitfilename}") - if os.path.isfile(waitfilename): - isKilled[0] = True - kill(pid) - break + if os.name == "nt": + steamApiPath = Path(factorioPath, "..", "steam_api64.dll") else: - time.sleep(0.4) - - killThread = threading.Thread(target=waitKill, args=(isKilled, pid)) - killThread.daemon = True - killThread.start() - + steamApiPath = Path(factorioPath, "..", "steam_api64.so") + if steamApiPath.exists(): # chances are this is a steam install.. + if os.name == "nt": + steamPath = Path(factorioPath, "..", "..", "..", "..", "..", "..", "steam.exe") + else: + steamPath = Path(factorioPath, "..", "..", "..", "..", "..", "..", "steam") + + if steamPath.exists(): # found a steam executable + usedSteamLaunchHack = True + exeWithArgs = [ + str(steamPath), + "-applaunch", + "427520" + ] + launchArgs + + if not usedSteamLaunchHack: # if non steam factorio, or if steam factorio but steam executable isnt found. + exeWithArgs = [ + str(factorioPath) + ] + launchArgs + + if args.verbose: + printErase(exeWithArgs) + + condition = mp.Condition() + results = manager.list() + + printErase("starting factorio") + startLogProcess = mp.Process( + target=startGameAndReadGameLogs, + args=(results, condition, exeWithArgs, usedSteamLaunchHack, tmpDir, pidBlacklist, rawTags, args) + ) + startLogProcess.daemon = True + startLogProcess.start() + + with condition: + condition.wait() + isSteam, pid = results[:] + + if isSteam is None: + raise Exception("isSteam error") + if pid is None: + raise Exception("pid error") + + while not datapath.exists(): + time.sleep(0.4) - if workthread and workthread.isAlive(): - #print("waiting for workthread") - workthread.join() + # empty autorun.lua + open("autorun.lua", 'w', encoding="utf-8").close() + + latest = [] + with datapath.open('r', encoding="utf-8") as f: + for line in f: + latest.append(line.rstrip("\n")) + if args.verbose: + printErase(latest) + + firstOutFolder, timestamp, surface, daytime = latest[-1].split(" ") + firstOutFolder = firstOutFolder.replace("/", " ") + waitfilename = Path(args.basepath, firstOutFolder, "images", timestamp, surface, daytime, "done.txt") + + isKilled = [False] + def waitKill(isKilled, pid): + while not isKilled[0]: + #print(f"Can I kill yet? {os.path.isfile(waitfilename)} {waitfilename}") + if os.path.isfile(waitfilename): + isKilled[0] = True + kill(pid) + break + else: + time.sleep(0.4) + + killThread = threading.Thread(target=waitKill, args=(isKilled, pid)) + killThread.daemon = True + killThread.start() + + if workthread and workthread.is_alive(): + #print("waiting for workthread") + workthread.join() + + timestamp = None + daytimeSurfaces = {} + for jindex, screenshot in enumerate(latest): + outFolder, timestamp, surface, daytime = list(map(lambda s: s.replace("|", " "), screenshot.split(" "))) + outFolder = outFolder.replace("/", " ") + print(f"Processing {outFolder}/{'/'.join([timestamp, surface, daytime])} ({len(latest) * index + jindex + 1 + daytimeIndex} of {len(latest) * len(saveGames) * len(daytimes)})") + + if daytime in daytimeSurfaces: + daytimeSurfaces[daytime].append(surface) + else: + daytimeSurfaces[daytime] = [surface] + #print("Cropping %s images" % screenshot) + crop(outFolder, timestamp, surface, daytime, args.basepath, args) + waitlocalfilename = os.path.join(args.basepath, outFolder, "Images", timestamp, surface, daytime, "done.txt") + if not os.path.exists(waitlocalfilename): + #print("waiting for done.txt") + while not os.path.exists(waitlocalfilename): + time.sleep(0.4) + def refZoom(): + needsThumbnail = index + 1 == len(saveGames) + #print("Crossreferencing %s images" % screenshot) + ref(outFolder, timestamp, surface, daytime, args.basepath, args) + #print("downsampling %s images" % screenshot) + zoom(outFolder, timestamp, surface, daytime, args.basepath, needsThumbnail, args) - timestamp = None - daytimeSurfaces = {} - for jindex, screenshot in enumerate(latest): - otherInputs = list(map(lambda s: s.replace("|", " "), screenshot.split(" "))) - outFolder = otherInputs.pop(0).replace("/", " ") - print("Processing {}/{} ({} of {})".format(outFolder, "/".join(otherInputs), len(latest) * index + jindex + 1, len(latest) * len(savenames))) + if jindex == len(latest) - 1: + print("zooming renderboxes", timestamp) + zoomRenderboxes(daytimeSurfaces, workfolder, timestamp, Path(args.basepath, firstOutFolder, "Images"), args) - timestamp = otherInputs[0] - if otherInputs[2] in daytimeSurfaces: - daytimeSurfaces[otherInputs[2]].append(otherInputs[1]) - else: - daytimeSurfaces[otherInputs[2]] = [otherInputs[1]] - - #print("Cropping %s images" % screenshot) - crop(outFolder, otherInputs[0], otherInputs[1], otherInputs[2], basepath, **kwargs) - waitlocalfilename = os.path.join(basepath, outFolder, "Images", otherInputs[0], otherInputs[1], otherInputs[2], "done.txt") - if not os.path.exists(waitlocalfilename): - #print("waiting for done.txt") - while not os.path.exists(waitlocalfilename): - time.sleep(0.4) + if screenshot != latest[-1]: + refZoom() + else: + startLogProcess.terminate() + # I have receieved a bug report from feidan in which he describes what seems like that this doesnt kill factorio? + onlyStall = isKilled[0] + isKilled[0] = True + kill(pid, onlyStall) - def refZoom(): - needsThumbnail = index + 1 == len(savenames) - #print("Crossreferencing %s images" % screenshot) - ref(outFolder, otherInputs[0], otherInputs[1], otherInputs[2], basepath, **kwargs) - #print("downsampling %s images" % screenshot) - zoom(outFolder, otherInputs[0], otherInputs[1], otherInputs[2], basepath, needsThumbnail, **kwargs) + if savename == saveGames[-1] and daytimeIndex == len(daytimes) - 1: + refZoom() - if jindex == len(latest) - 1: - print("zooming renderboxes", timestamp) - zoomRenderboxes(daytimeSurfaces, workfolder, timestamp, os.path.join(basepath, firstOutFolder, "Images"), **kwargs) + else: + workthread = threading.Thread(target=refZoom) + workthread.daemon = True + workthread.start() - if screenshot != latest[-1]: - refZoom() - else: - - startLogProcess.terminate() - # I have receieved a bug report from feidan in which he describes what seems like that this doesnt kill factorio? - - onlyStall = isKilled[0] - isKilled[0] = True - kill(pid, onlyStall) - if savename == savenames[-1]: - refZoom() - else: - workthread = threading.Thread(target=refZoom) - workthread.daemon = True - workthread.start() - - - - if os.path.isfile(os.path.join(workfolder, "mapInfo.out.json")): print("generating mapInfo.json") - with open(os.path.join(workfolder, "mapInfo.json"), 'r+', encoding='utf-8') as destf, open(os.path.join(workfolder, "mapInfo.out.json"), "r", encoding='utf-8') as srcf: + with Path(workfolder, "mapInfo.json").open('r+', encoding='utf-8') as destf, Path(workfolder, "mapInfo.out.json").open("r", encoding='utf-8') as srcf: data = json.load(destf) for mapIndex, mapStuff in json.load(srcf)["maps"].items(): for surfaceName, surfaceStuff in mapStuff["surfaces"].items(): @@ -663,7 +713,7 @@ def refZoom(): print("updating labels") tags = {} - with open(os.path.join(workfolder, "mapInfo.json"), 'r+', encoding='utf-8') as mapInfoJson: + with Path(workfolder, "mapInfo.json").open('r+', encoding='utf-8') as mapInfoJson: data = json.load(mapInfoJson) for mapStuff in data["maps"]: for surfaceName, surfaceStuff in mapStuff["surfaces"].items(): @@ -673,22 +723,22 @@ def refZoom(): tags[tag["iconType"] + tag["iconName"][0].upper() + tag["iconName"][1:]] = tag rmtree(os.path.join(workfolder, "Images", "labels"), ignore_errors=True) - + modVersions = sorted( map(lambda m: (m.group(2).lower(), (m.group(3), m.group(4), m.group(5), m.group(6) is None), m.group(1)), filter(lambda m: m, map(lambda f: re.search(r"^((.*)_(\d+)\.(\d+)\.(\d+))(\.zip)?$", f, flags=re.IGNORECASE), - os.listdir(os.path.join(basepath, kwargs["modpath"]))))), + os.listdir(os.path.join(args.basepath, args.mod_path))))), key = lambda t: t[1], reverse = True) rawTags["__used"] = True - if not kwargs["no-tags"]: + if args.tags: for _, tag in tags.items(): dest = os.path.join(workfolder, tag["iconPath"]) os.makedirs(os.path.dirname(dest), exist_ok=True) - + rawPath = rawTags[tag["iconType"] + tag["iconName"][0].upper() + tag["iconName"][1:]] @@ -707,7 +757,7 @@ def refZoom(): else: mod = next(mod for mod in modVersions if mod[0] == m.group(1).lower()) if not mod[1][3]: #true if mod is zip - zipPath = os.path.join(basepath, kwargs["modpath"], mod[2] + ".zip") + zipPath = os.path.join(args.basepath, args.mod_path, mod[2] + ".zip") with ZipFile(zipPath, 'r') as zipObj: if len(icons) == 1: zipInfo = zipObj.getinfo(os.path.join(mod[2], icon + ".png").replace('\\', '/')) @@ -717,18 +767,18 @@ def refZoom(): else: src = zipObj.extract(os.path.join(mod[2], icon + ".png").replace('\\', '/'), os.path.join(tempfile.gettempdir(), "FactorioMaps")) else: - src = os.path.join(basepath, kwargs["modpath"], mod[2], icon + ".png") - + src = os.path.join(args.basepath, args.mod_path, mod[2], icon + ".png") + if len(icons) == 1: if src is not None: img = Image.open(src) w, h = img.size - img = img.crop((0, 0, h, h)) + img = img.crop((0, 0, h, h)).resize((64, 64)) img.save(dest) else: newImg = Image.open(src) w, h = newImg.size - newImg = newImg.crop((0, 0, h, h)).convert("RGBA") + newImg = newImg.crop((0, 0, h, h)).resize((64, 64)).convert("RGBA") if len(iconColor) > 1: newImg = ImageChops.multiply(newImg, Image.new("RGBA", newImg.size, color=tuple(map(lambda s: int(round(float(s))), iconColor[1].split("%"))))) if i == 0: @@ -740,28 +790,34 @@ def refZoom(): + print("applying configuration") + with Path(workfolder, "mapInfo.json").open("r+", encoding='utf-8') as f: + mapInfo = json.load(f) + if args.default_timestamp != None or "defaultTimestamp" not in mapInfo["options"]: + if args.default_timestamp == None: + args.default_timestamp = -1 + mapInfo["options"]["defaultTimestamp"] = args.default_timestamp + f.seek(0) + json.dump(mapInfo, f) + f.truncate() - - #TODO: download leaflet shit - print("generating mapInfo.js") - with open(os.path.join(workfolder, "mapInfo.js"), 'w') as outf, open(os.path.join(workfolder, "mapInfo.json"), "r", encoding='utf-8') as inf: + with Path(workfolder, "mapInfo.js").open('w', encoding="utf-8") as outf, Path(workfolder, "mapInfo.json").open("r", encoding='utf-8') as inf: outf.write('"use strict";\nwindow.mapInfo = JSON.parse(') outf.write(json.dumps(inf.read())) outf.write(");") - - + + print("creating index.html") - copy("web/index.html", os.path.join(workfolder, "index.html")) - copy("web/index.css", os.path.join(workfolder, "index.css")) - copy("web/index.js", os.path.join(workfolder, "index.js")) + for fileName in ("index.html", "index.css", "index.js"): + copy(Path(__file__, "..", "web", fileName).resolve(), os.path.join(workfolder, fileName)) try: rmtree(os.path.join(workfolder, "lib")) except (FileNotFoundError, NotADirectoryError): pass - copytree("web/lib", os.path.join(workfolder, "lib")) + copytree(Path(__file__, "..", "web", "lib").resolve(), os.path.join(workfolder, "lib")) @@ -777,26 +833,7 @@ def refZoom(): except: pass - print("disabling FactorioMaps mod") - changeModlist(False) - - - - print("cleaning up") - for tmpDir in allTmpDirs: - try: - os.unlink(os.path.join(tmpDir, "script-output")) - rmtree(tmpDir) - except (FileNotFoundError, NotADirectoryError): - pass - - - - - - - - + changeModlist(args.mod_path, False) if __name__ == '__main__': - auto(*sys.argv[1:]) \ No newline at end of file + auto(*sys.argv[1:]) diff --git a/control.lua b/control.lua index 6fbebef..8e8f8d0 100644 --- a/control.lua +++ b/control.lua @@ -63,7 +63,7 @@ script.on_event(defines.events.on_tick, function(event) end fm.savename = fm.autorun.name or "" - fm.topfolder = "FactorioMaps/" .. fm.savename + fm.topfolder = fm.savename fm.autorun.tick = game.tick hour = math.ceil(fm.autorun.tick / 60 / 60 / 60) @@ -74,7 +74,7 @@ script.on_event(defines.events.on_tick, function(event) exists = false if fm.autorun.mapInfo.maps ~= nil then for _, map in pairs(fm.autorun.mapInfo.maps) do - if map.path == fm.autorun.filePath then + if map.path == fm.autorun.filePath and map.tick ~= fm.autorun.tick then exists = true break end @@ -87,9 +87,17 @@ script.on_event(defines.events.on_tick, function(event) end fm.API.pull() + if fm.autorun.surfaces == nil then - fm.autorun.surfaces = { fm.autorun.mapInfo.defaultSurface or "nauvis" } + if fm.autorun.mapInfo.defaultSurface == nil then + if game.surfaces["battle_surface_1"] then -- detect pvp scenario + fm.autorun.mapInfo.defaultSurface = "battle_surface_1" + else + fm.autorun.mapInfo.defaultSurface = "nauvis" + end + end + fm.autorun.surfaces = { fm.autorun.mapInfo.defaultSurface } else for index, surfaceName in pairs(fm.autorun.surfaces) do if player.surface.name == surfaceName then -- move surface the player is on to first @@ -150,12 +158,7 @@ script.on_event(defines.events.on_tick, function(event) latest = "" for _, surfaceName in pairs(fm.autorun.surfaces) do local surface = game.surfaces[surfaceName] - if fm.autorun.mapInfo.options.night and not surface.freeze_daytime then - latest = fm.autorun.name:sub(1, -2):gsub(" ", "/") .. " " .. fm.autorun.filePath .. " " .. surfaceName:gsub(" ", "|") .. " night\n" .. latest - end - if fm.autorun.mapInfo.options.day or (surface.freeze_daytime and fm.autorun.mapInfo.options.night) then - latest = fm.autorun.name:sub(1, -2):gsub(" ", "/") .. " " .. fm.autorun.filePath .. " " .. surfaceName:gsub(" ", "|") .. " day\n" .. latest - end + latest = fm.autorun.name:sub(1, -2):gsub(" ", "/") .. " " .. fm.autorun.filePath .. " " .. surfaceName:gsub(" ", "|") .. " " .. fm.autorun.daytime .. "\n" .. latest end game.write_file(fm.topfolder .. "latest.txt", latest, false, event.player_index) @@ -198,9 +201,11 @@ script.on_event(defines.events.on_tick, function(event) - if fm.autorun.mapInfo.options.day then + if fm.autorun.daytime == "day" then fm.currentSurface.daytime = 0 - fm.daytime = "day" + fm.generateMap(event) + else + fm.currentSurface.daytime = 0.5 fm.generateMap(event) end @@ -208,46 +213,17 @@ script.on_event(defines.events.on_tick, function(event) elseif fm.ticks < 2 then - if fm.autorun.mapInfo.options.day then - game.write_file(fm.topfolder .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/day/done.txt", "", false, event.player_index) - end + game.write_file(fm.topfolder .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.autorun.daytime .. "/done.txt", "", false, event.player_index) -- remove no path sign for key, entity in pairs(fm.currentSurface.find_entities_filtered({type="flying-text"})) do entity.destroy() end - if fm.autorun.mapInfo.options.night then - fm.currentSurface.daytime = 0.5 - fm.daytime = "night" - fm.generateMap(event) - end fm.ticks = 2 - - elseif fm.ticks < 3 then - - if fm.autorun.mapInfo.options.night then - game.write_file(fm.topfolder .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/night/done.txt", "", false, event.player_index) - end - - game.write_file(fm.topfolder .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/done.txt", "", false, event.player_index) - - - -- unfreeze all entities - for key, entity in pairs(fm.currentSurface.find_entities_filtered({})) do - entity.active = true - end - - - if #fm.autorun.surfaces > 0 then - fm.ticks = nil - else - fm.ticks = 3 - end else - fm.daytime = nil fm.topfolder = nil fm.done = true diff --git a/crop.py b/crop.py index 803c930..023fc52 100644 --- a/crop.py +++ b/crop.py @@ -1,88 +1,92 @@ -from PIL import Image import multiprocessing as mp -import os, math, sys, time, psutil, json +import os +import sys +import time +from argparse import Namespace from functools import partial +from pathlib import Path from shutil import get_terminal_size as tsize +import psutil +from PIL import Image - - ext = ".png" + def work(line, folder, progressQueue): - arg = line.rstrip('\n').split(" ", 5) - path = os.path.join(folder, arg[5]) - top = int(arg[0]) - left = int(arg[1]) - width = int(arg[2]) - height = int(arg[3]) - + arg = line.rstrip("\n").split(" ", 5) + path = Path(folder, arg.pop(5)) + arg = list(map(int, arg[:4])) + top, left, width, height = arg try: - Image.open(path).convert("RGB").crop((top, left, top + width, left + height)).save(path) + Image.open(path).convert("RGB").crop( + (top, left, top + width, left + height) + ).save(path) except IOError: progressQueue.put(False, True) return line except: progressQueue.put(False, True) import traceback + traceback.print_exc() pass return False progressQueue.put(True, True) return False - +def crop(outFolder, timestamp, surface, daytime, basePath=None, args: Namespace = Namespace()): + psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10) -def crop(*args, **kwargs): + subname = Path(timestamp, surface, daytime) + toppath = Path( + basePath if basePath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps"), + outFolder, + ) - psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) + imagePath = Path(toppath, "Images") - subname = os.path.join(*args[1:4]) - toppath = os.path.join((args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) + datapath = Path(imagePath, subname, "crop.txt") + maxthreads = args.cropthreads if args.cropthreads else args.maxthreads - basepath = os.path.join(toppath, "Images") - + while not datapath.exists(): + time.sleep(1) + print(f"crop {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="") - datapath = os.path.join(basepath, subname, "crop.txt") - maxthreads = int(kwargs["cropthreads" if kwargs["cropthreads"] else "maxthreads"]) - - - if not os.path.exists(datapath): - #print("waiting for game") - while not os.path.exists(datapath): - time.sleep(1) - - print("crop {:5.1f}% [{}]".format(0, " " * (tsize()[0]-15)), end="") - files = [] - with open(datapath, "r") as data: - assert(data.readline().rstrip('\n') == "v2") + with datapath.open("r", encoding="utf-8") as data: + assert data.readline().rstrip("\n") == "v2" for line in data: files.append(line) - + pool = mp.Pool(processes=maxthreads) - + m = mp.Manager() progressQueue = m.Queue() originalSize = len(files) doneSize = 0 + try: while len(files) > 0: - workers = pool.map_async(partial(work, folder=basepath, progressQueue=progressQueue), files, 128) + workers = pool.map_async( + partial(work, folder=imagePath, progressQueue=progressQueue), + files, + 128, + ) for _ in range(len(files)): if progressQueue.get(True): doneSize += 1 progress = float(doneSize) / originalSize - tsiz = tsize()[0]-15 - print("\rcrop {:5.1f}% [{}{}]".format(round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") + tsiz = tsize()[0] - 15 + print(f"\rcrop {round(progress * 100, 1):5.1f}% [{'=' * int(progress * tsiz)}{' ' * (tsiz - int(progress * tsiz))}]",end="",) workers.wait() files = [x for x in workers.get() if x] if len(files) > 0: time.sleep(10 if len(files) > 1000 else 1) - print("\rcrop {:5.1f}% [{}]".format(100, "=" * (tsize()[0]-15))) + print(f"\rcrop {100:5.1f}% [{'=' * (tsize()[0]-15)}]") except KeyboardInterrupt: time.sleep(0.2) @@ -92,13 +96,3 @@ def crop(*args, **kwargs): print(line) raise - - - - - - - - -if __name__ == '__main__': - crop(*sys.argv[1:]) \ No newline at end of file diff --git a/data-final-fixes.lua b/data-final-fixes.lua index c865876..f62798e 100644 --- a/data-final-fixes.lua +++ b/data-final-fixes.lua @@ -69,6 +69,22 @@ end +-- user_tiles = [] +-- for key, item in pairs(data.raw["item"]) do +-- if item.place_as_tile then + +-- end +-- end + + +-- for key, tile in pairs(data.raw["tile"]) do +-- no = "NO" +-- if tile.items_to_place_this then +-- no = "YES" +-- end +-- log(key .. " " .. no) +-- end + data.raw["utility-sprites"].default["ammo_icon"]["filename"] = "__L0laapk3_FactorioMaps__/graphics/empty64.png" data.raw["utility-sprites"].default["danger_icon"]["filename"] = "__L0laapk3_FactorioMaps__/graphics/empty64.png" diff --git a/generateMap.lua b/generateMap.lua index 7da5867..8f0e951 100644 --- a/generateMap.lua +++ b/generateMap.lua @@ -54,7 +54,7 @@ function fm.generateMap(data) -- delete folder (if it already exists) local basePath = fm.topfolder - local subPath = basePath .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.daytime + local subPath = basePath .. "Images/" .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.autorun.daytime game.remove_path(subPath) subPath = subPath .. "/" @@ -78,37 +78,27 @@ function fm.generateMap(data) if fm.tilenames == nil then - local blacklist = { - "water", - "dirt", - "grass", - "lab", - "out-of-map", - "desert", - "sand", - "tutorial", - "ghost" - } + local craftableItems = {} + for _, recipe in pairs(game.recipe_prototypes) do + for _, product in pairs(recipe.products) do + if product.type == "item" then + craftableItems[product.name] = true + end + end + end local tilenamedict = {} - for _, item in pairs(game.item_prototypes) do - if item.place_as_tile_result ~= nil and tilenamedict[item.place_as_tile_result.result.name] == nil then - for _, keyword in pairs(blacklist) do - if string.match(item.place_as_tile_result.result.name, keyword) then - tilenamedict[item.place_as_tile_result.result.name] = false - goto continue - end - end + for itemName, _ in pairs(craftableItems) do + item = game.item_prototypes[itemName] + if item.place_as_tile_result ~= nil and item.place_as_tile_result.result.autoplace_specification == nil then tilenamedict[item.place_as_tile_result.result.name] = true end ::continue:: end fm.tilenames = {} - for tilename, value in pairs(tilenamedict) do - if value then - fm.tilenames[#fm.tilenames+1] = tilename - end + for tilename, _ in pairs(tilenamedict) do + fm.tilenames[#fm.tilenames+1] = tilename end end @@ -165,7 +155,7 @@ function fm.generateMap(data) end if tonumber(mapTick) == fm.autorun.tick then for i, map in pairs(fm.autorun.mapInfo.maps) do - if map.tick == mapTick then + if map.tick == fm.autorun.tick then surfaceWasScanned = v[fm.currentSurface.name] ~= nil mapIndex = i break @@ -240,10 +230,10 @@ function fm.generateMap(data) -- build range for chunk in fm.currentSurface.get_chunks() do if fm.currentSurface.is_chunk_generated(chunk) then - log(chunk.x .. " " .. chunk.y) + -- log(chunk.x .. " " .. chunk.y) for _, force in pairs(game.forces) do if #force.players > 0 and force.is_chunk_charted(fm.currentSurface, chunk) then - log("charted by " .. force.name) + -- log("charted by " .. force.name) forceStats[force.name] = forceStats[force.name] + 1 imageStats.charted = imageStats.charted + 1 for gridX = chunk.x * tilesPerChunk / gridPixelSize, (chunk.x + 1) * tilesPerChunk / gridPixelSize - 1 do @@ -551,7 +541,7 @@ function fm.generateMap(data) game.write_file(basePath .. "chunkCache.json", prettyjson(fm.autorun.chunkCache), false, data.player_index) end - fm.autorun.mapInfo.maps[mapIndex].surfaces[fm.currentSurface.name][fm.daytime] = true + fm.autorun.mapInfo.maps[mapIndex].surfaces[fm.currentSurface.name][fm.autorun.daytime] = true -- todo: if fm.autorun.mapInfo.maps[mapIndex].surfaces[fm.currentSurface.name].hidden is true, only care about the chunks linked to by renderboxes. @@ -561,7 +551,7 @@ function fm.generateMap(data) - log("[info]Surface capture " .. fm.savename .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.daytime) + log("[info]Surface capture " .. fm.savename .. fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.autorun.daytime) @@ -605,7 +595,7 @@ function fm.generateMap(data) { x = (chunk.x+1) * gridPixelSize, y = (chunk.y+1) * gridPixelSize } } - capture(positionTable, fm.currentSurface, fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.daytime .. "/" .. maxZoom .. "/" .. chunk.x .. "/" .. chunk.y .. extension) + capture(positionTable, fm.currentSurface, fm.autorun.filePath .. "/" .. fm.currentSurface.name .. "/" .. fm.autorun.daytime .. "/" .. maxZoom .. "/" .. chunk.x .. "/" .. chunk.y .. extension) end @@ -621,14 +611,14 @@ function fm.generateMap(data) while #linkWorkList > 0 do local link = table.remove(linkWorkList) - local folder = fm.autorun.filePath .. "/" .. link.toSurface .. "/" .. fm.daytime .. "/" .. "renderboxes" .. "/" + local folder = fm.autorun.filePath .. "/" .. link.toSurface .. "/" .. fm.autorun.daytime .. "/" .. "renderboxes" .. "/" local filename = link.to[1].x .. "_" .. link.to[1].y .. "_" .. link.to[2].x .. "_" .. link.to[2].y local path = folder .. maxZoom .. "/" .. filename local surface = game.surfaces[link.toSurface] link.daynight = not surface.freeze_daytime if link.daynight then - surface.daytime = fm.daytime == "day" and 0 or 0.5 + surface.daytime = fm.autorun.daytime == "day" and 0 or 0.5 end diff --git a/info.json b/info.json index 8d16a11..2d300a5 100644 --- a/info.json +++ b/info.json @@ -1,6 +1,6 @@ { "name": "L0laapk3_FactorioMaps", - "version": "3.5.5", + "version": "4.0.0", "title": "FactorioMaps", "author": "L0laapk3", "contact": "https://github.com/L0laapk3/", diff --git a/makeZip.py b/makeZip.py index ee1d253..e4660bc 100644 --- a/makeZip.py +++ b/makeZip.py @@ -1,10 +1,9 @@ -import tempfile import os -from shutil import rmtree, copy, make_archive - import shutil -from updateLib import update as updateLib +import tempfile +from shutil import copy, make_archive, rmtree +from updateLib import update as updateLib folderName = os.path.basename(os.path.realpath(".")) tempPath = os.path.join(tempfile.gettempdir(), folderName) diff --git a/packages.txt b/packages.txt index 4eea34e..8915d61 100644 --- a/packages.txt +++ b/packages.txt @@ -2,3 +2,4 @@ PyTurboJPEG>=1.1.5 psutil>=5.4.8 numpy>=1.16.4 Pillow>=6.1.0 +orderedset>=2.0.1 \ No newline at end of file diff --git a/ref.py b/ref.py index eb44e8e..f0a24fe 100644 --- a/ref.py +++ b/ref.py @@ -1,4 +1,6 @@ +from argparse import Namespace import os, sys, math, time, json, psutil +from pathlib import Path from PIL import Image, ImageChops, ImageStat import multiprocessing as mp from functools import partial @@ -37,7 +39,7 @@ def compare(path, basePath, new, progressQueue): progressQueue.put(True, True) return (testResult, path[1:]) -def compare_renderbox(renderbox, basePath, new): +def compareRenderbox(renderbox, basePath, new): newPath = os.path.join(basePath, new, renderbox[0]) + ext testResult = False try: @@ -59,13 +61,13 @@ def neighbourScan(coord, keepList, cropList): surfaceName, daytime, z = coord[:3] x, y = int(coord[3]), int(os.path.splitext(coord[4])[0]) return (((surfaceName, daytime, z, str(x+1), str(y+1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x+1, y+1), 0) & 0b1000) \ - or ((surfaceName, daytime, z, str(x+1), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x+1, y-1), 0) & 0b0100) \ - or ((surfaceName, daytime, z, str(x-1), str(y+1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y+1), 0) & 0b0010) \ - or ((surfaceName, daytime, z, str(x-1), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y-1), 0) & 0b0001) \ - or ((surfaceName, daytime, z, str(x+1), str(y ) + ext) in keepList and cropList.get((surfaceName, daytime, z, x+1, y ), 0) & 0b1100) \ - or ((surfaceName, daytime, z, str(x-1), str(y ) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y ), 0) & 0b0011) \ - or ((surfaceName, daytime, z, str(x ), str(y+1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x , y+1), 0) & 0b1010) \ - or ((surfaceName, daytime, z, str(x ), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x , y-1), 0) & 0b0101), coord) + or ((surfaceName, daytime, z, str(x+1), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x+1, y-1), 0) & 0b0100) \ + or ((surfaceName, daytime, z, str(x-1), str(y+1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y+1), 0) & 0b0010) \ + or ((surfaceName, daytime, z, str(x-1), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y-1), 0) & 0b0001) \ + or ((surfaceName, daytime, z, str(x+1), str(y ) + ext) in keepList and cropList.get((surfaceName, daytime, z, x+1, y ), 0) & 0b1100) \ + or ((surfaceName, daytime, z, str(x-1), str(y ) + ext) in keepList and cropList.get((surfaceName, daytime, z, x-1, y ), 0) & 0b0011) \ + or ((surfaceName, daytime, z, str(x ), str(y+1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x , y+1), 0) & 0b1010) \ + or ((surfaceName, daytime, z, str(x ), str(y-1) + ext) in keepList and cropList.get((surfaceName, daytime, z, x , y-1), 0) & 0b0101), coord) @@ -94,33 +96,39 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64) - - -def ref(*args, **kwargs): +def ref( + outFolder: Path, + timestamp: str = None, + surfaceReference: str = None, + daytimeReference: str = None, + basepath: Path = None, + args: Namespace = Namespace(), +): psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) - - toppath = os.path.join((args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) - datapath = os.path.join(toppath, "mapInfo.json") - maxthreads = int(kwargs["refthreads" if kwargs["refthreads"] else"maxthreads"]) + workFolder = basepath if basepath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps") + topPath = Path(workFolder, outFolder) + dataPath = Path(topPath, "mapInfo.json") + maxthreads = args.refthreads if args.refthreads else args.maxthreads pool = mp.Pool(processes=maxthreads) - with open(datapath, "r", encoding="utf-8") as f: + with open(dataPath, "r", encoding="utf-8") as f: data = json.load(f) - if os.path.isfile(datapath[:-5] + ".out.json"): - with open(datapath[:-5] + ".out.json", "r", encoding="utf-8") as f: - outdata = json.load(f) + outFile = Path(topPath, "mapInfo.out.json") + if outFile.exists(): + with outFile.open("r", encoding="utf-8") as mapInfoOutFile: + outdata = json.load(mapInfoOutFile) else: outdata = {} - if len(args) > 1: + if timestamp: for i, mapObj in enumerate(data["maps"]): - if mapObj["path"] == args[1]: + if mapObj["path"] == timestamp: new = i break else: @@ -146,9 +154,9 @@ def ref(*args, **kwargs): firstRemoveList = [] cropList = {} didAnything = False - if len(args) <= 3 or daytime == args[3]: + if daytime is None or daytime == daytimeReference: for surfaceName, surface in newMap["surfaces"].items(): - if (len(args) <= 2 or surfaceName == args[2]) and daytime in surface and str(surface[daytime]) and (len(args) <= 3 or daytime == args[3]): + if (surfaceReference is None or surfaceName == surfaceReference) and daytime in surface and str(surface[daytime]) and (daytime is None or daytime == daytimeReference): didAnything = True z = surface["zoom"]["max"] @@ -156,7 +164,7 @@ def ref(*args, **kwargs): dayImages = [] newComparedSurfaces.append((surfaceName, daytime)) - + oldMapsList = [] for old in range(new): if surfaceName in data["maps"][old]["surfaces"]: @@ -164,7 +172,7 @@ def ref(*args, **kwargs): def readCropList(path, combinePrevious): - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: version = 2 if f.readline().rstrip('\n') == "v2" else 1 for line in f: if version == 1: @@ -173,18 +181,19 @@ def readCropList(path, combinePrevious): value = split[4] else: split = line.rstrip("\n").split(" ", 5) - pathSplit = split[5].split("/", 2) - if pathSplit[0] != str(z): + pathSplit = split[5].split("/", 5) + if pathSplit[3] != str(z): continue - key = (surfaceName, daytime, str(z), int(pathSplit[1]), int(os.path.splitext(pathSplit[2])[0])) + #(surfaceName, daytime, z, str(x+1), str(y+1) + ext) + key = (surfaceName, daytime, str(z), int(pathSplit[4]), int(os.path.splitext(pathSplit[5])[0])) value = split[2] - + cropList[key] = int(value, 16) | cropList.get(key, 0) if combinePrevious else int(value, 16) for old in oldMapsList: - readCropList(os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, "crop.txt"), False) - - readCropList(os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, "crop.txt"), True) + readCropList(os.path.join(topPath, "Images", data["maps"][old]["path"], surfaceName, daytime, "crop.txt"), False) + + readCropList(os.path.join(topPath, "Images", newMap["path"], surfaceName, daytime, "crop.txt"), True) @@ -193,26 +202,26 @@ def readCropList(path, combinePrevious): if surfaceName in data["maps"][old]["surfaces"] and daytime in surface and z == surface["zoom"]["max"]: if surfaceName not in allImageIndex: allImageIndex[surfaceName] = {} - path = os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z)) + path = os.path.join(topPath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): oldImages[(x, y.replace(ext, outext))] = data["maps"][old]["path"] if daytime != "day": - if not os.path.isfile(os.path.join(toppath, "Images", newMap["path"], surfaceName, "day", "ref.txt")): + if not os.path.isfile(os.path.join(topPath, "Images", newMap["path"], surfaceName, "day", "ref.txt")): print("WARNING: cannot find day surface to copy non-day surface from. running ref.py on night surfaces is not very accurate.") else: - if kwargs["verbose"]: print("found day surface, reuse results from ref.py from there") - - with open(os.path.join(toppath, "Images", newMap["path"], surfaceName, "day", "ref.txt"), "r") as f: + if args.verbose: print("found day surface, reuse results from ref.py from there") + + with Path(topPath, "Images", newMap["path"], surfaceName, "day", "ref.txt").open("r", encoding="utf-8") as f: for line in f: dayImages.append(tuple(line.rstrip("\n").split(" ", 2))) - + allDayImages[surfaceName] = dayImages - - path = os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, str(z)) + + path = os.path.join(topPath, "Images", newMap["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): if (x, os.path.splitext(y)[0]) in dayImages or (x, y.replace(ext, outext)) not in oldImages: @@ -220,22 +229,22 @@ def readCropList(path, combinePrevious): elif (x, y.replace(ext, outext)) in oldImages: compareList.append((oldImages[(x, y.replace(ext, outext))], surfaceName, daytime, str(z), x, y)) - + if not didAnything: continue - - if kwargs["verbose"]: print("found %s new images" % len(keepList)) + + if args.verbose: print("found %s new images" % len(keepList)) if len(compareList) > 0: - if kwargs["verbose"]: print("comparing %s existing images" % len(compareList)) + if args.verbose: print("comparing %s existing images" % len(compareList)) m = mp.Manager() progressQueue = m.Queue() - #compare(compareList[0], treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue) - workers = pool.map_async(partial(compare, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue), compareList, 128) + #compare(compareList[0], treshold=treshold, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue) + workers = pool.map_async(partial(compare, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue), compareList, 128) doneSize = 0 print("ref {:5.1f}% [{}]".format(0, " " * (tsize()[0]-15)), end="") for i in range(len(compareList)): @@ -249,30 +258,30 @@ def readCropList(path, combinePrevious): newList = [x[1] for x in [x for x in resultList if x[0]]] firstRemoveList += [x[1] for x in [x for x in resultList if not x[0]]] - if kwargs["verbose"]: print("found %s changed in %s images" % (len(newList), len(compareList))) + if args.verbose: print("found %s changed in %s images" % (len(newList), len(compareList))) keepList += newList print("\rref {:5.1f}% [{}]".format(100, "=" * (tsize()[0]-15))) - - if kwargs["verbose"]: print("scanning %s chunks for neighbour cropping" % len(firstRemoveList)) + + if args.verbose: print("scanning %s chunks for neighbour cropping" % len(firstRemoveList)) resultList = pool.map(partial(neighbourScan, keepList=keepList, cropList=cropList), firstRemoveList, 64) neighbourList = [x[1] for x in [x for x in resultList if x[0]]] removeList = [x[1] for x in [x for x in resultList if not x[0]]] - if kwargs["verbose"]: print("keeping %s neighbouring images" % len(neighbourList)) + if args.verbose: print("keeping %s neighbouring images" % len(neighbourList)) - if kwargs["verbose"]: print("deleting %s, keeping %s of %s existing images" % (len(removeList), len(keepList) + len(neighbourList), len(keepList) + len(neighbourList) + len(removeList))) + if args.verbose: print("deleting %s, keeping %s of %s existing images" % (len(removeList), len(keepList) + len(neighbourList), len(keepList) + len(neighbourList) + len(removeList))) - if kwargs["verbose"]: print("removing identical images") + if args.verbose: print("removing identical images") for x in removeList: - os.remove(os.path.join(toppath, "Images", newMap["path"], *x)) + os.remove(os.path.join(topPath, "Images", newMap["path"], *x)) - if kwargs["verbose"]: print("creating render index") + if args.verbose: print("creating render index") for surfaceName, daytime in newComparedSurfaces: z = surface["zoom"]["max"] - with open(os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, "ref.txt"), "w") as f: + with Path(topPath, "Images", newMap["path"], surfaceName, daytime, "ref.txt").open("w", encoding="utf-8") as f: for aList in (keepList, neighbourList): for coord in aList: if coord[0] == surfaceName and coord[1] == daytime and coord[2] == str(z): @@ -281,7 +290,7 @@ def readCropList(path, combinePrevious): - if kwargs["verbose"]: print("creating client index") + if args.verbose: print("creating client index") for aList in (keepList, neighbourList): for coord in aList: x = int(coord[3]) @@ -296,12 +305,12 @@ def readCropList(path, combinePrevious): allImageIndex[coord[0]][coord[1]][y].append(x) - - - if kwargs["verbose"]: print("comparing renderboxes") + + + if args.verbose: print("comparing renderboxes") if "renderboxesCompared" not in outdata["maps"][str(new)]: changed = True outdata["maps"][str(new)]["renderboxesCompared"] = True @@ -339,7 +348,7 @@ def readCropList(path, combinePrevious): compareList = compareList.values() - resultList = pool.map(partial(compare_renderbox, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])), compareList, 16) + resultList = pool.map(partial(compareRenderbox, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"])), compareList, 16) count = 0 for (isDifferent, path, oldPath, links) in resultList: @@ -348,12 +357,12 @@ def readCropList(path, combinePrevious): for (surfaceName, linkIndex) in links: outdata["maps"][str(new)]["surfaces"][surfaceName]["links"][linkIndex] = { "path": oldPath } - + else: count += 1 - if kwargs["verbose"]: print("removed %s of %s compared renderboxes, found %s new" % (count, len(compareList), totalCount)) - + if args.verbose: print("removed %s of %s compared renderboxes, found %s new" % (count, len(compareList), totalCount)) + @@ -374,47 +383,32 @@ def readCropList(path, combinePrevious): string = getBase64(y, False) isLastChangedImage = False isLastNightImage = False - + for x in range(min(xList), max(xList) + 2): - isChangedImage = x in xList #does the image exist at all? + isChangedImage = x in xList #does the image exist at all? isNightImage = daytime == "night" and (str(x), str(y)) not in allDayImages[surfaceName] #is this image only in night? if isLastChangedImage != isChangedImage or (isChangedImage and isLastNightImage != isNightImage): #differential encoding string += getBase64(x, isNightImage if isChangedImage else isLastNightImage) isLastChangedImage = isChangedImage isLastNightImage = isNightImage indexList.append(string) - - + + if surfaceName not in outdata["maps"][str(new)]["surfaces"]: outdata["maps"][str(new)]["surfaces"][surfaceName] = {} outdata["maps"][str(new)]["surfaces"][surfaceName]["chunks"] = '='.join(indexList) if len(indexList) > 0: changed = True - + if changed: - if kwargs["verbose"]: print("writing mapInfo.out.json") - with open(datapath[:-5] + ".out.json", "w+", encoding="utf-8") as f: + if args.verbose: print("writing mapInfo.out.json") + with outFile.open("w+", encoding="utf-8") as f: json.dump(outdata, f) - if kwargs["verbose"]: print("deleting empty folders") - for curdir, subdirs, files in os.walk(toppath, *args[1:4]): + if args.verbose: print("deleting empty folders") + for curdir, subdirs, files in os.walk(Path(topPath, timestamp, surfaceReference, daytimeReference)): if len(subdirs) == 0 and len(files) == 0: os.rmdir(curdir) - - - - - - - - - - - - - -if __name__ == '__main__': - ref(*sys.argv[1:]) \ No newline at end of file diff --git a/updateLib.py b/updateLib.py index 3b51f6e..45fd38b 100644 --- a/updateLib.py +++ b/updateLib.py @@ -1,12 +1,10 @@ -from shutil import rmtree, copytree -import os +from pathlib import Path +from shutil import copytree, rmtree +from tempfile import TemporaryDirectory from urllib.parse import urlparse -from urllib.request import urlretrieve, build_opener, install_opener -from tempfile import gettempdir +from urllib.request import build_opener, install_opener, urlretrieve - - -urlList = ( +URLLIST = ( "https://cdn.jsdelivr.net/npm/leaflet@1.6.0/dist/leaflet.css", "https://cdn.jsdelivr.net/npm/leaflet@1.6.0/dist/leaflet-src.min.js", "https://cdn.jsdelivr.net/npm/leaflet.fullscreen@1.4.5/Control.FullScreen.css", @@ -20,63 +18,49 @@ "https://factorio.com/static/img/favicon.ico", ) -CURRENTVERSION = 4 - - - +CURRENTVERSION = 4 def update(Force=True): - targetPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "web/lib") - + targetPath = Path(__file__, "..", "web", "lib") + if not Force: try: - with open(os.path.join(targetPath, "VERSION"), "r") as f: + with open(Path(targetPath, "VERSION"), "r") as f: if f.readline() == str(CURRENTVERSION): return False except FileNotFoundError: pass - tempPath = os.path.join(gettempdir(), "FactorioMapsTmpLib") - try: - rmtree(tempPath) - except (FileNotFoundError, NotADirectoryError): - pass - - os.makedirs(tempPath, exist_ok=True) + with TemporaryDirectory() as tempDir: + print(tempDir) + opener = build_opener() + opener.addheaders = [ + ("User-agent", "Mozilla/5.0 U GUYS SUCK WHY ARE YOU BLOCKING Python-urllib") + ] + install_opener(opener) - opener = build_opener() - opener.addheaders = [('User-agent', 'Mozilla/5.0 U GUYS SUCK WHY ARE YOU BLOCKING Python-urllib')] - install_opener(opener) + for url in URLLIST: + print(f"downloading {url}") + urlretrieve(url, Path(tempDir, Path(urlparse(url).path).name)) - for url in urlList: - print(f"downloading {url}") - urlretrieve(url, os.path.join(tempPath, os.path.basename(urlparse(url).path))) - - - try: - rmtree(targetPath) - except (FileNotFoundError, NotADirectoryError): - pass - - - copytree(tempPath, targetPath) - with open(os.path.join(targetPath, "VERSION"), "w") as f: - f.write(str(CURRENTVERSION)) - - - try: - rmtree(tempPath) - except (FileNotFoundError, NotADirectoryError): - pass - - return True + try: + rmtree(targetPath) + except (FileNotFoundError, NotADirectoryError): + pass + copytree(tempDir, targetPath) + with open(Path(targetPath, "VERSION"), "w") as f: + f.write(str(CURRENTVERSION)) + + if __name__ == "__main__": + input("Press Enter to continue...") + + return True - -if __name__ == '__main__': - update(True) \ No newline at end of file +if __name__ == "__main__": + update(True) diff --git a/updates.json b/updates.json index 297f4ce..82fe572 100644 --- a/updates.json +++ b/updates.json @@ -13,22 +13,52 @@ "2.4.0": "Fixed a bug in the image deduplicator", "2.4.1": "Added a simple gui with instructions", "2.4.2": "Fix for dual python installation", - "3.0.0": ["Tags can now be seen on the map!", "Linux support", "Switched to python 3"], + "3.0.0": [ + "Tags can now be seen on the map!", + "Linux support", + "Switched to python 3" + ], "3.0.1": "Added flags to set maximum threads used", "3.0.2": "Added flag to disable alt-mode", "3.1.0": "!Better implementation of tag image extraction", "3.1.1": "!Fixed bugs when no mod path is specified", "3.1.2": "!Fixed bug where the entire script-output folder is sometimes removed due to inconsistent symlink behavior", - "3.2.0": ["!Considerable speedup for most users that haven't bothered changing the factorio config.ini", "Loads of bugfixes"], + "3.2.0": [ + "!Considerable speedup for most users that haven't bothered changing the factorio config.ini", + "Loads of bugfixes" + ], "3.2.1": "Fixed various bugs", - "3.3.0": ["Updated for 0.17", "Some minor bugfixes"], + "3.3.0": [ + "Updated for 0.17", + "Some minor bugfixes" + ], "3.3.1": "Fixed a bug where some parts of the map could be missing if multiple factions were used", "3.3.2": "Many bugfixes.", - "3.4.0": ["Separated map tags to surface and snapshot level", "Thumbnail generation of the map"], - "3.5.0": ["!Added API for integration with i.a. Factorissimo", "Better dependency management", "Several bugfixes"], + "3.4.0": [ + "Separated map tags to surface and snapshot level", + "Thumbnail generation of the map" + ], + "3.5.0": [ + "!Added API for integration with i.a. Factorissimo", + "Better dependency management", + "Several bugfixes" + ], "3.5.1": "Removed dependency on package that isnt being updated anymore", "3.5.2": "Added support for unicode characters", "3.5.3": "Bugfixes", "3.5.4": "Extended API", - "3.5.5": "Updated for 0.18" + "3.5.5": "Updated for 0.18", + "4.0.0": [ + "!Fixed bug when dealing with near empty factorio config files", + "Better defaults for pvp scenario", + "Allow running for any working directory", + "Added wildcards for savefile names", + "Added option to output to any directory", + "Improved launching factorio trough steam", + "Better handling for modded ground tiles", + "Fixed nasty bug with image cross-referencing", + "Fixed scale of map label image sizes", + "Day and night snapshots are now completely identical instead of 1 tick apart", + "Added option to set the default snapshot when the page loads" + ] } \ No newline at end of file diff --git a/web/index.js b/web/index.js index 16a8a61..6af8996 100644 --- a/web/index.js +++ b/web/index.js @@ -420,12 +420,14 @@ if (countAvailableSaves > 0 || mapInfo.links && mapInfo.links.save) { const defaultSurface = mapInfo.defaultSurface || "nauvis"; let nightOpacity = 0; -const someSurfaces = mapInfo.maps[mapInfo.maps.length-1].surfaces; +const defaultMapPath = mapInfo.options.defaultTimestamp; +console.assert(0 <= defaultMapPath && defaultMapPath < mapInfo.maps.length, "Default map path is out of bounds."); +const someSurfaces = mapInfo.maps[defaultMapPath].surfaces; let currentSurface = defaultSurface in someSurfaces ? defaultSurface : Object.keys(someSurfaces).sort()[0] let loadLayer = someSurfaces[currentSurface].layers; let timestamp = (loadLayer.day || loadLayer.night).path; -let startZ = 16, startX = 0, startY = 0; +let startZ = 16, startX = NaN, startY = NaN; try { let split = window.location.hash.substr(1).split('/').map(decodeURIComponent); if (window.location.hash[0] == '#' && split[0] == "1") { @@ -445,6 +447,11 @@ try { window.location.href = "#"; window.location.reload(); } +if (isNaN(startX) || isNaN(startY)) { + let spawn = mapInfo.maps.find(m => m.path == timestamp).surfaces[currentSurface].spawn; + startX = -spawn.y / 2**(startZ-1); + startY = spawn.x / 2**(startZ-1); +} let lastHash = ""; diff --git a/zoom.py b/zoom.py index 47a2c58..42b9cdd 100644 --- a/zoom.py +++ b/zoom.py @@ -1,23 +1,25 @@ import json import math import multiprocessing as mp +from argparse import Namespace import os +from pathlib import Path import subprocess import sys import time -import numpy -from turbojpeg import TurboJPEG from shutil import get_terminal_size as tsize from sys import platform as _platform +import numpy import psutil from PIL import Image, ImageChops +from turbojpeg import TurboJPEG maxQuality = False # Set this to true if you want to compress/postprocess the images yourself later useBetterEncoder = True # Slower encoder that generates smaller images. quality = 80 - + EXT = ".png" OUTEXT = ".jpg" # format='JPEG' is hardcoded in places, meed to modify those, too. Most parameters are not supported outside jpeg. THUMBNAILEXT = ".png" @@ -28,7 +30,6 @@ MINRENDERBOXSIZE = 8 - def printErase(arg): try: tsiz = tsize()[0] @@ -40,63 +41,72 @@ def printErase(arg): # note that these are all 64 bit libraries since factorio doesnt support 32 bit. if os.name == "nt": - jpeg = TurboJPEG("mozjpeg/turbojpeg.dll") + jpeg = TurboJPEG(Path(__file__, "..", "mozjpeg/turbojpeg.dll").as_posix()) # elif _platform == "darwin": # I'm not actually sure if mac can run linux libraries or not. # jpeg = TurboJPEG("mozjpeg/libturbojpeg.dylib") # If anyone on mac has problems with the line below please make an issue :) else: - jpeg = TurboJPEG("mozjpeg/libturbojpeg.so") + jpeg = TurboJPEG(Path(__file__, "..", "mozjpeg/libturbojpeg.so").as_posix()) -def saveCompress(img, path, inpath=None): +def saveCompress(img, path: Path): if maxQuality: # do not waste any time compressing the image return img.save(path, subsampling=0, quality=100) - - out_file = open(path, 'wb') - out_file.write(jpeg.encode(numpy.array(img)[:, :, ::-1].copy() )) - out_file.close() + outFile = path.open("wb") + outFile.write(jpeg.encode(numpy.array(img)[:, :, ::-1].copy())) + outFile.close() + def simpleZoom(workQueue): for (folder, start, stop, filename) in workQueue: - path = os.path.join(folder, str(start), filename) - img = Image.open(path + EXT, mode='r').convert("RGB") + path = Path(folder, str(start), filename) + img = Image.open(path.with_suffix(EXT), mode="r").convert("RGB") if OUTEXT != EXT: - saveCompress(img, path + OUTEXT, path + EXT) - os.remove(path + EXT) + saveCompress(img, path.with_suffix(OUTEXT)) + path.with_suffix(EXT).unlink() for z in range(start - 1, stop - 1, -1): - if img.size[0] >= MINRENDERBOXSIZE*2 and img.size[1] >= MINRENDERBOXSIZE*2: - img = img.resize((img.size[0]//2, img.size[1]//2), Image.ANTIALIAS) - zFolder = os.path.join(folder, str(z)) - if not os.path.exists(zFolder): - os.mkdir(zFolder) - saveCompress(img, os.path.join(zFolder, filename + OUTEXT)) + if img.size[0] >= MINRENDERBOXSIZE * 2 and img.size[1] >= MINRENDERBOXSIZE * 2: + img = img.resize((img.size[0] // 2, img.size[1] // 2), Image.ANTIALIAS) + zFolder = Path(folder, str(z)) + if not zFolder.exists(): + zFolder.mkdir(parents=True) + saveCompress(img, Path(zFolder, filename).with_suffix(OUTEXT)) -def zoomRenderboxes(daytimeSurfaces, workfolder, timestamp, subpath, **kwargs): - with open(os.path.join(workfolder, "mapInfo.json"), 'r+') as mapInfoFile: +def zoomRenderboxes(daytimeSurfaces, toppath, timestamp, subpath, args): + with Path(toppath, "mapInfo.json").open("r+", encoding="utf-8") as mapInfoFile: mapInfo = json.load(mapInfoFile) - outFileExists = os.path.isfile(os.path.join(workfolder, "mapInfo.out.json")) - mapInfoOutFile = open(os.path.join(workfolder, "mapInfo.out.json"), 'r+') - if outFileExists: - outInfo = json.load(mapInfoOutFile) + outFile = Path(toppath, "mapInfo.out.json") + if outFile.exists(): + with outFile.open("r", encoding="utf-8") as mapInfoOutFile: + outInfo = json.load(mapInfoOutFile) else: - outInfo = { "maps": {} } + outInfo = {"maps": {}} + + mapLayer = None + mapIndex = None for i, m in enumerate(mapInfo["maps"]): if m["path"] == timestamp: mapLayer = m mapIndex = str(i) + if not mapLayer or not mapIndex: + raise Exception("mapLayer or mapIndex missing") + if mapIndex not in outInfo["maps"]: - outInfo["maps"][mapIndex] = { "surfaces": {} } + outInfo["maps"][mapIndex] = {"surfaces": {}} zoomWork = set() for daytime, activeSurfaces in daytimeSurfaces.items(): surfaceZoomLevels = {} for surfaceName in activeSurfaces: - surfaceZoomLevels[surfaceName] = mapLayer["surfaces"][surfaceName]["zoom"]["max"] - mapLayer["surfaces"][surfaceName]["zoom"]["min"] + surfaceZoomLevels[surfaceName] = ( + mapLayer["surfaces"][surfaceName]["zoom"]["max"] + - mapLayer["surfaces"][surfaceName]["zoom"]["min"] + ) for surfaceName, surface in mapLayer["surfaces"].items(): if "links" in surface: @@ -111,30 +121,41 @@ def zoomRenderboxes(daytimeSurfaces, workfolder, timestamp, subpath, **kwargs): totalZoomLevelsRequired = 0 for zoomSurface, zoomLevel in link["maxZoomFromSurfaces"].items(): if zoomSurface in surfaceZoomLevels: - totalZoomLevelsRequired = max(totalZoomLevelsRequired, zoomLevel + surfaceZoomLevels[zoomSurface]) + totalZoomLevelsRequired = max( + totalZoomLevelsRequired, + zoomLevel + surfaceZoomLevels[zoomSurface], + ) if not outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex]: outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex] = {} if "zoom" not in outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex]: outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex]["zoom"] = {} - link["zoom"]["min"] = link["zoom"]["max"] - totalZoomLevelsRequired outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex]["zoom"]["min"] = link["zoom"]["min"] - # an assumption is made that the total zoom levels required doesnt change between snapshots. if (link if "path" in link else outInfo["maps"][mapIndex]["surfaces"][surfaceName]["links"][linkIndex])["path"] == timestamp: - zoomWork.add((os.path.abspath(os.path.join(subpath, mapLayer["path"], link["toSurface"], daytime if link["daynight"] else "day", "renderboxes")), link["zoom"]["max"], link["zoom"]["min"], link["filename"])) - - - mapInfoOutFile.seek(0) - json.dump(outInfo, mapInfoOutFile) - mapInfoOutFile.truncate() - - - - maxthreads = int(kwargs["zoomthreads" if kwargs["zoomthreads"] else "maxthreads"]) + zoomWork.add( + ( + Path( + subpath, + mapLayer["path"], + link["toSurface"], + daytime if link["daynight"] else "day", + "renderboxes", + ).resolve(), + link["zoom"]["max"], + link["zoom"]["min"], + link["filename"], + ) + ) + + with outFile.open("w", encoding="utf-8") as mapInfoOutFile: + json.dump(outInfo, mapInfoOutFile) + mapInfoOutFile.truncate() + + maxthreads = args.zoomthreads if args.zoomthreads else args.maxthreads processes = [] zoomWork = list(zoomWork) for i in range(0, min(maxthreads, len(zoomWork))): @@ -143,73 +164,84 @@ def zoomRenderboxes(daytimeSurfaces, workfolder, timestamp, subpath, **kwargs): processes.append(p) for p in processes: p.join() - - - - - def work(basepath, pathList, surfaceName, daytime, size, start, stop, last, chunk, keepLast=False): - chunksize = 2**(start-stop) + chunksize = 2 ** (start - stop) if start > stop: for k in range(start, stop, -1): - x = chunksize*chunk[0] - y = chunksize*chunk[1] - for j in range(y, y + chunksize, 2): + x = chunksize * chunk[0] + y = chunksize * chunk[1] + for j in range(y, y + chunksize, 2): for i in range(x, x + chunksize, 2): - coords = [(0,0), (1,0), (0,1), (1,1)] - paths = [os.path.join(basepath, pathList[0], surfaceName, daytime, str(k), str(i+coord[0]), str(j+coord[1]) + EXT) for coord in coords] - - if any(os.path.isfile(path) for path in paths): - - if not os.path.exists(os.path.join(basepath, pathList[0], surfaceName, daytime, str(k-1), str(i//2))): + coords = [(0, 0), (1, 0), (0, 1), (1, 1)] + paths = [ + Path( + basepath, + pathList[0], + surfaceName, + daytime, + str(k), + str(i + coord[0]), + str(j + coord[1]), + ).with_suffix(EXT) + for coord in coords + ] + + if any(path.exists() for path in paths): + + if not Path(basepath, pathList[0], surfaceName, daytime, str(k - 1), str(i // 2)).exists(): try: - os.makedirs(os.path.join(basepath, pathList[0], surfaceName, daytime, str(k-1), str(i//2))) + Path(basepath, pathList[0], surfaceName, daytime, str(k - 1), str(i // 2)).mkdir(parents=True) except OSError: pass isOriginal = [] for m in range(len(coords)): - isOriginal.append(os.path.isfile(paths[m])) + isOriginal.append(paths[m].is_file()) if not isOriginal[m]: for n in range(1, len(pathList)): - paths[m] = os.path.join(basepath, pathList[n], surfaceName, daytime, str(k), str(i+coords[m][0]), str(j+coords[m][1]) + OUTEXT) - if os.path.isfile(paths[m]): + paths[m] = Path(basepath, pathList[n], surfaceName, daytime, str(k), str(i + coords[m][0]), str(j + coords[m][1])).with_suffix(OUTEXT) + if paths[m].is_file(): break + result = Image.new("RGB", (size, size), BACKGROUNDCOLOR) - result = Image.new('RGB', (size, size), BACKGROUNDCOLOR) - - imgs = [] + images = [] for m in range(len(coords)): - if (os.path.isfile(paths[m])): - img = Image.open(paths[m], mode='r').convert("RGB") - result.paste(box=(coords[m][0]*size//2, coords[m][1]*size//2), im=img.resize((size//2, size//2), Image.ANTIALIAS)) + if paths[m].is_file(): + img = Image.open(paths[m], mode="r").convert("RGB") + result.paste( + box=( + coords[m][0] * size // 2, + coords[m][1] * size // 2, + ), + im=img.resize( + (size // 2, size // 2), Image.ANTIALIAS + ), + ) if isOriginal[m]: - imgs.append((img, paths[m])) + images.append((img, paths[m])) + if k == last + 1: + saveCompress(result, Path(basepath, pathList[0], surfaceName, daytime, str(k - 1), str(i // 2), str(j // 2)).with_suffix(OUTEXT)) + if OUTEXT != EXT and (k != last + 1 or keepLast): + result.save(Path(basepath, pathList[0], surfaceName, daytime, str(k - 1), str(i // 2), str(j // 2), ).with_suffix(EXT)) - if k == last+1: - saveCompress(result, os.path.join(basepath, pathList[0], surfaceName, daytime, str(k-1), str(i//2), str(j//2) + OUTEXT)) - if OUTEXT != EXT and (k != last+1 or keepLast): - result.save(os.path.join(basepath, pathList[0], surfaceName, daytime, str(k-1), str(i//2), str(j//2) + EXT)) - if OUTEXT != EXT: - for img, path in imgs: - saveCompress(img, path.replace(EXT, OUTEXT), path) - os.remove(path) - + for img, path in images: + saveCompress(img, path.with_suffix(OUTEXT)) + path.unlink() chunksize = chunksize // 2 elif stop == last: - path = os.path.join(basepath, pathList[0], surfaceName, daytime, str(start), str(chunk[0]), str(chunk[1])) - img = Image.open(path + EXT, mode='r').convert("RGB") - saveCompress(img, path + OUTEXT, path + EXT) - os.remove(path + EXT) - + path = Path(basepath, pathList[0], surfaceName, daytime, str(start), str(chunk[0]), str(chunk[1])) + img = Image.open(path.with_suffix(EXT), mode="r").convert("RGB") + saveCompress(img, path.with_suffix(OUTEXT)) + path.with_suffix(EXT).unlink() + def thread(basepath, pathList, surfaceName, daytime, size, start, stop, last, allChunks, counter, resultQueue, keepLast=False): #print(start, stop, chunks) @@ -222,77 +254,83 @@ def thread(basepath, pathList, surfaceName, daytime, size, start, stop, last, al chunk = allChunks[i] work(basepath, pathList, surfaceName, daytime, size, start, stop, last, chunk, keepLast) resultQueue.put(True) - - - - - - +def zoom( + outFolder: Path, + timestamp: str = None, + surfaceReference: str = None, + daytimeReference: str = None, + basepath: Path = None, + needsThumbnail: bool = True, + args: Namespace = Namespace(), +): -def zoom(*args, **kwargs): + psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10) + workFolder = basepath if basepath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps") - psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) + topPath = Path(workFolder, outFolder) + dataPath = Path(topPath, "mapInfo.json") + imagePath = Path(topPath, "Images") + maxthreads = args.zoomthreads if args.zoomthreads else args.maxthreads - - needsThumbnail = (str(args[5]).lower() != "false") if len(args) > 5 else True - toppath = os.path.join((args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) - datapath = os.path.join(toppath, "mapInfo.json") - basepath = os.path.join(toppath, "Images") - maxthreads = int(kwargs["zoomthreads" if kwargs["zoomthreads"] else "maxthreads"]) - - - #print(basepath) - - - with open(datapath, "r", encoding="utf-8") as f: + with dataPath.open("r", encoding="utf-8") as f: data = json.load(f) for mapIndex, map in enumerate(data["maps"]): - if len(args) <= 1 or map["path"] == args[1]: + if timestamp is None or map["path"] == timestamp: for surfaceName, surface in map["surfaces"].items(): - if len(args) <= 2 or surfaceName == args[2]: + if surfaceReference is None or surfaceName == surfaceReference: maxzoom = surface["zoom"]["max"] minzoom = surface["zoom"]["min"] daytimes = [] - try: - if surface["day"]: daytimes.append("day") - except KeyError: pass - try: - if surface["night"]: daytimes.append("night") - except KeyError: pass + if "day" in surface: + daytimes.append("day") + if "night" in surface: + daytimes.append("night") for daytime in daytimes: - if len(args) <= 3 or daytime == args[3]: - if not os.path.isdir(os.path.join(toppath, "Images", str(map["path"]), surfaceName, daytime, str(maxzoom - 1))): - - print("zoom {:5.1f}% [{}]".format(0, " " * (tsize()[0]-15)), end="") - - generateThumbnail = needsThumbnail \ - and mapIndex == len(data["maps"]) - 1 \ - and surfaceName == ("nauvis" if "nauvis" in map["surfaces"] else sorted(map["surfaces"].keys())[0]) \ - and daytime == daytimes[0] + if daytimeReference is None or daytime == daytimeReference: + if not Path(topPath, "Images", str(map["path"]), surfaceName, daytime, str(maxzoom - 1)).is_dir(): + + print(f"zoom {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="") + + generateThumbnail = ( + needsThumbnail + and mapIndex == len(data["maps"]) - 1 + and surfaceName + == ( + "nauvis" + if "nauvis" in map["surfaces"] + else sorted(map["surfaces"].keys())[0] + ) + and daytime == daytimes[0] + ) allBigChunks = {} minX = float("inf") maxX = float("-inf") minY = float("inf") maxY = float("-inf") - imageSize = None - for xStr in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(maxzoom))): - x = int(xStr) + imageSize: int = None + for xStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom)).iterdir(): + x = int(xStr.name) minX = min(minX, x) maxX = max(maxX, x) - for yStr in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr)): + for yStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr).iterdir(): if imageSize is None: - imageSize = Image.open(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr, yStr), mode='r').size[0] - y = int(yStr.split('.', 2)[0]) + imageSize = Image.open(Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr, yStr), mode="r").size[0] + y = int(yStr.stem) minY = min(minY, y) maxY = max(maxY, y) - allBigChunks[(x >> maxzoom - minzoom, y >> maxzoom - minzoom)] = True + allBigChunks[ + ( + x >> maxzoom-minzoom, + y >> maxzoom-minzoom, + ) + ] = True if len(allBigChunks) <= 0: continue @@ -309,84 +347,126 @@ def zoom(*args, **kwargs): for pos in list(allBigChunks): for i in range(2**threadsplit): for j in range(2**threadsplit): - allChunks.append((pos[0]*(2**threadsplit) + i, pos[1]*(2**threadsplit) + j)) + allChunks.append( + ( + pos[0] * (2**threadsplit) + i, + pos[1] * (2**threadsplit) + j, + ) + ) threads = min(len(allChunks), maxthreads) processes = [] originalSize = len(allChunks) - + # print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList))) # print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks)))) - counter = mp.Value('i', originalSize) + counter = mp.Value("i", originalSize) resultQueue = mp.Queue() for _ in range(0, threads): - p = mp.Process(target=thread, args=(basepath, pathList, surfaceName, daytime, imageSize, maxzoom, minzoom + threadsplit, minzoom, allChunks, counter, resultQueue, generateThumbnail)) + p = mp.Process( + target=thread, + args=( + imagePath, + pathList, + surfaceName, + daytime, + imageSize, + maxzoom, + minzoom + threadsplit, + minzoom, + allChunks, + counter, + resultQueue, + generateThumbnail, + ), + ) p.start() processes.append(p) - + doneSize = 0 for _ in range(originalSize): resultQueue.get(True) doneSize += 1 progress = float(doneSize) / originalSize - tsiz = tsize()[0]-15 - print("\rzoom {:5.1f}% [{}{}]".format(round(progress * 98, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") + tsiz = tsize()[0] - 15 + print( + "\rzoom {:5.1f}% [{}{}]".format( + round(progress * 98, 1), + "=" * int(progress * tsiz), + " " * (tsiz - int(progress * tsiz)), + ), + end="", + ) for p in processes: p.join() - - - if threadsplit > 0: - #print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks)))) + # print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks)))) processes = [] i = len(allBigChunks) - 1 for chunk in list(allBigChunks): - p = mp.Process(target=work, args=(basepath, pathList, surfaceName, daytime, imageSize, minzoom + threadsplit, minzoom, minzoom, chunk, generateThumbnail)) + p = mp.Process( + target=work, + args=( + imagePath, + pathList, + surfaceName, + daytime, + imageSize, + minzoom + threadsplit, + minzoom, + minzoom, + chunk, + generateThumbnail, + ), + ) i = i - 1 p.start() processes.append(p) for p in processes: p.join() - if generateThumbnail: printErase("generating thumbnail") - minzoompath = os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(minzoom)) - - - thumbnail = Image.new('RGB', ((maxX-minX+1) * imageSize >> maxzoom-minzoom, (maxY-minY+1) * imageSize >> maxzoom-minzoom), BACKGROUNDCOLOR) + minzoompath = Path( + imagePath, + str(map["path"]), + surfaceName, + daytime, + str(minzoom), + ) + + if imageSize is None: + raise Exception("Missing imageSize for thumbnail generation") + + thumbnail = Image.new( + "RGB", + ( + (maxX - minX + 1) * imageSize >> maxzoom-minzoom, + (maxY - minY + 1) * imageSize >> maxzoom-minzoom, + ), + BACKGROUNDCOLOR, + ) bigMinX = minX >> maxzoom-minzoom bigMinY = minY >> maxzoom-minzoom xOffset = ((bigMinX * imageSize << maxzoom-minzoom) - minX * imageSize) >> maxzoom-minzoom yOffset = ((bigMinY * imageSize << maxzoom-minzoom) - minY * imageSize) >> maxzoom-minzoom for chunk in list(allBigChunks): - path = os.path.join(minzoompath, str(chunk[0]), str(chunk[1]) + EXT) - thumbnail.paste(box=(xOffset+(chunk[0]-bigMinX)*imageSize, yOffset+(chunk[1]-bigMinY)*imageSize), im=Image.open(path, mode='r').convert("RGB").resize((imageSize, imageSize), Image.ANTIALIAS)) + path = Path(minzoompath, str(chunk[0]), str(chunk[1])).with_suffix(EXT) + thumbnail.paste( + box=( + xOffset + (chunk[0] - bigMinX) * imageSize, + yOffset + (chunk[1] - bigMinY) * imageSize, + ), + im=Image.open(path, mode="r") + .convert("RGB") + .resize((imageSize, imageSize), Image.ANTIALIAS), + ) if OUTEXT != EXT: - os.remove(path) - - thumbnail.save(os.path.join(basepath, "thumbnail" + THUMBNAILEXT)) - - - - - print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0]-15))) - - - - - - - - - - - - + path.unlink() + thumbnail.save(Path(imagePath, "thumbnail" + THUMBNAILEXT)) -if __name__ == '__main__': - zoom(*sys.argv[1:]) + print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))