diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..c942cb5f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,59 @@ +name: Build + +on: + push: + pull_request: + +jobs: + build: + container: ghcr.io/zeldaret/oot-gc-vc-build:main + runs-on: ubuntu-latest + + steps: + # Checkout the repository (shallow clone) + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + # Set Git config + - name: Git config + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + # Copy the original files to the workspace + - name: Prepare + run: cp -R /orig . + + # Build the project + - name: Build + run: | + python configure.py --map --binutils /binutils --compilers /compilers + ninja all_source build/{mq-j,mq-u,mq-e,ce-j,ce-u,ce-e}/progress.json build/report.json + + # Upload progress if we're on the main branch + - name: Upload progress + if: github.ref == 'refs/heads/main' + continue-on-error: true + env: + PROGRESS_SLUG: oot-vc + PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} + run: | + for version in {mq-j,mq-u,mq-e,ce-j,ce-u,ce-e}; do + python tools/upload_progress.py -b https://progress.decomp.club/ \ + -p $PROGRESS_SLUG -v $version build/$version/progress.json + done + + # Upload map files + - name: Upload map + uses: actions/upload-artifact@v4 + with: + name: combined_maps + path: build/**/*.MAP + + # Upload progress report + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: combined_report + path: build/report.json diff --git a/configure.py b/configure.py index 3d3a3a2e..9b9a6544 100755 --- a/configure.py +++ b/configure.py @@ -83,6 +83,12 @@ type=Path, help="path to decomp-toolkit binary or source (optional)", ) +parser.add_argument( + "--objdiff", + metavar="BINARY | DIR", + type=Path, + help="path to objdiff-cli binary or source (optional)", +) parser.add_argument( "--sjiswrap", metavar="EXE", @@ -124,6 +130,7 @@ config.build_dir = args.build_dir config.dtk_path = args.dtk +config.objdiff_path = args.objdiff config.binutils_path = args.binutils config.compilers_path = args.compilers config.generate_map = args.map @@ -141,6 +148,7 @@ config.binutils_tag = "2.42-1" config.compilers_tag = "20231018" config.dtk_tag = "v0.8.3" +config.objdiff_tag = "v2.0.0-beta.5" config.sjiswrap_tag = "v1.1.1" config.wibo_tag = "0.6.11" config.linker_version = "GC/1.1" @@ -214,7 +222,7 @@ def GenericLib(lib_name: str, cflags: List[str], objects: List[Object]) -> Dict[ ### Link order # Not matching for any version -NonMatching = {} +NonMatching: List[str] = [] # Matching for all versions Matching = config.versions diff --git a/src/emulator/mcardGCN.c b/src/emulator/mcardGCN.c index 66778e0a..e4777af1 100644 --- a/src/emulator/mcardGCN.c +++ b/src/emulator/mcardGCN.c @@ -426,7 +426,7 @@ static bool mcardPoll(MemCard* pMCard) { } if (pMCard->pPollFunction != NULL) { - if (!simulatorTestReset(false, false, false, false)) { + if (!SIMULATOR_TEST_RESET(false, false, false, false)) { return false; } else { pMCard->pPollFunction(); @@ -1375,7 +1375,7 @@ bool mcardUpdate(void) { prevIndex = 100; counter = 0; while (true) { - if (!simulatorTestReset(false, false, false, false)) { + if (!SIMULATOR_TEST_RESET(false, false, false, false)) { return false; } @@ -1428,14 +1428,14 @@ bool mcardUpdate(void) { } if (toggle != true && mCard.writeStatus == 0 || mCard.saveToggle != true) { - if (!simulatorTestReset(false, false, true, false)) { + if (!SIMULATOR_TEST_RESET(false, false, true, false)) { return false; } if (gpSystem->eTypeROM == SRT_ZELDA1 && mCard.saveToggle == true) { do { mcardMenu(&mCard, MC_M_SV12, &command); - if (!simulatorTestReset(false, false, true, false)) { + if (!SIMULATOR_TEST_RESET(false, false, true, false)) { return false; } } while (mCard.wait == true); diff --git a/tools/download_tool.py b/tools/download_tool.py index 594f6476..37d61679 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -55,6 +55,21 @@ def dtk_url(tag: str) -> str: repo = "https://github.com/encounter/decomp-toolkit" return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" +def objdiff_cli_url(tag: str) -> str: + uname = platform.uname() + suffix = "" + system = uname.system.lower() + if system == "darwin": + system = "macos" + elif system == "windows": + suffix = ".exe" + arch = uname.machine.lower() + if arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/objdiff" + return f"{repo}/releases/download/{tag}/objdiff-cli-{system}-{arch}{suffix}" + def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" @@ -70,6 +85,7 @@ def wibo_url(tag: str) -> str: "binutils": binutils_url, "compilers": compilers_url, "dtk": dtk_url, + "objdiff-cli": objdiff_cli_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, } @@ -112,6 +128,6 @@ def main() -> None: data = data.replace(b"\xb9\x41\0\0\0\xf3\xa5\x8d\x44\x24\x04", b"\xb9\x51\0\0\0\xf3\xa5\x8d\x44\x24\x04") with open(compiler_path, "wb") as outfile: outfile.write(data) - + if __name__ == "__main__": main() diff --git a/tools/project.py b/tools/project.py index 457298fe..2f0cde32 100644 --- a/tools/project.py +++ b/tools/project.py @@ -28,31 +28,85 @@ f"\n(Current path: {sys.executable})" ) +Library = Dict[str, Any] + class Object: def __init__( self, completed_versions: List[str], name: str, **options: Any ) -> None: self.name = name - self.base_name = Path(name).with_suffix("") self.completed_versions = completed_versions self.options: Dict[str, Any] = { "add_to_all": True, "asflags": None, - "extra_asflags": None, + "asm_dir": None, + "asm_processor": False, "cflags": None, + "extra_asflags": None, "extra_cflags": None, + "host": None, + "lib": None, "mw_version": None, + "progress_category": None, "shift_jis": None, "source": name, - "asm_processor": False, + "src_dir": None, } self.options.update(options) + # Internal + self.src_path: Optional[Path] = None + self.asm_path: Optional[Path] = None + self.src_obj_path: Optional[Path] = None + self.asm_obj_path: Optional[Path] = None + self.host_obj_path: Optional[Path] = None + self.ctx_path: Optional[Path] = None + + def resolve(self, config: "ProjectConfig", lib: Library, version: str) -> "Object": + # Use object options, then library options + obj = Object(self.completed_versions, self.name, **lib) + for key, value in self.options.items(): + if value is not None or key not in obj.options: + obj.options[key] = value + + # Use default options from config + def set_default(key: str, value: Any) -> None: + if obj.options[key] is None: + obj.options[key] = value + + set_default("add_to_all", True) + set_default("asflags", config.asflags) + set_default("asm_dir", config.asm_dir) + set_default("host", False) + set_default("mw_version", config.linker_version) + set_default("shift_jis", config.shift_jis) + set_default("src_dir", config.src_dir) + + # Resolve paths + build_dir = config.out_path(version) + obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"] + if obj.options["asm_dir"] is not None: + obj.asm_path = ( + Path(obj.options["asm_dir"]) / obj.options["source"] + ).with_suffix(".s") + base_name = Path(self.name).with_suffix("") + obj.src_obj_path = build_dir / "src" / f"{base_name}.o" + obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o" + obj.host_obj_path = build_dir / "host" / f"{base_name}.o" + obj.ctx_path = build_dir / "src" / f"{base_name}.ctx" + return obj + def completed(self, version: str) -> bool: return version in self.completed_versions +class ProgressCategory: + def __init__(self, id: str, name: str) -> None: + self.id = id + self.name = name + + class ProjectConfig: def __init__(self) -> None: # Paths @@ -74,6 +128,8 @@ def __init__(self) -> None: self.wrapper: Optional[Path] = None # If None, download wibo on Linux self.sjiswrap_tag: Optional[str] = None # Git tag self.sjiswrap_path: Optional[Path] = None # If None, download + self.objdiff_tag: Optional[str] = None # Git tag + self.objdiff_path: Optional[Path] = None # If None, download # Project config self.build_rels: bool = True # Build REL files @@ -101,8 +157,9 @@ def __init__(self) -> None: self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( - True # Include individual modules, disable for large numbers of modules + False # Include individual modules, disable for large numbers of modules ) + self.progress_categories: List[ProgressCategory] = [] # Additional categories # Progress fancy printing self.progress_use_fancy: bool = False @@ -126,12 +183,17 @@ def validate(self) -> None: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + # Creates a map of object names to Object instances + # Options are fully resolved from the library and object + def objects(self, version: str) -> Dict[str, Object]: + out = {} for lib in self.libs or {}: - for obj in lib["objects"]: - if obj.name == name: - return lib, obj - return None + objects: List[Object] = lib["objects"] + for obj in objects: + if obj.name in out: + sys.exit(f"Duplicate object name {obj.name}") + out[obj.name] = obj.resolve(self, lib, version) + return out def check_sha_path(self, version: str) -> Path: return self.config_dir / version / "build.sha1" @@ -192,29 +254,32 @@ def versiontuple(v: str) -> Tuple[int, ...]: # Generate build.ninja and objdiff.json def generate_build(config: ProjectConfig) -> None: + config.validate() rebuild_configs = False + version_objects = {} build_configs = {} for version in config.versions: + objects = config.objects(version) build_config_path = config.out_path(version) / "config.json" build_config = load_build_config(config, build_config_path) if build_config is None: rebuild_configs = True else: + version_objects[version] = objects build_configs[version] = build_config - generate_build_ninja(config, build_configs, rebuild_configs) - generate_objdiff_config(config, build_configs) + generate_build_ninja(config, version_objects, build_configs, rebuild_configs) + generate_objdiff_config(config, version_objects, build_configs) # Generate build.ninja def generate_build_ninja( config: ProjectConfig, + version_objects: Dict[str, Dict[str, Object]], build_configs: Dict[str, Dict[str, Any]], rebuild_configs: bool, ) -> None: - config.validate() - out = io.StringIO() n = ninja_syntax.Writer(out) n.variable("ninja_required_version", "1.3") @@ -265,17 +330,27 @@ def generate_build_ninja( deps="gcc", ) + cargo_rule_written = False + + def write_cargo_rule(): + nonlocal cargo_rule_written + if not cargo_rule_written: + n.pool("cargo", 1) + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + pool="cargo", + depfile=Path("$target") / "release" / "$bin.d", + deps="gcc", + ) + cargo_rule_written = True + if config.dtk_path is not None and config.dtk_path.is_file(): dtk = config.dtk_path elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" - n.rule( - name="cargo", - command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", - description="CARGO $bin", - depfile=Path("$target") / "release" / "$bin.d", - deps="gcc", - ) + write_cargo_rule() n.build( outputs=dtk, rule="cargo", @@ -300,6 +375,35 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.dtk_tag missing") + if config.objdiff_path is not None and config.objdiff_path.is_file(): + objdiff = config.objdiff_path + elif config.objdiff_path is not None: + objdiff = build_tools_path / "release" / f"objdiff-cli{EXE}" + write_cargo_rule() + n.build( + outputs=objdiff, + rule="cargo", + inputs=config.objdiff_path / "Cargo.toml", + implicit=config.objdiff_path / "Cargo.lock", + variables={ + "bin": "objdiff-cli", + "target": build_tools_path, + }, + ) + elif config.objdiff_tag: + objdiff = build_tools_path / f"objdiff-cli{EXE}" + n.build( + outputs=objdiff, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "objdiff-cli", + "tag": config.objdiff_tag, + }, + ) + else: + sys.exit("ProjectConfig.objdiff_tag missing") + if config.sjiswrap_path: sjiswrap = config.sjiswrap_path elif config.sjiswrap_tag: @@ -378,6 +482,17 @@ def generate_build_ninja( n.newline() + ### + # Helper rule for downloading all tools + ### + n.comment("Download all tools") + n.build( + outputs="tools", + rule="phony", + inputs=[dtk, sjiswrap, wrapper, compilers, binutils, objdiff], + ) + n.newline() + ### # Build rules ### @@ -416,14 +531,19 @@ def generate_build_ninja( # MWCC with asm_processor and UTF-8 to Shift JIS wrapper mwcc_asm_processor_sjis_cmd = f'tools/asm_processor/compile.sh "{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD" "{gnu_as} $asflags" $in $out' - mwcc_asm_processor_sjis_implicit: List[Optional[Path]] = [*mwcc_asm_processor_implicit, sjiswrap] + mwcc_asm_processor_sjis_implicit: List[Optional[Path]] = [ + *mwcc_asm_processor_implicit, + sjiswrap, + ] if os.name != "nt": transform_dep = config.tools_dir / "transform_dep.py" mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" mwcc_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" mwcc_asm_processor_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" - mwcc_asm_processor_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_asm_processor_sjis_cmd += ( + f" && $python {transform_dep} $basefile.d $basefile.d" + ) mwcc_implicit.append(transform_dep) mwcc_sjis_implicit.append(transform_dep) mwcc_asm_processor_implicit.append(transform_dep) @@ -526,6 +646,14 @@ def generate_build_ninja( ) n.newline() + n.comment("Report") + n.rule( + name="report", + command=f"{objdiff} report generate -o $out", + description="REPORT", + ) + n.newline() + ### # Source files ### @@ -618,125 +746,124 @@ def write(self, n: ninja_syntax.Writer) -> None: ) n.newline() + report_path = config.build_dir / "report.json" + source_inputs: List[Path] = [] + for version, build_config in build_configs.items(): build_path = config.out_path(version) progress_path = build_path / "progress.json" - build_asm_path = build_path / "mod" - build_src_path = build_path / "src" - build_host_path = build_path / "host" link_steps: List[LinkStep] = [] used_compiler_versions: Set[str] = set() - source_inputs: List[Path] = [] host_source_inputs: List[Path] = [] source_added: Set[Path] = set() - def c_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path - ) -> Optional[Path]: - asflags_str = make_flags_str(config.asflags or []) - cflags_str = make_flags_str(options["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_flags_str(options["extra_cflags"]) + def c_build(obj: Object, src_path: Path) -> Optional[Path]: + asflags_str = make_flags_str(obj.options["asflags"]) + cflags_str = make_flags_str(obj.options["cflags"]) + if obj.options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) cflags_str += " " + extra_cflags_str cflags_str += f" -i build/{version}/include -DVERSION={version.upper().replace('-', '_')}" - used_compiler_versions.add(options["mw_version"]) - - src_obj_path = build_src_path / f"{obj.base_name}.o" - src_base_path = build_src_path / obj.base_name + used_compiler_versions.add(obj.options["mw_version"]) # Avoid creating duplicate build rules - if src_obj_path in source_added: - return src_obj_path - source_added.add(src_obj_path) - - shift_jis = options["shift_jis"] - if shift_jis is None: - shift_jis = config.shift_jis + if obj.src_obj_path is None or obj.src_obj_path in source_added: + return obj.src_obj_path + source_added.add(obj.src_obj_path) # Add MWCC build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed(version)})") - if options["asm_processor"]: + if obj.options["asm_processor"]: n.build( - outputs=src_obj_path, - rule="mwcc_asm_processor_sjis" if shift_jis else "mwcc_asm_processor", + outputs=obj.src_obj_path, + rule=( + "mwcc_asm_processor_sjis" + if obj.options["shift_jis"] + else "mwcc_asm_processor" + ), inputs=src_path, variables={ - "mw_version": Path(options["mw_version"]), + "mw_version": Path(obj.options["mw_version"]), "asflags": asflags_str, "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": src_base_path, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), }, - implicit=mwcc_asm_processor_sjis_implicit if shift_jis else mwcc_asm_processor_implicit, + implicit=( + mwcc_asm_processor_sjis_implicit + if obj.options["shift_jis"] + else mwcc_asm_processor_implicit + ), ) else: n.build( - outputs=src_obj_path, - rule="mwcc_sjis" if shift_jis else "mwcc", + outputs=obj.src_obj_path, + rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc", inputs=src_path, variables={ - "mw_version": Path(options["mw_version"]), + "mw_version": Path(obj.options["mw_version"]), "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": src_base_path, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), }, - implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + implicit=( + mwcc_sjis_implicit + if obj.options["shift_jis"] + else mwcc_implicit + ), ) # Add ctx build rule - ctx_path = build_src_path / f"{obj.base_name}.ctx" - n.build( - outputs=ctx_path, - rule="decompctx", - inputs=src_path, - implicit=decompctx, - ) + if obj.ctx_path is not None: + n.build( + outputs=obj.ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) # Add host build rule - if options.get("host", False): - host_obj_path = build_host_path / f"{obj.base_name}.o" - host_base_path = build_host_path / obj.base_name + if obj.options["host"] and obj.host_obj_path is not None: n.build( - outputs=host_obj_path, + outputs=obj.host_obj_path, rule="host_cc" if src_path.suffix == ".c" else "host_cpp", inputs=src_path, variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": host_base_path, + "basedir": os.path.dirname(obj.host_obj_path), + "basefile": obj.host_obj_path.with_suffix(""), }, ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) + if obj.options["add_to_all"]: + host_source_inputs.append(obj.host_obj_path) n.newline() - if options["add_to_all"]: - source_inputs.append(src_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj.src_obj_path) - return src_obj_path + return obj.src_obj_path def asm_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + obj: Object, src_path: Path, obj_path: Optional[Path] ) -> Optional[Path]: - asflags = options["asflags"] or config.asflags - if asflags is None: + if obj.options["asflags"] is None: sys.exit("ProjectConfig.asflags missing") - asflags_str = make_flags_str(asflags) - if options["extra_asflags"] is not None: - extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str = make_flags_str(obj.options["asflags"]) + if obj.options["extra_asflags"] is not None: + extra_asflags_str = make_flags_str(obj.options["extra_asflags"]) asflags_str += " " + extra_asflags_str - asm_obj_path = build_asm_path / f"{obj.base_name}.o" - # Avoid creating duplicate build rules - if asm_obj_path in source_added: - return asm_obj_path - source_added.add(asm_obj_path) + if obj_path is None or obj_path in source_added: + return obj_path + source_added.add(obj_path) # Add assembler build rule - n.comment(f"{obj.name}: {lib_name} (linked {obj.completed(version)})") + lib_name = obj.options["lib"] + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=asm_obj_path, + outputs=obj_path, rule="as", inputs=src_path, variables={"asflags": asflags_str}, @@ -744,57 +871,40 @@ def asm_build( ) n.newline() - if options["add_to_all"]: - source_inputs.append(asm_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj_path) - return asm_obj_path + return obj_path def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] - result = config.find_object(obj_name) - if not result: + obj = version_objects[version].get(obj_name) + if obj is None: if config.warn_missing_config and not build_obj["autogenerated"]: print(f"Missing configuration for {obj_name}") link_step.add(obj_path) return - lib, obj = result - lib_name = lib["lib"] - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - - unit_asm_path: Optional[Path] = None - if config.asm_dir is not None: - unit_asm_path = ( - Path(lib.get("asm_dir", config.asm_dir)) / options["source"] - ).with_suffix(".s") - - built_obj_path: Optional[Path] = None link_built_obj = obj.completed(version) - if unit_src_path.exists(): - if unit_src_path.suffix in (".c", ".cp", ".cpp"): + built_obj_path: Optional[Path] = None + if obj.src_path is not None and obj.src_path.exists(): + if obj.src_path.suffix in (".c", ".cp", ".cpp"): # Add MWCC & host build rules - built_obj_path = c_build(obj, options, lib_name, unit_src_path) - elif unit_src_path.suffix == ".s": + built_obj_path = c_build(obj, obj.src_path) + elif obj.src_path.suffix == ".s": # Add assembler build rule - built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + built_obj_path = asm_build(obj, obj.src_path, obj.src_obj_path) else: - sys.exit(f"Unknown source file type {unit_src_path}") + sys.exit(f"Unknown source file type {obj.src_path}") else: if config.warn_missing_source or obj.completed(version): - print(f"Missing source file {unit_src_path}") + print(f"Missing source file {obj.src_path}") link_built_obj = False # Assembly overrides - if unit_asm_path is not None and unit_asm_path.exists(): + if obj.asm_path is not None and obj.asm_path.exists(): link_built_obj = True - built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path) if link_built_obj and built_obj_path is not None: # Use the source-built object @@ -803,7 +913,10 @@ def add_unit(build_obj, link_step: LinkStep): # Use the original (extracted) object link_step.add(obj_path) else: - sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") + lib_name = obj.options["lib"] + sys.exit( + f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}" + ) # Add DOL link step link_step = LinkStep(build_config) @@ -819,7 +932,7 @@ def add_unit(build_obj, link_step: LinkStep): add_unit(unit, module_link_step) # Add empty object to empty RELs if len(module_link_step.inputs) == 0: - if not config.rel_empty_file: + if config.rel_empty_file is None: sys.exit("ProjectConfig.rel_empty_file missing") add_unit( { @@ -922,6 +1035,29 @@ def add_unit(build_obj, link_step: LinkStep): ) n.newline() + ### + # Helper rule for building all source files + ### + n.comment("Build all source files") + n.build( + outputs="all_source", + rule="phony", + inputs=source_inputs, + ) + n.newline() + + ### + # Generate progress report + ### + n.comment("Generate progress report") + report_implicit: List[str | Path] = [objdiff, "all_source"] + n.build( + outputs=report_path, + rule="report", + implicit=report_implicit, + ) + n.newline() + ### # Regenerate on change ### @@ -969,10 +1105,13 @@ def add_unit(build_obj, link_step: LinkStep): # Generate objdiff.json def generate_objdiff_config( - config: ProjectConfig, build_configs: Dict[str, Dict[str, Any]] + config: ProjectConfig, + version_objects: Dict[str, Dict[str, Object]], + build_configs: Dict[str, Dict[str, Any]], ) -> None: objdiff_config: Dict[str, Any] = { - "min_version": "1.0.0", + "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json", + "min_version": "2.0.0-alpha.5", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -988,6 +1127,7 @@ def generate_objdiff_config( "*.json", ], "units": [], + "progress_categories": [], } # decomp.me compiler name mapping @@ -998,67 +1138,48 @@ def generate_objdiff_config( "GC/1.2.5": "mwcc_233_163", "GC/1.2.5e": "mwcc_233_163e", "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3": "mwcc_242_53", "GC/1.3.2": "mwcc_242_81", "GC/1.3.2r": "mwcc_242_81r", "GC/2.0": "mwcc_247_92", "GC/2.5": "mwcc_247_105", "GC/2.6": "mwcc_247_107", "GC/2.7": "mwcc_247_108", - "GC/3.0": "mwcc_41_60831", - # "GC/3.0a3": "mwcc_41_51213", + "GC/3.0a3": "mwcc_41_51213", "GC/3.0a3.2": "mwcc_41_60126", - # "GC/3.0a3.3": "mwcc_41_60209", - # "GC/3.0a3.4": "mwcc_42_60308", - # "GC/3.0a5": "mwcc_42_60422", + "GC/3.0a3.3": "mwcc_41_60209", + "GC/3.0a3.4": "mwcc_42_60308", + "GC/3.0a5": "mwcc_42_60422", "GC/3.0a5.2": "mwcc_41_60831", + "GC/3.0": "mwcc_41_60831", + "Wii/1.0RC1": "mwcc_42_140", "Wii/0x4201_127": "mwcc_42_142", - # "Wii/1.0": "mwcc_43_145", - # "Wii/1.0RC1": "mwcc_42_140", "Wii/1.0a": "mwcc_42_142", + "Wii/1.0": "mwcc_43_145", "Wii/1.1": "mwcc_43_151", "Wii/1.3": "mwcc_43_172", - # "Wii/1.5": "mwcc_43_188", + "Wii/1.5": "mwcc_43_188", "Wii/1.6": "mwcc_43_202", "Wii/1.7": "mwcc_43_213", } - def add_unit(build_obj: Dict[str, Any], version: str, module_name: str) -> None: - if build_obj["autogenerated"]: - # Skip autogenerated objects - return - - build_path = config.out_path(version) + def add_unit(build_obj: Dict[str, Any], version: str, progress_categories: List[str]) -> None: obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") unit_config: Dict[str, Any] = { "name": Path(version) / base_object, "target_path": obj_path, + "metadata": { + "auto_generated": build_obj["autogenerated"], + }, } - result = config.find_object(obj_name) - if not result: + obj = version_objects[version].get(obj_name) + if obj is None or not obj.src_path or not obj.src_path.exists(): objdiff_config["units"].append(unit_config) return - lib, obj = result - src_dir = Path(lib.get("src_dir", config.src_dir)) - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = src_dir / str(options["source"]) - - if not unit_src_path.exists(): - objdiff_config["units"].append(unit_config) - return - - cflags = options["cflags"] - src_obj_path = build_path / "src" / f"{obj.base_name}.o" - src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" - + cflags = obj.options["cflags"] reverse_fn_order = False if type(cflags) is list: for flag in cflags: @@ -1077,36 +1198,79 @@ def keep_flag(flag): cflags = list(filter(keep_flag, cflags)) # Add appropriate lang flag - if unit_src_path.suffix in (".cp", ".cpp"): + if obj.src_path.suffix in (".cp", ".cpp"): cflags.insert(0, "-lang=c++") else: cflags.insert(0, "-lang=c") - unit_config["base_path"] = src_obj_path - unit_config["reverse_fn_order"] = reverse_fn_order - unit_config["complete"] = obj.completed(version) - compiler_version = COMPILER_MAP.get(options["mw_version"]) + unit_config["base_path"] = obj.src_obj_path + compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: - print(f"Missing scratch compiler mapping for {options['mw_version']}") + print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: + cflags_str = make_flags_str(cflags) + if obj.options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) + cflags_str += " " + extra_cflags_str unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, - "c_flags": make_flags_str(cflags), - "ctx_path": src_ctx_path, + "c_flags": cflags_str, + "ctx_path": obj.ctx_path, "build_ctx": True, } + category_opt: List[str] | str = obj.options["progress_category"] + if isinstance(category_opt, list): + progress_categories.extend(map(lambda x: f"{version}.{x}", category_opt)) + elif category_opt is not None: + progress_categories.append(f"{version}.{category_opt}") + unit_config["metadata"].update({ + "complete": obj.completed(version), + "reverse_fn_order": reverse_fn_order, + "source_path": obj.src_path, + "progress_categories": progress_categories, + }) objdiff_config["units"].append(unit_config) for version, build_config in build_configs.items(): # Add DOL units for unit in build_config["units"]: - add_unit(unit, version, build_config["name"]) + progress_categories = [version] + # Only include a "dol" category if there are any modules + # Otherwise it's redundant with the global report measures + if len(build_config["modules"]) > 0: + progress_categories.append(f"{version}.dol") + add_unit(unit, version, progress_categories) # Add REL units for module in build_config["modules"]: for unit in module["units"]: - add_unit(unit, version, module["name"]) + progress_categories = [version] + if config.progress_modules: + progress_categories.append(f"{version}.modules") + if config.progress_each_module: + progress_categories.append(f"{version}.{module['name']}") + add_unit(unit, version, progress_categories) + + # Add progress categories + def add_category(id: str, name: str): + objdiff_config["progress_categories"].append( + { + "id": id, + "name": name, + } + ) + + add_category(version, version) + if len(build_config["modules"]) > 0: + add_category(f"{version}.dol", "DOL") + if config.progress_modules: + add_category(f"{version}.modules", "Modules") + if config.progress_each_module: + for module in build_config["modules"]: + add_category(f"{version}.{module['name']}", module["name"]) + for category in config.progress_categories: + add_category(f"{version}.{category.id}", category.name) # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: @@ -1119,6 +1283,8 @@ def unix_path(input: Any) -> str: # Calculate, print and write progress to progress.json def calculate_progress(config: ProjectConfig, version: str) -> None: + config.validate() + objects = config.objects(version) out_path = config.out_path(version) build_config = load_build_config(config, out_path / "config.json") if not build_config: @@ -1153,12 +1319,8 @@ def add(self, build_obj: Dict[str, Any]) -> None: # Skip autogenerated objects return - result = config.find_object(build_obj["name"]) - if not result: - return - - _, obj = result - if not obj.completed(version) or obj.options["asm_processor"]: + obj = objects.get(build_obj["name"]) + if obj is None or not obj.completed(version) or obj.options["asm_processor"]: return self.code_progress += build_obj["code_size"] @@ -1172,26 +1334,52 @@ def code_frac(self) -> float: def data_frac(self) -> float: return self.data_progress / self.data_total + progress_units: Dict[str, ProgressUnit] = {} + if config.progress_all: + progress_units["all"] = ProgressUnit("All") + progress_units["dol"] = ProgressUnit("DOL") + if len(build_config["modules"]) > 0: + if config.progress_modules: + progress_units["modules"] = ProgressUnit("Modules") + if len(config.progress_categories) > 0: + for category in config.progress_categories: + progress_units[category.id] = ProgressUnit(category.name) + if config.progress_each_module: + for module in build_config["modules"]: + progress_units[module["name"]] = ProgressUnit(module["name"]) + + def add_unit(id: str, unit: Dict[str, Any]) -> None: + progress = progress_units.get(id) + if progress is not None: + progress.add(unit) + # Add DOL units - all_progress = ProgressUnit("All") if config.progress_all else None - dol_progress = ProgressUnit("DOL") for unit in build_config["units"]: - if all_progress: - all_progress.add(unit) - dol_progress.add(unit) + add_unit("all", unit) + add_unit("dol", unit) + obj = objects.get(unit["name"]) + if obj is not None: + category_opt = obj.options["progress_category"] + if isinstance(category_opt, list): + for id in category_opt: + add_unit(id, unit) + elif category_opt is not None: + add_unit(category_opt, unit) # Add REL units - rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: - progress = ProgressUnit(module["name"]) - modules_progress.append(progress) for unit in module["units"]: - if all_progress: - all_progress.add(unit) - if rels_progress: - rels_progress.add(unit) - progress.add(unit) + add_unit("all", unit) + add_unit("modules", unit) + add_unit(module["name"], unit) + obj = objects.get(unit["name"]) + if obj is not None: + category_opt = obj.options["progress_category"] + if isinstance(category_opt, list): + for id in category_opt: + add_unit(id, unit) + elif category_opt is not None: + add_unit(category_opt, unit) # Print human-readable progress print(f"{version} progress:") @@ -1219,15 +1407,8 @@ def print_category(unit: Optional[ProgressUnit]) -> None: ) ) - if all_progress: - print_category(all_progress) - print_category(dol_progress) - module_count = len(build_config["modules"]) - if module_count > 0: - print_category(rels_progress) - if config.progress_each_module: - for progress in modules_progress: - print_category(progress) + for progress in progress_units.values(): + print_category(progress) # Generate and write progress.json progress_json: Dict[str, Any] = {} @@ -1240,14 +1421,7 @@ def add_category(category: str, unit: ProgressUnit) -> None: "data/total": unit.data_total, } - if all_progress: - add_category("all", all_progress) - add_category("dol", dol_progress) - if len(build_config["modules"]) > 0: - if rels_progress: - add_category("modules", rels_progress) - if config.progress_each_module: - for progress in modules_progress: - add_category(progress.name, progress) + for id, progress in progress_units.items(): + add_category(id, progress) with open(out_path / "progress.json", "w", encoding="utf-8") as w: json.dump(progress_json, w, indent=4)