diff --git a/scripts/inference.py b/scripts/inference.py index 4121b1b95..08de992b8 100644 --- a/scripts/inference.py +++ b/scripts/inference.py @@ -18,15 +18,12 @@ from typing import List, Optional import subprocess from transformers import AutoTokenizer +import neural_speed model_maps = {"gpt_neox": "gptneox", "llama2": "llama", "gpt_bigcode": "starcoder"} build_path = Path(Path(__file__).parent.absolute(), "../build/") -def is_win(): - return sys.platform.startswith('win') - - def main(args_in: Optional[List[str]] = None) -> None: parser = argparse.ArgumentParser(description="main program llm running") parser.add_argument("--model_name", type=str, help="Model name: String", required=True) @@ -130,19 +127,8 @@ def main(args_in: Optional[List[str]] = None) -> None: args = parser.parse_args(args_in) print(args) model_name = model_maps.get(args.model_name, args.model_name) - if is_win(): - path = Path(args.build_dir, "./Bin/Release/run_{}.exe".format(model_name)) - else: - if args.one_click_run == "True": - import neural_speed - package_path = os.path.dirname(neural_speed.__file__) - path = Path(package_path, "./run_{}".format(model_name)) - else: - path = Path(args.build_dir, "./bin/run_{}".format(model_name)) - - if not path.exists(): - print("Please build graph first or select the correct model name.") - sys.exit(1) + package_path = os.path.dirname(neural_speed.__file__) + path = Path(package_path, "./run_{}".format(model_name)) cmd = [path] cmd.extend(["--model", args.model]) diff --git a/scripts/quantize.py b/scripts/quantize.py index 7034675a9..d71ba9fc2 100644 --- a/scripts/quantize.py +++ b/scripts/quantize.py @@ -17,15 +17,12 @@ import argparse from typing import List, Optional import subprocess +import neural_speed model_maps = {"gpt_neox": "gptneox", "llama2": "llama", "gpt_bigcode": "starcoder"} build_path = Path(Path(__file__).parent.absolute(), "../build/") -def is_win(): - return sys.platform.startswith('win') - - def str2bool(v): if isinstance(v, bool): return v @@ -100,19 +97,8 @@ def main(args_in: Optional[List[str]] = None) -> None: args = parser.parse_args(args_in) model_name = model_maps.get(args.model_name, args.model_name) - if is_win(): - path = Path(args.build_dir, "./Bin/Release/quant_{}.exe".format(model_name)) - else: - if args.one_click_run == "True": - import neural_speed - package_path = os.path.dirname(neural_speed.__file__) - path = Path(package_path, "./quant_{}".format(model_name)) - else: - path = Path(args.build_dir, "./bin/quant_{}".format(model_name)) - if not path.exists(): - print(path) - print("Please build graph first or select the correct model name.") - sys.exit(1) + package_path = os.path.dirname(neural_speed.__file__) + path = Path(package_path, "./quant_{}".format(model_name)) cmd = [path] cmd.extend(["--model_file", args.model_file])