update Model's compile depth (#808)

forge/test/models/pytorch/audio/whisper/test_whisper_0.py

@@ -37,7 +37,7 @@
 def generate_model_whisper_congen_hf_pytorch(test_device, variant):
     # Configurations
     compiler_cfg = _get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
+    compiler_cfg.compile_depth = forge.CompileDepth.POST_INITIAL_GRAPH_PASS
 
     class Wrapper(torch.nn.Module):
         def __init__(self, model):

forge/test/models/pytorch/multimodal/vilt/test_vilt.py

@@ -10,6 +10,7 @@
 from PIL import Image
 from transformers import ViltProcessor, ViltForQuestionAnswering, ViltForMaskedLM, ViltConfig
 from test.models.pytorch.multimodal.vilt.utils.model import ViLtEmbeddingWrapper, ViltModelWrapper
+from forge.op.eval.common import compare_with_golden
 
 url = "http://images.cocodataset.org/val2017/000000039769.jpg"
 image = Image.open(requests.get(url, stream=True).raw)
@@ -65,7 +66,6 @@ def generate_model_vilt_maskedlm_hf_pytorch(test_device, variant):
 
     # STEP 1: Set Forge configuration parameters
     compiler_cfg = forge.config._get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Set model configurations
     config = ViltConfig.from_pretrained(variant)
@@ -93,11 +93,19 @@ def generate_model_vilt_maskedlm_hf_pytorch(test_device, variant):
 variants = ["dandelin/vilt-b32-mlm"]
 
 
+@pytest.mark.xfail(reason="pcc=0.9498278562793674")
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.nightly
 def test_vilt_maskedlm_hf_pytorch(variant, test_device):
     model, inputs, _ = generate_model_vilt_maskedlm_hf_pytorch(
         test_device,
         variant,
     )
-    compiled_model = forge.compile(model, sample_inputs=[inputs[0], inputs[1]], module_name="pt_ViLt_maskedlm")
+    compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_ViLt_maskedlm")
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])

forge/test/models/pytorch/text/albert/test_albert.py

@@ -5,12 +5,14 @@
 from test.utils import download_model
 import forge
 from transformers import AlbertForMaskedLM, AlbertTokenizer, AlbertForTokenClassification
-
+from forge.op.eval.common import compare_with_golden
+import torch
 
 sizes = ["base", "large", "xlarge", "xxlarge"]
 variants = ["v1", "v2"]
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.parametrize("size", sizes, ids=sizes)
 @pytest.mark.nightly
@@ -22,7 +24,6 @@ def test_albert_masked_lm_pytorch(size, variant, test_device):
     model = download_model(AlbertForMaskedLM.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample
     sample_text = "The capital of France is [MASK]."
@@ -41,18 +42,26 @@ def test_albert_masked_lm_pytorch(size, variant, test_device):
     varaint_name = model_ckpt.replace("-", "_")
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name=f"pt_{varaint_name}_masked_lm")
 
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
+
 
 sizes = ["base", "large", "xlarge", "xxlarge"]
 variants = ["v1", "v2"]
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.parametrize("size", sizes, ids=sizes)
 @pytest.mark.nightly
 def test_albert_token_classification_pytorch(size, variant, test_device):
 
     compiler_cfg = forge.config._get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # NOTE: These model variants are pre-trined only. They need to be fine-tuned
     # on a downstream task. Code is for demonstration purposes only.
@@ -80,3 +89,11 @@ def test_albert_token_classification_pytorch(size, variant, test_device):
     inputs = [input_tokens["input_ids"], input_tokens["attention_mask"]]
     varaint_name = model_ckpt.replace("-", "_")
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name=f"pt_{varaint_name}_token_cls")
+
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])

forge/test/models/pytorch/text/bert/test_bert.py

@@ -11,6 +11,8 @@
     BertForSequenceClassification,
     BertForQuestionAnswering,
 )
+import torch
+from forge.op.eval.common import compare_with_golden
 
 
 def generate_model_bert_maskedlm_hf_pytorch(variant):
@@ -20,7 +22,6 @@ def generate_model_bert_maskedlm_hf_pytorch(variant):
     model = BertForMaskedLM.from_pretrained(model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample
     sample_text = "The capital of France is [MASK]."
@@ -37,12 +38,21 @@ def generate_model_bert_maskedlm_hf_pytorch(variant):
     return model, [input_tokens["input_ids"]], {}
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 def test_bert_masked_lm_pytorch(test_device):
     model, inputs, _ = generate_model_bert_maskedlm_hf_pytorch("bert-base-uncased")
 
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_bert_masked_lm")
 
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
+
 
 def generate_model_bert_qa_hf_pytorch(variant):
     # Load Bert tokenizer and model from HuggingFace
@@ -51,7 +61,6 @@ def generate_model_bert_qa_hf_pytorch(variant):
     model = download_model(BertForQuestionAnswering.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample from SQuADv1.1
     context = """Super Bowl 50 was an American football game to determine the champion of the National Football League
@@ -78,12 +87,21 @@ def generate_model_bert_qa_hf_pytorch(variant):
     return model, [input_tokens["input_ids"]], {}
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 def test_bert_question_answering_pytorch(test_device):
     model, inputs, _ = generate_model_bert_qa_hf_pytorch("bert-large-cased-whole-word-masking-finetuned-squad")
 
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_bert_qa")
 
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
+
 
 def generate_model_bert_seqcls_hf_pytorch(variant):
     # Load Bert tokenizer and model from HuggingFace
@@ -117,6 +135,14 @@ def test_bert_sequence_classification_pytorch(test_device):
 
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_bert_sequence_classification")
 
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
+
 
 def generate_model_bert_tkcls_hf_pytorch(variant):
     # Load Bert tokenizer and model from HuggingFace
@@ -125,7 +151,6 @@ def generate_model_bert_tkcls_hf_pytorch(variant):
     model = download_model(BertForTokenClassification.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample
     sample_text = "HuggingFace is a company based in Paris and New York"
@@ -142,8 +167,17 @@ def generate_model_bert_tkcls_hf_pytorch(variant):
     return model, [input_tokens["input_ids"]], {}
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 def test_bert_token_classification_pytorch(test_device):
     model, inputs, _ = generate_model_bert_tkcls_hf_pytorch("dbmdz/bert-large-cased-finetuned-conll03-english")
 
     compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_bert_sequence_classification")
+
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])

forge/test/models/pytorch/text/codegen/test_codegen.py

@@ -15,14 +15,16 @@
     # "Salesforce/codegen-350M-multi", # Currently not supported
     # "Salesforce/codegen-350M-nl", # Currently not supported
 ]
+import torch
+from forge.op.eval.common import compare_with_golden
 
 
+@pytest.mark.xfail(reason="RuntimeError: Tensor 41 - data type mismatch: expected Float32, got BFloat16")
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_codegen(test_device, variant):
     # Configurations
     compiler_cfg = forge.config._get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load model (with tokenizer)
     tokenizer = download_model(AutoTokenizer.from_pretrained, variant)
@@ -63,3 +65,11 @@ def forward(self, input_ids, attention_mask):
     compiled_model = forge.compile(
         framework_model, sample_inputs=inputs, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_"))
     )
+
+    co_out = compiled_model(*inputs)
+    fw_out = framework_model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])

forge/test/models/pytorch/text/dpr/test_dpr.py

@@ -12,10 +12,13 @@
     DPRQuestionEncoder,
     DPRQuestionEncoderTokenizer,
 )
+import torch
+from forge.op.eval.common import compare_with_golden
 
 variants = ["facebook/dpr-ctx_encoder-single-nq-base", "facebook/dpr-ctx_encoder-multiset-base"]
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_context_encoder_pytorch(variant, test_device):
@@ -27,7 +30,6 @@ def test_dpr_context_encoder_pytorch(variant, test_device):
     model = download_model(DPRContextEncoder.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample
     sample_text = "Hello, is my dog cute?"
@@ -45,11 +47,19 @@ def test_dpr_context_encoder_pytorch(variant, test_device):
     compiled_model = forge.compile(
         model, sample_inputs=inputs, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_"))
     )
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
 
 
 variants = ["facebook/dpr-question_encoder-single-nq-base", "facebook/dpr-question_encoder-multiset-base"]
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_question_encoder_pytorch(variant, test_device):
@@ -60,7 +70,6 @@ def test_dpr_question_encoder_pytorch(variant, test_device):
     model = download_model(DPRQuestionEncoder.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Load data sample
     sample_text = "Hello, is my dog cute?"
@@ -78,11 +87,19 @@ def test_dpr_question_encoder_pytorch(variant, test_device):
     compiled_model = forge.compile(
         model, sample_inputs=inputs, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_"))
     )
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
 
 
 variants = ["facebook/dpr-reader-single-nq-base", "facebook/dpr-reader-multiset-base"]
 
 
+@pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op")
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_reader_pytorch(variant, test_device):
@@ -93,7 +110,6 @@ def test_dpr_reader_pytorch(variant, test_device):
     model = download_model(DPRReader.from_pretrained, model_ckpt)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Data preprocessing
     input_tokens = tokenizer(
@@ -110,3 +126,10 @@ def test_dpr_reader_pytorch(variant, test_device):
     compiled_model = forge.compile(
         model, sample_inputs=inputs, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_"))
     )
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])

forge/test/models/pytorch/text/gpt2/test_gpt2.py

@@ -7,8 +7,10 @@
 import forge
 import os
 from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config
+from forge.op.eval.common import compare_with_golden
 
 
+@pytest.mark.xfail(reason="RuntimeError: Tensor 6 - data type mismatch: expected Float32, got BFloat16")
 @pytest.mark.nightly
 def test_gpt2_text_gen(test_device):
     # Load tokenizer and model from HuggingFace
@@ -20,7 +22,6 @@ def test_gpt2_text_gen(test_device):
     model = download_model(GPT2LMHeadModel.from_pretrained, "gpt2", config=config)
 
     compiler_cfg = forge.config._get_global_compiler_config()  # load global compiler config object
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
 
     # Wrapper to get around past key values
     class Wrapper(torch.nn.Module):
@@ -38,6 +39,14 @@ def forward(self, input_ids, attention_mask):
     inputs = [input_ids, attn_mask]
     compiled_model = forge.compile(Wrapper(model), sample_inputs=inputs, module_name="pt_gpt2_generation")
 
+    co_out = compiled_model(*inputs)
+    fw_out = model(*inputs)
+
+    co_out = [co.to("cpu") for co in co_out]
+    fw_out = [fw_out] if isinstance(fw_out, torch.Tensor) else fw_out
+
+    assert all([compare_with_golden(golden=fo, calculated=co, pcc=0.99) for fo, co in zip(fw_out, co_out)])
+
 
 class Wrapper(torch.nn.Module):
     def __init__(self, model):

forge/test/models/pytorch/text/opt/test_opt.py

@@ -85,7 +85,7 @@ def test_opt_qa(variant, test_device):
 def test_opt_sequence_classification(variant, test_device):
     # Set Forge configuration parameters
     compiler_cfg = forge.config._get_global_compiler_config()
-    compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH
+    compiler_cfg.compile_depth = forge.CompileDepth.INIT_COMPILE
 
     # Load tokenizer and model from HuggingFace
     # Variants: "facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"