From 5ca58226861861229de71a32e3f754649daf7126 Mon Sep 17 00:00:00 2001 From: xtinkt Date: Thu, 18 Jul 2024 09:47:16 +0000 Subject: [PATCH 1/2] draft test --- tests/test_speculative_generation.py | 119 +++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 18 deletions(-) diff --git a/tests/test_speculative_generation.py b/tests/test_speculative_generation.py index e3045dea..52044e5e 100644 --- a/tests/test_speculative_generation.py +++ b/tests/test_speculative_generation.py @@ -3,33 +3,116 @@ import pytest import torch +import transformers + +from petals import AutoDistributedModelForCausalLM from petals import AutoDistributedConfig, RemoteSequential from petals.server.block_functions import MAX_SHORT_INFERENCE_TOKENS from petals.server.from_pretrained import load_pretrained_block from test_utils import * +@pytest.fixture +def tokenizer(): + # We set use_fast=False since LlamaTokenizerFast is slow on load + return transformers.AutoTokenizer.from_pretrained(MODEL_NAME, use_fast=False) + + +@pytest.fixture +def model(): + return AutoDistributedModelForCausalLM.from_pretrained( + MODEL_NAME, initial_peers=INITIAL_PEERS, torch_dtype=torch.float32 + ) + +@pytest.fixture +def model2(): + return transformers.AutoModelForCausalLM.from_pretrained( + REF_NAME, low_cpu_mem_usage=True, torch_dtype=torch.float32 + ) + +@pytest.fixture +def ref_model(): + return transformers.AutoModelForCausalLM.from_pretrained( + MODEL_NAME, low_cpu_mem_usage=True, torch_dtype=torch.float32 + ) + +# @pytest.mark.forked +# def test_remote_block_with_cache_invalidation_exact_match(atol_forward=1e-4, atol_inference=1e-3): +# config = AutoDistributedConfig.from_pretrained(MODEL_NAME, initial_peers=INITIAL_PEERS) +# remote_sequential = RemoteSequential(config) + +# block_index = random.randint(0, config.num_hidden_layers - 1) +# remote_block = remote_sequential[block_index] + +# inputs = torch.randn(1, MAX_SHORT_INFERENCE_TOKENS - 50, config.hidden_size) +# short_inputs = torch.randn(1, MAX_SHORT_INFERENCE_TOKENS - 50, config.hidden_size) +# short_inputs[:, :2, :] = inputs[:, :2, :] + +# initial_outputs_inference = None +# secondary_outputs_inference = None +# with torch.inference_mode(): +# with remote_block.inference_session(max_length=inputs.shape[1]) as sess: +# initial_outputs_inference = sess.step(inputs) +# secondary_outputs_inference = sess.step(short_inputs[:, 2:, :], start_from_position=2) +# result = torch.cat([initial_outputs_inference[:, :2, :], secondary_outputs_inference], dim=1) + +# ref_block = load_pretrained_block(MODEL_NAME, block_index, torch_dtype=torch.float32) +# (outputs_local,) = ref_block(short_inputs) + +# assert torch.allclose(outputs_local, result, rtol=0, atol=atol_inference) + +# @pytest.mark.forked +# def test_speculative_greedy_generation(tokenizer, model, ref_model, max_new_tokens=4): +# inputs = tokenizer("A cat sat on a mat", return_tensors="pt")["input_ids"] + +# options = dict(max_new_tokens=max_new_tokens, do_sample=False) +# outputs = model.generate(inputs, **options) +# print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@", outputs.shape, outputs) +# ref_outputs = ref_model.generate(inputs, **options) +# assert torch.allclose( +# outputs, ref_outputs +# ), f"Greedy generation is not identical to HF with {multiple_calls=}, {inputs.shape=}" + @pytest.mark.forked -def test_remote_block_with_cache_invalidation_exact_match(atol_forward=1e-4, atol_inference=1e-3): - config = AutoDistributedConfig.from_pretrained(MODEL_NAME, initial_peers=INITIAL_PEERS) - remote_sequential = RemoteSequential(config) +def test_speculative_greedy_generation(tokenizer, model, model2, ref_model, max_new_tokens=50, batch_size=10): + inputs = tokenizer("A cat sat on a mat", return_tensors="pt")["input_ids"] + generated_ids = inputs + + with torch.no_grad(): + while generated_ids.shape[1] < max_new_tokens + inputs.shape[1]: + outputs2 = model2.generate(generated_ids, max_new_tokens=batch_size, do_sample=False) + new_tokens = outputs2[:, -batch_size:] + + random_pos = random.randrange(1, batch_size) + new_tokens[:, random_pos] = random.randrange(1, 100) - block_index = random.randint(0, config.num_hidden_layers - 1) - remote_block = remote_sequential[block_index] + combined_ids = torch.cat((generated_ids, new_tokens), dim=1) + logits = model(combined_ids, start_from_position=1).logits - inputs = torch.randn(1, MAX_SHORT_INFERENCE_TOKENS - 50, config.hidden_size) - short_inputs = torch.randn(1, MAX_SHORT_INFERENCE_TOKENS - 50, config.hidden_size) - short_inputs[:, :2, :] = inputs[:, :2, :] + # Найти первую позицию, где токены совпали + match_length = 0 + for i in range(batch_size): + top_predicted_id_model2 = new_tokens[:, i] + top_predicted_id_model = torch.argmax(logits[:, generated_ids.shape[1] + i - 1, :], dim=-1) + + if top_predicted_id_model2 == top_predicted_id_model: + match_length += 1 + else: + break + print(f"Принято {match_length} из {batch_size}") - initial_outputs_inference = None - secondary_outputs_inference = None - with torch.inference_mode(): - with remote_block.inference_session(max_length=inputs.shape[1]) as sess: - initial_outputs_inference = sess.step(inputs) - secondary_outputs_inference = sess.step(short_inputs[:, 2:, :], start_from_position=2) - result = torch.cat([initial_outputs_inference[:, :2, :], secondary_outputs_inference], dim=1) + if match_length > 0: + generated_ids = torch.cat((generated_ids, new_tokens[:, :match_length]), dim=1) + print(f"Всего {generated_ids.shape[1]}") + else: + break + + ref_outputs = ref_model.generate(inputs, max_new_tokens=max_new_tokens, do_sample=False) + + gen_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + ref_text = tokenizer.decode(ref_outputs[0], skip_special_tokens=True) - ref_block = load_pretrained_block(MODEL_NAME, block_index, torch_dtype=torch.float32) - (outputs_local,) = ref_block(short_inputs) + print(f"Generated by speculative decoding: {gen_text}") + print(f"Reference generation: {ref_text}") - assert torch.allclose(outputs_local, result, rtol=0, atol=atol_inference) + assert gen_text == ref_text, "The outputs do not match!" \ No newline at end of file From b9d4d06859f687bf2bd06e7459739e96cfc49a92 Mon Sep 17 00:00:00 2001 From: xtinkt Date: Thu, 18 Jul 2024 10:14:57 +0000 Subject: [PATCH 2/2] test --- tests/test_speculative_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_speculative_generation.py b/tests/test_speculative_generation.py index 52044e5e..2834410d 100644 --- a/tests/test_speculative_generation.py +++ b/tests/test_speculative_generation.py @@ -87,7 +87,7 @@ def test_speculative_greedy_generation(tokenizer, model, model2, ref_model, max_ new_tokens[:, random_pos] = random.randrange(1, 100) combined_ids = torch.cat((generated_ids, new_tokens), dim=1) - logits = model(combined_ids, start_from_position=1).logits + logits = model(combined_ids).logits # Найти первую позицию, где токены совпали match_length = 0