Skip to content

Commit

Permalink
var naming and add todo
Browse files Browse the repository at this point in the history
  • Loading branch information
winglian committed Dec 26, 2024
1 parent 57738d4 commit b246edd
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions src/axolotl/prompt_strategies/chat_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,20 +494,21 @@ def transform_logprobs(self, sample):
logprobs = sample.pop(self.logprobs_field)
target_seq_len = len(logprobs)
input_seq_len = len(sample["input_ids"])
padding_len = input_seq_len - target_seq_len
input_padding_len = input_seq_len - target_seq_len
top_k = len(logprobs[0])
target_logprobs = []
target_token_ids = []
target_mask = []

# fill with -inf for padding_len tokens for top_k tokens
# extend target_logprobs with a padding_len x top_k 2D list filled with -inf
for _ in range(padding_len):
for _ in range(input_padding_len):
target_logprobs.append([-float("inf")] * top_k)
target_token_ids.append(list(range(top_k)))
target_mask.append([0] * top_k)

for _ in range(target_seq_len):
# TODO also check against sample["labels"]
target_mask.append([1] * top_k)

for _, token_pos_logprobs in enumerate(logprobs):
Expand Down

0 comments on commit b246edd

Please sign in to comment.