Skip to content

Commit

Permalink
Enable DUS-based KV Cache update for OSS
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 705933157
  • Loading branch information
talumbau authored and copybara-github committed Dec 13, 2024
1 parent 7074ae7 commit 6a7ca32
Showing 1 changed file with 1 addition and 2 deletions.
3 changes: 1 addition & 2 deletions ai_edge_torch/generative/layers/kv_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from ai_edge_torch import hlfb
from ai_edge_torch.generative.layers import model_config
from ai_edge_torch.generative.utilities.dynamic_update_slice import dynamic_update_slice
import torch
import torch.utils._pytree as pytree

Expand Down Expand Up @@ -159,8 +160,6 @@ def update(
Returns:
KVCacheEntry: The updated KVCache entry based on the passed inputs.
"""
# Turn dynamic_update_slice updates off for now.
use_dus=False
update_kv_cache = _update_kv_impl if use_dus else _update_kv_base_impl
return update_kv_cache(cache, input_pos, k_slice, v_slice)

Expand Down

0 comments on commit 6a7ca32

Please sign in to comment.