|
def evict_host(self, num_tokens: int): |
|
leaves = self._collect_leaves() |
|
heapq.heapify(leaves) |
|
|
|
num_evicted = 0 |
|
while num_evicted < num_tokens and len(leaves): |
|
x = heapq.heappop(leaves) |
|
if x == self.root_node: |
|
break |
|
# only evict the host value of evicted nodes |
|
if not x.evicted: |
|
continue |
|
assert x.lock_ref == 0 and x.host_value is not None |
|
|
|
assert self.cache_controller.evict_host(x.host_value) > 0 |
|
for k, v in x.parent.children.items(): |
|
if v == x: |
|
break |
|
del x.parent.children[k] |
|
|
|
if len(x.parent.children) == 0 and x.parent.evicted: |
|
heapq.heappush(leaves, x.parent) |
@xiezhq-hermann num_evicted and num_tokens is not changed
while num_evicted < num_tokens:
sglang/python/sglang/srt/mem_cache/hiradix_cache.py
Lines 201 to 222 in 9eb49e8
@xiezhq-hermann num_evicted and num_tokens is not changed