Usage shown in func stream_filling_sequence's comments
filling_stream = stream_filling_sequence(
model, seq,
batch_size=1,
get_masks_and_position_ids=get_func,
strategy=strategy,
pre_image=pre_image,
image=torch_image,
)
for tokens, logit, mems in filling_stream:
pass # you can do something or nothing at all
output = strategy.finalize(tokens, mems)[0]
Usage shown in func stream_filling_sequence's comments