Update token length

2024-10-26 02:42:40 +08:00
parent 226f6f3d87
commit 87f8b7dba1
1 changed files with 1 additions and 1 deletions
--- a/lightrag/llm.py
+++ b/lightrag/llm.py
@@ -268,7 +268,7 @@ async def hf_model_if_cache(
    ).to("cuda")
    inputs = {k: v.to(hf_model.device) for k, v in input_ids.items()}
    output = hf_model.generate(
-        **input_ids, max_new_tokens=200, num_return_sequences=1, early_stopping=True
+        **input_ids, max_new_tokens=512, num_return_sequences=1, early_stopping=True
    )
    response_text = hf_tokenizer.decode(output[0][len(inputs["input_ids"][0]):], skip_special_tokens=True)
    if hashing_kv is not None: