Merge pull request LianjiaTech#138 from bestpredicts/main

解决colab环境不稳定导致quant cuda不存在的问题
zhiyou720 · Apr 9, 2023 · 4a2619f · 4a2619f
2 parents 78d2863 + 6236701
commit 4a2619f
Show file tree

Hide file tree

Showing 2 changed files with 1,448 additions and 773 deletions.
diff --git a/gptq/llama_inference.py b/gptq/llama_inference.py
@@ -133,6 +133,7 @@ def noop(*args, **kwargs):
                 temperature=args.temperature,
             )
         print("Assistant:\n") 
-        print(tokenizer.decode([el.item() for el in generated_ids[0]]))
+        print(tokenizer.decode([el.item() for el in generated_ids[0]])[len(inputs)+4:]) # generated_ids开头加上了bos_token,需要将inpu的内容截断,只输出Assistant 
         print("\n-------------------------------\n")
+        print("Human:") #每次终端用户输入前，加上Human提示。
         line = input()