Update app.py

app.py

@@ -1,19 +1,50 @@
+from transformers import pipeline, Conversation, AutoTokenizer, AutoModelForCausalLM
+from langchain.llms import HuggingFacePipeline
+from langchain.prompts import PromptTemplate
 from langchain_community.llms import HuggingFaceHub
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
 from langchain.chains import LLMChain
-from langchain.prompts import PromptTemplate
-from langchain_community.llms import LlamaCpp
+
+#1: "meta-llama/Llama-2-13b-chat-hf",
+#2: "BramVanroy/Llama-2-13b-chat-dutch"
+my_config = {'model_name': "meta-llama/Llama-2-13b-chat-hf", #"./Bram", #BramVanroy/Llama-2-13b-chat-dutch", 
+             'do_sample': True, 'temperature': 0.1, 
+             'repetition_penalty': 1.1, 'max_new_tokens': 500, }
+
+print(f"Selected model: {my_config['model_name']}")
+print(f"Parameters are: {my_config}")
 
 question = "Who won the FIFA World Cup in the year 1994? "
 template = """Question: {question}
 Answer: Let's think step by step."""
 
 prompt = PromptTemplate.from_template(template)
+def generate_with_llama_chat(my_config):
+    print('tokenizer')
+    tokenizer = AutoTokenizer.from_pretrained(my_config['model_name'])
+    print('causal')
+    model = AutoModelForCausalLM.from_pretrained(my_config['model_name'])
+    print('Pipeline')
+    chatbot = pipeline("text-generation",model=my_config['model_name'], 
+                       tokenizer=tokenizer,
+                       do_sample=my_config['do_sample'], 
+                       temperature=my_config['temperature'], 
+                       repetition_penalty=my_config['repetition_penalty'],
+                       #max_length=my_config['max_length'],
+                       max_new_tokens=my_config['max_new_tokens'], 
+                       model_kwargs={"device_map": "auto","load_in_8bit": True})
+    return chatbot
+
+llama_chat = generate_with_llama_chat(my_config)
+
+# Set up callback manager to print output word by word
+callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+
+llm = HuggingFacePipeline(pipeline=llama_chat, callback_manager=callback_manager)
+
 
-llm = HuggingFaceHub(
-    repo_id="BramVanroy/Llama-2-13b-chat-dutch", model_kwargs={"temperature": 0.5, "max_length": 64}
-)
 llm_chain = LLMChain(prompt=prompt, llm=llm)
 
 print(llm_chain.invoke(question))