Update app.py

app.py

@@ -246,8 +246,9 @@ class Translators:
     def rosetta(self):
         model = AutoModelForCausalLM.from_pretrained(
             self.model_name,
-            dtype=torch.bfloat16,
-            device_map="auto").to(self.device)
+            dtype=torch.float32, # bfloat16
+            low_cpu_mem_usage=True,
+            device_map="auto")
         tokenizer = AutoTokenizer.from_pretrained(self.model_name)
         context = {"tone": "Formal and precise"}        
         system = f"Translate the user's text to {self.tl}. Provide the final translation in a formal tone immediately immediately without any other text."
@@ -259,6 +260,7 @@ class Translators:
         print(prompt)
         inputs = tokenizer(prompt, return_tensors="pt").to(self.device)
         input_length = inputs["input_ids"].shape[1]       
+        model.eval()
         with torch.inference_mode():
             outputs = model.generate(
                 **inputs,