Update models/loader.py (#4)
Browse files- Update models/loader.py (876c070d6cc70e98fd2dfef6fd152a934b4ed962)
- models/loader.py +1 -1
models/loader.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
import torch
|
| 3 |
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
| 4 |
from backend.agents import ROLE_PROMPTS
|
| 5 |
-
|
| 6 |
# The following configs are no longer used for CPU, but kept for future GPU use.
|
| 7 |
QUANTIZATION_CONFIG = BitsAndBytesConfig(
|
| 8 |
load_in_4bit=True,
|
|
|
|
| 2 |
import torch
|
| 3 |
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
| 4 |
from backend.agents import ROLE_PROMPTS
|
| 5 |
+
torch.set_num_threads(2)
|
| 6 |
# The following configs are no longer used for CPU, but kept for future GPU use.
|
| 7 |
QUANTIZATION_CONFIG = BitsAndBytesConfig(
|
| 8 |
load_in_4bit=True,
|