Update requirements: Add comprehensive dependencies including sentencepiece for OpenLLM model compatibility
Browse files- requirements.txt +2 -1
requirements.txt
CHANGED
|
@@ -10,7 +10,7 @@ torchaudio>=2.0.0 # Audio processing utilities
|
|
| 10 |
transformers>=4.30.0 # Pre-trained models and training utilities
|
| 11 |
datasets>=2.12.0 # Dataset loading and processing
|
| 12 |
tokenizers>=0.13.0 # Fast tokenization library
|
| 13 |
-
sentencepiece>=0.1.99 # SentencePiece tokenization
|
| 14 |
huggingface_hub>=0.34.0 # Hugging Face Hub integration
|
| 15 |
accelerate>=0.20.0 # Distributed training acceleration
|
| 16 |
|
|
@@ -48,3 +48,4 @@ flake8>=6.0.0 # Code linting
|
|
| 48 |
# Note: These versions are compatible with Hugging Face Spaces
|
| 49 |
# and provide stable training performance for OpenLLM models
|
| 50 |
# Gradio 4.44.1 fixes compatibility issues with JSON components
|
|
|
|
|
|
| 10 |
transformers>=4.30.0 # Pre-trained models and training utilities
|
| 11 |
datasets>=2.12.0 # Dataset loading and processing
|
| 12 |
tokenizers>=0.13.0 # Fast tokenization library
|
| 13 |
+
sentencepiece>=0.1.99 # SentencePiece tokenization (CRITICAL for OpenLLM models)
|
| 14 |
huggingface_hub>=0.34.0 # Hugging Face Hub integration
|
| 15 |
accelerate>=0.20.0 # Distributed training acceleration
|
| 16 |
|
|
|
|
| 48 |
# Note: These versions are compatible with Hugging Face Spaces
|
| 49 |
# and provide stable training performance for OpenLLM models
|
| 50 |
# Gradio 4.44.1 fixes compatibility issues with JSON components
|
| 51 |
+
# SentencePiece is CRITICAL for OpenLLM model tokenization
|