Fix PunktTokenizer für Hugging Face Space

analyze_aspects.py

@@ -3,7 +3,7 @@
 
 #python /Users/fischer/Desktop/HanserMVP/scraping/analyze_aspects.py --isbn "9783446264199" --db-path /Users/fischer/Desktop/buch_datenbank.sqlite --languages de
 # python analyze_aspects.py --isbn "9783446264199" --db-path /Pfad/zur/sqlite.db --languages de
-
+# Fixing Punkt tokenizer bug
 import sqlite3
 import argparse
 import logging
@@ -13,7 +13,6 @@ from transformers import pipeline
 from collections import defaultdict
 import matplotlib.pyplot as plt
 
-
 def visualize_aspects(aspect_results: dict[str, list[float]], output_dir: Path, filename: str = "sentiment_aspekte.png"):
     output_dir.mkdir(parents=True, exist_ok=True)
 
@@ -42,7 +41,7 @@ def visualize_aspects(aspect_results: dict[str, list[float]], output_dir: Path,
 
 
 # NLTK punkt model for sentence tokenization
-nltk.download('punkt')
+nltk.download('punkt', download_dir='/home/user/nltk_data')
 from nltk import sent_tokenize
 
 # Logging Configuration
@@ -119,7 +118,13 @@ def analyze_quickwin(db_path: Path, isbn: str, device: int = -1, languages: list
             continue
 
         logger.info(f"Review ID {review_id} ({lang}) wird verarbeitet.")
-        sentences = sent_tokenize(text, language='german' if lang == 'de' else 'english')
+        import os
+        nltk.download('punkt')
+        nltk.data.path.append("/home/user/nltk_data")
+        os.environ["NLTK_DATA"] = "/home/user/nltk_data"
+
+        lang_map = {'de': 'german', 'en': 'english'}
+        sentences = sent_tokenize(text, language=lang_map.get(lang, 'english'))
 
         if lang == 'de':
             aspect_map = ASPECT_LABEL_MAP