Update app.py

app.py

@@ -9,7 +9,7 @@ from fastapi.responses import HTMLResponse, JSONResponse
 from pydantic import BaseModel
 from rank_bm25 import BM25Okapi
 from sentence_transformers import SentenceTransformer
-from transformers import T5Tokenizer, T5ForConditionalGeneration
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from collections import Counter
 from typing import Optional
 
@@ -28,9 +28,6 @@ STOPWORDS = {
     'would','could','should','its','if','about','than','then','too','s','t'
 }
 
-def _check(name):
-    import __main__
-    return hasattr(__main__, name)
 
 def preprocess(text, remove_stops=False):
     if not isinstance(text, str) or not text.strip(): return ''
@@ -44,9 +41,7 @@ def preprocess(text, remove_stops=False):
     return text
 
 
-# ── Load everything at startup ─────────────────────────────────
 CSV_PATH = os.getenv('CSV_PATH', 'Dataset-SA.csv')
-
 print('Loading data...')
 df_raw    = pd.read_csv(CSV_PATH)
 mask_bad  = pd.to_numeric(df_raw[COL_RATE], errors='coerce').isna()
@@ -67,37 +62,32 @@ df = df[df['clean_text'].str.len() > 15].reset_index(drop=True)
 df_sample = df.sample(n=min(1000, len(df)), random_state=42).reset_index(drop=True)
 print(f'Data: {len(df):,} rows | sample: {len(df_sample):,}')
 
-print('Loading SBERT (multilingual)...')
+print('Loading SBERT...')
 sbert     = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
 EMBED_DIM = sbert.get_sentence_embedding_dimension()
-embeddings = sbert.encode(
-    df_sample['clean_text'].tolist(), batch_size=16,
-    show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=False
-)
-
-print('Building FAISS index...')
+embeddings = sbert.encode(df_sample['clean_text'].tolist(), batch_size=16,
+                           show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=False)
+print('Building FAISS...')
 emb_norm = embeddings.copy().astype('float32')
 faiss.normalize_L2(emb_norm)
 index_flat = faiss.IndexFlatIP(EMBED_DIM)
 index_flat.add(emb_norm)
-
 print('Building BM25...')
 tokenized_corpus = [doc.split() for doc in df_sample['clean_text_bm25'].tolist()]
 bm25 = BM25Okapi(tokenized_corpus)
-
 print('Loading Flan-T5...')
-_tok   = T5Tokenizer.from_pretrained('google/flan-t5-base')
-_model = T5ForConditionalGeneration.from_pretrained('google/flan-t5-base')
+_tok   = AutoTokenizer.from_pretrained('google/flan-t5-base')
+_model = AutoModelForSeq2SeqLM.from_pretrained('google/flan-t5-base')
 _model.eval()
-
-print('Building autocomplete index...')
+print('Building autocomplete...')
 _all_words = Counter()
-for txt in df_sample['clean_text_bm25'].sample(min(2000, len(df_sample))):
+for txt in df_sample['clean_text_bm25'].sample(min(1000, len(df_sample))):
     _all_words.update(txt.split())
 _top_words = [w for w, c in _all_words.most_common(500) if len(w) > 3]
 _product_names = df_sample[COL_PRODUCT].str.lower().str[:50].drop_duplicates().head(200).tolist()
 _autocomplete_corpus = list(set(_top_words + _product_names))
-print(f'All models ready! Autocomplete: {len(_autocomplete_corpus)} suggestions')
+print(f'All models ready!')
+
 
 # ── Search cache (for performance) ────────────────────────────
 _cache = {}
@@ -183,16 +173,6 @@ def _similar(doc_id, k=5):
         if len(rows) == k: break
     return rows
 
-# ── Build autocomplete index ───────────────────────────────────
-print('Building autocomplete index...')
-_all_words = Counter()
-for txt in df_sample['clean_text_bm25'].sample(min(3000, len(df_sample))):
-    _all_words.update(txt.split())
-_top_words = [w for w, c in _all_words.most_common(500) if len(w) > 3]
-_product_names = df_sample[COL_PRODUCT].str.lower().str[:50].drop_duplicates().head(200).tolist()
-_autocomplete_corpus = list(set(_top_words + _product_names))
-print(f'Autocomplete: {len(_autocomplete_corpus)} suggestions ready')
-
 # ── FastAPI ────────────────────────────────────────────────────
 app_ui = FastAPI(title='FlipSearch Ultimate')
 app_ui.add_middleware(CORSMiddleware, allow_origins=['*'], allow_methods=['*'], allow_headers=['*'])
@@ -281,237 +261,105 @@ async def autocomplete(q: str = ''):
     matches = [w for w in _autocomplete_corpus if w.startswith(ql)][:8]
     return {'suggestions': matches}
 
-@app_ui.post('/analyse')
-def _analyse_uploaded_df(udf):
-    report = {}
-    report['rows']    = int(len(udf))
-    report['columns'] = list(udf.columns)
-    report['missing'] = {c: int(udf[c].isna().sum()) for c in udf.columns}
-
-    text_col = next((c for c in ['review','Review','text','Text','comment',
-                                  'Comment','description','feedback','body']
-                     if c in udf.columns), None)
-    if text_col is None:
-        obj = udf.select_dtypes(include='object').columns
-        text_col = obj[0] if len(obj) > 0 else None
-
-    rating_col = next((c for c in ['rating','Rating','rate','Rate',
-                                    'stars','score','Score']
-                       if c in udf.columns), None)
-
-    sent_col = next((c for c in ['sentiment','Sentiment','label','Label']
-                     if c in udf.columns), None)
-
-    report['detected'] = {
-        'text_column'     : text_col,
-        'rating_column'   : rating_col,
-        'sentiment_column': sent_col
-    }
-
-    if text_col:
-        lengths = udf[text_col].dropna().astype(str).apply(len)
-        words   = udf[text_col].dropna().astype(str).apply(lambda x: len(x.split()))
-        report['text_stats'] = {
-            'avg_length'     : round(float(lengths.mean()), 1),
-            'max_length'     : int(lengths.max()),
-            'min_length'     : int(lengths.min()),
-            'avg_words'      : round(float(words.mean()), 1),
-            'total_non_empty': int(
-                (udf[text_col].notna() &
-                 (udf[text_col].astype(str).str.len() > 0)).sum()
-            )
-        }
-        all_words = ' '.join(
-            udf[text_col].dropna().astype(str).str.lower().tolist()
-        )
-        word_freq = Counter(
-            w for w in re.sub(r'[^a-z\s]', '', all_words).split()
-            if w not in STOPWORDS and len(w) > 2
-        )
-        report['top_words'] = [
-            {'word': w, 'count': c}
-            for w, c in word_freq.most_common(15)
-        ]
-    else:
-        report['text_stats'] = {}
-        report['top_words']  = []
-
-    if rating_col:
-        rvals = pd.to_numeric(udf[rating_col], errors='coerce').dropna()
-        report['rating_stats'] = {
-            'avg'         : round(float(rvals.mean()), 2),
-            'distribution': {
-                str(int(k)): int(v)
-                for k, v in rvals.value_counts().sort_index().items()
-            }
-        }
-    else:
-        report['rating_stats'] = {}
-
-    if sent_col:
-        svals = udf[sent_col].dropna().astype(str).str.lower().str.strip()
-        report['sentiment_stats'] = {
-            str(k): int(v) for k, v in svals.value_counts().items()
-        }
-    else:
-        report['sentiment_stats'] = {}
-
-    if text_col:
-        sample_reviews = udf[text_col].dropna().astype(str).head(8).tolist()
-        sample_text    = ' '.join(sample_reviews)[:800]
-        prompt = (
-            'You are a data analyst reviewing customer feedback.\n'
-            f'Reviews: {sample_text}\n\n'
-            'Summarise: (1) overall satisfaction '
-            '(2) common praise (3) common complaints (4) product quality.\n'
-            'Answer:'
-        )
-        inputs = _tok(
-            prompt, return_tensors='pt',
-            max_length=512, truncation=True
-        )
-        with torch.no_grad():
-            out = _model.generate(
-                input_ids=inputs['input_ids'],
-                attention_mask=inputs['attention_mask'],
-                max_new_tokens=120, num_beams=2,
-                early_stopping=True, no_repeat_ngram_size=3
-            )
-        report['ai_summary'] = _tok.decode(
-            out[0], skip_special_tokens=True
-        ).strip()
-    else:
-        report['ai_summary'] = 'No text column detected for analysis.'
-
-    return report
-
-@app_ui.post('/search')
-async def search(req: SearchRequest):
-    t0 = time.time()
-    try:
-        answer = None
-        if req.mode == 'semantic':
-            results = _semantic(req.query, req.k)
-        elif req.mode == 'rag':
-            results, answer = _rag(req.query, req.k, req.alpha)
-        else:
-            results = _hybrid(req.query, req.k, req.alpha)
-        sent_dist = {}
-        rating_dist = {}
-        for r in results:
-            sent_dist[r['sentiment']]          = sent_dist.get(r['sentiment'], 0) + 1
-            rating_dist[str(r['rating'])]       = rating_dist.get(str(r['rating']), 0) + 1
-        return {
-            'results'    : results,
-            'answer'     : answer,
-            'latency'    : round(time.time()-t0, 2),
-            'mode'       : req.mode,
-            'sent_dist'  : sent_dist,
-            'rating_dist': rating_dist,
-        }
-    except Exception as e:
-        return {'results':[], 'answer':None, 'latency':0, 'mode':req.mode, 'error':str(e)}
-
-@app_ui.get('/similar/{doc_id}')
-async def similar(doc_id: int, k: int = 5):
-    try:
-        return {'results': _similar(doc_id, k)}
-    except Exception as e:
-        return {'results': [], 'error': str(e)}
-
-@app_ui.post('/compare')
-async def compare(req: CompareRequest):
-    try:
-        def get_doc(did):
-            if did < 0 or did >= len(df_sample): return None
-            row = df_sample.iloc[did]
-            return {
-                'doc_id'   : int(did),
-                'product'  : str(row[COL_PRODUCT]),
-                'review'   : str(row[COL_REVIEW]),
-                'rating'   : int(row[COL_RATE]),
-                'sentiment': str(row[COL_SENTIMENT]),
-                'price'    : int(row[COL_PRICE]),
-                'summary'  : str(row[COL_SUMMARY]),
-            }
-        a = get_doc(req.doc_id_a)
-        b = get_doc(req.doc_id_b)
-        if not a or not b:
-            return {'error': 'Invalid doc IDs'}
-        prompt = (f'Compare these two products based on customer reviews:\n'
-                  f'Product A: {a["product"]}\nReview A: {a["review"][:200]}\n'
-                  f'Product B: {b["product"]}\nReview B: {b["review"][:200]}\n'
-                  f'Which is better and why? Answer in 3 sentences:')
-        inputs = _tok(prompt, return_tensors='pt', max_length=512, truncation=True)
-        with torch.no_grad():
-            out = _model.generate(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'],
-                                  max_new_tokens=120, num_beams=2, early_stopping=True, no_repeat_ngram_size=3)
-        verdict = _tok.decode(out[0], skip_special_tokens=True).strip()
-        return {'doc_a': a, 'doc_b': b, 'verdict': verdict}
-    except Exception as e:
-        return {'error': str(e)}
-
-@app_ui.get('/autocomplete')
-async def autocomplete(q: str = ''):
-    if not q or len(q) < 2:
-        return {'suggestions': []}
-    ql = q.lower()
-    matches = [w for w in _autocomplete_corpus if w.startswith(ql)][:8]
-    return {'suggestions': matches}
 
 @app_ui.post('/analyse')
 async def analyse(file: UploadFile = File(...)):
     try:
-        content = await file.read()
+        MAX_SIZE_MB   = 50
+        ALLOWED_TYPES = {'.csv', '.xlsx', '.xls', '.json'}
         fname = file.filename.lower()
-        if fname.endswith('.csv'):    udf = pd.read_csv(io.BytesIO(content))
-        elif fname.endswith(('.xlsx','.xls')): udf = pd.read_excel(io.BytesIO(content))
-        elif fname.endswith('.json'): udf = pd.read_json(io.BytesIO(content))
-        else: return {'error': 'Use CSV, Excel, or JSON'}
-        if len(udf) == 0: return {'error': 'File is empty'}
+        ext   = os.path.splitext(fname)[1]
+        if ext not in ALLOWED_TYPES:
+            return {'success': False, 'error': 'File type not allowed. Use CSV, Excel or JSON.'}
+        content = await file.read()
+        if len(content) > MAX_SIZE_MB * 1024 * 1024:
+            return {'success': False, 'error': f'File too large. Max {MAX_SIZE_MB}MB.'}
+        if len(content) == 0:
+            return {'success': False, 'error': 'File is empty.'}
+        if ext == '.csv':
+            udf = pd.read_csv(io.BytesIO(content))
+        elif ext in ('.xlsx', '.xls'):
+            udf = pd.read_excel(io.BytesIO(content))
+        else:
+            udf = pd.read_json(io.BytesIO(content))
+        if len(udf) == 0:
+            return {'success': False, 'error': 'No data rows found.'}
+        if len(udf) > 500000:
+            return {'success': False, 'error': 'Too many rows. Max 500,000.'}
+        udf.columns = [str(c)[:50] for c in udf.columns]
+        for col in udf.select_dtypes(include='object').columns:
+            udf[col] = udf[col].astype(str).str[:1000]
         report = {}
         report['rows']    = int(len(udf))
         report['columns'] = list(udf.columns)
         report['missing'] = {c: int(udf[c].isna().sum()) for c in udf.columns}
-        text_col = next((c for c in ['review','Review','text','Text','comment','Comment','description','feedback'] if c in udf.columns), None)
+        text_col = next((c for c in ['review','Review','text','Text','comment',
+                                      'Comment','description','feedback','body']
+                         if c in udf.columns), None)
         if text_col is None:
             obj = udf.select_dtypes(include='object').columns
             text_col = obj[0] if len(obj) > 0 else None
-        rating_col   = next((c for c in ['rating','Rating','rate','Rate','stars','score'] if c in udf.columns), None)
-        sent_col     = next((c for c in ['sentiment','Sentiment','label','Label'] if c in udf.columns), None)
+        rating_col = next((c for c in ['rating','Rating','rate','Rate','stars','score','Score']
+                           if c in udf.columns), None)
+        sent_col   = next((c for c in ['sentiment','Sentiment','label','Label']
+                           if c in udf.columns), None)
         report['detected'] = {'text_column': text_col, 'rating_column': rating_col, 'sentiment_column': sent_col}
         if text_col:
             lengths = udf[text_col].dropna().astype(str).apply(len)
             words   = udf[text_col].dropna().astype(str).apply(lambda x: len(x.split()))
-            report['text_stats'] = {'avg_length': round(float(lengths.mean()),1), 'max_length': int(lengths.max()),
-                                     'avg_words': round(float(words.mean()),1), 'total_non_empty': int((udf[text_col].notna() & (udf[text_col].astype(str).str.len()>0)).sum())}
-            wf = Counter(w for w in re.sub(r'[^a-z\s]','', ' '.join(udf[text_col].dropna().astype(str).str.lower().tolist())).split()
+            report['text_stats'] = {
+                'avg_length'     : round(float(lengths.mean()), 1),
+                'max_length'     : int(lengths.max()),
+                'min_length'     : int(lengths.min()),
+                'avg_words'      : round(float(words.mean()), 1),
+                'total_non_empty': int((udf[text_col].notna() &
+                                       (udf[text_col].astype(str).str.len() > 0)).sum())
+            }
+            all_words = ' '.join(udf[text_col].dropna().astype(str).str.lower().tolist())
+            wf = Counter(w for w in re.sub(r'[^a-z\s]', '', all_words).split()
                          if w not in STOPWORDS and len(w) > 2)
-            report['top_words'] = [{'word':w,'count':c} for w,c in wf.most_common(15)]
+            report['top_words'] = [{'word': w, 'count': c} for w, c in wf.most_common(15)]
         else:
-            report['text_stats'] = {}; report['top_words'] = []
+            report['text_stats'] = {}
+            report['top_words']  = []
         if rating_col:
             rv = pd.to_numeric(udf[rating_col], errors='coerce').dropna()
-            report['rating_stats'] = {'avg': round(float(rv.mean()),2),
-                                       'distribution': {str(int(k)):int(v) for k,v in rv.value_counts().sort_index().items()}}
-        else: report['rating_stats'] = {}
+            report['rating_stats'] = {
+                'avg': round(float(rv.mean()), 2),
+                'distribution': {str(int(k)): int(v) for k, v in rv.value_counts().sort_index().items()}
+            }
+        else:
+            report['rating_stats'] = {}
         if sent_col:
             sv = udf[sent_col].dropna().astype(str).str.lower().str.strip()
-            report['sentiment_stats'] = {str(k):int(v) for k,v in sv.value_counts().items()}
-        else: report['sentiment_stats'] = {}
+            report['sentiment_stats'] = {str(k): int(v) for k, v in sv.value_counts().items()}
+        else:
+            report['sentiment_stats'] = {}
         if text_col:
             sample_text = ' '.join(udf[text_col].dropna().astype(str).head(8).tolist())[:800]
-            prompt = f'Analyse these reviews and summarise key themes and insights:\n{sample_text}\nSummary:'
+            prompt = (
+                'You are a data analyst. Read these customer reviews carefully.\n'
+                f'Reviews: {sample_text}\n\n'
+                'Summarise: (1) overall satisfaction (2) common praise '
+                '(3) common complaints (4) product quality.\nAnswer:'
+            )
             inputs = _tok(prompt, return_tensors='pt', max_length=512, truncation=True)
             with torch.no_grad():
-                out = _model.generate(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'],
-                                      max_new_tokens=120, num_beams=2, early_stopping=True, no_repeat_ngram_size=3)
+                out = _model.generate(
+                    input_ids=inputs['input_ids'],
+                    attention_mask=inputs['attention_mask'],
+                    max_new_tokens=120, num_beams=2,
+                    early_stopping=True, no_repeat_ngram_size=3
+                )
             report['ai_summary'] = _tok.decode(out[0], skip_special_tokens=True).strip()
         else:
             report['ai_summary'] = 'No text column detected.'
-        return {'success': True, 'filename': file.filename, 'report': report}
+        return {'success': True, 'filename': file.filename[:100], 'report': report}
+    except pd.errors.ParserError:
+        return {'success': False, 'error': 'Could not parse file. Check it is valid CSV/Excel/JSON.'}
     except Exception as e:
-        return {'success': False, 'error': str(e)}
+        return {'success': False, 'error': f'Analysis failed: {str(e)[:300]}'}
+
 
 @app_ui.get('/stats')
 async def stats():
@@ -1242,15 +1090,6 @@ function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').repl
 </html>"""
 
 
-@app_ui.get('/', response_class=HTMLResponse)
-async def root():
-    return HTML_UI
-
-
-@app_ui.get('/', response_class=HTMLResponse)
-async def root():
-    return HTML_UI
-
 
 if __name__ == '__main__':
     uvicorn.run(app_ui, host='0.0.0.0', port=7860)