Update app.py

app.py

@@ -185,9 +185,9 @@ class IntelligentAgent:
         
         return "\n\n".join(media_content) if media_content else ""
 
-    def _should_search(self, question: str, media_context: str) -> bool:
+    def _should_search(self, question: str, media_context: str = "") -> bool:
         """
-        Use LLM to determine if search is needed for the question.
+        Use LLM to determine if search is needed for the question, considering media context.
         Returns True if search is recommended, False otherwise.
         """
         decision_prompt = f"""Analyze this question and decide if it requires real-time information, recent data, or specific facts that might not be in your training data.
@@ -208,14 +208,18 @@ SEARCH IS NOT NEEDED for:
 - Definitions of well-established concepts
 - How-to instructions for common tasks
 - Creative writing or opinion-based responses
+- Questions that can be answered from attached media content
 
 Question: "{question}"
 
+{f"Media Context Available: {media_context[:500]}..." if media_context else "No media context available."}
+
 Respond with only "SEARCH" or "NO_SEARCH" followed by a brief reason (max 20 words).
 
 Example responses:
 - "SEARCH - Current weather data needed"
 - "NO_SEARCH - Mathematical concept, general knowledge sufficient"
+- "NO_SEARCH - Can be answered from attached image content"
 """
 
         try:
@@ -236,12 +240,16 @@ Example responses:
             # Default to search if decision fails
             return True
 
-    def _answer_with_llm(self, question: str) -> str:
+    def _answer_with_llm(self, question: str, media_context: str = "") -> str:
         """
-        Generate answer using LLM without search.
+        Generate answer using LLM without search, considering media context.
         """
+        context_section = f"\n\nMedia Context:\n{media_context}" if media_context else ""
+        
         answer_prompt = f"""You are a general AI assistant. I will ask you a question. YOUR ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
 
+{context_section}
+
 Question: {question}
 
 Answer:"""
@@ -253,9 +261,9 @@ Answer:"""
         except Exception as e:
             return f"Sorry, I encountered an error generating the response: {e}"
 
-    def _answer_with_search(self, question: str) -> str:
+    def _answer_with_search(self, question: str, media_context: str = "") -> str:
         """
-        Generate answer using search results and LLM.
+        Generate answer using search results and LLM, considering media context.
         """
         try:
             # Perform search
@@ -264,17 +272,11 @@ Answer:"""
             
             if self.debug:
                 print(f"Search results type: {type(search_results)}")
-                #print(f"Search results: {search_results}")
             
             if not search_results:
-                return "No search results found. Let me try to answer based on my knowledge:\n\n" + self._answer_with_llm(question)
+                return "No search results found. Let me try to answer based on my knowledge:\n\n" + self._answer_with_llm(question, media_context)
 
             # Format search results - handle different result formats
-            if self.debug:
-                print(f"First result type: {type(search_results[0]) if search_results else 'None'}")
-                print(f"First result: {search_results[0] if search_results else 'None'}")
-            
-            # If search_results is a string, use it directly
             if isinstance(search_results, str):
                 search_context = search_results
             else:
@@ -287,22 +289,25 @@ Answer:"""
                         link = result.get("link", "")
                         formatted_results.append(f"Title: {title}\nContent: {snippet}\nSource: {link}")
                     elif isinstance(result, str):
-                        # If result is a string, use it directly
                         formatted_results.append(result)
                     else:
-                        # Handle other formats
                         formatted_results.append(str(result))
                 
                 search_context = "\n\n".join(formatted_results)
 
-            # Generate answer using search context
+            # Generate answer using search context and media context
+            context_section = f"\n\nMedia Context:\n{media_context}" if media_context else ""
+            
             answer_prompt = f"""You are a general AI assistant. I will ask you a question. Based on the search results below, provide an answer to the question. If the search results don't fully answer the question, you can supplement with your general knowledge.
             Your ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+
 Question: {question}
 
 Search Results:
 {search_context}
 
+{context_section}
+
 Answer:"""
 
             try:
@@ -331,29 +336,37 @@ Answer:"""
                     return "Search completed but no usable results found."
 
         except Exception as e:
-            return f"Search failed: {e}. Let me try to answer based on my knowledge:\n\n" + self._answer_with_llm(question)
+            return f"Search failed: {e}. Let me try to answer based on my knowledge:\n\n" + self._answer_with_llm(question, media_context)
 
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, image_files: List[str] = None, audio_files: List[str] = None) -> str:
         """
-        Main entry point - decide whether to search and generate appropriate response.
+        Main entry point - process media files, decide whether to search, and generate appropriate response.
         """
         if self.debug:
             print(f"Agent received question: {question}")
+            print(f"Image files: {image_files}")
+            print(f"Audio files: {audio_files}")
 
         # Early validation
         if not question or not question.strip():
             return "Please provide a valid question."
 
         try:
+            # Process media files first
+            media_context = self._process_media_files(image_files, audio_files)
+            
+            if self.debug and media_context:
+                print(f"Media context: {media_context[:200]}...")
+
             # Decide whether to search
-            if self._should_search(question):
+            if self._should_search(question, media_context):
                 if self.debug:
                     print("Using search-based approach")
-                answer = self._answer_with_search(question)
+                answer = self._answer_with_search(question, media_context)
             else:
                 if self.debug:
                     print("Using LLM-only approach")
-                answer = self._answer_with_llm(question)
+                answer = self._answer_with_llm(question, media_context)
 
         except Exception as e:
             answer = f"Sorry, I encountered an error: {e}"
@@ -474,210 +487,4 @@ def start_answer_generation(model_choice: str):
     
     return f"Answer generation started using {model_choice}. Check progress."
 
-def get_generation_progress():
-    """
-    Get the current progress of answer generation.
-    """
-    if not processing_status["is_processing"] and processing_status["progress"] == 0:
-        return "Not started"
-    
-    if processing_status["is_processing"]:
-        progress = processing_status["progress"]
-        total = processing_status["total"]
-        status_msg = f"Generating answers... {progress}/{total} completed"
-        return status_msg
-    else:
-        # Generation completed
-        if cached_answers:
-            # Create DataFrame with results
-            display_data = []
-            for task_id, data in cached_answers.items():
-                display_data.append({
-                    "Task ID": task_id,
-                    "Question": data["question"][:100] + "..." if len(data["question"]) > 100 else data["question"],
-                    "Generated Answer": data["answer"][:200] + "..." if len(data["answer"]) > 200 else data["answer"]
-                })
-            
-            df = pd.DataFrame(display_data)
-            status_msg = f"Answer generation completed! {len(cached_answers)} answers ready for submission."
-            return status_msg, df
-        else:
-            return "Answer generation completed but no answers were generated."
-
-def submit_cached_answers(profile: gr.OAuthProfile | None):
-    """
-    Submit the cached answers to the evaluation API.
-    """
-    global cached_answers
-    
-    if not profile:
-        return "Please log in to Hugging Face first.", None
-    
-    if not cached_answers:
-        return "No cached answers available. Please generate answers first.", None
-    
-    username = profile.username
-    space_id = os.getenv("SPACE_ID")
-    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "Unknown"
-    
-    # Prepare submission payload
-    answers_payload = []
-    for task_id, data in cached_answers.items():
-        answers_payload.append({
-            "task_id": task_id,
-            "submitted_answer": data["answer"]
-        })
-    
-    submission_data = {
-        "username": username.strip(),
-        "agent_code": agent_code,
-        "answers": answers_payload
-    }
-    
-    # Submit to API
-    api_url = DEFAULT_API_URL
-    submit_url = f"{api_url}/submit"
-    
-    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
-    
-    try:
-        response = requests.post(submit_url, json=submission_data, timeout=60)
-        response.raise_for_status()
-        result_data = response.json()
-        
-        final_status = (
-            f"Submission Successful!\n"
-            f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', 'N/A')}% "
-            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
-            f"Message: {result_data.get('message', 'No message received.')}"
-        )
-        
-        # Create results DataFrame
-        results_log = []
-        for task_id, data in cached_answers.items():
-            results_log.append({
-                "Task ID": task_id,
-                "Question": data["question"],
-                "Submitted Answer": data["answer"]
-            })
-        
-        results_df = pd.DataFrame(results_log)
-        return final_status, results_df
-        
-    except requests.exceptions.HTTPError as e:
-        error_detail = f"Server responded with status {e.response.status_code}."
-        try:
-            error_json = e.response.json()
-            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
-        except:
-            error_detail += f" Response: {e.response.text[:500]}"
-        return f"Submission Failed: {error_detail}", None
-        
-    except requests.exceptions.Timeout:
-        return "Submission Failed: The request timed out.", None
-        
-    except Exception as e:
-        return f"Submission Failed: {e}", None
-
-def clear_cache():
-    """
-    Clear all cached data.
-    """
-    global cached_answers, cached_questions, processing_status
-    cached_answers = {}
-    cached_questions = []
-    processing_status = {"is_processing": False, "progress": 0, "total": 0}
-    return "Cache cleared successfully.", None
-
-# --- Enhanced Gradio Interface ---
-with gr.Blocks(title="Intelligent Agent with Conditional Search") as demo:
-    gr.Markdown("# Intelligent Agent with Conditional Search")
-    gr.Markdown("This agent uses an LLM to decide when search is needed, optimizing for both accuracy and efficiency.")
-
-    with gr.Row():
-        gr.LoginButton()
-        clear_btn = gr.Button("Clear Cache", variant="secondary")
-
-    with gr.Tab("Step 1: Fetch Questions"):
-        gr.Markdown("### Fetch Questions from API")
-        fetch_btn = gr.Button("Fetch Questions", variant="primary")
-        fetch_status = gr.Textbox(label="Fetch Status", lines=2, interactive=False)
-        questions_table = gr.DataFrame(label="Available Questions", wrap=True)
-        
-        fetch_btn.click(
-            fn=fetch_questions,
-            outputs=[fetch_status, questions_table]
-        )
-
-    with gr.Tab("Step 2: Generate Answers"):
-        gr.Markdown("### Generate Answers with Intelligent Search Decision")
-        
-        with gr.Row():
-            model_choice = gr.Dropdown(
-                choices=["Llama 3.1 8B", "Mistral 7B"],
-                value="Llama 3.1 8B",
-                label="Select Model"
-            )
-            generate_btn = gr.Button("Start Answer Generation", variant="primary")
-            refresh_btn = gr.Button("Refresh Progress", variant="secondary")
-        
-        generation_status = gr.Textbox(label="Generation Status", lines=2, interactive=False)
-        
-        generate_btn.click(
-            fn=start_answer_generation,
-            inputs=[model_choice],
-            outputs=[generation_status]
-        )
-        
-        refresh_btn.click(
-            fn=get_generation_progress,
-            outputs=[generation_status]
-        )
-
-    with gr.Tab("Step 3: Submit Results"):
-        gr.Markdown("### Submit Generated Answers")
-        submit_btn = gr.Button("Submit Cached Answers", variant="primary")
-        submission_status = gr.Textbox(label="Submission Status", lines=5, interactive=False)
-        final_results = gr.DataFrame(label="Final Submission Results", wrap=True)
-        
-        submit_btn.click(
-            fn=submit_cached_answers,
-            outputs=[submission_status, final_results]
-        )
-
-    # Clear cache functionality
-    clear_btn.click(
-        fn=clear_cache,
-        outputs=[fetch_status, questions_table]
-    )
-
-    # Auto-refresh progress every 5 seconds when generation is active
-    demo.load(
-        fn=get_generation_progress,
-        outputs=[generation_status]
-    )
-
-if __name__ == "__main__":
-    print("\n" + "-"*30 + " Intelligent Agent Starting " + "-"*30)
-    
-    space_host_startup = os.getenv("SPACE_HOST")
-    space_id_startup = os.getenv("SPACE_ID")
-
-    if space_host_startup:
-        print(f"✅ SPACE_HOST found: {space_host_startup}")
-        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
-    else:
-        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
-
-    if space_id_startup:
-        print(f"✅ SPACE_ID found: {space_id_startup}")
-        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
-    else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
-
-    print("-"*(60 + len(" Intelligent Agent Starting ")) + "\n")
-
-    print("Launching Intelligent Agent Interface...")
-    demo.launch(debug=True, share=False)
+def get_generation_prog