attempts to solve the api_key issue for huggingface , settings not appearing , set settings for audio , adds modal gpu , speech to text with mic input addon, adds graphs

pyproject.toml

@@ -14,7 +14,7 @@ dependencies = [
     "beautifulsoup4>=4.12", # HTML parsing
     "xmltodict>=0.13", # PubMed XML -> dict
     "huggingface-hub>=0.20.0", # Hugging Face Inference API
-    "gradio[mcp]>=6.0.0", # Chat interface with MCP server support (6.0 required for css in launch())
+    "gradio[mcp,oauth]>=6.0.0", # Chat interface with MCP server support (6.0 required for css in launch())
     "python-dotenv>=1.0", # .env loading
     "tenacity>=8.2", # Retry logic
     "structlog>=24.1", # Structured logging
@@ -26,15 +26,21 @@ dependencies = [
     "llama-index-llms-huggingface-api>=0.6.1",
     "llama-index-vector-stores-chroma>=0.5.3",
     "llama-index>=0.14.8",
-    # Audio/Image processing
     "gradio-client>=1.0.0", # For STT/OCR API calls
     "soundfile>=0.12.0", # For audio file I/O
     "pillow>=10.0.0", # For image processing
-    # TTS dependencies (for Modal GPU TTS)
     "torch>=2.0.0", # Required by Kokoro TTS
-    "transformers>=4.30.0", # Required by Kokoro TTS
+    "transformers>=4.57.2", # Required by Kokoro TTS
     "modal>=0.63.0", # Required for TTS GPU execution
-    # Note: Kokoro is installed in Modal image from: git+https://github.com/hexgrad/kokoro.git
+    "tokenizers>=0.22.0,<=0.23.0",
+    "rpds-py>=0.29.0",
+    "pydantic-ai-slim[huggingface]>=0.0.18",
+    "agent-framework-core>=1.0.0b251120,<2.0.0",
+    "chromadb>=0.4.0",
+    "sentence-transformers>=2.2.0",
+    "numpy<2.0",
+    "llama-index-llms-openai>=0.6.9",
+    "llama-index-embeddings-openai>=0.5.1",
 ]
 
 [project.optional-dependencies]
@@ -72,11 +78,11 @@ modal = [
     # Mario's Modal code execution + LlamaIndex RAG
     # Note: modal>=0.63.0 is now in main dependencies for TTS support
     "llama-index>=0.11.0",
-    "llama-index-llms-openai",
-    "llama-index-embeddings-openai",
+    "llama-index-llms-openai>=0.6.9",
+    "llama-index-embeddings-openai>=0.5.1",
     "llama-index-vector-stores-chroma",
     "chromadb>=0.4.0",
-    "numpy<2.0",  # chromadb compatibility: uses np.float_ removed in NumPy 2.0
+    "numpy<2.0", # chromadb compatibility: uses np.float_ removed in NumPy 2.0
 ]
 
 [build-system]

requirements.txt

@@ -43,6 +43,20 @@ llama-index-llms-huggingface>=0.6.1
 llama-index-llms-huggingface-api>=0.6.1
 llama-index-vector-stores-chroma>=0.5.3
 llama-index>=0.14.8
+
+# Audio/Image processing
+gradio-client>=1.0.0  # For STT/OCR API calls
+soundfile>=0.12.0  # For audio file I/O
+pillow>=10.0.0  # For image processing
+
+# TTS dependencies (for Modal GPU TTS)
+torch>=2.0.0  # Required by Kokoro TTS
+transformers>=4.30.0  # Required by Kokoro TTS
+modal>=0.63.0  # Required for TTS GPU execution
+# Note: Kokoro is installed in Modal image from: git+https://github.com/hexgrad/kokoro.git
+
+# Multi-agent orchestration (Advanced mode) - from optional magentic
+agent-framework-core>=1.0.0b251120,<2.0.0
 llama-index-llms-openai>=0.6.9
 llama-index-embeddings-openai>=0.5.1
 
@@ -57,5 +71,8 @@ numpy<2.0
 # Optional: Modal for code execution
 modal>=0.63.0
 
-# Pydantic AI with HuggingFace support
-pydantic-ai-slim[huggingface]>=0.0.18
+# LlamaIndex RAG - from optional modal
+llama-index-llms-openai
+llama-index-embeddings-openai
+
+pydantic-ai-slim[huggingface]>=0.0.18

src/app.py

@@ -501,16 +501,24 @@ async def research_agent(
     audio_input_data: tuple[int, np.ndarray] | None = None
 
     if isinstance(message, dict):
-        # MultimodalPostprocess format: {"text": str, "files": list[FileData]}
+        # MultimodalPostprocess format: {"text": str, "files": list[FileData], "audio": tuple | None}
         processed_text = message.get("text", "") or ""
         files = message.get("files", [])
+        # Check for audio input in message (Gradio may include it as a separate field)
+        audio_input_data = message.get("audio") or None
 
-        # Process multimodal input (images, audio files)
-        if files and settings.enable_image_input:
+        # Process multimodal input (images, audio files, audio input)
+        # Always process if we have files or audio input, not just when enable_image_input is True
+        if files or (audio_input_data is not None and settings.enable_audio_input):
             try:
                 multimodal_service = get_multimodal_service()
+                # Prepend audio/image text to original text (prepend_multimodal=True)
                 processed_text = await multimodal_service.process_multimodal_input(
-                    processed_text, files=files, hf_token=token_value
+                    processed_text,
+                    files=files,
+                    audio_input=audio_input_data,
+                    hf_token=token_value,
+                    prepend_multimodal=True,  # Prepend audio/image text to text input
                 )
             except Exception as e:
                 logger.warning("multimodal_processing_failed", error=str(e))
@@ -636,8 +644,8 @@ def create_demo() -> gr.Blocks:
             )
         
         # Create settings components
-        # Mode selector is visible in Settings accordion
-        # Model/provider selection hidden to avoid dropdown value mismatch errors
+        # Note: ChatInterface doesn't support additional_inputs_accordion parameter in Gradio 6.0
+        # Components are created outside accordion context to ensure they're accessible for additional_inputs
         mode_radio = gr.Radio(
             choices=["simple", "advanced", "iterative", "deep", "auto"],
             value="simple",
@@ -666,56 +674,59 @@ def create_demo() -> gr.Blocks:
             info="Enable graph-based workflow execution",
         )
 
-        # TTS Configuration (in Settings accordion)
-        with gr.Accordion("🎤 Audio Settings", open=False, visible=settings.enable_audio_output):
-            tts_voice_dropdown = gr.Dropdown(
-                choices=[
-                    "af_heart",
-                    "af_bella",
-                    "af_nicole",
-                    "af_aoede",
-                    "af_kore",
-                    "af_sarah",
-                    "af_nova",
-                    "af_sky",
-                    "af_alloy",
-                    "af_jessica",
-                    "af_river",
-                    "am_michael",
-                    "am_fenrir",
-                    "am_puck",
-                    "am_echo",
-                    "am_eric",
-                    "am_liam",
-                    "am_onyx",
-                    "am_santa",
-                    "am_adam",
-                ],
-                value=settings.tts_voice,
-                label="Voice",
-                info="Select TTS voice (American English voices: af_*, am_*)",
-            )
-            tts_speed_slider = gr.Slider(
-                minimum=0.5,
-                maximum=2.0,
-                value=settings.tts_speed,
-                step=0.1,
-                label="Speech Speed",
-                info="Adjust TTS speech speed (0.5x to 2.0x)",
-            )
-            tts_gpu_dropdown = gr.Dropdown(
-                choices=["T4", "A10", "A100", "L4", "L40S"],
-                value=settings.tts_gpu or "T4",
-                label="GPU Type",
-                info="Modal GPU type for TTS (T4 is cheapest, A100 is fastest). Note: GPU changes require app restart.",
-                visible=settings.modal_available,
-                interactive=False,  # GPU type set at function definition time, requires restart
-            )
-            enable_audio_output_checkbox = gr.Checkbox(
-                value=settings.enable_audio_output,
-                label="Enable Audio Output",
-                info="Generate audio responses using TTS",
-            )
+        # TTS Configuration components
+        # Note: These are created outside accordion to ensure accessibility for additional_inputs
+        # The ChatInterface will display them, but grouping in accordion is not supported via additional_inputs_accordion
+        tts_voice_dropdown = gr.Dropdown(
+            choices=[
+                "af_heart",
+                "af_bella",
+                "af_nicole",
+                "af_aoede",
+                "af_kore",
+                "af_sarah",
+                "af_nova",
+                "af_sky",
+                "af_alloy",
+                "af_jessica",
+                "af_river",
+                "am_michael",
+                "am_fenrir",
+                "am_puck",
+                "am_echo",
+                "am_eric",
+                "am_liam",
+                "am_onyx",
+                "am_santa",
+                "am_adam",
+            ],
+            value=settings.tts_voice,
+            label="TTS Voice",
+            info="Select TTS voice (American English voices: af_*, am_*)",
+            visible=settings.enable_audio_output,
+        )
+        tts_speed_slider = gr.Slider(
+            minimum=0.5,
+            maximum=2.0,
+            value=settings.tts_speed,
+            step=0.1,
+            label="TTS Speech Speed",
+            info="Adjust TTS speech speed (0.5x to 2.0x)",
+            visible=settings.enable_audio_output,
+        )
+        tts_gpu_dropdown = gr.Dropdown(
+            choices=["T4", "A10", "A100", "L4", "L40S"],
+            value=settings.tts_gpu or "T4",
+            label="TTS GPU Type",
+            info="Modal GPU type for TTS (T4 is cheapest, A100 is fastest). Note: GPU changes require app restart.",
+            visible=settings.modal_available and settings.enable_audio_output,
+            interactive=False,  # GPU type set at function definition time, requires restart
+        )
+        enable_audio_output_checkbox = gr.Checkbox(
+            value=settings.enable_audio_output,
+            label="Enable Audio Output",
+            info="Generate audio responses using TTS",
+        )
 
         # Hidden text components for model/provider (not dropdowns to avoid value mismatch)
         # These will be empty by default and use defaults in configure_orchestrator
@@ -787,7 +798,8 @@ def create_demo() -> gr.Blocks:
             ],
             cache_examples=False,  # CRITICAL: Disable example caching to prevent examples from running at startup
             # Examples will only run when user explicitly clicks them (after login)
-            additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=True, visible=True),
+            # Note: additional_inputs_accordion is not a valid parameter in Gradio 6.0 ChatInterface
+            # Components will be displayed in the order provided
             additional_inputs=[
                 mode_radio,
                 hf_model_dropdown,

src/services/multimodal_processing.py

@@ -36,6 +36,7 @@ class MultimodalService:
         files: list[FileData] | None = None,
         audio_input: tuple[int, Any] | None = None,
         hf_token: str | None = None,
+        prepend_multimodal: bool = True,
     ) -> str:
         """Process multimodal input (text + images + audio) and return combined text.
 
@@ -44,26 +45,24 @@ class MultimodalService:
             files: List of uploaded files (images, audio, etc.)
             audio_input: Audio input tuple (sample_rate, audio_array)
             hf_token: HuggingFace token for authenticated Gradio Spaces
+            prepend_multimodal: If True, prepend audio/image text to original text; otherwise append
 
         Returns:
             Combined text from all inputs
         """
+        multimodal_parts: list[str] = []
         text_parts: list[str] = []
 
-        # Add original text if present
-        if text and text.strip():
-            text_parts.append(text.strip())
-
-        # Process audio input
+        # Process audio input first
         if audio_input is not None and settings.enable_audio_input:
             try:
                 transcribed = await self.audio.process_audio_input(audio_input, hf_token=hf_token)
                 if transcribed:
-                    text_parts.append(f"[Audio transcription: {transcribed}]")
+                    multimodal_parts.append(transcribed)
             except Exception as e:
                 logger.warning("audio_processing_failed", error=str(e))
 
-        # Process uploaded files
+        # Process uploaded files (images and audio files)
         if files:
             for file_data in files:
                 file_path = file_data.path if isinstance(file_data, FileData) else str(file_data)
@@ -73,7 +72,7 @@ class MultimodalService:
                     try:
                         extracted_text = await self.ocr.extract_text(file_path, hf_token=hf_token)
                         if extracted_text:
-                            text_parts.append(f"[Image OCR: {extracted_text}]")
+                            multimodal_parts.append(extracted_text)
                     except Exception as e:
                         logger.warning("image_ocr_failed", file_path=file_path, error=str(e))
 
@@ -86,8 +85,20 @@ class MultimodalService:
                     except Exception as e:
                         logger.warning("audio_file_processing_failed", file_path=file_path, error=str(e))
 
+        # Add original text if present
+        if text and text.strip():
+            text_parts.append(text.strip())
+
+        # Combine parts based on prepend_multimodal flag
+        if prepend_multimodal:
+            # Prepend: multimodal content first, then original text
+            combined_parts = multimodal_parts + text_parts
+        else:
+            # Append: original text first, then multimodal content
+            combined_parts = text_parts + multimodal_parts
+
         # Combine all text parts
-        combined_text = "\n\n".join(text_parts) if text_parts else ""
+        combined_text = "\n\n".join(combined_parts) if combined_parts else ""
 
         logger.info(
             "multimodal_input_processed",

src/utils/config.py

@@ -140,6 +140,30 @@ class Settings(BaseSettings):
         description="Automatically ingest evidence into RAG",
     )
 
+    # Audio/TTS Configuration
+    enable_audio_input: bool = Field(
+        default=True,
+        description="Enable audio input (speech-to-text) in multimodal interface",
+    )
+    enable_audio_output: bool = Field(
+        default=True,
+        description="Enable audio output (text-to-speech) for responses",
+    )
+    tts_voice: str = Field(
+        default="af_heart",
+        description="TTS voice ID for Kokoro TTS (e.g., af_heart, am_michael)",
+    )
+    tts_speed: float = Field(
+        default=1.0,
+        ge=0.5,
+        le=2.0,
+        description="TTS speech speed multiplier (0.5x to 2.0x)",
+    )
+    tts_gpu: str | None = Field(
+        default=None,
+        description="Modal GPU type for TTS (T4, A10, A100, L4, L40S). None uses default T4.",
+    )
+
     @property
     def modal_available(self) -> bool:
         """Check if Modal credentials are configured."""

uv.lock

@@ -1108,30 +1108,39 @@ name = "deepcritical"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "agent-framework-core" },
     { name = "anthropic" },
     { name = "beautifulsoup4" },
+    { name = "chromadb" },
     { name = "duckduckgo-search" },
-    { name = "gradio", extra = ["mcp"] },
+    { name = "gradio", extra = ["mcp", "oauth"] },
     { name = "gradio-client" },
     { name = "httpx" },
     { name = "huggingface-hub" },
     { name = "limits" },
     { name = "llama-index" },
+    { name = "llama-index-embeddings-openai" },
     { name = "llama-index-llms-huggingface" },
     { name = "llama-index-llms-huggingface-api" },
+    { name = "llama-index-llms-openai" },
     { name = "llama-index-vector-stores-chroma" },
     { name = "modal" },
+    { name = "numpy" },
     { name = "openai" },
     { name = "pillow" },
     { name = "pydantic" },
     { name = "pydantic-ai" },
+    { name = "pydantic-ai-slim", extra = ["huggingface"] },
     { name = "pydantic-graph" },
     { name = "pydantic-settings" },
     { name = "python-dotenv" },
     { name = "requests" },
+    { name = "rpds-py" },
+    { name = "sentence-transformers" },
     { name = "soundfile" },
     { name = "structlog" },
     { name = "tenacity" },
+    { name = "tokenizers" },
     { name = "torch" },
     { name = "transformers" },
     { name = "xmltodict" },
@@ -1169,7 +1178,6 @@ modal = [
     { name = "llama-index-embeddings-openai" },
     { name = "llama-index-llms-openai" },
     { name = "llama-index-vector-stores-chroma" },
-    { name = "modal" },
     { name = "numpy" },
 ]
 
@@ -1181,23 +1189,27 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "agent-framework-core", specifier = ">=1.0.0b251120,<2.0.0" },
     { name = "agent-framework-core", marker = "extra == 'magentic'", specifier = ">=1.0.0b251120,<2.0.0" },
     { name = "anthropic", specifier = ">=0.18.0" },
     { name = "beautifulsoup4", specifier = ">=4.12" },
+    { name = "chromadb", specifier = ">=0.4.0" },
     { name = "chromadb", marker = "extra == 'embeddings'", specifier = ">=0.4.0" },
     { name = "chromadb", marker = "extra == 'modal'", specifier = ">=0.4.0" },
     { name = "duckduckgo-search", specifier = ">=5.0" },
-    { name = "gradio", extras = ["mcp"], specifier = ">=6.0.0" },
+    { name = "gradio", extras = ["mcp", "oauth"], specifier = ">=6.0.0" },
     { name = "gradio-client", specifier = ">=1.0.0" },
     { name = "httpx", specifier = ">=0.27" },
     { name = "huggingface-hub", specifier = ">=0.20.0" },
     { name = "limits", specifier = ">=3.0" },
     { name = "llama-index", specifier = ">=0.14.8" },
     { name = "llama-index", marker = "extra == 'modal'", specifier = ">=0.11.0" },
-    { name = "llama-index-embeddings-openai", marker = "extra == 'modal'" },
+    { name = "llama-index-embeddings-openai", specifier = ">=0.5.1" },
+    { name = "llama-index-embeddings-openai", marker = "extra == 'modal'", specifier = ">=0.5.1" },
     { name = "llama-index-llms-huggingface", specifier = ">=0.6.1" },
     { name = "llama-index-llms-huggingface-api", specifier = ">=0.6.1" },
-    { name = "llama-index-llms-openai", marker = "extra == 'modal'" },
+    { name = "llama-index-llms-openai", specifier = ">=0.6.9" },
+    { name = "llama-index-llms-openai", marker = "extra == 'modal'", specifier = ">=0.6.9" },
     { name = "llama-index-vector-stores-chroma", specifier = ">=0.5.3" },
     { name = "llama-index-vector-stores-chroma", marker = "extra == 'modal'" },
     { name = "mkdocs", marker = "extra == 'dev'", specifier = ">=1.6.0" },
@@ -1206,8 +1218,8 @@ requires-dist = [
     { name = "mkdocs-mermaid2-plugin", marker = "extra == 'dev'", specifier = ">=1.1.0" },
     { name = "mkdocs-minify-plugin", marker = "extra == 'dev'", specifier = ">=0.8.0" },
     { name = "modal", specifier = ">=0.63.0" },
-    { name = "modal", marker = "extra == 'modal'", specifier = ">=0.63.0" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" },
+    { name = "numpy", specifier = "<2.0" },
     { name = "numpy", marker = "extra == 'embeddings'", specifier = "<2.0" },
     { name = "numpy", marker = "extra == 'modal'", specifier = "<2.0" },
     { name = "openai", specifier = ">=1.0.0" },
@@ -1215,6 +1227,7 @@ requires-dist = [
     { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.7" },
     { name = "pydantic", specifier = ">=2.7" },
     { name = "pydantic-ai", specifier = ">=0.0.16" },
+    { name = "pydantic-ai-slim", extras = ["huggingface"], specifier = ">=0.0.18" },
     { name = "pydantic-graph", specifier = ">=1.22.0" },
     { name = "pydantic-settings", specifier = ">=2.2" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
@@ -1225,13 +1238,16 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.0" },
     { name = "requests", specifier = ">=2.32.5" },
     { name = "respx", marker = "extra == 'dev'", specifier = ">=0.21" },
+    { name = "rpds-py", specifier = ">=0.29.0" },
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" },
+    { name = "sentence-transformers", specifier = ">=2.2.0" },
     { name = "sentence-transformers", marker = "extra == 'embeddings'", specifier = ">=2.2.0" },
     { name = "soundfile", specifier = ">=0.12.0" },
     { name = "structlog", specifier = ">=24.1" },
     { name = "tenacity", specifier = ">=8.2" },
+    { name = "tokenizers", specifier = ">=0.22.0,<=0.23.0" },
     { name = "torch", specifier = ">=2.0.0" },
-    { name = "transformers", specifier = ">=4.30.0" },
+    { name = "transformers", specifier = ">=4.57.2" },
     { name = "typer", marker = "extra == 'dev'", specifier = ">=0.9.0" },
     { name = "xmltodict", specifier = ">=0.13" },
 ]
@@ -1748,6 +1764,10 @@ mcp = [
     { name = "mcp" },
     { name = "pydantic" },
 ]
+oauth = [
+    { name = "authlib" },
+    { name = "itsdangerous" },
+]
 
 [[package]]
 name = "gradio-client"
@@ -2160,6 +2180,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/32/4b/b99e37f88336009971405cbb7630610322ed6fbfa31e1d7ab3fbf3049a2d/invoke-2.2.1-py3-none-any.whl", hash = "sha256:2413bc441b376e5cd3f55bb5d364f973ad8bdd7bf87e53c79de3c11bf3feecc8", size = 160287, upload-time = "2025-10-11T00:36:33.703Z" },
 ]
 
+[[package]]
+name = "itsdangerous"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
+]
+
 [[package]]
 name = "jaraco-classes"
 version = "3.4.0"