Upload 3 files

hub_dashboard.html

@@ -1436,6 +1436,60 @@ td.rc {
   color: var(--cyan) !important;
 }
 
+/* ════════════════════════════════════════════════════════════════════════════
+   TRAINING PANEL KPI CELLS
+════════════════════════════════════════════════════════════════════════════ */
+.training-kpi-cell {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  padding: 20px var(--sp-lg);
+  gap: 6px;
+  text-align: center;
+  position: relative;
+  transition: background 0.3s ease;
+}
+.training-kpi-cell:hover {
+  background: rgba(255,255,255,0.02);
+}
+.training-kpi-label {
+  font-size: 0.62rem;
+  font-weight: 800;
+  letter-spacing: 0.2em;
+  text-transform: uppercase;
+  color: var(--t3);
+}
+.training-kpi-value {
+  font-size: 1.65rem;
+  font-weight: 800;
+  letter-spacing: -0.02em;
+  line-height: 1;
+  font-variant-numeric: tabular-nums;
+  transition: all 0.4s ease;
+}
+.training-kpi-sub {
+  font-size: 0.62rem;
+  color: var(--t3);
+  font-weight: 500;
+  letter-spacing: 0.04em;
+  min-height: 14px;
+}
+/* Terminal scrollbar styling */
+#training-log-terminal::-webkit-scrollbar { width: 4px; }
+#training-log-terminal::-webkit-scrollbar-thumb { background: rgba(0,212,255,0.2); border-radius: 4px; }
+#training-log-terminal::-webkit-scrollbar-track { background: transparent; }
+
+/* Log line colorings */
+.tlog-debug  { color: var(--t3); }
+.tlog-info   { color: var(--t2); }
+.tlog-signal { color: var(--cyan); }
+.tlog-trade  { color: var(--amber); }
+.tlog-error  { color: var(--red); }
+.tlog-warn   { color: var(--amber); }
+.tlog-ts     { color: var(--t4); margin-right: 6px; }
+.tlog-badge  { font-weight: 800; margin-right: 4px; }
+
 </style>
 
 <script>
@@ -1653,22 +1707,68 @@ td.rc {
         <div id="detail-panel"></div>
       </div>
 
+      <!-- TRAINING PANEL -->
       <div id="bottom-grid">
-        <div class="panel" id="feed-panel">
+        <div class="panel" id="training-panel">
           <div class="panel-hd">
             <div class="panel-hd-left">
-              <div class="panel-hd-pip" style="background:#a855f7;box-shadow:0 0 10px rgba(168,85,247,0.5)"></div>
-              <span class="panel-hd-label">Live Operation Feed</span>
+              <div class="panel-hd-pip" style="background:var(--cyan);box-shadow:0 0 10px var(--cyan-glow)"></div>
+              <span class="panel-hd-label">Training Logs</span>
+            </div>
+            <div style="display:flex;align-items:center;gap:12px">
+              <span class="panel-hd-meta" id="training-meta">awaiting training stream…</span>
+              <div style="display:flex;align-items:center;gap:6px;padding:4px 10px;border-radius:var(--pill-r);border:1px solid rgba(0,212,255,0.2);background:rgba(0,212,255,0.06)">
+                <div class="pring live" id="training-dot"><div class="core"></div></div>
+                <span style="font-size:0.6rem;font-weight:800;letter-spacing:0.12em;color:var(--cyan)">LIVE</span>
+              </div>
             </div>
-            <span class="panel-hd-meta" id="feed-meta">syncing...</span>
           </div>
-          <div class="feed-cols hd">
-            <span>Target / T-Minus</span>
-            <span>Vector</span>
-            <span style="text-align:right">Delta</span>
+
+          <!-- Terminal log stream -->
+          <div id="training-log-terminal" style="
+            font-family:'SF Mono','Fira Code','Cascadia Code',monospace;
+            font-size:0.72rem;
+            line-height:1.65;
+            color:var(--t2);
+            background:rgba(0,0,0,0.5);
+            border-bottom:1px solid rgba(255,255,255,0.04);
+            padding:16px 20px;
+            height:200px;
+            overflow-y:auto;
+            letter-spacing:0.01em;
+            scrollbar-width:thin;
+            scrollbar-color:rgba(0,212,255,0.2) transparent;
+          ">
+            <div style="color:var(--t3);font-style:italic">Awaiting training data stream…</div>
           </div>
-          <div id="feed-body">
-            <div class="feed-cols row"><div class="f-info"><span class="f-asset" style="color:var(--t4)">Standing by...</span></div></div>
+
+          <!-- KPI strip: Loss · Learning Rate · Step · Assets -->
+          <div style="display:grid;grid-template-columns:repeat(4,1fr);border-top:1px solid rgba(255,255,255,0.04)">
+
+            <div class="training-kpi-cell" style="border-right:1px solid rgba(255,255,255,0.04)">
+              <div class="training-kpi-label">Loss</div>
+              <div class="training-kpi-value" id="trn-loss" style="color:var(--red);text-shadow:0 0 12px rgba(255,68,102,0.45)">—</div>
+              <div class="training-kpi-sub" id="trn-loss-trend"></div>
+            </div>
+
+            <div class="training-kpi-cell" style="border-right:1px solid rgba(255,255,255,0.04)">
+              <div class="training-kpi-label">Learning Rate</div>
+              <div class="training-kpi-value" id="trn-lr" style="color:var(--amber);text-shadow:0 0 12px rgba(255,170,0,0.4)">—</div>
+              <div class="training-kpi-sub" id="trn-lr-sub">scheduler active</div>
+            </div>
+
+            <div class="training-kpi-cell" style="border-right:1px solid rgba(255,255,255,0.04)">
+              <div class="training-kpi-label">Step</div>
+              <div class="training-kpi-value" id="trn-step" style="color:var(--cyan);text-shadow:0 0 12px rgba(0,212,255,0.4)">—</div>
+              <div class="training-kpi-sub" id="trn-step-sub">gradient updates</div>
+            </div>
+
+            <div class="training-kpi-cell">
+              <div class="training-kpi-label">Assets</div>
+              <div class="training-kpi-value" id="trn-assets" style="color:var(--green);text-shadow:0 0 12px rgba(0,255,136,0.4)">—</div>
+              <div class="training-kpi-sub" id="trn-assets-sub">in cluster</div>
+            </div>
+
           </div>
         </div>
       </div>
@@ -3650,6 +3750,188 @@ setInterval(function(){
   if (aTab && aTab.classList.contains('active')) renderAssetsTab();
 }, 2000);
 
+// ════════════════════════════════════════════════════════════════════════════
+//  TRAINING PANEL — live log stream + KPI metrics
+//  Polls /api/ranker/logs/recent?category=training (DEBUG) + all recent logs
+//  Falls back gracefully if endpoint is unavailable
+// ════════════════════════════════════════════════════════════════════════════
+(function() {
+  const MAX_LINES   = 120;   // keep last N lines in terminal
+  const POLL_MS     = 2000;
+  let _termLines    = [];
+  let _lastLogTs    = null;
+  let _prevLoss     = null;
+
+  // Format a log entry into a coloured terminal line
+  function formatLogLine(entry) {
+    const ts  = (entry.timestamp || '').slice(11, 19); // HH:MM:SS  (no ms for raw lines)
+    const cat = (entry.category  || '').toUpperCase();
+    const lvl = (entry.level     || '').toUpperCase();
+
+    // Prefer structured message; strip raw-line boilerplate if the API returns the full line
+    let msg = entry.message || '';
+    // Strip leading "[YYYY-MM-DD HH:MM:SS] | LEVEL | CATEGORY | " prefix (raw line format)
+    msg = msg.replace(/^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\]\s*\|\s*\w+\s*\|\s*\w+\s*\|\s*/, '');
+    // Strip trailing JSON blob — already structured in entry.data
+    msg = msg.replace(/\s*\{.*\}\s*$/, '');
+    msg = msg.trim();
+
+    let lineClass = 'tlog-info';
+    if (lvl === 'DEBUG')                              lineClass = 'tlog-debug';
+    else if (lvl === 'ERROR' || lvl === 'CRITICAL')   lineClass = 'tlog-error';
+    else if (lvl === 'WARNING')                       lineClass = 'tlog-warn';
+    else if (cat === 'SIGNAL')                        lineClass = 'tlog-signal';
+    else if (cat === 'TRADE')                         lineClass = 'tlog-trade';
+    else if (cat === 'TRAINING')                      lineClass = 'tlog-debug';
+
+    const catCol = cat === 'SIGNAL'   ? 'var(--cyan)'  :
+                   cat === 'TRADE'    ? 'var(--amber)'  :
+                   cat === 'TRAINING' ? '#a855f7'       :
+                   cat === 'RANKING'  ? 'var(--green)'  : 'var(--t3)';
+
+    const catLabel = cat || lvl;
+
+    return `<div class="${lineClass}" style="display:flex;gap:10px;padding:1px 0">` +
+           `<span class="tlog-ts" style="flex-shrink:0;width:68px">${ts}</span>` +
+           `<span style="color:${catCol};font-weight:700;flex-shrink:0;width:76px">${catLabel}</span>` +
+           `<span>${escHtml(msg)}</span>` +
+           `</div>`;
+  }
+
+  function escHtml(s) {
+    return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+  }
+
+  function updateTerminal(entries) {
+    const term = document.getElementById('training-log-terminal');
+    if (!term) return;
+
+    // Append only NEW entries (filter by timestamp > _lastLogTs)
+    const newEntries = _lastLogTs
+      ? entries.filter(e => (e.timestamp || '') > _lastLogTs)
+      : entries;
+
+    if (!newEntries.length) return;
+
+    _lastLogTs = newEntries[newEntries.length - 1].timestamp || _lastLogTs;
+
+    newEntries.forEach(e => _termLines.push(formatLogLine(e)));
+    if (_termLines.length > MAX_LINES) _termLines = _termLines.slice(-MAX_LINES);
+
+    const atBottom = term.scrollHeight - term.clientHeight - term.scrollTop < 40;
+    term.innerHTML = _termLines.join('');
+    if (atBottom) term.scrollTop = term.scrollHeight;
+  }
+
+  // Regex to extract training fields directly from the raw message string.
+  // Matches the format produced by ranker_logging.py training_update():
+  //   "step=14 | loss=0.0278 | lr=0.000300 | assets=4"
+  // Also matches the full pipe-delimited log line (the API returns the raw line as "message").
+  const _TRAINING_MSG_RE = /step=(\d+).*?loss=([\d.]+).*?lr=([\d.eE+\-]+).*?assets=(\d+)/;
+
+  function _extractTrainingData(entry) {
+    // Priority 1: structured "data" field from the enriched API endpoint
+    if (entry.data && typeof entry.data === 'object' && entry.data.step != null) {
+      return entry.data;
+    }
+    // Priority 2: JSON blob embedded in "metadata" (RankerLogger.to_dict())
+    if (entry.metadata && entry.metadata.step != null) {
+      return entry.metadata;
+    }
+    // Priority 3: Regex over the raw message / log line string
+    const m = _TRAINING_MSG_RE.exec(entry.message || '');
+    if (m) {
+      return {
+        step:        parseInt(m[1], 10),
+        loss:        parseFloat(m[2]),
+        lr:          parseFloat(m[3]),
+        asset_count: parseInt(m[4], 10),
+      };
+    }
+    return null;
+  }
+
+  function updateTrainingKPIs(entries) {
+    // Accept entries whose category field OR raw message text indicate TRAINING
+    const trainEntries = entries
+      .filter(e => {
+        const cat = (e.category || '').toUpperCase();
+        const msg = (e.message  || '').toUpperCase();
+        return cat === 'TRAINING' || msg.includes('| TRAINING ');
+      })
+      .map(e => ({ entry: e, data: _extractTrainingData(e) }))
+      .filter(x => x.data !== null)
+      .sort((a, b) => (a.entry.timestamp || '') < (b.entry.timestamp || '') ? -1 : 1);
+
+    if (!trainEntries.length) return;
+
+    const { entry: latest, data: d } = trainEntries[trainEntries.length - 1];
+
+    // Loss
+    const lossEl      = document.getElementById('trn-loss');
+    const lossTrendEl = document.getElementById('trn-loss-trend');
+    if (lossEl && d.loss != null) {
+      const v = parseFloat(d.loss);
+      lossEl.textContent = v.toFixed(4);
+      if (lossTrendEl && _prevLoss != null) {
+        const delta = v - _prevLoss;
+        lossTrendEl.textContent = (delta < 0 ? '▼ ' : '▲ ') + Math.abs(delta).toFixed(4);
+        lossTrendEl.style.color = delta < 0 ? 'var(--green)' : 'var(--red)';
+      }
+      _prevLoss = v;
+    }
+
+    // Learning rate — display as fixed 6dp (matches log format) or sci notation if tiny
+    const lrEl = document.getElementById('trn-lr');
+    if (lrEl && d.lr != null) {
+      const lr = parseFloat(d.lr);
+      lrEl.textContent = lr < 0.001 ? lr.toExponential(3) : lr.toFixed(6);
+    }
+
+    // Step
+    const stepEl    = document.getElementById('trn-step');
+    const stepSubEl = document.getElementById('trn-step-sub');
+    if (stepEl && d.step != null) {
+      stepEl.textContent = parseInt(d.step).toLocaleString();
+      if (stepSubEl) stepSubEl.textContent = 'gradient updates';
+    }
+
+    // Assets
+    const assetsEl = document.getElementById('trn-assets');
+    if (assetsEl) {
+      const n = d.asset_count ?? d.assets ?? null;
+      if (n != null) assetsEl.textContent = n;
+    }
+
+    // Header meta timestamp
+    const metaEl = document.getElementById('training-meta');
+    if (metaEl && latest.timestamp) {
+      metaEl.textContent = 'last update ' + (latest.timestamp || '').slice(11, 19);
+    }
+  }
+
+  async function pollTrainingLogs() {
+    try {
+      const res = await fetch('/api/ranker/logs/recent?limit=80');
+      if (!res.ok) throw new Error(res.status);
+      const json = await res.json();
+      const entries = json.logs || [];
+      updateTerminal(entries);
+      updateTrainingKPIs(entries);
+    } catch(err) {
+      // Silently tolerate connection errors — dashboard may run offline
+      const term = document.getElementById('training-log-terminal');
+      if (term && !_termLines.length) {
+        term.innerHTML = '<div class="tlog-error">Error loading logs — ' + escHtml(err.message) + '</div>';
+      }
+    }
+  }
+
+  // Init: first poll immediately, then repeat
+  pollTrainingLogs();
+  setInterval(pollTrainingLogs, POLL_MS);
+})();
+
 </script>
 </body>
 </html>

hub_dashboard_service.py

@@ -620,6 +620,14 @@ def api_logs_recent():
     # Reverse so newest is first
     raw_lines.reverse()
 
+    # Pre-compiled patterns for structured field extraction
+    _CAT_RE      = re.compile(r'\|\s*(INFO|DEBUG|WARNING|ERROR|CRITICAL)\s*\|\s*([A-Z_]+)\s*\|')
+    _ASSET_RE    = re.compile(r'\|\s*([A-Z][A-Z0-9_]+)\s*\|', re.IGNORECASE)
+    _TRAINING_RE = re.compile(
+        r'step=(\d+)\s*\|\s*loss=([\d.]+)\s*\|\s*lr=([\d.eE+\-]+)\s*\|\s*assets=(\d+)'
+    )
+    _JSON_RE = re.compile(r'(\{.*\})\s*$')
+
     for line in raw_lines[:limit]:
         if not line.strip():
             continue
@@ -628,14 +636,46 @@ def api_logs_recent():
             continue
 
         ts = ts_match.group(1)
-        level_match = re.search(r'\|\s*(INFO|DEBUG|WARNING|ERROR)\s*\|', line)
-        level = level_match.group(1) if level_match else "INFO"
+
+        # Extract level AND category from the pipe-delimited format:
+        # "[ts] | LEVEL    | CATEGORY        | ..."
+        cat_m = _CAT_RE.search(line)
+        level    = cat_m.group(1) if cat_m else "INFO"
+        category_field = cat_m.group(2).strip() if cat_m else ""
 
         if category:
             if category.upper() not in line.upper():
                 continue
 
-        logs.append({"timestamp": ts, "level": level, "message": line.strip()})
+        # Build base entry
+        entry = {
+            "timestamp":  ts,
+            "level":      level,
+            "category":   category_field,
+            "message":    line.strip(),
+            "data":       None,
+        }
+
+        # Parse structured data for TRAINING entries
+        if category_field == "TRAINING":
+            tm = _TRAINING_RE.search(line)
+            if tm:
+                entry["data"] = {
+                    "step":        int(tm.group(1)),
+                    "loss":        float(tm.group(2)),
+                    "lr":          float(tm.group(3)),
+                    "asset_count": int(tm.group(4)),
+                }
+            else:
+                # Fallback: try the trailing JSON blob
+                jm = _JSON_RE.search(line)
+                if jm:
+                    try:
+                        entry["data"] = json.loads(jm.group(1))
+                    except ValueError:
+                        pass
+
+        logs.append(entry)
 
     return jsonify({"logs": logs, "count": len(logs)})
 

ranker_logging.py

@@ -277,7 +277,7 @@ class RankerLogger:
         """Get most recent n log entries, optionally filtered by category."""
         with self._lock:
             if category:
-                entries = list(self._by_category.get(category, []))
+                entries = list(self._by_category.get(category.upper(), []))
             else:
                 entries = list(self._buffer)
         return [e.to_dict() for e in entries[-n:]]