ci: add provider health-check workflow to catch broken fetchers

scripts/healthcheck.js runs every fetcher and fails if one returns
too few models or throws — the failure mode that silently broke the
Nebius and Mistral scrapers. Keyed providers are neutral (SKIP) when
their secret is absent.

healthcheck.yml runs it on PRs and pushes touching scripts/**, on a
daily schedule, and on demand; a second job verifies the app builds.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/healthcheck.yml

@@ -0,0 +1,54 @@
+name: Provider health check
+
+# Catches upstream breakage: if a provider changes its page/API shape and a
+# fetcher starts returning 0 models (or throws), this run goes red and notifies.
+on:
+  schedule:
+    - cron: '0 12 * * *'   # daily at 12:00 UTC (independent of the 06:00 data update)
+  pull_request:
+    paths:
+      - 'scripts/**'
+      - '.github/workflows/healthcheck.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'scripts/providers/**'
+      - 'scripts/fetch-providers.js'
+      - 'scripts/healthcheck.js'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  # Verify every provider fetcher still returns a sane number of models.
+  fetchers:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+      - run: npm install
+      - name: Run fetcher health check
+        env:
+          REQUESTY_API_KEY: ${{ secrets.REQUESTY_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          NSCALE_API_KEY: ${{ secrets.NSCALE_API_KEY }}
+          NEBIUS_API_KEY: ${{ secrets.NEBIUS_API_KEY }}
+        run: node scripts/healthcheck.js
+
+  # Verify the app still builds (TypeScript + Vite).
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+      - run: npm install
+      - run: npm run build

package.json

@@ -25,7 +25,8 @@
     "fetch:benchmarks": "node scripts/fetch-benchmarks.js",
     "push:hf": "node scripts/sync-hf.js",
     "server": "node server.js",
-    "dev:api": "node server.js"
+    "dev:api": "node server.js",
+    "healthcheck": "node scripts/healthcheck.js"
   },
   "keywords": [],
   "author": "",

scripts/healthcheck.js

@@ -0,0 +1,111 @@
+'use strict';
+
+/**
+ * Provider fetcher health check.
+ *
+ * Runs every provider fetcher and asserts it still returns a sane number of
+ * models. This catches the failure mode where an upstream page/API changes
+ * shape and a scraper silently returns 0 (or throws) — as happened when Nebius
+ * rebranded to Token Factory and Mistral moved to Astro.
+ *
+ * Exit code:
+ *   0 — all checked providers healthy
+ *   1 — one or more providers broke (below their minimum, or threw)
+ *
+ * Providers that require an API key are only checked when that key is present
+ * in the environment; otherwise they are reported as SKIP (neutral), so the
+ * check still passes on forks / environments without secrets.
+ *
+ * Usage: node scripts/healthcheck.js
+ */
+
+const path = require('path');
+
+// key: provider module under scripts/providers/
+// min: lowest model count we still consider healthy (set well below the live
+//      count so normal catalog churn never trips it, but a broken scraper does)
+// keyEnv: env var required to reach the source (null = public scrape)
+const CHECKS = [
+  { key: 'scaleway',          min: 8,   keyEnv: null },
+  { key: 'ovhcloud',          min: 10,  keyEnv: null },
+  { key: 'stackit',           min: 4,   keyEnv: null },
+  { key: 'mistral',           min: 10,  keyEnv: null },
+  { key: 'langdock',          min: 12,  keyEnv: null },
+  { key: 'groq',              min: 5,   keyEnv: null },
+  { key: 'infomaniak',        min: 6,   keyEnv: null },
+  { key: 'ionos',             min: 5,   keyEnv: null },
+  { key: 'black-forest-labs', min: 6,   keyEnv: null },
+  { key: 'nebius',            min: 10,  keyEnv: 'NEBIUS_API_KEY' },
+  { key: 'nscale',            min: 5,   keyEnv: 'NSCALE_API_KEY' },
+  { key: 'requesty',          min: 40,  keyEnv: 'REQUESTY_API_KEY' },
+  { key: 'openrouter',        min: 80,  keyEnv: 'OPENROUTER_API_KEY' },
+];
+
+const TIMEOUT_MS = 120000;
+
+function loadFetcher(key) {
+  const mod = require(path.join(__dirname, 'providers', key));
+  const fn = Object.values(mod).find((v) => typeof v === 'function');
+  if (!fn) throw new Error(`module ${key} exports no fetcher function`);
+  return { fn, providerName: mod.providerName || key };
+}
+
+function withTimeout(promise, ms, label) {
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms)),
+  ]).catch((e) => { throw new Error(`${label}: ${e.message}`); });
+}
+
+async function main() {
+  const results = [];
+
+  for (const check of CHECKS) {
+    const { key, min, keyEnv } = check;
+    let providerName = key;
+
+    if (keyEnv && !process.env[keyEnv]) {
+      results.push({ key, providerName, status: 'SKIP', count: null, note: `no ${keyEnv}` });
+      continue;
+    }
+
+    try {
+      const { fn, providerName: name } = loadFetcher(key);
+      providerName = name;
+      const models = await withTimeout(Promise.resolve().then(fn), TIMEOUT_MS, key);
+      const count = Array.isArray(models) ? models.length : 0;
+      if (count >= min) {
+        results.push({ key, providerName, status: 'PASS', count, note: `>= ${min}` });
+      } else {
+        results.push({ key, providerName, status: 'FAIL', count, note: `expected >= ${min}` });
+      }
+    } catch (err) {
+      results.push({ key, providerName, status: 'FAIL', count: null, note: err.message });
+    }
+  }
+
+  // Report
+  const pad = (s, n) => String(s).padEnd(n);
+  console.log('\nProvider fetcher health check\n' + '='.repeat(60));
+  for (const r of results) {
+    const mark = r.status === 'PASS' ? '✓' : r.status === 'SKIP' ? '–' : '✗';
+    const count = r.count == null ? '' : `${r.count} models`;
+    console.log(`  ${mark} ${pad(r.status, 5)} ${pad(r.providerName, 20)} ${pad(count, 12)} ${r.note}`);
+  }
+  console.log('='.repeat(60));
+
+  const failed = results.filter((r) => r.status === 'FAIL');
+  const passed = results.filter((r) => r.status === 'PASS').length;
+  const skipped = results.filter((r) => r.status === 'SKIP').length;
+  console.log(`  ${passed} passed, ${failed.length} failed, ${skipped} skipped\n`);
+
+  if (failed.length) {
+    console.error('BROKEN FETCHERS: ' + failed.map((r) => r.providerName).join(', '));
+    process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error('Health check crashed:', err);
+  process.exit(1);
+});