From 7f1c278db817acfd315b15f1eac5a87fde4fe5bd Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:27:02 -0700 Subject: [PATCH] fix(photon): intercept console.log so 'stream interrupted' bursts escalate spectrum-ts routes stream telemetry through @photon-ai/otel's createLogger, which sends severity>=ERROR to console.error and WARN/INFO to console.log. The two lines the health monitor keys off land on different channels: log.error("stream persistently failing") -> console.error (caught), but log.warn("stream interrupted; reconnecting") -> console.log (was missed). The original interception patched console.error only, so the recovering-> degraded escalation counter never saw the interrupt bursts that are the primary silent-inbound symptom. Verified live against spectrum-ts 3.1.0 + @photon-ai/otel: 3 real log.warn('stream interrupted') calls now escalate to degraded -> process.exit(75) -> adapter reconnect. Adds a shared classifyStreamLog() fed by both console.error and console.log, plus a regression test asserting both channels are intercepted. --- plugins/platforms/photon/sidecar/index.mjs | 35 ++++++++++++++----- .../platforms/photon/test_spectrum_patch.py | 17 +++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/plugins/platforms/photon/sidecar/index.mjs b/plugins/platforms/photon/sidecar/index.mjs index 5b5d20ed826..9dec22ddfa4 100644 --- a/plugins/platforms/photon/sidecar/index.mjs +++ b/plugins/platforms/photon/sidecar/index.mjs @@ -168,22 +168,41 @@ function markStreamRecovering(reason) { } } +function classifyStreamLog(text) { + if (!text.includes("[spectrum.stream]")) return; + const reason = text.split("\n", 1)[0]; + if (text.includes("persistently failing")) { + markStreamDegraded(reason); + } else if (text.includes("stream interrupted")) { + markStreamRecovering(reason); + } +} + +// spectrum-ts routes its stream telemetry through @photon-ai/otel's +// createLogger, which sends severity >= ERROR to console.error and +// everything else (WARN/INFO) to console.log. The two lines we key off +// land on *different* channels: `log.error("stream persistently failing")` +// -> console.error, but `log.warn("stream interrupted; reconnecting")` +// -> console.log. Patch both so the recovering/degraded counters see the +// interrupt bursts, not just the terminal "persistently failing" line. const originalConsoleError = console.error.bind(console); console.error = (...args) => { const text = args .map((arg) => (arg && arg.stack ? arg.stack : String(arg))) .join(" "); - if (text.includes("[spectrum.stream]")) { - const reason = text.split("\n", 1)[0]; - if (text.includes("persistently failing")) { - markStreamDegraded(reason); - } else if (text.includes("stream interrupted")) { - markStreamRecovering(reason); - } - } + classifyStreamLog(text); originalConsoleError(...args); }; +const originalConsoleLog = console.log.bind(console); +console.log = (...args) => { + const text = args + .map((arg) => (arg && arg.stack ? arg.stack : String(arg))) + .join(" "); + classifyStreamLog(text); + originalConsoleLog(...args); +}; + if (!projectId || !projectSecret || !sharedToken) { console.error( "photon-sidecar: PHOTON_PROJECT_ID, PHOTON_PROJECT_SECRET and " + diff --git a/tests/plugins/platforms/photon/test_spectrum_patch.py b/tests/plugins/platforms/photon/test_spectrum_patch.py index 548466ed77a..8e875ea788c 100644 --- a/tests/plugins/platforms/photon/test_spectrum_patch.py +++ b/tests/plugins/platforms/photon/test_spectrum_patch.py @@ -26,6 +26,23 @@ def test_sidecar_healthz_reports_stream_health() -> None: assert "process.exit(75);" in index +def test_sidecar_intercepts_both_console_channels() -> None: + """spectrum-ts routes its stream telemetry through @photon-ai/otel, which + sends severity >= ERROR to console.error and WARN/INFO to console.log. + The two lines the health monitor keys off land on *different* channels: + `log.error("stream persistently failing")` -> console.error, but + `log.warn("stream interrupted; reconnecting")` -> console.log. Patching + only console.error would miss every interrupt burst (the primary silent- + inbound symptom), so both channels must be intercepted. + """ + index = Path("plugins/platforms/photon/sidecar/index.mjs").read_text(encoding="utf-8") + assert "function classifyStreamLog(" in index + assert "console.error = (...args) =>" in index + assert "console.log = (...args) =>" in index + # Both wrappers must feed the shared classifier. + assert index.count("classifyStreamLog(text)") >= 2 + + def test_sidecar_labels_catchup_internal_errors_as_upstream_photon() -> None: """Photon cloud stream failures should not look like local auth problems.""" index = Path("plugins/platforms/photon/sidecar/index.mjs").read_text(encoding="utf-8")