mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-27 17:58:07 +00:00
29fc746350
* feat(conformance): vector generator — native renderers as executable spec - scripts/generate_conformance_vectors.py: renders a 44-case corpus (markdown grid + scar tissue + adversarial agent output) through the NATIVE renderers (Telegram MarkdownV2 format_message, Slack mrkdwn, WhatsApp, Discord) and emits per-platform JSON vectors stamped with the oracle commit. Expect semantics: parity | semantic | divergent(note). - tests/conformance/: committed vectors + 7 behavior-contract tests (determinism, self-free oracle invocation, shape, scar-bug coverage, committed-vectors-reproduce lockstep — the openapi.json discipline). - Consumed by gateway-gateway's conformance runner (committed vectors + sender-level vitest suite + weekly refresh workflow). * fix(conformance): explicit utf-8 encoding on vector file I/O (Windows footguns gate)
322 lines
15 KiB
JSON
322 lines
15 KiB
JSON
{
|
|
"$comment": "GENERATED — do not hand-edit. Regenerate with hermes-agent scripts/generate_conformance_vectors.py; the native renderers are the oracle (executable spec).",
|
|
"oracle": {
|
|
"repo": "NousResearch/hermes-agent",
|
|
"commit": "40eebc7d70f3d8e95c429c8aa482f31ba6c867f6",
|
|
"generator": "scripts/generate_conformance_vectors.py",
|
|
"generator_version": 1
|
|
},
|
|
"platform": "slack",
|
|
"vectors": [
|
|
{
|
|
"id": "plain-text",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "Just a plain sentence.",
|
|
"native_output": "Just a plain sentence."
|
|
},
|
|
{
|
|
"id": "bold",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "This is **bold** text.",
|
|
"native_output": "This is *bold* text."
|
|
},
|
|
{
|
|
"id": "italic",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "This is *italic* text.",
|
|
"native_output": "This is _italic_ text."
|
|
},
|
|
{
|
|
"id": "bold-italic",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "Mix of **bold** and *italic* in one line.",
|
|
"native_output": "Mix of *bold* and _italic_ in one line."
|
|
},
|
|
{
|
|
"id": "strikethrough",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "This is ~~struck~~ text.",
|
|
"native_output": "This is ~struck~ text."
|
|
},
|
|
{
|
|
"id": "inline-code",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "Run `pip install hermes` to start.",
|
|
"native_output": "Run `pip install hermes` to start."
|
|
},
|
|
{
|
|
"id": "fenced-code",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "```\nprint('hello')\n```",
|
|
"native_output": "```\nprint('hello')\n```"
|
|
},
|
|
{
|
|
"id": "fenced-code-lang",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "```python\ndef f(x):\n return x * 2\n```",
|
|
"native_output": "```\ndef f(x):\n return x * 2\n```"
|
|
},
|
|
{
|
|
"id": "link",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "See [the docs](https://example.com/docs) for more.",
|
|
"native_output": "See <https://example.com/docs|the docs> for more."
|
|
},
|
|
{
|
|
"id": "link-parens-url",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "See [spec](https://example.com/a_(b)) here.",
|
|
"native_output": "See <https://example.com/a_(b)|spec> here."
|
|
},
|
|
{
|
|
"id": "header-h1",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "# Big Title\nBody follows.",
|
|
"native_output": "*Big Title*\nBody follows."
|
|
},
|
|
{
|
|
"id": "header-h2",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "## Section\nBody follows.",
|
|
"native_output": "*Section*\nBody follows."
|
|
},
|
|
{
|
|
"id": "header-h3",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "### Sub-section\nBody follows.",
|
|
"native_output": "*Sub-section*\nBody follows."
|
|
},
|
|
{
|
|
"id": "ul-list",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "- first\n- second\n- third",
|
|
"native_output": "- first\n- second\n- third"
|
|
},
|
|
{
|
|
"id": "ol-list",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "1. first\n2. second\n3. third",
|
|
"native_output": "1. first\n2. second\n3. third"
|
|
},
|
|
{
|
|
"id": "nested-list",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "- outer\n - inner one\n - inner two\n- outer two",
|
|
"native_output": "- outer\n - inner one\n - inner two\n- outer two"
|
|
},
|
|
{
|
|
"id": "blockquote",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "> quoted wisdom\nregular line",
|
|
"native_output": "> quoted wisdom\nregular line"
|
|
},
|
|
{
|
|
"id": "hrule",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "above\n\n---\n\nbelow",
|
|
"native_output": "above\n\n---\n\nbelow"
|
|
},
|
|
{
|
|
"id": "table-simple",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
|
|
"native_output": "```\n| name | value |\n| ---- | ----- |\n| a | 1 |\n| b | 2 |\n```"
|
|
},
|
|
{
|
|
"id": "emoji",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "Done ✅ with 🎉 emoji 👀 test.",
|
|
"native_output": "Done ✅ with 🎉 emoji 👀 test."
|
|
},
|
|
{
|
|
"id": "cjk",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "中文测试：**粗体** 和 `代码` 混排。",
|
|
"native_output": "中文测试：*粗体* 和 `代码` 混排。"
|
|
},
|
|
{
|
|
"id": "bare-url",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "Visit https://example.com/path?q=1&r=2 today.",
|
|
"native_output": "Visit https://example.com/path?q=1&r=2 today."
|
|
},
|
|
{
|
|
"id": "mixed-document",
|
|
"category": "grid",
|
|
"expect": "parity",
|
|
"input": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x).",
|
|
"native_output": "*Report*\n\nStatus: *green*. Details in `runbook.md`.\n\n- item _one_\n- item *two*\n\n```\nmake deploy\n```\n\nSee <https://grafana.example.com/d/x|dashboard>."
|
|
},
|
|
{
|
|
"id": "mdv2-reserved-chars",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win.",
|
|
"native_output": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win."
|
|
},
|
|
{
|
|
"id": "mdv2-underscores",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "snake_case_name and file_name.py in prose.",
|
|
"native_output": "snake_case_name and file_name.py in prose."
|
|
},
|
|
{
|
|
"id": "mdv2-brackets",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "Array[0] and dict{key} and (parens) live here.",
|
|
"native_output": "Array[0] and dict{key} and (parens) live here."
|
|
},
|
|
{
|
|
"id": "slack-bold-conversion",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "**important** word",
|
|
"native_output": "*important* word"
|
|
},
|
|
{
|
|
"id": "slack-link-conversion",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "[click here](https://example.com)",
|
|
"native_output": "<https://example.com|click here>"
|
|
},
|
|
{
|
|
"id": "slack-broadcast-mention",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "Hey <!everyone> and <!channel> and <!here>!",
|
|
"native_output": "Hey <!everyone> and <!channel> and <!here>!"
|
|
},
|
|
{
|
|
"id": "fence-lang-tag-slack",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "```text\nliteral first line issue\n```",
|
|
"native_output": "```\nliteral first line issue\n```"
|
|
},
|
|
{
|
|
"id": "backslash-in-code",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```",
|
|
"native_output": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```"
|
|
},
|
|
{
|
|
"id": "backtick-in-fence",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "```\nuse `inline` inside fence\n```",
|
|
"native_output": "```\nuse `inline` inside fence\n```"
|
|
},
|
|
{
|
|
"id": "header-with-bold",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "## The **Real** Deal",
|
|
"native_output": "*The Real Deal*"
|
|
},
|
|
{
|
|
"id": "table-cjk",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
|
|
"native_output": "```\n| 名前 | 値 |\n| ---- | --- |\n| 中文 | 42 |\n| b | 2 |\n```"
|
|
},
|
|
{
|
|
"id": "link-display-escapes",
|
|
"category": "scar",
|
|
"expect": "parity",
|
|
"input": "[v2.0 (beta)](https://example.com/v2)",
|
|
"native_output": "<https://example.com/v2|v2.0 (beta)>"
|
|
},
|
|
{
|
|
"id": "media-tag",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "Here you go\nMEDIA:/tmp/output.png\ndone",
|
|
"native_output": "Here you go\nMEDIA:/tmp/output.png\ndone"
|
|
},
|
|
{
|
|
"id": "unclosed-fence",
|
|
"category": "adversarial",
|
|
"expect": "divergent",
|
|
"input": "```python\nprint('never closed')",
|
|
"native_output": "```python\nprint('never closed')",
|
|
"note": "unterminated fence handling differs; both degrade without dropping content"
|
|
},
|
|
{
|
|
"id": "pathological-nesting",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "**bold *italic ~~struck `code` struck~~ italic* bold**",
|
|
"native_output": "*bold *italic ~~struck `code` struck~~ italic* bold*"
|
|
},
|
|
{
|
|
"id": "placeholder-injection",
|
|
"category": "adversarial",
|
|
"expect": "divergent",
|
|
"input": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
|
"native_output": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
|
"note": "\\x00SL tokens are the native renderer's own placeholder alphabet; connector uses a different scheme"
|
|
},
|
|
{
|
|
"id": "triple-markers",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "***what is this*** and ____that____",
|
|
"native_output": "*_what is this_* and ____that____"
|
|
},
|
|
{
|
|
"id": "empty-string",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "",
|
|
"native_output": ""
|
|
},
|
|
{
|
|
"id": "whitespace-only",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": " \n\t\n ",
|
|
"native_output": " \n\t\n "
|
|
},
|
|
{
|
|
"id": "long-line",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ",
|
|
"native_output": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "
|
|
},
|
|
{
|
|
"id": "many-fences",
|
|
"category": "adversarial",
|
|
"expect": "parity",
|
|
"input": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail",
|
|
"native_output": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"
|
|
}
|
|
]
|
|
}
|