mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-26 17:38:36 +00:00
fix(skills): read OOXML parts as bytes and form JSON as UTF-8 in office skill scripts
The bundled office skills (#68595) read user documents and agent-authored payloads with the locale-default codec: - docx/powerpoint validators/base.py opened OOXML part XML in text mode before handing it to lxml. On Windows (cp1251/GBK) the bytes decode to mojibake that lxml then parses, so validation runs against silently corrupted document text; on locales where the UTF-8 bytes don't decode the validator crashes with UnicodeDecodeError instead of validating. Opening as bytes lets lxml honor the encoding declared in the XML prolog. - The pdf form scripts (fill_fillable_fields, fill_pdf_form_with_annotations, create_validation_image, check_bounding_boxes) read the fields JSON the agent authors — UTF-8 by construction — with the locale codec, so non-ASCII form values (any Cyrillic/CJK/accented input) either crash or get written into the user's PDF as mojibake. The json.dump writers use ensure_ascii=True and were already safe; only the readers needed pinning. Adds a contract test asserting every document/payload reader is locale-independent, plus a live regression test that runs check_bounding_boxes.py on a non-ASCII fields.json under a forced non-UTF-8 locale — it fails without the fix on both POSIX (C locale) and Windows (cp1251 chokes on the 0x98 byte of U+2018).
This commit is contained in:
parent
607f647141
commit
adecb0d1a9
7 changed files with 91 additions and 6 deletions
|
|
@ -783,7 +783,7 @@ class BaseSchemaValidator:
|
|||
try:
|
||||
schema = _load_schema(str(schema_path))
|
||||
|
||||
with open(xml_file, "r") as f:
|
||||
with open(xml_file, "rb") as f:
|
||||
xml_doc = lxml.etree.parse(f)
|
||||
|
||||
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ if __name__ == "__main__":
|
|||
if len(sys.argv) != 2:
|
||||
print("Usage: check_bounding_boxes.py [fields.json]")
|
||||
sys.exit(1)
|
||||
with open(sys.argv[1]) as f:
|
||||
with open(sys.argv[1], encoding="utf-8") as f:
|
||||
messages = get_bounding_box_messages(f)
|
||||
for msg in messages:
|
||||
print(msg)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from PIL import Image, ImageDraw
|
|||
|
||||
|
||||
def create_validation_image(page_number, fields_json_path, input_path, output_path):
|
||||
with open(fields_json_path, 'r') as f:
|
||||
with open(fields_json_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
img = Image.open(input_path)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from extract_form_field_info import get_field_info
|
|||
|
||||
|
||||
def fill_pdf_fields(input_pdf_path: str, fields_json_path: str, output_pdf_path: str):
|
||||
with open(fields_json_path) as f:
|
||||
with open(fields_json_path, encoding="utf-8") as f:
|
||||
fields = json.load(f)
|
||||
fields_by_page = {}
|
||||
for field in fields:
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def transform_from_pdf_coords(bbox, pdf_height):
|
|||
|
||||
def fill_pdf_form(input_pdf_path, fields_json_path, output_pdf_path):
|
||||
|
||||
with open(fields_json_path, "r") as f:
|
||||
with open(fields_json_path, "r", encoding="utf-8") as f:
|
||||
fields_data = json.load(f)
|
||||
|
||||
reader = PdfReader(input_pdf_path)
|
||||
|
|
|
|||
|
|
@ -783,7 +783,7 @@ class BaseSchemaValidator:
|
|||
try:
|
||||
schema = _load_schema(str(schema_path))
|
||||
|
||||
with open(xml_file, "r") as f:
|
||||
with open(xml_file, "rb") as f:
|
||||
xml_doc = lxml.etree.parse(f)
|
||||
|
||||
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
|
||||
|
|
|
|||
|
|
@ -124,3 +124,88 @@ def test_docs_pages_generated():
|
|||
assert (docs_dir / f"productivity-{name}.md").exists(), (
|
||||
f"missing generated docs page for {name}; run website/scripts/generate-skill-docs.py"
|
||||
)
|
||||
|
||||
|
||||
# Document/payload readers must not depend on the host locale. OOXML part
|
||||
# files are UTF-8 (declared in the XML prolog) and the form-fields JSON the
|
||||
# agent authors is UTF-8, but these sites used locale-default text mode: on
|
||||
# Windows (cp1251/GBK/cp932) the validators parsed silently mojibake'd
|
||||
# document text and the pdf fill scripts wrote mojibake'd values into the
|
||||
# user's form — or crashed outright where the bytes don't decode.
|
||||
_ENCODING_SENSITIVE_READS = [
|
||||
(
|
||||
"docx/scripts/office/validators/base.py",
|
||||
'with open(xml_file, "rb") as f:',
|
||||
),
|
||||
(
|
||||
"powerpoint/scripts/office/validators/base.py",
|
||||
'with open(xml_file, "rb") as f:',
|
||||
),
|
||||
(
|
||||
"pdf/scripts/check_bounding_boxes.py",
|
||||
'with open(sys.argv[1], encoding="utf-8") as f:',
|
||||
),
|
||||
(
|
||||
"pdf/scripts/create_validation_image.py",
|
||||
"with open(fields_json_path, 'r', encoding='utf-8') as f:",
|
||||
),
|
||||
(
|
||||
"pdf/scripts/fill_fillable_fields.py",
|
||||
'with open(fields_json_path, encoding="utf-8") as f:',
|
||||
),
|
||||
(
|
||||
"pdf/scripts/fill_pdf_form_with_annotations.py",
|
||||
'with open(fields_json_path, "r", encoding="utf-8") as f:',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rel_path,expected", _ENCODING_SENSITIVE_READS)
|
||||
def test_document_readers_are_locale_independent(rel_path, expected):
|
||||
"""XML parts are opened as bytes (lxml honors the XML prolog) and JSON
|
||||
payloads as UTF-8 — never with the locale-default codec."""
|
||||
source = (SKILLS / "productivity" / rel_path).read_text(encoding="utf-8")
|
||||
assert expected in source, (
|
||||
f"{rel_path}: locale-dependent read of a UTF-8 document/payload"
|
||||
)
|
||||
|
||||
|
||||
def test_check_bounding_boxes_reads_utf8_fields_json(tmp_path):
|
||||
"""Run the real script on a UTF-8 fields.json with non-ASCII field
|
||||
descriptions under a forced non-UTF-8 locale. Without the explicit
|
||||
encoding the json.load crashes (C locale on POSIX; cp1251 chokes on
|
||||
the 0x98 byte of U+2018 on Windows)."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = _skill_dir("pdf") / "scripts" / "check_bounding_boxes.py"
|
||||
fields = {
|
||||
"form_fields": [
|
||||
{
|
||||
"description": "Фамилия (‘label’)",
|
||||
"page_number": 1,
|
||||
"label_bounding_box": [0, 0, 10, 10],
|
||||
"entry_bounding_box": [20, 20, 30, 40],
|
||||
}
|
||||
]
|
||||
}
|
||||
fields_json = tmp_path / "fields.json"
|
||||
fields_json.write_bytes(
|
||||
json.dumps(fields, ensure_ascii=False, indent=2).encode("utf-8")
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env.update({"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "0", "PYTHONIOENCODING": "utf-8"})
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script), str(fields_json)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"script failed under non-UTF-8 locale:\n{result.stderr}"
|
||||
)
|
||||
assert "SUCCESS" in result.stdout
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue