fix(skills): read OOXML parts as bytes and form JSON as UTF-8 in office skill scripts

The bundled office skills (#68595) read user documents and agent-authored
payloads with the locale-default codec:

- docx/powerpoint validators/base.py opened OOXML part XML in text mode
  before handing it to lxml. On Windows (cp1251/GBK) the bytes decode to
  mojibake that lxml then parses, so validation runs against silently
  corrupted document text; on locales where the UTF-8 bytes don't decode
  the validator crashes with UnicodeDecodeError instead of validating.
  Opening as bytes lets lxml honor the encoding declared in the XML prolog.

- The pdf form scripts (fill_fillable_fields, fill_pdf_form_with_annotations,
  create_validation_image, check_bounding_boxes) read the fields JSON the
  agent authors — UTF-8 by construction — with the locale codec, so
  non-ASCII form values (any Cyrillic/CJK/accented input) either crash or
  get written into the user's PDF as mojibake. The json.dump writers use
  ensure_ascii=True and were already safe; only the readers needed pinning.

Adds a contract test asserting every document/payload reader is
locale-independent, plus a live regression test that runs
check_bounding_boxes.py on a non-ASCII fields.json under a forced
non-UTF-8 locale — it fails without the fix on both POSIX (C locale)
and Windows (cp1251 chokes on the 0x98 byte of U+2018).
This commit is contained in:
solyanviktor-star 2026-07-23 23:13:01 +03:00 committed by Teknium
parent 607f647141
commit adecb0d1a9
7 changed files with 91 additions and 6 deletions

View file

@ -783,7 +783,7 @@ class BaseSchemaValidator:
try:
schema = _load_schema(str(schema_path))
with open(xml_file, "r") as f:
with open(xml_file, "rb") as f:
xml_doc = lxml.etree.parse(f)
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)

View file

@ -59,7 +59,7 @@ if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: check_bounding_boxes.py [fields.json]")
sys.exit(1)
with open(sys.argv[1]) as f:
with open(sys.argv[1], encoding="utf-8") as f:
messages = get_bounding_box_messages(f)
for msg in messages:
print(msg)

View file

@ -7,7 +7,7 @@ from PIL import Image, ImageDraw
def create_validation_image(page_number, fields_json_path, input_path, output_path):
with open(fields_json_path, 'r') as f:
with open(fields_json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
img = Image.open(input_path)

View file

@ -9,7 +9,7 @@ from extract_form_field_info import get_field_info
def fill_pdf_fields(input_pdf_path: str, fields_json_path: str, output_pdf_path: str):
with open(fields_json_path) as f:
with open(fields_json_path, encoding="utf-8") as f:
fields = json.load(f)
fields_by_page = {}
for field in fields:

View file

@ -32,7 +32,7 @@ def transform_from_pdf_coords(bbox, pdf_height):
def fill_pdf_form(input_pdf_path, fields_json_path, output_pdf_path):
with open(fields_json_path, "r") as f:
with open(fields_json_path, "r", encoding="utf-8") as f:
fields_data = json.load(f)
reader = PdfReader(input_pdf_path)

View file

@ -783,7 +783,7 @@ class BaseSchemaValidator:
try:
schema = _load_schema(str(schema_path))
with open(xml_file, "r") as f:
with open(xml_file, "rb") as f:
xml_doc = lxml.etree.parse(f)
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)

View file

@ -124,3 +124,88 @@ def test_docs_pages_generated():
assert (docs_dir / f"productivity-{name}.md").exists(), (
f"missing generated docs page for {name}; run website/scripts/generate-skill-docs.py"
)
# Document/payload readers must not depend on the host locale. OOXML part
# files are UTF-8 (declared in the XML prolog) and the form-fields JSON the
# agent authors is UTF-8, but these sites used locale-default text mode: on
# Windows (cp1251/GBK/cp932) the validators parsed silently mojibake'd
# document text and the pdf fill scripts wrote mojibake'd values into the
# user's form — or crashed outright where the bytes don't decode.
_ENCODING_SENSITIVE_READS = [
(
"docx/scripts/office/validators/base.py",
'with open(xml_file, "rb") as f:',
),
(
"powerpoint/scripts/office/validators/base.py",
'with open(xml_file, "rb") as f:',
),
(
"pdf/scripts/check_bounding_boxes.py",
'with open(sys.argv[1], encoding="utf-8") as f:',
),
(
"pdf/scripts/create_validation_image.py",
"with open(fields_json_path, 'r', encoding='utf-8') as f:",
),
(
"pdf/scripts/fill_fillable_fields.py",
'with open(fields_json_path, encoding="utf-8") as f:',
),
(
"pdf/scripts/fill_pdf_form_with_annotations.py",
'with open(fields_json_path, "r", encoding="utf-8") as f:',
),
]
@pytest.mark.parametrize("rel_path,expected", _ENCODING_SENSITIVE_READS)
def test_document_readers_are_locale_independent(rel_path, expected):
"""XML parts are opened as bytes (lxml honors the XML prolog) and JSON
payloads as UTF-8 never with the locale-default codec."""
source = (SKILLS / "productivity" / rel_path).read_text(encoding="utf-8")
assert expected in source, (
f"{rel_path}: locale-dependent read of a UTF-8 document/payload"
)
def test_check_bounding_boxes_reads_utf8_fields_json(tmp_path):
"""Run the real script on a UTF-8 fields.json with non-ASCII field
descriptions under a forced non-UTF-8 locale. Without the explicit
encoding the json.load crashes (C locale on POSIX; cp1251 chokes on
the 0x98 byte of U+2018 on Windows)."""
import json
import os
import subprocess
import sys
script = _skill_dir("pdf") / "scripts" / "check_bounding_boxes.py"
fields = {
"form_fields": [
{
"description": "Фамилия (label)",
"page_number": 1,
"label_bounding_box": [0, 0, 10, 10],
"entry_bounding_box": [20, 20, 30, 40],
}
]
}
fields_json = tmp_path / "fields.json"
fields_json.write_bytes(
json.dumps(fields, ensure_ascii=False, indent=2).encode("utf-8")
)
env = dict(os.environ)
env.update({"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "0", "PYTHONIOENCODING": "utf-8"})
result = subprocess.run(
[sys.executable, str(script), str(fields_json)],
capture_output=True,
text=True,
env=env,
timeout=60,
)
assert result.returncode == 0, (
f"script failed under non-UTF-8 locale:\n{result.stderr}"
)
assert "SUCCESS" in result.stdout