From adecb0d1a93bf9aaf31f6662332409d69653e85e Mon Sep 17 00:00:00 2001 From: solyanviktor-star <233359899+solyanviktor-star@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:13:01 +0300 Subject: [PATCH] fix(skills): read OOXML parts as bytes and form JSON as UTF-8 in office skill scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundled office skills (#68595) read user documents and agent-authored payloads with the locale-default codec: - docx/powerpoint validators/base.py opened OOXML part XML in text mode before handing it to lxml. On Windows (cp1251/GBK) the bytes decode to mojibake that lxml then parses, so validation runs against silently corrupted document text; on locales where the UTF-8 bytes don't decode the validator crashes with UnicodeDecodeError instead of validating. Opening as bytes lets lxml honor the encoding declared in the XML prolog. - The pdf form scripts (fill_fillable_fields, fill_pdf_form_with_annotations, create_validation_image, check_bounding_boxes) read the fields JSON the agent authors — UTF-8 by construction — with the locale codec, so non-ASCII form values (any Cyrillic/CJK/accented input) either crash or get written into the user's PDF as mojibake. The json.dump writers use ensure_ascii=True and were already safe; only the readers needed pinning. Adds a contract test asserting every document/payload reader is locale-independent, plus a live regression test that runs check_bounding_boxes.py on a non-ASCII fields.json under a forced non-UTF-8 locale — it fails without the fix on both POSIX (C locale) and Windows (cp1251 chokes on the 0x98 byte of U+2018). --- .../docx/scripts/office/validators/base.py | 2 +- .../pdf/scripts/check_bounding_boxes.py | 2 +- .../pdf/scripts/create_validation_image.py | 2 +- .../pdf/scripts/fill_fillable_fields.py | 2 +- .../scripts/fill_pdf_form_with_annotations.py | 2 +- .../scripts/office/validators/base.py | 2 +- tests/skills/test_office_document_skills.py | 85 +++++++++++++++++++ 7 files changed, 91 insertions(+), 6 deletions(-) diff --git a/skills/productivity/docx/scripts/office/validators/base.py b/skills/productivity/docx/scripts/office/validators/base.py index 91f2fb834129..f928b2ca5bec 100644 --- a/skills/productivity/docx/scripts/office/validators/base.py +++ b/skills/productivity/docx/scripts/office/validators/base.py @@ -783,7 +783,7 @@ class BaseSchemaValidator: try: schema = _load_schema(str(schema_path)) - with open(xml_file, "r") as f: + with open(xml_file, "rb") as f: xml_doc = lxml.etree.parse(f) xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc) diff --git a/skills/productivity/pdf/scripts/check_bounding_boxes.py b/skills/productivity/pdf/scripts/check_bounding_boxes.py index 2cc5e348f359..b9c6ea76556e 100644 --- a/skills/productivity/pdf/scripts/check_bounding_boxes.py +++ b/skills/productivity/pdf/scripts/check_bounding_boxes.py @@ -59,7 +59,7 @@ if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: check_bounding_boxes.py [fields.json]") sys.exit(1) - with open(sys.argv[1]) as f: + with open(sys.argv[1], encoding="utf-8") as f: messages = get_bounding_box_messages(f) for msg in messages: print(msg) diff --git a/skills/productivity/pdf/scripts/create_validation_image.py b/skills/productivity/pdf/scripts/create_validation_image.py index 10eadd8124b1..ce98a8597508 100644 --- a/skills/productivity/pdf/scripts/create_validation_image.py +++ b/skills/productivity/pdf/scripts/create_validation_image.py @@ -7,7 +7,7 @@ from PIL import Image, ImageDraw def create_validation_image(page_number, fields_json_path, input_path, output_path): - with open(fields_json_path, 'r') as f: + with open(fields_json_path, 'r', encoding='utf-8') as f: data = json.load(f) img = Image.open(input_path) diff --git a/skills/productivity/pdf/scripts/fill_fillable_fields.py b/skills/productivity/pdf/scripts/fill_fillable_fields.py index 51c2600f3895..7bc4b1c6f54d 100644 --- a/skills/productivity/pdf/scripts/fill_fillable_fields.py +++ b/skills/productivity/pdf/scripts/fill_fillable_fields.py @@ -9,7 +9,7 @@ from extract_form_field_info import get_field_info def fill_pdf_fields(input_pdf_path: str, fields_json_path: str, output_pdf_path: str): - with open(fields_json_path) as f: + with open(fields_json_path, encoding="utf-8") as f: fields = json.load(f) fields_by_page = {} for field in fields: diff --git a/skills/productivity/pdf/scripts/fill_pdf_form_with_annotations.py b/skills/productivity/pdf/scripts/fill_pdf_form_with_annotations.py index b430069fd016..204e94098fcc 100644 --- a/skills/productivity/pdf/scripts/fill_pdf_form_with_annotations.py +++ b/skills/productivity/pdf/scripts/fill_pdf_form_with_annotations.py @@ -32,7 +32,7 @@ def transform_from_pdf_coords(bbox, pdf_height): def fill_pdf_form(input_pdf_path, fields_json_path, output_pdf_path): - with open(fields_json_path, "r") as f: + with open(fields_json_path, "r", encoding="utf-8") as f: fields_data = json.load(f) reader = PdfReader(input_pdf_path) diff --git a/skills/productivity/powerpoint/scripts/office/validators/base.py b/skills/productivity/powerpoint/scripts/office/validators/base.py index 91f2fb834129..f928b2ca5bec 100644 --- a/skills/productivity/powerpoint/scripts/office/validators/base.py +++ b/skills/productivity/powerpoint/scripts/office/validators/base.py @@ -783,7 +783,7 @@ class BaseSchemaValidator: try: schema = _load_schema(str(schema_path)) - with open(xml_file, "r") as f: + with open(xml_file, "rb") as f: xml_doc = lxml.etree.parse(f) xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc) diff --git a/tests/skills/test_office_document_skills.py b/tests/skills/test_office_document_skills.py index 4804769d8618..1557182cf750 100644 --- a/tests/skills/test_office_document_skills.py +++ b/tests/skills/test_office_document_skills.py @@ -124,3 +124,88 @@ def test_docs_pages_generated(): assert (docs_dir / f"productivity-{name}.md").exists(), ( f"missing generated docs page for {name}; run website/scripts/generate-skill-docs.py" ) + + +# Document/payload readers must not depend on the host locale. OOXML part +# files are UTF-8 (declared in the XML prolog) and the form-fields JSON the +# agent authors is UTF-8, but these sites used locale-default text mode: on +# Windows (cp1251/GBK/cp932) the validators parsed silently mojibake'd +# document text and the pdf fill scripts wrote mojibake'd values into the +# user's form — or crashed outright where the bytes don't decode. +_ENCODING_SENSITIVE_READS = [ + ( + "docx/scripts/office/validators/base.py", + 'with open(xml_file, "rb") as f:', + ), + ( + "powerpoint/scripts/office/validators/base.py", + 'with open(xml_file, "rb") as f:', + ), + ( + "pdf/scripts/check_bounding_boxes.py", + 'with open(sys.argv[1], encoding="utf-8") as f:', + ), + ( + "pdf/scripts/create_validation_image.py", + "with open(fields_json_path, 'r', encoding='utf-8') as f:", + ), + ( + "pdf/scripts/fill_fillable_fields.py", + 'with open(fields_json_path, encoding="utf-8") as f:', + ), + ( + "pdf/scripts/fill_pdf_form_with_annotations.py", + 'with open(fields_json_path, "r", encoding="utf-8") as f:', + ), +] + + +@pytest.mark.parametrize("rel_path,expected", _ENCODING_SENSITIVE_READS) +def test_document_readers_are_locale_independent(rel_path, expected): + """XML parts are opened as bytes (lxml honors the XML prolog) and JSON + payloads as UTF-8 — never with the locale-default codec.""" + source = (SKILLS / "productivity" / rel_path).read_text(encoding="utf-8") + assert expected in source, ( + f"{rel_path}: locale-dependent read of a UTF-8 document/payload" + ) + + +def test_check_bounding_boxes_reads_utf8_fields_json(tmp_path): + """Run the real script on a UTF-8 fields.json with non-ASCII field + descriptions under a forced non-UTF-8 locale. Without the explicit + encoding the json.load crashes (C locale on POSIX; cp1251 chokes on + the 0x98 byte of U+2018 on Windows).""" + import json + import os + import subprocess + import sys + + script = _skill_dir("pdf") / "scripts" / "check_bounding_boxes.py" + fields = { + "form_fields": [ + { + "description": "Фамилия (‘label’)", + "page_number": 1, + "label_bounding_box": [0, 0, 10, 10], + "entry_bounding_box": [20, 20, 30, 40], + } + ] + } + fields_json = tmp_path / "fields.json" + fields_json.write_bytes( + json.dumps(fields, ensure_ascii=False, indent=2).encode("utf-8") + ) + + env = dict(os.environ) + env.update({"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "0", "PYTHONIOENCODING": "utf-8"}) + result = subprocess.run( + [sys.executable, str(script), str(fields_json)], + capture_output=True, + text=True, + env=env, + timeout=60, + ) + assert result.returncode == 0, ( + f"script failed under non-UTF-8 locale:\n{result.stderr}" + ) + assert "SUCCESS" in result.stdout