From 049af61d64b6ede4ee43580d9b390f343be5bc04 Mon Sep 17 00:00:00 2001 From: chancelu Date: Fri, 17 Jul 2026 18:18:26 +0800 Subject: [PATCH] fix: handle non-UTF-8 files in OpenClaw migration script --- .../scripts/openclaw_to_hermes.py | 41 ++++-- tests/skills/test_openclaw_migration.py | 138 ++++++++++++++++++ 2 files changed, 170 insertions(+), 9 deletions(-) diff --git a/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py b/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py index c108a99ea5a6..7f6e9b3fb52e 100644 --- a/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py +++ b/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py @@ -312,7 +312,7 @@ def sha256_file(path: Path) -> str: def read_text(path: Path) -> str: - return path.read_text(encoding="utf-8") + return path.read_text(encoding="utf-8", errors="replace") def normalize_text(text: str) -> str: @@ -349,7 +349,12 @@ def resolve_secret_input(value: Any, env: Optional[Dict[str, str]] = None) -> Op def load_yaml_file(path: Path) -> Dict[str, Any]: if yaml is None or not path.exists(): return {} - data = yaml.safe_load(path.read_text(encoding="utf-8")) + try: + # errors="replace" means read_text() cannot raise UnicodeDecodeError; + # OSError covers races like the file disappearing or being unreadable. + data = yaml.safe_load(path.read_text(encoding="utf-8", errors="replace")) + except (yaml.YAMLError, OSError): + return {} return data if isinstance(data, dict) else {} @@ -367,7 +372,13 @@ def parse_env_file(path: Path) -> Dict[str, str]: if not path.exists(): return {} data: Dict[str, str] = {} - for raw_line in path.read_text(encoding="utf-8").splitlines(): + try: + # errors="replace" means read_text() cannot raise UnicodeDecodeError; + # OSError covers races like the file disappearing or being unreadable. + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return {} + for raw_line in lines: line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue @@ -1208,10 +1219,13 @@ class Migrator: return try: - data = json.loads(source.read_text(encoding="utf-8")) + data = json.loads(source.read_text(encoding="utf-8", errors="replace")) except json.JSONDecodeError as exc: self.record("command-allowlist", source, destination, "error", f"Invalid JSON: {exc}") return + except OSError as exc: + self.record("command-allowlist", source, destination, "error", f"Could not read file: {exc}") + return patterns: List[str] = [] agents = data.get("agents", {}) @@ -1262,9 +1276,11 @@ class Migrator: config_path = self.source_root / name if config_path.exists(): try: - data = json.loads(config_path.read_text(encoding="utf-8")) + # errors="replace" means read_text() cannot raise + # UnicodeDecodeError; OSError covers unreadable/vanished files. + data = json.loads(config_path.read_text(encoding="utf-8", errors="replace")) return data if isinstance(data, dict) else {} - except json.JSONDecodeError: + except (json.JSONDecodeError, OSError): continue return {} @@ -1343,9 +1359,11 @@ class Migrator: allowlist_path = self.source_root / "credentials" / "telegram-default-allowFrom.json" if allowlist_path.exists(): try: - allow_data = json.loads(allowlist_path.read_text(encoding="utf-8")) + allow_data = json.loads(allowlist_path.read_text(encoding="utf-8", errors="replace")) except json.JSONDecodeError: self.record("messaging-settings", allowlist_path, self.target_root / ".env", "error", "Invalid JSON in Telegram allowlist file") + except OSError as exc: + self.record("messaging-settings", allowlist_path, self.target_root / ".env", "error", f"Could not read Telegram allowlist file: {exc}") else: allow_from = allow_data.get("allowFrom", []) if isinstance(allow_from, list): @@ -1617,7 +1635,7 @@ class Migrator: auth_profiles_path = self.source_root / "agents" / "main" / "agent" / "auth-profiles.json" if auth_profiles_path.exists(): try: - profiles = json.loads(auth_profiles_path.read_text(encoding="utf-8")) + profiles = json.loads(auth_profiles_path.read_text(encoding="utf-8", errors="replace")) if isinstance(profiles, dict): # auth-profiles.json wraps profiles in a "profiles" key profile_entries = profiles.get("profiles", profiles) if isinstance(profiles.get("profiles"), dict) else profiles @@ -1901,7 +1919,12 @@ class Migrator: all_incoming: List[str] = [] for md_file in md_files: - entries = extract_markdown_entries(read_text(md_file)) + try: + # read_text() uses errors="replace" so it cannot raise + # UnicodeDecodeError; OSError covers unreadable/vanished files. + entries = extract_markdown_entries(read_text(md_file)) + except OSError: + continue all_incoming.extend(entries) if not all_incoming: diff --git a/tests/skills/test_openclaw_migration.py b/tests/skills/test_openclaw_migration.py index 0b331c402386..e75dad8792a2 100644 --- a/tests/skills/test_openclaw_migration.py +++ b/tests/skills/test_openclaw_migration.py @@ -1107,3 +1107,141 @@ def test_migrate_model_config_no_catalog_leaves_value_alone(tmp_path: Path): {"agents": {"defaults": {"model": "some-model-id"}}}, ) assert _extract_model(parsed) == "some-model-id" + + +# ── non-UTF-8 tolerance (issue #8901) ─────────────────────────────────────── + + +def _write_invalid_utf8_json(path: Path, prefix: bytes, valid_value: bytes, suffix: bytes) -> None: + """Write a JSON-shaped file containing one invalid UTF-8 byte (0xB3) inside + a string value, alongside a separate, validly-encoded value. Used to check + that a single bad byte does not prevent the rest of the file's data from + being read (relies on read_text(..., errors="replace")).""" + path.write_bytes(prefix + b"\xb3" + valid_value + suffix) + + +def test_command_allowlist_handles_invalid_utf8_bytes(tmp_path: Path): + """exec-approvals.json with a non-UTF-8 byte should not abort migration; + valid patterns elsewhere in the same file must still be imported.""" + mod = load_module() + source = tmp_path / ".openclaw" + target = tmp_path / ".hermes" + source.mkdir() + target.mkdir() + + _write_invalid_utf8_json( + source / "exec-approvals.json", + prefix=b'{"agents": {"*": {"allowlist": [{"pattern": "/bad', + valid_value=b'"}, {"pattern": "/usr/bin/*"}]}}}', + suffix=b"", + ) + (target / "config.yaml").write_text("command_allowlist: []\n", encoding="utf-8") + + migrator = mod.Migrator( + source_root=source, target_root=target, execute=True, + workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None, + selected_options={"command-allowlist"}, + ) + report = migrator.migrate() + + items = [i for i in report["items"] if i["kind"] == "command-allowlist"] + assert items and items[0]["status"] == "migrated" + config_text = (target / "config.yaml").read_text(encoding="utf-8") + assert "/usr/bin/*" in config_text + + +def test_messaging_settings_handles_invalid_utf8_in_telegram_allowlist(tmp_path: Path): + """Telegram allowFrom file with a non-UTF-8 byte should not abort migration; + valid user IDs elsewhere in the same file must still be imported.""" + mod = load_module() + source = tmp_path / ".openclaw" + target = tmp_path / ".hermes" + source.mkdir() + target.mkdir() + + creds_dir = source / "credentials" + creds_dir.mkdir() + _write_invalid_utf8_json( + creds_dir / "telegram-default-allowFrom.json", + prefix=b'{"allowFrom": ["bad', + valid_value=b'", "123456789"]}', + suffix=b"", + ) + + migrator = mod.Migrator( + source_root=source, target_root=target, execute=True, + workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None, + selected_options={"messaging-settings"}, + ) + report = migrator.migrate() + + items = [i for i in report["items"] if i["kind"] == "messaging-settings"] + assert items and items[0]["status"] == "migrated" + env_text = (target / ".env").read_text(encoding="utf-8") + assert "123456789" in env_text + + +def test_provider_keys_handles_invalid_utf8_in_auth_profiles(tmp_path: Path): + """auth-profiles.json with a non-UTF-8 byte should not abort migration; + a valid provider key elsewhere in the same file must still be imported.""" + mod = load_module() + source = tmp_path / ".openclaw" + target = tmp_path / ".hermes" + source.mkdir() + target.mkdir() + + agent_dir = source / "agents" / "main" / "agent" + agent_dir.mkdir(parents=True) + _write_invalid_utf8_json( + agent_dir / "auth-profiles.json", + prefix=b'{"profiles": {"broken": {"key": "bad', + valid_value=b'"}, "openrouter-main": {"key": "sk-or-valid-key"}}}', + suffix=b"", + ) + (source / "openclaw.json").write_text(json.dumps({}), encoding="utf-8") + + migrator = mod.Migrator( + source_root=source, target_root=target, execute=True, + workspace_target=None, overwrite=False, migrate_secrets=True, output_dir=None, + selected_options={"provider-keys"}, + ) + report = migrator.migrate() + + items = [i for i in report["items"] if i["kind"] == "provider-keys"] + assert items and items[0]["status"] == "migrated" + env_text = (target / ".env").read_text(encoding="utf-8") + assert "OPENROUTER_API_KEY=sk-or-valid-key" in env_text + + +def test_daily_memory_skips_undecodable_file_but_merges_others(tmp_path: Path): + """A daily-memory .md file with invalid UTF-8 bytes should not abort the + merge; entries from the other, cleanly-encoded file must still land.""" + mod = load_module() + source = tmp_path / ".openclaw" + target = tmp_path / ".hermes" + target.mkdir() + + mem_dir = source / "workspace" / "memory" + mem_dir.mkdir(parents=True) + (mem_dir / "2026-03-01.md").write_text( + "# March 1 Notes\n\n- User prefers dark mode\n", + encoding="utf-8", + ) + # errors="replace" means this file is still readable (bad byte becomes + # U+FFFD), so its valid heading/entries should also survive alongside it. + (mem_dir / "2026-03-02.md").write_bytes( + b"# March 2 Notes\n\n- Working on \xb3migration project\n" + ) + + migrator = mod.Migrator( + source_root=source, target_root=target, execute=True, + workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None, + selected_options={"daily-memory"}, + ) + report = migrator.migrate() + + items = [i for i in report["items"] if i["kind"] == "daily-memory"] + assert items and items[0]["status"] == "migrated" + content = (target / "memories" / "MEMORY.md").read_text(encoding="utf-8") + assert "dark mode" in content + assert "migration project" in content