mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-25 17:18:11 +00:00
fix: handle non-UTF-8 files in OpenClaw migration script
This commit is contained in:
chancelu
Teknium
parent
60dbce7c34
commit
049af61d64
2 changed files with 170 additions and 9 deletions
|
|
@ -312,7 +312,7 @@ def sha256_file(path: Path) -> str:
|
|||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
|
|
@ -349,7 +349,12 @@ def resolve_secret_input(value: Any, env: Optional[Dict[str, str]] = None) -> Op
|
|||
def load_yaml_file(path: Path) -> Dict[str, Any]:
|
||||
if yaml is None or not path.exists():
|
||||
return {}
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
# errors="replace" means read_text() cannot raise UnicodeDecodeError;
|
||||
# OSError covers races like the file disappearing or being unreadable.
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8", errors="replace"))
|
||||
except (yaml.YAMLError, OSError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
|
|
@ -367,7 +372,13 @@ def parse_env_file(path: Path) -> Dict[str, str]:
|
|||
if not path.exists():
|
||||
return {}
|
||||
data: Dict[str, str] = {}
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
# errors="replace" means read_text() cannot raise UnicodeDecodeError;
|
||||
# OSError covers races like the file disappearing or being unreadable.
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return {}
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
|
|
@ -1208,10 +1219,13 @@ class Migrator:
|
|||
return
|
||||
|
||||
try:
|
||||
data = json.loads(source.read_text(encoding="utf-8"))
|
||||
data = json.loads(source.read_text(encoding="utf-8", errors="replace"))
|
||||
except json.JSONDecodeError as exc:
|
||||
self.record("command-allowlist", source, destination, "error", f"Invalid JSON: {exc}")
|
||||
return
|
||||
except OSError as exc:
|
||||
self.record("command-allowlist", source, destination, "error", f"Could not read file: {exc}")
|
||||
return
|
||||
|
||||
patterns: List[str] = []
|
||||
agents = data.get("agents", {})
|
||||
|
|
@ -1262,9 +1276,11 @@ class Migrator:
|
|||
config_path = self.source_root / name
|
||||
if config_path.exists():
|
||||
try:
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
# errors="replace" means read_text() cannot raise
|
||||
# UnicodeDecodeError; OSError covers unreadable/vanished files.
|
||||
data = json.loads(config_path.read_text(encoding="utf-8", errors="replace"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
return {}
|
||||
|
||||
|
|
@ -1343,9 +1359,11 @@ class Migrator:
|
|||
allowlist_path = self.source_root / "credentials" / "telegram-default-allowFrom.json"
|
||||
if allowlist_path.exists():
|
||||
try:
|
||||
allow_data = json.loads(allowlist_path.read_text(encoding="utf-8"))
|
||||
allow_data = json.loads(allowlist_path.read_text(encoding="utf-8", errors="replace"))
|
||||
except json.JSONDecodeError:
|
||||
self.record("messaging-settings", allowlist_path, self.target_root / ".env", "error", "Invalid JSON in Telegram allowlist file")
|
||||
except OSError as exc:
|
||||
self.record("messaging-settings", allowlist_path, self.target_root / ".env", "error", f"Could not read Telegram allowlist file: {exc}")
|
||||
else:
|
||||
allow_from = allow_data.get("allowFrom", [])
|
||||
if isinstance(allow_from, list):
|
||||
|
|
@ -1617,7 +1635,7 @@ class Migrator:
|
|||
auth_profiles_path = self.source_root / "agents" / "main" / "agent" / "auth-profiles.json"
|
||||
if auth_profiles_path.exists():
|
||||
try:
|
||||
profiles = json.loads(auth_profiles_path.read_text(encoding="utf-8"))
|
||||
profiles = json.loads(auth_profiles_path.read_text(encoding="utf-8", errors="replace"))
|
||||
if isinstance(profiles, dict):
|
||||
# auth-profiles.json wraps profiles in a "profiles" key
|
||||
profile_entries = profiles.get("profiles", profiles) if isinstance(profiles.get("profiles"), dict) else profiles
|
||||
|
|
@ -1901,7 +1919,12 @@ class Migrator:
|
|||
|
||||
all_incoming: List[str] = []
|
||||
for md_file in md_files:
|
||||
entries = extract_markdown_entries(read_text(md_file))
|
||||
try:
|
||||
# read_text() uses errors="replace" so it cannot raise
|
||||
# UnicodeDecodeError; OSError covers unreadable/vanished files.
|
||||
entries = extract_markdown_entries(read_text(md_file))
|
||||
except OSError:
|
||||
continue
|
||||
all_incoming.extend(entries)
|
||||
|
||||
if not all_incoming:
|
||||
|
|
|
|||
|
|
@ -1107,3 +1107,141 @@ def test_migrate_model_config_no_catalog_leaves_value_alone(tmp_path: Path):
|
|||
{"agents": {"defaults": {"model": "some-model-id"}}},
|
||||
)
|
||||
assert _extract_model(parsed) == "some-model-id"
|
||||
|
||||
|
||||
# ── non-UTF-8 tolerance (issue #8901) ───────────────────────────────────────
|
||||
|
||||
|
||||
def _write_invalid_utf8_json(path: Path, prefix: bytes, valid_value: bytes, suffix: bytes) -> None:
|
||||
"""Write a JSON-shaped file containing one invalid UTF-8 byte (0xB3) inside
|
||||
a string value, alongside a separate, validly-encoded value. Used to check
|
||||
that a single bad byte does not prevent the rest of the file's data from
|
||||
being read (relies on read_text(..., errors="replace"))."""
|
||||
path.write_bytes(prefix + b"\xb3" + valid_value + suffix)
|
||||
|
||||
|
||||
def test_command_allowlist_handles_invalid_utf8_bytes(tmp_path: Path):
|
||||
"""exec-approvals.json with a non-UTF-8 byte should not abort migration;
|
||||
valid patterns elsewhere in the same file must still be imported."""
|
||||
mod = load_module()
|
||||
source = tmp_path / ".openclaw"
|
||||
target = tmp_path / ".hermes"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
|
||||
_write_invalid_utf8_json(
|
||||
source / "exec-approvals.json",
|
||||
prefix=b'{"agents": {"*": {"allowlist": [{"pattern": "/bad',
|
||||
valid_value=b'"}, {"pattern": "/usr/bin/*"}]}}}',
|
||||
suffix=b"",
|
||||
)
|
||||
(target / "config.yaml").write_text("command_allowlist: []\n", encoding="utf-8")
|
||||
|
||||
migrator = mod.Migrator(
|
||||
source_root=source, target_root=target, execute=True,
|
||||
workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None,
|
||||
selected_options={"command-allowlist"},
|
||||
)
|
||||
report = migrator.migrate()
|
||||
|
||||
items = [i for i in report["items"] if i["kind"] == "command-allowlist"]
|
||||
assert items and items[0]["status"] == "migrated"
|
||||
config_text = (target / "config.yaml").read_text(encoding="utf-8")
|
||||
assert "/usr/bin/*" in config_text
|
||||
|
||||
|
||||
def test_messaging_settings_handles_invalid_utf8_in_telegram_allowlist(tmp_path: Path):
|
||||
"""Telegram allowFrom file with a non-UTF-8 byte should not abort migration;
|
||||
valid user IDs elsewhere in the same file must still be imported."""
|
||||
mod = load_module()
|
||||
source = tmp_path / ".openclaw"
|
||||
target = tmp_path / ".hermes"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
|
||||
creds_dir = source / "credentials"
|
||||
creds_dir.mkdir()
|
||||
_write_invalid_utf8_json(
|
||||
creds_dir / "telegram-default-allowFrom.json",
|
||||
prefix=b'{"allowFrom": ["bad',
|
||||
valid_value=b'", "123456789"]}',
|
||||
suffix=b"",
|
||||
)
|
||||
|
||||
migrator = mod.Migrator(
|
||||
source_root=source, target_root=target, execute=True,
|
||||
workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None,
|
||||
selected_options={"messaging-settings"},
|
||||
)
|
||||
report = migrator.migrate()
|
||||
|
||||
items = [i for i in report["items"] if i["kind"] == "messaging-settings"]
|
||||
assert items and items[0]["status"] == "migrated"
|
||||
env_text = (target / ".env").read_text(encoding="utf-8")
|
||||
assert "123456789" in env_text
|
||||
|
||||
|
||||
def test_provider_keys_handles_invalid_utf8_in_auth_profiles(tmp_path: Path):
|
||||
"""auth-profiles.json with a non-UTF-8 byte should not abort migration;
|
||||
a valid provider key elsewhere in the same file must still be imported."""
|
||||
mod = load_module()
|
||||
source = tmp_path / ".openclaw"
|
||||
target = tmp_path / ".hermes"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
|
||||
agent_dir = source / "agents" / "main" / "agent"
|
||||
agent_dir.mkdir(parents=True)
|
||||
_write_invalid_utf8_json(
|
||||
agent_dir / "auth-profiles.json",
|
||||
prefix=b'{"profiles": {"broken": {"key": "bad',
|
||||
valid_value=b'"}, "openrouter-main": {"key": "sk-or-valid-key"}}}',
|
||||
suffix=b"",
|
||||
)
|
||||
(source / "openclaw.json").write_text(json.dumps({}), encoding="utf-8")
|
||||
|
||||
migrator = mod.Migrator(
|
||||
source_root=source, target_root=target, execute=True,
|
||||
workspace_target=None, overwrite=False, migrate_secrets=True, output_dir=None,
|
||||
selected_options={"provider-keys"},
|
||||
)
|
||||
report = migrator.migrate()
|
||||
|
||||
items = [i for i in report["items"] if i["kind"] == "provider-keys"]
|
||||
assert items and items[0]["status"] == "migrated"
|
||||
env_text = (target / ".env").read_text(encoding="utf-8")
|
||||
assert "OPENROUTER_API_KEY=sk-or-valid-key" in env_text
|
||||
|
||||
|
||||
def test_daily_memory_skips_undecodable_file_but_merges_others(tmp_path: Path):
|
||||
"""A daily-memory .md file with invalid UTF-8 bytes should not abort the
|
||||
merge; entries from the other, cleanly-encoded file must still land."""
|
||||
mod = load_module()
|
||||
source = tmp_path / ".openclaw"
|
||||
target = tmp_path / ".hermes"
|
||||
target.mkdir()
|
||||
|
||||
mem_dir = source / "workspace" / "memory"
|
||||
mem_dir.mkdir(parents=True)
|
||||
(mem_dir / "2026-03-01.md").write_text(
|
||||
"# March 1 Notes\n\n- User prefers dark mode\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# errors="replace" means this file is still readable (bad byte becomes
|
||||
# U+FFFD), so its valid heading/entries should also survive alongside it.
|
||||
(mem_dir / "2026-03-02.md").write_bytes(
|
||||
b"# March 2 Notes\n\n- Working on \xb3migration project\n"
|
||||
)
|
||||
|
||||
migrator = mod.Migrator(
|
||||
source_root=source, target_root=target, execute=True,
|
||||
workspace_target=None, overwrite=False, migrate_secrets=False, output_dir=None,
|
||||
selected_options={"daily-memory"},
|
||||
)
|
||||
report = migrator.migrate()
|
||||
|
||||
items = [i for i in report["items"] if i["kind"] == "daily-memory"]
|
||||
assert items and items[0]["status"] == "migrated"
|
||||
content = (target / "memories" / "MEMORY.md").read_text(encoding="utf-8")
|
||||
assert "dark mode" in content
|
||||
assert "migration project" in content
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue