fix(memory): degrade undecodable preferences/projects to empty

soroush5 · Sep 6, 2026 · Sep 6, 2026 · 2186415dae23d71ea21dca51bb30e2a4d6155801
commit 2186415dae23d71ea21dca51bb30e2a4d6155801
@@ -99,6 +99,25 @@ def _is_corruption_error(exc: BaseException) -> bool:
     return any(marker in msg for marker in _DB_CORRUPTION_MARKERS)
 
 
+def _read_text_lossy(path: Path) -> str:
+    """Read a user-editable state file, tolerating one bad byte (#8247).
+
+    Crash mid-write, editor slip, or a synced file can leave a single
+    non-UTF-8 byte in an otherwise valid file. Strict decoding would kill
+    every reader (dashboard, prompt assembly, index rebuild); lossy
+    decoding keeps the surviving valid content, which also keeps
+    read-merge-write baselines intact. Single read: the bytes are decoded
+    strict first and re-decoded lossy only on failure, so the corrupt path
+    costs no second I/O.
+    """
+    raw = path.read_bytes()
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        logger.warning("memory file is not valid UTF-8: %s", path)
+        return raw.decode("utf-8", errors="replace")
+
+
 def workspace_dir() -> Path:
     return config_dir() / WORKSPACE_DIR_NAME
 
@@ -290,7 +309,20 @@ def init(self) -> None:
     def read_preferences(self) -> str:
         """Read user preferences markdown file."""
         if self._preferences_file.exists():
-            return self._preferences_file.read_text(encoding="utf-8")
+            try:
+                return self._preferences_file.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                # One bad byte (crash mid-write, editor slip, synced file)
+                # must degrade, not kill the dashboard tab, prompt assembly,
+                # or the repair path (#8247). Decode lossy rather than
+                # returning "": an empty baseline would pass the
+                # compare-and-swap check in write_preferences and let a
+                # consolidation overwrite the surviving valid content.
+                logger.warning(
+                    "preferences file is not valid UTF-8: %s",
+                    self._preferences_file,
+                )
+                return self._preferences_file.read_text(encoding="utf-8", errors="replace")
         return ""
 
     def write_preferences(self, content: str, *, expected_baseline: str | None = None) -> bool:
@@ -346,7 +378,15 @@ def add_preference(self, preference: str) -> None:
     def read_projects(self) -> str:
         """Read active projects markdown file."""
         if self._projects_file.exists():
-            return self._projects_file.read_text(encoding="utf-8")
+            try:
+                return self._projects_file.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                # Same lossy fallback as read_preferences (#8247).
+                logger.warning(
+                    "projects file is not valid UTF-8: %s",
+                    self._projects_file,
+                )
+                return self._projects_file.read_text(encoding="utf-8", errors="replace")
         return ""
 
     def write_projects(self, content: str, *, expected_baseline: str | None = None) -> bool:
@@ -469,7 +509,7 @@ def append_history(self, entry: str) -> None:
                     )
                 content = ""
                 if path.exists():
-                    content = path.read_text(encoding="utf-8")
+                    content = _read_text_lossy(path)
                 if not content:
                     date = datetime.now().strftime("%Y-%m-%d")
                     content = f"# {date}\n"
@@ -535,7 +575,7 @@ def _read_recent_history_uncached(self, days: int, today: _date) -> str:
             path = self._history_dir / f"{day.strftime('%Y-%m-%d')}.md"
             if not path.exists():
                 continue
-            content = path.read_text(encoding="utf-8").strip()
+            content = _read_text_lossy(path).strip()
             if not content:
                 continue
 
@@ -988,12 +1028,15 @@ def _index_file(self, path: Path, content: str) -> None:
     def rebuild_index(self) -> int:
         """Rebuild the full FTS index from all memory files. Returns file count."""
         files: list[tuple[str, str]] = []
-        for path in (self._preferences_file, self._projects_file):
+        for reader, path in (
+            (self.read_preferences, self._preferences_file),
+            (self.read_projects, self._projects_file),
+        ):
             if path.exists():
-                files.append((str(path), path.read_text(encoding="utf-8")))
+                files.append((str(path), reader()))
         if self._history_dir.exists():
             for path in self._history_dir.glob("*.md"):
-                files.append((str(path), path.read_text(encoding="utf-8")))
+                files.append((str(path), _read_text_lossy(path)))
 
         conn = None
         try:

@@ -105,6 +105,63 @@ def test_read_concatenates_both_sections(self, tmp_path):
         assert "wxyv" in combined
         assert combined.index("qqzz") < combined.index("wxyv")
 
+    def test_non_utf8_preferences_decode_lossy(self, tmp_path):
+        # One bad byte must not kill the dashboard tab, prompt assembly,
+        # or the repair path (#8247). Decode lossy so the surviving valid
+        # content stays in the read baseline: an empty baseline would pass
+        # the write_preferences CAS check and let a consolidation wipe it.
+        store = MemoryStore(workspace=tmp_path)
+        store.init()
+        prefs = tmp_path / "memory" / "preferences.md"
+        prefs.write_bytes(b"# prefs\n\xff\xfe bad bytes\n")
+
+        content = store.read_preferences()
+        assert "# prefs" in content
+        assert "bad bytes" in content
+        # The repair path reads first: it must work, not crash.
+        store.add_preference("dark mode")
+        assert "dark mode" in store.read_preferences()
+
+    def test_non_utf8_projects_decode_lossy(self, tmp_path):
+        store = MemoryStore(workspace=tmp_path)
+        store.init()
+        projects = tmp_path / "memory" / "projects.md"
+        projects.write_bytes(b"# projects\n\xff bad\n")
+
+        content = store.read_projects()
+        assert "# projects" in content
+        assert "bad" in content
+
+    def test_consolidation_roundtrip_keeps_valid_content(self, tmp_path):
+        # A read-merge-write computed from a corrupt file must keep the
+        # surviving valid content: the lossy baseline matches on re-read,
+        # so CAS passes, and only the bad byte is lost.
+        store = MemoryStore(workspace=tmp_path)
+        store.init()
+        prefs = tmp_path / "memory" / "preferences.md"
+        prefs.write_bytes(b"# prefs\n\xff bad\n")
+
+        baseline = store.read_preferences()
+        assert store.write_preferences(baseline + "\n- extra", expected_baseline=baseline) is True
+        assert "# prefs" in store.read_preferences()
+
+    def test_sibling_readers_tolerate_bad_bytes(self, tmp_path):
+        # The same one-bad-byte crash lived at four sibling sites
+        # (rebuild_index, append_history, recent-history assembly).
+        from datetime import date
+
+        store = MemoryStore(workspace=tmp_path)
+        store.init()
+        history_dir = tmp_path / "memory" / "history"
+        history_dir.mkdir(parents=True, exist_ok=True)
+        today_file = history_dir / f"{date.today().strftime('%Y-%m-%d')}.md"
+        today_file.write_bytes(b"# today\n\xff bad\n")
+
+        assert store.rebuild_index() >= 1
+        store.append_history("another entry")
+        assert "another entry" in today_file.read_text(encoding="utf-8", errors="replace")
+        assert "today" in store.read_recent_history(days=1)
+
 
 class TestPruneHistory:
     def test_returns_zero_when_history_dir_absent(self, tmp_path):