feat: add rebuild-fts command for FTS5 shadow-table corruption (#287) (#288)

Closes #287.

## What this adds

- **`msgvault rebuild-fts`** — drops and recreates `messages_fts`, then
repopulates it from `messages` / `message_bodies` / `message_recipients`
/ `participants` via the existing batched backfill. Recovery path for
`malformed inverted index for FTS5 table main.messages_fts` in `verify`
output — the case where SQLite's own `rebuild` pragma and `delete-all`
don't help on a contentful FTS5 table. Wraps
`SQLITE_BUSY`/`SQLITE_LOCKED` as "stop msgvault serve / MCP and retry."
- **`verify` ordering** — integrity check now runs before OAuth setup,
so corrupt databases surface the repair hint even with an expired token.
- **`verify` hints** — split between FTS-only corruption (points at
`rebuild-fts`) and core-table corruption (points at `.recover`).
- **`docs/recovery.md`** — covers both paths and the contentful-FTS5
caveat.

## What this does not cover

- Core-table B-tree corruption (e.g., `Rowid out of order` in `messages`
/ `message_bodies`). `rebuild-fts` is strictly for the derived index;
core corruption still needs `.recover`.
- `_synchronous=FULL` default. Worth considering as archival-durability
hardening in a separate PR; not framed as the fix for #287.

## Usage

```
msgvault rebuild-fts
```

Stop `msgvault serve` and MCP clients first — needs an exclusive write
lock. Peak extra disk ≈ size of the FTS5 shadow tables (a few percent of
the DB).

Author: wesm

cmd/msgvault/cmd/rebuild_fts.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/wesm/msgvault/internal/store"
+)
+
+var rebuildFTSCmd = &cobra.Command{
+	Use:   "rebuild-fts",
+	Short: "Rebuild the full-text search index from scratch",
+	Long: `Drop and recreate the messages_fts virtual table, then repopulate it
+from messages / message_bodies / message_recipients / participants.
+
+Use this to recover from FTS5 shadow-table corruption that surfaces as
+"malformed inverted index for FTS5 table main.messages_fts" in
+'msgvault verify' output. SQLite's own 'rebuild' pragma reads from the
+same corrupt shadow tables and cannot clear this state.
+
+This command only fixes the derived search index. Core-table corruption
+(e.g., "Rowid out of order" in messages / message_bodies B-trees) requires
+a different recovery path — see 'msgvault verify' output.
+
+Peak extra disk usage is roughly the size of the FTS5 shadow tables
+(a few percent of the SQLite database). Stop 'msgvault serve' and any
+MCP clients before running this command — it needs an exclusive write lock.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		dbPath := cfg.DatabaseDSN()
+		s, err := store.Open(dbPath)
+		if err != nil {
+			return fmt.Errorf("open database: %w", err)
+		}
+		defer func() { _ = s.Close() }()
+
+		if err := s.InitSchema(); err != nil {
+			return fmt.Errorf("init schema: %w", err)
+		}
+
+		fmt.Fprintln(os.Stderr, "Rebuilding full-text search index...")
+		n, err := s.RebuildFTS(func(done, total int64) {
+			if total <= 0 {
+				return
+			}
+			if done > total {
+				done = total
+			}
+			pct := int(done * 100 / total)
+			barWidth := 30
+			filled := barWidth * pct / 100
+			bar := strings.Repeat("=", filled) +
+				strings.Repeat(" ", barWidth-filled)
+			fmt.Fprintf(os.Stderr, "\r  [%s] %3d%%", bar, pct)
+		})
+		if err != nil {
+			fmt.Fprintln(os.Stderr)
+			if s.IsBusyError(err) {
+				return fmt.Errorf(
+					"database is busy — stop 'msgvault serve' and any MCP " +
+						"clients, then retry",
+				)
+			}
+			return fmt.Errorf("rebuild FTS: %w", err)
+		}
+		fmt.Fprintf(os.Stderr,
+			"\r  [%s] 100%%  %d messages indexed.\n",
+			strings.Repeat("=", 30), n)
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(rebuildFTSCmd)
+}

cmd/msgvault/cmd/verify.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
 	"github.com/mattn/go-isatty"
@@ -53,6 +54,33 @@ Examples:
 			return fmt.Errorf("init schema: %w", err)
 		}
 
+		// Run SQLite integrity check before any Gmail work. Users with a
+		// corrupt database should see the repair hint even if their OAuth
+		// token is expired or the network is down.
+		var dbCorrupt bool
+		if !verifySkipDBCheck {
+			fmt.Println("Running database integrity check...")
+			integrityErrors, err := runIntegrityCheck(s)
+			if err != nil {
+				return fmt.Errorf("integrity check failed: %w", err)
+			}
+			if len(integrityErrors) == 0 {
+				fmt.Println("  Database integrity: OK")
+			} else {
+				dbCorrupt = true
+				fmt.Printf("  Database integrity: FAILED (%d errors)\n", len(integrityErrors))
+				for i, ie := range integrityErrors {
+					if i >= 10 {
+						fmt.Printf("  ... and %d more errors\n", len(integrityErrors)-10)
+						break
+					}
+					fmt.Printf("  - %s\n", ie)
+				}
+				printIntegrityRecoveryHint(integrityErrors)
+			}
+			fmt.Println()
+		}
+
 		// Look up source to get OAuth app binding
 		appName := ""
 		src, srcErr := findGmailSource(s, email)
@@ -72,10 +100,6 @@ Examples:
 			return err
 		}
 
-		if err := s.InitSchema(); err != nil {
-			return fmt.Errorf("init schema: %w", err)
-		}
-
 		// Create OAuth manager and get token source
 		oauthMgr, err := oauth.NewManager(clientSecretsPath, cfg.TokensDir(), logger)
 		if err != nil {
@@ -106,35 +130,6 @@ Examples:
 		client := gmail.NewClient(tokenSource, gmail.WithLogger(logger))
 		defer func() { _ = client.Close() }()
 
-		// Run SQLite integrity check first (offline, no Gmail needed)
-		var dbCorrupt bool
-		if !verifySkipDBCheck {
-			fmt.Println("Running database integrity check...")
-			integrityErrors, err := runIntegrityCheck(s)
-			if err != nil {
-				return fmt.Errorf("integrity check failed: %w", err)
-			}
-			if len(integrityErrors) == 0 {
-				fmt.Println("  Database integrity: OK")
-			} else {
-				dbCorrupt = true
-				fmt.Printf("  Database integrity: FAILED (%d errors)\n", len(integrityErrors))
-				for i, ie := range integrityErrors {
-					if i >= 10 {
-						fmt.Printf("  ... and %d more errors\n", len(integrityErrors)-10)
-						break
-					}
-					fmt.Printf("  - %s\n", ie)
-				}
-				fmt.Println()
-				fmt.Println("  The database has corruption. Consider:")
-				fmt.Println("    1. Back up the database file before any repair attempts")
-				fmt.Println("    2. Run: sqlite3 msgvault.db '.recover' | sqlite3 recovered.db")
-				fmt.Println("    3. Or export to SQL and reimport: sqlite3 msgvault.db .dump | sqlite3 new.db")
-			}
-			fmt.Println()
-		}
-
 		// Get Gmail profile
 		profile, err := client.GetProfile(ctx)
 		if err != nil {
@@ -286,6 +281,49 @@ func runIntegrityCheck(s *store.Store) ([]string, error) {
 	return errors, rows.Err()
 }
 
+// printIntegrityRecoveryHint prints repair guidance tailored to the kind of
+// corruption reported. FTS5 shadow-table corruption is fixable with the
+// lightweight `rebuild-fts` command; core B-tree corruption needs `.recover`,
+// which requires free disk roughly equal to the database size.
+func printIntegrityRecoveryHint(integrityErrors []string) {
+	var ftsErrs, coreErrs int
+	for _, e := range integrityErrors {
+		if isFTSIntegrityError(e) {
+			ftsErrs++
+		} else {
+			coreErrs++
+		}
+	}
+
+	fmt.Println()
+	fmt.Println("  Back up msgvault.db before attempting any repair.")
+	fmt.Println()
+
+	if ftsErrs > 0 {
+		fmt.Println("  Search index (FTS5) corruption:")
+		fmt.Println("    Run: msgvault rebuild-fts")
+		fmt.Println("    Drops and recreates messages_fts from the core tables.")
+		fmt.Println("    SQLite's 'rebuild' pragma reads from the corrupt shadow")
+		fmt.Println("    tables and cannot clear this state.")
+		fmt.Println()
+	}
+
+	if coreErrs > 0 {
+		fmt.Println("  Core table corruption (e.g., Rowid out of order in messages")
+		fmt.Println("  or message_bodies):")
+		fmt.Println("    Run: sqlite3 msgvault.db '.recover' | sqlite3 recovered.db")
+		fmt.Println("    (requires free disk roughly equal to the database size)")
+		fmt.Println("    Alternative: sqlite3 msgvault.db .dump | sqlite3 new.db")
+	}
+}
+
+// isFTSIntegrityError reports whether an integrity-check line describes
+// corruption in the FTS5 search index rather than the core tables.
+func isFTSIntegrityError(msg string) bool {
+	return strings.Contains(msg, "messages_fts") ||
+		strings.Contains(msg, "FTS5")
+}
+
 func init() {
 	verifyCmd.Flags().IntVar(&verifySampleSize, "sample", 100, "Number of messages to sample for MIME verification")
 	verifyCmd.Flags().BoolVar(&verifySkipDBCheck, "skip-db-check", false, "Skip SQLite integrity check")

cmd/msgvault/cmd/verify_test.go

@@ -0,0 +1,41 @@
+package cmd
+
+import "testing"
+
+// TestIsFTSIntegrityError_Classification verifies that the hint-classifier
+// cleanly separates FTS5 shadow-table errors (which rebuild-fts can fix)
+// from core-table errors (which need .recover). Messages come from real
+// PRAGMA integrity_check output; the shapes below are what users will see.
+func TestIsFTSIntegrityError_Classification(t *testing.T) {
+	tests := []struct {
+		msg    string
+		wantFT bool
+	}{
+		{
+			msg:    "malformed inverted index for FTS5 table main.messages_fts",
+			wantFT: true,
+		},
+		{
+			msg:    "row 42 missing from index messages_fts_idx",
+			wantFT: true,
+		},
+		{
+			msg:    "Tree 26 page 8231140 cell 2: Rowid 421177 out of order",
+			wantFT: false,
+		},
+		{
+			msg:    "non-unique entry in index sqlite_autoindex_messages_1",
+			wantFT: false,
+		},
+		{
+			msg:    "",
+			wantFT: false,
+		},
+	}
+
+	for _, tc := range tests {
+		if got := isFTSIntegrityError(tc.msg); got != tc.wantFT {
+			t.Errorf("isFTSIntegrityError(%q) = %v, want %v", tc.msg, got, tc.wantFT)
+		}
+	}
+}

docs/recovery.md

@@ -0,0 +1,87 @@
+# Database Recovery
+
+`msgvault verify you@gmail.com` runs `PRAGMA integrity_check` against the
+SQLite database before the Gmail comparison step. When that check fails,
+the recovery path depends on which part of the database is affected.
+
+## Search-index (FTS5) corruption
+
+Symptom (from `msgvault verify` output):
+
+```
+malformed inverted index for FTS5 table main.messages_fts
+```
+
+Fix:
+
+```
+msgvault rebuild-fts
+```
+
+This drops `messages_fts` and recreates it from the core tables
+(`messages`, `message_bodies`, `message_recipients`, `participants`). Peak
+extra disk usage is roughly the size of the FTS5 shadow tables — a few
+percent of the SQLite database.
+
+Stop `msgvault serve` and any MCP clients before running; `rebuild-fts`
+needs an exclusive write lock and will fail with a "database is busy"
+message otherwise.
+
+### Why SQLite's own rebuild pragma does not work
+
+`INSERT INTO messages_fts(messages_fts) VALUES('rebuild')` regenerates the
+FTS5 inverted index from the contentful shadow tables themselves. If those
+shadow tables are already malformed, the pragma reads the corruption right
+back out.
+
+`INSERT INTO messages_fts(messages_fts) VALUES('delete-all')` is rejected
+with `'delete-all' may only be used with a contentless or external content
+fts5 table`. msgvault's `messages_fts` is contentful by design (it stores
+its own copy of the searchable text), so `delete-all` is not available.
+
+`rebuild-fts` sidesteps both: it drops the virtual table entirely — which
+removes the shadow tables — then recreates it fresh and repopulates from
+the core tables.
+
+## Core-table corruption
+
+Symptom:
+
+```
+Tree 26 page 8231140 cell 2: Rowid 421177 out of order
+non-unique entry in index sqlite_autoindex_messages_1
+```
+
+Fix (requires free disk roughly equal to the size of the database):
+
+```
+sqlite3 ~/.msgvault/msgvault.db '.recover' | sqlite3 ~/.msgvault/recovered.db
+mv ~/.msgvault/msgvault.db ~/.msgvault/msgvault.db.bak
+mv ~/.msgvault/recovered.db ~/.msgvault/msgvault.db
+msgvault verify you@gmail.com
+```
+
+A leaner alternative that works on cleaner corruption:
+
+```
+sqlite3 ~/.msgvault/msgvault.db .dump | sqlite3 ~/.msgvault/new.db
+```
+
+If free disk is tight, individual corrupt rows can sometimes be repaired
+by hand — delete and re-insert the affected row(s) from their source
+(MIME blob, etc.). This is a last resort and only advisable if you can
+identify the specific rows flagged by `integrity_check`.
+
+## Before any repair
+
+Back up the database file. If the repair tool is interrupted or makes
+things worse, the backup is the only way back:
+
+```
+cp ~/.msgvault/msgvault.db ~/.msgvault/msgvault.db.bak
+```
+
+If the database has any activity, also copy the `-wal` and `-shm` sidecar
+files at the same instant (or run `msgvault` once to checkpoint the WAL
+into the main file before copying). A bare `.db` copy without its sidecars
+can itself be a source of corruption.

internal/store/dialect.go

@@ -90,6 +90,14 @@ type Dialect interface {
 	// (e.g., PostgreSQL includes tsvector in its main schema).
 	SchemaFTS() string
 
+	// FTSRebuildSchema tears down and recreates the FTS infrastructure from
+	// scratch — the caller is expected to follow up with a full backfill.
+	// Used to recover from malformed FTS shadow-table state that in-place
+	// rebuild operations (e.g., SQLite's rebuild pragma) cannot clear.
+	// SQLite: DROP TABLE IF EXISTS messages_fts + re-execute schema_sqlite.sql.
+	// PostgreSQL: TODO (REINDEX / recompute tsvector column).
+	FTSRebuildSchema(db *sql.DB) error
+
 	// Connection lifecycle
 
 	// InitConn performs driver-specific connection initialization.
@@ -128,4 +136,10 @@ type Dialect interface {
 	// This handles SQLite < 3.35 which doesn't support RETURNING.
 	// Always false for PostgreSQL (which always supports RETURNING).
 	IsReturningError(err error) bool
+
+	// IsBusyError returns true if the error indicates the database is held
+	// by another connection, either busy (SQLITE_BUSY) or locked
+	// (SQLITE_LOCKED). Used to surface actionable errors from maintenance
+	// commands that need exclusive access.
+	IsBusyError(err error) bool
 }