feat(server): operator removal of stale sessions/machines (SPEC-004 Task 5, server)

Admin-gated soft-delete + purge so operators can clear ghost machines/sessions
(the ~15-rows-for-one-host accumulation) from the console.

- migration 009: deleted_at on connect_sessions + connect_machines, with partial
  indexes WHERE deleted_at IS NULL.
- DELETE /api/machines/:agent_id?purge=true and DELETE /api/sessions/:id?purge=true
  soft-delete the row and purge the in-memory session (remove_session); the
  non-purge path keeps the legacy hard-delete / live-only disconnect. POST
  /api/machines/bulk-remove handles multi-select (batch cap 500). All admin-gated
  (AdminUser -> 403; tightens the prior any-user delete) and audited to
  connect_session_events (actor + target + trusted client IP).
- list/get queries filter deleted_at IS NULL so removed units leave the console;
  upsert revives (deleted_at = NULL) a genuinely-reconnecting machine. The
  keyed-reattach identity resolver (get_machine_by_id) is intentionally unfiltered.

Dashboard removal UI is the A3b follow-up. 86 server tests pass; fmt/clippy/test
clean. Implements specs/v2-stable-identity/plan.md Task 5 (server portion).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

server/src/db/machines.rs

@@ -58,6 +58,12 @@ pub struct Machine {
     /// machine binding stays authoritative and the claimed uid is NOT used to dedup
     /// (see `upsert_machine`).
     pub machine_uid: Option<String>,
+    /// Soft-delete marker (migration 009). When non-null the machine was
+    /// operator-purged (Task 5): it is excluded from every list/get query and is
+    /// never restored by the startup reconcile, but the row (and its audit
+    /// history) is retained. NULL = live. Nullable, so it is read NULL-tolerantly
+    /// in the manual `FromRow` below.
+    pub deleted_at: Option<DateTime<Utc>>,
 }
 
 impl<'r> FromRow<'r, PgRow> for Machine {
@@ -75,6 +81,8 @@ impl<'r> FromRow<'r, PgRow> for Machine {
             site: row.try_get("site")?,
             // Schema-nullable (migration 008); decode directly as Option.
             machine_uid: row.try_get("machine_uid")?,
+            // Schema-nullable (migration 009); decode directly as Option.
+            deleted_at: row.try_get("deleted_at")?,
             // Nullable-with-default columns mapped to non-`Option` Rust types: read as
             // `Option<T>` and fall back to the type default so a NULL cell never errors.
             is_elevated: row
@@ -129,6 +137,12 @@ impl<'r> FromRow<'r, PgRow> for Machine {
 /// aid, not a trust boundary. Only the un-keyed path supplies it; the keyed path's
 /// authority lives in the key→machine binding upstream (see
 /// `relay::agent_ws_handler`), never here.
+///
+/// SOFT-DELETE REVIVE (Task 5): both `ON CONFLICT DO UPDATE` arms clear
+/// `deleted_at` (set it back to NULL). A machine that was operator-purged but then
+/// genuinely reconnects is a live machine again, so it must reappear in the console
+/// rather than stay hidden behind a stale soft-delete marker. A purge of a truly
+/// gone host is permanent precisely because such a host never upserts again.
 pub async fn upsert_machine(
     pool: &PgPool,
     agent_id: &str,
@@ -156,7 +170,8 @@ pub async fn upsert_machine(
                     agent_id = EXCLUDED.agent_id,
                     hostname = EXCLUDED.hostname,
                     status = 'online',
-                    last_seen = NOW()
+                    last_seen = NOW(),
+                    deleted_at = NULL
                 RETURNING *
                 "#,
             )
@@ -178,7 +193,8 @@ pub async fn upsert_machine(
                 ON CONFLICT (agent_id) DO UPDATE SET
                     hostname = EXCLUDED.hostname,
                     status = 'online',
-                    last_seen = NOW()
+                    last_seen = NOW(),
+                    deleted_at = NULL
                 RETURNING *
                 "#,
             )
@@ -222,24 +238,31 @@ pub async fn update_machine_status(
     Ok(())
 }
 
-/// Get all persistent machines (for restore on startup)
+/// Get all persistent machines (for the dashboard list AND the startup restore).
+///
+/// Excludes operator-purged rows (`deleted_at IS NOT NULL`, migration 009 / Task 5):
+/// a soft-deleted machine must not reappear in `/api/machines` and must not be
+/// re-restored into the in-memory session manager on startup. This is the filter
+/// that makes the ghost-row purge stick.
 pub async fn get_all_machines(pool: &PgPool) -> Result<Vec<Machine>, sqlx::Error> {
     sqlx::query_as::<_, Machine>(
-        "SELECT * FROM connect_machines WHERE is_persistent = true ORDER BY hostname",
+        "SELECT * FROM connect_machines WHERE is_persistent = true AND deleted_at IS NULL ORDER BY hostname",
     )
     .fetch_all(pool)
     .await
 }
 
-/// Get machine by agent_id
+/// Get machine by agent_id (live rows only — excludes soft-deleted, Task 5).
 pub async fn get_machine_by_agent_id(
     pool: &PgPool,
     agent_id: &str,
 ) -> Result<Option<Machine>, sqlx::Error> {
-    sqlx::query_as::<_, Machine>("SELECT * FROM connect_machines WHERE agent_id = $1")
-        .bind(agent_id)
-        .fetch_optional(pool)
-        .await
+    sqlx::query_as::<_, Machine>(
+        "SELECT * FROM connect_machines WHERE agent_id = $1 AND deleted_at IS NULL",
+    )
+    .bind(agent_id)
+    .fetch_optional(pool)
+    .await
 }
 
 /// Get machine by its primary-key UUID (`connect_machines.id`).
@@ -248,6 +271,13 @@ pub async fn get_machine_by_agent_id(
 /// `verify_agent_key` back to its canonical `agent_id`, so persistent reattach
 /// binds to the authenticated identity rather than a client-supplied query
 /// param (Task 3 identity binding).
+///
+/// NOTE (Task 5): this deliberately does NOT filter `deleted_at IS NULL`. It is
+/// the authenticated-identity resolver for a `cak_`-keyed agent's reattach, not a
+/// dashboard read. If a previously operator-purged machine genuinely reconnects
+/// with a valid key, it must resolve so `upsert_machine` can revive it (the
+/// upsert clears `deleted_at`). The dashboard get-by-id path is
+/// `get_machine_by_agent_id`, which IS filtered.
 pub async fn get_machine_by_id(
     pool: &PgPool,
     machine_id: Uuid,
@@ -269,7 +299,11 @@ pub async fn mark_machine_offline(pool: &PgPool, agent_id: &str) -> Result<(), s
     Ok(())
 }
 
-/// Delete a machine record
+/// Hard-delete a machine record (legacy path, retained for backward compatibility).
+///
+/// Cascades to `connect_sessions` / `connect_session_events` via the FKs, so it
+/// also destroys audit history. The Task-5 operator-removal flow prefers
+/// [`soft_delete_machine`] instead, which keeps the row for the audit trail.
 pub async fn delete_machine(pool: &PgPool, agent_id: &str) -> Result<(), sqlx::Error> {
     sqlx::query("DELETE FROM connect_machines WHERE agent_id = $1")
         .bind(agent_id)
@@ -278,6 +312,24 @@ pub async fn delete_machine(pool: &PgPool, agent_id: &str) -> Result<(), sqlx::E
     Ok(())
 }
 
+/// Soft-delete (operator purge) a single machine by `agent_id` (Task 5).
+///
+/// Sets `deleted_at = NOW()` so the row is excluded from every list/get query and
+/// the startup reconcile, while retaining the row and its `connect_session_events`
+/// history for the audit trail. Only flips rows that are still live
+/// (`deleted_at IS NULL`), so a re-purge is a no-op rather than overwriting the
+/// original removal instant. Returns the number of rows affected (0 = unknown or
+/// already-purged `agent_id`), letting the caller distinguish a 404 from a success.
+pub async fn soft_delete_machine(pool: &PgPool, agent_id: &str) -> Result<u64, sqlx::Error> {
+    let result = sqlx::query(
+        "UPDATE connect_machines SET deleted_at = NOW() WHERE agent_id = $1 AND deleted_at IS NULL",
+    )
+    .bind(agent_id)
+    .execute(pool)
+    .await?;
+    Ok(result.rows_affected())
+}
+
 /// Update machine organization, site, and tags
 pub async fn update_machine_metadata(
     pool: &PgPool,
@@ -441,4 +493,157 @@ mod tests {
 
         cleanup(&pool, &["null-1", "null-2"], &[]).await;
     }
+
+    /// Helper: does `get_all_machines` (the dashboard list / startup restore query)
+    /// currently return a row with this agent_id?
+    async fn list_contains(pool: &PgPool, agent_id: &str) -> bool {
+        get_all_machines(pool)
+            .await
+            .expect("list machines")
+            .iter()
+            .any(|m| m.agent_id == agent_id)
+    }
+
+    /// Task 5: soft-deleting a machine sets `deleted_at` and excludes it from BOTH
+    /// the list query and the by-agent_id get — the core operator-removal guarantee
+    /// (a purged ghost row must not reappear in /api/machines).
+    #[tokio::test]
+    async fn soft_delete_machine_hides_from_list_and_get() {
+        let Some(pool) = test_pool().await else {
+            return; // no TEST_DATABASE_URL: skip (runs in CI)
+        };
+        let agent = "test-softdel-agent-001";
+        cleanup(&pool, &[agent], &[]).await;
+
+        let m = upsert_machine(&pool, agent, "SOFTDEL-HOST", true, None)
+            .await
+            .expect("create machine");
+        assert!(m.deleted_at.is_none(), "fresh row must be live");
+        assert!(
+            list_contains(&pool, agent).await,
+            "live machine must be listed"
+        );
+        assert!(
+            get_machine_by_agent_id(&pool, agent)
+                .await
+                .expect("get")
+                .is_some(),
+            "live machine must be gettable"
+        );
+
+        // Soft-delete.
+        let affected = soft_delete_machine(&pool, agent)
+            .await
+            .expect("soft delete");
+        assert_eq!(affected, 1, "exactly one live row flips to deleted");
+
+        // Excluded from list and get.
+        assert!(
+            !list_contains(&pool, agent).await,
+            "soft-deleted machine must NOT be listed"
+        );
+        assert!(
+            get_machine_by_agent_id(&pool, agent)
+                .await
+                .expect("get after delete")
+                .is_none(),
+            "soft-deleted machine must NOT be gettable by agent_id"
+        );
+
+        // The row still exists with a non-null deleted_at (history retained).
+        let deleted_at: Option<DateTime<Utc>> =
+            sqlx::query_scalar("SELECT deleted_at FROM connect_machines WHERE agent_id = $1")
+                .bind(agent)
+                .fetch_one(&pool)
+                .await
+                .expect("row still present");
+        assert!(
+            deleted_at.is_some(),
+            "row must be retained with deleted_at set"
+        );
+
+        // Re-purge is a no-op (does not overwrite the original instant).
+        let again = soft_delete_machine(&pool, agent)
+            .await
+            .expect("re-soft-delete");
+        assert_eq!(
+            again, 0,
+            "re-purge of an already-deleted row affects 0 rows"
+        );
+
+        cleanup(&pool, &[agent], &[]).await;
+    }
+
+    /// Task 5: a genuine reconnect (upsert) of a previously soft-deleted machine
+    /// REVIVES it — `deleted_at` is cleared so it reappears in the console. A purge
+    /// only sticks for a host that never upserts again.
+    #[tokio::test]
+    async fn upsert_revives_soft_deleted_machine() {
+        let Some(pool) = test_pool().await else {
+            return; // no TEST_DATABASE_URL: skip (runs in CI)
+        };
+        let agent = "test-revive-agent-001";
+        cleanup(&pool, &[agent], &[]).await;
+
+        upsert_machine(&pool, agent, "REVIVE-HOST", true, None)
+            .await
+            .expect("create");
+        soft_delete_machine(&pool, agent)
+            .await
+            .expect("soft delete");
+        assert!(!list_contains(&pool, agent).await, "purged: hidden");
+
+        // Reconnect.
+        let revived = upsert_machine(&pool, agent, "REVIVE-HOST", true, None)
+            .await
+            .expect("reconnect upsert");
+        assert!(
+            revived.deleted_at.is_none(),
+            "reconnect must clear deleted_at"
+        );
+        assert!(
+            list_contains(&pool, agent).await,
+            "revived machine must be listed again"
+        );
+
+        cleanup(&pool, &[agent], &[]).await;
+    }
+
+    /// Task 5: bulk soft-delete (as the bulk endpoint does, one id at a time)
+    /// removes every listed id from the live list.
+    #[tokio::test]
+    async fn bulk_soft_delete_hides_all_listed() {
+        let Some(pool) = test_pool().await else {
+            return; // no TEST_DATABASE_URL: skip (runs in CI)
+        };
+        let agents = ["test-bulk-a", "test-bulk-b", "test-bulk-c"];
+        cleanup(&pool, &agents, &[]).await;
+
+        for (i, a) in agents.iter().enumerate() {
+            upsert_machine(&pool, a, &format!("BULK-HOST-{i}"), true, None)
+                .await
+                .expect("create bulk machine");
+        }
+        for a in &agents {
+            assert!(list_contains(&pool, a).await, "{a} listed before bulk");
+        }
+
+        // Purge all three (the bulk endpoint loops soft_delete_machine per id).
+        let mut removed = 0u64;
+        for a in &agents {
+            removed += soft_delete_machine(&pool, a)
+                .await
+                .expect("bulk soft delete");
+        }
+        assert_eq!(removed, 3, "all three live rows flipped to deleted");
+
+        for a in &agents {
+            assert!(
+                !list_contains(&pool, a).await,
+                "{a} must be hidden after bulk purge"
+            );
+        }
+
+        cleanup(&pool, &agents, &[]).await;
+    }
 }