mcardle-features-dashboard

README.md

@@ -29,8 +29,52 @@ To learn more about Next.js, take a look at the following resources:
 
 You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
 
-## Deploy on Vercel
+## Persistent deployment (pm2)
 
-The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+The dashboard runs on a remote machine under [pm2](https://pm2.keymetrics.io/), which keeps the Next.js server alive across SSH disconnects and (once configured) reboots. Config lives in `ecosystem.config.js` at the repo root.
 
-Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
+Postgres credentials (`PG_user`, `PG_host`, `PG_database`, `PG_password`, `PG_port`) are read from `.env.local` by `next start` itself — pm2 doesn't need them duplicated in its own env block. `.env.local` is gitignored, so copy it to the remote machine out-of-band (not via git).
+
+### One-time setup on the remote machine
+
+```bash
+npm install -g pm2
+```
+
+### First deploy
+
+```bash
+cd /path/to/dpinterview-web
+npm install
+npm run build
+pm2 start ecosystem.config.js
+```
+
+### Survive reboots
+
+```bash
+pm2 save       # snapshot the currently running process list
+pm2 startup    # prints an OS-specific command — copy/paste and run it (needs sudo once)
+```
+
+Without `pm2 startup`, pm2 keeps the app alive across SSH disconnects but not across a machine reboot.
+
+### Everyday commands
+
+| Task | Command |
+|---|---|
+| List running apps | `pm2 list` |
+| Tail logs | `pm2 logs dpinterview-web` |
+| Restart (e.g. after deploy) | `pm2 restart dpinterview-web` |
+| Stop | `pm2 stop dpinterview-web` |
+| Remove from pm2 | `pm2 delete dpinterview-web` |
+| Live CPU/mem monitor | `pm2 monit` |
+
+### Redeploying a new version
+
+```bash
+git pull
+npm install
+npm run build
+pm2 restart dpinterview-web
+```

ecosystem.config.js

@@ -0,0 +1,19 @@
+module.exports = {
+  apps: [
+    {
+      name: "dpinterview-web",
+      cwd: __dirname,
+      script: "node_modules/.bin/next",
+      args: "start",
+      env: {
+        NODE_ENV: "production",
+        PORT: 3001,
+        // PG_user: "...",
+        // PG_host: "...",
+        // PG_database: "...",
+        // PG_password: "...",
+        // PG_port: "5432",
+      },
+    },
+  ],
+};

package.json

@@ -58,5 +58,8 @@
     "eslint-config-next": "15.2.4",
     "tailwindcss": "^4",
     "typescript": "^5"
+  },
+  "allowScripts": {
+    "sharp@0.33.5": true
   }
 }

src/app/api/v1/issues/dashboard-actions/route.ts

@@ -0,0 +1,34 @@
+import { DashboardActions } from "@/lib/models/DashboardActions";
+import { OVERRIDE_LEDGER_ACTIONS } from "@/lib/types/dashboard_actions";
+
+/**
+ * Handles the GET request to fetch rows from the dashboard_actions ledger,
+ * scoped to the manual override actions (audio QC bypass, runsheet datetime
+ * match) - not every dashboard_actions row, since the same table also logs
+ * routine edits (mark_primary, clear_role, etc.) unrelated to QC overrides.
+ *
+ * @param {Request} request - The incoming request object.
+ * @returns {Promise<Response>} - A promise that resolves to a Response object containing the fetched rows in JSON format.
+ */
+export async function GET(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000);
+    const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0);
+    const study_id = url.searchParams.get("study_id") ?? undefined;
+    const subject_id = url.searchParams.get("subject_id") ?? undefined;
+
+    const { rows, totalRows } = await DashboardActions.getByActions(
+        [...OVERRIDE_LEDGER_ACTIONS],
+        limit,
+        offset,
+        { study_id, subject_id }
+    );
+
+    const metadata = { totalRows, limit, offset };
+
+    return new Response(JSON.stringify({ metadata, rows }), {
+        headers: {
+            "Content-Type": "application/json",
+        },
+    });
+}

src/app/api/v1/issues/unresolved/audio-qc-failed/override/route.ts

@@ -0,0 +1,66 @@
+import { Transcribeme } from "@/lib/models/Transcribeme";
+
+/**
+ * Manually overrides a failed audio QC result, marking it approved for
+ * transcription despite failing the automated check. One-directional: the
+ * dpinterview push runners pick this up and push the file to TranscribeMe,
+ * an external action that can't be undone from here.
+ */
+export async function POST(request: Request): Promise<Response> {
+    try {
+        const body = await request.json();
+        const {
+            aqc_source_path,
+            interview_name,
+            source_type,
+            study_id,
+            subject_id,
+            journal_name,
+            aqc_metrics,
+            aqc_fail_reasons,
+        } = body;
+        if (!aqc_source_path) {
+            return new Response(JSON.stringify({ error: "Missing aqc_source_path parameter" }), {
+                status: 400,
+                headers: { "Content-Type": "application/json" },
+            });
+        }
+
+        try {
+            const { DashboardActions } = await import("@/lib/models/DashboardActions");
+            await DashboardActions.recordAction(
+                interview_name || "unknown",
+                "override_audio_qc",
+                aqc_source_path,
+                source_type || "other",
+                {
+                    study_id,
+                    subject_id,
+                    // interview_name is the "unknown" placeholder above for audio
+                    // journals (that column is reserved for real interview_name
+                    // values) - journal_name carries the actual identifier so the
+                    // ledger can still link/group audio-journal overrides.
+                    interview_name: interview_name ?? journal_name,
+                    source_type,
+                    aqc_metrics,
+                    aqc_fail_reasons,
+                }
+            );
+        } catch (e) {
+            // If DashboardActions fails, continue but log error
+            console.error("Failed to record dashboard action", e);
+        }
+
+        await Transcribeme.setAudioQcOverride(aqc_source_path);
+
+        return new Response(JSON.stringify({ success: true }), {
+            status: 200,
+            headers: { "Content-Type": "application/json" },
+        });
+    } catch (error: any) {
+        return new Response(JSON.stringify({ error: error.message || "Unknown error" }), {
+            status: 500,
+            headers: { "Content-Type": "application/json" },
+        });
+    }
+}

src/app/api/v1/issues/unresolved/audio-qc-failed/route.ts

@@ -0,0 +1,27 @@
+import { Transcribeme } from "@/lib/models/Transcribeme";
+
+/**
+ * Handles the GET request to fetch AMPSCZ interviews / audio journals whose
+ * combined audio failed the pre-transcription audio QC (transcribeme.audio_qc).
+ * Defaults to non-overridden failures only; pass ?includeOverridden=true to
+ * also return failures that have already been manually overridden.
+ *
+ * @param {Request} request - The incoming request object.
+ * @returns {Promise<Response>} - A promise that resolves to a Response object containing the fetched rows in JSON format.
+ */
+export async function GET(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000);
+    const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0);
+    const includeOverridden = url.searchParams.get("includeOverridden") === "true";
+
+    const { rows, totalRows } = await Transcribeme.getFailedAudioQc(limit, offset, includeOverridden);
+
+    const metadata = { totalRows, limit, offset, includeOverridden };
+
+    return new Response(JSON.stringify({ metadata, rows }), {
+        headers: {
+            "Content-Type": "application/json",
+        },
+    });
+}

src/app/api/v1/issues/unresolved/awaiting-vendor-transcription/route.ts

@@ -0,0 +1,24 @@
+import { Transcribeme } from "@/lib/models/Transcribeme";
+
+/**
+ * Handles the GET request to fetch AMPSCZ interviews / audio journals that
+ * have been pushed to TranscribeMe but have no delivered transcript yet.
+ *
+ * @param {Request} request - The incoming request object.
+ * @returns {Promise<Response>} - A promise that resolves to a Response object containing the fetched rows in JSON format.
+ */
+export async function GET(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000);
+    const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0);
+
+    const { rows, totalRows } = await Transcribeme.getAwaitingVendor(limit, offset);
+
+    const metadata = { totalRows, limit, offset };
+
+    return new Response(JSON.stringify({ metadata, rows }), {
+        headers: {
+            "Content-Type": "application/json",
+        },
+    });
+}

src/app/api/v1/issues/unresolved/missing/route.ts

@@ -23,6 +23,20 @@ export async function GET(request: Request): Promise<Response> {
     const url = new URL(request.url);
     const limit = url.searchParams.get('limit') || 5;
     const offset = url.searchParams.get('offset') || 0;
+    const study_id = url.searchParams.get('study_id');
+    const subject_id = url.searchParams.get('subject_id');
+
+    const params: string[] = [];
+    const extraConditions: string[] = [];
+    if (study_id) {
+        params.push(study_id);
+        extraConditions.push(`e.study_id = $${params.length}`);
+    }
+    if (subject_id) {
+        params.push(subject_id);
+        extraConditions.push(`e.subject_id = $${params.length}`);
+    }
+    const extraWhere = extraConditions.length > 0 ? `AND ${extraConditions.join(' AND ')}` : '';
 
     const baseQuery = `
     SELECT
@@ -48,14 +62,15 @@ export async function GET(request: Request): Promise<Response> {
                 AND i.interview_type = e.expected_interview_type
                 AND ABS(e.expected_interview_day - ip.interview_day) <= 10
         )
+        ${extraWhere}
 `;
 
     const countQuery = `SELECT COUNT(*) FROM (${baseQuery}) AS total_count`;
-    const countResult = await connection.query(countQuery);
+    const countResult = await connection.query(countQuery, params);
     const totalRows = countResult.rows[0].count;
 
     const limitedQuery = `${baseQuery} LIMIT ${limit} OFFSET ${offset}`;
-    const { rows } = await connection.query(limitedQuery);
+    const { rows } = await connection.query(limitedQuery, params);
 
     const metadata = {
         query: unformatSQL(limitedQuery),

src/app/api/v1/issues/unresolved/pending-transcription-push/route.ts

@@ -0,0 +1,24 @@
+import { Transcribeme } from "@/lib/models/Transcribeme";
+
+/**
+ * Handles the GET request to fetch AMPSCZ interviews / audio journals that
+ * passed audio QC but have not yet been pushed to TranscribeMe for transcription.
+ *
+ * @param {Request} request - The incoming request object.
+ * @returns {Promise<Response>} - A promise that resolves to a Response object containing the fetched rows in JSON format.
+ */
+export async function GET(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000);
+    const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0);
+
+    const { rows, totalRows } = await Transcribeme.getPendingPush(limit, offset);
+
+    const metadata = { totalRows, limit, offset };
+
+    return new Response(JSON.stringify({ metadata, rows }), {
+        headers: {
+            "Content-Type": "application/json",
+        },
+    });
+}

src/app/api/v1/issues/unresolved/pipeline-failures/datetime-override/route.ts

@@ -0,0 +1,71 @@
+import { DatetimeOverrides } from "@/lib/models/DatetimeOverrides";
+
+/**
+ * Looks up pending/consumed datetime overrides for a set of raw
+ * pipeline_failures identifiers, so the UI can show "pending crawler
+ * pickup" for a failure that already has an override recorded.
+ */
+export async function GET(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const identifiersParam = url.searchParams.get("identifiers");
+    const identifiers = identifiersParam ? identifiersParam.split(",").filter(Boolean) : [];
+
+    const overrides = await DatetimeOverrides.getByIdentifiers(identifiers);
+
+    return new Response(JSON.stringify({ overrides }), {
+        headers: { "Content-Type": "application/json" },
+    });
+}
+
+/**
+ * Records a staff-confirmed datetime for a raw file/directory that failed to
+ * date-parse (a "datetime_parse" pipeline_failures row), after a human has
+ * matched it to a runsheet entry on the Runsheet Match page. Does not mark
+ * the pipeline_failures row resolved directly - the Python crawler consumes
+ * this override on its next pass, imports the file normally, and resolves
+ * the ledger row itself.
+ */
+export async function POST(request: Request): Promise<Response> {
+    try {
+        const body = await request.json();
+        const { pf_identifier, study_id, subject_id, override_datetime } = body;
+
+        if (!pf_identifier || !override_datetime) {
+            return new Response(
+                JSON.stringify({ error: "Missing pf_identifier or override_datetime parameter" }),
+                { status: 400, headers: { "Content-Type": "application/json" } }
+            );
+        }
+
+        try {
+            const { DashboardActions } = await import("@/lib/models/DashboardActions");
+            await DashboardActions.recordAction(
+                "unknown",
+                "datetime_override_pipeline_failure",
+                pf_identifier,
+                "file_path",
+                { study_id, subject_id, override_datetime }
+            );
+        } catch (e) {
+            // If DashboardActions fails, continue but log error
+            console.error("Failed to record dashboard action", e);
+        }
+
+        await DatetimeOverrides.create(
+            pf_identifier,
+            study_id ?? null,
+            subject_id ?? null,
+            override_datetime
+        );
+
+        return new Response(JSON.stringify({ success: true }), {
+            status: 200,
+            headers: { "Content-Type": "application/json" },
+        });
+    } catch (error: any) {
+        return new Response(JSON.stringify({ error: error.message || "Unknown error" }), {
+            status: 500,
+            headers: { "Content-Type": "application/json" },
+        });
+    }
+}