diff --git a/README.md b/README.md index e215bc4..b4308ca 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,52 @@ To learn more about Next.js, take a look at the following resources: You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! -## Deploy on Vercel +## Persistent deployment (pm2) -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. +The dashboard runs on a remote machine under [pm2](https://pm2.keymetrics.io/), which keeps the Next.js server alive across SSH disconnects and (once configured) reboots. Config lives in `ecosystem.config.js` at the repo root. -Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. +Postgres credentials (`PG_user`, `PG_host`, `PG_database`, `PG_password`, `PG_port`) are read from `.env.local` by `next start` itself โ€” pm2 doesn't need them duplicated in its own env block. `.env.local` is gitignored, so copy it to the remote machine out-of-band (not via git). + +### One-time setup on the remote machine + +```bash +npm install -g pm2 +``` + +### First deploy + +```bash +cd /path/to/dpinterview-web +npm install +npm run build +pm2 start ecosystem.config.js +``` + +### Survive reboots + +```bash +pm2 save # snapshot the currently running process list +pm2 startup # prints an OS-specific command โ€” copy/paste and run it (needs sudo once) +``` + +Without `pm2 startup`, pm2 keeps the app alive across SSH disconnects but not across a machine reboot. + +### Everyday commands + +| Task | Command | +|---|---| +| List running apps | `pm2 list` | +| Tail logs | `pm2 logs dpinterview-web` | +| Restart (e.g. after deploy) | `pm2 restart dpinterview-web` | +| Stop | `pm2 stop dpinterview-web` | +| Remove from pm2 | `pm2 delete dpinterview-web` | +| Live CPU/mem monitor | `pm2 monit` | + +### Redeploying a new version + +```bash +git pull +npm install +npm run build +pm2 restart dpinterview-web +``` diff --git a/ecosystem.config.js b/ecosystem.config.js new file mode 100644 index 0000000..5de51b9 --- /dev/null +++ b/ecosystem.config.js @@ -0,0 +1,19 @@ +module.exports = { + apps: [ + { + name: "dpinterview-web", + cwd: __dirname, + script: "node_modules/.bin/next", + args: "start", + env: { + NODE_ENV: "production", + PORT: 3001, + // PG_user: "...", + // PG_host: "...", + // PG_database: "...", + // PG_password: "...", + // PG_port: "5432", + }, + }, + ], +}; diff --git a/package.json b/package.json index 11bffc1..613e047 100644 --- a/package.json +++ b/package.json @@ -58,5 +58,8 @@ "eslint-config-next": "15.2.4", "tailwindcss": "^4", "typescript": "^5" + }, + "allowScripts": { + "sharp@0.33.5": true } } diff --git a/src/app/api/v1/issues/dashboard-actions/route.ts b/src/app/api/v1/issues/dashboard-actions/route.ts new file mode 100644 index 0000000..401ff91 --- /dev/null +++ b/src/app/api/v1/issues/dashboard-actions/route.ts @@ -0,0 +1,34 @@ +import { DashboardActions } from "@/lib/models/DashboardActions"; +import { OVERRIDE_LEDGER_ACTIONS } from "@/lib/types/dashboard_actions"; + +/** + * Handles the GET request to fetch rows from the dashboard_actions ledger, + * scoped to the manual override actions (audio QC bypass, runsheet datetime + * match) - not every dashboard_actions row, since the same table also logs + * routine edits (mark_primary, clear_role, etc.) unrelated to QC overrides. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + const study_id = url.searchParams.get("study_id") ?? undefined; + const subject_id = url.searchParams.get("subject_id") ?? undefined; + + const { rows, totalRows } = await DashboardActions.getByActions( + [...OVERRIDE_LEDGER_ACTIONS], + limit, + offset, + { study_id, subject_id } + ); + + const metadata = { totalRows, limit, offset }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v1/issues/unresolved/audio-qc-failed/override/route.ts b/src/app/api/v1/issues/unresolved/audio-qc-failed/override/route.ts new file mode 100644 index 0000000..8a9d0b3 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/audio-qc-failed/override/route.ts @@ -0,0 +1,66 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +/** + * Manually overrides a failed audio QC result, marking it approved for + * transcription despite failing the automated check. One-directional: the + * dpinterview push runners pick this up and push the file to TranscribeMe, + * an external action that can't be undone from here. + */ +export async function POST(request: Request): Promise { + try { + const body = await request.json(); + const { + aqc_source_path, + interview_name, + source_type, + study_id, + subject_id, + journal_name, + aqc_metrics, + aqc_fail_reasons, + } = body; + if (!aqc_source_path) { + return new Response(JSON.stringify({ error: "Missing aqc_source_path parameter" }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + + try { + const { DashboardActions } = await import("@/lib/models/DashboardActions"); + await DashboardActions.recordAction( + interview_name || "unknown", + "override_audio_qc", + aqc_source_path, + source_type || "other", + { + study_id, + subject_id, + // interview_name is the "unknown" placeholder above for audio + // journals (that column is reserved for real interview_name + // values) - journal_name carries the actual identifier so the + // ledger can still link/group audio-journal overrides. + interview_name: interview_name ?? journal_name, + source_type, + aqc_metrics, + aqc_fail_reasons, + } + ); + } catch (e) { + // If DashboardActions fails, continue but log error + console.error("Failed to record dashboard action", e); + } + + await Transcribeme.setAudioQcOverride(aqc_source_path); + + return new Response(JSON.stringify({ success: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (error: any) { + return new Response(JSON.stringify({ error: error.message || "Unknown error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/src/app/api/v1/issues/unresolved/audio-qc-failed/route.ts b/src/app/api/v1/issues/unresolved/audio-qc-failed/route.ts new file mode 100644 index 0000000..f5de8c8 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/audio-qc-failed/route.ts @@ -0,0 +1,27 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +/** + * Handles the GET request to fetch AMPSCZ interviews / audio journals whose + * combined audio failed the pre-transcription audio QC (transcribeme.audio_qc). + * Defaults to non-overridden failures only; pass ?includeOverridden=true to + * also return failures that have already been manually overridden. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + const includeOverridden = url.searchParams.get("includeOverridden") === "true"; + + const { rows, totalRows } = await Transcribeme.getFailedAudioQc(limit, offset, includeOverridden); + + const metadata = { totalRows, limit, offset, includeOverridden }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v1/issues/unresolved/awaiting-vendor-transcription/route.ts b/src/app/api/v1/issues/unresolved/awaiting-vendor-transcription/route.ts new file mode 100644 index 0000000..e98749d --- /dev/null +++ b/src/app/api/v1/issues/unresolved/awaiting-vendor-transcription/route.ts @@ -0,0 +1,24 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +/** + * Handles the GET request to fetch AMPSCZ interviews / audio journals that + * have been pushed to TranscribeMe but have no delivered transcript yet. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + + const { rows, totalRows } = await Transcribeme.getAwaitingVendor(limit, offset); + + const metadata = { totalRows, limit, offset }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v1/issues/unresolved/missing/route.ts b/src/app/api/v1/issues/unresolved/missing/route.ts index 9bbeaad..e76e435 100644 --- a/src/app/api/v1/issues/unresolved/missing/route.ts +++ b/src/app/api/v1/issues/unresolved/missing/route.ts @@ -23,6 +23,20 @@ export async function GET(request: Request): Promise { const url = new URL(request.url); const limit = url.searchParams.get('limit') || 5; const offset = url.searchParams.get('offset') || 0; + const study_id = url.searchParams.get('study_id'); + const subject_id = url.searchParams.get('subject_id'); + + const params: string[] = []; + const extraConditions: string[] = []; + if (study_id) { + params.push(study_id); + extraConditions.push(`e.study_id = $${params.length}`); + } + if (subject_id) { + params.push(subject_id); + extraConditions.push(`e.subject_id = $${params.length}`); + } + const extraWhere = extraConditions.length > 0 ? `AND ${extraConditions.join(' AND ')}` : ''; const baseQuery = ` SELECT @@ -48,14 +62,15 @@ export async function GET(request: Request): Promise { AND i.interview_type = e.expected_interview_type AND ABS(e.expected_interview_day - ip.interview_day) <= 10 ) + ${extraWhere} `; const countQuery = `SELECT COUNT(*) FROM (${baseQuery}) AS total_count`; - const countResult = await connection.query(countQuery); + const countResult = await connection.query(countQuery, params); const totalRows = countResult.rows[0].count; const limitedQuery = `${baseQuery} LIMIT ${limit} OFFSET ${offset}`; - const { rows } = await connection.query(limitedQuery); + const { rows } = await connection.query(limitedQuery, params); const metadata = { query: unformatSQL(limitedQuery), diff --git a/src/app/api/v1/issues/unresolved/pending-transcription-push/route.ts b/src/app/api/v1/issues/unresolved/pending-transcription-push/route.ts new file mode 100644 index 0000000..1a9698b --- /dev/null +++ b/src/app/api/v1/issues/unresolved/pending-transcription-push/route.ts @@ -0,0 +1,24 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +/** + * Handles the GET request to fetch AMPSCZ interviews / audio journals that + * passed audio QC but have not yet been pushed to TranscribeMe for transcription. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + + const { rows, totalRows } = await Transcribeme.getPendingPush(limit, offset); + + const metadata = { totalRows, limit, offset }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v1/issues/unresolved/pipeline-failures/datetime-override/route.ts b/src/app/api/v1/issues/unresolved/pipeline-failures/datetime-override/route.ts new file mode 100644 index 0000000..5e6a809 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/pipeline-failures/datetime-override/route.ts @@ -0,0 +1,71 @@ +import { DatetimeOverrides } from "@/lib/models/DatetimeOverrides"; + +/** + * Looks up pending/consumed datetime overrides for a set of raw + * pipeline_failures identifiers, so the UI can show "pending crawler + * pickup" for a failure that already has an override recorded. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const identifiersParam = url.searchParams.get("identifiers"); + const identifiers = identifiersParam ? identifiersParam.split(",").filter(Boolean) : []; + + const overrides = await DatetimeOverrides.getByIdentifiers(identifiers); + + return new Response(JSON.stringify({ overrides }), { + headers: { "Content-Type": "application/json" }, + }); +} + +/** + * Records a staff-confirmed datetime for a raw file/directory that failed to + * date-parse (a "datetime_parse" pipeline_failures row), after a human has + * matched it to a runsheet entry on the Runsheet Match page. Does not mark + * the pipeline_failures row resolved directly - the Python crawler consumes + * this override on its next pass, imports the file normally, and resolves + * the ledger row itself. + */ +export async function POST(request: Request): Promise { + try { + const body = await request.json(); + const { pf_identifier, study_id, subject_id, override_datetime } = body; + + if (!pf_identifier || !override_datetime) { + return new Response( + JSON.stringify({ error: "Missing pf_identifier or override_datetime parameter" }), + { status: 400, headers: { "Content-Type": "application/json" } } + ); + } + + try { + const { DashboardActions } = await import("@/lib/models/DashboardActions"); + await DashboardActions.recordAction( + "unknown", + "datetime_override_pipeline_failure", + pf_identifier, + "file_path", + { study_id, subject_id, override_datetime } + ); + } catch (e) { + // If DashboardActions fails, continue but log error + console.error("Failed to record dashboard action", e); + } + + await DatetimeOverrides.create( + pf_identifier, + study_id ?? null, + subject_id ?? null, + override_datetime + ); + + return new Response(JSON.stringify({ success: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (error: any) { + return new Response(JSON.stringify({ error: error.message || "Unknown error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/src/app/api/v1/issues/unresolved/pipeline-failures/resolve/route.ts b/src/app/api/v1/issues/unresolved/pipeline-failures/resolve/route.ts new file mode 100644 index 0000000..bd80159 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/pipeline-failures/resolve/route.ts @@ -0,0 +1,45 @@ +import { PipelineFailures } from "@/lib/models/PipelineFailures"; + +export async function POST(request: Request): Promise { + try { + const body = await request.json(); + const { pf_stage, pf_identifier, pf_identifier_type, note } = body; + if (!pf_stage || !pf_identifier) { + return new Response(JSON.stringify({ error: "Missing pf_stage or pf_identifier parameter" }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + + // Record dashboard action. dashboard_actions.interview_name is NOT NULL, + // but not every ledger row is tied to an interview (e.g. study/batch/other + // identifier types) - only pass the identifier through as interview_name + // when it actually is one; da_target_id/da_target_type carry the real + // identifier either way. + try { + const { DashboardActions } = await import("@/lib/models/DashboardActions"); + await DashboardActions.recordAction( + pf_identifier_type === "interview_name" ? pf_identifier : "unknown", + "resolve_pipeline_failure", + pf_identifier, + pf_identifier_type || "other", + note ? { pf_stage, note } : { pf_stage } + ); + } catch (e) { + // If DashboardActions fails, continue but log error + console.error("Failed to record dashboard action", e); + } + + await PipelineFailures.resolve(pf_stage, pf_identifier, note); + + return new Response(JSON.stringify({ success: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (error: any) { + return new Response(JSON.stringify({ error: error.message || "Unknown error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/src/app/api/v1/issues/unresolved/pipeline-failures/route.ts b/src/app/api/v1/issues/unresolved/pipeline-failures/route.ts new file mode 100644 index 0000000..fb84d67 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/pipeline-failures/route.ts @@ -0,0 +1,33 @@ +import { PipelineFailures } from "@/lib/models/PipelineFailures"; + +/** + * Handles the GET request to fetch rows from the pipeline_ledger.pipeline_failures + * ledger. Defaults to unresolved failures only; pass ?includeResolved=true to + * also return failures that have already been marked resolved. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + const includeResolved = url.searchParams.get("includeResolved") === "true"; + const study_id = url.searchParams.get("study_id") ?? undefined; + const subject_id = url.searchParams.get("subject_id") ?? undefined; + const error_code = url.searchParams.get("errorCode") ?? undefined; + + const { rows, totalRows } = await PipelineFailures.getAll(includeResolved, limit, offset, { + study_id, + subject_id, + error_code, + }); + + const metadata = { totalRows, limit, offset, includeResolved }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v1/issues/unresolved/transcript-not-imported/route.ts b/src/app/api/v1/issues/unresolved/transcript-not-imported/route.ts new file mode 100644 index 0000000..3d7c475 --- /dev/null +++ b/src/app/api/v1/issues/unresolved/transcript-not-imported/route.ts @@ -0,0 +1,25 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +/** + * Handles the GET request to fetch AMPSCZ interviews / audio journals whose + * transcript was delivered by TranscribeMe (transcribeme.transcribeme_pull) + * but has not yet been registered in transcript_files by the import crawler. + * + * @param {Request} request - The incoming request object. + * @returns {Promise} - A promise that resolves to a Response object containing the fetched rows in JSON format. + */ +export async function GET(request: Request): Promise { + const url = new URL(request.url); + const limit = Math.min(Math.max(parseInt(url.searchParams.get("limit") ?? "", 10) || 50, 1), 5000); + const offset = Math.max(parseInt(url.searchParams.get("offset") ?? "", 10) || 0, 0); + + const { rows, totalRows } = await Transcribeme.getDownloadedNotImported(limit, offset); + + const metadata = { totalRows, limit, offset }; + + return new Response(JSON.stringify({ metadata, rows }), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v2/interviews/[interview_name]/transcription-status/route.ts b/src/app/api/v2/interviews/[interview_name]/transcription-status/route.ts new file mode 100644 index 0000000..5808ce4 --- /dev/null +++ b/src/app/api/v2/interviews/[interview_name]/transcription-status/route.ts @@ -0,0 +1,26 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +export async function GET( + _: Request, + props: { params: Promise<{ interview_name: string }> } +): Promise { + const params = await props.params; + const interview_name = params.interview_name; + + if (!interview_name) { + return new Response(JSON.stringify({ error: "Missing interview_name parameter" }), { + status: 400, + headers: { + "Content-Type": "application/json", + }, + }); + } + + const status = await Transcribeme.getStatusForInterview(interview_name); + + return new Response(JSON.stringify(status), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/api/v3/studies/[study_id]/subjects/[subject_id]/audioJournals/[journal_name]/transcription-status/route.ts b/src/app/api/v3/studies/[study_id]/subjects/[subject_id]/audioJournals/[journal_name]/transcription-status/route.ts new file mode 100644 index 0000000..9f01733 --- /dev/null +++ b/src/app/api/v3/studies/[study_id]/subjects/[subject_id]/audioJournals/[journal_name]/transcription-status/route.ts @@ -0,0 +1,26 @@ +import { Transcribeme } from "@/lib/models/Transcribeme"; + +export async function GET( + _: Request, + props: { params: Promise<{ study_id: string; subject_id: string; journal_name: string }> } +): Promise { + const params = await props.params; + const { study_id, subject_id, journal_name } = params; + + if (!study_id || !subject_id || !journal_name) { + return new Response(JSON.stringify({ error: "Missing study_id, subject_id or journal_name parameter" }), { + status: 400, + headers: { + "Content-Type": "application/json", + }, + }); + } + + const status = await Transcribeme.getStatusForAudioJournal(study_id, subject_id, journal_name); + + return new Response(JSON.stringify(status), { + headers: { + "Content-Type": "application/json", + }, + }); +} diff --git a/src/app/interviews/[interview_name]/page.tsx b/src/app/interviews/[interview_name]/page.tsx index addc89a..f555507 100644 --- a/src/app/interviews/[interview_name]/page.tsx +++ b/src/app/interviews/[interview_name]/page.tsx @@ -31,6 +31,7 @@ import InterviewRunsheet from '@/components/domain/InterviewRunsheet'; import QcForm from '@/components/domain/QcForm'; import Transcript from '@/components/domain/TranscriptE'; import InterviewPdfReport from '@/components/domain/InterviewPdfReport'; +import TranscriptionPipelineStatus from '@/components/domain/TranscriptionPipelineStatus'; const { Paragraph } = AntTypography; @@ -415,6 +416,7 @@ export default function Page({ ๐Ÿšฉ QC Issues ๐Ÿ“„ PDF Report ๐Ÿ“„ Transcript + ๐Ÿ”Š Transcription Pipeline @@ -441,12 +443,22 @@ export default function Page({ - + + + Transcription Pipeline + + + + diff --git a/src/app/issues/audioQcFailed/page.tsx b/src/app/issues/audioQcFailed/page.tsx new file mode 100644 index 0000000..b15f252 --- /dev/null +++ b/src/app/issues/audioQcFailed/page.tsx @@ -0,0 +1,198 @@ +'use client' +import * as React from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; +import Button from '@mui/material/Button'; +import Switch from '@mui/material/Switch'; +import FormControlLabel from '@mui/material/FormControlLabel'; + +import { FailedAudioQcRow } from '@/lib/types/transcribeme'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = FailedAudioQcRow & { id: number }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'source_type', label: 'Source' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, + { + field: 'aqc_fail_reasons', + label: 'Fail Reason', + // A row can fail for several reasons at once, so it's counted once per reason + // rather than once per distinct reason-combination. + extractKeys: (row) => row.aqc_fail_reasons + ? Object.entries(row.aqc_fail_reasons).filter(([, failed]) => failed).map(([reason]) => reason) + : [], + }, +]; + +function linkFor(row: FailedAudioQcRow): string { + if (row.source_type === 'audio_journal') { + return `/studies/${row.study_id}/subjects/${row.subject_id}/journals/${row.interview_name}`; + } + return `/interviews/${row.interview_name}`; +} + +async function overrideAudioQc(row: FailedAudioQcRow): Promise { + const response = await fetch('/api/v1/issues/unresolved/audio-qc-failed/override', { + method: 'POST', + body: JSON.stringify({ + aqc_source_path: row.aqc_source_path, + interview_name: row.source_type === 'interview' ? row.interview_name : undefined, + source_type: row.source_type, + // Carried into dashboard_actions.da_metadata for the override ledger - + // lets reporting group/link without re-joining transcribeme.audio_qc, + // and preserves what the automated check found (useful for later + // analysis of which QC checks tend to be false positives). + study_id: row.study_id, + subject_id: row.subject_id, + journal_name: row.source_type === 'audio_journal' ? row.interview_name : undefined, + aqc_metrics: row.aqc_metrics, + aqc_fail_reasons: row.aqc_fail_reasons, + }), + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + throw new Error('Failed to override audio QC'); + } +} + +export default function AudioQcFailedIssues() { + const [rows, setRows] = useState(null); + const [includeOverridden, setIncludeOverridden] = useState(false); + + const loadRows = useCallback(() => { + setRows(null); + fetch(`/api/v1/issues/unresolved/audio-qc-failed?limit=2000&includeOverridden=${includeOverridden}`) + .then((res) => res.json()) + .then((data) => setRows(data.rows)); + }, [includeOverridden]); + + useEffect(() => { + loadRows(); + }, [loadRows]); + + const handleOverride = useCallback((row: FailedAudioQcRow) => { + const confirmed = window.confirm( + `Push "${row.interview_name}" to TranscribeMe despite failing audio QC?\n\n` + + 'This cannot be undone from the dashboard - once a push runner picks it up, ' + + 'the file is uploaded to TranscribeMe.' + ); + if (!confirmed) { + return; + } + + const promise = overrideAudioQc(row).then(() => loadRows()); + toast.promise(promise, { + loading: 'Overriding audio QC...', + success: 'Audio QC overridden - will be pushed to TranscribeMe', + error: 'Failed to override audio QC', + }); + }, [loadRows]); + + const columns: GridColDef[] = React.useMemo(() => [ + { + field: 'interview_name', + headerName: 'Name', + width: 300, + renderCell: (params) => ( + {params.value} + ) + }, + { + field: 'aqc_override', + headerName: 'QC Override', + width: 140, + sortable: false, + filterable: false, + renderCell: (params) => ( + params.row.aqc_override ? ( + โœ… Bypassed + ) : ( + + ) + ), + }, + { field: 'source_type', headerName: 'Source', width: 130 }, + { field: 'subject_id', headerName: 'Subject ID', width: 150 }, + { field: 'study_id', headerName: 'Study ID', width: 150 }, + { + field: 'aqc_fail_reasons', + headerName: 'Fail Reasons', + width: 350, + valueGetter: (value) => value ? Object.keys(value).join(', ') : '', + }, + { field: 'aqc_timestamp', headerName: 'QC Timestamp', width: 200 }, + ], [handleOverride]); + + const gridRows: GridRow[] | null = useMemo( + () => rows?.map((row, index) => ({ id: index, ...row })) ?? null, + [rows] + ); + + const dataGridProps: MuiDataGridProps | null = useMemo(() => { + if (!gridRows) return null; + return { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20], + selectable: true, + }; + }, [gridRows, columns]); + + return ( +
+ + Failed Audio QC + + + + Before being sent to TranscribeMe, combined audio is checked for basic + quality issues (silence, clipping, DC offset, voice activity). These + interviews / audio journals failed that check and are not being + transcribed until someone looks at why - or manually overrides the + result below. + + + setIncludeOverridden(e.target.checked)} />} + label="Show overridden files" + sx={{ mb: 3 }} + /> + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No audio QC failures found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} recordings failed audio QC: + + +
+ +
+ + )} +
+ ); +} diff --git a/src/app/issues/awaitingVendorTranscription/page.tsx b/src/app/issues/awaitingVendorTranscription/page.tsx new file mode 100644 index 0000000..6d6d4b1 --- /dev/null +++ b/src/app/issues/awaitingVendorTranscription/page.tsx @@ -0,0 +1,114 @@ +'use client' +import * as React from 'react'; +import { useEffect, useState } from 'react'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; + +import { AwaitingVendorRow } from '@/lib/types/transcribeme'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'source_type', label: 'Source' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, + { field: 'source_language', label: 'Language' }, +]; + +function linkFor(row: AwaitingVendorRow): string { + if (row.source_type === 'audio_journal') { + return `/studies/${row.study_id}/subjects/${row.subject_id}/journals/${row.interview_name}`; + } + return `/interviews/${row.interview_name}`; +} + +export default function AwaitingVendorTranscriptionIssues() { + const [dataGridProps, setDataGridProps] = useState(null); + + const columns: GridColDef[] = React.useMemo(() => [ + { + field: 'interview_name', + headerName: 'Name', + width: 350, + renderCell: (params) => ( + {params.value} + ) + }, + { field: 'source_type', headerName: 'Source', width: 130 }, + { field: 'subject_id', headerName: 'Subject ID', width: 150 }, + { field: 'study_id', headerName: 'Study ID', width: 150 }, + { field: 'source_language', headerName: 'Language', width: 120 }, + { field: 'sftp_upload_timestamp', headerName: 'Pushed At', width: 200 }, + { + field: 'hours_waiting', + headerName: 'Hours Waiting', + width: 140, + valueGetter: (value) => typeof value === 'number' ? Math.round(value) : value, + }, + ], []); + + useEffect(() => { + fetch('/api/v1/issues/unresolved/awaiting-vendor-transcription?limit=2000') + .then((res) => res.json()) + .then((data) => { + const gridRows = data.rows.map((row: AwaitingVendorRow, index: number) => ({ + id: index, + ...row, + })); + + const props: MuiDataGridProps = { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20], + selectable: true + }; + setDataGridProps(props); + }); + }, [columns]); + + return ( +
+ + Awaiting Vendor Transcription + + + + These interviews / audio journals have been pushed to TranscribeMe + but no transcript has been pulled back yet. A long wait here may mean + the file is stuck at the vendor rather than simply queued. + + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No recordings awaiting vendor transcription found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} recordings are awaiting a transcript from TranscribeMe: + + +
+ +
+ + )} +
+ ); +} diff --git a/src/app/issues/missing/page.tsx b/src/app/issues/missing/page.tsx index 41026cf..52de7fe 100644 --- a/src/app/issues/missing/page.tsx +++ b/src/app/issues/missing/page.tsx @@ -9,6 +9,17 @@ import Alert from '@mui/joy/Alert'; import { InterviewIssue } from '@/app/api/v1/issues/unresolved/missing/route'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = InterviewIssue & { id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, + { field: 'event_name', label: 'REDCap Event' }, + { field: 'expected_day', label: 'Expected Day' }, +]; export default function Issues() { const [dataGridProps, setDataGridProps] = useState(null); @@ -123,6 +134,7 @@ export default function Issues() { The following {dataGridProps.rows.length} runsheets no not have any uploaded data: +
diff --git a/src/app/issues/multiCombinedAudio/page.tsx b/src/app/issues/multiCombinedAudio/page.tsx index 9359f10..24df42e 100644 --- a/src/app/issues/multiCombinedAudio/page.tsx +++ b/src/app/issues/multiCombinedAudio/page.tsx @@ -8,6 +8,15 @@ import Link from '@mui/material/Link'; import { InterviewIssue } from '@/app/api/v1/issues/unresolved/multiple-combined-audio/route'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = InterviewIssue & { id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; export default function Issues() { const [dataGridProps, setDataGridProps] = useState(null); @@ -102,6 +111,11 @@ export default function Issues() { The following {dataGridProps.rows.length} interviews have multiple combined audio files: +
diff --git a/src/app/issues/multiPart/page.tsx b/src/app/issues/multiPart/page.tsx index 2508964..65c8471 100644 --- a/src/app/issues/multiPart/page.tsx +++ b/src/app/issues/multiPart/page.tsx @@ -8,6 +8,15 @@ import Link from '@mui/material/Link'; import { InterviewIssue } from '@/app/api/v1/issues/unresolved/multi-part/route'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = InterviewIssue & { id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; export default function Issues() { const [dataGridProps, setDataGridProps] = useState(null); @@ -102,6 +111,11 @@ export default function Issues() { The following {dataGridProps.rows.length} interviews have multiple parts: +
diff --git a/src/app/issues/noRunsheet/page.tsx b/src/app/issues/noRunsheet/page.tsx index 3e9ded3..2196b87 100644 --- a/src/app/issues/noRunsheet/page.tsx +++ b/src/app/issues/noRunsheet/page.tsx @@ -9,6 +9,15 @@ import Alert from '@mui/joy/Alert'; import {DbInterview} from '@/lib/types/interview'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = { id: string; interview_name: string; interview_type: string; subject_id: string; study_id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; export default function MissingRunsheet() { const [dataGridProps, setDataGridProps] = useState(null); @@ -117,6 +126,7 @@ export default function MissingRunsheet() { The following {dataGridProps.rows.length} interviews have data uploaded, but no runsheets associated with them. +
diff --git a/src/app/issues/noTranscript/page.tsx b/src/app/issues/noTranscript/page.tsx index c63760b..1ad46c0 100644 --- a/src/app/issues/noTranscript/page.tsx +++ b/src/app/issues/noTranscript/page.tsx @@ -9,6 +9,15 @@ import Alert from '@mui/joy/Alert'; import {DbInterview} from '@/lib/types/interview'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = { id: string; interview_name: string; interview_type: string; subject_id: string; study_id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; export default function MissingTranscrips() { const [dataGridProps, setDataGridProps] = useState(null); @@ -117,6 +126,7 @@ export default function MissingTranscrips() { The following {dataGridProps.rows.length} interviews have data uploaded, but no transcripts associated with them. +
diff --git a/src/app/issues/overrideLedger/page.tsx b/src/app/issues/overrideLedger/page.tsx new file mode 100644 index 0000000..ef57028 --- /dev/null +++ b/src/app/issues/overrideLedger/page.tsx @@ -0,0 +1,227 @@ +'use client' +import * as React from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; +import Alert from '@mui/joy/Alert'; + +import { DbDashboardAction } from '@/lib/types/dashboard_actions'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +const ACTION_LABELS: Record = { + override_audio_qc: 'Audio QC Bypass', + datetime_override_pipeline_failure: 'Runsheet Datetime Match', +}; + +// study_id/subject_id aren't real dashboard_actions columns - every override +// call site records them inside da_metadata instead, so they're pulled out +// here into real fields the grid/aggregation can group and link on. +type LedgerRow = DbDashboardAction & { + id: number; + ledger_study_id: string | null; + ledger_subject_id: string | null; +}; + +function metaString(row: DbDashboardAction, key: string): string | null { + const value = row.da_metadata?.[key]; + return typeof value === 'string' ? value : null; +} + +function copyToClipboard(text: string) { + navigator.clipboard.writeText(text); + toast.success('Copied to clipboard'); +} + +// Mirrors the linkFor helpers on the audioQcFailed / pipelineFailures pages - +// prefer a direct interview/journal link when we know the name, otherwise +// fall back to the subject page, otherwise there's nothing linkable. +function linkFor(row: LedgerRow): string | null { + const interviewName = metaString(row, 'interview_name'); + const sourceType = metaString(row, 'source_type'); + if (interviewName && sourceType === 'audio_journal' && row.ledger_study_id && row.ledger_subject_id) { + return `/studies/${row.ledger_study_id}/subjects/${row.ledger_subject_id}/journals/${interviewName}`; + } + if (interviewName) { + return `/interviews/${interviewName}`; + } + if (row.ledger_study_id && row.ledger_subject_id) { + return `/studies/${row.ledger_study_id}/subjects/${row.ledger_subject_id}`; + } + return null; +} + +function detailsFor(row: LedgerRow): string { + const meta = row.da_metadata ?? {}; + const overrideDatetime = metaString(row, 'override_datetime'); + if (overrideDatetime) { + return `New datetime: ${overrideDatetime}`; + } + const reasons = meta.aqc_fail_reasons as Record | undefined; + if (reasons) { + const failed = Object.entries(reasons).filter(([, wasFailed]) => wasFailed).map(([reason]) => reason); + return failed.length ? `Failed: ${failed.join(', ')}` : 'No fail reasons recorded'; + } + return ''; +} + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'da_action', label: 'Override Type' }, + { field: 'ledger_study_id', label: 'Study ID' }, + { field: 'ledger_subject_id', label: 'Subject ID' }, + { + field: 'da_metadata', + label: 'Audio QC Fail Reason', + // A row can fail for several reasons at once, so it's counted once per + // reason rather than once per distinct reason-combination. Rows with no + // aqc_fail_reasons (e.g. datetime overrides) contribute nothing here. + extractKeys: (row) => { + const reasons = row.da_metadata?.aqc_fail_reasons as Record | undefined; + return reasons + ? Object.entries(reasons).filter(([, wasFailed]) => wasFailed).map(([reason]) => reason) + : []; + }, + }, +]; + +const FETCH_LIMIT = 3000; + +export default function OverrideLedgerIssues() { + const [rows, setRows] = useState(null); + const [totalRows, setTotalRows] = useState(null); + + const loadRows = useCallback(() => { + setRows(null); + fetch(`/api/v1/issues/dashboard-actions?limit=${FETCH_LIMIT}`) + .then((res) => res.json()) + .then((data) => { + setRows(data.rows); + setTotalRows(data.metadata?.totalRows ?? null); + }); + }, []); + + useEffect(() => { + loadRows(); + }, [loadRows]); + + const columns: GridColDef[] = useMemo(() => [ + { field: 'da_timestamp', headerName: 'When', width: 200 }, + { + field: 'da_action', + headerName: 'Override Type', + width: 210, + valueGetter: (value) => ACTION_LABELS[value as string] ?? value, + }, + { + field: 'da_target_id', + headerName: 'File / Identifier', + width: 320, + renderCell: (params) => ( + copyToClipboard(params.value)}> + {params.value} + + ), + }, + { + field: 'linked_name', + headerName: 'Interview / Subject', + width: 220, + sortable: false, + filterable: false, + renderCell: (params) => { + const href = linkFor(params.row as LedgerRow); + const label = metaString(params.row, 'interview_name') ?? params.row.ledger_subject_id ?? ''; + if (!href) return label || null; + return {label}; + }, + }, + { field: 'ledger_study_id', headerName: 'Study ID', width: 130 }, + { field: 'ledger_subject_id', headerName: 'Subject ID', width: 130 }, + { + field: 'details', + headerName: 'Details', + width: 320, + sortable: false, + filterable: false, + valueGetter: (_value, row) => detailsFor(row as LedgerRow), + }, + { field: 'da_user_id', headerName: 'Performed By', width: 150 }, + ], []); + + const gridRows: LedgerRow[] | null = useMemo( + () => rows?.map((row) => ({ + id: row.da_id, + ...row, + ledger_study_id: metaString(row, 'study_id'), + ledger_subject_id: metaString(row, 'subject_id'), + })) ?? null, + [rows] + ); + + const dataGridProps: MuiDataGridProps | null = useMemo(() => { + if (!gridRows) return null; + return { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20, 50], + selectable: true, + }; + }, [gridRows, columns]); + + return ( +
+ + Override Ledger + + + + Audit trail of manual overrides performed from the dashboard: audio QC + bypasses and runsheet datetime matches. Use this to report on how many + files were addressed through these mechanisms, and to trace back an + individual action if it needs to be reviewed or undone. + + + {totalRows !== null && totalRows >= FETCH_LIMIT && ( + + This page fetches at most {FETCH_LIMIT} rows, and the ledger currently has{' '} + {totalRows}{totalRows > FETCH_LIMIT ? '+' : ''} matching entries. Counts and + aggregates below may be incomplete โ€” contact a maintainer to raise the limit + or add pagination. + + )} + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No manual overrides recorded yet. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} override actions were recorded: + + + + + + + )} +
+ ); +} diff --git a/src/app/issues/page.tsx b/src/app/issues/page.tsx index 47907e9..b259e39 100644 --- a/src/app/issues/page.tsx +++ b/src/app/issues/page.tsx @@ -75,6 +75,76 @@ export default function Home() { + +
+ + ๐ŸŽ™๏ธ Failed Audio QC + + + Combined audio that failed pre-transcription QC (silence, clipping, voice activity) and is not being transcribed. + +
+ + +
+ + ๐Ÿ“ค Pending Transcription Push + + + Audio that passed QC but has not yet been pushed to TranscribeMe. + +
+ + +
+ + โณ Awaiting Vendor Transcription + + + Audio pushed to TranscribeMe with no transcript delivered back yet. + +
+ + +
+ + ๐Ÿ“ฅ Transcript Not Imported + + + Transcripts delivered by TranscribeMe that have not yet appeared in transcript_files. + +
+ + +
+ + ๐Ÿงพ Pipeline Failures + + + Errors raised across pipeline stages/crawlers, with occurrence counts and resolution tracking. + +
+ + +
+ + ๐Ÿ”— Runsheet Match + + + Match malformed interview files (datetime_parse failures) to missing runsheet entries, subject by subject. + +
+ + +
+ + ๐Ÿ“‹ Override Ledger + + + Audit trail of manual overrides (audio QC bypasses, runsheet datetime matches) - which files were addressed and by what mechanism. + +
+ diff --git a/src/app/issues/pendingTranscriptionPush/page.tsx b/src/app/issues/pendingTranscriptionPush/page.tsx new file mode 100644 index 0000000..ff842d8 --- /dev/null +++ b/src/app/issues/pendingTranscriptionPush/page.tsx @@ -0,0 +1,103 @@ +'use client' +import * as React from 'react'; +import { useEffect, useState } from 'react'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; + +import { PendingPushRow } from '@/lib/types/transcribeme'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'source_type', label: 'Source' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; + +function linkFor(row: PendingPushRow): string { + if (row.source_type === 'audio_journal') { + return `/studies/${row.study_id}/subjects/${row.subject_id}/journals/${row.interview_name}`; + } + return `/interviews/${row.interview_name}`; +} + +export default function PendingTranscriptionPushIssues() { + const [dataGridProps, setDataGridProps] = useState(null); + + const columns: GridColDef[] = React.useMemo(() => [ + { + field: 'interview_name', + headerName: 'Name', + width: 350, + renderCell: (params) => ( + {params.value} + ) + }, + { field: 'source_type', headerName: 'Source', width: 130 }, + { field: 'subject_id', headerName: 'Subject ID', width: 150 }, + { field: 'study_id', headerName: 'Study ID', width: 150 }, + { field: 'aqc_timestamp', headerName: 'QC Passed At', width: 200 }, + ], []); + + useEffect(() => { + fetch('/api/v1/issues/unresolved/pending-transcription-push?limit=2000') + .then((res) => res.json()) + .then((data) => { + const gridRows = data.rows.map((row: PendingPushRow, index: number) => ({ + id: index, + ...row, + })); + + const props: MuiDataGridProps = { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20], + selectable: true + }; + setDataGridProps(props); + }); + }, [columns]); + + return ( +
+ + Pending Transcription Push + + + + These interviews / audio journals passed audio QC but have not yet + been pushed to TranscribeMe over SFTP. Usually this clears on the + next push run - a growing backlog here means the push job may be + stuck. + + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No recordings pending push found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} recordings passed QC but have not been pushed: + + +
+ +
+ + )} +
+ ); +} diff --git a/src/app/issues/pipelineFailures/page.tsx b/src/app/issues/pipelineFailures/page.tsx new file mode 100644 index 0000000..93a9f49 --- /dev/null +++ b/src/app/issues/pipelineFailures/page.tsx @@ -0,0 +1,217 @@ +'use client' +import * as React from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; +import Button from '@mui/material/Button'; +import Switch from '@mui/material/Switch'; +import FormControlLabel from '@mui/material/FormControlLabel'; +import Alert from '@mui/joy/Alert'; + +import { PipelineFailureRow } from '@/lib/types/pipeline_failures'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +type GridRow = PipelineFailureRow & { id: number }; + +// Not every ledger row is tied to an interview - only interview_name/study/subject +// identifiers (and file_path/batch/other rows where the study+subject happen to +// be known) can be drilled into. Everything else falls back to copyable text, +// since there's no portable way to link an arbitrary file_path in this app today. +function linkFor(row: PipelineFailureRow): string | null { + if (row.pf_identifier_type === 'interview_name') { + return `/interviews/${row.pf_identifier}`; + } + if (row.pf_study_id && row.pf_subject_id) { + return `/studies/${row.pf_study_id}/subjects/${row.pf_subject_id}`; + } + if (row.pf_identifier_type === 'study' && row.pf_study_id) { + return `/studies/${row.pf_study_id}`; + } + return null; +} + +function copyToClipboard(text: string) { + navigator.clipboard.writeText(text); + toast.success('Copied to clipboard'); +} + +async function resolveFailure(row: PipelineFailureRow): Promise { + const note = window.prompt('Optional note for resolving this failure:') ?? undefined; + + const response = await fetch('/api/v1/issues/unresolved/pipeline-failures/resolve', { + method: 'POST', + body: JSON.stringify({ + pf_stage: row.pf_stage, + pf_identifier: row.pf_identifier, + pf_identifier_type: row.pf_identifier_type, + note, + }), + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + throw new Error('Failed to resolve failure'); + } +} + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'pf_stage', label: 'Stage' }, + { field: 'pf_error_code', label: 'Error Code' }, + { field: 'pf_identifier_type', label: 'Identifier Type' }, + { field: 'pf_study_id', label: 'Study ID' }, + { field: 'pf_resolved', label: 'Resolved' }, +]; + +const FETCH_LIMIT = 3000; + +export default function PipelineFailuresIssues() { + const [rows, setRows] = useState(null); + const [totalRows, setTotalRows] = useState(null); + const [includeResolved, setIncludeResolved] = useState(false); + + const loadRows = useCallback(() => { + setRows(null); + fetch(`/api/v1/issues/unresolved/pipeline-failures?limit=${FETCH_LIMIT}&includeResolved=${includeResolved}`) + .then((res) => res.json()) + .then((data) => { + setRows(data.rows); + setTotalRows(data.metadata?.totalRows ?? null); + }); + }, [includeResolved]); + + useEffect(() => { + loadRows(); + }, [loadRows]); + + const handleResolve = useCallback((row: PipelineFailureRow) => { + const promise = resolveFailure(row).then(() => loadRows()); + toast.promise(promise, { + loading: 'Resolving failure...', + success: 'Failure resolved', + error: 'Failed to resolve failure', + }); + }, [loadRows]); + + const columns: GridColDef[] = useMemo(() => [ + { field: 'pf_stage', headerName: 'Stage', width: 180 }, + { field: 'pf_error_code', headerName: 'Error Code', width: 200 }, + { + field: 'pf_identifier', + headerName: 'Identifier', + width: 320, + renderCell: (params) => { + const href = linkFor(params.row); + if (href) { + return {params.value}; + } + return ( + copyToClipboard(params.value)}> + {params.value} + + ); + }, + }, + { field: 'pf_identifier_type', headerName: 'Identifier Type', width: 140 }, + { field: 'pf_study_id', headerName: 'Study ID', width: 130 }, + { field: 'pf_subject_id', headerName: 'Subject ID', width: 130 }, + { field: 'pf_error', headerName: 'Error', width: 350 }, + { field: 'pf_occurrence_count', headerName: 'Occurrences', width: 120, type: 'number' }, + { field: 'pf_first_seen_at', headerName: 'First Seen', width: 200 }, + { field: 'pf_last_seen_at', headerName: 'Last Seen', width: 200 }, + { field: 'pf_resolved', headerName: 'Resolved', width: 110, type: 'boolean' }, + { field: 'pf_resolved_note', headerName: 'Resolved Note', width: 250 }, + { + field: 'actions', + headerName: 'Actions', + width: 120, + sortable: false, + filterable: false, + renderCell: (params) => ( + params.row.pf_resolved ? null : ( + + ) + ), + }, + ], [handleResolve]); + + const gridRows: GridRow[] | null = useMemo( + () => rows?.map((row) => ({ id: row.pf_id, ...row })) ?? null, + [rows] + ); + + const dataGridProps: MuiDataGridProps | null = useMemo(() => { + if (!gridRows) return null; + return { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20, 50], + selectable: true, + }; + }, [gridRows, columns]); + + return ( +
+ + Pipeline Failures + + + + Errors raised across every pipeline stage/crawler, deduplicated by stage + + identifier (recurrences bump the occurrence count and refresh "last seen" + instead of creating a new row). + + + setIncludeResolved(e.target.checked)} />} + label="Show resolved failures" + sx={{ mb: 3 }} + /> + + {totalRows !== null && totalRows >= FETCH_LIMIT && ( + + This page fetches at most {FETCH_LIMIT} rows, and the ledger currently has{' '} + {totalRows}{totalRows > FETCH_LIMIT ? '+' : ''} unresolved failures matching this filter. + Counts and aggregates below may be incomplete โ€” contact a maintainer to raise the limit + or add pagination. + + )} + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No pipeline failures found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} failures were found: + + + + + + + )} +
+ ); +} diff --git a/src/app/issues/runsheetMatch/[study_id]/[subject_id]/page.tsx b/src/app/issues/runsheetMatch/[study_id]/[subject_id]/page.tsx new file mode 100644 index 0000000..ba0e7f5 --- /dev/null +++ b/src/app/issues/runsheetMatch/[study_id]/[subject_id]/page.tsx @@ -0,0 +1,322 @@ +'use client' +import * as React from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import Typography from '@mui/joy/Typography'; +import Alert from '@mui/joy/Alert'; +import Link from '@mui/material/Link'; +import Button from '@mui/material/Button'; + +import { PipelineFailureRow } from '@/lib/types/pipeline_failures'; +import { InterviewIssue } from '@/app/api/v1/issues/unresolved/missing/route'; +import { DbInterviewEnhanced } from '@/lib/types/interview'; + +function copyToClipboard(text: string) { + navigator.clipboard.writeText(text); + toast.success('Copied to clipboard'); +} + +// pf_identifier is a raw path like ".../raw/{subject_id}/interviews/{interview_type}/{leaf}" +// - the leaf is what failed to parse, but the segment structure above it is +// reliable, so the interview_type can still be read off the path. +function extractInterviewType(pfIdentifier: string): string | null { + const parts = pfIdentifier.split('/'); + const idx = parts.indexOf('interviews'); + if (idx === -1 || idx + 1 >= parts.length) return null; + return parts[idx + 1]; +} + +type TimelineEntry = { + key: string; + date: string | null; + interview_type: string; + event_name: string | null; + interview_name: string; + status: 'matched' | 'missing'; + label: string; + missingRow?: InterviewIssue; +}; + +export default function RunsheetMatchDetail({ + params, +}: { + params: Promise<{ study_id: string; subject_id: string }>; +}) { + const [studyId, setStudyId] = useState(''); + const [subjectId, setSubjectId] = useState(''); + const [failures, setFailures] = useState(null); + const [missing, setMissing] = useState(null); + const [matched, setMatched] = useState(null); + const [pendingIdentifiers, setPendingIdentifiers] = useState>(new Set()); + + const [selectedFailure, setSelectedFailure] = useState(null); + const [selectedMissing, setSelectedMissing] = useState(null); + const [overrideDate, setOverrideDate] = useState(''); + const [submitting, setSubmitting] = useState(false); + + const loadData = useCallback((study: string, subject: string) => { + setFailures(null); + setMissing(null); + setMatched(null); + fetch(`/api/v1/issues/unresolved/pipeline-failures?limit=3000&includeResolved=false&errorCode=datetime_parse&study_id=${study}&subject_id=${subject}`) + .then((res) => res.json()) + .then((data) => setFailures(data.rows)); + fetch(`/api/v1/issues/unresolved/missing?limit=3000&study_id=${study}&subject_id=${subject}`) + .then((res) => res.json()) + .then((data) => setMissing(data.rows)); + fetch(`/api/v3/studies/${study}/subjects/${subject}/interviews`) + .then((res) => res.json()) + .then((data) => setMatched(Array.isArray(data) ? data : [])); + }, []); + + useEffect(() => { + (async () => { + const resolved = await params; + setStudyId(resolved.study_id); + setSubjectId(resolved.subject_id); + loadData(resolved.study_id, resolved.subject_id); + })(); + }, [params, loadData]); + + // Best-effort: mark any failure that already has a pending (unconsumed) + // override as such, so staff don't re-submit a match that's already + // waiting on the crawler's next pass. + useEffect(() => { + if (!failures || failures.length === 0) { + setPendingIdentifiers(new Set()); + return; + } + const identifiers = failures.map((f) => f.pf_identifier).join(','); + fetch(`/api/v1/issues/unresolved/pipeline-failures/datetime-override?identifiers=${encodeURIComponent(identifiers)}`) + .then((res) => res.json()) + .then((data) => { + const pending = new Set( + (data.overrides ?? []) + .filter((o: { do_consumed_at: string | null }) => !o.do_consumed_at) + .map((o: { do_identifier: string }) => o.do_identifier) + ); + setPendingIdentifiers(pending); + }) + .catch(() => undefined); + }, [failures]); + + const timeline: TimelineEntry[] = useMemo(() => { + if (!matched || !missing) return []; + const matchedEntries: TimelineEntry[] = matched.map((m) => ({ + key: `matched-${m.interview_name}`, + date: m.interview_datetime ? new Date(m.interview_datetime).toISOString() : null, + interview_type: m.interview_type, + event_name: null, + interview_name: m.interview_name, + status: 'matched', + label: m.interview_name, + })); + const missingEntries: TimelineEntry[] = missing.map((m) => ({ + key: `missing-${m.interview_name}`, + date: m.expected_date ? new Date(m.expected_date).toISOString() : null, + interview_type: m.interview_type, + event_name: m.event_name ?? null, + interview_name: m.interview_name, + status: 'missing', + label: `${m.interview_type} - expected day ${m.expected_day}`, + missingRow: m, + })); + return [...matchedEntries, ...missingEntries].sort((a, b) => (a.date ?? '').localeCompare(b.date ?? '')); + }, [matched, missing]); + + const unmatchedFiles: PipelineFailureRow[] = useMemo(() => { + if (!failures) return []; + return [...failures].sort((a, b) => { + const ta = extractInterviewType(a.pf_identifier) ?? ''; + const tb = extractInterviewType(b.pf_identifier) ?? ''; + return ta.localeCompare(tb) || a.pf_identifier.localeCompare(b.pf_identifier); + }); + }, [failures]); + + const handleSelectMissing = (entry: TimelineEntry) => { + if (entry.status !== 'missing' || !entry.missingRow) return; + setSelectedMissing(entry.missingRow); + setOverrideDate(entry.missingRow.expected_date ? String(entry.missingRow.expected_date).slice(0, 10) : ''); + }; + + const handleConfirmMatch = async () => { + if (!selectedFailure || !overrideDate) return; + setSubmitting(true); + try { + const response = await fetch('/api/v1/issues/unresolved/pipeline-failures/datetime-override', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + pf_identifier: selectedFailure.pf_identifier, + study_id: studyId, + subject_id: subjectId, + override_datetime: overrideDate, + }), + }); + if (!response.ok) throw new Error('Failed to submit match'); + toast.success('Match submitted - will be picked up on the crawler\'s next pass'); + setPendingIdentifiers((prev) => new Set(prev).add(selectedFailure.pf_identifier)); + setSelectedFailure(null); + setSelectedMissing(null); + setOverrideDate(''); + } catch { + toast.error('Failed to submit match'); + } finally { + setSubmitting(false); + } + }; + + const loading = !failures || !missing || !matched; + + return ( +
+ + Runsheet Match: {subjectId} + + + ← Back to subjects · Study {studyId} + + + + Select an unmatched file on the right and a missing runsheet entry on the left, then + confirm the match. The corrected datetime is picked up by the pipeline on its next + run, which imports the file normally and resolves the pipeline failure - nothing on + disk is renamed, and no note is required. + + + {loading ? ( +
+
+
+ Loading data... +
+
+ ) : ( + <> +
+
+ + Timeline (matched + missing runsheet entries) + + {timeline.length === 0 ? ( + Nothing to show. + ) : ( +
+ {timeline.map((entry) => { + const selectable = entry.status === 'missing'; + const selected = selectedMissing && entry.missingRow === selectedMissing; + return ( +
selectable && handleSelectMissing(entry)} + className={[ + 'border rounded p-2 text-sm', + entry.status === 'matched' ? 'bg-green-50 border-green-200' : 'bg-amber-50 border-amber-200', + selectable ? 'cursor-pointer' : '', + selected ? 'ring-2 ring-blue-500' : '', + ].join(' ')} + > +
+ + {entry.status === 'matched' ? 'โœ“ Matched' : 'โ—‹ Missing'} - {entry.interview_type} + {entry.event_name && ( + · {entry.event_name} + )} + +
+ {entry.date ? entry.date.slice(0, 10) : 'unknown date'} + e.stopPropagation()} + > + View + +
+
+
{entry.label}
+
+ ); + })} +
+ )} +
+ +
+ + Unmatched files (datetime_parse failures) + + {unmatchedFiles.length === 0 ? ( + No unresolved datetime_parse failures for this subject. + ) : ( +
+ {unmatchedFiles.map((failure) => { + const pending = pendingIdentifiers.has(failure.pf_identifier); + const selected = selectedFailure?.pf_identifier === failure.pf_identifier; + return ( +
!pending && setSelectedFailure(failure)} + className={[ + 'border rounded p-2 text-sm', + pending ? 'bg-blue-50 border-blue-200' : 'bg-red-50 border-red-200 cursor-pointer', + selected ? 'ring-2 ring-blue-500' : '', + ].join(' ')} + > +
+ {failure.pf_identifier} + +
+
+ {extractInterviewType(failure.pf_identifier) ?? 'unknown type'} + {pending && ' โ€” pending crawler pickup'} +
+
+ ); + })} +
+ )} +
+
+ + {selectedFailure && ( +
+ Confirm Match + {selectedFailure.pf_identifier} + + {selectedMissing + ? `Matched to: ${selectedMissing.interview_type} - ${selectedMissing.event_name} (expected day ${selectedMissing.expected_day})` + : 'Select a missing runsheet entry on the left, or enter a date directly.'} + +
+ + setOverrideDate(e.target.value)} + className="border rounded px-2 py-1 text-sm" + /> + + +
+
+ )} + + )} +
+ ); +} diff --git a/src/app/issues/runsheetMatch/page.tsx b/src/app/issues/runsheetMatch/page.tsx new file mode 100644 index 0000000..5eeafa7 --- /dev/null +++ b/src/app/issues/runsheetMatch/page.tsx @@ -0,0 +1,149 @@ +'use client' +import * as React from 'react'; +import { useEffect, useMemo, useState } from 'react'; + +import Typography from '@mui/joy/Typography'; +import Alert from '@mui/joy/Alert'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; + +import { PipelineFailureRow } from '@/lib/types/pipeline_failures'; +import { InterviewIssue } from '@/app/api/v1/issues/unresolved/missing/route'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +const FETCH_LIMIT = 3000; + +type SubjectGroupRow = { + id: string; + study_id: string; + subject_id: string; + failureCount: number; + missingCount: number; + lastFailureSeen: string; +}; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'pf_study_id', label: 'Study ID' }, + { field: 'pf_subject_id', label: 'Subject ID' }, +]; + +export default function RunsheetMatchIndex() { + const [failures, setFailures] = useState(null); + const [missing, setMissing] = useState(null); + + useEffect(() => { + fetch(`/api/v1/issues/unresolved/pipeline-failures?limit=${FETCH_LIMIT}&includeResolved=false&errorCode=datetime_parse`) + .then((res) => res.json()) + .then((data) => setFailures(data.rows)); + fetch(`/api/v1/issues/unresolved/missing?limit=${FETCH_LIMIT}`) + .then((res) => res.json()) + .then((data) => setMissing(data.rows)); + }, []); + + const subjectRows: SubjectGroupRow[] | null = useMemo(() => { + if (!failures || !missing) return null; + + const missingCounts = new Map(); + for (const row of missing) { + const key = `${row.study_id}::${row.subject_id}`; + missingCounts.set(key, (missingCounts.get(key) ?? 0) + 1); + } + + const groups = new Map(); + for (const row of failures) { + if (!row.pf_study_id || !row.pf_subject_id) continue; + const key = `${row.pf_study_id}::${row.pf_subject_id}`; + const existing = groups.get(key); + const lastSeen = new Date(row.pf_last_seen_at).toISOString(); + if (existing) { + existing.failureCount += 1; + if (lastSeen > existing.lastFailureSeen) existing.lastFailureSeen = lastSeen; + } else { + groups.set(key, { + id: key, + study_id: row.pf_study_id, + subject_id: row.pf_subject_id, + failureCount: 1, + missingCount: missingCounts.get(key) ?? 0, + lastFailureSeen: lastSeen, + }); + } + } + + return Array.from(groups.values()).sort((a, b) => b.failureCount - a.failureCount); + }, [failures, missing]); + + const columns: GridColDef[] = useMemo(() => [ + { + field: 'subject_id', + headerName: 'Subject', + width: 200, + renderCell: (params) => ( + + {params.value} + + ), + }, + { field: 'study_id', headerName: 'Study ID', width: 130 }, + { field: 'failureCount', headerName: 'Unmatched Files', width: 150, type: 'number' }, + { field: 'missingCount', headerName: 'Missing Interviews', width: 160, type: 'number' }, + { field: 'lastFailureSeen', headerName: 'Last Seen', width: 200 }, + ], []); + + const dataGridProps: MuiDataGridProps | null = useMemo(() => { + if (!subjectRows) return null; + return { + columns, + rows: subjectRows, + height: 670, + pageSizeOptions: [10, 20, 50], + }; + }, [subjectRows, columns]); + + return ( +
+ + Runsheet Match + + + + Subjects with unresolved "datetime_parse" pipeline failures - raw interview + files/directories whose names couldn't be date-parsed, so they were skipped + entirely and never imported. These are strong candidates for the "Missing + Interviews" on the same subject: matching a malformed file to its runsheet + entry recovers it without renaming anything on disk. Click a subject to compare its + unmatched files against its missing runsheet entries. + + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No subjects with unresolved datetime_parse failures found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} subjects have unresolved datetime_parse failures: + + + + + + + )} +
+ ); +} diff --git a/src/app/issues/transcriptNotImported/page.tsx b/src/app/issues/transcriptNotImported/page.tsx new file mode 100644 index 0000000..3cb0cc4 --- /dev/null +++ b/src/app/issues/transcriptNotImported/page.tsx @@ -0,0 +1,103 @@ +'use client' +import * as React from 'react'; +import { useEffect, useState } from 'react'; + +import Typography from '@mui/joy/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import Link from '@mui/material/Link'; + +import { TranscriptNotImportedRow } from '@/lib/types/transcribeme'; +import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'source_type', label: 'Source' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; + +function linkFor(row: TranscriptNotImportedRow): string { + if (row.source_type === 'audio_journal') { + return `/studies/${row.study_id}/subjects/${row.subject_id}/journals/${row.interview_name}`; + } + return `/interviews/${row.interview_name}`; +} + +export default function TranscriptNotImportedIssues() { + const [dataGridProps, setDataGridProps] = useState(null); + + const columns: GridColDef[] = React.useMemo(() => [ + { + field: 'interview_name', + headerName: 'Name', + width: 350, + renderCell: (params) => ( + {params.value} + ) + }, + { field: 'source_type', headerName: 'Source', width: 130 }, + { field: 'subject_id', headerName: 'Subject ID', width: 150 }, + { field: 'study_id', headerName: 'Study ID', width: 150 }, + { field: 'sftp_download_timestamp', headerName: 'Downloaded At', width: 200 }, + ], []); + + useEffect(() => { + fetch('/api/v1/issues/unresolved/transcript-not-imported?limit=2000') + .then((res) => res.json()) + .then((data) => { + const gridRows = data.rows.map((row: TranscriptNotImportedRow, index: number) => ({ + id: index, + ...row, + })); + + const props: MuiDataGridProps = { + columns, + rows: gridRows, + height: 670, + pageSizeOptions: [10, 20], + selectable: true + }; + setDataGridProps(props); + }); + }, [columns]); + + return ( +
+ + Transcript Downloaded, Not Yet Imported + + + + TranscribeMe has delivered these transcripts and they have been + downloaded, but they have not yet shown up in transcript_files. + This usually means the transcript import crawler needs to run, or + is failing to parse the file name. + + + {!dataGridProps ? ( +
+
+
+ Loading data... +
+
+ ) : dataGridProps.rows.length === 0 ? ( +
+ + No un-imported transcripts found. + +
+ ) : ( + <> + + The following {dataGridProps.rows.length} transcripts have been downloaded but not imported: + + +
+ +
+ + )} +
+ ); +} diff --git a/src/app/issues/unlabelledAudio/page.tsx b/src/app/issues/unlabelledAudio/page.tsx index abf32f5..86aed40 100644 --- a/src/app/issues/unlabelledAudio/page.tsx +++ b/src/app/issues/unlabelledAudio/page.tsx @@ -7,9 +7,18 @@ import { GridColDef } from '@mui/x-data-grid'; import Link from '@mui/material/Link'; import MuiDataGrid, { MuiDataGridProps } from '@/components/mui/MuiDataGrid'; +import AggregationSummary, { GroupByOption } from '@/components/mui/AggregationSummary'; import { DbInterview } from '@/lib/types/interview'; +type GridRow = { id: string; interview_name: string; interview_type: string; subject_id: string; study_id: string }; + +const GROUP_BY_OPTIONS: GroupByOption[] = [ + { field: 'interview_type', label: 'Interview Type' }, + { field: 'study_id', label: 'Study ID' }, + { field: 'subject_id', label: 'Subject ID' }, +]; + export default function UnlabelledAudioIssues() { const [dataGridProps, setDataGridProps] = useState(null); @@ -80,6 +89,7 @@ export default function UnlabelledAudioIssues() { The following {dataGridProps.rows.length} interviews have atleast one unassigned role from the diarized audio. +
diff --git a/src/app/studies/[study_id]/subjects/[subject_id]/journals/[journal_name]/page.tsx b/src/app/studies/[study_id]/subjects/[subject_id]/journals/[journal_name]/page.tsx index 1e4909f..ce591ee 100644 --- a/src/app/studies/[study_id]/subjects/[subject_id]/journals/[journal_name]/page.tsx +++ b/src/app/studies/[study_id]/subjects/[subject_id]/journals/[journal_name]/page.tsx @@ -15,6 +15,7 @@ import TabPanel from '@mui/joy/TabPanel'; import { DbAudioJournal } from '@/lib/types/audio_journals'; import Transcript from '@/components/domain/TranscriptE'; +import TranscriptionPipelineStatus from '@/components/domain/TranscriptionPipelineStatus'; import { toast } from "sonner"; @@ -134,6 +135,7 @@ export default function Page({ ๐Ÿ“„ Transcript + ๐Ÿ”Š Transcription Pipeline @@ -146,6 +148,18 @@ export default function Page({ updateAudioTime={updateAudioTime} /> + + + Transcription Pipeline + + + + diff --git a/src/components/app-sidebar.tsx b/src/components/app-sidebar.tsx index 6745b8e..3fdf088 100644 --- a/src/components/app-sidebar.tsx +++ b/src/components/app-sidebar.tsx @@ -66,6 +66,41 @@ export const navData = { url: "/issues/noTranscript", isActive: false, }, + { + title: "Failed Audio QC", + url: "/issues/audioQcFailed", + isActive: false, + }, + { + title: "Pending Transcription Push", + url: "/issues/pendingTranscriptionPush", + isActive: false, + }, + { + title: "Awaiting Vendor Transcription", + url: "/issues/awaitingVendorTranscription", + isActive: false, + }, + { + title: "Transcript Not Imported", + url: "/issues/transcriptNotImported", + isActive: false, + }, + { + title: "Pipeline Failures", + url: "/issues/pipelineFailures", + isActive: false, + }, + { + title: "Runsheet Match", + url: "/issues/runsheetMatch", + isActive: false, + }, + { + title: "Override Ledger", + url: "/issues/overrideLedger", + isActive: false, + }, ], }, { diff --git a/src/components/domain/TranscriptionPipelineStatus.tsx b/src/components/domain/TranscriptionPipelineStatus.tsx new file mode 100644 index 0000000..67ef28b --- /dev/null +++ b/src/components/domain/TranscriptionPipelineStatus.tsx @@ -0,0 +1,162 @@ +"use client" +import { useEffect, useState } from 'react'; + +import { Descriptions } from 'antd'; +import type { DescriptionsProps } from 'antd'; +import { Empty } from 'antd'; + +import Skeleton from '@mui/material/Skeleton'; +import Chip from '@mui/joy/Chip'; + +import { TranscriptionPipelineStatus as StatusType } from '@/lib/types/transcribeme'; + +export type TranscriptionPipelineStatusProps = { + identifier: string; + identifier_type: 'interview' | 'audio_journal'; + study_id?: string; + subject_id?: string; +}; + +export default function TranscriptionPipelineStatus(props: TranscriptionPipelineStatusProps) { + const { identifier, identifier_type, study_id, subject_id } = props; + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + + useEffect(() => { + const fetchInterviewStatus = async (interviewName: string) => { + const res = await fetch(`/api/v2/interviews/${interviewName}/transcription-status`); + if (res.ok) { + setStatus(await res.json()); + } + setLoading(false); + }; + + const fetchJournalStatus = async (studyId: string, subjectId: string, journalName: string) => { + const res = await fetch( + `/api/v3/studies/${studyId}/subjects/${subjectId}/audioJournals/${journalName}/transcription-status` + ); + if (res.ok) { + setStatus(await res.json()); + } + setLoading(false); + }; + + if (!identifier) { + return; + } + if (identifier_type === 'interview') { + fetchInterviewStatus(identifier); + } else if (identifier_type === 'audio_journal') { + if (!study_id || !subject_id) { + setLoading(false); + return; + } + fetchJournalStatus(study_id, subject_id, identifier); + } + }, [identifier, identifier_type, study_id, subject_id]); + + if (loading) { + return ; + } + + if (!status || !status.wav_conversion) { + return ( +
+ +
+ ); + } + + const { wav_conversion, audio_qc, push, pull } = status; + + const wavConversionItems: DescriptionsProps['items'] = [ + { key: 'source', label: 'Source Audio', children: wav_conversion.wc_source_path }, + { key: 'destination', label: 'Converted WAV', children: wav_conversion.wc_destination_path }, + { key: 'duration', label: 'Duration (s)', children: wav_conversion.wc_duration_s }, + { key: 'timestamp', label: 'Converted At', children: new Date(wav_conversion.wc_timestamp).toLocaleString() }, + ]; + + const audioQcItems: DescriptionsProps['items'] = audio_qc ? [ + { key: 'passed', label: 'Passed', children: audio_qc.aqc_passed ? 'โœ… Yes' : 'โŒ No' }, + { key: 'override', label: 'Manual Override', children: audio_qc.aqc_override ? 'Yes' : 'No' }, + { + key: 'fail_reasons', + label: 'Fail Reasons', + children: audio_qc.aqc_fail_reasons && Object.keys(audio_qc.aqc_fail_reasons).length > 0 + ? Object.keys(audio_qc.aqc_fail_reasons).join(', ') + : 'None', + }, + { + key: 'metrics', + label: 'Metrics', + children: 
{JSON.stringify(audio_qc.aqc_metrics, null, 2)}
, + }, + { key: 'timestamp', label: 'QC Timestamp', children: new Date(audio_qc.aqc_timestamp).toLocaleString() }, + ] : []; + + const pushItems: DescriptionsProps['items'] = push ? [ + { key: 'language', label: 'Source Language', children: push.source_language }, + { key: 'destination', label: 'Destination Path', children: push.transcription_destination_path }, + { key: 'timestamp', label: 'Pushed At', children: new Date(push.sftp_upload_timestamp).toLocaleString() }, + ] : []; + + const pullItems: DescriptionsProps['items'] = pull ? [ + { key: 'downloaded', label: 'Downloaded At', children: new Date(pull.sftp_download_timestamp).toLocaleString() }, + { key: 'archive', label: 'Archived (Vendor)', children: pull.sftp_archive_path }, + { key: 'completed_audio', label: 'Completed Audio Path', children: pull.completed_audio_file_path }, + ] : []; + + return ( +
+
+
+ 1. WAV Conversion + Done +
+ +
+ +
+
+ 2. Audio QC + {audio_qc ? ( + + {audio_qc.aqc_passed ? 'Passed' : 'Failed'} + + ) : ( + Not Run + )} +
+ {audio_qc ? ( + + ) : ( + + )} +
+ +
+
+ 3. Pushed to TranscribeMe + {push ? 'Done' : 'Not Yet'} +
+ {push ? ( + + ) : ( + + )} +
+ +
+
+ 4. Pulled from TranscribeMe + {pull ? 'Done' : 'Not Yet'} +
+ {pull ? ( + + ) : ( + + )} +
+
+ ); +} diff --git a/src/components/mui/AggregationSummary.tsx b/src/components/mui/AggregationSummary.tsx new file mode 100644 index 0000000..8cf7a16 --- /dev/null +++ b/src/components/mui/AggregationSummary.tsx @@ -0,0 +1,107 @@ +'use client' +import * as React from 'react'; + +import Typography from '@mui/joy/Typography'; +import Select from '@mui/material/Select'; +import MenuItem from '@mui/material/MenuItem'; + +export type GroupByOption = { + field: keyof T; + label: string; + // For multi-valued fields (e.g. a Record of flags) where a single + // row can belong to several buckets at once - one row is counted once per key returned. + extractKeys?: (row: T) => string[]; +}; +export type SumOption = { field: keyof T; label: string }; + +function toKey(raw: unknown): string { + return raw === null || raw === undefined || raw === '' ? '(none)' : String(raw); +} + +function aggregate(rows: T[], option: GroupByOption, sumField?: keyof T) { + const groups = new Map(); + + for (const row of rows) { + const keys = option.extractKeys + ? option.extractKeys(row) + : [toKey((row as Record)[option.field as string])]; + const bucketKeys = keys.length > 0 ? keys : ['(none)']; + + const sumValue = sumField ? (row as Record)[sumField as string] : undefined; + + for (const key of bucketKeys) { + const existing = groups.get(key) ?? { count: 0, sum: 0 }; + existing.count += 1; + if (typeof sumValue === 'number') { + existing.sum += sumValue; + } + groups.set(key, existing); + } + } + + return Array.from(groups.entries()) + .map(([key, value]) => ({ key, ...value })) + .sort((a, b) => (sumField ? b.sum - a.sum : b.count - a.count)); +} + +export type AggregationSummaryProps = { + rows: T[]; + groupByOptions: GroupByOption[]; + sumField?: SumOption; + defaultGroupBy?: keyof T; +}; + +export default function AggregationSummary({ + rows, + groupByOptions, + sumField, + defaultGroupBy, +}: AggregationSummaryProps) { + const [groupBy, setGroupBy] = React.useState(defaultGroupBy ?? groupByOptions[0].field); + + const selectedOption = groupByOptions.find((o) => o.field === groupBy) ?? groupByOptions[0]; + + const aggregationRows = React.useMemo( + () => aggregate(rows, selectedOption, sumField?.field), + [rows, selectedOption, sumField] + ); + + const currentLabel = selectedOption.label; + + return ( +
+
+ + Aggregate by + + +
+ + + + + + {sumField && } + + + + {aggregationRows.map((agg) => ( + + + + {sumField && } + + ))} + +
{currentLabel}RowsSum({sumField.label})
{agg.key}{agg.count}{agg.sum}
+
+ ); +} diff --git a/src/lib/models/DashboardActions.ts b/src/lib/models/DashboardActions.ts index f746bf6..cde284d 100644 --- a/src/lib/models/DashboardActions.ts +++ b/src/lib/models/DashboardActions.ts @@ -1,7 +1,52 @@ import { getConnection } from "@/lib/db"; +import { DbDashboardAction } from "@/lib/types/dashboard_actions"; export class DashboardActions { + // Backs the Override Ledger reporting page - filtered to a caller-supplied + // set of da_action values (e.g. OVERRIDE_LEDGER_ACTIONS) since this table + // also logs routine, non-override dashboard edits (mark_primary, clear_role, + // etc.) that shouldn't count toward the override audit trail. + static async getByActions( + actions: string[], + limit: number, + offset: number, + filters: { study_id?: string; subject_id?: string } = {} + ): Promise<{ rows: DbDashboardAction[]; totalRows: number }> { + const connection = getConnection(); + + const conditions: string[] = ["da_action = ANY($1)"]; + const params: (string | string[])[] = [actions]; + + // study_id/subject_id aren't real columns on this general-purpose + // table - every override call site is expected to record them inside + // da_metadata instead. + if (filters.study_id) { + params.push(filters.study_id); + conditions.push(`da_metadata->>'study_id' = $${params.length}`); + } + if (filters.subject_id) { + params.push(filters.subject_id); + conditions.push(`da_metadata->>'subject_id' = $${params.length}`); + } + + const baseQuery = ` + SELECT * + FROM dashboard_actions + WHERE ${conditions.join(" AND ")} + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`, params); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY da_timestamp DESC LIMIT $${params.length + 1} OFFSET $${params.length + 2}`, + [...params, limit, offset] + ); + + return { rows: rows as DbDashboardAction[], totalRows }; + } + static async recordAction( interview_name: string, action: string, diff --git a/src/lib/models/DatetimeOverrides.ts b/src/lib/models/DatetimeOverrides.ts new file mode 100644 index 0000000..15fba81 --- /dev/null +++ b/src/lib/models/DatetimeOverrides.ts @@ -0,0 +1,48 @@ +import { getConnection } from "@/lib/db"; + +// pipeline_ledger.datetime_overrides +// Mirrors pipeline/models/datetime_overrides.py on the dpinterview side. +// Staff-confirmed event datetime for a raw file/directory that failed to +// date-parse - consumed by the Python crawler on its next pass, which then +// resolves the corresponding pipeline_failures row itself. Written here, not +// resolved here: this table only records intent. + +export class DatetimeOverrides { + static async create( + identifier: string, + study_id: string | null, + subject_id: string | null, + override_datetime: string + ): Promise { + const connection = getConnection(); + await connection.query( + ` + INSERT INTO pipeline_ledger.datetime_overrides ( + do_identifier, do_study_id, do_subject_id, do_override_datetime + ) VALUES ($1, $2, $3, $4) + ON CONFLICT (do_identifier) DO UPDATE SET + do_study_id = EXCLUDED.do_study_id, + do_subject_id = EXCLUDED.do_subject_id, + do_override_datetime = EXCLUDED.do_override_datetime, + do_consumed_at = NULL + `, + [identifier, study_id, subject_id, override_datetime] + ); + } + + static async getByIdentifiers(identifiers: string[]): Promise<{ do_identifier: string; do_consumed_at: Date | null }[]> { + if (identifiers.length === 0) { + return []; + } + const connection = getConnection(); + const { rows } = await connection.query( + ` + SELECT do_identifier, do_consumed_at + FROM pipeline_ledger.datetime_overrides + WHERE do_identifier = ANY($1) + `, + [identifiers] + ); + return rows; + } +} diff --git a/src/lib/models/PipelineFailures.ts b/src/lib/models/PipelineFailures.ts new file mode 100644 index 0000000..8027c0a --- /dev/null +++ b/src/lib/models/PipelineFailures.ts @@ -0,0 +1,66 @@ +import { getConnection } from "@/lib/db"; + +import { PipelineFailureRow } from "@/lib/types/pipeline_failures"; + +export class PipelineFailures { + static async getAll( + includeResolved: boolean, + limit: number, + offset: number, + filters?: { study_id?: string; subject_id?: string; error_code?: string } + ): Promise<{ rows: PipelineFailureRow[]; totalRows: number }> { + const connection = getConnection(); + + const conditions: string[] = []; + const params: (string | number)[] = []; + + if (!includeResolved) { + conditions.push("pf_resolved IS FALSE"); + } + if (filters?.study_id) { + params.push(filters.study_id); + conditions.push(`pf_study_id = $${params.length}`); + } + if (filters?.subject_id) { + params.push(filters.subject_id); + conditions.push(`pf_subject_id = $${params.length}`); + } + if (filters?.error_code) { + params.push(filters.error_code); + conditions.push(`pf_error_code = $${params.length}`); + } + + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + const baseQuery = ` + SELECT * + FROM pipeline_ledger.pipeline_failures + ${whereClause} + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`, params); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY pf_last_seen_at DESC LIMIT $${params.length + 1} OFFSET $${params.length + 2}`, + [...params, limit, offset] + ); + + return { rows: rows as PipelineFailureRow[], totalRows }; + } + + // Mirrors dpinterview's pipeline/helpers/db.py:resolve_failure - that + // function has no call sites on the Python side today, so this is + // currently the only place a failure ever gets marked resolved. + static async resolve(pf_stage: string, pf_identifier: string, note?: string): Promise { + const connection = getConnection(); + + await connection.query( + ` + UPDATE pipeline_ledger.pipeline_failures + SET pf_resolved = TRUE, pf_resolved_at = CURRENT_TIMESTAMP, pf_resolved_note = $3 + WHERE pf_stage = $1 AND pf_identifier = $2 + `, + [pf_stage, pf_identifier, note ?? null] + ); + } +} diff --git a/src/lib/models/Transcribeme.ts b/src/lib/models/Transcribeme.ts new file mode 100644 index 0000000..598ef29 --- /dev/null +++ b/src/lib/models/Transcribeme.ts @@ -0,0 +1,280 @@ +import { getConnection } from "@/lib/db"; + +import { + DbAudioQc, + DbTranscribemePull, + DbTranscribemePush, + DbWavConversion, + FailedAudioQcRow, + PendingPushRow, + AwaitingVendorRow, + TranscriptNotImportedRow, + TranscriptionPipelineStatus, +} from "@/lib/types/transcribeme"; + +// transcribeme.* rows trace back to a source file that is either a combined +// interview audio file or an audio journal recording - there's no direct +// study_id/subject_id column on any transcribeme table, so every query joins +// through one of these two chains off `wc.wc_source_path` / `wc_destination_path`. +// AMPSCZ is the only study writing to transcribeme.*, so no study_id filter is needed. +// +// These are INNER joins on purpose: every browse query below UNIONs an +// interview-side query with a journal-side query, and a given wc_source_path +// belongs to exactly one of the two trees, never both. Using LEFT JOIN here +// let every row leak into *both* branches - matched with real data in the +// branch it actually belongs to, and as an all-NULL "ghost" row with no +// name/subject/study in the other branch. INNER JOIN drops the ghost row by +// only letting a wav_conversion row through the branch it actually resolves in. +const INTERVIEW_JOIN = ` + JOIN interview_files ifi ON ifi.interview_file = wc.wc_source_path + JOIN interview_parts ip ON ip.interview_path = ifi.interview_path + JOIN interviews i ON i.interview_name = ip.interview_name +`; + +const JOURNAL_JOIN = ` + JOIN audio_journals aj ON aj.aj_path = wc.wc_source_path +`; + +export class Transcribeme { + static async getFailedAudioQc( + limit: number, + offset: number, + includeOverridden: boolean = false + ): Promise<{ rows: FailedAudioQcRow[]; totalRows: number }> { + const connection = getConnection(); + + const overrideFilter = includeOverridden ? "" : "AND aqc.aqc_override IS FALSE"; + + const baseQuery = ` + SELECT 'interview' AS source_type, i.interview_name, i.subject_id, i.study_id, + aqc.aqc_source_path, aqc.aqc_metrics, aqc.aqc_fail_reasons, aqc.aqc_timestamp, aqc.aqc_override + FROM transcribeme.wav_conversion wc + JOIN transcribeme.audio_qc aqc ON aqc.aqc_source_path = wc.wc_destination_path + ${INTERVIEW_JOIN} + WHERE aqc.aqc_passed IS FALSE ${overrideFilter} + + UNION ALL + + SELECT 'audio_journal' AS source_type, aj.aj_name AS interview_name, aj.subject_id, aj.study_id, + aqc.aqc_source_path, aqc.aqc_metrics, aqc.aqc_fail_reasons, aqc.aqc_timestamp, aqc.aqc_override + FROM transcribeme.wav_conversion wc + JOIN transcribeme.audio_qc aqc ON aqc.aqc_source_path = wc.wc_destination_path + ${JOURNAL_JOIN} + WHERE aqc.aqc_passed IS FALSE ${overrideFilter} + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY aqc_timestamp DESC LIMIT $1 OFFSET $2`, + [limit, offset] + ); + + return { rows: rows as FailedAudioQcRow[], totalRows }; + } + + // Sets the manual QC override flag. The transcribeme push runners + // (dpinterview) pick this up, relocate the file from rejected_audio/ to + // pending_audio/, and push it through despite aqc_passed staying FALSE. + // One-directional by design - once a push runner grabs the file it's + // uploaded to TranscribeMe's SFTP server, an external action that can't + // be undone from here. + static async setAudioQcOverride(aqc_source_path: string): Promise { + const connection = getConnection(); + + await connection.query( + `UPDATE transcribeme.audio_qc SET aqc_override = TRUE WHERE aqc_source_path = $1`, + [aqc_source_path] + ); + } + + static async getPendingPush(limit: number, offset: number): Promise<{ rows: PendingPushRow[]; totalRows: number }> { + const connection = getConnection(); + + const baseQuery = ` + SELECT 'interview' AS source_type, i.interview_name, i.subject_id, i.study_id, + wc.wc_destination_path, aqc.aqc_timestamp + FROM transcribeme.wav_conversion wc + JOIN transcribeme.audio_qc aqc ON aqc.aqc_source_path = wc.wc_destination_path + ${INTERVIEW_JOIN} + WHERE aqc.aqc_passed IS TRUE + AND wc.wc_destination_path NOT IN (SELECT transcription_source_path FROM transcribeme.transcribeme_push) + + UNION ALL + + SELECT 'audio_journal' AS source_type, aj.aj_name AS interview_name, aj.subject_id, aj.study_id, + wc.wc_destination_path, aqc.aqc_timestamp + FROM transcribeme.wav_conversion wc + JOIN transcribeme.audio_qc aqc ON aqc.aqc_source_path = wc.wc_destination_path + ${JOURNAL_JOIN} + WHERE aqc.aqc_passed IS TRUE + AND wc.wc_destination_path NOT IN (SELECT transcription_source_path FROM transcribeme.transcribeme_push) + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY aqc_timestamp ASC LIMIT $1 OFFSET $2`, + [limit, offset] + ); + + return { rows: rows as PendingPushRow[], totalRows }; + } + + static async getAwaitingVendor(limit: number, offset: number): Promise<{ rows: AwaitingVendorRow[]; totalRows: number }> { + const connection = getConnection(); + + const baseQuery = ` + SELECT 'interview' AS source_type, i.interview_name, i.subject_id, i.study_id, + tp.transcription_source_path, tp.source_language, tp.sftp_upload_timestamp, + EXTRACT(EPOCH FROM (NOW() - tp.sftp_upload_timestamp)) / 3600.0 AS hours_waiting + FROM transcribeme.transcribeme_push tp + LEFT JOIN transcribeme.wav_conversion wc ON wc.wc_destination_path = tp.transcription_source_path + ${INTERVIEW_JOIN} + WHERE tp.transcription_destination_path NOT IN ( + SELECT transcription_destination_path FROM transcribeme.transcribeme_pull + ) + + UNION ALL + + SELECT 'audio_journal' AS source_type, aj.aj_name AS interview_name, aj.subject_id, aj.study_id, + tp.transcription_source_path, tp.source_language, tp.sftp_upload_timestamp, + EXTRACT(EPOCH FROM (NOW() - tp.sftp_upload_timestamp)) / 3600.0 AS hours_waiting + FROM transcribeme.transcribeme_push tp + LEFT JOIN transcribeme.wav_conversion wc ON wc.wc_destination_path = tp.transcription_source_path + ${JOURNAL_JOIN} + WHERE tp.transcription_destination_path NOT IN ( + SELECT transcription_destination_path FROM transcribeme.transcribeme_pull + ) + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY hours_waiting DESC LIMIT $1 OFFSET $2`, + [limit, offset] + ); + + return { rows: rows as AwaitingVendorRow[], totalRows }; + } + + static async getDownloadedNotImported(limit: number, offset: number): Promise<{ rows: TranscriptNotImportedRow[]; totalRows: number }> { + const connection = getConnection(); + + // NOTE: the audio-journal transcript importer (5_import_journal_transcripts.py) + // writes identifier_type = 'audioJounal' - that's an upstream typo, not a bug + // here. It must be matched exactly or every journal row will look "not imported". + const baseQuery = ` + SELECT 'interview' AS source_type, i.interview_name, i.subject_id, i.study_id, + pull.transcription_destination_path, pull.sftp_download_timestamp + FROM transcribeme.transcribeme_pull pull + LEFT JOIN transcribeme.transcribeme_push tp ON tp.transcription_destination_path = pull.transcription_destination_path + LEFT JOIN transcribeme.wav_conversion wc ON wc.wc_destination_path = tp.transcription_source_path + ${INTERVIEW_JOIN} + WHERE NOT EXISTS ( + SELECT 1 FROM transcript_files tf + WHERE tf.identifier_name = i.interview_name AND tf.identifier_type = 'interview' + ) + + UNION ALL + + SELECT 'audio_journal' AS source_type, aj.aj_name AS interview_name, aj.subject_id, aj.study_id, + pull.transcription_destination_path, pull.sftp_download_timestamp + FROM transcribeme.transcribeme_pull pull + LEFT JOIN transcribeme.transcribeme_push tp ON tp.transcription_destination_path = pull.transcription_destination_path + LEFT JOIN transcribeme.wav_conversion wc ON wc.wc_destination_path = tp.transcription_source_path + ${JOURNAL_JOIN} + WHERE NOT EXISTS ( + SELECT 1 FROM transcript_files tf + WHERE tf.identifier_name = aj.aj_name AND tf.identifier_type = 'audioJounal' + ) + `; + + const countResult = await connection.query(`SELECT COUNT(*) FROM (${baseQuery}) AS total`); + const totalRows = parseInt(countResult.rows[0].count, 10); + + const { rows } = await connection.query( + `${baseQuery} ORDER BY sftp_download_timestamp DESC LIMIT $1 OFFSET $2`, + [limit, offset] + ); + + return { rows: rows as TranscriptNotImportedRow[], totalRows }; + } + + static async getStatusForInterview(interview_name: string): Promise { + const connection = getConnection(); + + const result = await connection.query( + ` + SELECT wc.wc_source_path, wc.wc_destination_path, wc.wc_duration_s, wc.wc_timestamp + FROM transcribeme.wav_conversion wc + JOIN interview_files ifi ON ifi.interview_file = wc.wc_source_path + JOIN interview_parts ip ON ip.interview_path = ifi.interview_path + WHERE ip.interview_name = $1 + ORDER BY wc.wc_timestamp DESC + LIMIT 1 + `, + [interview_name] + ); + + return Transcribeme.buildStatusFromWavConversion(result.rows[0] ?? null); + } + + static async getStatusForAudioJournal( + study_id: string, + subject_id: string, + journal_name: string + ): Promise { + const connection = getConnection(); + + const result = await connection.query( + ` + SELECT wc.wc_source_path, wc.wc_destination_path, wc.wc_duration_s, wc.wc_timestamp + FROM transcribeme.wav_conversion wc + JOIN audio_journals aj ON aj.aj_path = wc.wc_source_path + WHERE aj.study_id = $1 AND aj.subject_id = $2 AND aj.aj_name = $3 + ORDER BY wc.wc_timestamp DESC + LIMIT 1 + `, + [study_id, subject_id, journal_name] + ); + + return Transcribeme.buildStatusFromWavConversion(result.rows[0] ?? null); + } + + private static async buildStatusFromWavConversion( + wav_conversion: DbWavConversion | null + ): Promise { + if (!wav_conversion) { + return { wav_conversion: null, audio_qc: null, push: null, pull: null }; + } + + const connection = getConnection(); + + const aqcResult = await connection.query( + `SELECT * FROM transcribeme.audio_qc WHERE aqc_source_path = $1`, + [wav_conversion.wc_destination_path] + ); + const audio_qc: DbAudioQc | null = aqcResult.rows[0] ?? null; + + const pushResult = await connection.query( + `SELECT * FROM transcribeme.transcribeme_push WHERE transcription_source_path = $1`, + [wav_conversion.wc_destination_path] + ); + const push: DbTranscribemePush | null = pushResult.rows[0] ?? null; + + let pull: DbTranscribemePull | null = null; + if (push) { + const pullResult = await connection.query( + `SELECT * FROM transcribeme.transcribeme_pull WHERE transcription_destination_path = $1`, + [push.transcription_destination_path] + ); + pull = pullResult.rows[0] ?? null; + } + + return { wav_conversion, audio_qc, push, pull }; + } +} diff --git a/src/lib/types/dashboard_actions.ts b/src/lib/types/dashboard_actions.ts index 5808213..b9a644f 100644 --- a/src/lib/types/dashboard_actions.ts +++ b/src/lib/types/dashboard_actions.ts @@ -2,15 +2,28 @@ // interview_name varchar(255) NOT NULL, // da_action varchar(255) NOT NULL, // da_user_id varchar(255) NOT NULL, -// da_target_id varchar(255) NOT NULL, -// da_target_type varchar(255) NOT NULL, +// da_target_id varchar(255) NULL, +// da_target_type varchar(255) NULL, +// da_metadata jsonb NULL, // da_timestamp timestamp DEFAULT CURRENT_TIMESTAMP NULL, export type DbDashboardAction = { da_id: number; interview_name: string; da_action: string; da_user_id: string; - da_target_id: string; - da_target_type: string; + da_target_id: string | null; + da_target_type: string | null; + da_metadata: Record | null; da_timestamp: Date | null; -} \ No newline at end of file +} + +// The manual override/remediation features that write to dashboard_actions - +// this is the scope of the "Override Ledger" reporting page. Add here if a +// new override-style action is introduced elsewhere; not every da_action in +// the table is an override (e.g. mark_primary/clear_role are routine edits). +export const OVERRIDE_LEDGER_ACTIONS = [ + "override_audio_qc", + "datetime_override_pipeline_failure", +] as const; + +export type OverrideLedgerAction = (typeof OVERRIDE_LEDGER_ACTIONS)[number]; \ No newline at end of file diff --git a/src/lib/types/pipeline_failures.ts b/src/lib/types/pipeline_failures.ts new file mode 100644 index 0000000..3909aaa --- /dev/null +++ b/src/lib/types/pipeline_failures.ts @@ -0,0 +1,49 @@ +// pipeline_ledger.pipeline_failures +// Mirrors pipeline/models/pipeline_failures.py on the dpinterview side. +// Kept in its own schema there (not `public`) so it can be permissioned/retained +// separately from the rest of the application tables. + +export type PipelineFailureIdentifierType = + | "file_path" + | "study" + | "interview_name" + | "subject" + | "batch" + | "other"; + +export type PipelineFailureErrorCode = + | "datetime_parse" + | "subject_id_parse" + | "filename_parse" + | "consent_date_missing" + | "missing_file" + | "db_write_failure" + | "data_dictionary_import_failed" + | "ffprobe_streams_missing" + | "openface_datatype_cast_failed" + | "openface_load_failed" + | "decryption_failed" + | "llm_prompt_build_failed" + | "llm_language_identification_failed" + | "transcribeme_pull_failed" + | "interview_not_in_study_list" + | "crawler_stage_failed" + | "other"; + +export type PipelineFailureRow = { + pf_id: number; + pf_stage: string; + pf_error_code: PipelineFailureErrorCode; + pf_identifier_type: PipelineFailureIdentifierType; + pf_identifier: string; + pf_study_id: string | null; + pf_subject_id: string | null; + pf_error: string; + pf_error_type: string | null; + pf_occurrence_count: number; + pf_first_seen_at: Date; + pf_last_seen_at: Date; + pf_resolved: boolean; + pf_resolved_at: Date | null; + pf_resolved_note: string | null; +}; diff --git a/src/lib/types/transcribeme.ts b/src/lib/types/transcribeme.ts new file mode 100644 index 0000000..6f7e206 --- /dev/null +++ b/src/lib/types/transcribeme.ts @@ -0,0 +1,91 @@ +// transcribeme.wav_conversion +export type DbWavConversion = { + wc_source_path: string; + wc_destination_path: string; + wc_duration_s: number; + wc_timestamp: Date; +}; + +// transcribeme.audio_qc +export type DbAudioQc = { + aqc_source_path: string; + aqc_passed: boolean; + aqc_metrics: Record; + aqc_fail_reasons: Record | null; + aqc_duration_s: number; + aqc_timestamp: Date; + aqc_override: boolean; +}; + +// transcribeme.transcribeme_push +export type DbTranscribemePush = { + transcription_source_path: string; + source_language: string; + sftp_upload_path: string; + transcription_destination_path: string; + sftp_upload_duration_s: number; + sftp_upload_timestamp: Date; +}; + +// transcribeme.transcribeme_pull +export type DbTranscribemePull = { + transcription_destination_path: string; + sftp_download_path: string; + sftp_archive_path: string; + sftp_download_duration_s: number; + sftp_download_timestamp: Date; + completed_audio_file_path: string; +}; + +// Every transcribeme.* row traces back to either a combined interview audio +// file or an audio journal - the two identifier trees join back differently +// (see Transcribeme model), so every browse row is tagged with which one it is. +export type TranscriptionSourceType = "interview" | "audio_journal"; + +export type FailedAudioQcRow = { + source_type: TranscriptionSourceType; + interview_name: string; + subject_id: string; + study_id: string; + aqc_source_path: string; + aqc_metrics: Record; + aqc_fail_reasons: Record | null; + aqc_timestamp: Date; + aqc_override: boolean; +}; + +export type PendingPushRow = { + source_type: TranscriptionSourceType; + interview_name: string; + subject_id: string; + study_id: string; + wc_destination_path: string; + aqc_timestamp: Date; +}; + +export type AwaitingVendorRow = { + source_type: TranscriptionSourceType; + interview_name: string; + subject_id: string; + study_id: string; + transcription_source_path: string; + source_language: string; + sftp_upload_timestamp: Date; + hours_waiting: number; +}; + +export type TranscriptNotImportedRow = { + source_type: TranscriptionSourceType; + interview_name: string; + subject_id: string; + study_id: string; + transcription_destination_path: string; + sftp_download_timestamp: Date; +}; + +export type TranscriptionPipelineStatus = { + wav_conversion: DbWavConversion | null; + audio_qc: DbAudioQc | null; + push: DbTranscribemePush | null; + pull: DbTranscribemePull | null; +};