Add survey psychometric validity assistant

survey-psychometric-validity-assistant/README.md

@@ -0,0 +1,26 @@
+# Survey Psychometric Validity Assistant
+
+Self-contained SCIBASE issue #16 slice for AI-assisted peer review of survey manuscripts. The assistant blocks or escalates AI review output when a submitted survey scale has psychometric validity gaps that would make the downstream review misleading.
+
+The module uses synthetic packets only and has no network calls, external AI APIs, private manuscripts, credentials, or live data.
+
+## What it checks
+
+- reverse-coded item handling drift
+- low Cronbach alpha or missing internal-consistency evidence
+- factor-loading cross-loads and weak primary loadings
+- confirmatory factor analysis sample-size shortfall
+- missing or inconsistent Likert anchors
+- construct-claim mismatch between manuscript claims and validated scales
+- release readiness for AI-generated peer-review output
+
+## Run
+
+```bash
+npm run check
+npm test
+npm run demo
+npm run demo:video
+```
+
+Demo outputs are written to `reports/`.

survey-psychometric-validity-assistant/demo.js

@@ -0,0 +1,29 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { evaluateSurveyPacket, renderMarkdownReport } from "./src/guard.js";
+import { cleanPacket, riskyPacket } from "./src/samplePackets.js";
+
+const reportsDir = new URL("./reports/", import.meta.url);
+await mkdir(reportsDir, { recursive: true });
+
+const clean = evaluateSurveyPacket(cleanPacket);
+const risky = evaluateSurveyPacket(riskyPacket);
+
+await writeFile(new URL("clean-result.json", reportsDir), JSON.stringify(clean, null, 2));
+await writeFile(new URL("risky-result.json", reportsDir), JSON.stringify(risky, null, 2));
+await writeFile(new URL("risky-report.md", reportsDir), renderMarkdownReport(risky));
+
+const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="960" height="540" viewBox="0 0 960 540">
+  <rect width="960" height="540" fill="#f8fafc"/>
+  <rect x="54" y="58" width="852" height="424" rx="8" fill="#ffffff" stroke="#cbd5e1"/>
+  <text x="86" y="118" font-family="Arial, sans-serif" font-size="34" font-weight="700" fill="#111827">Survey Psychometric Validity Assistant</text>
+  <text x="86" y="168" font-family="Arial, sans-serif" font-size="22" fill="#334155">Packet: ${risky.packetId}</text>
+  <text x="86" y="218" font-family="Arial, sans-serif" font-size="28" font-weight="700" fill="#991b1b">Decision: ${risky.decision}</text>
+  <text x="86" y="268" font-family="Arial, sans-serif" font-size="22" fill="#334155">Severity score: ${risky.severityScore} | Findings: ${risky.findingCount}</text>
+  <text x="86" y="326" font-family="Arial, sans-serif" font-size="20" fill="#111827">Top gates: reverse coding, reliability, CFA sample size, factor cross-loads</text>
+  <text x="86" y="372" font-family="Arial, sans-serif" font-size="20" fill="#111827">Action: hold AI peer-review release for psychometric review</text>
+  <rect x="86" y="414" width="660" height="16" fill="#fee2e2"/>
+  <rect x="86" y="414" width="540" height="16" fill="#ef4444"/>
+</svg>`;
+await writeFile(new URL("summary.svg", reportsDir), svg);
+
+console.log(JSON.stringify({ clean: clean.decision, risky: risky.decision, reportsDir: reportsDir.pathname }, null, 2));

survey-psychometric-validity-assistant/make-demo-video.js

@@ -0,0 +1,39 @@
+import { spawnSync } from "node:child_process";
+import { mkdir } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+
+const reportsDir = new URL("./reports/", import.meta.url);
+await mkdir(reportsDir, { recursive: true });
+
+const output = new URL("demo.mp4", reportsDir);
+const font = "C\\:/Windows/Fonts/arial.ttf";
+const filter = [
+  "color=c=0xf8fafc:s=960x540:d=5:r=12",
+  "drawbox=x=48:y=52:w=864:h=436:color=0xcbd5e1:t=2",
+  `drawtext=fontfile='${font}':text='Survey Psychometric Validity Assistant':x=78:y=92:fontsize=34:fontcolor=0x111827`,
+  `drawtext=fontfile='${font}':text='AI review release gate for survey manuscripts':x=78:y=146:fontsize=23:fontcolor=0x334155`,
+  `drawtext=fontfile='${font}':text='Checks reverse coding, reliability, factor loadings, CFA sample size':x=78:y=204:fontsize=22:fontcolor=0x111827`,
+  `drawtext=fontfile='${font}':text='Risky packet decision HOLD':x=78:y=276:fontsize=34:fontcolor=0x991b1b`,
+  `drawtext=fontfile='${font}':text='Action - psychometric reviewer required before AI output is trusted':x=78:y=344:fontsize=22:fontcolor=0x111827`,
+  "drawbox=x=78:y=402:w=660:h=18:color=0xfee2e2:t=18",
+  "drawbox=x=78:y=402:w=540:h=18:color=0xef4444:t=18"
+].join(",");
+
+const result = spawnSync("ffmpeg", [
+  "-y",
+  "-f",
+  "lavfi",
+  "-i",
+  filter,
+  "-pix_fmt",
+  "yuv420p",
+  "-movflags",
+  "+faststart",
+  fileURLToPath(output)
+], { stdio: "inherit" });
+
+if (result.status !== 0) {
+  throw new Error(`ffmpeg failed with status ${result.status}`);
+}
+
+console.log(`wrote ${fileURLToPath(output)}`);

survey-psychometric-validity-assistant/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "survey-psychometric-validity-assistant",
+  "version": "1.0.0",
+  "description": "Deterministic survey psychometric validity assistant for SCIBASE issue #16.",
+  "type": "module",
+  "scripts": {
+    "check": "node --check src/guard.js && node --check src/samplePackets.js && node --check test.js && node --check demo.js && node --check make-demo-video.js",
+    "test": "node test.js",
+    "demo": "node demo.js",
+    "demo:video": "node make-demo-video.js"
+  },
+  "license": "MIT"
+}

survey-psychometric-validity-assistant/reports/clean-result.json

@@ -0,0 +1,13 @@
+{
+  "packetId": "survey-clean-001",
+  "title": "Validated Remote Collaboration Burnout Scale",
+  "decision": "RELEASE",
+  "severityScore": 0,
+  "findingCount": 0,
+  "findings": [],
+  "releaseGate": {
+    "canReleaseAiReview": true,
+    "requiresPsychometricReviewer": false,
+    "rationale": "Psychometric evidence is sufficient for AI peer-review release."
+  }
+}

survey-psychometric-validity-assistant/reports/risky-report.md

@@ -0,0 +1,30 @@
+# Survey Psychometric Validity Report
+
+Packet: survey-risky-017
+Decision: HOLD
+Severity score: 39
+Findings: 17
+
+## Findings
+- [medium] thin_likert_anchor_set: Survey reports 3 Likert anchors; expected at least 5.
+- [high] cfa_sample_size_shortfall: CFA sample size 96 is below 280 for 28 parameters.
+- [high] low_internal_consistency: Workload Anxiety Draft Scale reports alpha 0.58, below 0.7.
+- [medium] weak_primary_loading: wa_1 has primary loading 0.42, below 0.5.
+- [critical] reverse_coding_drift: wa_2 is reverse coded but not reverse scored before scale scoring.
+- [medium] weak_primary_loading: wa_2 has primary loading 0.39, below 0.5.
+- [medium] weak_primary_loading: wa_3 has primary loading 0.44, below 0.5.
+- [medium] weak_primary_loading: wa_4 has primary loading 0.31, below 0.5.
+- [high] factor_cross_loading: wa_1 has secondary loading 0.41, above 0.32.
+- [high] factor_cross_loading: wa_3 has secondary loading 0.37, above 0.32.
+- [high] missing_internal_consistency: Productivity Perception Index does not report Cronbach alpha or equivalent reliability evidence.
+- [medium] weak_primary_loading: ppi_1 has primary loading 0.47, below 0.5.
+- [medium] weak_primary_loading: ppi_2 has primary loading 0.36, below 0.5.
+- [high] factor_cross_loading: ppi_2 has secondary loading 0.34, above 0.32.
+- [high] construct_claim_mismatch: Manuscript claims anxiety, but no submitted scale validates that construct directly.
+- [high] construct_claim_mismatch: Manuscript claims productivity, but no submitted scale validates that construct directly.
+- [high] construct_claim_mismatch: Manuscript claims clinical burnout, but no submitted scale validates that construct directly.
+
+## Release gate
+- Can release AI review: false
+- Requires psychometric reviewer: true
+- Rationale: Psychometric validity issues must be reviewed before AI output is trusted.

survey-psychometric-validity-assistant/reports/risky-result.json

@@ -0,0 +1,188 @@
+{
+  "packetId": "survey-risky-017",
+  "title": "AI Workload Anxiety and Productivity Claims",
+  "decision": "HOLD",
+  "severityScore": 39,
+  "findingCount": 17,
+  "findings": [
+    {
+      "severity": "medium",
+      "code": "thin_likert_anchor_set",
+      "message": "Survey reports 3 Likert anchors; expected at least 5.",
+      "evidence": {
+        "anchors": [
+          "Never",
+          "Sometimes",
+          "Often"
+        ]
+      }
+    },
+    {
+      "severity": "high",
+      "code": "cfa_sample_size_shortfall",
+      "message": "CFA sample size 96 is below 280 for 28 parameters.",
+      "evidence": {
+        "sampleSize": 96,
+        "plannedCfaParameters": 28,
+        "minimumCfaSample": 280
+      }
+    },
+    {
+      "severity": "high",
+      "code": "low_internal_consistency",
+      "message": "Workload Anxiety Draft Scale reports alpha 0.58, below 0.7.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "alpha": 0.58
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "wa_1 has primary loading 0.42, below 0.5.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_1",
+        "loading": 0.42
+      }
+    },
+    {
+      "severity": "critical",
+      "code": "reverse_coding_drift",
+      "message": "wa_2 is reverse coded but not reverse scored before scale scoring.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_2"
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "wa_2 has primary loading 0.39, below 0.5.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_2",
+        "loading": 0.39
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "wa_3 has primary loading 0.44, below 0.5.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_3",
+        "loading": 0.44
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "wa_4 has primary loading 0.31, below 0.5.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_4",
+        "loading": 0.31
+      }
+    },
+    {
+      "severity": "high",
+      "code": "factor_cross_loading",
+      "message": "wa_1 has secondary loading 0.41, above 0.32.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_1",
+        "secondaryLoading": 0.41
+      }
+    },
+    {
+      "severity": "high",
+      "code": "factor_cross_loading",
+      "message": "wa_3 has secondary loading 0.37, above 0.32.",
+      "evidence": {
+        "scale": "Workload Anxiety Draft Scale",
+        "itemId": "wa_3",
+        "secondaryLoading": 0.37
+      }
+    },
+    {
+      "severity": "high",
+      "code": "missing_internal_consistency",
+      "message": "Productivity Perception Index does not report Cronbach alpha or equivalent reliability evidence.",
+      "evidence": {
+        "scale": "Productivity Perception Index"
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "ppi_1 has primary loading 0.47, below 0.5.",
+      "evidence": {
+        "scale": "Productivity Perception Index",
+        "itemId": "ppi_1",
+        "loading": 0.47
+      }
+    },
+    {
+      "severity": "medium",
+      "code": "weak_primary_loading",
+      "message": "ppi_2 has primary loading 0.36, below 0.5.",
+      "evidence": {
+        "scale": "Productivity Perception Index",
+        "itemId": "ppi_2",
+        "loading": 0.36
+      }
+    },
+    {
+      "severity": "high",
+      "code": "factor_cross_loading",
+      "message": "ppi_2 has secondary loading 0.34, above 0.32.",
+      "evidence": {
+        "scale": "Productivity Perception Index",
+        "itemId": "ppi_2",
+        "secondaryLoading": 0.34
+      }
+    },
+    {
+      "severity": "high",
+      "code": "construct_claim_mismatch",
+      "message": "Manuscript claims anxiety, but no submitted scale validates that construct directly.",
+      "evidence": {
+        "claim": "anxiety",
+        "availableConstructs": [
+          "workload anxiety",
+          "productivity perception"
+        ]
+      }
+    },
+    {
+      "severity": "high",
+      "code": "construct_claim_mismatch",
+      "message": "Manuscript claims productivity, but no submitted scale validates that construct directly.",
+      "evidence": {
+        "claim": "productivity",
+        "availableConstructs": [
+          "workload anxiety",
+          "productivity perception"
+        ]
+      }
+    },
+    {
+      "severity": "high",
+      "code": "construct_claim_mismatch",
+      "message": "Manuscript claims clinical burnout, but no submitted scale validates that construct directly.",
+      "evidence": {
+        "claim": "clinical burnout",
+        "availableConstructs": [
+          "workload anxiety",
+          "productivity perception"
+        ]
+      }
+    }
+  ],
+  "releaseGate": {
+    "canReleaseAiReview": false,
+    "requiresPsychometricReviewer": true,
+    "rationale": "Psychometric validity issues must be reviewed before AI output is trusted."
+  }
+}