Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions external-validity-transfer-assistant/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# External Validity Transfer Assistant

This module is a focused slice for SCIBASE issue #16, AI-Powered Research Assistant Suite.

It adds a deterministic research-assistant review gate for external validity and population-transfer risk before AI-generated peer-review packets are shown to authors, reviewers, funders, or lab leads.

## What It Checks

- Whether broad manuscript claims are backed by evidence from the asserted populations.
- Whether claimed deployment settings are covered by linked study artifacts.
- Whether assay or instrument contexts match the manuscript language.
- Whether runtime environments have reproducible rerun evidence.
- Whether strong claims are missing required subgroup coverage.
- Whether a broad transfer claim lacks external validation.

## Outputs

The demo creates:

- `reports/summary.json`: structured review packet.
- `reports/reviewer-packet.md`: reviewer-facing findings, actions, and research gaps.
- `reports/summary.svg`: visual transfer-risk summary.

## Why This Is Distinct

This is not another broad AI assistant, preregistration checker, retraction sentinel, prompt-safety guard, statistical review, benchmark-leakage auditor, figure/table checker, supplement-readiness module, funding/COI checker, or evidence-trace assistant.

It focuses specifically on whether a manuscript's claims transfer beyond the exact population, setting, assay, and runtime contexts represented by the linked evidence.

## Local Validation

```bash
npm run check
npm test
npm run demo
```

The module uses synthetic data only. It makes no network calls and uses no credentials, private manuscripts, protected health information, payment data, or external APIs.
174 changes: 174 additions & 0 deletions external-validity-transfer-assistant/demo-video.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
const fs = require("fs");
const os = require("os");
const path = require("path");
const { execFileSync } = require("child_process");

const reportDir = path.join(__dirname, "reports");
const outputPath = path.join(reportDir, "demo.webm");

const chromeCandidates = [
process.env.CHROME_PATH,
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
"C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe"
].filter(Boolean);

function findBrowser() {
const found = chromeCandidates.find((candidate) => fs.existsSync(candidate));
if (!found) {
throw new Error("Chrome or Edge was not found. Set CHROME_PATH to generate reports/demo.webm.");
}
return found;
}

function fileUrl(filePath) {
return `file:///${filePath.replace(/\\/g, "/")}`;
}

const html = String.raw`<!doctype html>
<html>
<head>
<meta charset="utf-8">
<title>External validity transfer assistant demo</title>
<style>
html, body { margin: 0; background: #0f172a; }
canvas { width: 960px; height: 540px; }
pre { white-space: pre-wrap; word-break: break-all; color: #0f172a; font-size: 1px; }
</style>
</head>
<body>
<canvas id="stage" width="960" height="540"></canvas>
<pre id="out">recording</pre>
<script>
const canvas = document.getElementById("stage");
const ctx = canvas.getContext("2d");
const out = document.getElementById("out");
const cards = [
["Population transfer", "Pediatric and underrepresented ancestry evidence is missing.", 0],
["Setting transfer", "Community and international site claims need validation.", 1],
["Runtime transfer", "CPU-only deployment language needs rerun evidence.", 2],
["Research gaps", "Prioritize pediatric validation, ancestry-balanced cohorts, and clinic-like reruns.", 3]
];

function roundRect(x, y, w, h, r) {
ctx.beginPath();
ctx.moveTo(x + r, y);
ctx.arcTo(x + w, y, x + w, y + h, r);
ctx.arcTo(x + w, y + h, x, y + h, r);
ctx.arcTo(x, y + h, x, y, r);
ctx.arcTo(x, y, x + w, y, r);
ctx.closePath();
}

function draw(frame) {
const t = frame / 48;
ctx.fillStyle = "#f8fafc";
ctx.fillRect(0, 0, 960, 540);
ctx.fillStyle = "#0f172a";
ctx.font = "bold 34px Arial";
ctx.fillText("External Validity Transfer Assistant", 48, 64);
ctx.font = "20px Arial";
ctx.fillStyle = "#475569";
ctx.fillText("SCIBASE #16 demo packet: peer review, reproducibility actions, and research gaps", 48, 98);

ctx.fillStyle = "#e2e8f0";
roundRect(48, 128, 864, 28, 8);
ctx.fill();
ctx.fillStyle = "#0f766e";
roundRect(48, 128, 864 * Math.min(1, t), 28, 8);
ctx.fill();

cards.forEach(([title, text, index]) => {
const y = 190 + index * 78;
const active = Math.floor(t * 4.2) >= index;
ctx.fillStyle = active ? "#ffffff" : "#eef2f7";
ctx.strokeStyle = active ? "#0f766e" : "#cbd5e1";
ctx.lineWidth = active ? 3 : 1;
roundRect(48, y, 864, 54, 8);
ctx.fill();
ctx.stroke();
ctx.fillStyle = active ? "#0f766e" : "#64748b";
ctx.font = "bold 18px Arial";
ctx.fillText(title, 70, y + 22);
ctx.fillStyle = "#334155";
ctx.font = "16px Arial";
ctx.fillText(text, 70, y + 44);
});

ctx.fillStyle = "#64748b";
ctx.font = "15px Arial";
ctx.fillText("Synthetic data only. No credentials, private manuscripts, PHI, external APIs, or network calls.", 48, 504);
}

async function main() {
if (!window.MediaRecorder) {
out.textContent = "ERROR: MediaRecorder unavailable";
return;
}
draw(0);
const stream = canvas.captureStream(12);
const recorder = new MediaRecorder(stream, { mimeType: "video/webm;codecs=vp8" });
const chunks = [];
recorder.ondataavailable = (event) => {
if (event.data && event.data.size > 0) {
chunks.push(event.data);
}
};
recorder.onstop = () => {
const blob = new Blob(chunks, { type: "video/webm" });
const reader = new FileReader();
reader.onloadend = () => {
out.textContent = reader.result;
};
reader.readAsDataURL(blob);
};
recorder.start();
let frame = 0;
const timer = setInterval(() => {
draw(frame);
frame += 1;
if (frame >= 48) {
clearInterval(timer);
recorder.stop();
stream.getTracks().forEach((track) => track.stop());
}
}, 83);
}

main();
</script>
</body>
</html>`;

fs.mkdirSync(reportDir, { recursive: true });

const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-validity-demo-"));
const htmlPath = path.join(tempDir, "demo.html");
const profileDir = path.join(tempDir, "profile");
fs.writeFileSync(htmlPath, html, "utf8");

const browserPath = findBrowser();
const stdout = execFileSync(
browserPath,
[
"--headless=new",
"--disable-gpu",
"--disable-dev-shm-usage",
"--autoplay-policy=no-user-gesture-required",
"--run-all-compositor-stages-before-draw",
"--virtual-time-budget=7000",
`--user-data-dir=${profileDir}`,
"--dump-dom",
fileUrl(htmlPath)
],
{ encoding: "utf8", maxBuffer: 30 * 1024 * 1024 }
);

const match = stdout.match(/data:video\/webm;base64,([A-Za-z0-9+/=]+)/);
if (!match) {
throw new Error(`Demo video generation failed. Browser output ended with: ${stdout.slice(-600)}`);
}

fs.writeFileSync(outputPath, Buffer.from(match[1], "base64"));
console.log(`Generated ${path.relative(process.cwd(), outputPath)}`);
22 changes: 22 additions & 0 deletions external-validity-transfer-assistant/demo.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const fs = require("fs");
const path = require("path");
const { project } = require("./sample-data");
const { buildReviewPacket, renderMarkdownReport, renderSvgSummary } = require("./index");

const reportDir = path.join(__dirname, "reports");
fs.mkdirSync(reportDir, { recursive: true });

const packet = buildReviewPacket(project);

fs.writeFileSync(
path.join(reportDir, "summary.json"),
`${JSON.stringify(packet, null, 2)}\n`,
"utf8"
);
fs.writeFileSync(path.join(reportDir, "reviewer-packet.md"), renderMarkdownReport(packet), "utf8");
fs.writeFileSync(path.join(reportDir, "summary.svg"), renderSvgSummary(packet), "utf8");

console.log(`Generated reports for ${packet.assistant}`);
console.log(`Decision: ${packet.decision}`);
console.log(`Average score: ${packet.averageScore}`);
console.log(`Findings: ${packet.peerReviewSuggestions.length}`);
Loading