> ## Documentation Index
> Fetch the complete documentation index at: https://fuguai.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# Screen a literature search in the right order

> Thousands of papers, a review's written criteria, and a reading order in which the papers that belong come early.

export const SCREEN_PUZZLE = [{
  "title": "D‐penicillamine versus zinc sulfate as first‐line therapy for Wilson's disease",
  "kept": true,
  "inReview": true,
  "p": 0.85,
  "why": "A head-to-head trial of two of the four drugs: in the review, and hunch was sure."
}, {
  "title": "Poster Presentations",
  "kept": true,
  "inReview": false,
  "p": 0.02,
  "why": "Listed as a conference's poster session. The record hunch read holds one poster abstract, about another disorder, so it said no. The screeners kept it; it did not make the review."
}, {
  "title": "Hepatic manifestations of Wilson’s disease: 12-year experience in a Swiss tertiary referral centre",
  "kept": true,
  "inReview": true,
  "p": 0.08,
  "why": "A single hospital's patients over 12 years. It made the review, so it compared treatments, which neither title nor abstract makes plain. hunch was nearly sure it wouldn't."
}, {
  "title": "Cause of death in Wilson disease",
  "kept": true,
  "inReview": false,
  "p": 0.06,
  "why": "Another group of patients followed over time. The screeners kept it too; after reading the full text, the reviewers left it out."
}, {
  "title": "The Efficacy of Oral Zinc Therapy as an Alternative to Penicillamine for Wilson's Disease",
  "kept": false,
  "inReview": false,
  "p": 0.08,
  "why": "Zinc against penicillamine sounds exactly right. It is a letter to a journal, and the criteria exclude letters. Asked without the publication type, hunch said yes (0.84); with it, no."
}, {
  "title": "A new risk locus in the ZEB2 gene for schizophrenia in the Han Chinese population",
  "kept": false,
  "inReview": false,
  "p": 0.02,
  "why": "A genetics study of schizophrenia that the literature search swept in. Nobody's yes."
}];

export const SCREEN_CURVES = {
  "anxiety": {
    "title": "Anxiety therapy",
    "papers": 9883,
    "inReview": 72,
    "hunch": [0.0, 8.3, 16.7, 26.4, 31.9, 40.3, 45.8, 47.2, 48.6, 54.2, 58.3, 59.7, 59.7, 62.5, 66.7, 66.7, 66.7, 69.4, 69.4, 69.4, 69.4, 72.2, 72.2, 76.4, 76.4, 77.8, 77.8, 77.8, 79.2, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 83.3, 86.1, 87.5, 87.5, 88.9, 88.9, 88.9, 88.9, 90.3, 90.3, 91.7, 94.4, 94.4, 94.4, 94.4, 97.2, 98.6, 98.6, 98.6, 98.6, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
    "active": [0.0, 1.4, 5.6, 11.1, 15.3, 26.4, 30.6, 36.1, 43.1, 43.1, 44.4, 50.0, 52.8, 56.9, 59.7, 63.9, 68.1, 72.2, 76.4, 77.8, 80.6, 80.6, 81.9, 81.9, 83.3, 83.3, 83.3, 87.5, 88.9, 88.9, 90.3, 91.7, 91.7, 93.1, 94.4, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 97.2, 98.6, 98.6, 98.6, 98.6, 98.6, 98.6, 98.6, 98.6, 98.6, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]
  },
  "diet_risk": {
    "title": "Diet and risk",
    "papers": 5244,
    "inReview": 142,
    "hunch": [0.0, 14.8, 22.5, 30.3, 34.5, 39.4, 43.0, 46.5, 51.4, 55.6, 59.2, 64.8, 65.5, 66.9, 68.3, 69.0, 70.4, 71.8, 72.5, 73.9, 76.1, 77.5, 77.5, 77.5, 78.9, 78.9, 79.6, 81.0, 81.7, 83.1, 83.8, 84.5, 85.2, 85.2, 86.6, 86.6, 87.3, 87.3, 88.0, 88.0, 88.0, 88.7, 89.4, 90.1, 90.1, 90.1, 90.1, 90.1, 90.1, 90.8, 91.5, 92.3, 93.0, 93.0, 94.4, 94.4, 94.4, 94.4, 94.4, 95.1, 95.1, 95.1, 95.8, 95.8, 95.8, 96.5, 97.2, 97.9, 98.6, 98.6, 98.6, 98.6, 98.6, 98.6, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
    "active": [0.0, 8.5, 12.0, 14.1, 21.1, 25.4, 28.2, 29.6, 31.0, 33.1, 35.2, 38.0, 42.3, 46.5, 50.0, 51.4, 52.1, 52.1, 54.2, 54.2, 54.2, 54.9, 55.6, 55.6, 56.3, 56.3, 56.3, 56.3, 56.3, 56.3, 56.3, 57.7, 57.7, 57.7, 59.2, 59.9, 62.0, 62.7, 63.4, 65.5, 66.2, 67.6, 69.0, 69.0, 69.0, 71.1, 72.5, 73.9, 74.6, 75.4, 76.1, 76.1, 76.1, 76.1, 76.8, 78.9, 79.6, 79.6, 79.6, 80.3, 80.3, 81.0, 82.4, 83.1, 83.8, 84.5, 85.2, 85.2, 85.2, 87.3, 88.0, 88.7, 88.7, 89.4, 89.4, 90.1, 90.1, 90.8, 90.8, 90.8, 90.8, 91.5, 91.5, 91.5, 92.3, 93.0, 93.0, 93.0, 93.0, 93.0, 93.0, 93.0, 93.0, 93.7, 93.7, 93.7, 93.7, 93.7, 93.7, 93.7, 94.4, 94.4, 94.4, 94.4, 95.1, 95.1, 95.1, 95.1, 95.1, 95.1, 95.8, 95.8, 96.5, 96.5, 96.5, 96.5, 96.5, 96.5, 96.5, 97.2, 97.9, 97.9, 97.9, 97.9, 97.9, 97.9, 97.9, 97.9, 97.9, 97.9, 98.6, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 99.3, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]
  },
  "wilson": {
    "title": "Wilson disease",
    "papers": 2896,
    "inReview": 26,
    "hunch": [0.0, 19.2, 23.1, 23.1, 23.1, 26.9, 26.9, 26.9, 26.9, 26.9, 26.9, 26.9, 30.8, 34.6, 34.6, 34.6, 38.5, 42.3, 42.3, 42.3, 42.3, 42.3, 46.2, 46.2, 46.2, 50.0, 53.8, 53.8, 57.7, 57.7, 61.5, 65.4, 69.2, 73.1, 73.1, 73.1, 76.9, 84.6, 84.6, 84.6, 84.6, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 92.3, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
    "active": [0.0, 3.8, 15.4, 34.6, 46.2, 57.7, 57.7, 57.7, 65.4, 65.4, 69.2, 76.9, 76.9, 76.9, 76.9, 76.9, 80.8, 80.8, 80.8, 80.8, 80.8, 80.8, 80.8, 80.8, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 84.6, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 88.5, 92.3, 92.3, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 96.2, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]
  }
};

export const PaperPuzzle = ({papers}) => {
  const [picks, setPicks] = useState({});
  const teal = "#7D969B", red = "#C64D35", grey = "rgba(128,128,128,0.25)";
  const btn = (k, v, label) => <button key={label} onClick={() => picks[k] === undefined && setPicks({
    ...picks,
    [k]: v
  })} disabled={picks[k] !== undefined} style={{
    border: `1px solid ${picks[k] === v ? "currentColor" : grey}`,
    borderRadius: 8,
    padding: "3px 12px",
    background: "transparent",
    color: "inherit",
    font: "inherit",
    fontSize: 13,
    cursor: picks[k] === undefined ? "pointer" : "default"
  }}>{label}</button>;
  return <div className="not-prose hunch-widget" style={{
    border: `1px solid ${grey}`,
    borderRadius: 12,
    padding: 16,
    margin: "16px 0"
  }}>
      <div style={{
    fontSize: 13,
    opacity: 0.7
  }}>Would you read the full paper? You see the title; hunch also read the abstract, where there was one, and the publication type.</div>
      {papers.map((it, k) => {
    const shown = picks[k] !== undefined;
    return <div key={k} style={{
      padding: "10px 0",
      borderTop: k ? `1px solid ${grey}` : "none"
    }}>
            <div style={{
      display: "flex",
      justifyContent: "space-between",
      alignItems: "center",
      gap: 10,
      flexWrap: "wrap"
    }}>
              <span style={{
      fontSize: 15,
      flex: "1 1 240px"
    }}>{it.title}</span>
              <span style={{
      display: "flex",
      gap: 6
    }}>{btn(k, true, "Read it")}{btn(k, false, "Skip")}</span>
            </div>
            {shown && <div style={{
      fontSize: 13,
      marginTop: 6,
      lineHeight: 1.5
    }}>
                <div style={{
      display: "flex",
      gap: 14,
      flexWrap: "wrap",
      marginBottom: 3
    }}>
                  <span>Screeners: <b>{it.kept ? "read it" : "skipped"}</b></span>
                  <span>In the review: <b style={{
      color: it.inReview ? teal : "inherit"
    }}>{it.inReview ? "yes" : "no"}</b></span>
                  <span>hunch: <b>p(yes) {it.p.toFixed(2)}</b></span>
                </div>
                <span style={{
      opacity: 0.85
    }}>{it.why}</span>
              </div>}
          </div>;
  })}
    </div>;
};

export const FoundCurve = ({data}) => {
  const names = Object.keys(data);
  const [review, setReview] = useState(names[0]);
  const [step, setStep] = useState(80);
  const d = data[review];
  const teal = "#7D969B", gold = "#C9A227", grey = "rgba(128,128,128,0.25)";
  const W = 320, H = 170, L = 8, B = 8, maxStep = 200;
  const x = i => L + i / maxStep * (W - L - 4), yv = v => H - B - v / 100 * (H - B - 6);
  const path = arr => arr.map((v, i) => `${i ? "L" : "M"}${x(i).toFixed(1)},${yv(v).toFixed(1)}`).join("");
  const read = step / 2, n = Math.round(d.papers * read / 100);
  const by = (arr, pct) => arr.findIndex(v => v >= pct) / 2;
  return <div className="not-prose hunch-widget" style={{
    border: `1px solid ${grey}`,
    borderRadius: 12,
    padding: 16,
    margin: "16px 0"
  }}>
      <div style={{
    display: "flex",
    gap: 6,
    flexWrap: "wrap",
    marginBottom: 10
  }}>
        {names.map(k => <button key={k} onClick={() => setReview(k)} style={{
    border: `1px solid ${k === review ? "currentColor" : grey}`,
    borderRadius: 8,
    padding: "4px 10px",
    background: "transparent",
    color: "inherit",
    font: "inherit",
    fontSize: 13,
    cursor: "pointer",
    opacity: k === review ? 1 : 0.75
  }}>{data[k].title}</button>)}
      </div>
      <label style={{
    display: "block",
    fontSize: 14
  }}>
        Read the first <b>{read}%</b> of the pile: {n.toLocaleString("en")} of {d.papers.toLocaleString("en")} papers
        <input type="range" min={0} max={maxStep} value={step} onChange={e => setStep(+e.target.value)} aria-label="Share of the pile read" style={{
    width: "100%",
    accentColor: teal,
    marginTop: 6
  }} />
      </label>
      <svg viewBox={`0 0 ${W} ${H}`} style={{
    width: "100%",
    maxWidth: 560,
    display: "block",
    margin: "4px 0"
  }} role="img" aria-label={`After reading ${read}%, hunch's order has found ${d.hunch[step]}% of the review's papers, active learning ${d.active[step]}%`}>
        <line x1={L} y1={yv(0)} x2={W - 4} y2={yv(0)} stroke={grey} />
        <line x1={L} y1={yv(100)} x2={W - 4} y2={yv(100)} stroke={grey} strokeDasharray="3 3" />
        <path d={path(d.active)} fill="none" stroke="rgba(128,128,128,0.6)" strokeWidth={2} />
        <path d={path(d.hunch)} fill="none" stroke={teal} strokeWidth={2.5} />
        <line x1={x(step)} y1={yv(0)} x2={x(step)} y2={yv(100)} stroke={gold} strokeWidth={1.5} />
      </svg>
      <div style={{
    fontSize: 12,
    opacity: 0.7,
    display: "flex",
    justifyContent: "space-between"
  }}><span>read 0%</span><span>dashed line: all {d.inReview} found</span><span>100%</span></div>
      <div style={{
    fontSize: 14,
    marginTop: 10,
    lineHeight: 1.7
  }}>
        <div><span style={{
    color: teal
  }}>━</span> In hunch's order: <b>{d.hunch[step]}%</b> of the {d.inReview} papers in the review found; 95% of them by {by(d.hunch, 95)}%, all by {by(d.hunch, 100)}%</div>
        <div><span style={{
    opacity: 0.6
  }}>━</span> Active learning: <b>{d.active[step]}%</b>; 95% by {by(d.active, 95)}%, all by {by(d.active, 100)}%</div>
      </div>
    </div>;
};

A systematic review of therapy for anxiety began with a literature search that returned 9,883 papers. In the end, 72 of them were in the review. To find those 72, the reviewers read the title and abstract of all 9,883. Could they have known which ones to skip?

<Info>
  Systematic reviews decide which treatments doctors use, which policies governments fund and which findings a field believes. The slowest part is screening: reading thousands of titles and abstracts against criteria written before the search. The criteria are the part hunch runs.
</Info>

## Try it first

A review of Wilson disease, a rare disorder in which copper builds up in the body, asked which of four drugs works best. Its rule began: *"We included WD patients of any age or stage. The study drug had to be one of four established therapies, namely DPen, trientine, TTM or Zn."* It excluded, among others, *"abstract-only publications"*.

<PaperPuzzle papers={SCREEN_PUZZLE} />

What decides a paper is often not in its title. Whether a hospital's twelve years of patients compared the drugs is in the full text; that a paper is a letter is in its metadata. Screening doesn't decide what goes in the review; it decides what gets read.

## 1. Get the papers

```bash theme={null}
uv run --with synergy-dataset \
  python -m synergy_dataset get
D=prototype/examples/screening
uv run python $D/fetch.py
```

[SYNERGY](https://github.com/asreview/synergy-dataset) is an open collection of real systematic reviews: every paper each search found, whether the screeners kept it after reading its title and abstract, and whether it ended up in the review. `fetch.py` adds what a screening tool shows: each paper's type (article, review, letter, ...) and journal. Abstracts can't be republished, so the first command shows a legal note and `fetch.py` rebuilds them on your machine only, with PubMed filling gaps. Some papers still have only a title.

## 2. Paste the review's criteria into a question

```yaml prototype/examples/screening/anxiety.yml theme={null}
judgment: anxiety
source: .cache/van_Dis_2019.csv
key: id
state: [title, type, venue, abstract]
questions:
  anxiety:
    type: noul      # yes or no, with how sure
    instructions: >-
      `title`, `type` (article, review,
      letter, ...), `venue` and `abstract`
      describe a paper found by the
      literature search for a systematic
      review ... The
      review's eligibility criteria, as its
      authors published them: "Randomized
      clinical trials were included that
      examined effects of CBT ..." This is
      title-and-abstract screening: could
      this paper meet those criteria, so
      that someone should read the full
      text? ... when what you can see leaves
      it open, "yes".
    gold: gold      # the screeners' decision
```

```bash theme={null}
hunch run $D/anxiety.yml --max-cost 0.5
```

It reads all 9,883 papers in under three minutes, for \$0.41. Nobody screened a single paper first: the question is the criteria, word for word.

## 3. Read in hunch's order

Sort the pile by how sure hunch is that a paper could qualify, and read from the top. Drag the line to see how many of the papers that ended up in the review you have found:

<FoundCurve data={SCREEN_CURVES} />

In the anxiety review, all 72 are in the first 30% of the pile. The other 6,900 papers could have gone unread without losing one of them.

## 4. Decide when to stop

A reviewer can't see the curve; they don't know which papers are in the review until they've read them. So the stopping rule has to be set in advance, and checked. `act` says how sure a "no" must be to set a paper aside unread, and `min_recall` says how many of the screeners' papers must still be read:

```yaml prototype/examples/screening/anxiety.yml theme={null}
    act: {yes: 1.0, no: 0.96}
tests:
  anxiety: {min_recall: 0.92}
```

```text theme={null}
PASS recall of yes 96.1%
  (95% CI 94.4%–97.3%)
  of 689 gold-yes rows at act 0.96:
  a person reads 41% of rows, the
  confident "no" answers are set aside
  (min 92% on the interval's lower bound)
```

The bar of 0.96 was chosen on half the papers and tested on the other half, where it kept 95% of what the screeners kept and set aside none of the papers that ended up in the review. Choosing it needs screening decisions; ranking the pile did not.

## The lesson

The tool this field already uses is active learning: a model that watches each screening decision and moves papers like the ones kept up the pile. It is a strong baseline. On the anxiety review it beats hunch everywhere: 95% of the papers in the review after 17% of the pile, against hunch's 28%. On the diet review hunch wins everywhere: 30% against 52%. On Wilson disease it's split. Active learning finds what the screeners kept sooner (95% after 44–51% of the pile, against hunch's 77%); hunch finds the papers that ended up in the review sooner (95% after 28%, against 37%).

The difference is what each one learns from. Active learning copies the screeners, including the papers they keep just in case. hunch reads the criteria, before anyone has screened a paper. When the two disagree, as on Wilson disease, the disagreement is worth reading: it is where the screeners' habits and the written rule part.

## Use it on your data

Export your search results with titles and abstracts, paste your criteria into `instructions`, screen a few hundred papers to set the bar, and `hunch test` tells you how much you can safely leave unread.

## How it was measured

<AccordionGroup>
  <Accordion title="The data">
    Three reviews from [SYNERGY](https://doi.org/10.34894/HE6NAQ) (De Bruin et al. 2023, CC0): van Dis et al. 2020 on cognitive behavioural therapy for anxiety (9,883 papers, 689 kept at screening, 72 in the review); Moran et al. 2020 on whether poor nutrition makes animals take more risks (5,244; 619; 142); Appenzeller-Herzog et al. 2019 on drugs for Wilson disease (2,896; 146; 26). `gold` is the screeners' title-and-abstract decision; `included` whether the paper ended up in the review, which every such paper had passed at screening. Each paper also carries its type and journal from OpenAlex, which hunch sees and active learning doesn't (it reads title and abstract). OpenAlex no longer has most abstracts; `fetch.py` fills gaps from PubMed, leaving 90%, 71% and 63% of the papers with one. Each spec quotes the review's criteria as SYNERGY records them. This page shows paper titles (OpenAlex metadata, CC0) and quotes no abstract. A first run without type and journal cost \$0.73; it said yes to letters and reviews the criteria exclude, which an adversarial review caught. A fourth review, on class size in schools, was dropped because PubMed covers almost none of its papers.
  </Accordion>

  <Accordion title="The results, in full">
    From `measure.py`; the share of the pile read, in order of hunch's p(yes), with 95% intervals from 1,000 bootstrap resamples.

    | Review | Reader | 95% of the screeners' papers | Papers in the review: 95% | all but one | all |
    | - | - | - | - | - | - |
    | Anxiety | hunch | 37.3% (34.0–41.4) | 27.8% | 28.0% | 30.4% |
    | | active learning | 31.7–32.3% | 16.7–17.2% | 18.9–24.8% | 26.7–29.2% |
    | Diet and risk | hunch | 44.9% (42.0–46.9) | 29.5% | 36.8% | 41.9% |
    | | active learning | 52.6–52.7% | 51.9–52.2% | 64.9–65.3% | 70.7–70.9% |
    | Wilson disease | hunch | 77.3% (46.3–97.3) | 27.6% | 27.6% | 38.9% |
    | | active learning | 44.2–50.9% | 36.8–37.3% | 36.8–37.3% | 72.4–72.7% |

    Active learning's ranges span its three runs. With few papers in each review (72, 142, 26), the last one or two decide the "all" column: active learning's last paper on Wilson disease and on diet has no abstract, so read the "95%" and "all but one" columns as the steadier comparison. Work saved at 95% of the screeners' papers (WSS\@95) in hunch's order: 57.7%, 50.1%, 17.7%. Papers with only a title are harder: in hunch's order, 95% of the screeners' title-only papers take 63%, 76% and 72% of the title-only pile, against 36%, 32% and 93% with an abstract (on Wilson disease the screeners kept many papers whose abstracts the criteria rule out).
  </Accordion>

  <Accordion title="How active learning was run">
    ASReview 3.0.8's simulator (`baseline.py`), its default model (elas\_u4), starting from one kept and one rejected paper, three times from different starting papers per review; it learns from the screeners' decisions, hunch from none, and it stops once it has found every paper the screeners kept. Starting it instead from hunch's ten likeliest papers, as a reviewer who screens those first would (8, 10 and 8 of them kept), changed almost nothing: 95% of the papers in the review after 16.4%, 52.2% and 36.0% of the pile, all of them after 28.5%, 70.8% and 72.3%. The runs are local and free.
  </Accordion>

  <Accordion title="Wilson disease: where hunch and the screeners part">
    Reading in hunch's order, 95% of what the Wilson disease screeners kept takes 77% of the pile, and a recall bar for them would mean reading almost everything, so that spec has none. Among the papers they kept are records of conference poster sessions and cohorts of patients followed over time, which might compare the drugs and can only be told apart by the full text. hunch mostly says no to both; among the papers in the review, the ones it was least sure of (p(yes) 0.06 to 0.11) are such cohorts and one OpenAlex types as a review. Still, 25 of the 26 are in the first 28% of the pile, and all of them in the first 39%.
  </Accordion>

  <Accordion title="The recall bars were chosen on half the papers">
    `act` for "no" is the lowest bar that keeps 95% of the screeners' papers on one half of each review (split by a hash of the paper's id), then checked on the other half: anxiety 0.96 (95.3% kept, 40.2% read, none of the 38 papers in the review set aside), diet and risk 0.96 (96.1%, 45.8%, none of 58). The `min_recall` checks in the specs run on all papers, so they are regression bars; the half-split numbers are the honest ones.
  </Accordion>

  <Accordion title="Limits">
    Three reviews, all from one collection. The criteria are those the reviews published, which can be sharper than the protocol the screeners worked from. The papers and reviews are published online, and the model may have seen some of them. Screening decisions are only as good as the screeners: a paper they wrongly skipped counts against nobody.
  </Accordion>

  <Accordion title="Cost">
    All three reviews, 18,023 papers, with type and journal: $0.76 (anxiety $0.41 in 165 seconds). Before that, $0.69 for the run without them and $0.04 for a 300-paper trial of each: $1.49 in all. The answers are kept, so `measure.py` and the widgets rebuild for $0.
  </Accordion>
</AccordionGroup>