diff --git a/.github/workflows/matched-history-budget-2026.yml b/.github/workflows/matched-history-budget-2026.yml new file mode 100644 index 0000000..e3f39df --- /dev/null +++ b/.github/workflows/matched-history-budget-2026.yml @@ -0,0 +1,71 @@ +name: "Paper 1 - 2026 matched-history-budget experiment" +on: + push: + branches: + - research/kt-methodological-2026-history-budget + paths: + - METHODOLOGICAL_2026_PROTOCOL.md + - scripts/matched_history_budget_2026.py + - .github/workflows/matched-history-budget-2026.yml + workflow_dispatch: +permissions: + contents: write +jobs: + matched-budget: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned scientific stack + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Code compilation and causal prefix smoke checks + run: | + python -m py_compile scripts/matched_history_budget_2026.py + PYTHONPATH=src python scripts/matched_history_budget_2026.py --smoke + - name: Execute same-target independent-cohort budget comparison + run: PYTHONPATH=src python -u scripts/matched_history_budget_2026.py + - name: Validate learner and calculation invariants + run: | + python - <<'PY' + import json + a=json.load(open("results/matched_history_budget_2026.json")) + assert a["status"]=="passed_all_computational_gates",a.get("diagnostic_failures") + assert a["source"]["source_sha256"]=="75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0" + assert a["source"]["cohort_overlap_previous"]==0 + assert a["source"]["stage3_learners"]==4000 + assert a["design"]["test_learners"]==600 + assert set(a["comparisons"])=={"1","20"} + for b in a["comparisons"].values(): + assert set(b["models"])=={"window_count","ordered_logistic","window_gru"} + for x in b["models"].values(): + assert 010000 + for x in b["paired_GRU_minus"].values(): + assert 400 <= x["n_learners"] <= 600 and x["n_bootstrap"]==2000 + assert x["n_interactions"]==b["models"]["window_gru"]["n"] + print("Verified fresh learner cohort and history-parity output gates.") + PY + - name: Upload aggregate evidence even if a research gate fails + uses: actions/upload-artifact@v4 + if: always() + with: + name: matched-history-budget-2026-aggregate + path: results/matched_history_budget_2026.json + if-no-files-found: warn + - name: Archive de-identified aggregate results + if: success() + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/matched_history_budget_2026.json + if ! git diff --cached --quiet; then + git commit -m "research: archive matched-history budget empirical results" + git push origin HEAD:research/kt-methodological-2026-history-budget + fi diff --git a/METHODOLOGICAL_2026_PROTOCOL.md b/METHODOLOGICAL_2026_PROTOCOL.md new file mode 100644 index 0000000..973af25 --- /dev/null +++ b/METHODOLOGICAL_2026_PROTOCOL.md @@ -0,0 +1,32 @@ +# Frozen 2026 methodological extension: matched history budgets (26 October 2026) + +This document is committed **before any stage-3 outcome is evaluated**. Previous ASSISTments 2009 mirror, corrected-source 2009, and 2015 stage-1/2 results have been examined, so this is a **new, prospectively specified 2015 holdout analysis conditional on earlier model development**, not a public registry preregistration and not a new educational intervention. + +## Research question and defensible novelty +How do the *number of available prior student responses* and the *representation of their order* interact with the predictive differences between count-based generalized PFA and a small recurrent knowledge tracer, on the **same response targets**? This study does not claim to invent information-parity evaluation (see Padi et al., 2026) or cross-cohort benchmarks (Polydoros, 2026). The contribution tested here is a frozen **history-budget response curve** on a fresh learner cohort, linked to independent corrected-source/mirror provenance audits. + +## Official source and untouched sample +Use exact original bytes of **ASSISTments 2015 Skill Builder Data**, SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`, downloaded only through the official portal link (as in existing scripts). The source contains 19,840 learners eligible under fixed existing stage-1/2 filters, ordered within learner by numeric `log_id`; use `sequence_id` as the 2015 skill-builder unit. + +Recreate stage-1 sampled learners with `random.Random(20261022).sample(sorted_eligible_ids, 5000)`; remove them. Recreate stage-2 sampled learners with `random.Random(20261023).sample(sorted_remaining, 5000)`; remove them. Independently sample **4,000** learners with `random.Random(20261026).sample(sorted_remaining_after_both,4000)`. Assert absolutely zero overlap with all previously evaluated 10,000 learners. Learner-disjoint **70/15/15** train/validation/test, split seed **20261026** (2,800/600/600). Test uses only responses at index t>=1 (the first response has no past to compare), regardless of history budget. + +## Models and information budget +History budgets `K=1` and `K=20`, determined by number of *prior response events*, never time. At target t, each model may access only current skill ID and the preceding `min(K,t)` pairs (skill ID and correctness); **never current correctness**. Training and inference both use exactly this restriction. + +1. **Window-count PFA+**: logistic regression with current skill one-hot, within-current-skill correct/incorrect counts and all-skill correct/incorrect counts derived *only* from the last K prior events. Regularized logistic regression `C=1.0`, `lbfgs`, `tol=1e-7`, `max_iter=1600`, fitted on training interactions only. Standardize numerical features on training only. +2. **Ordered-feature logistic**: same as window-count PFA+, plus correctness of latest prior interaction, latest occurrence of current skill (0 absent; +/-1 for correct/incorrect) and distance to most recent same-skill observation capped at K+1. The selected features are all derivable from the identical allowed history. This tests a transparent nonlinear-history proxy without upgrading PFA to an unrelated data stream. +3. **Window-GRU**: train a compact recurrent model on only the last K (skill,response) tokens with the current skill query embedded separately; variable-length sequences padded only to batch maximum; final valid hidden state is read per target. Embeddings 24/12, hidden 48; Adam LR0.002, batch 256, 12 epochs max, early stopping patience3 on *validation only*, seed42. Same model capacity across K. + +Each K uses the exact same train/validation/test labels and learners, no within-test tuning. Models retrain for each K. This is **information-source parity**, not parameter/complexity parity, and does not identify architectural causal effects. + +## Estimands and numerical criteria +Primary descriptive outputs: for each model and K report held-out ROC-AUC, Brier, log loss, ECE10, held-out interactions, and learner count; report GRU minus ordered-logistic and GRU minus count-logistic pooled interaction-weighted Brier differences with paired **2000 learner-cluster bootstrap percentile CIs** (seed 20261026). Report change in each model's Brier moving K=1 to K=20 and GRU-vs-ordered gap change. These comparisons are **exploratory multiple comparisons**; intervals are descriptive and not a controlled family-wise hypothesis rejection or causal effect. Do not label the result a state-of-the-art benchmark or universal replication. + +Sanity gates: checksum, exclusion counts, zero prior-cohort overlap, exact test IDs and target index agreement across all models/K, full train-only preprocessing, history prefix tests verifying the target answer is absent and only previous K events used, numerical convergence no warnings, finite outputs, numeric probabilities in [0,1], 2000 valid learner bootstrap draws. If any fail, output `inconclusive` and preserve diagnostics; never fabricate or quietly substitute results. + +## Non-overclaiming and next steps +Only merge observed results into manuscript after actual GitHub Actions real-data execution and critical result inspection. Earlier corrected-2009 and 2015 evidence remains separately reported. A 2026 survey/literature novelty statement must explicitly recognize recent matched-input and leakage papers. Additional modern datasets and AKT are useful but NOT claimed executed in this extension. No student-level data exported; only cohort hashes and aggregate predictions/statistics retained. + +## Post-execution correction to a software assertion (26 October 2026; no model/test retuning) + +The predeclared test split comprises **600 distinct learners**, but 18 test learners have no response at index t>=1 and therefore generate no history-eligible targets. **582 learners / 20,592 interactions** are evaluable on the common response target set. The first Actions run 40310088864 successfully executed both K budgets and wrote aggregate metrics (`status: passed_all_computational_gates`) but the downstream release assertion incorrectly demanded 600 **evaluated** learners. The workflow assertion was changed to verify **up to 600** evaluation learners, requiring more than 400, and exact event counts across paired models. There was no change to sampling, models, features, targets, seeds, or inferential analysis. The initial diagnostic artifact is retained. This correction and the initially failed pipeline must be disclosed as technical QA history; the new clean workflow run is the final computational release gate. diff --git a/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md b/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md new file mode 100644 index 0000000..5ae140e --- /dev/null +++ b/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md @@ -0,0 +1,41 @@ +# Executed 2026 matched-history-budget results — 26 October 2026 + +**Scientific outcome:** An interpretable order-aware logistic regression scored better Brier loss than the compact GRU on the same next-response targets under both fixed observable history budgets. The earlier GRU-over-PFA result does not generalize to every classical baseline under matched input stream availability. + +The official 2015 source SHA-256 was verified; stage 3 excluded the full 5,000-learner stage-1 sample and full 5,000-learner stage-2 sample. Of 19,840 source-eligible students, 9,840 were eligible for further sampling and 4,000 were selected for stage 3 (138,129 source events). Learner splits: 2,800 train / 600 validation / 600 test. **582** test learners produced **20,592** eligible targets at t>=1. The other 18 test learners had no post-first response and were properly excluded from next-response metrics. + +## Actual same-target results + +| Previous history K | Model | Brier (lower) | ROC-AUC (higher) | Log loss | ECE-10 | +|---|---|---:|---:|---:|---:| +| 1 | Count-based PFA+ | 0.177882 | 0.684440 | 0.534589 | 0.009320 | +| 1 | Order-aware logistic | **0.177882** | 0.684436 | 0.534589 | 0.009321 | +| 1 | GRU | 0.180003 | 0.670909 | 0.540658 | 0.005781 | +| 20 | Count-based PFA+ | 0.176674 | 0.691058 | 0.531590 | 0.006126 | +| 20 | Order-aware logistic | **0.173885** | 0.705044 | 0.524095 | 0.007251 | +| 20 | GRU | 0.175140 | 0.699361 | 0.527153 | 0.011658 | + +Paired learner-cluster bootstrap, 2,000 draws: **GRU minus order-aware Brier K=1 +0.002121** (95% interval [0.001198,0.003013]), **K=20 +0.001255** ([0.000075,0.002426]). GRU-minus-count at K=20 is **−0.001533**, interval [−0.002761, −0.000279], demonstrating the difference that comparator choice makes. + +Improvements in Brier from K=1 to K=20: count-based 0.001208, order-aware 0.003997, GRU 0.004863. Interpret intervals descriptively: fixed data split/model fits, one GRU seed, no multiple-comparison correction. + +## Audited execution status + +- New protocol committed **before evaluating stage-3 cohort**: [frozen protocol](../METHODOLOGICAL_2026_PROTOCOL.md) +- [Python implementation](../scripts/matched_history_budget_2026.py) +- [Real-data, checksum-verified and learner-disjoint aggregate JSON](../results/matched_history_budget_2026.json) +- [Successful GitHub Actions run 40310733288](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/40310733288). +- [Final 13-page author-review manuscript](main_methodological_2026.tex). +- [41 internal manuscript and numerical QA checks](METHODOLOGICAL_2026_FINAL_QA.json). +- [Figure generator](generate_figures.py). +- User-facing final PDF, minimal arXiv ZIP, and complete evidence package are preserved in ChatGPT Library folder `/Paper1_Methodological_2026`. + +### Transparency on prior release assertion +The initial run 40310088864 completed all model training, both history horizons and aggregate output. Its release gate nevertheless failed on a mistaken assertion requiring 600 **evaluated** learners, confusing split members with learners having t>=1 targets. The corrected check permits 582 observed evaluable test learners and verifies exactly equal event counts across models, without changing any data, model, tuning, or observed result. The final run succeeds. + +### Contribution boundaries +The study is **not** the first knowledge-tracing information-parity comparison, not the first source correction study and not a model architecture innovation. Polydoros (2026, DOI 10.5281/zenodo.22131530), Padi et al. (2026, arXiv:2609.28502), and Badran & Preisach (2026, DOI 10.1007/s42979-026-05344-3) are material prior work. The incremental contribution is a frozen, newly nonoverlapping 2015 cohort with explicitly constrained K=1 versus K=20 history budgets and order-feature comparator, interpreted alongside independently verified 2009 source mismatch. + +There is no basis for causal effect on educational outcomes, current schooling deployment, universal superiority of logistic models, or cross-year frozen-weight transfer. Next strengthening steps would be independent contemporary datasets, multiple splits/seeds and attention baselines; **none are claimed completed**. + +**Status:** Audited working preprint for named human author's scientific approval, not externally peer reviewed and not submitted to arXiv. diff --git a/paper/METHODOLOGICAL_2026_FINAL_QA.json b/paper/METHODOLOGICAL_2026_FINAL_QA.json new file mode 100644 index 0000000..d5c43ba --- /dev/null +++ b/paper/METHODOLOGICAL_2026_FINAL_QA.json @@ -0,0 +1,181 @@ +{ + "date": "2026-10-26", + "scope": "Internal computation/transcription/document QA, not outside peer review", + "checks_total": 41, + "passed": 41, + "failed": 0, + "checks": [ + { + "name": "Official 2015 CSV SHA-256 confirmed", + "passed": true + }, + { + "name": "Fresh cohort has no overlap with first/second sample", + "passed": true + }, + { + "name": "Correct stage3 learner splits", + "passed": true + }, + { + "name": "Result scientifically completed", + "passed": true + }, + { + "name": "Both history budgets present", + "passed": true + }, + { + "name": "Same 20592 targets across both budgets", + "passed": true + }, + { + "name": "582 evaluable history-eligible test learners", + "passed": true + }, + { + "name": "2k learner cluster resamples", + "passed": true + }, + { + "name": "No logistic convergence warnings", + "passed": true + }, + { + "name": "All six models report bounded metrics", + "passed": true + }, + { + "name": "Manuscript Brier 1 window_count", + "passed": true + }, + { + "name": "Manuscript AUC 1 window_count", + "passed": true + }, + { + "name": "Manuscript Brier 1 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript AUC 1 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript Brier 1 window_gru", + "passed": true + }, + { + "name": "Manuscript AUC 1 window_gru", + "passed": true + }, + { + "name": "Positive GRU-minus-ordered contrast for K=1", + "passed": true + }, + { + "name": "Paired interval excludes zero positive at K=1", + "passed": true + }, + { + "name": "Manuscript Brier 20 window_count", + "passed": true + }, + { + "name": "Manuscript AUC 20 window_count", + "passed": true + }, + { + "name": "Manuscript Brier 20 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript AUC 20 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript Brier 20 window_gru", + "passed": true + }, + { + "name": "Manuscript AUC 20 window_gru", + "passed": true + }, + { + "name": "Positive GRU-minus-ordered contrast for K=20", + "passed": true + }, + { + "name": "Paired interval excludes zero positive at K=20", + "passed": true + }, + { + "name": "New figure exists", + "passed": true + }, + { + "name": "Multiple comparison caveat", + "passed": true + }, + { + "name": "Late arxiv file includes AI-assistance caveat", + "passed": true + }, + { + "name": "No novel architect invention claim", + "passed": true + }, + { + "name": "Inherited figure kt_auc.png present", + "passed": true + }, + { + "name": "Inherited figure kt_brier.png present", + "passed": true + }, + { + "name": "Inherited figure robust_comparison.png present", + "passed": true + }, + { + "name": "arXiv archive ZIP integrity", + "passed": true + }, + { + "name": "Clean-room pdflatex pass 1", + "passed": true + }, + { + "name": "Clean-room pdflatex pass 2", + "passed": true + }, + { + "name": "Final PDF contains 20,592 evaluation count", + "passed": true + }, + { + "name": "Final PDF contains new method section", + "passed": true + }, + { + "name": "No LaTeX overfull warning", + "passed": true + }, + { + "name": "No unresolved citations", + "passed": true + }, + { + "name": "Final manuscript PDF hash matches clean-room compiled PDF", + "passed": true + } + ], + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "stage3_results_sha256": "e9a8cba68c2cb5b945d3a286aa27c0bfdc24b8841cd5052badc5ea4ac2e7381d", + "arxiv_source_sha256": "6d15e720f8bb613e40d9f4c1e6e4f9118d162092125dc749122f92cc2edcb6ff", + "pdf_sha256": "b0783067f84ac3c5b1f6b478f673e59465d6ecc2655833a0443831af4546609a", + "author_approval": false, + "external_peer_review": false, + "modern_independent_source_validation": false, + "scientific_caveat": "Single stage-3 split and GRU seed; matched input-stream availability does not match parameter count or optimization; descriptive bootstrap comparisons and 2009 source audit only" +} \ No newline at end of file diff --git a/paper/METHODOLOGICAL_2026_UPGRADE.md b/paper/METHODOLOGICAL_2026_UPGRADE.md new file mode 100644 index 0000000..9231195 --- /dev/null +++ b/paper/METHODOLOGICAL_2026_UPGRADE.md @@ -0,0 +1,97 @@ +> **ARCHIVED DRAFT NOTICE (26 October 2026):** This file preserves the design-stage manuscript with PENDING fields. Its outcomes are now executed and documented in [METHODOLOGICAL_2026_EXECUTED_RESULTS.md](METHODOLOGICAL_2026_EXECUTED_RESULTS.md) and the complete [main_methodological_2026.tex](main_methodological_2026.tex). Do not submit this archived draft as the final paper.\n\n# Toward a History-Budget Sensitivity Audit for Knowledge Tracing: Source Provenance, Information-Restricted Baselines, and Cross-Year Evidence + +**26 October 2026 — research draft, not a completed new experiment or a submission-ready replacement.** The numerical results of the newly frozen Stage-3 history-budget study must be inserted only after independent runner success and author QA. Earlier verified numerical results below are historical evidence and were already used for earlier preprint drafts. + +**Author:** Devis Wawan Saputra (authorship and contribution disclosure require personal approval) + +## Abstract (conditional working version) + +Knowledge tracing benchmarks often compare model families whose access to historical response information and whose underlying data transformations differ. Rather than treating historical model rankings as architecture effects, this study audits three separate sources of benchmark sensitivity: event-level provenance, the amount of prior learner history made available to predictors, and representation of that allowed history. Previous documented re-executions on a pinned third-party ASSISTments 2009 mirror and an independently sourced corrected 2009 release reveal nonidentical source representations: 817 of 4,148 shared learners have differing skill–correctness sequences, and the mirror omits 8,774 labeled events relative to the corrected official source. An earlier fresh, nonoverlapping ASSISTments 2015 test of 750 learners found an interaction-weighted Brier difference (extended GRU minus enriched PFA) of −0.004681 (95% learner-cluster percentile interval [−0.006073, −0.003346]), conditional on separately retrained models. We now prospectively freeze a third, disjoint 2015 learner cohort to estimate predictive sensitivity to history budgets of one versus 20 preceding interactions, comparing window-based count logistic regression, an ordered-feature logistic baseline, and a compact window-GRU on identical response targets. **Results from this latest experiment are pending execution/verification and are not yet asserted here.** The study is a test of benchmark interpretation, not a novel model architecture, causal educational intervention, or generalization guarantee. + +## 1. Introduction and specific research contribution + +Claims that sequential architectures outperform interpretable baselines are sensitive to the availability and representation of input data. Reproducible benchmarking therefore requires (a) identifying the exact event source; (b) fixing the observable learner history permitted at prediction time; (c) matching target events, learner splits, preprocessing scope, and uncertainty estimands; and (d) distinguishing computational/predictive results from causal pedagogical value. + +**Research questions:** + +- **RQ1 — provenance:** Can the common transformed ASSISTments 2009 mirror stand in for the source-corrected official 2009 Skill Builder data without changing its labeled event population? Earlier checksum-verified direct audits show **no**. +- **RQ2 — history budget:** Holding cohort, splits, test targets and model capacity fixed *within model family*, how does restricting the previous observable response stream to K=1 versus K=20 alter test Brier score and discrimination? +- **RQ3 — order representation:** At the same history budget and on exactly the same target responses, what is the descriptive predictive gap between a logistic model with summary counts, a logistic model augmented with derivable order proxies, and a small recurrent history encoder? +- **RQ4 — continuity across years:** Do earlier corrected-2009 and fresh 2015 direction-specific results survive careful qualification when placed alongside a new 2015 window study? This is comparison of the published scopes, **not** frozen-weight transfer nor a controlled cross-year experimental effect. + +The study's proposed incremental contribution is an **explicit two-axis source/history-window audit**, not the first proposal of input-matched knowledge tracing. + +## 2. Literature positioning (necessary non-overclaiming) + +- Polydoros (2026), *Cross-Cohort Knowledge-Tracing Benchmark*, Zenodo v2.0.1 (https://doi.org/10.5281/zenodo.22131530), compares PFA, BKT, DKT and SAKT on corrected ASSISTments 2009 and 2015. Hence neither cross-year comparison nor student-level cluster uncertainty is new here. +- Padi et al. (2026), *Stable and Faithful Explanations for Knowledge Tracing*, arXiv:2609.28502, explicitly studies information-matched baselines and duplicated KC records. Hence information parity and detecting leakage are not new constructs here. +- Badran and Preisach (2026), *Enhancing Knowledge Tracing through Leakage-Free and Recency-Aware Embeddings*, SN Computer Science 7:774, DOI 10.1007/s42979-026-05344-3, addresses multi-KC leakage and recency. This paper does **not** claim first detection of KC leakage, nor does it introduce their recency encoding. +- Gervet et al. (2020), *When Is Deep Learning the Best Approach to Knowledge Tracing?*, Journal of Educational Data Mining 12(3), demonstrates model-dependent comparisons on KT benchmarks. +- Piech et al. (2015), *Deep Knowledge Tracing*, NeurIPS 28, introduces neural sequential models and is background, not a method newly invented here. +- Eedi NeurIPS 2020 challenge (Wang et al., 2020, arXiv:2007.12061) publishes a larger, newer-source benchmark under CC BY-NC-ND 4.0; no experiment on Eedi is claimed in the current work. + +**What remains distinct and testable:** An explicitly pre-specified new learner subset with same-target restriction to previous 1 versus 20 response events and three representations, interpreted alongside a concrete **different-source 2009 count/sequence discrepancy**. Novelty is bounded; review by a subject specialist is necessary. + +## 3. Sources and prior verified findings + +### 3.1 Corrected 2009 vs transformed mirror + +Pinned third-party mirror SHA-256: `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`. Corrected official 2009 source SHA-256: `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`. The mirror includes 4,148 learner sequences and 274,331 labeled events; corrected official source includes 4,163 learners and 283,105 labeled events; 817/4,148 common learners differ in at least the count or sequence of skill–response pairs. These facts **reject exact source parity** and explain why old mirror-based test metrics cannot be silently labeled "corrected official 2009" results. + +### 3.2 Earlier predictive reanalyses + +Corrected official 2009 exploratory reanalysis: enriched PFA AUC 0.69508/Brier 0.196755 versus extended GRU AUC 0.73243/Brier 0.185074; 625 held-out learners/48,842 responses, with GRU minus PFA paired pooled Brier −0.011681 (2,000 learner-cluster bootstrap 95% interval [−0.013718, −0.009862]). This is **exploratory**, and its split/representation is not held fixed against the mirror benchmark; no model ranking change is causally attributed to source correction. + +2015 independent-year Stage 2: exclude 5,000 previously tested learners before selecting 5,000 new learners. Enriched PFA AUC 0.681286/Brier 0.186954 and GRU AUC 0.699683/Brier 0.182273 on 750 test learners/26,697 events, Brier delta −0.004681 [−0.006073, −0.003346]. Stage-1 result was discarded as confirmatory evidence after baseline numerical nonconvergence. Different-year models were retrained, not transported unchanged. + +## 4. New window-budget experiment — frozen before fresh data evaluation + +Complete operational specification and decision gates are in [METHODOLOGICAL_2026_PROTOCOL.md](../METHODOLOGICAL_2026_PROTOCOL.md); executable implementation is [scripts/matched_history_budget_2026.py](../scripts/matched_history_budget_2026.py). + +Official ASSISTments 2015 CSV SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`. Exclude all Stage-1 and Stage-2 learners (10,000 IDs), sample a new 4,000 from the remainder with seed 20261026, and split by learner with the same seed 2,800/600/600 for train/validation/test. Predictions for every model and history budget concern exactly the event targets at t>=1; the current response is never an input. + +- **K=1:** the single immediately preceding observed (skill, response) pair and current skill ID. +- **K=20:** at most 20 immediately preceding observed (skill, response) pairs and current skill ID. +- **Window-count logistic (PFA+):** current skill, previous same-skill success/failure counts and all-skill success/failure counts within the K-window. +- **Ordered-feature logistic:** same features, plus previous response outcome, most recent same-skill response sign and lag since that same-skill observation (all derivable from the same window). +- **Window-GRU:** a compact recurrent model reading only K preceding event tokens, current skill as prediction query, with train-only fitting and validation-only early stopping. + +All models share the **observable input stream**, learner populations, and test events; architectures, feature engineering and parameter counts differ. Numeric classifiers must converge and GRU training must complete under identical frozen hyperparameters across budgets. + +### Predeclared descriptive primary table — do not fill until real results exist + +| History budget | Model | Test Brier | Test ROC-AUC | Test log loss | Learners / test events | +|---|---|---:|---:|---:|---| +| K=1 | Window-count PFA+ | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=1 | Ordered-feature logistic | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=1 | Window-GRU | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Window-count PFA+ | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Ordered-feature logistic | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Window-GRU | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | + +Matched comparison: GRU Brier minus ordered-feature logistic Brier at each K, paired learner-cluster bootstrap (2,000 repetitions); positive/negative differences interpreted descriptively, and compared across budgets. All repeated choices are labeled exploratory with no family-wise multiplicity guarantee. + +## 5. Validity threats and interpretation bounds + +- New sample comes from the *same historical platform/year* as Stage 1 and 2; no modern-school deployment or external ecological validity follows. +- Split-seed and model-hyperparameter sensitivity are not completely assessed by a single stage-3 split and GRU seed. +- Ordered-feature logistic transforms historical signals explicitly, whereas recurrent models learn their own representations; equal input *availability* is not equal expressive capacity or optimization. +- Sequence labeling is based on recorded `log_id`, not on proof of clinical/educational learning states. Learner-based partitions prevent train–test ID overlap but do not rule out all upstream duplicated-question artifacts. +- Correction of 2009 data is a provenance result. A direct *paired* source-effect estimate would require aligned exact learner-event IDs and targets, unavailable from the transformed mirror. +- Bootstrap CIs are conditional on analyzed learners/model fits and not prospective causal intervals; model selection, multiple comparisons and the reviewed historical corpus limit confirmation language. +- Eedi and attention-model benchmarks are promising additional contexts but have **not** been executed and must not appear as study results. +- Disclosure of substantial generative AI involvement in coding/manuscript drafting is required by author policy; author must independently check data, code, citations and final interpretation. + +## 6. Conditional scientific conclusion + +**Until Stage-3 completes:** Earlier corrected-source and independent-year historical evidence shows reproducible predictive advantages of the tested GRU over the specific summary PFA baseline, but cannot attribute them to recurrence itself. The newly specified history-budget experiment is not claimed as executed, favorable, significant or publication-ready yet. + +**After execution:** The conclusion must be driven by observed changes and confidence intervals; an absence of predicted differences, reversal, or numerical failure is a reportable result rather than grounds for secretly retuning and replacing the study. + +## Links and reproducibility + +- [Official ASSISTments 2009 and corrected file](https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010). +- [Official ASSISTments 2015](https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data). +- [Prior corrected source reanalysis](../results/corrected_2009_sensitivity.json). +- [Prior independent-year 2015 result](../results/confirmatory_assist2015_stage2.json). +- New Stage-3 results: [planned fixed output path](../results/matched_history_budget_2026.json), only valid when source file is actually created by the successful real-data pipeline. diff --git a/paper/generate_figures.py b/paper/generate_figures.py index 565d1c3..93ec4e1 100644 --- a/paper/generate_figures.py +++ b/paper/generate_figures.py @@ -65,4 +65,24 @@ def bars(items,metric,title,filename,ylim=None): bars([('Enriched PFA (converged)',ms['enriched_pfa']),('Extended GRU',ms['extended_gru'])], 'brier','Fresh ASSISTments 2015: 750 learners; 26,697 held-out responses', 'cohort2015_brier.png',ylim=(0,.23)) -print('Generated four manuscript figures from executed JSON.') \ No newline at end of file +print('Generated four manuscript figures from executed JSON.') + +# 2026 matched-history-budget figure, recomputed from executed aggregate evidence. +budget=read("results/matched_history_budget_2026.json") +assert budget["status"]=="passed_all_computational_gates" +assert budget["source"]["cohort_overlap_previous"]==0 +fig,ax=plt.subplots(figsize=(7.25,3.35)) +for name,label in [("window_count","Window-count PFA+"),("ordered_logistic","Order-aware logistic"),("window_gru","Window-GRU")]: + y=[budget["comparisons"][str(k)]["models"][name]["brier"] for k in (1,20)] + ax.plot([1,20],y,marker="o",linewidth=1.8,markersize=6,label=label) + for k,v in zip((1,20),y): + ax.annotate(f"{v:.4f}",(k,v),xytext=(3,7 if name!="window_count" else -14),textcoords="offset points",fontsize=7) +ax.set_xticks([1,20],["1 prior event","20 prior events"]) +ax.set_ylabel("Interaction-pooled test Brier score (lower better)") +ax.set_xlabel("Information budget per next-response prediction") +ax.grid(axis="y",alpha=.27) +ax.legend(loc="upper right",fontsize=8,frameon=False) +fig.tight_layout() +fig.savefig(OUT/"history_budget_brier.png",dpi=190) +plt.close(fig) +print("Generated new 2026 matched-budget figure.") diff --git a/paper/main_methodological_2026.tex b/paper/main_methodological_2026.tex new file mode 100644 index 0000000..f5695d0 --- /dev/null +++ b/paper/main_methodological_2026.tex @@ -0,0 +1,261 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.74in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\hypersetup{pdftitle={When Knowledge-Tracing Rankings Depend on Available History},pdfauthor={Devis Wawan Saputra}} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{When Knowledge-Tracing Rankings Depend on Available History\\\large A Source-Correction and Matched-Information Audit on ASSISTments 2009--2015} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{26 October 2026} +\begin{document} +\maketitle +\begin{abstract} +Knowledge-tracing benchmarks often compare models that have different access to a learner's previous responses. We examine how data provenance, history length, and representation affect those comparisons. First, we compare a pinned ASSISTments 2009 sequence mirror against the corrected official release. Of 4,148 learners represented in both, 817 have different ordered skill--response sequences; the mirror also contains fewer labeled events. Second, we retain earlier learner-disjoint model evaluations on the 2009 and 2015 datasets, distinguishing exploratory results from a corrective 2015 replication. Finally, we evaluate a new, nonoverlapping sample of 4,000 ASSISTments 2015 learners using three models constrained to the same one-event or 20-event prior-response window. On 20,592 test responses from 582 learners, the 20-event Brier scores were 0.17667 for count-based logistic regression, 0.17389 for an order-aware logistic model, and 0.17514 for a GRU. The GRU-minus-order-aware difference was 0.00125 (95\% learner-cluster bootstrap interval [0.00008, 0.00243]); at one prior event, the corresponding difference was 0.00212 [0.00120, 0.00301]. Earlier comparisons had favored the GRU against simpler history summaries. The newer analysis instead favors a logistic comparator that represents order using the same permitted event stream. These results concern predictive performance on particular historical cohorts and model settings. They neither isolate an architectural effect nor establish generalization to contemporary classrooms. + +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; information parity; history window; model ranking sensitivity; reproducibility. + +\section{Introduction} +Knowledge tracing predicts a student's next response from earlier practice. Binary correctness histories allow comparisons among simple frequency estimates, probabilistic mastery models, logistic models, and neural sequence encoders. Such comparisons are useful only if the underlying event records and the information supplied to each predictor are described accurately. + +Two problems recur in published benchmarks. Datasets distributed under the same name may reflect different filtering and serialization decisions. Moreover, a recurrent predictor may see answers on other skills and their order, while a conventional performance-factors model sees counts for the queried skill. A performance difference in that setting combines model specification with differences in available history. It cannot be attributed to recurrence alone. + +We investigate these problems using the ASSISTments 2009--2010 and 2015 Skill Builder releases. We first document what changes between a widely used transformed 2009 mirror and the source-corrected official records. We then retain the earlier held-out results, including a failed optimization diagnostic and its independently sampled follow-up. The main new experiment restricts all models to the same prior-response window and scores their predictions on precisely the same held-out response events. We ask whether the GRU advantage reported against a count-based baseline persists after adding explicit order information to an interpretable logistic model, and whether that comparison changes between windows of one and 20 previous events. + +\section{Related Work and Positioning} +Bayesian Knowledge Tracing (BKT) represents learning through an evolving latent mastery probability \citep{corbett1995}. Performance Factors Analysis (PFA) uses previous successes and failures as predictors in a logistic model \citep{pavlik2009}; Deep Knowledge Tracing introduced recurrent sequence encoders \citep{piech2015}. Later studies and surveys found that comparisons vary with datasets, splits, preprocessing, and baseline implementation \citep{gervet2020,abdelrahman2023}. + +This is an active methodological literature. A 2026 cross-cohort replication examines BKT, PFA, and neural methods using corrected ASSISTments releases \citep{polydoros2026}. Padi and colleagues test information-matched prediction and identify leakage associated with duplicated skill records \citep{padi2026}. Badran and Preisach examine leakage-free and recency-aware sequence representations \citep{badran2026}. Thus, correcting historical data and equalizing input information are not novel ideas by themselves. Our narrower test is a frozen history-window comparison on a new 2015 learner sample, reported alongside a concrete check of the 2009 mirror's provenance. We do not claim a new knowledge-tracing architecture or a comprehensive ranking of existing models. + +\section{Study Design and Evidence} +The results come from three related but distinct analyses. The original 2009 study reproduces six models on a fixed learner-level split of a transformed mirror. Additional 2009 analyses vary PFA features and GRU training; since their test set had already been inspected, these are exploratory. A separate audit compares the mirror with the corrected official source and refits PFA and GRU to that source. + +The 2015 study proceeded in stages. A first 5,000-learner sample produced a nonconverged PFA fit. A second, nonoverlapping 5,000-learner sample used a corrected solver and provided a new held-out comparison. After those results were available, we fixed the protocol for a third sample of 4,000 learners, excluding everyone from the first two samples. That third analysis, comparing one- and 20-event history windows, is the present paper's central methodological experiment. Its design was recorded before examining the third sample's results, although the choice of research question followed earlier analyses and is not a public preregistration. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and observed differences from the corrected source} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Direct provenance audit.} The ASSISTments maintainers corrected an earlier Skill Builder release after identifying duplicated records \citep{assistmentsdata}. We downloaded the official corrected student--problem CSV directly from the file they link (SHA-256 \texttt{162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da}) and compared it with the exact pinned mirror used for the model run. The corrected file contains 346,860 student--problem records, of which 283,105 contain a usable binary outcome and nonmissing skill identifier, from 4,163 learners. The mirror comprises 274,331 responses from 4,148 learners. All 4,148 mirror learner IDs occur in the corrected source; 15 additional learners occur only in the corrected source. Among the 4,148 shared learners, 3,331 have identical sequences of normalized (skill identifier, binary response) pairs when official records are ordered by numeric \texttt{order\_id}; 817 do not. The same 3,331 satisfy event-count and skill--response-multiset equality. Accordingly, \emph{exact whole-dataset parity is empirically disproved}, not merely unverified. The original mirror-derived predictive results remain correct for the pinned mirror, but cannot be represented as estimates from the corrected official 2009 release. This audit does not identify which upstream transformation rule caused the discrepancies. Although corrected-source learner and event keys can be checked directly, the mirror omits original student--problem event IDs, so a complete one-to-one event crosswalk and independently timestamped chronology are not recoverable. Numeric \texttt{order\_id} was used as a documented ordering convention, not as proof of real-world time order. The full aggregate audit, source hashes and code are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168}{39773920168}; individual student data are not redistributed. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{2009 Benchmark and Source Sensitivity} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Direct corrected-source parity test} +The corrected official release has 4,163 learners, including the 4,148 with sequences in the mirror. Normalizing the skill IDs and sorting the corrected file by numeric \texttt{order\_id} within each learner gives exact sequence agreement for 3,331 shared learner IDs; it fails for 817. Table~\ref{tab:parity} shows the necessary-condition comparison. Thus the frozen 2009 mirror results cannot be relabeled as corrected-release results. Ordered pair equality provides some external corroboration for unaffected learners, but it does not restore original problem-level identifiers to the mirror or prove that the \texttt{order\_id} convention captures the physical event chronology. +\begin{table}[ht]\centering\small +\caption{Official corrected 2009 skill-builder records compared with the pinned Atomi sequence mirror. Official labeled rows exclude records without a usable skill identifier.} +\label{tab:parity} +\begin{tabular}{lrr}\toprule +Measure & Corrected official & Pinned mirror\\\midrule +Unique learners & 4,163 & 4,148\\ +Skill-labeled responses & 283,105 & 274,331\\ +Shared learner IDs & \multicolumn{2}{c}{4,148}\\ +Identical ordered skill--response sequences & \multicolumn{2}{c}{3,331 shared learners}\\ +Different ordered sequences & \multicolumn{2}{c}{817 shared learners}\\\bottomrule +\end{tabular} +\end{table} + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\subsection{Exploratory predictive sensitivity on corrected official 2009 data} +To determine how much the corrected source changes the predictive comparison, we ran a separate, prewritten \emph{exploratory} pipeline on the official corrected release rather than treating the mirror as equivalent. We excluded unlabeled skill records, normalized skill identifiers, ordered each learner's records by numeric \texttt{order\_id}, and used a whole-learner 70/15/15 split with seed 20261024. The enriched PFA retained its same-skill and across-skill history summaries, and the GRU retained prior ordered interactions, seed 42, 25-epoch maximum and validation-based stopping. Both models were refitted exclusively on corrected-source training learners. This is not a fully untouched confirmatory comparison, because the same 2009 academic setting and modeling direction were previously analyzed. + +On 625 corrected-source test learners and 48,842 held-out interactions, the enriched PFA achieved ROC-AUC 0.6951, Brier 0.19675, and log loss 0.5809; the GRU achieved ROC-AUC 0.7324, Brier 0.18507, and log loss 0.5512. The paired interaction-weighted Brier difference (GRU minus PFA) was $-0.01168$ (95\% learner-cluster bootstrap percentile interval $[-0.01372,-0.00986]$, 2,000 resamples). The PFA optimizer converged in 930 iterations, and the GRU's best validation epoch was 13. These source-corrected figures are not directly numerically comparable with the 2009 mirror test split because its learner membership and preprocessing differ. The new analysis audits predictive sensitivity; it does not restore mirror/event parity or identify architectural causes. The aggregate code and outputs are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969}{39774414969}. + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + + +\section{2026 Matched-History-Budget Study on a New ASSISTments 2015 Cohort} +\subsection{Frozen stage-3 design and disjoint history budgets} +After completing the previous 2015 pilot and disjoint second-stage replication, we froze a separate methodological protocol \emph{before} evaluating a third learner sample (see \href{https://github.com/devissaputra/knowledge_tracing_benchmark/blob/research/kt-methodological-2026-history-budget/METHODOLOGICAL_2026_PROTOCOL.md}{frozen protocol}). The official 2015 source hash remained \texttt{75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0}. Using the original 19,840 eligible learner IDs and the published stage-1 and stage-2 random-seed sampling algorithms, we excluded \emph{all 10,000 learners} selected earlier. From the 9,840 remaining learners, we sampled a third 4,000 with seed 20261026, yielding 138,129 interactions. The learner-disjoint train/validation/test split contained 2,800/600/600 learners (the same seed). + +Only responses after a learner's first recorded event are eligible targets, ensuring every evaluated response has at least one observed prior event. Consequently 18 of the 600 test learners contributed no history-eligible target, leaving \textbf{582 evaluable learners and 20,592 common next-response targets}. This distinction required correction of a downstream workflow assertion after the first computational run, with no changes to the model, split, data, features, training seeds or bootstrap. The initial failed release gate and corrected gate are preserved in the public workflow record. + +For an event at position $t\geq1$, the permissible input comprises its current skill identifier and only the immediately preceding $\min(K,t)$ observed skill--correctness pairs, for a history budget $K\in\{1,20\}$. The current response is never an input. Count-based and neural models use the \emph{same observable information stream and identical response targets}. Training, validation and test learners remain disjoint; supervised fitting and standardization use training data only. Predictions on held-out learners are sequential online predictions, permitted to condition on \emph{earlier, already observed} responses within that learner. Such history access is not test-label leakage for the current event. + +\subsection{Three fixed representations and descriptive estimands} +\textbf{Window-count PFA+:} Regularized logistic regression ($C=1$) with current-skill indicators and previous correct/incorrect counts for the queried skill and across all skills, computed strictly inside the last-$K$ window. \textbf{Order-aware logistic:} The identical count features plus the most recent prior correctness value, the latest result for the current skill (or absence flag) and lag to that skill within the window, all derivable from the permitted events. Both use LBFGS, standardized numeric features, 1,600-iteration ceiling and recorded convergence warnings. \textbf{Window-GRU:} Last-$K$ skill/outcome tokens enter a 48-unit hidden-state GRU with a separate 12-dimensional current-skill query embedding; token embedding 24, Adam learning rate 0.002, batch 256, up to 12 epochs, patience 3 on validation, seed 42. It sees no additional response history beyond the allowed K. Parameter counts and inductive biases remain unequal, so any gap cannot identify the isolated architecture effect. + +Primary descriptive metrics are test Brier score, AUC, log loss and ECE-10. The principal model contrasts are paired \emph{interaction-weighted} GRU-minus-logistic Brier differences, with 2,000 learner-cluster percentile bootstrap resamples. These intervals describe sampling uncertainty conditional on one model fit and test split, without corrections for multiple models or budgets. + +\subsection{Real-data results: order-aware logistic reverses the previous model ranking} +\begin{table}[t] +\centering\small +\caption{Same-target Stage-3 test performance at fixed prior-interaction budgets; $n=20{,}592$ responses from 582 evaluable learners at \emph{every} row. Lower Brier/log loss/ECE and higher AUC are better.} +\label{tab:budgetstudy} +\begin{tabular}{r l r r r r} +\toprule +$K$ & Model & Brier & AUC & Log loss & ECE-10\\ +\midrule +1 & Window-count PFA+ & 0.17788 & 0.68444 & 0.53459 & 0.00932 \\ +1 & Order-aware logistic & 0.17788 & 0.68444 & 0.53459 & 0.00932 \\ +1 & Window-GRU & 0.18000 & 0.67091 & 0.54066 & 0.00578 \\ +20 & Window-count PFA+ & 0.17667 & 0.69106 & 0.53159 & 0.00613 \\ +20 & Order-aware logistic & 0.17389 & 0.70504 & 0.52409 & 0.00725 \\ +20 & Window-GRU & 0.17514 & 0.69936 & 0.52715 & 0.01166 \\ +\bottomrule +\end{tabular} +\end{table} +\begin{figure}[t] +\centering\includegraphics[width=.88\linewidth]{history_budget_brier.png} +\caption{Frozen Stage-3 budget sweep with shared train/validation/test learner partitions and exactly matching held-out prediction events. Error intervals for paired model differences are reported in text, not on the points.} +\label{fig:historybudget} +\end{figure} + +The order-aware logistic model had the lowest Brier score at both history budgets. With only one previous response, count and order-aware models are numerically nearly identical, as expected: the single response already determines the recent-history summary. At 20 prior events, explicit order features improve the logistic Brier score from 0.17667 to 0.17389; the corresponding GRU is 0.17514. Moving from $K=1$ to $K=20$, Brier improves by \textbf{0.00121} for window-count, \textbf{0.00400} for order-aware logistic and \textbf{0.00486} for GRU. The original GRU advantage over standard PFA thus does not universally carry over when a transparent ordered-history comparator is given the same observable response window. + +At $K=1$, GRU minus order-aware logistic had pooled Brier difference $+0.002121$ with a 95\% paired learner-cluster bootstrap interval $[+0.001198,+0.003013]$. Relative to window-count PFA+, the corresponding GRU difference was $+0.002121$, interval $[+0.001198,+0.003013]$. + +At $K=20$, GRU minus order-aware logistic had pooled Brier difference $+0.001255$ with a 95\% paired learner-cluster bootstrap interval $[+0.000075,+0.002426]$. Relative to window-count PFA+, the corresponding GRU difference was $-0.001533$, interval $[-0.002761,-0.000279]$. + + + +For both history budgets, the logistic solvers reported no convergence warnings. The GRU selected epoch 6 using validation loss. The preprocessing hash, nonoverlapping cohort, same-label target identity and numerical sanity gates passed. No family-wise significance claim or empirical generalization beyond the study's single held-out source is made. In particular, these confidence intervals are descriptive, not evidence that ordering \emph{causes} improved learning. + +\section{Discussion and Threats to Validity} +\textbf{Model ranking depends on the comparator.} On the third 2015 cohort, adding order-related features to logistic regression reduced Brier loss from 0.17667 to 0.17389 at $K=20$, below the GRU's 0.17514. At $K=1$, the two logistic specifications are nearly identical, and both score below the GRU. This does not show that logistic regression is intrinsically more accurate than recurrent models. Their feature engineering, parameter counts, optimization, and inductive biases differ. It does show that a GRU advantage against a coarse count baseline need not survive comparison with a stronger, transparent representation of the same history. + +\textbf{Dataset identity matters.} The 2009 transformed mirror contains fewer labeled records than the corrected official source, and 817 shared learner sequences differ. Models trained and evaluated on the mirror therefore describe that specific transformation. The separate corrected-source results are not a paired estimate of the effect of preprocessing: their splits and target events differ. The mirror lacks original problem/event IDs needed for an exact row-level crosswalk. + +\textbf{Inference remains conditional.} The 2009 exploratory revisions were made after inspecting the original holdout. In the first 2015 pilot, the PFA optimizer failed to converge; a revised solver was used on a disjoint second sample. The third protocol was fixed before its new learners were evaluated, but it was designed in light of the earlier experiments. Bootstrap intervals resample learners within the evaluated test split; they do not incorporate alternative training splits, tuning choices, multiple-testing adjustments, or shifts to other institutions. + +The 2015 tests also use \texttt{sequence\_id} as the unit being predicted and order records by \texttt{log\_id}. Those coding and ordering decisions may not capture every aspect of the original learning activity. The matched-window experiment uses one split, one GRU seed, and two history lengths. Repeating it with additional seeds, modern attention-based baselines, and a genuinely different source would show whether the ranking is stable. Finally, accurate prediction of historical correctness does not establish that using any of these models improves instruction, fairness, or learning outcomes. + +\section{Summary of Findings} +The source audit rejects exact parity between the 2009 mirror and corrected release. The original six-model results reproduce on that mirror, but exploratory changes to model features or training cannot be treated as new confirmatory tests. The second 2015 sample reproduced the direction of the GRU--enriched-PFA Brier difference, conditional on the revised numerical solver and different model inputs. The third sample gives the clearest answer to the new question: with identical response targets and the same permitted event stream, an order-aware logistic model outperforms the tested GRU at both history lengths in Brier score. The contrast is small, especially at 20 events, and is specific to the implemented models and sample. + +\section{Conclusion} +The transformed ASSISTments 2009 mirror is not equivalent to the corrected official source. The earlier GRU advantage on historical learner cohorts also depends on its comparison model: in a new 2015 evaluation with identical response targets, the order-aware logistic predictor achieved lower Brier loss than a compact GRU when both were given one or 20 preceding interactions. Allowing 20 previous responses improved all three tested models relative to a one-response window. The appropriate conclusion is that knowledge-tracing rankings are sensitive to source construction, history length, and representation. Stability across seeds, other datasets, and stronger neural baselines remains untested. + +\section*{Data availability and research integrity} +The 2009 corrected-source data and 2015 Skill Builder source are available from the ASSISTments data portal \citep{assistmentsdata,assistments2015}. Source checksums, preprocessing and model code, learner-sampling protocols, and aggregate results are available at \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The initial mirror reproduction, corrected-source audit, 2015 pilot and second sample, and third-sample analysis were executed in separate recorded GitHub Actions runs. The successful third-sample execution is \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/40310733288}{run 40310733288}; its first execution required correction of a release assertion that confused all test-split learners with evaluable learners. No model settings, targets, or cohort selections changed as part of that correction. Student-level source records are not redistributed. Code and manuscript revisions remain in \href{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/3}{the research branch (draft pull request)}. + +\section*{Use of computational and writing tools} +Generative AI tools assisted with analysis-code development, research documentation, editing, and technical quality checks. The reported numerical results are linked to executed code and aggregate evidence. The named human author remains responsible for personally checking the methods, sources, computations, and claims before releasing this manuscript. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Badran and Preisach(2026)]{badran2026} Badran, Y. and Preisach, C. (2026). Enhancing knowledge tracing through leakage-free and recency-aware embeddings. \emph{SN Computer Science}, 7, article 774. \url{https://doi.org/10.1007/s42979-026-05344-3}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file diff --git a/results/matched_history_budget_2026.json b/results/matched_history_budget_2026.json new file mode 100644 index 0000000..89401fe --- /dev/null +++ b/results/matched_history_budget_2026.json @@ -0,0 +1,291 @@ +{ + "status": "passed_all_computational_gates", + "source": { + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "eligible_learners": 19840, + "stage1_excluded": 5000, + "stage2_excluded": 5000, + "eligible_unused_pool": 9840, + "stage3_learners": 4000, + "stage3_interactions": 138129, + "learner_set_sha256": "0d9bef7d4d9d9ce4064eb63a04e7bba488ae72cd919bd7730bccd9bdecbb19f1", + "cohort_overlap_previous": 0, + "invalid_source_rows_excluded": 24830 + }, + "design": { + "predeclared_protocol": "METHODOLOGICAL_2026_PROTOCOL.md", + "stage3_sampling_seed": 20261026, + "learner_partition_seed": 20261026, + "train_learners": 2800, + "validation_learners": 600, + "test_learners": 600, + "history_budgets": [ + 1, + 20 + ], + "model_types": [ + "window_count", + "ordered_logistic", + "window_gru" + ], + "GRU_seed": 42, + "bootstrap_replicates": 2000, + "same_targets": "Every model at every K predicts responses t >= 1 for identical 600 test learners.", + "selection": "Fixed before new cohort test; prior research decisions were informed by other 2009/2015 data.", + "inference": "Descriptive exploratory bootstrap contrasts, not multiplicity-corrected confirmatory tests." + }, + "comparisons": { + "1": { + "models": { + "window_count": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17788199203683247, + "log_loss": 0.5345889370393874, + "average_precision": 0.8482492230416429, + "ece_10": 0.009320407240567558, + "roc_auc": 0.6844395899296832 + }, + "ordered_logistic": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.1778819783584268, + "log_loss": 0.5345889271360941, + "average_precision": 0.8482469144579249, + "ece_10": 0.009320949287155366, + "roc_auc": 0.6844363662020748 + }, + "window_gru": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.18000310877303297, + "log_loss": 0.5406576923145966, + "average_precision": 0.838027003779582, + "ece_10": 0.005780779843442258, + "roc_auc": 0.6709091368043554 + } + }, + "paired_GRU_minus": { + "window_count": { + "difference_candidate_minus_reference": 0.0021211167362004853, + "bootstrap_95_interval": [ + 0.0011984039269019259, + 0.003013353764954072 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + }, + "ordered_logistic": { + "difference_candidate_minus_reference": 0.002121130414606117, + "bootstrap_95_interval": [ + 0.0011983198118476887, + 0.003013421140517525 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + } + }, + "diagnostics": { + "window_count": { + "iterations": 295, + "warnings": [] + }, + "ordered_logistic": { + "iterations": 367, + "warnings": [] + }, + "window_gru": { + "best_epoch": 6, + "epochs_completed": 9, + "history": [ + { + "epoch": 1, + "train_bce": 0.5700001594834665, + "validation_bce": 0.5546365849513333 + }, + { + "epoch": 2, + "train_bce": 0.5496423426455948, + "validation_bce": 0.5503481250851094 + }, + { + "epoch": 3, + "train_bce": 0.545433577422607, + "validation_bce": 0.5471623890125249 + }, + { + "epoch": 4, + "train_bce": 0.5431627397913593, + "validation_bce": 0.5474760850748747 + }, + { + "epoch": 5, + "train_bce": 0.5419399417416683, + "validation_bce": 0.5466526810471217 + }, + { + "epoch": 6, + "train_bce": 0.5412652839078146, + "validation_bce": 0.5449585088872168 + }, + { + "epoch": 7, + "train_bce": 0.5408221157525078, + "validation_bce": 0.5450970664384037 + }, + { + "epoch": 8, + "train_bce": 0.5403140336999217, + "validation_bce": 0.545603285447066 + }, + { + "epoch": 9, + "train_bce": 0.5400628717056964, + "validation_bce": 0.5456161411414004 + } + ], + "train_targets": 93426, + "validation_targets": 20111, + "test_targets": 20592 + } + }, + "elapsed_seconds": 14.7 + }, + "20": { + "models": { + "window_count": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17667353711786812, + "log_loss": 0.5315903261834504, + "average_precision": 0.8522443340072219, + "ece_10": 0.006126123934703186, + "roc_auc": 0.6910582372474963 + }, + "ordered_logistic": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17388534919067833, + "log_loss": 0.5240946395813895, + "average_precision": 0.8610186198519205, + "ece_10": 0.0072511782088527195, + "roc_auc": 0.7050441361763368 + }, + "window_gru": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17514020613173276, + "log_loss": 0.5271526751223963, + "average_precision": 0.8589845012762509, + "ece_10": 0.011658007349593165, + "roc_auc": 0.6993609963629661 + } + }, + "paired_GRU_minus": { + "window_count": { + "difference_candidate_minus_reference": -0.0015333309861353841, + "bootstrap_95_interval": [ + -0.0027605673893992813, + -0.0002792867452439877 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + }, + "ordered_logistic": { + "difference_candidate_minus_reference": 0.0012548569410544136, + "bootstrap_95_interval": [ + 7.536414859135726e-05, + 0.002425946875021449 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + } + }, + "diagnostics": { + "window_count": { + "iterations": 299, + "warnings": [] + }, + "ordered_logistic": { + "iterations": 334, + "warnings": [] + }, + "window_gru": { + "best_epoch": 6, + "epochs_completed": 9, + "history": [ + { + "epoch": 1, + "train_bce": 0.562216157968126, + "validation_bce": 0.5460942462327485 + }, + { + "epoch": 2, + "train_bce": 0.5360034013242106, + "validation_bce": 0.53683507956098 + }, + { + "epoch": 3, + "train_bce": 0.5258215163299819, + "validation_bce": 0.5301942893069468 + }, + { + "epoch": 4, + "train_bce": 0.5190114440188723, + "validation_bce": 0.5288743185004735 + }, + { + "epoch": 5, + "train_bce": 0.5149992852793813, + "validation_bce": 0.5286781249708553 + }, + { + "epoch": 6, + "train_bce": 0.5117375197071786, + "validation_bce": 0.5277301228198755 + }, + { + "epoch": 7, + "train_bce": 0.5087343534964336, + "validation_bce": 0.5304507112539802 + }, + { + "epoch": 8, + "train_bce": 0.5056064738932391, + "validation_bce": 0.5304346994831836 + }, + { + "epoch": 9, + "train_bce": 0.5025589439044289, + "validation_bce": 0.5334981878134205 + } + ], + "train_targets": 93426, + "validation_targets": 20111, + "test_targets": 20592 + } + }, + "elapsed_seconds": 36.92 + } + }, + "within_model_brier_k20_minus_k1": { + "window_count": -0.0012084549189643468, + "ordered_logistic": -0.003996629167748483, + "window_gru": -0.00486290264130021 + }, + "gru_minus_ordered_gap_k20_minus_k1": -0.0008662734735517034, + "diagnostic_failures": [], + "interpretation": "Same target set and same last-K learner response information; representations/capacities still unequal. Single stage-3 sample after prior research; no familywise or causal claim." +} diff --git a/scripts/matched_history_budget_2026.py b/scripts/matched_history_budget_2026.py new file mode 100644 index 0000000..6487791 --- /dev/null +++ b/scripts/matched_history_budget_2026.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Frozen 2026 methodology study: matched history budgets on unused ASSISTments 2015 learners. + +Read METHODOLOGICAL_2026_PROTOCOL.md. Source- and algorithm-level gates reject +unverified substitutions. Only aggregate results are written to the repository. +""" +from __future__ import annotations + +import argparse, hashlib, json, random, sys, time, warnings +from pathlib import Path +from dataclasses import dataclass +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler +import torch +from torch import nn +from torch.utils.data import Dataset, DataLoader +from torch.nn.utils.rnn import pad_sequence + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT/"src")) +sys.path.insert(0, str(ROOT/"scripts")) +import run_experiment as kt +from robustness_extension import paired_cluster_bootstrap +from confirmatory_2015 import download_official_csv +from confirmatory_2015_stage2 import SOURCE_SHA256 + +FIRST=20261022 +SECOND=20261023 +THIRD=20261026 +SAMPLE=4000 +HORIZONS=[1,20] +BOOTSTRAPS=2000 +SKILL_COL="skill" +COUNT_COLS=["within_success","within_failure","all_success","all_failure"] +ORDER_COLS=COUNT_COLS+["prev_outcome","prev_same_outcome","lag_to_same"] + +def official_unused_cohort(path:Path): + observed_hash=hashlib.sha256(path.read_bytes()).hexdigest() + if observed_hash!=SOURCE_SHA256: raise RuntimeError("official 2015 SHA-256 differs") + df=pd.read_csv(path,low_memory=False) + df.columns=[str(c).strip().lower() for c in df.columns] + required=["user_id","log_id","sequence_id","correct"] + if any(c not in df for c in required) or len(set(df.columns))!=len(df.columns): + raise ValueError("source columns differ") + n_original=len(df) + df=df[required].copy() + missing=df.isna().any(axis=1) + df["log_id"]=pd.to_numeric(df["log_id"],errors="coerce") + df["correct"]=pd.to_numeric(df["correct"],errors="coerce") + invalid=missing|df.isna().any(axis=1)|~df["correct"].isin([0,1]) + excluded=int(invalid.sum()) + if n_original!=708631 or excluded!=24830: + raise ValueError("source row-validation totals differ from prior audits") + df=df.loc[~invalid].copy() + if df.duplicated(["user_id","log_id"]).any(): + raise ValueError("Duplicate learner-log identifiers") + if not np.isfinite(df["log_id"]).all() or (df["log_id"]!=df["log_id"].astype("int64")).any(): + raise ValueError("invalid chronological event IDs") + df["user_id"]=df["user_id"].astype(str) + df["sequence_id"]=df["sequence_id"].astype(str) + df["correct"]=df["correct"].astype(int) + users=sorted(df["user_id"].unique()) + if len(users)!=19840:raise ValueError("eligible learner population changed") + pilot=set(random.Random(FIRST).sample(users,5000)) + after_pilot=[x for x in users if x not in pilot] + stage2=set(random.Random(SECOND).sample(after_pilot,5000)) + untouched=[x for x in after_pilot if x not in stage2] + chosen=set(random.Random(THIRD).sample(untouched,SAMPLE)) + if chosen&pilot or chosen&stage2 or len(chosen)!=SAMPLE:raise ValueError("Cohort reused!") + df=df[df["user_id"].isin(chosen)].sort_values(["user_id","log_id"],kind="stable") + rows=[{"user_id":str(uid),"skill_ids":g["sequence_id"].tolist(),"grades":g["correct"].tolist()} + for uid,g in df.groupby("user_id",sort=True)] + if len(rows)!=SAMPLE or kt.validate_unique_learner_rows(rows)!=SAMPLE: + raise ValueError("Unmatched learner count") + cohort_fingerprint=hashlib.sha256("\n".join(sorted(chosen)).encode()).hexdigest() + return rows,{"source_sha256":observed_hash, "eligible_learners":len(users), + "stage1_excluded":len(pilot),"stage2_excluded":len(stage2), + "eligible_unused_pool":len(untouched),"stage3_learners":len(chosen), + "stage3_interactions":int(len(df)), "learner_set_sha256":cohort_fingerprint, + "cohort_overlap_previous":0,"invalid_source_rows_excluded":excluded} + +def frame(rows,k): + """Every target uses only the k preceding labeled events, never its own label.""" + feats=[];labels=[];learners=[];skills=[];positions=[] + for row in rows: + uid=str(row["user_id"]) + ss,ys=kt.aligned(row) + for t in range(1,len(ys)): + recent=list(zip(ss[max(0,t-k):t],ys[max(0,t-k):t])) + current=ss[t] + within=[int(y) for skill,y in recent if skill==current] + total=[int(y) for skill,y in recent] + lag=next((j for j,(skill,y) in enumerate(reversed(recent),1) if skill==current),k+1) + last_same=next((2*int(y)-1 for skill,y in reversed(recent) if skill==current),0) + feats.append({"skill":current,"within_success":sum(within), + "within_failure":len(within)-sum(within), + "all_success":sum(total),"all_failure":len(total)-sum(total), + "prev_outcome":2*int(recent[-1][1])-1, + "prev_same_outcome":last_same,"lag_to_same":lag}) + labels.append(ys[t]);learners.append(uid);skills.append(current);positions.append(t) + X=pd.DataFrame(feats) + ids=pd.MultiIndex.from_arrays([learners,positions],names=["learner","target_position"]) + if ids.has_duplicates:raise ValueError("duplicate target identifiers") + return X,np.asarray(labels,dtype=int),np.asarray(learners),np.asarray(skills),np.asarray(positions) + +def fit_logit(train_f,test_f,which): + cols=COUNT_COLS if which=="window_count" else ORDER_COLS + X,y,*_=train_f; xx,yy,learner,skill,positions=test_f + pre=ColumnTransformer([("skill",OneHotEncoder(handle_unknown="ignore"),[SKILL_COL]), + ("hist",StandardScaler(),cols)]) + estimator=LogisticRegression(C=1.,solver="lbfgs",max_iter=1600,tol=1e-7) + model=Pipeline([("pre",pre),("model",estimator)]) + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter("always",ConvergenceWarning) + model.fit(X[[SKILL_COL]+cols],y) + warn=[str(q.message) for q in seen if issubclass(q.category,ConvergenceWarning)] + prob=model.predict_proba(xx[[SKILL_COL]+cols])[:,1] + return predictions(yy,prob,learner,skill,positions),{"iterations":int(model["model"].n_iter_[0]),"warnings":warn} + +def predictions(y,p,learners,skills,positions): + result=kt.prediction_frame(y,p,learners,skills,np.zeros(len(y),dtype=bool)) + result["target_position"]=np.asarray(positions,dtype=int) + return result + +class HistoryTargets(Dataset): + def __init__(self,rows,k,vocab): + self.data=[] + self.index=[] + for ri,row in enumerate(rows): + ss,yy=kt.aligned(row) + idx=np.array([vocab.get(s,0) for s in ss],dtype=np.int32) + lab=np.array(yy,dtype=np.int8) + self.data.append((idx,lab,str(row["user_id"]),ss)) + self.index.extend((ri,t) for t in range(1,len(yy))) + self.k=k + def __len__(self):return len(self.index) + def __getitem__(self,i): + ri,t=self.index[i] + skills,answers,uid,sknames=self.data[ri] + lo=max(0,t-self.k) + # Encoding of previous events only; t's outcome appears only as label. + tokens=(1+2*skills[lo:t]+answers[lo:t]).astype(np.int64) + if not len(tokens):raise AssertionError("no history on valid target") + return torch.from_numpy(tokens),int(skills[t]),int(answers[t]) +def collate(batch): + tokens=pad_sequence([z[0] for z in batch],batch_first=True,padding_value=0) + q=torch.tensor([z[1] for z in batch],dtype=torch.long) + y=torch.tensor([z[2] for z in batch],dtype=torch.float32) + lengths=torch.tensor([len(z[0]) for z in batch],dtype=torch.long) + return tokens,q,y,lengths + +class WindowGRU(nn.Module): + def __init__(self,nskills): + super().__init__() + self.tokens=nn.Embedding(2*(nskills+1)+1,24,padding_idx=0) + self.query=nn.Embedding(nskills+1,12) + self.gru=nn.GRU(24,48,batch_first=True) + self.head=nn.Linear(48+12,1) + def forward(self,x,q,length): + h,_=self.gru(self.tokens(x)) + idx=(length-1).to(h.device) + final=h[torch.arange(len(idx),device=h.device),idx] + return self.head(torch.cat([final,self.query(q)],dim=-1)).squeeze(-1) + +def fit_gru(train,val,test,k,vocab,epochs=12): + torch.manual_seed(42);np.random.seed(42);torch.set_num_threads(2) + td=HistoryTargets(train,k,vocab) + vd=HistoryTargets(val,k,vocab) + ed=HistoryTargets(test,k,vocab) + gen=torch.Generator().manual_seed(42) + tl=DataLoader(td,batch_size=256,shuffle=True,collate_fn=collate,generator=gen,num_workers=0) + vl=DataLoader(vd,batch_size=512,shuffle=False,collate_fn=collate,num_workers=0) + el=DataLoader(ed,batch_size=512,shuffle=False,collate_fn=collate,num_workers=0) + m=WindowGRU(len(vocab));opt=torch.optim.Adam(m.parameters(),lr=0.002) + criterion=nn.BCEWithLogitsLoss() + best=float("inf");best_epoch=0;best_weights=None;stale=0;history=[] + def evaluate(loader,trainmode): + m.train(trainmode);loss_sum=0;num=0 + ps=[] + for x,q,y,lengths in loader: + if trainmode:opt.zero_grad() + with torch.set_grad_enabled(trainmode): + logits=m(x,q,lengths);loss=criterion(logits,y) + if trainmode:loss.backward();opt.step() + loss_sum+=float(loss.detach())*len(y);num+=len(y) + if not trainmode:ps.extend(torch.sigmoid(logits).detach().numpy().tolist()) + return loss_sum/max(num,1),ps + for epoch in range(1,epochs+1): + tr,_=evaluate(tl,True) + with torch.no_grad():v,_=evaluate(vl,False) + history.append({"epoch":epoch,"train_bce":tr,"validation_bce":v}) + if v=3:break + m.load_state_dict(best_weights) + with torch.no_grad():_,prob=evaluate(el,False) + ff=frame(test,k) + if len(ff[1])!=len(prob):raise RuntimeError("GRU targets misaligned with logistic") + out=predictions(ff[1],np.asarray(prob),ff[2],ff[3],ff[4]) + return out,{"best_epoch":best_epoch,"epochs_completed":len(history), + "history":history, "train_targets":len(td),"validation_targets":len(vd), + "test_targets":len(ed)} + +def align(*preds): + first=preds[0] + for p in preds[1:]: + for col in ["learner","target_position","skill","y"]: + if not np.array_equal(first[col].to_numpy(),p[col].to_numpy()): + raise ValueError(f"Models do not evaluate same targets: {col}") +def main(): + parser=argparse.ArgumentParser() + parser.add_argument("--csv",default="data/cache/assistments_2015_official.csv") + parser.add_argument("--output",default="results/matched_history_budget_2026.json") + parser.add_argument("--smoke",action="store_true") + a=parser.parse_args() + if a.smoke: + synthetic=[{"user_id":"A","skill_ids":["x","y","x","x"],"grades":[0,1,1,0]}, + {"user_id":"B","skill_ids":["y","x","y"],"grades":[1,0,1]}] + one=frame(synthetic,1) + twenty=frame(synthetic,20) + # Target at t=2: previous last event is y=1, current x=1 must not leak. + assert one[0].iloc[1]["within_success"]==0 + assert one[0].iloc[1]["prev_outcome"]==1 + assert twenty[0].iloc[1]["within_failure"]==1 + assert one[1].tolist()==twenty[1].tolist() + altered=[dict(r) for r in synthetic] + altered[0]=dict(synthetic[0],grades=[0,1,0,0]) + assert frame(altered,1)[0].iloc[1].equals(one[0].iloc[1]) + vocab={"x":1,"y":2} + ds=HistoryTargets(synthetic,1,vocab) + assert ds[1][0].tolist()==[1+2*2+1] + print("SMOKE PASS: k restrictions, label exclusion, same targets") + return + source=Path(a.csv) + download_official_csv(source) + rows,manifest=official_unused_cohort(source) + train,val,test=kt.split_rows(rows,seed=THIRD) + train_ids={r["user_id"] for r in train};val_ids={r["user_id"] for r in val};test_ids={r["user_id"] for r in test} + if train_ids&val_ids or train_ids&test_ids or val_ids&test_ids:raise ValueError("learner leakage") + if (len(train),len(val),len(test))!=(2800,600,600):raise ValueError("split size wrong") + vocab=kt.skill_vocabulary(train) + output={"status":"started","source":manifest,"design":{ + "predeclared_protocol":"METHODOLOGICAL_2026_PROTOCOL.md", + "stage3_sampling_seed":THIRD,"learner_partition_seed":THIRD, + "train_learners":len(train),"validation_learners":len(val),"test_learners":len(test), + "history_budgets":HORIZONS,"model_types":["window_count","ordered_logistic","window_gru"], + "GRU_seed":42,"bootstrap_replicates":BOOTSTRAPS, + "same_targets":"Every model at every K predicts responses t >= 1 for identical 600 test learners.", + "selection":"Fixed before new cohort test; prior research decisions were informed by other 2009/2015 data.", + "inference":"Descriptive exploratory bootstrap contrasts, not multiplicity-corrected confirmatory tests." + },"comparisons":{}} + preds_by_k={} + guards=[] + for k in HORIZONS: + ts=time.monotonic() + train_f=frame(train,k) + test_f=frame(test,k) + all_preds={};diagnostics={} + for which in ["window_count","ordered_logistic"]: + all_preds[which],diagnostics[which]=fit_logit(train_f,test_f,which) + if diagnostics[which]["warnings"]:guards.append(which+"_"+str(k)+"_nonconverged") + all_preds["window_gru"],diagnostics["window_gru"]=fit_gru(train,val,test,k,vocab) + align(*all_preds.values()) + if len(all_preds["window_gru"])==0:raise ValueError("zero test events") + vals={name:kt.metric_bundle(z["y"],z["p"]) for name,z in all_preds.items()} + contrasts={ref:paired_cluster_bootstrap(all_preds[ref],all_preds["window_gru"], + repetitions=BOOTSTRAPS,seed=THIRD+int(k)) + for ref in ["window_count","ordered_logistic"]} + output["comparisons"][str(k)]={"models":vals,"paired_GRU_minus":contrasts, + "diagnostics":diagnostics, + "elapsed_seconds":round(time.monotonic()-ts,2)} + preds_by_k[k]=all_preds + print("FINISHED_HORIZON",k,json.dumps({"brier":{key:value["brier"] for key,value in vals.items()}, + "gru_minus_ordered_ci":contrasts["ordered_logistic"]["bootstrap_95_interval"]}),flush=True) + # Same exact learners and target ordinals between history budgets, with identical label stream. + align(*[p for k in HORIZONS for p in preds_by_k[k].values()]) + output["within_model_brier_k20_minus_k1"]={ + name:float(output["comparisons"]["20"]["models"][name]["brier"]- + output["comparisons"]["1"]["models"][name]["brier"]) + for name in ["window_count","ordered_logistic","window_gru"]} + first=output["comparisons"]["1"]["paired_GRU_minus"]["ordered_logistic"]["difference_candidate_minus_reference"] + last=output["comparisons"]["20"]["paired_GRU_minus"]["ordered_logistic"]["difference_candidate_minus_reference"] + output["gru_minus_ordered_gap_k20_minus_k1"]=float(last-first) + output["status"]="passed_all_computational_gates" if not guards else "inconclusive_diagnostic_fail" + output["diagnostic_failures"]=guards + output["interpretation"]="Same target set and same last-K learner response information; representations/capacities still unequal. Single stage-3 sample after prior research; no familywise or causal claim." + out=Path(a.output);out.parent.mkdir(exist_ok=True,parents=True) + out.write_text(json.dumps(output,indent=2,allow_nan=False)+"\n") + print("FINAL_RESULTS",json.dumps({"status":output["status"], + "delta_k20_minus_k1":output["within_model_brier_k20_minus_k1"], + "gap_change":output["gru_minus_ordered_gap_k20_minus_k1"], + "path":str(out)},indent=2),flush=True) + if guards:sys.exit(3) +if __name__=="__main__":main()