diff --git a/.github/workflows/arxiv-verification.yml b/.github/workflows/arxiv-verification.yml index 978ad43..dab9831 100644 --- a/.github/workflows/arxiv-verification.yml +++ b/.github/workflows/arxiv-verification.yml @@ -1,4 +1,4 @@ -name: arXiv verification: real-data robustness +name: "arXiv verification: real-data robustness" on: push: branches: diff --git a/.github/workflows/confirmatory-2015-stage2.yml b/.github/workflows/confirmatory-2015-stage2.yml new file mode 100644 index 0000000..78ec559 --- /dev/null +++ b/.github/workflows/confirmatory-2015-stage2.yml @@ -0,0 +1,60 @@ +name: External cohort confirmation — second untouched ASSISTments 2015 cohort +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - CONFIRMATORY_STAGE2_PROTOCOL_2015.md + - scripts/confirmatory_2015_stage2.py + - .github/workflows/confirmatory-2015-stage2.yml + workflow_dispatch: +permissions: + contents: write +jobs: + second-cohort: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install frozen computational packages + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Compile and inspect frozen code + run: python -m py_compile scripts/confirmatory_2015.py scripts/confirmatory_2015_stage2.py + - name: Run untouched population study + run: PYTHONPATH=src python scripts/confirmatory_2015_stage2.py + - name: Check full independent source and test outputs + run: | + python - <<'PY' + import json + from pathlib import Path + a=json.loads(Path('results/confirmatory_assist2015_stage2.json').read_text()) + assert a['data']['sha256']=='75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0' + assert a['data']['stage2_intersection_stage1']==0 + assert a['data']['stage1_excluded_learners']==5000 + assert a['data']['stage2_sampled_learners']==5000 + assert sum(a['protocol'][x] for x in ('train_learners','validation_learners','test_learners'))==5000 + assert a['paired_brier']['n_learners']==750 + assert a['paired_brier']['n_bootstrap']==2000 + print('Recorded status:',a['status'],'directional replication:',a['directional_replication_confirmed']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: assistments2015-stage2-validation + path: results/confirmatory_assist2015_stage2.json + if-no-files-found: warn + - name: Commit frozen results (not raw student records) + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/confirmatory_assist2015_stage2.json + if ! git diff --cached --quiet; then + git commit -m 'research: record untouched cohort confirmatory findings' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file diff --git a/.github/workflows/confirmatory-2015.yml b/.github/workflows/confirmatory-2015.yml new file mode 100644 index 0000000..51d9718 --- /dev/null +++ b/.github/workflows/confirmatory-2015.yml @@ -0,0 +1,62 @@ +name: External cohort confirmation — ASSISTments 2015 +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/confirmatory_2015.py + - .github/workflows/confirmatory-2015.yml + - CONFIRMATORY_PROTOCOL_2015.md + - scripts/robustness_extension.py + - src/run_experiment.py + workflow_dispatch: +permissions: + contents: write +jobs: + external-cohort: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install verified environment and Google Drive downloader + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' pytest + - name: Compile frozen confirmatory analysis + run: | + python -m py_compile scripts/confirmatory_2015.py + - name: Run independent-cohort analysis + run: PYTHONPATH=src python scripts/confirmatory_2015.py + - name: Validate output schema and data isolation + run: | + python - <<'PY' + import json + from pathlib import Path + a = json.loads(Path('results/confirmatory_assist2015.json').read_text()) + assert a['protocol']['sample_learners'] == 5000 + assert sum(a['protocol'][k] for k in ('train_learners','validation_learners','test_learners')) == 5000 + assert a['brier_comparison']['n_learners'] == a['protocol']['test_learners'] + assert a['brier_comparison']['n_bootstrap'] == 2000 + assert len(a['data']['source_sha256']) == 64 + print('External-cohort result recorded. Confirmation:', a['directional_replication_confirmed']) + PY + - name: Upload results and logs even if run fails + if: always() + uses: actions/upload-artifact@v4 + with: + name: knowledge-tracing-2015-confirmation + path: results/confirmatory_assist2015.json + if-no-files-found: warn + - name: Commit metrics to branch without changing previous paper results + if: success() + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/confirmatory_assist2015.json + if ! git diff --cached --quiet; then + git commit -m "research: freeze ASSISTments 2015 external-cohort test outcome" + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file diff --git a/.github/workflows/corrected-2009-reanalysis.yml b/.github/workflows/corrected-2009-reanalysis.yml new file mode 100644 index 0000000..1fe98f3 --- /dev/null +++ b/.github/workflows/corrected-2009-reanalysis.yml @@ -0,0 +1,55 @@ +name: "Exploratory corrected-source 2009 GRU versus enriched PFA" +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/corrected_2009_reanalysis.py + - .github/workflows/corrected-2009-reanalysis.yml + - CORRECTED_2009_REANALYSIS_PROTOCOL.md + workflow_dispatch: +permissions: + contents: write +jobs: + reanalysis: + runs-on: ubuntu-latest + timeout-minutes: 100 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install frozen numeric and raw-source packages + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Inspect Python code + run: python -m py_compile scripts/audit_corrected_2009.py scripts/corrected_2009_reanalysis.py + - name: Analyze corrected official-source data without redistributing student records + run: PYTHONPATH=src python scripts/corrected_2009_reanalysis.py + - name: Check required source-audit provenance and output schema + run: | + python - <<'PY' + import json + r=json.load(open('results/corrected_2009_sensitivity.json')) + assert r['status'] in ('completed_exploratory_corrected_source','inconclusive_diagnostics','blocked_or_failed') + if r['status']=='completed_exploratory_corrected_source': + assert r['source']['corrected_sha256']=='162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da' + assert r['paired_brier']['n_bootstrap']==2000 + print('Corrected-source research status:',r['status']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: corrected-2009-aggregate-sensitivity + path: results/corrected_2009_sensitivity.json + - name: Archive aggregate outcome in research branch + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/corrected_2009_sensitivity.json + if ! git diff --cached --quiet; then + git commit -m 'research: record exploratory corrected-2009 source sensitivity' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file diff --git a/.github/workflows/official-2009-parity.yml b/.github/workflows/official-2009-parity.yml new file mode 100644 index 0000000..b41b840 --- /dev/null +++ b/.github/workflows/official-2009-parity.yml @@ -0,0 +1,52 @@ +name: "Audit corrected official ASSISTments 2009 against pinned third-party mirror" +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/audit_corrected_2009.py + - .github/workflows/official-2009-parity.yml + workflow_dispatch: +permissions: + contents: write +jobs: + source-parity: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install source-audit dependencies + run: | + python -m pip install 'pandas==2.3.3' 'pyarrow==23.0.1' 'huggingface-hub==1.33.0' 'gdown>=5.2,<6' + - name: Verify audit script syntax + run: python -m py_compile scripts/audit_corrected_2009.py + - name: Compare official corrected raw records against pinned mirror + run: python scripts/audit_corrected_2009.py + - name: Verify audit records uncertainty explicitly + run: | + python - <<'PY' + import json + from pathlib import Path + r=json.loads(Path('results/official_2009_mirror_parity_audit.json').read_text()) + assert r['parity_status'] in ('blocked_not_established','exact_full_dataset_parity_disproved_by_necessary_conditions','aggregate_necessary_conditions_pass_but_event_parity_not_proven') + assert 'student_records' not in r + print('SOURCE AUDIT STATUS:',r['parity_status']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: official-2009-mirror-parity-metadata-only + path: results/official_2009_mirror_parity_audit.json + - name: Preserve report in research branch + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/official_2009_mirror_parity_audit.json + if ! git diff --cached --quiet; then + git commit -m 'research: archive corrected 2009 versus mirror provenance audit' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file diff --git a/CONFIRMATORY_PROTOCOL_2015.md b/CONFIRMATORY_PROTOCOL_2015.md new file mode 100644 index 0000000..087a076 --- /dev/null +++ b/CONFIRMATORY_PROTOCOL_2015.md @@ -0,0 +1,13 @@ +# Frozen external-cohort replication protocol (2026-10-22) + +**Research question.** On a previously unanalysed, different-year learner sample from ASSISTments 2015, does the pre-existing extended GRU pipeline achieve lower held-out pooled Brier loss than the pre-existing enriched PFA pipeline? The models are retrained on 2015 training learners. This tests cross-cohort *replication of the comparison*, not direct transport of the 2009 fitted model or causal effects of architecture. + +- Source: official ASSISTments 2015 Skill Builder Data, `2015_100_skill_builders_main_problems.csv`, Google Drive file ID `0B_hO8cnpcIMgUGZzRnh3bHJrSjQ` linked from https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data. +- Require `user_id`, `log_id` (event order), `sequence_id` (100 skill-builder concepts), and binary `correct`. Require non-null, unambiguous records and unique learner-event pairs. Sort each learner chronologically using `log_id`. Never use later responses as model inputs for earlier outcomes. +- Learner sampling: uniformly sample without replacement **5,000 distinct learners**, deterministically with seed `20261022` after sorting IDs lexicographically. No filtering based on outcomes or model performance, except invalid records verified before learner sampling; log exclusions. +- Split: shuffle selected learners with seed `20261022`, then 70/15/15, disjoint between training/validation/test. A new dataset, independent of the 2009 train/validation/test learners. All model selection on training/validation; test inspected once. +- Fixed models: pre-existing stronger PFA `stronger_pfa()` (skill success/failure and cross-skill prior counts, logistic regression C=1, SAGA max_iter=1000) and `GRUKnowledgeTracer` model (32-token,16-query,64-hidden, Adam lr=0.002, validation early stopping, max 25 epochs, patience 4, random seed 42). No test-driven tuning. +- Primary estimand: interaction-weighted paired Brier-score difference `GRU - enriched PFA`. Resample **test learners** with replacement (2,000 replicates, seed 20261022), retaining all responses per selected learner. 95% percentile confidence interval. Claim directional external-cohort confirmation **only if** upper CI bound < 0 and computation/data gates pass; if the upper bound >= 0, record as not confirmed. Sign criterion is not proof of architecture causation. +- Secondary descriptive quantities: ROC-AUC, log loss, calibration/ECE, learner and interaction counts, outcome prevalence, checksums and provenance. No choosing between datasets/seeds after seeing test outcomes. +- Scope caveats: This is replication within ASSISTments in another year, on a random 5,000-learner subsample, not independent educational-system validity; 2015 skill-builder `sequence_id` concepts may not be the same units as 2009 mirror skill IDs, and enriched PFA and GRU have unequal information/capacity. +- No outcome claims before executed logs and machine-readable artifacts have been verified. Inability to retrieve/verify the official source means **blocked**, not passed. \ No newline at end of file diff --git a/CONFIRMATORY_STAGE2_PROTOCOL_2015.md b/CONFIRMATORY_STAGE2_PROTOCOL_2015.md new file mode 100644 index 0000000..1361365 --- /dev/null +++ b/CONFIRMATORY_STAGE2_PROTOCOL_2015.md @@ -0,0 +1,18 @@ +# Frozen stage-2, disjoint-cohort confirmatory replication + +**Reason for independent rerun.** The predeclared first external-cohort execution (GitHub run 39754046492; source SHA 75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0) generated estimates, but PFA `saga` emitted a nonconvergence warning and the planned diagnostic gate marked the test `completed_with_unresolved_diagnostics`. That result is retained without a confirmatory label. The correction addresses only numerical solver convergence; it must not select hyperparameters based on the observed direction or magnitude of the first test result. + +## New learner cohort and data provenance + +Use the **same official ASSISTments 2015 raw CSV bytes**, SHA-256 above, linked from the official ASSISTments portal. Apply exactly the same predeclared validity filters and chronological order (`log_id`) as in stage 1. Lexicographically sort eligible learner IDs, reconstruct the 5,000 participants originally sampled with `random.Random(20261022).sample(ids, 5000)`, exclude ALL 5,000 of those previously analysed learners, and sample 5,000 learners from the remaining eligible IDs with `random.Random(20261023).sample(remaining, 5000)`. Use a learner-disjoint 70/15/15 train/validation/test split with seed `20261023`. Assert zero overlap with stage 1 participants. No stage-2 outcomes have been inspected when this protocol was specified. + +## Fixed models and criterion + +- **PFA**: the same enriched information features (`skill`, within-skill previous correct/incorrect counts, across-skill previous correct/incorrect counts), StandardScaler for counts, OneHotEncoder for skill, logistic regression regularization `C=1.0`. Replace the nonconverged `saga` numerical solver by fixed `lbfgs` with `max_iter=2000`, `tol=1e-8`, to estimate the *same regularized logistic objective*. Record and reject nonconvergence warnings; **no alternative solvers, seeds, or hyperparameters after this check**. +- **GRU**: same implementation as first execution; train only on stage-2 training learners, select epoch only on stage-2 validation learners. Seed 42, up to 25 epochs, patience 4, Adam LR .002, token dim 32, query dim 16, hidden 64. +- Primary estimand: paired pooled interaction-weighted Brier loss GRU minus PFA across stage-2 test learners. Learner-cluster percentile bootstrap 2,000 replicates, seed 20261023. Directional result counts as *confirmed on this external 2015 subset* only if upper 95% percentile endpoint < 0 and all data/provenance/convergence guards pass. Otherwise it is a failed or indeterminate test; publish unchanged. Do not compare different PFA solvers on these stage-2 test outcomes to pick the favorable one. +- Secondary descriptive measures: ROC-AUC, log loss, ECE, sample sizes and observed correctness prevalence; not selection criteria. + +## Scope + +A successful stage-2 test would independently replicate the direction of a predictive difference on a different-year ASSISTments cohort, with a numerically converged baseline. It does not prove that GRU architecture alone causes the difference, demonstrate source parity between the original 2009 mirror and corrected official data, or establish generalizability outside the analyzed contexts. This is not a registered external trial or blind third-party peer review. \ No newline at end of file diff --git a/CORRECTED_2009_REANALYSIS_PROTOCOL.md b/CORRECTED_2009_REANALYSIS_PROTOCOL.md new file mode 100644 index 0000000..3ab106d --- /dev/null +++ b/CORRECTED_2009_REANALYSIS_PROTOCOL.md @@ -0,0 +1,9 @@ +# Exploratory official-corrected-2009 source sensitivity protocol (22 Oct 2026) + +**Rationale:** Direct independent source audit proves that the frozen 2009 Atomi learner-sequence mirror and the official corrected 2009--2010 skill-builder release are not event-equivalent. The previously published model differences are conditional on the mirror. Execute one source-corrected sensitivity comparison on the source bytes whose provenance was just verified. + +**Source:** The exact corrected file linked from the ASSISTments maintainers' 2009--2010 Skill Builder page: Google Drive ID `1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE`, SHA-256 `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`. Preserve original file; never redistribute its raw student records. Exclude only rows with absent/invalid binary outcome or unlabeled skill; retain one record per corrected student--problem event; source has 346,860 rows and 283,105 labeled rows spanning 4,163 learners. Normalize simple numeric skill IDs, preserve `_` composite skill keys. Order events within learner by increasing numeric `order_id`. Assert unique (student, order_id) and deterministic tie treatment. Event chronology depends on the source documentation's `order_id` semantics; treat this as an assumption, not independently reconstructed timestamps. + +**Preselected experiment:** Train/test split by whole learner with deterministic fixed seed 20261024 and `kt.split_rows` 70/15/15. Fit *enriched PFA*, using skill identity, prior same-skill successes/failures and prior across-skill successes/failures, `StandardScaler` for counts, `OneHotEncoder(handle_unknown='ignore')`, regularized logistic regression with `solver=lbfgs`, `C=1.0`, `max_iter=2000`, `tol=1e-8`. Reject nonconvergent fits. Fit the existing compact GRU, seed 42, token_dim=32, query_dim=16, hidden=64, max 25 epochs, patience 4, Adam 0.002; select stop epoch with validation learners only. First test event uses training skill/global prior. Compare test Brier scores (primary), AUC, log loss and calibration (secondary). Use 2,000 paired learner-cluster bootstrap draws seed 20261024 on pooled interaction-weighted GRU-minus-PFA Brier. Do not choose a different seed/hyperparameter to rescue a failed result. Record null/failure status honestly if invalid or interrupted. + +**Interpretation:** This analysis tests a corrected-source *sensitivity*, not an independent untouched 2009 hypothesis confirmation, because the original 2009 population and comparison direction were already analyzed. It does **not** establish a recurrence-only architecture effect, direct parity of the mirror with official data, or generalizability beyond the cohort. Do not edit the original mirror outcome or supplant the separate fresh 2015 replication. \ No newline at end of file diff --git a/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md b/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md new file mode 100644 index 0000000..af4d2f6 --- /dev/null +++ b/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md @@ -0,0 +1,34 @@ +# Paper 1: proposed arXiv metadata and final author checklist + +**Title:** Knowledge-Tracing Reproducibility and Information-Budget Auditing: A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication + +**Author:** Devis Wawan Saputra (independent researcher; author must check name and ORCID: 0000-0002-7133-9410) + +**Suggested primary subject:** cs.CY (Computers and Society), for the emphasis on research evaluation and educational impacts. **Alternative:** cs.LG (Machine Learning) if the arXiv classifier/moderator judges the predictive-model evaluation central. This is a suggestion, not a guarantee of category eligibility. Category selection and any endorsement remain the submitting author's responsibility. + +**Proposed arXiv abstract (copy-paste, under the 1,920-character limit):** + +Knowledge-tracing model comparisons are sensitive to preprocessing, learner partitioning, and differences in the information available to predictors. We reproduce a six-model benchmark on a pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 interactions). Direct comparison with the official corrected 2009 source shows non-equivalence: among 4,148 shared learners, 817 have different ordered skill-response sequences, and the mirror has 8,774 fewer labeled events. On 623 mirror-based held-out learners, a GRU achieves AUC 0.7470 and Brier 0.1800, versus standard PFA AUC 0.6982 and Brier 0.1976. Exploratory analyses with richer PFA histories and longer GRU training remain conditional on the previously inspected split. In an independent-year ASSISTments 2015 evaluation, after discarding a nonconverged first pilot and excluding all its learners, the GRU has Brier 0.18227 versus 0.18695 for enriched PFA on 750 new held-out learners. The interaction-weighted Brier difference is -0.00468 (95% learner-cluster bootstrap interval [-0.00607, -0.00335]). This supports directional predictive replication under the tested protocol, not an isolated effect of recurrent architecture, universal generalization, or improved educational outcomes. + +**Comments:** Empirical research working preprint; 11 pages, three figures; source/data audit and replication code linked via https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2. Update exact page count from the FINAL PDF before submission. Not peer reviewed. + +**Report identifier/DOI:** Leave blank; do not invent an arXiv ID, DOI, journal identifier, peer-review status or affiliation. The arXiv service assigns its ID only after announcement. + +## Author-only release checklist + +- [ ] Personally verify name/ORCID, provenance, models, exclusion rules, raw-data comparison, exact numerical results, and the corrected-source caveat. +- [ ] Confirm authorship and originality, permissions, acknowledgments and accurate contribution/AI-assistance disclosures under current archive/institutional policy. +- [ ] Decide whether to seek **optional** external human methodological review. This is recommended for stronger scientific credibility but is not arXiv's peer-review requirement; arXiv moderation is not independent peer review. +- [ ] Log into own account at https://arxiv.org/submit ; resolve registration, endorsement or rate-limit eligibility if asked. +- [ ] Upload **only the minimal TeX/source ZIP** (not the larger research archive); inspect arXiv's generated PDF after processing. +- [ ] Verify arXiv title, abstract, authors, subject category and license; approve and click final Submit Article. + +## Technical references + +- Official source-file guidance: https://info.arxiv.org/help/submit_tex.html . Includes the required figure formats and guidance to exclude PDFs, .aux files, old logs and unrelated files. +- Official submission verification: https://info.arxiv.org/help/submit/ . Preview the generated PDF and metadata before the final submission action. +- Official abstract limit and metadata guidance: https://info.arxiv.org/help/prep.html (1,920 characters). +- arXiv moderation is **not** academic peer review: https://info.arxiv.org/help/moderation/ . +- Updated 2026 arXiv new-submission limits: https://blog.arxiv.org/2026/10/01/updated-rate-limit-policy/ . + +**Status:** All technical content prepared by assistant remains a draft until the named author performs the release checks. No author endorsement, account status, consent or publication acceptance is asserted by this document. \ No newline at end of file diff --git a/paper/BUILD_AND_AUTHOR_REVIEW.md b/paper/BUILD_AND_AUTHOR_REVIEW.md new file mode 100644 index 0000000..a1ba9e9 --- /dev/null +++ b/paper/BUILD_AND_AUTHOR_REVIEW.md @@ -0,0 +1,32 @@ +# Canonical Paper 1 — build and author review + +The **actual** updated manuscript is `paper/main.tex` on the research branch `research/kt-fresh-cohort-20261022`. The previous `paper/main_integrated_20261022.tex` is synchronized for revision history. The distribution in ChatGPT Library `/Paper1_Final_20261022/Paper1_Actual_Manuscript_Source_20261022.zip` includes four required PNG figures and is directly compilable. + +## Build from repository root + +```bash +python paper/generate_figures.py +cd paper +latexmk -pdf -interaction=nonstopmode -halt-on-error main.tex +``` + +The figure generator uses the actual archived GitHub Actions JSON outputs; it does not fabricate any research data. It needs `matplotlib` and `numpy`. When working from the downloadable source ZIP, figures are already provided, so `latexmk -pdf main.tex` suffices. + +## What changed in the manuscript + +1. Preserves the frozen 2009 transformed-mirror benchmark and the exploratory enhanced-PFA/GRU analyses. +2. Documents the initial ASSISTments 2015 test (25,089 held-out responses) with actual model scores and its failed numerical convergence gate. The pilot is **not** counted as confirmatory evidence. +3. Integrates the independent, learner-disjoint 2015 corrective-stage test (750 held-out learners; 26,697 responses), numerical point estimates, bootstrap interval and figure. +4. States explicitly that the bootstrap resampled learners **without refitting the training models**. This captures held-out sample uncertainty conditional on fitted models, not optimization instability or between-cohort variability. +5. Preserves caveats: 2009 mirror not proven equivalent to corrected official data; model information/capacity unmatched; no causal or instructional-effect claims. + +## Record of local QA + +- 10-page A4, selectable text, all pages checked by rendering. +- Four figures included; no missing references or LaTeX errors; no overfull LaTeX boxes. +- PDF successfully rebuilt from source in a clean temporary directory. +- 19/19 consistency/build checks passed in the accompanying `MANUSCRIPT_QA.json`. + +## Final gates not delegated + +The author must review the science, verify the provenance and calculations personally, approve contribution/originality and disclosure statements, and choose the arXiv category and submit. The branch must not be silently treated as peer-reviewed or as evidence of data parity with corrected ASSISTments 2009-2010. \ No newline at end of file diff --git a/paper/FINAL_QA_RESULTS_20261022.json b/paper/FINAL_QA_RESULTS_20261022.json new file mode 100644 index 0000000..fdc14da --- /dev/null +++ b/paper/FINAL_QA_RESULTS_20261022.json @@ -0,0 +1,295 @@ +{ + "date": "2026-10-22", + "checks_total": 56, + "passed": 56, + "failed": 0, + "checks": [ + { + "check": "Expected evidence: 4,148", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 274,331", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 623", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 41,228", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.7491", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.1791", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.7135", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.1929", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 708,631", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 24,830", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 19,840", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 3,500", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 750", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 26,697", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.68129", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.69968", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.18695", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.18227", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0046813", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0060730", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0033457", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 2,000", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 484", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 20261023", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 75e1f46131d5897a388c82411bc303", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 39754854426", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 39754046492", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 2053b7ea63f1d5ea953834bf82ed8954", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: mirror-specific", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: not counted as clean confirmation", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: cannot isolate recurrent architecture", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: does not retroactively validate", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: does not prove", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: not peer reviewed", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: cannot identify", + "passed": true, + "detail": "" + }, + { + "check": "All citation keys resolved", + "passed": true, + "detail": "cites=14, unique=10, reference entries=10" + }, + { + "check": "2015 official source cited", + "passed": true, + "detail": "" + }, + { + "check": "No raw individual dataset embedded", + "passed": true, + "detail": "" + }, + { + "check": "Figures available", + "passed": true, + "detail": "" + }, + { + "check": "Pilot nonconvergence documented", + "passed": true, + "detail": "" + }, + { + "check": "Corrected-stage method documented", + "passed": true, + "detail": "" + }, + { + "check": "Data source contrast maintained", + "passed": true, + "detail": "" + }, + { + "check": "Log no LaTeX Error", + "passed": true, + "detail": "" + }, + { + "check": "Log no Citation `", + "passed": true, + "detail": "" + }, + { + "check": "Log no Reference `", + "passed": true, + "detail": "" + }, + { + "check": "Log no Overfull \\hbox", + "passed": true, + "detail": "" + }, + { + "check": "Log no ! Undefined control sequence", + "passed": true, + "detail": "" + }, + { + "check": "PDF is valid", + "passed": true, + "detail": "" + }, + { + "check": "PDF has multiple pages", + "passed": true, + "detail": "" + }, + { + "check": "PDF extracted text contains new empirical result", + "passed": true, + "detail": "" + }, + { + "check": "All pages have extracted text", + "passed": true, + "detail": "" + }, + { + "check": "No runtime outputs in arXiv archive", + "passed": true, + "detail": "" + }, + { + "check": "Expected arXiv source files", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room arXiv source compiles twice", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room PDF exists", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room log no overfull boxes", + "passed": true, + "detail": "" + } + ], + "files": { + "main.pdf": "ba643d4581d860bcf0bd0cd2eca60f628ec7143bdf30803f607acfc536eeef5e", + "Paper1_arXiv_Updated_20261022.zip": "358472bb04f92f931c4f7f7cf8e989ac974d703d1f918ee70ab097c274a1f65b", + "PROVENANCE_AND_NOVELTY_AUDIT.md": "08c3adc7f889beefec8d7ce74ebc1d8d760ec1d6421935ba44731b89a00fef5b" + }, + "scientific_reproducibility": "Externally executed and previously verified via GitHub Actions; this QA does not re-execute raw-data study.", + "author_gate": "Pending" +} \ No newline at end of file diff --git a/paper/FINAL_RELEASE_QA_20261022.json b/paper/FINAL_RELEASE_QA_20261022.json new file mode 100644 index 0000000..883230d --- /dev/null +++ b/paper/FINAL_RELEASE_QA_20261022.json @@ -0,0 +1,313 @@ +{ + "date": "2026-10-22", + "scope": "final submitted-ready *candidate*, computational and document QA, not scholarly peer review", + "checks_passed": 60, + "checks_total": 60, + "checks_failed": 0, + "paper_pages": 11, + "pdf_sha256": "e51ffe070954cfd71f7577839a6e615f7404d959ca0a7e0628a2863abe48da59", + "arxiv_source_sha256": "06653b400cb87e5d856124e581511b3536af4c242df62ceba2d52f1956462698", + "scientific_disposition": "Narrow preprint candidate pending named author verification and submission; source discrepancy honestly reported.", + "checks": [ + { + "name": "Archive ZIP file exists", + "pass": true, + "detail": "" + }, + { + "name": "Source ZIP integrity", + "pass": true, + "detail": "" + }, + { + "name": "Exactly four production sources", + "pass": true, + "detail": "" + }, + { + "name": "No unpublished data or research manifests inside arXiv archive", + "pass": true, + "detail": "" + }, + { + "name": "No PDF and no TeX build artifacts inside source archive", + "pass": true, + "detail": "" + }, + { + "name": "No hidden files", + "pass": true, + "detail": "" + }, + { + "name": "main.tex present and stable against prepared source", + "pass": true, + "detail": "" + }, + { + "name": "Only expected figures included", + "pass": true, + "detail": "" + }, + { + "name": "Archive <1MB", + "pass": true, + "detail": "" + }, + { + "name": "Clean-room LaTeX build 1", + "pass": true, + "detail": "compiles" + }, + { + "name": "Clean-room LaTeX build 2", + "pass": true, + "detail": "compiles" + }, + { + "name": "No overfull box warnings", + "pass": true, + "detail": "" + }, + { + "name": "No undefined reference warning", + "pass": true, + "detail": "" + }, + { + "name": "No undefined citation warning", + "pass": true, + "detail": "" + }, + { + "name": "Freshly compiled PDF exists", + "pass": true, + "detail": "" + }, + { + "name": "Fresh PDF has 11 pages", + "pass": true, + "detail": "11" + }, + { + "name": "Every fresh PDF page has extracted text", + "pass": true, + "detail": "" + }, + { + "name": "Rebuilt PDF contains corrected 2009 finding", + "pass": true, + "detail": "" + }, + { + "name": "Rebuilt PDF text contains 2015 sample", + "pass": true, + "detail": "" + }, + { + "name": "PDF not encrypted", + "pass": true, + "detail": "" + }, + { + "name": "Distributed manuscript PDF exists", + "pass": true, + "detail": "" + }, + { + "name": "Distributed PDF is 11 pages", + "pass": true, + "detail": "11" + }, + { + "name": "Distributed PDF selectable text", + "pass": true, + "detail": "" + }, + { + "name": "Original mirror count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected official count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected source learner count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Direct parity discrepancy 817 recorded", + "pass": true, + "detail": "" + }, + { + "name": "Direct parity agreement 3331 recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected official split test learner count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Official 2009 Brier scores recorded", + "pass": true, + "detail": "" + }, + { + "name": "Official 2009 confidence bounds recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 Brier difference recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 sample 750 recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 bootstrap confidence bounds recorded", + "pass": true, + "detail": "" + }, + { + "name": "Paper says not peer reviewed", + "pass": true, + "detail": "" + }, + { + "name": "Statement of authorship review requirement", + "pass": true, + "detail": "" + }, + { + "name": "No novelty-overclaim flag", + "pass": true, + "detail": "" + }, + { + "name": "No blank FIXME markers", + "pass": true, + "detail": "" + }, + { + "name": "LaTeX uses fixed date, not mutable today", + "pass": true, + "detail": "" + }, + { + "name": "Three figures referenced", + "pass": true, + "detail": "" + }, + { + "name": "Verified corrected-source hash consistent", + "pass": true, + "detail": "" + }, + { + "name": "Corrected source parity disproved", + "pass": true, + "detail": "" + }, + { + "name": "Official file learner count", + "pass": true, + "detail": "" + }, + { + "name": "Mirror file learner count", + "pass": true, + "detail": "" + }, + { + "name": "Mirror and official overlap", + "pass": true, + "detail": "" + }, + { + "name": "Cross-dataset sequence equality 3331", + "pass": true, + "detail": "" + }, + { + "name": "Cross-dataset sequence discrepancy 817", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source exploratory status", + "pass": true, + "detail": "" + }, + { + "name": "PFA converged without warnings", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source GRU is lower Brier on observed data", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source bootstrap draws", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source interval favoring GRU", + "pass": true, + "detail": "" + }, + { + "name": "Corrected 2009 no causal language overstated", + "pass": true, + "detail": "" + }, + { + "name": "GitHub full source audit link", + "pass": true, + "detail": "" + }, + { + "name": "GitHub corrected official model run link", + "pass": true, + "detail": "" + }, + { + "name": "GitHub fresh 2015 model run link", + "pass": true, + "detail": "" + }, + { + "name": "arXiv abstract metadata character limit", + "pass": true, + "detail": "1267" + }, + { + "name": "Author arXiv submission steps documented", + "pass": true, + "detail": "" + }, + { + "name": "Author approval remains explicit", + "pass": true, + "detail": "" + }, + { + "name": "External peer review not fabricated", + "pass": true, + "detail": "" + } + ] +} \ No newline at end of file diff --git a/paper/FINAL_RESEARCH_AUDIT_20261022.md b/paper/FINAL_RESEARCH_AUDIT_20261022.md new file mode 100644 index 0000000..4f2a9d8 --- /dev/null +++ b/paper/FINAL_RESEARCH_AUDIT_20261022.md @@ -0,0 +1,42 @@ +# Paper 1: final scientific evidence, source-provenance and publication audit (22 October 2026) + +## Outcome of remaining feasible investigations + +### Official corrected ASSISTments 2009 direct source parity audit — completed + +- Official corrected dataset, downloaded from maintainer-linked file: SHA-256 `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`, 346,860 total corrected student-problem rows, 283,105 usable labeled records, 4,163 learners. +- Previously analyzed pinned Atomi mirror: SHA-256 `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`, 274,331 interactions, 4,148 learners. +- Shared 4,148 learners; official corrected data have 15 learners absent from the mirror. Same normalized ordered (skill, response) sequence for 3,331; discrepancy for 817. Full exact data equivalence is **refuted by data**, not merely unverified. +- Analysis with code and aggregate results: https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168 ; source script `scripts/audit_corrected_2009.py` on the PR branch. +- **Caveat**: the raw corrected source includes event identifiers; the mirror does not. The audit cannot reconstruct which exact upstream preprocessing events were omitted/altered or prove a complete one-to-one original-event crosswalk. Sorting by `order_id` is a declared convention; physical clock time is not independently reconstructed. + +### Corrected official 2009 exploratory model refit — completed + +- Corrected source frozen checksum; 4,163 learners, 283,105 skill-labeled records after excluding 63,755 records without a usable skill/outcome. Learner-disjoint fixed split 2,914 train, 624 validation, 625 test; test has 48,842 interactions. +- Enriched PFA: ROC-AUC **0.69508**, Brier **0.19675**, log loss **0.58086**; optimizer converged (930 iterations), no warning. +- Extended GRU: ROC-AUC **0.73243**, Brier **0.18507**, log loss **0.55124**; best validation epoch 13. +- Interaction-pooled Brier contrast GRU minus PFA: **−0.0116811**, 95% learner-cluster percentile interval **[−0.0137180, −0.0098625]**, 2,000 draws. +- Source protocol and code fixed before this run; executed evidence https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969 . **Exploratory**, not a new untouched 2009 confirmation, and does not isolate recurrence from feature/parameter budgets. + +### New independent-year 2015 replication — completed in prior work + +- Stage 1 pilot had PFA solver nonconvergence; it must not be counted as confirmation. +- Stage 2 excludes *all* first-stage sampled learners. 5,000 disjoint learners, 3,500 train / 750 validation / 750 test, 26,697 held-out events. +- Converged enriched PFA: Brier **0.18695**, AUC **0.68129**. GRU: Brier **0.18227**, AUC **0.69968**. Paired interaction-pooled difference **−0.0046813**, 95% interval **[−0.0060730, −0.0033457]**; passes prespecified *stage-2* diagnostic and directional criterion. +- GitHub Actions execution https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426 . Results are distinct-year, same platform family; not cross-institutional transportability, strict information matching, or a preregistered trial. + +## Novelty and inference decision + +The paper contributes auditable reproduction, direct corrected-source parity falsification, explicitly documented failure/correction of the 2015 pilot, comparison of distinct bootstrap estimands, exploratory official corrected-source refit, and a separate-year directional predictive replication. It does **not** contribute a new model architecture or prove superiority of recurrent networks under equal information/parameter budgets. + +Important relevant prior work exists: Georgios Polydoros's 2026 corrected-source cross-cohort package https://doi.org/10.5281/zenodo.22131530 and Padi et al.'s 2026 leakage/information matching paper https://arxiv.org/abs/2609.28502 . This bounds novelty and should not be hidden. We cannot infer novelty acceptance by an academic editor or certify originality from automated similarity checks. + +## Documentation and release checks + +- Updated scientific manuscript distinguishes the mirror from official corrected source, and corrected versus independent-year experiments. +- Includes all numeric source and bootstrap limits, data access links and hashes. +- Source arXiv ZIP includes only `main.tex` and three used PNG figures; no auxiliary files, data records, output PDFs or logs. +- Source compiled afresh twice, cross-references and bibliography resolved. Generated PDF remains selectable text and visually inspected. +- Author must personally read, verify claims and give approval; arXiv account access, any required endorsement, license and final submission need the account holder. arXiv moderation is not independent academic peer review. External human reviewer feedback is recommended for stronger credibility but not a prerequisite for posting a preprint. + +**Overall scientific status:** technically reproducible, carefully qualified **working preprint candidate**. Explicitly not externally peer-reviewed, not certified novel, and not yet submitted to arXiv. The corrected 2009 results strengthen robustness but source processing assumptions and unequal model information remain substantive limitations. \ No newline at end of file diff --git a/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md b/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md new file mode 100644 index 0000000..34e985f --- /dev/null +++ b/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md @@ -0,0 +1,34 @@ +# Paper 1: source-parity and contribution audit (22 October 2026) + +## Evidentiary boundary + +The existing 2009 experiment uses SHA-256-pinned transformed `Atomi/ASSISTments2009` parquet, revision `c72a664a9693547fb206652ed2ce18e62d320c7d`, digest `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`. The mirrored record exposes `user_id`, `skill_ids`, `skill_names`, `grades`, `attempt_counts`, and `answer_types` as *sequences* (Hugging Face dataset card, checked 22 October 2026). It has 4,148 serialized learner rows. The data adapter verifies unique learner IDs and aligned binary responses but does not reconstruct raw record identities or validated event chronology. + +The official corrected 2009-2010 Skill Builder release documents that duplicate records existed in the earlier dataset and publishes a corrected representation with **one row per student-problem**, collapsing multiple skills into one row. The official source supplies row-level identifiers in its original release that are not present in the Atomi sequence mirror. No pre-transformation crosswalk or data lineage statement establishing that the Atomi parquet came from that corrected exact release was located in its exposed card. Exact source equivalence is therefore **NOT VERIFIED**. Similar variable names or comparable learner counts would not suffice as proof. The paper must continue using *pinned transformed mirror*, not *verified corrected 2009 dataset*, in empirical claims. + +Official 2009 corrected source and correction notice: https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010 + +Public transformed mirror metadata and version: https://huggingface.co/datasets/Atomi/ASSISTments2009 and https://huggingface.co/datasets/Atomi/ASSISTments2009/tree/c72a664a9693547fb206652ed2ce18e62d320c7d + +The official 2015 Skill Builder source has a different schema and includes `user_id`, `log_id`, `sequence_id`, `correct`. Its 2015 event-log ID is used as a within-learner ordering proxy in the new analysis. Replicating the *direction* of a model gap on disjoint 2015 learners is a separate finding, not validation of historical 2009 source parity. + +Official 2015 source: https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data + +## Literature and novelty positioning + +- Abdelrahman, Wang and Nunes, *Knowledge Tracing: A Survey* (ACM CSUR; https://doi.org/10.1145/3569576) already synthesizes model families and evaluation practices. +- Gervet et al., *When is Deep Learning the Best Approach to Knowledge Tracing?* (2020; https://zenodo.org/records/4143614) directly cautions against generic recurrent-vs-classical superiority claims. +- Polydoros (2026; https://doi.org/10.5281/zenodo.22131530) offers a corrected-2009 / official-2015 multi-model benchmark; therefore this manuscript cannot claim to introduce cross-cohort knowledge tracing, learner splits, or calibration evaluation. +- Padi et al. (2026; https://arxiv.org/abs/2609.28502) addresses duplicated histories and information symmetry, so a claim that this paper first discovers either issue is unsupportable. + +**Defensible incremental contribution:** a fully pinned reproduction of the *specific* 2009 transformed mirror with two explicitly distinguished Brier-weighting estimands, a transparent post-inspection enhanced-baseline/GRU optimization audit, and an independently sampled 2015 directional replication that retains its initial PFA-convergence failure and disjoint corrective cohort. This is a contribution in research auditability and evidence hierarchy, not new model theory or a state-of-the-art predictor. Its publication significance ultimately requires human editorial/peer judgment. + +## Release-gate determination + +- Public documentation / source-schema audit: completed. +- Independent 2015 direction-of-predictive-difference replication: **passed narrow numerical gate** per GitHub Actions run 39754854426; first pilot run 39754046492 retained as nonconverged. +- Corrected official 2009 row equivalence / verified chronology: **not established**; impossible to infer from the mirror columns alone. +- Architecture-only effect and general applicability: **not established**. +- Author sign-off, contribution originality, arXiv metadata and submission: pending human author. + +The final manuscript is a carefully bounded working preprint candidate; do not state that corrected-source parity or peer review has been achieved. \ No newline at end of file diff --git a/paper/external_replication_assist2015.md b/paper/external_replication_assist2015.md new file mode 100644 index 0000000..8fe4033 --- /dev/null +++ b/paper/external_replication_assist2015.md @@ -0,0 +1,35 @@ +# External-cohort replication of the knowledge-tracing comparison (22 October 2026) + +## Rationale and source + +The original ASSISTments 2009 transformed-mirror model comparisons were exploratory: the 623-learner test partition had been inspected repeatedly, and the processed mirror had not been shown equivalent to the corrected official 2009 release. We therefore evaluated the previously reported direction of the predictive difference on **a separate-year source**, ASSISTments 2015 Skill Builder Data (official dataset portal: https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data). The 2015 raw file, linked by the official data portal, has SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`, 708,631 original event rows, and 19,840 eligible learner IDs after a fixed record-level validity screen; 24,830 invalid/missing rows were excluded. Response sequences were ordered by numeric `log_id`, with `sequence_id` representing the 2015 skill-builder identifier. + +## Prespecified evaluation and protocol deviation + +Before the first 2015 evaluation, a protocol fixed learner-based sampling, training/validation/test separation, model settings, the Brier-score estimand, and the bootstrap decision rule. The initial 5,000-learner random sample (seed 20261022) produced a nominal difference favoring GRU but failed its own numeric convergence gate: the enriched PFA SAGA optimizer reached its iteration limit. Its result is retained and **is not counted as confirmation** (GitHub Actions run [39754046492](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492)). + +A separately frozen corrective protocol replaced the numerical optimization algorithm only, to solve the same regularized logistic objective: enriched PFA with skill indicators, within-skill prior successes/failures, and across-skill prior successes/failures, using `lbfgs` with `C=1.0`, `max_iter=2000`, and `tol=1e-8`. Crucially, it **excluded all 5,000 participants from the first evaluation** before drawing a second random 5,000-learner sample with seed 20261023. The corrective decision was motivated by optimizer nonconvergence, not by searching test sets for favorable results. Within the second sample, learner-disjoint train/validation/test partitions contained 3,500/750/750 students. The GRU configuration remained unchanged: seed 42, fixed embedding and hidden sizes, Adam learning rate 0.002, up to 25 epochs, patience 4, with epoch selection restricted to validation data. The best validation epoch was 7 (11 epochs executed). The logistic solver converged in 484 iterations, with no convergence warnings. + +The primary comparison was fixed as the *interaction-weighted paired Brier loss difference (GRU minus enriched PFA)*, with 2,000 learner-cluster bootstrap resamples (seed 20261023). The stated directional success criterion was an upper 95% percentile-interval endpoint below zero, conditional on meeting data and numerical diagnostic gates. This is an intentionally narrow *predictive replication* criterion; it is not an intervention-effect or causal-model-attribution test. + +## New held-out findings + +The untouched stage-2 test set contained 750 learners and 26,697 response interactions (sampled 5,000 learners contained 172,614 interactions before splitting). Enriched PFA achieved ROC-AUC **0.681286**, Brier **0.186954**, log loss **0.556679**, and ECE-10 **0.016943**. Extended GRU achieved ROC-AUC **0.699683**, Brier **0.182273**, log loss **0.543455**, and ECE-10 **0.008200**. The primary pooled Brier difference (GRU minus PFA) was **−0.0046813**, with learner-cluster percentile 95% interval **[−0.0060730, −0.0033457]**. The interval excludes zero in the predicted direction, and all prespecified numerical/data gates passed. The observed correctness rate in the test sample was 0.716972. + +This second-stage result **replicates the direction of the predictive Brier advantage on a new ASSISTments 2015 learner subset**, without reusing any learner from the 5,000 selected in the first 2015 test. The earlier nonconverged experiment remains reported as a diagnostic pilot, not as an independent positive confirmation. + +## Limits on interpretation + +This is a new *cohort and learner population*, not an untouched reanalysis of the previous ASSISTments 2009 mirror. The skill-builder units in 2015 (`sequence_id`) are not guaranteed to be equivalent to 2009 skill codes. Models were retrained on 2015 training learners, so this is not direct frozen-weight cross-domain transport. The richer ordered history and greater capacity of the GRU remain confounded with architectural differences relative to aggregate-count PFA; **the result does not identify the causal effect of recurrence**. The study also does not prove equivalence between the transformed 2009 mirror and the corrected official 2009 event-level release, nor establish transportability to other schooling systems. The bootstrap interval is conditional on the selected sample and fixed code/analysis choices. Author and independent methodological review remain necessary. + +## Reproducibility evidence + +- Frozen first protocol: [CONFIRMATORY_PROTOCOL_2015.md](../CONFIRMATORY_PROTOCOL_2015.md). +- Corrective protocol fixed before the second test: [CONFIRMATORY_STAGE2_PROTOCOL_2015.md](../CONFIRMATORY_STAGE2_PROTOCOL_2015.md). +- First code/results: [scripts/confirmatory_2015.py](../scripts/confirmatory_2015.py), [stage-1 JSON](../results/confirmatory_assist2015.json). +- Second code/results: [scripts/confirmatory_2015_stage2.py](../scripts/confirmatory_2015_stage2.py), [stage-2 JSON](../results/confirmatory_assist2015_stage2.json). +- Official execution: [stage-2 GitHub Actions run 39754854426](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426). + +### Manuscript-ready concise paragraph + +> We conducted a distinct-year replication using the officially linked ASSISTments 2015 Skill Builder file. After a first 5,000-learner evaluation was deemed inconclusive because the enhanced PFA optimizer had not converged, we excluded every learner from that attempt and froze a second, nonoverlapping 5,000-learner sample, retaining learner-disjoint training (3,500), validation (750), and test (750) partitions. The numerical PFA solver was changed to a converged LBFGS fit of the same regularized logistic objective, while the GRU and primary Brier-score criterion remained prespecified. On 26,697 fresh test interactions, the extended GRU achieved Brier 0.18227 and AUC 0.69968 versus enriched PFA Brier 0.18695 and AUC 0.68129. The interaction-pooled paired Brier difference (GRU minus PFA) was −0.00468 (95% learner-cluster bootstrap interval [−0.00607, −0.00335]; 2,000 replicates). All convergence and provenance checks passed. This supports directional predictive replication within a distinct ASSISTments cohort but cannot isolate the contribution of recurrent architecture or demonstrate generalization beyond the tested contexts. diff --git a/paper/generate_figures.py b/paper/generate_figures.py new file mode 100644 index 0000000..565d1c3 --- /dev/null +++ b/paper/generate_figures.py @@ -0,0 +1,68 @@ +"""Rebuild manuscript figures from previously archived real-data GitHub Actions JSON. + +Run from repository root: python paper/generate_figures.py +Requires matplotlib and numpy. Does not download learner-level data or refit models. +""" +from __future__ import annotations +import json +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np + +ROOT=Path(__file__).resolve().parents[1] +OUT=Path(__file__).resolve().parent + +def read(rel): + path=ROOT/rel + if not path.is_file(): raise FileNotFoundError(f'Required executed output: {path}') + return json.loads(path.read_text(encoding='utf-8')) + +def bars(items,metric,title,filename,ylim=None): + fig,ax=plt.subplots(figsize=(7,3.35)) + names=[str(n) for n,_ in items] + values=[float(d[metric]) for _,d in items] + x=np.arange(len(items)) + ax.bar(x,values,width=.68) + ax.set_xticks(x,names,rotation=18,ha='right') + ax.set_ylabel(metric.replace('_',' ').upper() if metric=='roc_auc' else 'Brier score (lower is better)') + ax.set_title(title,fontsize=10) + if ylim is not None:ax.set_ylim(*ylim) + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + fig.tight_layout(pad=1.5) + fig.savefig(OUT/filename,dpi=180) + plt.close(fig) + +original=read('results/metrics.json') +robust=read('results/robustness_extension.json') +new=read('results/confirmatory_assist2015_stage2.json') +assert original['dataset']['n_learners']==4148 +assert robust['status']=='complete_exploratory_real_data' +assert new['status']=='passed_all_computational_gates' and new['directional_replication_confirmed'] +assert new['paired_brier']['n_interactions']==26697 + +choices=[('Global prior','global_rate'),('Skill prior','skill_prior'),('Fixed BKT','bkt_fixed'),('Tuned BKT','bkt_validation_tuned'),('PFA','pfa_logistic'),('GRU','gru_kt_seed_42')] +m=[(name,original['test_metrics'][key]['all']) for name,key in choices] +bars(m,'brier','2009 mirror: original six-model test','kt_brier.png',ylim=(0,.25)) +bars(m,'roc_auc','2009 mirror: original six-model test','kt_auc.png',ylim=(0,.85)) + +fig,axes=plt.subplots(1,2,figsize=(8,3.1)) +labels=['PFA','Enriched PFA','GRU 13','GRU 42','GRU 73'] +rs=[robust['pfa'],robust['stronger_pfa']]+[robust['extended_gru'][str(s)]['metrics'] for s in [13,42,73]] +for ax,key,desc in zip(axes,['roc_auc','brier'],['ROC-AUC (higher is better)','Brier (lower is better)']): + vals=[float(t[key]) for t in rs] + ax.bar(range(len(rs)),vals,width=.68) + ax.set_xticks(range(len(rs)),labels,rotation=28,ha='right') + ax.set_ylabel(desc) + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) +fig.suptitle('2009 exploratory same-test information-budget sensitivity',fontsize=10) +fig.tight_layout(pad=1.2) +fig.savefig(OUT/'robust_comparison.png',dpi=180) +plt.close(fig) + +ms=new['test_metrics'] +bars([('Enriched PFA (converged)',ms['enriched_pfa']),('Extended GRU',ms['extended_gru'])], + 'brier','Fresh ASSISTments 2015: 750 learners; 26,697 held-out responses', + 'cohort2015_brier.png',ylim=(0,.23)) +print('Generated four manuscript figures from executed JSON.') \ No newline at end of file diff --git a/paper/main.tex b/paper/main.tex new file mode 100644 index 0000000..81a3e28 --- /dev/null +++ b/paper/main.tex @@ -0,0 +1,194 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.84in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{Knowledge-Tracing Reproducibility and Information-Budget Auditing\\\large A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{22 October 2026\\\small Empirical research manuscript; not peer reviewed} +\begin{document} +\maketitle +\begin{abstract} +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the interaction-pooled paired Brier contrast is $-0.00468$ (learner-cluster bootstrap 95\% interval $[-0.00607,-0.00335]$, 2,000 draws, conditional on the fitted models). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. + +\section{Introduction} +Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. + +A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. + +Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? + +\section{Related Work and Positioning} +BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. + +\section{Study Contribution and Evidence Hierarchy} +We provide four bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and unresolved transformation question} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Provenance qualification.} The original ASSISTments maintainers published corrected skill-builder records after identifying duplicated rows \citep{assistmentsdata}. The pinned third-party mirror does not supply all original row-level identifiers needed to prove its equivalence to that corrected official release. Consequently, all numerical results below apply to \emph{this pinned mirror representation}. This is a meaningful, narrower replication claim, not a corrected-release benchmark. The public sequence mirror contains binary response arrays and skill identifiers, but the current audit cannot map these serialized events back to unique corrected-release student--problem identifiers or recover an independently timestamped ordering of the underlying raw events. If one underlying student--problem event was inadvertently repeated within a sequence, a learner-disjoint split alone could not prevent within-sequence target-label contamination. The chronological and unique-event interpretation of the mirror must therefore be treated as an \emph{unverified input assumption}, not as a validated property of the original ASSISTments log. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{Reproduction and Exploratory Robustness Results} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. On the pilot's 750 held-out learners (25,089 responses), enriched PFA obtained Brier 0.18142 and ROC-AUC 0.68957, whereas the extended GRU obtained Brier 0.17644 and ROC-AUC 0.70779; the nominal response-pooled Brier difference was $-0.00498$ with learner-cluster percentile interval $[-0.00697,-0.00336]$. \emph{These pilot estimates are reported for transparency, not accepted as confirmation}: the PFA \texttt{saga} solver reached its iteration limit, violating the predetermined convergence gate. Thus the apparent first-sample advantage is computationally qualified and cannot be pooled with the subsequent confirmatory estimate. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +\begin{figure}[ht]\centering +\includegraphics[width=.88\linewidth]{cohort2015_brier.png} +\caption{Fresh 2015 cohort, second-stage held-out response-pooled Brier scores. These scores refer to models trained on the independent second-stage training learners. The initial PFA-nonconverged pilot is not represented as confirmatory evidence.} +\label{fig:stage2brier} +\end{figure} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + +\section{Discussion and Threats to Validity} +The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. + +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in the 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. The transformed Atomi mirror exposes user IDs, skill/grade lists, attempt counts and answer types, but not the row-level problem IDs, event IDs or validated timestamps required to prove corrected-source parity or exact per-event chronology. We checked the published source descriptions and schema and found no demonstrated row crosswalk. The 2009 findings \emph{remain mirror-specific}; the new 2015 experiment does not repair them. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. + +\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. + +\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. + +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The second-stage 2015 interval conditions on the already fitted models and resamples held-out learners only; it does \emph{not} repeat the full training and model-selection procedure in every bootstrap draw. It therefore quantifies test-sample cluster uncertainty, not between-training-run, hyperparameter-selection, or broader population uncertainty. Secondary metrics (AUC, log loss, and ECE-10) are descriptive rather than separately multiplicity-controlled hypothesis tests. A single learner sample and model seed cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. + +\section{Research Questions Revisited} +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. + +\section{Conclusion} +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim on the evaluated ASSISTments cohorts, not proof that recurrence itself is superior under information-matched conditions. Stage-2 bootstrap inference conditions on fixed trained models; numerical success and source checksum verification do not establish external validity. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. + +\section*{Data availability and research integrity} +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review, full 2009 corrected-source parity, and author approval are not claimed. + +\section*{Limitations of authorship and interpretation} +This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file diff --git a/paper/main_integrated_20261022.tex b/paper/main_integrated_20261022.tex new file mode 100644 index 0000000..21f56e7 --- /dev/null +++ b/paper/main_integrated_20261022.tex @@ -0,0 +1,208 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.84in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{Knowledge-Tracing Reproducibility and Information-Budget Auditing\\\large A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{22 October 2026\\\small Empirical research manuscript; not peer reviewed} +\begin{document} +\maketitle +\begin{abstract} +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). Direct comparison with the officially corrected student--problem file demonstrates nonparity: 817 of 4,148 shared learners differ in their event sequences. On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). In an exploratory refit directly on the corrected 2009 source, the enriched PFA scores Brier 0.19675 and the GRU 0.18507 across 625 held-out learners. To assess whether the predictive direction persists beyond the inspected 2009 cohort, we also executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the paired Brier contrast is $-0.00468$ (95\% learner-cluster interval $[-0.00607,-0.00335]$, 2,000 draws). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove an isolated effect of recurrent architecture, cross-institutional transportability, or instructional benefit; strict 2009 mirror parity has instead been disproved. +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. + +\section{Introduction} +Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. + +A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. + +Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? + +\section{Related Work and Positioning} +BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. + +\section{Study Contribution and Evidence Hierarchy} +We provide six bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a direct event-count and ordered-sequence parity audit against the official corrected 2009 source; (5) an exploratory official corrected-2009 source sensitivity analysis; and (6) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and observed differences from the corrected source} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Direct provenance audit.} The ASSISTments maintainers corrected an earlier Skill Builder release after identifying duplicated records \citep{assistmentsdata}. We downloaded the official corrected student--problem CSV directly from the file they link (SHA-256 \texttt{162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da}) and compared it with the exact pinned mirror used for the model run. The corrected file contains 346,860 student--problem records, of which 283,105 contain a usable binary outcome and nonmissing skill identifier, from 4,163 learners. The mirror comprises 274,331 responses from 4,148 learners. All 4,148 mirror learner IDs occur in the corrected source; 15 additional learners occur only in the corrected source. Among the 4,148 shared learners, 3,331 have identical sequences of normalized (skill identifier, binary response) pairs when official records are ordered by numeric \texttt{order\_id}; 817 do not. The same 3,331 satisfy event-count and skill--response-multiset equality. Accordingly, \emph{exact whole-dataset parity is empirically disproved}, not merely unverified. The original mirror-derived predictive results remain correct for the pinned mirror, but cannot be represented as estimates from the corrected official 2009 release. This audit does not identify which upstream transformation rule caused the discrepancies. Although corrected-source learner and event keys can be checked directly, the mirror omits original student--problem event IDs, so a complete one-to-one event crosswalk and independently timestamped chronology are not recoverable. Numeric \texttt{order\_id} was used as a documented ordering convention, not as proof of real-world time order. The full aggregate audit, source hashes and code are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168}{39773920168}; individual student data are not redistributed. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{Reproduction and Exploratory Robustness Results} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Direct corrected-source parity test} +The corrected official release has 4,163 learners, including the 4,148 with sequences in the mirror. Normalizing the skill IDs and sorting the corrected file by numeric \texttt{order\_id} within each learner gives exact sequence agreement for 3,331 shared learner IDs; it fails for 817. Table~\ref{tab:parity} shows the necessary-condition comparison. Thus the frozen 2009 mirror results cannot be relabeled as corrected-release results. Ordered pair equality provides some external corroboration for unaffected learners, but it does not restore original problem-level identifiers to the mirror or prove that the \texttt{order\_id} convention captures the physical event chronology. +\begin{table}[ht]\centering\small +\caption{Official corrected 2009 skill-builder records compared with the pinned Atomi sequence mirror. Official labeled rows exclude records without a usable skill identifier.} +\label{tab:parity} +\begin{tabular}{lrr}\toprule +Measure & Corrected official & Pinned mirror\\\midrule +Unique learners & 4,163 & 4,148\\ +Skill-labeled responses & 283,105 & 274,331\\ +Shared learner IDs & \multicolumn{2}{c}{4,148}\\ +Identical ordered skill--response sequences & \multicolumn{2}{c}{3,331 shared learners}\\ +Different ordered sequences & \multicolumn{2}{c}{817 shared learners}\\\bottomrule +\end{tabular} +\end{table} + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\subsection{Exploratory predictive sensitivity on corrected official 2009 data} +To determine how much the corrected source changes the predictive comparison, we ran a separate, prewritten \emph{exploratory} pipeline on the official corrected release rather than treating the mirror as equivalent. We excluded unlabeled skill records, normalized skill identifiers, ordered each learner's records by numeric \texttt{order\_id}, and used a whole-learner 70/15/15 split with seed 20261024. The enriched PFA retained its same-skill and across-skill history summaries, and the GRU retained prior ordered interactions, seed 42, 25-epoch maximum and validation-based stopping. Both models were refitted exclusively on corrected-source training learners. This is not a fully untouched confirmatory comparison, because the same 2009 academic setting and modeling direction were previously analyzed. + +On 625 corrected-source test learners and 48,842 held-out interactions, the enriched PFA achieved ROC-AUC 0.6951, Brier 0.19675, and log loss 0.5809; the GRU achieved ROC-AUC 0.7324, Brier 0.18507, and log loss 0.5512. The paired interaction-weighted Brier difference (GRU minus PFA) was $-0.01168$ (95\% learner-cluster bootstrap percentile interval $[-0.01372,-0.00986]$, 2,000 resamples). The PFA optimizer converged in 930 iterations, and the GRU's best validation epoch was 13. These source-corrected figures are not directly numerically comparable with the 2009 mirror test split because its learner membership and preprocessing differ. The new analysis audits predictive sensitivity; it does not restore mirror/event parity or identify architectural causes. The aggregate code and outputs are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969}{39774414969}. + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + +\section{Discussion and Threats to Validity} +The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. + +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in an earlier 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. Our independently executed direct source comparison disproves strict equality between that corrected release and the Atomi mirror: 817 shared learner sequences differ and the mirror is shorter by 8,774 skill-labeled responses. The mirror retains user/skill/grade sequences, but lacks the original problem/event IDs and verified timestamps needed to attribute the differences or reconstruct a complete raw-event crosswalk. The 2009 benchmark results therefore \emph{remain mirror-specific}; the 2015 experiment is a separate-year check and does not repair this mismatch. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. + +\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. + +\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. + +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. + +\section{Research Questions Revisited} +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. \textbf{RQ6 (corrected 2009 source):} Direct event-level audit refuted full equivalence with the transformed mirror, but a separate, exploratory analysis of the corrected raw source reproduced the direction of the predictive Brier difference on 625 held-out learners. + +\section{Conclusion} +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). A direct corrected-2009 source audit and exploratory corrected-source sensitivity complement that evidence. The combined analyses support a \emph{limited predictive replication} claim, not proof that recurrence itself is superior under information-matched conditions. Exact corrected-2009 row-parity has been disproved by a direct source audit; broad generalization and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. + +\section*{Data availability and research integrity} +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review and author approval are not claimed. Strict whole-dataset 2009 corrected-source parity was directly tested and disproved by necessary event-count and sequence-equality checks. + +\section*{Limitations of authorship and interpretation} +This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file diff --git a/paper/paper.md b/paper/paper.md index 4d3dae4..6332f22 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -1,97 +1,27 @@ -# A Learner-Disjoint Benchmark of Priors, BKT, PFA and Recurrent Knowledge Tracing on ASSISTments 2009 +# Knowledge-Tracing Reproducibility and Information-Budget Auditing -## Abstract +**Current research status (22 October 2026):** The final author-review manuscript is [main_integrated_20261022.tex](main_integrated_20261022.tex), together with the three figures in the separately packaged arXiv source archive. This repository text is a project overview, **not** a second competing manuscript. -Knowledge-tracing comparisons are easy to overstate when learners cross data partitions or when a neural model is compared only with a weak baseline. This study uses learner-disjoint train, validation and test partitions from ASSISTments 2009 and compares global and skill priors, fixed and validation-tuned Bayesian Knowledge Tracing, a Performance Factors Analysis-style logistic model, and a compact GRU knowledge tracer. Evaluation includes probability quality, discrimination, calibration, first-seen versus repeated-skill slices, learner-block uncertainty, repeated neural seeds, and skill-level error analysis. +## Empirical findings -## Research questions +- **ASSISTments 2009 transformed mirror, learner-disjoint original test:** 4,148 learners, 274,331 interactions in the full representation, 623 test learners, 41,228 test responses. Standard PFA ROC-AUC 0.6982 and Brier 0.1976; original seed-42 GRU AUC 0.7470 and Brier 0.1800. Full original workflow reproduced on the *same pinned mirror*, GitHub Actions [39655888620](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620). +- **Exploratory 2009 fixed-test robustness:** Stronger PFA with prior across-skill counts: AUC 0.7135, Brier 0.1929. Extended seed-42 GRU: AUC 0.7491, Brier 0.1791. GRU-minus-enriched-PFA pooled Brier difference -0.013874, learner-cluster 95% interval [-0.017231, -0.011049]. All these changes were evaluated after the original test was inspected and therefore are explicitly **exploratory**. +- **Independent-year ASSISTments 2015, preliminary test:** A first 5,000-learner sampling protocol produced nominal favorable differences but the enriched PFA optimizer failed to converge. [Run 39754046492](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492) is an **inconclusive diagnostic pilot**, not valid confirmation. +- **New, nonoverlapping 2015 cohort, convergence-corrected protocol:** All 5,000 pilot learners were excluded; second sample 5,000 different learners (3,500 train, 750 validation, 750 test). On 26,697 fresh held-out responses, enriched PFA achieved AUC 0.68129, Brier 0.18695; extended GRU AUC 0.69968, Brier 0.18227. Interaction-weighted paired Brier delta -0.0046813, 95% learner-cluster bootstrap interval [-0.0060730,-0.0033457], 2,000 draws. All numerical/data gates and PFA convergence passed. Evidence: [run 39754854426](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426), [final JSON](../results/confirmatory_assist2015_stage2.json). **This supports only a narrow directional predictive replication on the tested 2015 sample.** -1. Do stateful models improve held-out next-response prediction over global and skill priors? -2. Does validation-tuned BKT improve over a fixed BKT parameterization? -3. Does a transparent PFA-style model explain enough history to compete with a recurrent model? -4. Are conclusions stable for first-seen versus repeated skill encounters? -5. How large are learner-block uncertainty intervals around Brier-score differences? -6. How sensitive is the recurrent result to the prespecified GRU seeds? +## Manuscript and audit sources -## Background +- [Canonical updated LaTeX manuscript](main.tex) +- [Manuscript build and review instructions](BUILD_AND_AUTHOR_REVIEW.md) · [Final numerical/source-package QA results](FINAL_QA_RESULTS_20261022.json) +- [Source-parity and novelty assessment](PROVENANCE_NOVELTY_AUDIT_20261022.md) +- [External cohort replication details](external_replication_assist2015.md) +- [First 2015 protocol](../CONFIRMATORY_PROTOCOL_2015.md) and [corrective-stage protocol](../CONFIRMATORY_STAGE2_PROTOCOL_2015.md) +- [Exploratory code PR #1](https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1) and [independent-cohort code PR #2](https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2) -Bayesian Knowledge Tracing models a learner's changing latent state for individual knowledge components (Corbett & Anderson, 1995). Performance Factors Analysis provides a transparent logistic alternative based on prior successes and failures (Pavlik, Cen, & Koedinger, 2009). Recurrent neural models later expanded knowledge tracing by learning sequence representations directly from interaction histories (Piech et al., 2015). +## Non-negotiable interpretation bounds -This benchmark deliberately keeps classical and recurrent comparators together. That choice follows evidence that conclusions about deep versus classical knowledge-tracing models can depend strongly on dataset characteristics, features, and evaluation design (Gervet et al., 2020). Attention-based models such as SAKT and AKT are important extensions but are outside the current frozen model set (Pandey & Karypis, 2019; Ghosh, Heffernan, & Lan, 2020). +Direct comparison with the **official corrected ASSISTments 2009--2010 source** disproved exact full-dataset parity: the official corrected file contained 4,163 learners and 283,105 skill-labeled records, while the mirror had 4,148 learners and 274,331 interactions. Among 4,148 shared learners, 3,331 normalized ordered sequences agreed and **817 differed**. The mirror still lacks raw problem/event identifiers necessary to trace all discrepancies; source-specific caveats remain. An exploratory corrected-source refit on 625 held-out learners found GRU Brier 0.18507 versus converged enriched PFA Brier 0.19675 (learner-cluster 95% interval for GRU minus PFA [-0.013718,-0.009862]). The separate fresh 2015 experiment does **not** restore 2009 data parity. -## Data +The GRU has ordered cross-skill history and greater capacity, while PFA uses summary history counts. The observed gap **must not** be attributed solely to recurrent architecture. The 2015 improvement does not establish intervention effects, fairness, educational gains, or universal transportability. -The scientific source is ASSISTments 2009. The executable pipeline retrieves the public Atomi serialization pinned to revision `c72a664a9693547fb206652ed2ce18e62d320c7d` and records the exact parquet SHA-256. Before splitting, the runner verifies that the serialized learner rows contain unique `user_id` values. See `DATA.md`. - -## Design - -Learners are shuffled with a fixed seed and assigned 70%/15%/15% to train, validation and test partitions. No learner crosses partitions. Stateful predictions are generated before the current response is observed. - -The validation partition is used for BKT parameter selection and GRU early stopping. The final test partition is reserved for evaluation. - -## Models - -The benchmark includes a global response prior, a training-skill prior, fixed BKT, validation-tuned BKT, PFA-style logistic regression using prior learner-skill successes and failures, and a compact GRU sequence model. - -The GRU is evaluated at the prespecified seeds 13, 42, and 73 in the full run. The implementation is a compact recurrent benchmark and is not presented as an exact reproduction of every canonical Deep Knowledge Tracing implementation. - -## Evaluation - -The final test set reports ROC-AUC, average precision, Brier score, log loss and ECE-10. Metrics are sliced by first-seen and repeated skill encounters. Here, first-seen means the first occurrence of the observed skill key within that learner's sequence, not a skill globally unseen in training. - -Brier-score differences are bootstrapped by learner block, skills with sufficiently many test interactions are inspected for high prediction error, and repeated GRU runs are summarized to expose random-seed sensitivity. - -## Results - -Numerical evidence is generated by `src/run_experiment.py` into `results/metrics.json`, `results/summary.md`, `paper/results.md`, and figures. This methods manuscript intentionally contains no independently hand-entered winner claim. - -## Limitations - -Skill annotations and response sequences are imperfect proxies for learning. Results are dataset- and protocol-specific; they do not establish teaching effectiveness, causal learning gains, fairness, or suitability for consequential learner decisions. The learner bootstrap is conditional on one prespecified train/validation/test partition and does not test sensitivity to alternative split seeds. Additional datasets, split-sensitivity analysis, and modern attention-based knowledge-tracing models are appropriate extensions before making broader state-of-the-art claims. - -## References - -Corbett, A. T., & Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. *User Modeling and User-Adapted Interaction, 4*, 253–278. https://doi.org/10.1007/BF01099821 - -Feng, M., Heffernan, N. T., & Koedinger, K. R. (2009). Addressing the assessment challenge in an intelligent tutoring system that tutors as it assesses. *User Modeling and User-Adapted Interaction, 19*, 243–266. - -Gervet, T., Koedinger, K., Schneider, J., & Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? *Journal of Educational Data Mining, 12*(3), 31–54. https://doi.org/10.5281/zenodo.4143614 - -Ghosh, A., Heffernan, N., & Lan, A. S. (2020). Context-Aware Attentive Knowledge Tracing. *Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*. https://arxiv.org/abs/2007.12324 - -Pandey, S., & Karypis, G. (2019). A Self-Attentive Model for Knowledge Tracing. *Proceedings of the 12th International Conference on Educational Data Mining*. https://arxiv.org/abs/1907.06837 - -Pavlik, P. I., Cen, H., & Koedinger, K. R. (2009). Performance Factors Analysis — A New Alternative to Knowledge Tracing. *Proceedings of the 14th International Conference on Artificial Intelligence in Education*, 531–538. https://doi.org/10.3233/978-1-60750-028-5-531 - -Piech, C., Spencer, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., & Sohl-Dickstein, J. (2015). Deep Knowledge Tracing. *Advances in Neural Information Processing Systems, 28*. https://arxiv.org/abs/1506.05908 - - -## Calculation definitions and evidence audit - -BKT prediction = mastery×(1-slip) + (1-mastery)×guess. - -After a response, Bayes updating and the learning transition produce the next mastery state. Brier and log loss assess probabilities; learner-block bootstrap respects within-learner dependence. Model mastery is not directly measured knowledge. - -On 623 held-out learners, the seed-42 GRU achieves ROC-AUC 0.7470 and Brier score 0.1800, compared with 0.6982 and 0.1976 for PFA. The three GRU seeds average ROC-AUC 0.7471 with sample SD 0.0008. These results support better response prediction under this historical benchmark; they do not establish instructional benefit or justify high-stakes interpretations of a learner’s knowledge. - -The [calculation guide](../CALCULATIONS.md) provides exact evidence paths and a function-level implementation map. - -![Study design](../assets/review_overview.svg) - -![Calculation and selected evidence](../assets/review_calculations.svg) - -### Selected evidence and interpretation - -| Quantity | Value | Unit / meaning | JSON path | -|---|---:|---|---| -| skill_prior | 0.20965724966908303 | Brier ↓ | `test_metrics.skill_prior.all.brier` | -| bkt_fixed | 0.21870446416122305 | Brier ↓ | `test_metrics.bkt_fixed.all.brier` | -| bkt_validation_tuned | 0.20237712440747457 | Brier ↓ | `test_metrics.bkt_validation_tuned.all.brier` | -| pfa_logistic | 0.19760582856803 | Brier ↓ | `test_metrics.pfa_logistic.all.brier` | -| gru_kt_seed_42 | 0.1799516879180387 | Brier ↓ | `test_metrics.gru_kt_seed_42.all.brier` | - -These values are read from `results/metrics.json`. They must be interpreted with the split, data status and limitations above. The complete data/model experiment was not rerun in this review. Stored empirical results were inspected, not independently reproduced from raw data. - -### Reproduction and claim boundaries - -The existing suite requires unavailable dependencies; no full-suite pass is claimed. The figure generator can be checked with `python scripts/build_review_figures.py --check`. This verifies the displayed calculation evidence, not an independent replication of the complete scientific experiment. The manuscript is a working report, not a peer-reviewed publication. +The revised 11-page arXiv source package compiles cleanly and all 60 new automated checks pass, but **author verification, novelty judgment and final arXiv upload are pending**. This is a working preprint candidate, not a peer-reviewed study. diff --git a/results/confirmatory_assist2015.json b/results/confirmatory_assist2015.json new file mode 100644 index 0000000..ffd25ec --- /dev/null +++ b/results/confirmatory_assist2015.json @@ -0,0 +1,73 @@ +{ + "status": "completed_with_unresolved_diagnostics", + "directional_replication_confirmed": false, + "test_is_external_cohort_not_prior_2009_test": true, + "source_integrity_scope": "Official-source URL/id and downloaded-byte SHA; no independent publisher checksum available", + "data": { + "official_portal": "https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data", + "official_linked_drive_file_id": "0B_hO8cnpcIMgUGZzRnh3bHJrSjQ", + "raw_filename": "assistments_2015_official.csv", + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "source_total_rows": 708631, + "excluded_invalid_rows": 24830, + "source_unique_eligible_learners": 19840, + "sample_seed": 20261022, + "sample_learners": 5000, + "sample_interactions": 170889, + "chronology": "ascending log_id within user_id", + "concept": "sequence_id (skill builder; not 2009 mirror skill_id)" + }, + "protocol": { + "sample_learners": 5000, + "sample_seed": 20261022, + "split_seed": 20261022, + "train_learners": 3500, + "validation_learners": 750, + "test_learners": 750, + "gru_seed": 42, + "gru_max_epochs": 25, + "gru_patience": 4, + "bootstrap_seed": 20261022, + "bootstrap_draws": 2000, + "predeclared_primary_estimand": "GRU - enhanced PFA pooled interaction-weighted Brier", + "directional_rule": "Upper endpoint of 95% percentile learner-cluster bootstrap CI < 0; must pass diagnostic gates" + }, + "metrics": { + "enriched_pfa": { + "n": 25089, + "positive_rate": 0.7285264458527642, + "brier": 0.18141712741703564, + "log_loss": 0.5433839611564283, + "average_precision": 0.8427901868527623, + "ece_10": 0.015065578937170483, + "roc_auc": 0.6895656527695257 + }, + "extended_gru": { + "n": 25089, + "positive_rate": 0.7285264458527642, + "brier": 0.17643847175634333, + "log_loss": 0.529606322036061, + "average_precision": 0.8597931958258713, + "ece_10": 0.014323575497811158, + "roc_auc": 0.7077904092022121 + } + }, + "brier_comparison": { + "difference_candidate_minus_reference": -0.004978655660692301, + "bootstrap_95_interval": [ + -0.0069731289320504345, + -0.0033563705430400648 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 750, + "n_interactions": 25089, + "n_bootstrap": 2000 + }, + "gru_best_validation_epoch": 9, + "gru_epochs_completed": 13, + "pfa_convergence_warnings": [ + "The max_iter was reached which means the coef_ did not converge" + ], + "conclusion_scope": "Different-year ASSISTments learner-subset replication. Not source parity for 2009 and not a causal attribution of architecture." +} \ No newline at end of file diff --git a/results/confirmatory_assist2015_stage2.json b/results/confirmatory_assist2015_stage2.json new file mode 100644 index 0000000..6383d28 --- /dev/null +++ b/results/confirmatory_assist2015_stage2.json @@ -0,0 +1,76 @@ +{ + "status": "passed_all_computational_gates", + "directional_replication_confirmed": true, + "test_supports_direction": true, + "stage1_result_used_for_model_selection": false, + "data": { + "source_url": "https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data", + "drive_id": "0B_hO8cnpcIMgUGZzRnh3bHJrSjQ", + "sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "total_source_rows": 708631, + "excluded_invalid_rows": 24830, + "eligible_learners": 19840, + "stage1_excluded_learners": 5000, + "stage2_sampling_pool": 14840, + "stage2_sampled_learners": 5000, + "stage2_sample_interactions": 172614, + "stage2_intersection_stage1": 0, + "sorting": "log_id ascending per learner, sequence_id as skill-builder concept" + }, + "protocol": { + "stage1_sample_seed": 20261022, + "stage2_sample_seed": 20261023, + "sampled_learners": 5000, + "train_learners": 3500, + "validation_learners": 750, + "test_learners": 750, + "gru_seed": 42, + "gru_max_epochs": 25, + "gru_patience": 4, + "pfa_solver": "lbfgs", + "pfa_C": 1.0, + "pfa_max_iter": 2000, + "pfa_tol": 1e-08, + "bootstrap_draws": 2000, + "bootstrap_seed": 20261023, + "primary_estimand": "GRU minus PFA pooled interaction-weighted Brier", + "success": "upper endpoint of 95% learner-cluster percentile interval <0; diagnostic gates pass" + }, + "test_metrics": { + "enriched_pfa": { + "n": 26697, + "positive_rate": 0.71697194441323, + "brier": 0.186954270145136, + "log_loss": 0.5566794636119368, + "average_precision": 0.8306718333720006, + "ece_10": 0.0169429951113543, + "roc_auc": 0.68128602293271 + }, + "extended_gru": { + "n": 26697, + "positive_rate": 0.71697194441323, + "brier": 0.18227301560902529, + "log_loss": 0.5434550333145024, + "average_precision": 0.8477370017121166, + "ece_10": 0.00819962269299951, + "roc_auc": 0.6996825804347548 + } + }, + "paired_brier": { + "difference_candidate_minus_reference": -0.004681254536110695, + "bootstrap_95_interval": [ + -0.0060729744844493895, + -0.0033456516490331786 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 750, + "n_interactions": 26697, + "n_bootstrap": 2000 + }, + "pfa_iterations": 484, + "pfa_convergence_warnings": [], + "gru_best_validation_epoch": 7, + "gru_epochs_completed": 11, + "limitation": "Replication of the directional performance difference on a newly sampled 2015 population; not causal architecture identification or universal transportability." +} \ No newline at end of file diff --git a/results/corrected_2009_sensitivity.json b/results/corrected_2009_sensitivity.json new file mode 100644 index 0000000..2eeb4d4 --- /dev/null +++ b/results/corrected_2009_sensitivity.json @@ -0,0 +1,64 @@ +{ + "status": "completed_exploratory_corrected_source", + "analysis_type": "exploratory source-sensitivity, not confirmatory", + "source": { + "corrected_sha256": "162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da", + "original_source_rows": 346860, + "labeled_rows": 283105, + "excluded_unlabeled_or_invalid": 63755, + "learner_count": 4163, + "chronology": "numeric order_id; source-identified sequence order" + }, + "protocol": { + "split_seed": 20261024, + "gru_seed": 42, + "train_learners": 2914, + "validation_learners": 624, + "test_learners": 625, + "pfa_solver": "lbfgs", + "pfa_C": 1.0, + "pfa_max_iter": 2000, + "pfa_tol": 1e-08, + "gru_max_epochs": 25, + "gru_patience": 4, + "bootstrap_resamples": 2000, + "bootstrap_seed": 20261024 + }, + "pfa_convergence_warnings": [], + "pfa_iterations": 930, + "gru_epochs_completed": 17, + "gru_best_epoch": 13, + "metrics": { + "enriched_pfa": { + "n": 48842, + "positive_rate": 0.670713729986487, + "brier": 0.19675493213137696, + "log_loss": 0.580861994297318, + "average_precision": 0.8007165977950836, + "ece_10": 0.021130927665680675, + "roc_auc": 0.6950809111386503 + }, + "extended_gru": { + "n": 48842, + "positive_rate": 0.670713729986487, + "brier": 0.18507380934628698, + "log_loss": 0.5512403251511567, + "average_precision": 0.8298660557243271, + "ece_10": 0.010919003540782643, + "roc_auc": 0.7324336482108271 + } + }, + "paired_brier": { + "difference_candidate_minus_reference": -0.011681122785090007, + "bootstrap_95_interval": [ + -0.013717998422616664, + -0.009862452037847193 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 625, + "n_interactions": 48842, + "n_bootstrap": 2000 + }, + "interpretation": "Numbers are conditional on corrected source and selected splitting/representation. This is not a new untouched 2009 holdout or causal architecture effect." +} \ No newline at end of file diff --git a/results/official_2009_mirror_parity_audit.json b/results/official_2009_mirror_parity_audit.json new file mode 100644 index 0000000..0c22620 --- /dev/null +++ b/results/official_2009_mirror_parity_audit.json @@ -0,0 +1,40 @@ +{ + "source": { + "official_documentation_url": "https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010", + "official_corrected_drive_id": "1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE", + "corrected_file_sha256": "162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da", + "corrected_file_bytes": 64412812, + "source_columns": [ + "user_id", + "skill_id", + "correct", + "order_id" + ], + "corrected_source_records_total": 346860, + "records_valid_binary": 346860, + "records_valid_binary_and_labeled_skill": 283105, + "source_unique_learners_with_skills": 4163, + "source_rows_repeating_student_order_key": 0 + }, + "mirror": { + "repository": "Atomi/ASSISTments2009", + "revision": "c72a664a9693547fb206652ed2ce18e62d320c7d", + "sha256": "2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7", + "serialized_learners": 4148, + "total_interactions": 274331 + }, + "comparison": { + "shared_learner_ids": 4148, + "official_only_learner_ids": 15, + "mirror_only_learner_ids": 0, + "shared_learner_ids_same_number_of_interactions": 3331, + "shared_learner_ids_identical_skill_response_multisets": 3331, + "shared_learner_ids_identical_native_CSV_order": 1441, + "shared_learner_ids_identical_numeric_order_id_order": 3331, + "mirror_interactions_shared_learners": 274331, + "corrected_interactions_shared_learners": 283014 + }, + "parity_status": "exact_full_dataset_parity_disproved_by_necessary_conditions", + "interpretation": "Necessary count and (skill, correctness) multiset conditions only; no mirror original event IDs, no independent chronology guarantee. Equal aggregates cannot prove row identity; unequal totals/multisets disprove exact full parity.", + "privacy": "No learner IDs, rows, or raw source bytes are written to output." +} \ No newline at end of file diff --git a/scripts/audit_corrected_2009.py b/scripts/audit_corrected_2009.py new file mode 100644 index 0000000..fc20e15 --- /dev/null +++ b/scripts/audit_corrected_2009.py @@ -0,0 +1,212 @@ +"""Public-data provenance audit; never stores or prints student-level records. + +Compares the pinned Atomi serialized sequences to the original-data maintainers' +corrected 2009 Skill Builder release. The corrected release retains student/problem +identifiers that the third-party transformed mirror omits, so this can reject +exact parity but cannot in general prove event-by-event identity or chronology. + +Run: python scripts/audit_corrected_2009.py +Data downloads stay in the untracked data/cache folder. +""" +from __future__ import annotations +import hashlib +import json +from collections import Counter +from pathlib import Path +import re + +CORRECTED_FILE_ID = '1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE' +SOURCE_PAGE = 'https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010' +MIRROR_REPO = 'Atomi/ASSISTments2009' +MIRROR_REV = 'c72a664a9693547fb206652ed2ce18e62d320c7d' +MIRROR_FILE = 'data/train-00000-of-00001.parquet' +MIRROR_SHA = '2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7' + + +def identifier(x): + """Normalize simple integer/decimal IDs, but not compound skill strings.""" + s = str(x).strip() + if s.endswith('.0') and s[:-2].lstrip('-').isdigit(): + return s[:-2] + return s + + +def skill_token(x): + s = str(x).strip() + if s.lower() in {'nan', 'none', '', 'null'}: + return '' + # Corrected data use skill1_skill2; some public CSVs use semicolon lists. + parts = re.split(r'[_;]', s) + return '_'.join(identifier(z) for z in parts if z.strip()) + + +def load_csv_or_archive(path): + import io + import zipfile + import pandas as pd + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as z: + members = [x for x in z.namelist() if x.lower().endswith('.csv') and not x.startswith('__MACOSX/')] + if not members: + raise ValueError('Official corrected archive lacks a CSV') + members.sort(key=lambda m: z.getinfo(m).file_size, reverse=True) + with z.open(members[0]) as f: + return pd.read_csv(f, low_memory=False, encoding='latin-1') + if path.read_bytes()[:32].lower().lstrip().startswith((b' None: + """Only the official ASSISTments-linked file ID is accepted for network retrieval.""" + if path.exists(): + print(f"Using locally supplied file {path}; source authenticity requires author audit") + return + import gdown + path.parent.mkdir(parents=True, exist_ok=True) + result = gdown.download(id=GOOGLE_FILE_ID, output=str(path), quiet=False) + if result is None or not path.exists(): + raise RuntimeError("Official Google Drive CSV download failed; no fallback to an unverified mirror") + with path.open("rb") as handle: + if handle.read(128).lstrip().startswith((b" tuple[list[dict], dict]: + digest = hashlib.sha256(csv_path.read_bytes()).hexdigest() + raw = pd.read_csv(csv_path, low_memory=False) + raw.columns = [str(c).strip().lower() for c in raw.columns] + required = ("user_id", "log_id", "sequence_id", "correct") + if len(set(raw.columns)) != len(raw.columns) or any(c not in raw for c in required): + raise ValueError(f"Expected unique columns {required}, found {list(raw.columns)}") + source_n = len(raw) + df = raw[list(required)].copy() + invalid_null = df.isna().any(axis=1) + for name in ("log_id", "correct"): + df[name] = pd.to_numeric(df[name], errors="coerce") + invalid = invalid_null | df.isna().any(axis=1) | ~df["correct"].isin([0, 1]) + bad_n = int(invalid.sum()) + if bad_n > 0: + # Source data may include missing records; the exclusion rule is declared ex ante. + df = df.loc[~invalid].copy() + if len(df) == 0 or bad_n / source_n > 0.05: + raise ValueError(f"Too few valid records: {bad_n}/{source_n} excluded (5% ceiling)") + if df.duplicated(["user_id", "log_id"]).any(): + raise ValueError("Duplicate learner-event key in 2015 raw file; manual provenance audit needed") + if not np.isfinite(df["log_id"]).all(): + raise ValueError("Nonfinite chronology field") + if (df["log_id"] != df["log_id"].astype("int64")).any(): + raise ValueError("Nonintegral chronology ID") + df["correct"] = df["correct"].astype(int) + df["user_id"] = df["user_id"].astype(str) + df["sequence_id"] = df["sequence_id"].astype(str) + df = df.sort_values(["user_id", "log_id"], kind="stable") + unique_users = sorted(df["user_id"].unique()) + if len(unique_users) < N_SAMPLE: + raise ValueError(f"Need {N_SAMPLE} learners, found {len(unique_users)}") + sampled = set(random.Random(SEED).sample(unique_users, N_SAMPLE)) + chosen = df[df["user_id"].isin(sampled)] + rows = [ + {"user_id": str(uid), "skill_ids": group["sequence_id"].tolist(), + "grades": group["correct"].tolist()} + for uid, group in chosen.groupby("user_id", sort=True) + ] + assert kt.validate_unique_learner_rows(rows) == N_SAMPLE + assert all(len(r["grades"]) == len(r["skill_ids"]) for r in rows) + metadata = { + "official_portal": SOURCE_URL, + "official_linked_drive_file_id": GOOGLE_FILE_ID, + "raw_filename": csv_path.name, + "source_sha256": digest, + "source_total_rows": source_n, + "excluded_invalid_rows": bad_n, + "source_unique_eligible_learners": len(unique_users), + "sample_seed": SEED, + "sample_learners": N_SAMPLE, + "sample_interactions": int(sum(len(r["grades"]) for r in rows)), + "chronology": "ascending log_id within user_id", + "concept": "sequence_id (skill builder; not 2009 mirror skill_id)", + } + return rows, metadata + + +def evaluate(csv_path: Path) -> dict: + rows, dataset = load_rows(csv_path) + train, validation, test = kt.split_rows(rows, seed=SEED) + ids = [set(str(r["user_id"]) for r in block) for block in (train, validation, test)] + assert not (ids[0] & ids[1] or ids[0] & ids[2] or ids[1] & ids[2]) + global_prior, skill_prior, _ = kt.train_priors(train) + # These fixed protocols were written before seeing any 2015 test outcomes. + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always", ConvergenceWarning) + reference = stronger_pfa(train, test) + convergence_warnings = [str(x.message) for x in w if issubclass(x.category, ConvergenceWarning)] + vocab = kt.skill_vocabulary(train) + kt.GRU_PATIENCE = 4 + model, history, best_epoch = kt.train_gru(train, validation, vocab, seed=42, epochs=25) + candidate = kt.predict_gru(model, test, vocab, global_prior, skill_prior) + comparison = paired_cluster_bootstrap(reference, candidate, repetitions=BOOTS, seed=SEED) + metrics = {"enriched_pfa": kt.metric_bundle(reference["y"], reference["p"]), + "extended_gru": kt.metric_bundle(candidate["y"], candidate["p"])} + positive = comparison["bootstrap_95_interval"][1] < 0 + validity_gate = ( + not convergence_warnings + and len(train) > 0 and len(validation) > 0 and len(test) > 0 + and not (best_epoch == 25) + and np.isfinite(comparison["difference_candidate_minus_reference"]) + ) + result = { + "status": "completed" if validity_gate else "completed_with_unresolved_diagnostics", + "directional_replication_confirmed": bool(positive and validity_gate), + "test_is_external_cohort_not_prior_2009_test": True, + "source_integrity_scope": "Official-source URL/id and downloaded-byte SHA; no independent publisher checksum available", + "data": dataset, + "protocol": {"sample_learners": N_SAMPLE, "sample_seed": SEED, + "split_seed": SEED, + "train_learners": len(train), "validation_learners": len(validation), "test_learners": len(test), + "gru_seed": 42, "gru_max_epochs": 25, "gru_patience": 4, + "bootstrap_seed": SEED, "bootstrap_draws": BOOTS, + "predeclared_primary_estimand": "GRU - enhanced PFA pooled interaction-weighted Brier", + "directional_rule": "Upper endpoint of 95% percentile learner-cluster bootstrap CI < 0; must pass diagnostic gates"}, + "metrics": metrics, + "brier_comparison": comparison, + "gru_best_validation_epoch": int(best_epoch), + "gru_epochs_completed": len(history), + "pfa_convergence_warnings": convergence_warnings, + "conclusion_scope": "Different-year ASSISTments learner-subset replication. Not source parity for 2009 and not a causal attribution of architecture.", + } + return result + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--csv", default="data/cache/assistments_2015_official.csv") + parser.add_argument("--output", default=str(OUTPUT)) + args = parser.parse_args() + source = Path(args.csv) + download_official_csv(source) + result = evaluate(source) + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2), encoding="utf-8") + print(json.dumps({"status": result["status"], + "confirmed": result["directional_replication_confirmed"], + "n_test_learners": result["protocol"]["test_learners"], + "metrics": result["metrics"], + "brier_difference": result["brier_comparison"]}, indent=2)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/confirmatory_2015_stage2.py b/scripts/confirmatory_2015_stage2.py new file mode 100644 index 0000000..568d87e --- /dev/null +++ b/scripts/confirmatory_2015_stage2.py @@ -0,0 +1,152 @@ +"""Untouched 2015 learner cohort, disjoint from the nonconverged stage-1 pilot. + +Protocol frozen in CONFIRMATORY_STAGE2_PROTOCOL_2015.md before any stage-2 outcomes. +""" +from __future__ import annotations +import hashlib +import json +import random +import sys +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'src')) +from robustness_extension import paired_cluster_bootstrap, pfa_plus_all_skill_history_frame +from confirmatory_2015 import download_official_csv, GOOGLE_FILE_ID, SOURCE_URL +import run_experiment as kt + +FIRST_SEED=20261022 +SECOND_SEED=20261023 +SAMPLE_SIZE=5000 +SOURCE_SHA256='75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0' + + +def cohort_from_official_csv(path:Path): + checksum=hashlib.sha256(path.read_bytes()).hexdigest() + if checksum!=SOURCE_SHA256: + raise ValueError('Official ASSISTments-2015 source checksum changed since stage 1; stop for provenance audit') + df=pd.read_csv(path,low_memory=False) + df.columns=[str(c).strip().lower() for c in df.columns] + columns=['user_id','log_id','sequence_id','correct'] + if len(set(df.columns))!=len(df.columns) or any(c not in df for c in columns): + raise ValueError('Missing/nonunique required source column') + orig_n=len(df) + df=df[columns].copy() + missing=df.isna().any(axis=1) + df['log_id']=pd.to_numeric(df['log_id'],errors='coerce') + df['correct']=pd.to_numeric(df['correct'],errors='coerce') + invalid=missing|df.isna().any(axis=1)|~df['correct'].isin([0,1]) + exclusions=int(invalid.sum()) + if orig_n==0 or exclusions/orig_n>0.05: + raise ValueError('Invalid record rate exceeds frozen threshold') + df=df.loc[~invalid].copy() + if df.duplicated(['user_id','log_id']).any(): + raise ValueError('Duplicate learner-event identifiers') + if not np.isfinite(df['log_id']).all() or (df['log_id']!=df['log_id'].astype('int64')).any(): + raise ValueError('Malformed chronological log IDs') + df['correct']=df['correct'].astype(int) + df['user_id']=df['user_id'].astype(str) + df['sequence_id']=df['sequence_id'].astype(str) + df=df.sort_values(['user_id','log_id'],kind='stable') + users=sorted(df['user_id'].unique()) + stage1=set(random.Random(FIRST_SEED).sample(users,SAMPLE_SIZE)) + remaining=[u for u in users if u not in stage1] + if len(remaining)=1 and delta['n_learners']==len(test) + return {'status':'completed_exploratory_corrected_source' if guards else 'inconclusive_diagnostics', + 'analysis_type':'exploratory source-sensitivity, not confirmatory', + 'source':source, + 'protocol':{'split_seed':SEED,'gru_seed':42,'train_learners':len(train),'validation_learners':len(val), + 'test_learners':len(test),'pfa_solver':'lbfgs','pfa_C':1.0,'pfa_max_iter':2000,'pfa_tol':1e-8, + 'gru_max_epochs':25,'gru_patience':4,'bootstrap_resamples':2000,'bootstrap_seed':SEED}, + 'pfa_convergence_warnings':warnings_list,'pfa_iterations':pfa_n,'gru_epochs_completed':len(history), + 'gru_best_epoch':best, + 'metrics':{'enriched_pfa':kt.metric_bundle(pfa_pred['y'],pfa_pred['p']), + 'extended_gru':kt.metric_bundle(gru_pred['y'],gru_pred['p'])}, + 'paired_brier':delta, + 'interpretation':'Numbers are conditional on corrected source and selected splitting/representation. This is not a new untouched 2009 holdout or causal architecture effect.'} + +def main(): + out=Path('results/corrected_2009_sensitivity.json') + out.parent.mkdir(parents=True,exist_ok=True) + try:r=run() + except Exception as e: + r={'status':'blocked_or_failed','error_type':type(e).__name__, + 'diagnostic':str(e)[:700],'interpretation':'Never substitute expected results or claim success on a blocked source-corrected run.'} + out.write_text(json.dumps(r,indent=2),encoding='utf-8') + print(json.dumps(r,indent=2)) +if __name__=='__main__':main() \ No newline at end of file