From 68cc90e1ed176c32acba15b43a53accf3bef9ce5 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:28:37 +0700 Subject: [PATCH 1/4] research: add real-data robustness, richer PFA and paired learner bootstrap --- scripts/robustness_extension.py | 121 ++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 scripts/robustness_extension.py diff --git a/scripts/robustness_extension.py b/scripts/robustness_extension.py new file mode 100644 index 0000000..4682ae1 --- /dev/null +++ b/scripts/robustness_extension.py @@ -0,0 +1,121 @@ +"""Independent real-data robustness extension for the ASSISTments knowledge tracing benchmark. + +Execution: + PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 + +This reuses the pinned dataset adapter in src/run_experiment.py. The original test +partition has been inspected already; this analysis is EXPLORATORY, not independent +confirmation. No synthetic observations or fabricated experiment outcomes. +""" +from __future__ import annotations +import argparse +import json +import sys +from pathlib import Path +from collections import defaultdict +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +import run_experiment as kt + +def paired_cluster_bootstrap(reference, candidate, repetitions=2000, seed=20261022): + """Resample learners, preserve all responses, estimate pooled interaction-weighted Brier difference.""" + if repetitions < 200: + raise ValueError("Use at least 200 bootstrap repetitions") + for col in ("learner", "y", "skill"): + if not np.array_equal(reference[col].to_numpy(), candidate[col].to_numpy()): + raise ValueError("Misaligned prediction frames: " + col) + diff = (candidate["p"].to_numpy()-candidate["y"].to_numpy())**2 - (reference["p"].to_numpy()-reference["y"].to_numpy())**2 + records = pd.DataFrame({"learner":reference["learner"].astype(str).to_numpy(), "loss_diff":diff}) + grp = records.groupby("learner", sort=False)["loss_diff"].agg(["sum","count"]) + sums = grp["sum"].to_numpy(float) + counts = grp["count"].to_numpy(float) + rng = np.random.default_rng(seed) + sampled = np.empty(repetitions) + for i in range(repetitions): + index = rng.integers(0,len(sums),size=len(sums)) + sampled[i] = sums[index].sum()/counts[index].sum() + return {"difference_candidate_minus_reference":float(sums.sum()/counts.sum()), + "bootstrap_95_interval":[float(x) for x in np.quantile(sampled,[.025,.975])], + "resampling_unit":"learner", "estimand_unit":"interaction", + "n_learners":len(sums), "n_interactions":int(counts.sum()), "n_bootstrap":repetitions} + +def pfa_plus_all_skill_history_frame(rows): + features=[]; y=[]; users=[]; skills=[]; first=[] + for row in rows: + skill_success=defaultdict(int);skill_failure=defaultdict(int) + total_success=0;total_failure=0;seen=set() + sequence, outcomes=kt.aligned(row) + for s,response in zip(sequence,outcomes): + features.append({"skill":s,"skill_success":skill_success[s],"skill_failure":skill_failure[s], + "all_success":total_success,"all_failure":total_failure}) + y.append(response);users.append(str(row["user_id"]));skills.append(s);first.append(s not in seen) + if response: + skill_success[s]+=1;total_success+=1 + else: + skill_failure[s]+=1;total_failure+=1 + seen.add(s) + return pd.DataFrame(features),np.asarray(y,dtype=int),users,skills,first + +def stronger_pfa(train, test): + train_x,train_y,*_=pfa_plus_all_skill_history_frame(train) + test_x,test_y,users,skills,first=pfa_plus_all_skill_history_frame(test) + model=Pipeline([ + ("prep",ColumnTransformer([ + ("skill",OneHotEncoder(handle_unknown="ignore"),["skill"]), + ("history",StandardScaler(),["skill_success","skill_failure","all_success","all_failure"])])), + ("clf",LogisticRegression(solver="saga",max_iter=1000,C=1.0,random_state=42))]) + model.fit(train_x,train_y) + predictions=model.predict_proba(test_x)[:,1] + return kt.prediction_frame(test_y,predictions,users,skills,first) + +def run(args): + rows, data_record=kt.load_real_rows() + train, validation, test=kt.split_rows(rows,kt.SEED) + global_rate,skill_prior,_=kt.train_priors(train) + pfa=kt.predict_pfa(kt.fit_pfa(train,kt.SEED),test) + enhanced=stronger_pfa(train,test) + vocabulary=kt.skill_vocabulary(train) + kt.GRU_PATIENCE=args.patience + output={} + for seed in args.seeds: + model,history,best_epoch=kt.train_gru(train,validation,vocabulary,seed,epochs=args.epochs) + prediction=kt.predict_gru(model,test,vocabulary,global_rate,skill_prior) + output[str(seed)]={"best_epoch":best_epoch, "epochs_completed":len(history), + "best_epoch_at_training_cap":best_epoch==args.epochs, + "training_history":history, + "metrics":kt.metric_bundle(prediction["y"],prediction["p"]), + "first_and_repeated":kt.sliced_metrics(prediction), + "paired_vs_pfa":paired_cluster_bootstrap(pfa,prediction,args.boots,seed+100), + "paired_vs_stronger_pfa":paired_cluster_bootstrap(enhanced,prediction,args.boots,seed+200)} + record={"status":"complete_exploratory_real_data","dataset":data_record, + "qualification":"Prior final test data were already inspected; this is not an untouched holdout confirmation.", + "source_code":"scripts/robustness_extension.py", + "split_seed":kt.SEED, "train_learners":len(train),"validation_learners":len(validation),"test_learners":len(test), + "max_epochs":args.epochs,"patience":args.patience,"seeds":args.seeds, + "pfa":kt.metric_bundle(pfa["y"],pfa["p"]), + "stronger_pfa":kt.metric_bundle(enhanced["y"],enhanced["p"]), + "stronger_minus_original_pfa":paired_cluster_bootstrap(pfa,enhanced,args.boots), + "extended_gru":output} + path=Path(args.output) + path.parent.mkdir(parents=True,exist_ok=True) + path.write_text(json.dumps(record,indent=2),encoding="utf-8") + print(json.dumps({"output":str(path),"seed_results":{s:{"auc":v["metrics"]["roc_auc"],"brier":v["metrics"]["brier"], + "best_epoch":v["best_epoch"]} for s,v in output.items()}},indent=2)) + +if __name__=="__main__": + parser=argparse.ArgumentParser() + parser.add_argument("--epochs",type=int,default=25) + parser.add_argument("--patience",type=int,default=4) + parser.add_argument("--boots",type=int,default=2000) + parser.add_argument("--seeds",type=int,nargs="+",default=[13,42,73]) + parser.add_argument("--output",default="results/robustness_extension.json") + args=parser.parse_args() + if args.epochs<=8 or args.patience<2: parser.error("Set epochs > 8 and patience >= 2") + run(args) From 716f0fb31fad73f62a4452d779ddde7b52792756 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:29:07 +0700 Subject: [PATCH 2/4] ci: run full empirical reproducibility and robustness on GitHub Actions --- .github/workflows/arxiv-verification.yml | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/arxiv-verification.yml diff --git a/.github/workflows/arxiv-verification.yml b/.github/workflows/arxiv-verification.yml new file mode 100644 index 0000000..978ad43 --- /dev/null +++ b/.github/workflows/arxiv-verification.yml @@ -0,0 +1,67 @@ +name: arXiv verification: real-data robustness +on: + push: + branches: + - research/kt-arxiv-verification-20261022 + workflow_dispatch: +permissions: + contents: write +jobs: + verify: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned scientific packages + run: | + python -m pip install --upgrade pip + pip install -r requirements-repro.txt + pip install pytest + - name: Run repository unit tests + run: PYTHONPATH=src python -m pytest -q + - name: Reproduce original real-data experiment + run: PYTHONPATH=src python src/run_experiment.py + - name: Run enhanced baselines and extended GRU on actual data + run: PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 --seeds 13 42 73 + - name: Validate empirical outputs + run: | + python - <<'PY' + import json + from pathlib import Path + original=json.loads(Path("results/metrics.json").read_text()) + extension=json.loads(Path("results/robustness_extension.json").read_text()) + assert original["dataset"]["n_learners"]==extension["dataset"]["n_learners"] + assert original["dataset"]["n_interactions"]==extension["dataset"]["n_interactions"] + assert extension["status"]=="complete_exploratory_real_data" + for run in extension["extended_gru"].values(): + assert len(run["training_history"])>=1 + assert run["paired_vs_pfa"]["n_learners"]==623 + assert run["paired_vs_stronger_pfa"]["n_interactions"]==41228 + print("REAL-DATA VERIFICATION PASSED: numeric artifacts match sample identifiers") + PY + - name: Upload verification results + if: always() + uses: actions/upload-artifact@v4 + with: + name: kt-arxiv-verification + path: | + results/metrics.json + results/summary.md + results/robustness_extension.json + results/figures/ + if-no-files-found: warn + - name: Commit empirical records to verification branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/metrics.json results/summary.md results/figures/ results/robustness_extension.json + if git diff --cached --quiet; then + echo "No changes to empirical artifacts" + else + git commit -m "research: record rerun original and extended real-data results" + git push origin HEAD:research/kt-arxiv-verification-20261022 + fi From 13cab4779a1e5c34c7fed9ec3279f41ae618ffed Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:32:21 +0700 Subject: [PATCH 3/4] test: verify pooled learner bootstrap and leakage-free enriched PFA features --- tests/test_arxiv_robustness.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_arxiv_robustness.py diff --git a/tests/test_arxiv_robustness.py b/tests/test_arxiv_robustness.py new file mode 100644 index 0000000..9082967 --- /dev/null +++ b/tests/test_arxiv_robustness.py @@ -0,0 +1,35 @@ +"""Unit tests for manuscript robustness extension (no empirical datasets or synthetic study claims).""" +import importlib.util +import unittest +from pathlib import Path +import pandas as pd + +ROOT=Path(__file__).resolve().parents[1] +SPEC=importlib.util.spec_from_file_location("arxiv_robustness",ROOT/"scripts"/"robustness_extension.py") +MODULE=importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +class RobustnessTests(unittest.TestCase): + def test_cluster_bootstrap_preserves_interaction_weighted_target(self): + reference=pd.DataFrame({"learner":["a","a","a","b"],"y":[0,0,0,0], + "skill":["x","x","x","x"],"p":[0.,0.,0.,0.]}) + candidate=reference.copy() + candidate["p"]=[1.,1.,1.,0.] + result=MODULE.paired_cluster_bootstrap(reference,candidate,repetitions=300,seed=2) + self.assertAlmostEqual(result["difference_candidate_minus_reference"],.75) + self.assertEqual(result["n_learners"],2) + self.assertEqual(result["n_interactions"],4) + candidate.loc[0,"learner"]="different" + with self.assertRaises(ValueError): + MODULE.paired_cluster_bootstrap(reference,candidate,repetitions=300) + def test_cross_skill_history_uses_only_previous_outcomes(self): + rows=[{"user_id":"one","skill_ids":["x","x","y"],"grades":[1,0,1]}] + x,y,users,skills,first=MODULE.pfa_plus_all_skill_history_frame(rows) + self.assertEqual(y.tolist(),[1,0,1]) + self.assertEqual(x["all_success"].tolist(),[0,1,1]) + self.assertEqual(x["all_failure"].tolist(),[0,0,1]) + self.assertEqual(x["skill_success"].tolist(),[0,1,0]) + self.assertEqual(first,[True,False,True]) + +if __name__=="__main__": + unittest.main() From cab0ca88b821a7077e2976811fcc55b8952ec562 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:47:24 +0700 Subject: [PATCH 4/4] ci: execute full real-data reproduction and robustness in PR with downloadable evidence --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d202d70..b510abb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,50 @@ jobs: pip install -r requirements.txt pip install pytest - run: PYTHONPATH=src python -m pytest -q + + empirical-arxiv-check: + if: github.event_name == 'pull_request' && github.head_ref == 'research/kt-arxiv-verification-20261022' + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - name: Install reproducibility environment + run: | + python -m pip install --upgrade pip + pip install -r requirements-repro.txt + pip install pytest + - name: Rerun original experiment with real ASSISTments mirror + run: PYTHONPATH=src python src/run_experiment.py + - name: Execute extended GRU and richer PFA robustness + run: PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 --seeds 13 42 73 + - name: Validate outputs against real-data identifiers + run: | + python - <<'PY' + import json + from pathlib import Path + original=json.loads(Path("results/metrics.json").read_text()) + extension=json.loads(Path("results/robustness_extension.json").read_text()) + assert original["dataset"]["n_learners"]==extension["dataset"]["n_learners"]==4148 + assert original["dataset"]["n_interactions"]==extension["dataset"]["n_interactions"]==274331 + assert original["dataset"]["parquet_sha256"]==extension["dataset"]["parquet_sha256"] + assert extension["status"]=="complete_exploratory_real_data" + assert len(extension["extended_gru"])==3 + for run in extension["extended_gru"].values(): + assert run["paired_vs_pfa"]["n_learners"]==623 + assert run["paired_vs_stronger_pfa"]["n_interactions"]==41228 + print("Original + robustness real-data results: numerical validation passed") + PY + - name: Upload numerical evidence for manuscript update + if: always() + uses: actions/upload-artifact@v4 + with: + name: assistments-arxiv-reproduction + path: | + results/metrics.json + results/summary.md + results/robustness_extension.json + results/figures/ + if-no-files-found: warn