diff --git a/.github/workflows/wca-assignment-retry.yml b/.github/workflows/wca-assignment-retry.yml new file mode 100644 index 0000000..c1d2040 --- /dev/null +++ b/.github/workflows/wca-assignment-retry.yml @@ -0,0 +1,128 @@ +name: Temporary WCA assignment retry + +on: + pull_request: + paths: + - .github/workflows/wca-assignment-retry.yml + +permissions: + contents: read + actions: read + +jobs: + retry: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Download initial scan + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh run download 30577152714 --repo coder13/Competitor-groups --name wca-assignment-scan-2026-07-30 --dir input + + - name: Retry rate-limited WCIF requests + shell: python + run: | + import csv, json, os, random, time, urllib.error, urllib.request + from collections import Counter + from concurrent.futures import ThreadPoolExecutor, as_completed + + API = "https://www.worldcubeassociation.org/api/v0" + HEADERS = {"User-Agent": "competitiongroups.com assignment usage analysis retry"} + OUT = "assignment-retry" + os.makedirs(OUT, exist_ok=True) + + with open("input/all_competitions.csv", newline="", encoding="utf-8") as source: + failed = [row for row in csv.DictReader(source) if row["wcif_status"] != "ok"] + print(f"Retrying {len(failed)} competitions", flush=True) + + def get_json(url, attempts=10): + last = None + for attempt in range(attempts): + try: + request = urllib.request.Request(url, headers=HEADERS) + with urllib.request.urlopen(request, timeout=45) as response: + return json.loads(response.read().decode(response.headers.get_content_charset() or "utf-8")) + except urllib.error.HTTPError as error: + last = error + if error.code in (404, 422): + raise + retry_after = error.headers.get("Retry-After") if error.headers else None + delay = float(retry_after) if retry_after and retry_after.isdigit() else min(3 * (2 ** attempt), 90) + time.sleep(delay + random.random()) + except (urllib.error.URLError, TimeoutError) as error: + last = error + time.sleep(min(2 ** attempt, 30) + random.random()) + raise last + + def scan(row): + cid = row["competition_id"] + result = { + "competition_id": cid, + "persons_total": 0, + "persons_with_assignments": 0, + "assignment_count": 0, + "competitor_assignment_count": 0, + "staff_assignment_count": 0, + "other_assignment_count": 0, + "assignment_codes": "", + "has_person_assignments": False, + "has_competitor_assignments": False, + "has_staff_assignments": False, + "wcif_status": "ok", + "inference": "no assignment data", + } + try: + wcif = get_json(f"{API}/competitions/{cid}/wcif/public") + persons = wcif.get("persons") or [] + result["persons_total"] = len(persons) + codes = Counter() + for person in persons: + assignments = person.get("assignments") or [] + result["persons_with_assignments"] += bool(assignments) + for assignment in assignments: + codes[assignment.get("assignmentCode") or "unknown"] += 1 + result["assignment_count"] = sum(codes.values()) + result["competitor_assignment_count"] = codes.get("competitor", 0) + result["staff_assignment_count"] = sum(v for k, v in codes.items() if k.startswith("staff-")) + result["other_assignment_count"] = result["assignment_count"] - result["competitor_assignment_count"] - result["staff_assignment_count"] + result["assignment_codes"] = ";".join(f"{k}:{v}" for k, v in sorted(codes.items())) + result["has_person_assignments"] = result["assignment_count"] > 0 + result["has_competitor_assignments"] = result["competitor_assignment_count"] > 0 + result["has_staff_assignments"] = result["staff_assignment_count"] > 0 + if result["has_person_assignments"]: + result["inference"] = "potential Competition Groups user: assignment data present" + except urllib.error.HTTPError as error: + result["wcif_status"] = f"http_{error.code}" + except Exception as error: + result["wcif_status"] = f"error:{type(error).__name__}" + time.sleep(0.35 + random.random() * 0.3) + return result + + results = [] + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(scan, row) for row in failed] + for index, future in enumerate(as_completed(futures), 1): + results.append(future.result()) + if index % 10 == 0 or index == len(futures): + print(f"Retried {index}/{len(futures)}", flush=True) + + fields = list(results[0]) if results else [] + with open(f"{OUT}/retry_results.csv", "w", newline="", encoding="utf-8") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader(); writer.writerows(sorted(results, key=lambda row: row["competition_id"])) + summary = { + "retried": len(results), + "successful": sum(row["wcif_status"] == "ok" for row in results), + "remaining_failures": dict(Counter(row["wcif_status"] for row in results if row["wcif_status"] != "ok")), + "with_assignments": sum(bool(row["has_person_assignments"]) for row in results), + } + with open(f"{OUT}/retry_summary.json", "w", encoding="utf-8") as output: + json.dump(summary, output, indent=2) + print(json.dumps(summary, indent=2), flush=True) + + - name: Upload retry results + uses: actions/upload-artifact@v4 + with: + name: wca-assignment-retry-2026-07-30 + path: assignment-retry/ + retention-days: 7 diff --git a/.github/workflows/wca-assignment-scan-v2.yml b/.github/workflows/wca-assignment-scan-v2.yml new file mode 100644 index 0000000..14a89d8 --- /dev/null +++ b/.github/workflows/wca-assignment-scan-v2.yml @@ -0,0 +1,160 @@ +name: Temporary WCA assignment scan v2 + +on: + pull_request: + paths: + - .github/workflows/wca-assignment-scan-v2.yml + +permissions: + contents: read + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Scan public WCIF assignment data + shell: python + run: | + import csv, json, os, time, urllib.error, urllib.parse, urllib.request + from collections import Counter + from concurrent.futures import ThreadPoolExecutor, as_completed + from datetime import date + + START = date.fromisoformat("2026-04-30") + END = date.fromisoformat("2026-07-30") + API = "https://www.worldcubeassociation.org/api/v0" + HEADERS = {"User-Agent": "competitiongroups.com assignment usage analysis"} + OUT = "assignment-scan-v2" + os.makedirs(OUT, exist_ok=True) + + def get_json(url, attempts=4): + last = None + for attempt in range(attempts): + try: + request = urllib.request.Request(url, headers=HEADERS) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read().decode(response.headers.get_content_charset() or "utf-8")) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as error: + last = error + if getattr(error, "code", None) in (404, 422): + raise + time.sleep(min(2 ** attempt, 8)) + raise last + + competitions = [] + page_signatures = set() + for page in range(1, 101): + query = urllib.parse.urlencode({"start": START.isoformat(), "end": END.isoformat(), "page": page}) + batch = get_json(f"{API}/competitions?{query}") + if not batch: + break + signature = tuple(item["id"] for item in batch) + if signature in page_signatures: + print(f"Duplicate page detected at page {page}; stopping pagination") + break + page_signatures.add(signature) + competitions.extend(batch) + print(f"Competition page {page}: {len(batch)}", flush=True) + if len(batch) < 25: + break + else: + raise RuntimeError("Competition pagination exceeded 100 pages") + + competitions = { + item["id"]: item for item in competitions + if START <= date.fromisoformat(item["start_date"]) <= END + } + competitions = sorted(competitions.values(), key=lambda item: (item["start_date"], item["id"])) + print(f"Competitions in final date range: {len(competitions)}", flush=True) + + def scan(item): + cid = item["id"] + row = { + "competition_id": cid, + "name": item.get("name", ""), + "start_date": item.get("start_date", ""), + "end_date": item.get("end_date", ""), + "country_iso2": item.get("country_iso2", ""), + "city": item.get("city", ""), + "cancelled": bool(item.get("cancelled_at")), + "persons_total": 0, + "persons_with_assignments": 0, + "assignment_count": 0, + "competitor_assignment_count": 0, + "staff_assignment_count": 0, + "other_assignment_count": 0, + "assignment_codes": "", + "has_person_assignments": False, + "has_competitor_assignments": False, + "has_staff_assignments": False, + "wcif_status": "ok", + "inference": "no assignment data", + "wca_url": item.get("url") or f"https://www.worldcubeassociation.org/competitions/{cid}", + "competitiongroups_url": f"https://www.competitiongroups.com/competitions/{cid}", + } + try: + wcif = get_json(f"{API}/competitions/{cid}/wcif/public") + persons = wcif.get("persons") or [] + row["persons_total"] = len(persons) + codes = Counter() + for person in persons: + assignments = person.get("assignments") or [] + row["persons_with_assignments"] += bool(assignments) + for assignment in assignments: + codes[assignment.get("assignmentCode") or "unknown"] += 1 + row["assignment_count"] = sum(codes.values()) + row["competitor_assignment_count"] = codes.get("competitor", 0) + row["staff_assignment_count"] = sum(v for k, v in codes.items() if k.startswith("staff-")) + row["other_assignment_count"] = row["assignment_count"] - row["competitor_assignment_count"] - row["staff_assignment_count"] + row["assignment_codes"] = ";".join(f"{k}:{v}" for k, v in sorted(codes.items())) + row["has_person_assignments"] = row["assignment_count"] > 0 + row["has_competitor_assignments"] = row["competitor_assignment_count"] > 0 + row["has_staff_assignments"] = row["staff_assignment_count"] > 0 + if row["has_person_assignments"]: + row["inference"] = "potential Competition Groups user: assignment data present" + except urllib.error.HTTPError as error: + row["wcif_status"] = f"http_{error.code}" + except Exception as error: + row["wcif_status"] = f"error:{type(error).__name__}" + return row + + rows = [] + with ThreadPoolExecutor(max_workers=20) as executor: + futures = [executor.submit(scan, item) for item in competitions] + for index, future in enumerate(as_completed(futures), 1): + rows.append(future.result()) + if index % 50 == 0 or index == len(futures): + print(f"Scanned {index}/{len(futures)}", flush=True) + + rows.sort(key=lambda item: (item["start_date"], item["competition_id"])) + assignment_rows = [row for row in rows if row["has_person_assignments"]] + fields = list(rows[0]) if rows else [] + for filename, output_rows in (("all_competitions.csv", rows), ("competitions_with_assignments.csv", assignment_rows)): + with open(f"{OUT}/{filename}", "w", newline="", encoding="utf-8") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader(); writer.writerows(output_rows) + + summary = { + "date_window": {"start": START.isoformat(), "end": END.isoformat(), "basis": "competition start_date"}, + "competitions_total": len(rows), + "competitions_cancelled": sum(row["cancelled"] for row in rows), + "competitions_with_assignments": len(assignment_rows), + "assignment_coverage_percent": round(100 * len(assignment_rows) / len(rows), 2) if rows else 0, + "competitions_with_competitor_assignments": sum(row["has_competitor_assignments"] for row in rows), + "competitions_with_staff_assignments": sum(row["has_staff_assignments"] for row in rows), + "total_person_assignments": sum(row["assignment_count"] for row in rows), + "wcif_failures": dict(Counter(row["wcif_status"] for row in rows if row["wcif_status"] != "ok")), + "by_country": dict(Counter(row["country_iso2"] for row in assignment_rows).most_common()), + "by_month": dict(sorted(Counter(row["start_date"][:7] for row in assignment_rows).items())), + } + with open(f"{OUT}/summary.json", "w", encoding="utf-8") as output: + json.dump(summary, output, indent=2, ensure_ascii=False) + print(json.dumps(summary, indent=2), flush=True) + + - name: Upload scan results + uses: actions/upload-artifact@v4 + with: + name: wca-assignment-scan-v2-2026-07-30 + path: assignment-scan-v2/ + retention-days: 7 diff --git a/.github/workflows/wca-assignment-scan.yml b/.github/workflows/wca-assignment-scan.yml new file mode 100644 index 0000000..4519283 --- /dev/null +++ b/.github/workflows/wca-assignment-scan.yml @@ -0,0 +1,194 @@ +name: Temporary WCA assignment scan + +on: + pull_request: + paths: + - .github/workflows/wca-assignment-scan.yml + +permissions: + contents: read + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Scan public WCIF assignment data + shell: python + run: | + import csv + import json + import os + import re + import time + import urllib.error + import urllib.parse + import urllib.request + from collections import Counter + from concurrent.futures import ThreadPoolExecutor, as_completed + from datetime import date + + START = date.fromisoformat("2026-04-30") + END = date.fromisoformat("2026-07-30") + API = "https://www.worldcubeassociation.org/api/v0" + HEADERS = {"User-Agent": "competitiongroups.com assignment usage analysis"} + OUT = "assignment-scan" + os.makedirs(OUT, exist_ok=True) + + def get(url, attempts=5): + last = None + for attempt in range(attempts): + try: + request = urllib.request.Request(url, headers=HEADERS) + with urllib.request.urlopen(request, timeout=45) as response: + return response.read(), response.headers.get_content_charset() or "utf-8" + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as error: + last = error + code = getattr(error, "code", None) + if code in (404, 422): + raise + time.sleep(min(2 ** attempt, 12)) + raise last + + competitions = [] + page = 1 + while True: + query = urllib.parse.urlencode({ + "start": START.isoformat(), + "end": END.isoformat(), + "page": page, + }) + body, charset = get(f"{API}/competitions?{query}") + batch = json.loads(body.decode(charset)) + if not batch: + break + competitions.extend(batch) + print(f"Competition page {page}: {len(batch)}") + if len(batch) < 25: + break + page += 1 + + # Enforce the requested range explicitly in case API boundary semantics change. + competitions = [ + competition for competition in competitions + if START <= date.fromisoformat(competition["start_date"]) <= END + ] + deduped = {competition["id"]: competition for competition in competitions} + competitions = sorted(deduped.values(), key=lambda item: (item["start_date"], item["id"])) + print(f"Competitions in final date range: {len(competitions)}") + + def scan(competition): + competition_id = competition["id"] + row = { + "competition_id": competition_id, + "name": competition.get("name", ""), + "start_date": competition.get("start_date", ""), + "end_date": competition.get("end_date", ""), + "country_iso2": competition.get("country_iso2", ""), + "city": competition.get("city", ""), + "cancelled": bool(competition.get("cancelled_at")), + "persons_total": 0, + "persons_with_assignments": 0, + "assignment_count": 0, + "competitor_assignment_count": 0, + "staff_assignment_count": 0, + "other_assignment_count": 0, + "assignment_codes": "", + "has_person_assignments": False, + "has_competitor_assignments": False, + "has_staff_assignments": False, + "competitiongroups_link_on_wca_page": False, + "wcif_status": "ok", + "inference": "no assignment data", + "wca_url": competition.get("url") or f"https://www.worldcubeassociation.org/competitions/{competition_id}", + "competitiongroups_url": f"https://www.competitiongroups.com/competitions/{competition_id}", + } + try: + body, charset = get(f"{API}/competitions/{competition_id}/wcif/public") + wcif = json.loads(body.decode(charset)) + persons = wcif.get("persons") or [] + row["persons_total"] = len(persons) + codes = Counter() + for person in persons: + assignments = person.get("assignments") or [] + if assignments: + row["persons_with_assignments"] += 1 + for assignment in assignments: + code = assignment.get("assignmentCode") or "unknown" + codes[code] += 1 + row["assignment_count"] = sum(codes.values()) + row["competitor_assignment_count"] = codes.get("competitor", 0) + row["staff_assignment_count"] = sum(count for code, count in codes.items() if code.startswith("staff-")) + row["other_assignment_count"] = row["assignment_count"] - row["competitor_assignment_count"] - row["staff_assignment_count"] + row["assignment_codes"] = ";".join(f"{code}:{count}" for code, count in sorted(codes.items())) + row["has_person_assignments"] = row["assignment_count"] > 0 + row["has_competitor_assignments"] = row["competitor_assignment_count"] > 0 + row["has_staff_assignments"] = row["staff_assignment_count"] > 0 + except urllib.error.HTTPError as error: + row["wcif_status"] = f"http_{error.code}" + except Exception as error: + row["wcif_status"] = f"error:{type(error).__name__}" + + if row["has_person_assignments"]: + try: + body, charset = get(row["wca_url"], attempts=3) + html = body.decode(charset, errors="replace") + row["competitiongroups_link_on_wca_page"] = bool( + re.search(r"(?:www\.)?competitiongroups\.com", html, re.I) + ) + except Exception: + pass + row["inference"] = ( + "direct WCA-page link and assignment data" + if row["competitiongroups_link_on_wca_page"] + else "potential Competition Groups user: assignment data present" + ) + return row + + rows = [] + with ThreadPoolExecutor(max_workers=8) as executor: + futures = {executor.submit(scan, competition): competition["id"] for competition in competitions} + for index, future in enumerate(as_completed(futures), 1): + rows.append(future.result()) + if index % 25 == 0 or index == len(futures): + print(f"Scanned {index}/{len(futures)}") + + rows.sort(key=lambda item: (item["start_date"], item["competition_id"])) + fields = list(rows[0].keys()) if rows else [] + with open(f"{OUT}/all_competitions.csv", "w", newline="", encoding="utf-8") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + assignment_rows = [row for row in rows if row["has_person_assignments"]] + with open(f"{OUT}/competitions_with_assignments.csv", "w", newline="", encoding="utf-8") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + writer.writerows(assignment_rows) + + summary = { + "date_window": {"start": START.isoformat(), "end": END.isoformat(), "basis": "competition start_date"}, + "competitions_total": len(rows), + "competitions_cancelled": sum(row["cancelled"] for row in rows), + "competitions_with_assignments": len(assignment_rows), + "assignment_coverage_percent": round(100 * len(assignment_rows) / len(rows), 2) if rows else 0, + "competitions_with_competitor_assignments": sum(row["has_competitor_assignments"] for row in rows), + "competitions_with_staff_assignments": sum(row["has_staff_assignments"] for row in rows), + "competitions_with_direct_wca_page_link": sum(row["competitiongroups_link_on_wca_page"] for row in rows), + "total_person_assignments": sum(row["assignment_count"] for row in rows), + "wcif_failures": Counter(row["wcif_status"] for row in rows if row["wcif_status"] != "ok"), + "by_country": Counter(row["country_iso2"] for row in assignment_rows), + "by_month": Counter(row["start_date"][:7] for row in assignment_rows), + } + summary["wcif_failures"] = dict(summary["wcif_failures"]) + summary["by_country"] = dict(summary["by_country"].most_common()) + summary["by_month"] = dict(sorted(summary["by_month"].items())) + with open(f"{OUT}/summary.json", "w", encoding="utf-8") as output: + json.dump(summary, output, indent=2, ensure_ascii=False) + print(json.dumps(summary, indent=2)) + + - name: Upload scan results + uses: actions/upload-artifact@v4 + with: + name: wca-assignment-scan-2026-07-30 + path: assignment-scan/ + retention-days: 7