diff --git a/.cursor/rules/hatchet-docs.mdc b/.cursor/rules/hatchet-docs.mdc new file mode 100644 index 0000000..7738938 --- /dev/null +++ b/.cursor/rules/hatchet-docs.mdc @@ -0,0 +1,12 @@ +--- +description: Hatchet documentation MCP server +alwaysApply: true +--- + +When working with Hatchet (task queues, workflows, durable execution), use the +Hatchet MCP docs server for accurate, up-to-date API reference and examples. + +MCP server URL: https://docs.hatchet.run/api/mcp + +Use the search_docs tool to find relevant documentation pages, or get_full_docs +for comprehensive context. Documentation covers Python, TypeScript, and Go SDKs. diff --git a/.env.scythe.training b/.env.scythe.training new file mode 100644 index 0000000..d2e3634 --- /dev/null +++ b/.env.scythe.training @@ -0,0 +1,9 @@ +SCYTHE_WORKER_SLOTS=1 +SCYTHE_WORKER_DOES_FAN=False +SCYTHE_WORKER_DOES_LEAF=True +SCYTHE_WORKER_HAS_GPU=True + +SCYTHE_TIMEOUT_EXPERIMENT_SCHEDULE=2h +SCYTHE_TIMEOUT_EXPERIMENT_EXECUTION=1h +SCYTHE_TIMEOUT_SCATTER_GATHER_SCHEDULE=10h +SCYTHE_TIMEOUT_SCATTER_GATHER_EXECUTION=10h diff --git a/.github/.DS_Store b/.github/.DS_Store new file mode 100644 index 0000000..4e72eb9 Binary files /dev/null and b/.github/.DS_Store differ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 740628c..b11a228 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Generate Prisma client + run: uv run epi prisma generate + - name: Check typing run: uv run pyright @@ -64,5 +67,12 @@ jobs: - name: Set up the environment uses: ./.github/actions/setup-python-env + - name: Check out globi-data (for the generated data pages) + uses: actions/checkout@v4 + continue-on-error: true + with: + repository: MITSustainableDesignLab/globi-data + path: .globi-data + - name: Check if documentation can be built run: uv run mkdocs build -s diff --git a/.github/workflows/on-release-main.yml b/.github/workflows/on-release-main.yml index a8acee8..540fe2c 100644 --- a/.github/workflows/on-release-main.yml +++ b/.github/workflows/on-release-main.yml @@ -80,5 +80,12 @@ jobs: restore-keys: | mkdocs-material- + - name: Check out globi-data (for the generated data pages) + uses: actions/checkout@v4 + continue-on-error: true + with: + repository: MITSustainableDesignLab/globi-data + path: .globi-data + - name: Deploy documentation run: uv run mkdocs gh-deploy --force diff --git a/.gitignore b/.gitignore index b31cb9f..1cfa5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -217,3 +217,6 @@ inputs/ .env.local.hatchet .env.local.host.hatchet + +scratch/ +.globi-data/ diff --git a/Makefile b/Makefile index 87de19d..fa38b10 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,11 @@ docs-test: ## Test if documentation can be built without warnings or errors docs: ## Build and serve the documentation @uv run mkdocs serve +.PHONY: docs-internal +docs-internal: ## Serve the docs plus internal pages from GLOBI_INTERNAL_DOCS (never deploy these) + @test -n "$(GLOBI_INTERNAL_DOCS)" || (echo "Set GLOBI_INTERNAL_DOCS to the private discussion folder" && exit 1) + @uv run mkdocs serve + .PHONY: docs-deploy docs-deploy: ## Build and serve the documentation @uv run mkdocs gh-deploy @@ -71,10 +76,30 @@ simulations-native: ## Run the simulations fanouts-native: ## Run the fanouts @uv run --env-file .env.$(AWS_ENV).aws --env-file .env.$(HATCHET_ENV).hatchet --env-file .env.scythe.storage --env-file .env.scythe.fanouts worker +.PHONY: training-native +training-native: ## Run the training + @uv run --env-file .env.$(AWS_ENV).aws --env-file .env.$(HATCHET_ENV).hatchet --env-file .env.scythe.storage --env-file .env.scythe.training worker + .PHONY: viz-native viz-native: ## Run the visualization tool # TODO: possibly add env vars to the command @uv run streamlit run src/globi/tools/visualization/main.py +##################### Local Surrogate Testing (no Hatchet / S3) ##################### +SURROGATE_CONFIG ?= tests/data/surrogate_local/validation-dummy.yml +SURROGATE_WORKERS ?= 1 + +.PHONY: surrogate-local +surrogate-local: ## Train surrogate(s) locally, in-process. SURROGATE_CONFIG=... SURROGATE_WORKERS=N + @uv run globi tests surrogate-local --config $(SURROGATE_CONFIG) --workers $(SURROGATE_WORKERS) + +.PHONY: surrogate-validate +surrogate-validate: ## Validate trained surrogate(s) against deterministic EnergyPlus runs + @uv run globi tests surrogate-validate --config $(SURROGATE_CONFIG) --workers $(SURROGATE_WORKERS) + +.PHONY: test-energyplus +test-energyplus: ## Run the pytest suite including the EnergyPlus-backed surrogate test + @uv run pytest --run-energyplus + .PHONY: env-debug env-debug: ## Merge env files into .env.debug for the VS Code/Cursor debugger @uv run python scripts/merge_env_for_debug.py diff --git a/docker-compose.yml b/docker-compose.yml index c5a1d8a..a66b0b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,31 @@ services: - ./inputs:/code/inputs - ./outputs:/code/outputs - ./tests/data/e2e:/code/tests/data/e2e + training: + image: ${AWS_ACCOUNT_ID:-123456789012}.dkr.ecr.${AWS_REGION:-us-east-1}.amazonaws.com/hatchet/globi:${IMAGE_TAG:-latest} + build: + context: . + dockerfile: src/globi/worker/Dockerfile + args: + EP_VERSION: ${EP_VERSION:-25.2.0} + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + env_file: + - .env + - .env.${AWS_ENV:-local}.aws + - .env.${HATCHET_ENV:-local}.hatchet + - .env.scythe.storage + - .env.scythe.training + deploy: + mode: replicated + replicas: ${TRAINING_REPLICAS:-0} + resources: + reservations: + devices: + - capabilities: [gpu] # Requests access to all GPUs + volumes: + - ./inputs:/code/inputs + - ./outputs:/code/outputs + - ./tests/data/e2e:/code/tests/data/e2e fanouts: image: ${AWS_ACCOUNT_ID:-123456789012}.dkr.ecr.${AWS_REGION:-us-east-1}.amazonaws.com/hatchet/globi:${IMAGE_TAG:-latest} build: diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 0000000..7bc41ff Binary files /dev/null and b/docs/.DS_Store differ diff --git a/docs/data/index.md b/docs/data/index.md new file mode 100644 index 0000000..af5f4ed --- /dev/null +++ b/docs/data/index.md @@ -0,0 +1,86 @@ +# Data & outputs + +globi runs the simulations. Their input files and the records of their results +are kept in a separate repository, +[globi-data](https://github.com/MITSustainableDesignLab/globi-data). This +section explains how that repository is organized and shows what is in it. +Most pages here are **generated from globi-data at build time**, so they show +the data as it stands rather than a copy that can drift. + +## Where everything lives + +```mermaid +flowchart LR + subgraph src["Partner source data (private)"] + cfg["partner working files"] + gis[("building footprints
(GIS, ~1 GB each)")] + end + subgraph gd["globi-data (git)"] + inp["inputs/partners/<Partner>/vNNN
frozen input bundles"] + out["outputs/runs/…/vNNN
output-version.yml"] + sur["outputs/surrogates/…/vNNN
surrogate-version.yml"] + end + subgraph s3["S3 · ml-for-bem"] + pq[("final/*.pq
~11 GB of results")] + mdl[("surrogate model")] + end + globi(["globi
simulation code"]) + + cfg -- "globi-data inputs new" --> inp + gis -. "size + sha256 only" .-> inp + inp --> globi + globi -- "EnergyPlus runs" --> pq + pq -. "where + fingerprints" .-> out + inp -. "pinned by sha256" .-> out + globi -- "training" --> mdl + mdl -. "where + metrics" .-> sur + inp -. "pinned by sha256" .-> sur +``` + +Solid arrows move files. Dashed arrows record facts about files that stay +where they are. + +Two things decide where a file lives: whether it may be published, and how +big it is. Building footprints are partner data and are not published. +Simulation results are too large for git. For both, globi-data records a +fingerprint (sha256), which is enough to prove later that a result came from +particular inputs without publishing the inputs themselves. + +| What | Stored in | Size | Versioned by | +| ----------------------------------------------------------------------------------- | -------------------- | -------------------- | ----------------------------------------------------------------- | +| Input bundles (component library, semantic fields, component map, weather, configs) | globi-data `inputs/` | a few MB per partner | `inputs/partners//vNNN/` | +| Building footprints (GIS) | not published | ~1 GB per partner | size and sha256 recorded in each input version's `external_files` | +| Simulation results (parquet) | S3 | ~11 GB in total | recorded in `outputs/runs/…/vNNN/output-version.yml` | +| Trained surrogate models | S3 | varies | recorded in `outputs/surrogates//vNNN/` | + +## Pages in this section + +- [**Output structure**](outputs-structure.md) explains how results are + organized, what each output record contains, and how results are marked + stale. +- [**Run catalog**](runs/index.md) lists every recorded run, with a coverage + matrix, a timeline, and a page per partner. +- [**Results status**](status.md) shows which results still match the current + inputs. +- [**Surrogate models**](surrogates.md) lists the trained surrogates and their + feature contracts. +- [**Input bundles**](inputs.md) shows the current input version for each + partner. +- **Reference** holds globi-data's own manifest schema and how-to guides, + pulled in unchanged. + +## Building these pages + +The generated pages come from a local checkout of globi-data. The build looks +for one at `$GLOBI_DATA_ROOT`, then `../globi-data` and `../../globi-data` +(relative to `mkdocs.yml`), then `.globi-data`: + +```bash +export GLOBI_DATA_ROOT=~/globi-data +make docs # uv run mkdocs serve +``` + +If no checkout is found, the build still succeeds and each generated page says +the data was missing. Set `GLOBI_DATA_REQUIRED=1` to make that an error +instead. The generator is +[`mkdocs_hooks/globi_data.py`](https://github.com/mitsustainabledesignlab/globi/blob/main/mkdocs_hooks/globi_data.py). diff --git a/docs/data/outputs-structure.md b/docs/data/outputs-structure.md new file mode 100644 index 0000000..67395e1 --- /dev/null +++ b/docs/data/outputs-structure.md @@ -0,0 +1,311 @@ +# Output structure + +Every simulation run leaves two things behind: + +1. **The results.** These are parquet files in S3, written by scythe under a + run prefix. +2. **A record of the results.** This is a small YAML file in globi-data. It + says where the parquet files are and exactly which inputs and code produced + them. + +This page covers the record: how it is laid out, what it contains, and how it +is used to tell whether results are still current. + +## The directory tree + +``` +outputs/ +├── README.md ← generated catalog +├── STATUS.md ← generated: which results are stale +├── index.yml ← generated: machine-readable catalog +├── runs/ +│ └── / +│ └── / ← tmy | ftmy +│ └── / ← e.g. Baseline, Retrofit, BAU, withAC +│ ├── CURRENT.yml ← which version is current (mutable) +│ ├── versions.yml ← status of every version (mutable) +│ └── v001/ ← frozen once published +│ ├── output-version.yml +│ └── README.md +├── batches/ +│ └── /vNNN/ ← run families too big to list one by one +└── surrogates/ + └── / + ├── versions.yml + └── vNNN/ + ├── surrogate-version.yml + └── transforms.yml +``` + +Each level of `runs/` is one axis of the experiment design: + +| Level | Meaning | Values today | +| ------------ | ----------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `` | A region, with its own input bundle | Ahmedabad, Lisbon, Manchester, Nairobi, Sao_Paulo_Bela_Vista, Singapore, Trondheim | +| `` | Which weather file drove the run | `tmy` (current climate), `ftmy` (future, 2050) | +| `` | The partner's scenario, as named in its experiment manifest | one `baseline` and one `alternative` per partner | +| `vNNN` | The globi-data version of this record | `v001` for everything backfilled so far | + +`//` is called the **experiment**, and it matches +the experiment name used when the run was submitted. Every experiment has its own +`CURRENT.yml` and `versions.yml`, so a single run can be re-done or retracted +without touching its neighbors. + +!!! info "Why weather is a directory for runs but not for surrogates" +A simulation run uses exactly one weather file, so weather is one of the +axes that tell runs apart. A surrogate takes weather as a model _input_ +and covers current and future climate in one model, so +`outputs/surrogates//vNNN/` has no `` level. + +## Mutable pointers, frozen versions + +The same rule applies to inputs, outputs and surrogates. A `vNNN/` directory +is never edited after it is published. Everything that can change sits next +to the version directories, not inside them. + +```mermaid +flowchart LR + cur["CURRENT.yml
current: v002"] --> v2 + ver["versions.yml
v001: superseded
v002: active"] + subgraph frozen["frozen (listed in published.yml)"] + v1["v001/"] + v2["v002/"] + end + ver -.-> v1 + ver -.-> v2 +``` + +`CURRENT.yml` is a file rather than a symlink, because symlinks break on +Windows checkouts and show up as a 20-byte text blob on GitHub. A consumer +resolves versions in this order: an explicit `vNNN`, then `CURRENT.yml`, then +the highest version present (with a warning). + +A version's `status` in `versions.yml` follows this lifecycle: + +```mermaid +stateDiagram-v2 + [*] --> draft: created on a branch + draft --> active: merged (published) + active --> superseded: a newer version becomes current + active --> retracted: globi-data … retract + superseded --> retracted + retracted --> [*] +``` + +Retracting a version only changes `versions.yml`. The frozen directory stays +readable, so anything that pinned it can still be traced. + +## From an output record to the files in S3 + +The record points into S3. It does not reconstruct the location from +configuration. The path in S3 mirrors the experiment name, followed by two +levels that scythe adds: + +```text +s3://ml-for-bem/globi-partners/Lisbon/tmy/Baseline/v1.0.0/2026-06-08_13-37-12/final/EnergyAndPeak.pq + └───┬────┘ └─────┬──────┘ └────────┬────────┘ └─┬──┘ └────────┬────────┘ └─────┬─────┘ + bucket │ experiment │ run_timestamp │ + bucket_prefix version result_key +``` + +| Segment | Field in `output-version.yml` | Set by | +| --------------------- | ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `ml-for-bem` | `provenance.storage.bucket` | scythe storage config | +| `globi-partners` | `provenance.storage.bucket_prefix` | parsed from the observed URI | +| `Lisbon/tmy/Baseline` | `experiment` | the experiment name, same as the directory in globi-data | +| `v1.0.0` | `version` / `provenance.run.scythe_version` | scythe, per submitted run | +| `2026-06-08_13-37-12` | `run_timestamp` | scythe, when the run was submitted | +| `final/.pq` | `result_files[].s3_uri` | one parquet per result key: `EnergyAndPeak`, `EnergyAndPeakAnnual`, `BasicOverheating`, `ExceedanceDegreeHours`, `HeatIndexCategories` | + +Everything up to the timestamp is `provenance.storage.run_prefix`. + +!!! warning "Two different \"versions\"" +In `output-version.yml`, **`version`** is the _scythe run version_ +(`v1.0.0`), part of the S3 path. The globi-data version of the record is +**`output_version`** (`v001`). The field name `version` is fixed for +compatibility with the legacy `latest-results --yaml-dir` export. + +!!! warning "`bucket_prefix` comes from the URI, never the environment" +The recorded URIs use `globi-partners`, while the run environment +currently sets `SCYTHE_STORAGE_BUCKET_PREFIX=globi`. A URI rebuilt from the environment +would point to the wrong place, so the prefix is always parsed from the +URI that was actually observed. + +## Anatomy of `output-version.yml` + +The file has three parts: a block kept for compatibility with existing tools, +the version envelope, and the provenance. + +```yaml +schema_version: 1 +kind: output-version + +# ── legacy-compatible block: what `latest-results --yaml-dir` writes ── +experiment: Lisbon/tmy/Baseline +version: v1.0.0 # (1)! +run_timestamp: 2026-06-08_13-37-12 +results: # (2)! + EnergyAndPeak: s3://ml-for-bem/globi-partners/Lisbon/tmy/Baseline/v1.0.0/2026-06-08_13-37-12/final/EnergyAndPeak.pq + # …one entry per result key + +# ── version envelope ── +output_version: v001 # (3)! +parent_version: null +status: active +summary: Backfilled from latest-results.csv; first recorded results for this experiment. +partner: Lisbon +weather: tmy +scenario_kind: baseline # (4)! +scenario_value: Baseline + +# ── provenance: what produced these numbers ── +provenance: + input: + version: v001 + manifest_path: inputs/partners/Lisbon/v001/input-version.yml + manifest_sha256: 2a587ca3… # (5)! + bundle_sha256: f6bb368c… + confidence: inferred # (6)! + gis_file: { sha256: 9a544bc4…, bytes: 1129469887 } + code: + globi: { commit: c2d84e47…, describe: v0.2.0-96-gc2d84e4 } + scythe_engine: null + python: 3.13.12 + storage: + bucket: ml-for-bem + bucket_prefix: globi-partners + run_prefix: s3://ml-for-bem/globi-partners/Lisbon/tmy/Baseline/v1.0.0/2026-06-08_13-37-12 + run: + scythe_version: v1.0.0 + run_timestamp: 2026-06-08_13-37-12 + hatchet_workflow_run_id: null # (7)! + +result_files: # (8)! + - result_key: EnergyAndPeak + s3_uri: s3://…/final/EnergyAndPeak.pq + bytes: null + sha256: null + sha256_source: none +totals: { file_count: 5, bytes: 0 } +``` + +1. The **scythe run version**, part of the S3 path. It is not the globi-data version. +2. A flat `key → URI` map, identical to what existing consumers read. +3. The globi-data version of this record, matching the `vNNN/` directory it lives in. +4. `baseline` or `alternative`. `scenario_value` is the partner's own name for the scenario. +5. The sha256 of the pinned `input-version.yml`'s bytes. If that file changes, these results are flagged **tampered**. +6. `exact` means recorded when the run was submitted, `inferred` means assumed during backfill, and `unknown` means nothing was pinned. `inferred` is never upgraded to `exact` without evidence. +7. Filled in for runs recorded when they are submitted. Backfilled records leave it null. +8. One entry per parquet file. `sha256_source: computed` means the bytes were hashed. `none` means the fingerprint has not been computed yet. + +### What is pinned, and how + +An output record points at an input version, and the input version points at +its files. Each link is a hash, so any change along the chain can be detected: + +```mermaid +flowchart TB + o["output-version.yml
Lisbon/tmy/Baseline v001"] + m["input-version.yml
Lisbon v001"] + f1["components-lib.db"] + f2["semantic-fields.yaml"] + f3["component-map.yaml"] + f4["weather/*.zip"] + g[("building footprints
(not published)")] + c["globi commit"] + r[("S3 run prefix
final/*.pq")] + + o -- "manifest_sha256" --> m + o -- "bundle_sha256" --> m + m -- "sha256 per file" --> f1 & f2 & f3 & f4 + m -- "sha256 + bytes" --> g + o -- "commit" --> c + o -- "s3_uri (+ sha256 when computed)" --> r +``` + +- **A file's `sha256`** detects an edit to that file. +- **`bundle_sha256`** covers every simulation-affecting file together. It is + the bundle's content identity, and it leaves out READMEs, so fixing + documentation cannot invalidate results. +- **`manifest_sha256`** covers the bytes of `input-version.yml` and is what + an output record pins. + +## Staleness: is this result still current? + +`globi-data status` checks every experiment's current output version and +writes the verdict to `outputs/STATUS.md`. The checks run in this order, and +the first one that matches decides the verdict: + +```mermaid +flowchart TD + A{"Input version
recorded?"} -- no --> U["⚪ unknown"] + A -- yes --> B{"Pinned version
still exists?"} + B -- no --> BP["🔴 broken"] + B -- yes --> C{"Pinned files and manifest
match their hashes?"} + C -- no --> T["🔴 tampered"] + C -- yes --> D{"Pinned version is
the partner's current?"} + D -- no --> E{"Any later version
simulation-affecting?"} + E -- yes --> S["🔴 stale · re-run"] + E -- no --> MD["🔵 docs changed only"] + D -- yes --> F{"globi commit differs
from the run's?"} + F -- yes --> CD["🟡 code changed"] + F -- no --> G{"confidence ==
inferred?"} + G -- yes --> I["⚪ assumed"] + G -- no --> OK["🟢 up to date"] +``` + +The `impact` field on each input version makes this work. A version that only +touches documentation is classified `metadata-only`, and moving past it never +marks results stale. Without that field, a typo fix in a README would flag +every experiment for a re-run. + +The current verdicts are on the [Results status](status.md) page. + +## Batches + +Some run families have too many members to give each its own directory. MA, +for example, allocates about 355 building specs to each of about 1,600 census +tracts. These use `outputs/batches//vNNN/` with the same version +envelope and provenance, but `results:` is replaced by a `result_index` that +points at a `results.csv`, plus an `allocation` block that describes how specs +were split. + +## Surrogates + +A trained surrogate is recorded like a run, with two differences. Its record, +`surrogate-version.yml`, describes a **model contract** rather than a list of +result files. And the contract itself, `transforms.yml`, is kept in git, where +anyone can read it without credentials: + +| Part | Where | What it says | +| --------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `model` | S3 (`model.s3_uri`) | backend (`xgb`/`lgb`/`nn`), format, hyperparameters | +| `transforms.yml` | git, pinned by sha256 | ordered features, scalers, the category list for every categorical feature, target names | +| `features` | manifest | each input's source: `semantic`, `weather`, `geometry` or `extra_space`, plus a default where the user cannot supply it | +| `weather_options` | manifest | each weather category mapped to a file in the pinned input bundle | +| `metrics` | manifest | per-target R², CV(RMSE) and NMBE on held-out folds | +| `provenance.training` | manifest | experiment id, earlier iterations, sampler, sample count | + +`globi-data check` cross-checks these parts. Features must match +`transforms.yml`. Every trained category must be a valid option in the pinned +semantic fields. Every weather option must resolve to a file in the bundle. +See [Surrogate models](surrogates.md) for what is recorded today. + +## Common tasks + +```bash +globi-data outputs list # every experiment +globi-data outputs show Lisbon/tmy/Baseline # its current record +globi-data outputs download Lisbon/tmy/Baseline \ + --dest ./results # fetch the parquet (needs AWS credentials) +globi-data outputs verify --all # confirm the S3 objects still exist +globi-data status # recompute STATUS.md +``` + +!!! note "A URI is a record, not a guarantee" +A URI in an output record says where the files _were_. Run +`globi-data outputs verify ` before relying on an object still being +there. + +For the full field-by-field reference, see +[Manifest schema](reference/schema.md). diff --git a/docs/stylesheets/globi-data.css b/docs/stylesheets/globi-data.css new file mode 100644 index 0000000..ca020aa --- /dev/null +++ b/docs/stylesheets/globi-data.css @@ -0,0 +1,344 @@ +/* Charts and tables on the generated "Data & outputs" pages. */ + +:root { + --gd-surface: #fcfcfb; + --gd-ink: #0b0b0b; + --gd-ink-2: #52514e; + --gd-muted: #898781; + --gd-grid: #e1e0d9; + --gd-axis: #c3c2b7; + --gd-border: rgba(11, 11, 11, 0.1); + --gd-series-1: #2a78d6; + --gd-series-2: #eb6834; + --gd-good: #0ca30c; + --gd-info: #6da7ec; + --gd-warning: #fab219; + --gd-critical: #d03b3b; + --gd-neutral: #898781; +} + +[data-md-color-scheme="slate"] { + --gd-surface: #1a1a19; + --gd-ink: #ffffff; + --gd-ink-2: #c3c2b7; + --gd-grid: #2c2c2a; + --gd-axis: #383835; + --gd-border: rgba(255, 255, 255, 0.1); + --gd-series-1: #3987e5; + --gd-series-2: #d95926; +} + +/* Stat tiles */ +.gd-tiles { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); + gap: 0.75rem; + margin: 1rem 0 1.5rem; +} +.gd-tile { + border: 1px solid var(--gd-border); + border-radius: 0.4rem; + padding: 0.7rem 0.9rem; +} +.gd-tile__label { + font-size: 0.7rem; + color: var(--gd-ink-2); +} +.gd-tile__value { + font-size: 1.6rem; + font-weight: 600; + line-height: 1.3; + color: var(--gd-ink); +} +.gd-tile__sub { + font-size: 0.65rem; + color: var(--gd-muted); +} + +/* Status chips: the icon and label carry the meaning, color supports it. */ +.gd-chip { + display: inline-block; + white-space: nowrap; + font-size: 0.68rem; + padding: 0 0.45em; + border-radius: 1em; + border: 1px solid var(--gd-border); + color: var(--gd-ink); +} +.gd-chip > span { + font-size: 0.8em; +} +.gd-chip--good > span { + color: var(--gd-good); +} +.gd-chip--info > span { + color: var(--gd-info); +} +.gd-chip--warning > span { + color: var(--gd-warning); +} +.gd-chip--critical > span { + color: var(--gd-critical); +} +.gd-chip--neutral > span { + color: var(--gd-neutral); +} + +/* Figures */ +.md-typeset figure.gd-figure { + display: block; + width: 100%; + max-width: 48rem; + margin: 1rem 0 1.5rem; +} +.gd-figure figcaption { + font-size: 0.72rem; + color: var(--gd-ink-2); + text-align: left; + margin-bottom: 0.4rem; + max-width: none; +} +.gd-chart { + width: 100%; + height: auto; + max-width: 48rem; + font-family: var(--md-text-font-family, system-ui, sans-serif); +} +.gd-chart .gd-grid { + stroke: var(--gd-grid); + stroke-width: 1; +} +.gd-chart .gd-baseline { + stroke: var(--gd-axis); + stroke-width: 1; +} +.gd-chart .gd-axis { + fill: var(--gd-muted); + font-size: 11px; + font-variant-numeric: tabular-nums; +} +.gd-chart .gd-label { + fill: var(--gd-ink-2); + font-size: 12px; +} +.gd-chart .gd-value { + fill: var(--gd-ink); + font-size: 12px; + font-variant-numeric: tabular-nums; +} +.gd-chart .gd-dot { + stroke: var(--md-default-bg-color, var(--gd-surface)); + stroke-width: 2; + cursor: pointer; +} +.gd-chart .gd-dot:hover { + r: 7; +} +.gd-chart .gd-series-1 { + fill: var(--gd-series-1); +} +.gd-chart .gd-series-2 { + fill: var(--gd-series-2); +} +.gd-chart .gd-bar:hover { + opacity: 0.8; +} + +.gd-legend { + display: flex; + flex-wrap: wrap; + gap: 0.4rem 1.1rem; + font-size: 0.7rem; + color: var(--gd-ink-2); + margin-bottom: 0.5rem; +} +.gd-legend b { + color: var(--gd-ink); + font-weight: 600; +} +.gd-key { + display: inline-block; + width: 0.65rem; + height: 0.65rem; + border-radius: 50%; + margin-right: 0.35rem; + vertical-align: -0.05rem; +} +.gd-key.gd-series-1 { + background: var(--gd-series-1); +} +.gd-key.gd-series-2 { + background: var(--gd-series-2); +} + +/* 100% status bar: 2px surface gap between segments */ +.gd-stack { + display: flex; + gap: 2px; + height: 1.1rem; + max-width: 48rem; +} +.gd-seg { + border-radius: 2px; + min-width: 4px; +} +.gd-status-good { + background: var(--gd-good); +} +.gd-status-info { + background: var(--gd-info); +} +.gd-status-warning { + background: var(--gd-warning); +} +.gd-status-critical { + background: var(--gd-critical); +} +.gd-status-neutral { + background: var(--gd-neutral); +} + +/* Coverage matrix */ +.gd-scroll { + overflow-x: auto; +} +.md-typeset table.gd-matrix { + border-collapse: collapse; + font-size: 0.72rem; + display: table; +} +.md-typeset table.gd-matrix th, +.md-typeset table.gd-matrix td { + border: 1px solid var(--gd-border); + padding: 0.45rem 0.6rem; + vertical-align: top; +} +.md-typeset table.gd-matrix thead th { + text-align: center; +} +.md-typeset table.gd-matrix tbody th { + text-align: left; + white-space: nowrap; +} +.md-typeset table.gd-matrix th small { + color: var(--gd-muted); + font-weight: 400; +} +.md-typeset table.gd-matrix td.gd-empty { + color: var(--gd-muted); + text-align: center; +} + +.gd-source { + font-size: 0.68rem; + color: var(--gd-muted); + margin-top: 2rem; +} + +/* Publishing options page */ +.gd-chart .po-range { + fill: var(--gd-series-1); + fill-opacity: 0.3; +} +.gd-chart .po-mid { + fill: var(--gd-series-1); + stroke: var(--md-default-bg-color, var(--gd-surface)); + stroke-width: 2; +} + +.po-flow { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: 1.6rem; + margin: 1rem 0 1.2rem; +} +.po-step { + position: relative; + border: 1px solid var(--gd-border); + border-radius: 0.4rem; + padding: 0.6rem 0.8rem; + font-size: 0.72rem; + line-height: 1.5; +} +.po-step--us { + border-color: var(--gd-series-1); + border-width: 2px; +} +.po-step:not(:last-child)::after { + content: "→"; + position: absolute; + right: -1.25rem; + top: 50%; + transform: translateY(-50%); + color: var(--gd-muted); + font-size: 1rem; +} +@media (max-width: 45em) { + .po-flow { + grid-template-columns: 1fr; + gap: 1.4rem; + } + .po-step:not(:last-child)::after { + content: "↓"; + right: auto; + left: 50%; + top: auto; + bottom: -1.3rem; + transform: translateX(-50%); + } +} +.md-typeset .po-matrix table:not([class]) { + font-size: 0.68rem; +} +.md-typeset .po-matrix table:not([class]) td:first-child { + min-width: 11rem; +} +.md-typeset .po-matrix table:not([class]) th, +.md-typeset .po-matrix table:not([class]) td { + padding: 0.6em 0.7em; +} +.po-matrix table td:not(:first-child) { + text-align: center; +} +.gd-chart .po-range.gd-s2 { + fill: var(--gd-series-2); +} +.gd-chart .po-mid.gd-s2 { + fill: var(--gd-series-2); +} +.gd-chart .po-truth { + stroke: var(--gd-ink); + stroke-width: 2; + stroke-linecap: round; +} +.gd-chart .po-bar { + stroke: var(--gd-series-1); + stroke-opacity: 0.45; + stroke-width: 2; + stroke-linecap: round; +} +.gd-chart .po-dot { + fill: var(--gd-ink); + stroke: var(--md-default-bg-color, var(--gd-surface)); + stroke-width: 1; +} +.gd-chart .po-dot--out { + fill: var(--md-default-bg-color, var(--gd-surface)); + stroke: var(--gd-ink); + stroke-width: 1.5; +} +.gd-chart--square { + max-width: 20rem; +} +.po-side { + display: flex; + flex-wrap: wrap; + gap: 1rem 2rem; + align-items: center; +} +.po-side > svg { + flex: 0 1 20rem; +} +.po-side > div { + flex: 1 1 16rem; +} diff --git a/docs/tutorials/surrogate/local-testing.md b/docs/tutorials/surrogate/local-testing.md new file mode 100644 index 0000000..615e4fc --- /dev/null +++ b/docs/tutorials/surrogate/local-testing.md @@ -0,0 +1,146 @@ +# Test the surrogate pipeline locally + +GloBI can replace EnergyPlus with a machine-learned _surrogate_: a regressor trained on +simulations of buildings whose uncertain attributes (semantic fields, window-to-wall +ratio, floor-to-floor height, ...) are drawn from **priors**, rather than fixed by the +GIS data. In production this runs as the `iterative_training` Hatchet workflow with +results on S3. + +This page describes the **local harness** in `globi.validation`, which runs the same +stages in-process on one machine (no Hatchet server, no S3), on a small sample, and +then **validates** the surrogate against the standard _deterministic_ GloBI run — the +same buildings simulated with EnergyPlus using exactly the semantic field values in the +GIS data. + +The local harness is a **plumbing test**: it proves that prior-sampled training specs +and GIS-defined hold-out specs go through the real runnable to the same feature index, +and that a trained surrogate can predict the hold-out. At the sizes that run on a +laptop the metrics are noise; a hold-out validation worth reading is an at-scale run on +the Hatchet stack (see below). + +## The pipeline, stage by stage + +| Stage | Hatchet task (`pipelines/training.py`) | Local equivalent (`globi.validation.local_runner`) | +| ----------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1. Sample | `create_simulations` | `sample_training_specs` — pick rows of the GIS _context_ (stratified by weather file) and fill in prior-sampled attributes with `Priors.sample`; validate each row into a `GloBIBuildingSpec` | +| 2. Simulate | scythe scatter/gather of `simulate_globi_building` | `simulate_specs_locally` — runs the registered task in-process via `mock_run`, with a per-spec parquet cache and optional worker processes | +| 3. Combine | `combine_results` | `combine_with_previous` — append this iteration's results to the growing simulation cache | +| 4. Train | `start_training` → `train_regressor_with_cv_fold` × folds | `train_folds_locally` — one `TrainFoldSpec.train()` per fold, saving `regressor` + `transforms.yml` | +| 5. Evaluate | `evaluate_training` | fold-averaged test metrics (`fold_test_averages`) + `ConvergenceThresholdsByTarget.run` | +| 6. Recurse | `transition_recursion` | the loop in `run_local_iterative_training`: stop when converged or at `max_iters`, else bump `current_iter` and go to 1 | +| 7. Finalize | `finalize` | `summary.yml` + `combined_metrics/` | + +Features are the result frame's **index** (every scalar `GloBIBuildingSpec` field plus +`computed_features`: `feature.geometry.*`, `feature.semantic.*`, +`feature.extra_spaces.*`); targets are the flattened result **columns**, named +`///` (e.g. +`EnergyAndPeakAnnual/Energy/Raw/Heating`, kWh/m²). + +## Quick start (no EnergyPlus needed) + +The `dummy_simulation` runnable is an analytic stand-in for EnergyPlus; the whole loop +runs in well under a minute for all three backends: + +```bash +uv sync --all-extras # xgboost, lightgbm, torch +make surrogate-local # == globi tests surrogate-local --config tests/data/surrogate_local/validation-dummy.yml +make surrogate-validate +``` + +## Real run on the e2e test region + +`tests/data/surrogate_local/validation-e2e.yml` uses the 1000-building test region in +`tests/data/e2e/`. EnergyPlus must be installed locally (see `energyplus_versions.json`) +and the Boston EPW is downloaded once. + +```bash +make surrogate-local SURROGATE_CONFIG=tests/data/surrogate_local/validation-e2e.yml SURROGATE_WORKERS=4 +make surrogate-validate SURROGATE_CONFIG=tests/data/surrogate_local/validation-e2e.yml SURROGATE_WORKERS=4 +``` + +The profile is 16 training + 4 validation simulations (≈30 s with 4 workers on a +laptop). The simulation cache (keyed on the spec _and the contents_ of the db / +semantic fields / component map files) makes re-runs incremental. + +### What happens + +1. `n_test` buildings are chosen at random from the GIS data and **held out** + (`/holdout.json`). They never enter the training context. The file also + records the settings that chose them (`seed`, `n_test`, manifest, GIS/semantic + fields hash); running with different settings against the same `workdir` is an + error, since re-drawing the hold-out after training would let training buildings + leak into the validation set. Change `n_test`/`seed` together with `workdir`. +2. The remaining buildings become the training **context** (`context.parquet`): + footprint geometry, neighbours, floors and weather file only. Semantic fields, wwr, + f2f height, basement/attic are stripped and re-sampled from priors + (`default_priors_from_semantic_fields`: uniform over each categorical field's + options, uniform over `[Min, Max]` for numeric fields). +3. Training runs as above, one surrogate per configured backend, into + `/training//`. +4. The validation step simulates the held-out buildings **deterministically** (semantic fields + exactly as in the GIS data, no priors) and predicts them with the ensemble of fold + models. Outputs in `/validation//`: + - `report.md` — global metrics (MAE/RMSE/R²/CV(RMSE)/nMBE/MAPE), area-weighted stock + totals, per-fold vs ensemble, and breakdowns by every `feature.semantic.*` column; + - `per_building.parquet`, `predictions.parquet`, `metrics_*.parquet`, + `aggregate.parquet`. + +Categorical values that appear in the validation set but never in training are flagged +(`has_unseen_category`) and metrics are additionally reported on the seen-only subset. + +## Configuration + +`LocalValidationConfig` (`globi.validation.config`) is a small yaml; see the two fixtures +for annotated examples. Notable knobs: + +- `backends`: a list of `ml_backend: xgb | lgb | nn` entries with their `hp` / `trainer` + configs — one surrogate is trained per entry. +- `n_per_iter`, `min_per_stratum`, `max_iters`, `n_folds`, `thresholds`. +- `targets_globs` (default `EnergyAndPeakAnnual/Energy/Raw/*`). +- `exclude_columns` (default `building_id`, `rotated_rectangle`, `height`). + +## Pitfalls worth knowing + +- **Leaky index columns.** `rotated_rectangle` and `building_id` are unique per building + and would be index-encoded as categorical features; `height` is not used by the + simulator (it derives from `f2f_height × num_floors`). Keep them in `exclude_columns`. +- **Targets must share an index.** Restrict `targets_globs` to one result key + (`EnergyAndPeakAnnual/...`); mixing in the monthly `EnergyAndPeak` frame or overheating + frames fails the "identical indices" check in `TrainFoldSpec.combined_data`. +- **NN batch size.** The NN trainer drops incomplete batches (train _and_ validation). + With tiny data, `batch_size` must be smaller than the rows in one fold or the model + early-stops untrained. +- **LightGBM on small data.** Set `hp.min_data_in_leaf` (lightgbm's default of 20 + yields near-constant predictions on tens of samples). +- **Basement/attic priors.** With `fixed_basement_attic: true` (default) the + `GloBIBuildingSpec` validators are deterministic; otherwise attic height / use + fractions are drawn from numpy's global RNG (seeded by the harness). +- **Hatchet env.** Importing `globi.pipelines` builds a Hatchet client; the harness + (and `tests/conftest.py`) populates offline-safe defaults from + `.env.local.host.hatchet.example` when the variables are unset. + +## Running the real workflow on the local docker stack + +The Hatchet workflow itself can be exercised with docker + hatchet-lite + localstack S3 +(no AWS account): + +```bash +make hatchet-token && make engine # hatchet-lite, localstack, fanout + simulation workers +make training-native # a training worker on the host (the compose `training` + # service requests a GPU; the host worker just carries + # the HAS_GPU label and falls back to CPU) +make cli-native tests surrogate-stack-manifest -- --config tests/data/surrogate_local/validation-e2e.yml --out outputs/surrogate-stack/training.yml +make cli-native submit surrogate -- --path outputs/surrogate-stack/training.yml +``` + +`surrogate-stack-manifest` uploads the context, component library, semantic fields and +component map to the configured scythe bucket and writes a `ProgressiveTrainingSpec` +with s3 references; the run then proceeds exactly as in production. Progress is visible +in the Hatchet UI (see _Simulation Tasking_). + +## Tests + +```bash +uv run pytest # fast: dummy runnable, all backends (~45 s) +uv run pytest --run-energyplus # + the EnergyPlus-backed smoke run (~90 s total) +``` diff --git a/infra/sst.config.ts b/infra/sst.config.ts index 8398979..b344c22 100644 --- a/infra/sst.config.ts +++ b/infra/sst.config.ts @@ -49,6 +49,8 @@ export default $config({ ? aws.s3.BucketV2.get("Storage", process.env.EXISTING_BUCKET) : new sst.aws.Bucket("Storage"); + const bucketPrefix = process.env.BUCKET_PREFIX || "globi"; + const bucketName = bucket instanceof aws.s3.BucketV2 ? bucket.bucket : bucket.name; @@ -88,7 +90,7 @@ export default $config({ // SCYTHE_WORKER_MAX_RUNS: "1", SCYTHE_WORKER_SLOTS: "1", SCYTHE_STORAGE_BUCKET: bucketName, - SCYTHE_STORAGE_BUCKET_PREFIX: "globi", + SCYTHE_STORAGE_BUCKET_PREFIX: bucketPrefix, SCYTHE_TIMEOUT_SCATTER_GATHER_SCHEDULE: "10h", SCYTHE_TIMEOUT_SCATTER_GATHER_EXECUTION: "10h", SCYTHE_TIMEOUT_EXPERIMENT_SCHEDULE: "10h", diff --git a/mkdocs.yml b/mkdocs.yml index c8663a7..36cbfd9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,9 +17,30 @@ nav: - Run Simulations: - Simulation Tasking: tutorials/run-simulations/simulation_tasking.md - Simulate Building: tutorials/run-simulations/simulate_building.md + - Surrogate Models: + - Local Testing & Validation: tutorials/surrogate/local-testing.md + - Data & outputs: + - data/index.md + - Output structure: data/outputs-structure.md + - Run catalog: + - data/runs/index.md + - Results status: data/status.md + - Surrogate models: data/surrogates.md + - Input bundles: data/inputs.md + - Reference: + - Manifest schema: data/reference/schema.md + - What makes up a region: data/reference/data-components.md + - Adding an input version: data/reference/add-a-version.md + - Downloading inputs: data/reference/download-inputs.md - Reference: - CLI: reference/cli.md +hooks: + - mkdocs_hooks/globi_data.py + +extra_css: + - stylesheets/globi-data.css + plugins: - search - mkdocstrings: @@ -77,6 +98,8 @@ markdown_extensions: - toc: permalink: "#" - admonition + - attr_list + - md_in_html - pymdownx.details - pymdownx.arithmatex: generic: true diff --git a/mkdocs_hooks/globi_data.py b/mkdocs_hooks/globi_data.py new file mode 100644 index 0000000..7611cec --- /dev/null +++ b/mkdocs_hooks/globi_data.py @@ -0,0 +1,1128 @@ +"""MkDocs hook: build the "Data & outputs" pages from a globi-data checkout. + +globi-data (https://github.com/MITSustainableDesignLab/globi-data) holds the +versioned inputs and the provenance records of every simulation run. Rather +than copy its contents into these docs, this hook reads a checkout at build +time and generates: + +- ``data/runs/index.md`` and one ``data/runs/.md`` per partner, +- ``data/status.md``, ``data/inputs.md`` and ``data/surrogates.md``, +- ``data/reference/*.md``, globi-data's own reference docs with links rewritten. + +The checkout is found, in order, at ``$GLOBI_DATA_ROOT``, ``../globi-data``, +``../../globi-data`` (both relative to ``mkdocs.yml``), or ``.globi-data``. +Pages that must not be published (discussion drafts, results from real partner +buildings) live outside this public repository. Set ``GLOBI_INTERNAL_DOCS`` to a +folder of Markdown files and they are added under ``internal/`` with a banner. The +build refuses to ``gh-deploy`` while that variable is set. + +If no globi-data checkout exists the pages are still generated, each saying the data was not +available, so a build never fails for lack of the sibling repository. Set +``GLOBI_DATA_REQUIRED=1`` to make that an error instead. +""" + +from __future__ import annotations + +import html +import logging +import os +import posixpath +import re +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml +from mkdocs.exceptions import PluginError +from mkdocs.structure.files import File, Files + +log = logging.getLogger("mkdocs.hooks.globi_data") + +GITHUB = "https://github.com/MITSustainableDesignLab/globi-data" +BLOB = f"{GITHUB}/blob/main" +TREE = f"{GITHUB}/tree/main" + +# globi-data docs pulled in verbatim (links rewritten), keyed by source path. +REFERENCE_DOCS = { + "docs/SCHEMA.md": "data/reference/schema.md", + "docs/DATA-COMPONENTS.md": "data/reference/data-components.md", + "docs/HOW-TO-ADD-A-VERSION.md": "data/reference/add-a-version.md", + "docs/HOW-TO-DOWNLOAD-INPUTS.md": "data/reference/download-inputs.md", +} +# The partner source repository is private and must never be named on the +# public site. globi-data's own docs do name it, so these rewrites are applied +# to them on the way in, and `on_page_markdown` fails the build if any page +# still matches PRIVATE_PATTERN. If globi-data rewords a passage, the guard +# fires and a rewrite here needs updating. +PRIVATE_PATTERN = re.compile(r"partners[-_]data|globi-partners latest-results", re.I) +GIS_NOT_PUBLISHED = ( + "Building footprints are partner data and are not published. Each version\n" + "records their exact size and fingerprint, so anyone who holds the file can\n" + "confirm it is the one these results were built from." +) +PUBLIC_REWRITES: dict[str, list[tuple[str, str]]] = { + "docs/HOW-TO-DOWNLOAD-INPUTS.md": [ + ( + r"## The building footprints are not here\n.*?(?=\n## )", + "## The building footprints are not here\n\n" + GIS_NOT_PUBLISHED + "\n", + ), + ], + "docs/HOW-TO-ADD-A-VERSION.md": [ + (r"--from \S*partners-data\S*", "--from "), + ], + "docs/SCHEMA.md": [ + ( + r" repo: globi-partners-data\n remote: \S+\n", + " repo: \n", + ), + (r"repo: globi-partners-data", "repo: "), + ( + r"`globi-partners latest-results --yaml-dir`", + "the legacy `latest-results --yaml-dir` export", + ), + ( + r"`globi-partners-data/\.env` currently\nsets", + "the run environment currently\nsets", + ), + (r"globi_partners_data: \{[^}]*\}", "source_data: {...}"), + ], + "docs/DATA-COMPONENTS.md": [ + ( + r"the preprocessor in `globi-partners-data`\n\(`packages/globi/src/globi/", + "the preprocessor in globi\n(`src/globi/", + ), + ], +} + +# Other globi-data files that have a generated page here. +PAGE_FOR = { + "outputs/STATUS.md": "data/status.md", + "outputs/README.md": "data/runs/index.md", + "inputs/README.md": "data/inputs.md", +} + +# Status keys follow globi-data's STATE_BADGES (src/globi_data/render.py). +# Each maps to (css class, icon, label); the class picks a status color. +STATUS_STYLE = { + "ok": ("good", "●", "up to date"), + "metadata-drift": ("info", "●", "docs changed only"), + "code-drift": ("warning", "▲", "code changed"), + "inferred": ("neutral", "○", "assumed"), + "unpinned": ("neutral", "○", "unknown"), + "stale": ("critical", "■", "stale"), + "broken-pin": ("critical", "■", "broken"), + "tampered": ("critical", "■", "tampered"), +} +LABEL_TO_STATUS = {label: key for key, (_, _, label) in STATUS_STYLE.items()} +LABEL_TO_STATUS["broken / tampered"] = "broken-pin" +STATUS_ORDER = list(STATUS_STYLE) +# globi-data's SEVERITY order (src/globi_data/status.py). +STATUS_ORDER_WORST_FIRST = [ + "tampered", + "broken-pin", + "stale", + "code-drift", + "unpinned", + "inferred", + "metadata-drift", + "ok", +] + +WEATHER_LABEL = {"tmy": "Current climate (TMY)", "ftmy": "Future climate (fTMY 2050)"} + + +# ---------------------------------------------------------------- data model + + +@dataclass +class Experiment: + """The current output version of one ``//``.""" + + name: str + partner: str + weather: str + scenario: str + scenario_kind: str + current: str + manifest: dict[str, Any] + versions: list[dict[str, Any]] + status: str = "unpinned" + status_detail: str = "" + + @property + def run_time(self) -> datetime | None: + """Parse ``run_timestamp``, or None if it is not a timestamp.""" + ts = self.manifest.get("run_timestamp") + try: + return datetime.strptime(str(ts), "%Y-%m-%d_%H-%M-%S") + except ValueError: + return None + + @property + def path(self) -> str: + """The experiment's directory, relative to the globi-data root.""" + return f"outputs/runs/{self.name}" + + +@dataclass +class GlobiData: + """Everything the generated pages read from a globi-data checkout.""" + + root: Path | None + commit: str | None = None + experiments: list[Experiment] = field(default_factory=list) + inputs: list[dict[str, Any]] = field(default_factory=list) + surrogates: list[dict[str, Any]] = field(default_factory=list) + batches: list[dict[str, Any]] = field(default_factory=list) + status_generated_at: str | None = None + + @property + def partners(self) -> list[str]: + """Partners with at least one recorded experiment.""" + return sorted({e.partner for e in self.experiments}) + + +def _yaml(path: Path) -> Any: + with path.open() as f: + return yaml.safe_load(f) + + +def locate_root(config_dir: Path) -> Path | None: + """Find a globi-data checkout; see the module docstring for the order.""" + candidates = [] + if env := os.environ.get("GLOBI_DATA_ROOT"): + candidates.append(Path(env).expanduser()) + candidates += [ + config_dir.parent / "globi-data", + config_dir.parent.parent / "globi-data", + config_dir / ".globi-data", + ] + for c in candidates: + path = c if c.is_absolute() else config_dir / c + if (path / "published.yml").exists() or (path / "inputs").is_dir(): + return path.resolve() + return None + + +def _git_commit(root: Path) -> str | None: + head = root / ".git" / "HEAD" + try: + ref = head.read_text().strip() + if ref.startswith("ref: "): + ref_path = root / ".git" / ref[5:] + if ref_path.exists(): + return ref_path.read_text().strip()[:7] + packed = (root / ".git" / "packed-refs").read_text() + m = re.search(rf"^([0-9a-f]{{40}}) {re.escape(ref[5:])}$", packed, re.M) + return m.group(1)[:7] if m else None + return ref[:7] + except OSError: + return None + + +def _parse_status(root: Path) -> tuple[dict[str, tuple[str, str]], str | None]: + """Read the generated ``outputs/STATUS.md`` rather than recompute it.""" + path = root / "outputs" / "STATUS.md" + if not path.exists(): + return {}, None + text = path.read_text() + generated = re.search(r"_Generated (.+?)\._", text) + rows = {} + for line in text.splitlines(): + m = re.match( + r"\|\s*\[([^\]]+)\]\([^)]*\)\s*\|\s*(.+?)\s*\|.*\|\s*(.*?)\s*\|\s*$", line + ) + if not m: + continue + badge = re.sub(r"^[^\w]+", "", m.group(2)).strip() + rows[m.group(1)] = (LABEL_TO_STATUS.get(badge, "unpinned"), m.group(3)) + return rows, generated.group(1) if generated else None + + +def load(root: Path | None) -> GlobiData: + """Read every manifest the generated pages need.""" + data = GlobiData(root=root) + if root is None: + return data + data.commit = _git_commit(root) + status, data.status_generated_at = _parse_status(root) + + for current_file in sorted((root / "outputs" / "runs").glob("*/*/*/CURRENT.yml")): + exp_dir = current_file.parent + current = _yaml(current_file) + manifest_path = exp_dir / current["manifest"] + if not manifest_path.exists(): + log.warning("globi-data: %s points at a missing manifest", current_file) + continue + manifest = _yaml(manifest_path) + versions_file = exp_dir / "versions.yml" + versions = ( + _yaml(versions_file).get("versions", []) if versions_file.exists() else [] + ) + name = manifest.get("experiment") or str( + exp_dir.relative_to(root / "outputs" / "runs") + ) + exp = Experiment( + name=name, + partner=manifest.get("partner", exp_dir.parts[-3]), + weather=manifest.get("weather", exp_dir.parts[-2]), + scenario=manifest.get("scenario_value", exp_dir.parts[-1]), + scenario_kind=manifest.get("scenario_kind") or "", + current=current["current"], + manifest=manifest, + versions=versions, + ) + exp.status, exp.status_detail = status.get(name, ("unpinned", "")) + data.experiments.append(exp) + + index = root / "inputs" / "index.yml" + if index.exists(): + data.inputs = _yaml(index).get("partners", []) + + for manifest_path in sorted( + (root / "outputs" / "surrogates").glob("*/v*/surrogate-version.yml") + ): + data.surrogates.append( + _yaml(manifest_path) | {"_dir": manifest_path.parent.relative_to(root)} + ) + + for manifest_path in sorted((root / "outputs" / "batches").glob("*/v*/*.yml")): + doc = _yaml(manifest_path) + if isinstance(doc, dict) and doc.get("kind", "").endswith("version"): + data.batches.append(doc | {"_dir": manifest_path.parent.relative_to(root)}) + return data + + +# ---------------------------------------------------------------- rendering helpers + + +def esc(value: Any) -> str: + """HTML-escape a value for use in text or an attribute.""" + return html.escape(str(value), quote=True) + + +def chip(status: str) -> str: + """A status chip: icon and label, with color as support.""" + cls, icon, label = STATUS_STYLE.get(status, STATUS_STYLE["unpinned"]) + return f' {label}' + + +def short(sha: str | None, n: int = 12) -> str: + """A shortened hash in code style, or a dash.""" + return f"`{sha[:n]}`" if sha else "—" + + +def md_table(headers: list[str], rows: list[list[Any]]) -> str: + """Render a Markdown table.""" + out = ["| " + " | ".join(headers) + " |", "|" + "---|" * len(headers)] + out += [ + "| " + " | ".join(str(c).replace("|", "\\|") for c in row) + " |" + for row in rows + ] + return "\n".join(out) + + +def unavailable(data: GlobiData) -> str | None: + """The warning shown on data pages when no checkout was found.""" + if data.root is not None: + return None + return ( + '!!! warning "globi-data was not found at build time"\n' + " This page is generated from a checkout of\n" + f" [globi-data]({GITHUB}). Clone it beside this repository, or set\n" + " `GLOBI_DATA_ROOT`, and rebuild the docs.\n" + ) + + +def source_note(data: GlobiData, path: str) -> str: + """A footer naming the globi-data file a page was generated from.""" + at = f" at commit `{data.commit}`" if data.commit else "" + return ( + f'

Generated from globi-data/{path}' + f"{at}. Edit the data there, not this page.

" + ) + + +def stat_tiles(tiles: list[tuple[str, Any, str]]) -> str: + """A row of stat tiles: (label, value, optional sub-line).""" + cells = "".join( + f'
{esc(label)}
' + f'
{esc(value)}
' + + (f'
{sub}
' if sub else "") + + "
" + for label, value, sub in tiles + ) + return f'
{cells}
' + + +# ---------------------------------------------------------------- charts + + +def timeline_svg(experiments: list[Experiment]) -> str: + """Dot plot of when each run happened: one row per partner, colored by weather.""" + runs = [e for e in experiments if e.run_time] + if not runs: + return "" + partners = sorted({e.partner for e in runs}) + t0 = min(e.run_time for e in runs) # type: ignore[type-var] + t1 = max(e.run_time for e in runs) # type: ignore[type-var] + day0 = t0.replace(hour=0, minute=0, second=0) + n_days = max((t1 - day0).days + 1, 2) + left, right, top, row_h = 168, 24, 16, 34 + width = 720 + plot_w = width - left - right + height = top + row_h * len(partners) + 40 + + def x(t: datetime) -> float: + return left + plot_w * (t - day0).total_seconds() / (n_days * 86400) + + parts = [ + f'', + 'Simulation run dates by partner, colored by weather', + ] + step = 1 if n_days <= 14 else 7 + for d in range(0, n_days + 1, step): + tx = left + plot_w * d / n_days + parts.append( + f'' + ) + if d < n_days and (d % (2 * step) == 0 or n_days <= 8): + label = day0.toordinal() + d + parts.append( + f'' + f"{datetime.fromordinal(label).strftime('%b %-d')}" + ) + for i, partner in enumerate(partners): + cy = top + row_h * i + row_h / 2 + parts.append( + f'' + f"{esc(partner.replace('_', ' '))}" + ) + parts.append( + f'' + ) + for e in sorted( + (r for r in runs if r.partner == partner), key=lambda r: r.weather != "tmy" + ): + dy = -5 if e.weather == "tmy" else 5 + series = 1 if e.weather == "tmy" else 2 + tip = f"{e.name} · run {e.run_time:%Y-%m-%d %H:%M} · {e.manifest.get('version', '')}" # type: ignore[union-attr] + parts.append( + f'{esc(tip)}' # type: ignore[arg-type] + ) + parts.append("") + legend = ( + '
' + 'tmy (current climate)' + 'ftmy (future climate)
' + ) + return f'
{legend}{"".join(parts)}
' + + +def hbar_svg( + title: str, rows: list[tuple[str, float]], fmt: str = "{:.2f}", unit: str = "" +) -> str: + """Horizontal bars for one measure; value labelled at each bar's tip.""" + if not rows: + return "" + vmax = max(v for _, v in rows) or 1.0 + left, right, bar, gap, top = 170, 64, 18, 12, 8 + width = 640 + plot_w = width - left - right + height = top + len(rows) * (bar + gap) + 8 + parts = [ + f'', + f'', + ] + for i, (label, value) in enumerate(rows): + y = top + i * (bar + gap) + w = max(plot_w * value / vmax, 2) + r = min(4, w / 2) + path = ( + f"M{left},{y} h{w - r:.1f} a{r},{r} 0 0 1 {r},{r} v{bar - 2 * r} " + f"a{r},{r} 0 0 1 -{r},{r} h-{w - r:.1f} z" + ) + text = fmt.format(value) + unit + parts.append( + f'{esc(label)}' + f'{esc(label)}: {esc(text)}' + f'{esc(text)}' + ) + parts.append("") + return f'
{esc(title)}
{"".join(parts)}
' + + +def status_bar(counts: Counter[str]) -> str: + """A single 100% bar of experiments by status, with a legend of counts.""" + total = sum(counts.values()) + if not total: + return "" + segs, legend = [], [] + for key in STATUS_ORDER: + n = counts.get(key, 0) + if not n: + continue + cls, icon, label = STATUS_STYLE[key] + segs.append( + f'' + ) + legend.append( + f'{icon} {label} {n}' + ) + return ( + f'
{"".join(legend)}
' + f'
' + ) + + +# ---------------------------------------------------------------- pages + + +def anchor(e: Experiment) -> str: + """The heading anchor of an experiment on its partner page.""" + return re.sub(r"[^a-z0-9]+", "-", f"{e.weather}-{e.scenario}".lower()).strip("-") + + +def coverage_matrix(data: GlobiData) -> str: + """Partners x (weather, scenario kind), each cell the scenario and its status.""" + cols = [(w, k) for w in ("tmy", "ftmy") for k in ("baseline", "alternative")] + by_key = {(e.partner, e.weather, e.scenario_kind): e for e in data.experiments} + head = "".join(f"{w}
{k}" for w, k in cols) + rows = [] + for p in data.partners: + cells = [] + for w, k in cols: + e = by_key.get((p, w, k)) + if e is None: + cells.append('—') + else: + cells.append( + f'{esc(e.scenario)}
{chip(e.status)}' + f" {esc(e.current)}" + ) + rows.append( + f'{esc(p.replace("_", " "))}{"".join(cells)}' + ) + return ( + '
' + f"{head}{''.join(rows)}
Partner
" + ) + + +def page_runs_index(data: GlobiData) -> str: + """The run catalog: tiles, coverage matrix, timeline and table.""" + out = ["# Run catalog", ""] + if msg := unavailable(data): + return "\n".join([*out, msg]) + exps = data.experiments + n_files = sum( + len(e.manifest.get("result_files") or e.manifest.get("results") or []) + for e in exps + ) + confidence = Counter( + e.manifest.get("provenance", {}).get("input", {}).get("confidence") + for e in exps + ) + out += [ + "Every simulation run recorded in globi-data, one entry per", + "`//`. The parquet results stay in S3; what is", + "shown here is the record of where they are and what produced them. See", + "[Output structure](../outputs-structure.md) for how these records are laid out.", + "", + stat_tiles([ + ("Partners", len(data.partners), ""), + ( + "Experiments", + len(exps), + f"{len(exps) // max(len(data.partners), 1)} per partner", + ), + ("Result files", n_files, "parquet, in S3"), + ( + "Input pinned exactly", + confidence.get("exact", 0), + f"{confidence.get('inferred', 0)} inferred", + ), + ]), + "", + "## Coverage", + "", + "Each partner runs two scenarios (a baseline and an alternative) under two", + "weather files. Each cell links to that run's provenance.", + "", + coverage_matrix(data), + "", + "## When the runs happened", + "", + "Hover a dot for the run's experiment and timestamp; click it for the details.", + "", + timeline_svg(exps), + "", + "## All experiments", + "", + ] + rows = [] + for e in exps: + prov = e.manifest.get("provenance", {}) + rows.append([ + f"[{e.name}]({e.partner}.md#{anchor(e)}) `{e.current}`", + f"{e.manifest.get('version', '')} · {e.run_time:%Y-%m-%d}" + if e.run_time + else "", + f"`{prov.get('input', {}).get('version', '—')}` {prov.get('input', {}).get('confidence', '—')}", + chip(e.status), + ]) + out.append(md_table(["Experiment", "Scythe run", "Input", "Status"], rows)) + if data.batches: + out += ["", "## Batches", ""] + out.append( + md_table( + ["Batch", "Version", "Status", "Summary"], + [ + [ + f"[{b.get('experiment', b['_dir'])}]({TREE}/{b['_dir']})", + f"`{b.get('output_version', '')}`", + b.get("status", ""), + b.get("summary", ""), + ] + for b in data.batches + ], + ) + ) + out += ["", source_note(data, "outputs/index.yml")] + return "\n".join(out) + + +def lineage_mermaid(partner: str, exps: list[Experiment]) -> str: + """A flowchart from input versions to runs to their S3 prefixes.""" + lines = ["```mermaid", "flowchart LR"] + inputs = sorted({ + e.manifest.get("provenance", {}).get("input", {}).get("version", "?") + for e in exps + }) + for v in inputs: + lines.append(f' in_{v}[("inputs/{partner}/{v}")]') + for i, e in enumerate(exps): + inp = e.manifest.get("provenance", {}).get("input", {}) + edge = "-->" if inp.get("confidence") == "exact" else "-.->" + lines.append( + f' in_{inp.get("version", "?")} {edge} r{i}["{e.weather} / {e.scenario}
{e.current}"]' + ) + lines.append( + f' r{i} --> s{i}[/"S3: {e.manifest.get("version", "")}
{e.manifest.get("run_timestamp", "")}"/]' + ) + lines += [ + "```", + "", + "Solid arrow: input version recorded at run time. " + "Dashed: inferred during backfill.", + ] + return "\n".join(lines) + + +def page_partner(data: GlobiData, partner: str) -> str: + """One partner's experiments with full provenance.""" + exps = sorted( + (e for e in data.experiments if e.partner == partner), + key=lambda e: (e.weather != "tmy", e.scenario_kind != "baseline"), + ) + out = [f"# {partner.replace('_', ' ')}", ""] + inp = next((i for i in data.inputs if i["partner"] == partner), None) + if inp: + out += [ + f"Current input bundle: [`{inp['current']}`]({TREE}/inputs/partners/{partner}/{inp['current']}) " + f"({inp.get('bundle_kind', '')}, {len(inp.get('versions', []))} version(s)).", + "", + ] + out += ["## Lineage", "", lineage_mermaid(partner, exps), ""] + for e in exps: + m = e.manifest + prov = m.get("provenance", {}) + pin, code, store = ( + prov.get("input", {}), + prov.get("code", {}), + prov.get("storage", {}), + ) + globi = code.get("globi") or {} + out += [ + f"## {WEATHER_LABEL.get(e.weather, e.weather)} · {e.scenario} {{ #{anchor(e)} }}", + "", + f"{chip(e.status)} {esc(e.status_detail)}", + "", + f"`{e.name}` · output `{e.current}` · {esc(m.get('summary', ''))}", + "", + md_table( + ["", ""], + [ + ["Scenario", f"{e.scenario} ({e.scenario_kind})"], + [ + "Input version", + f"[`{pin.get('version')}`]({BLOB}/{pin.get('manifest_path', '')}) — " + f"confidence **{pin.get('confidence')}**", + ], + ["Input bundle sha256", short(pin.get("bundle_sha256"))], + [ + "GIS file (not published)", + f"{short((pin.get('gis_file') or {}).get('sha256'))}, " + f"{((pin.get('gis_file') or {}).get('bytes') or 0) / 1e6:,.0f} MB", + ], + [ + "globi", + f"`{globi.get('describe') or short(globi.get('commit'))}`", + ], + [ + "Scythe run", + f"{m.get('version', '')} · `{m.get('run_timestamp', '')}`", + ], + ["S3 run prefix", f"`{store.get('run_prefix', '')}`"], + ], + ), + "", + '??? note "Result files"', + "", + " " + + md_table( + ["Key", "Object", "sha256"], + [ + [ + f"`{r['result_key']}`", + f"`final/{posixpath.basename(r['s3_uri'])}`", + short(r.get("sha256")) + if r.get("sha256") + else f"not computed ({r.get('sha256_source')})", + ] + for r in m.get("result_files", []) + ], + ).replace("\n", "\n "), + "", + " Download with `globi-data outputs download " + + e.name + + " --dest ./results`.", + "", + ] + if len(e.versions) > 1: + out += [ + md_table( + ["Version", "Status", "Created", "Summary"], + [ + [ + f"`{v['version']}`", + v.get("status", ""), + str(v.get("created_at", ""))[:10], + v.get("summary", ""), + ] + for v in e.versions + ], + ), + "", + ] + out += [ + f"[Manifest on GitHub]({BLOB}/{e.path}/{e.current}/output-version.yml)", + "", + ] + out.append(source_note(data, f"outputs/runs/{partner}")) + return "\n".join(out) + + +def page_status(data: GlobiData) -> str: + """Every experiment's staleness verdict.""" + out = ["# Results status", ""] + if msg := unavailable(data): + return "\n".join([*out, msg]) + at = ( + f" globi-data last regenerated it at {data.status_generated_at}." + if data.status_generated_at + else "" + ) + out += [ + "Whether each set of results still matches the inputs and code that are current.", + f"This is read from globi-data's generated `outputs/STATUS.md`.{at}", + "", + status_bar(Counter(e.status for e in data.experiments)), + "", + md_table( + ["Status", "Meaning"], + [ + [chip("ok"), "Built from the current inputs."], + [ + chip("stale"), + "Inputs changed in a way that affects results. Re-run.", + ], + [ + chip("metadata-drift"), + "Inputs changed, but only documentation. No re-run needed.", + ], + [chip("code-drift"), "Simulation code moved on since this run."], + [ + chip("inferred"), + "Which inputs produced this was inferred during backfill, not recorded.", + ], + [ + chip("broken-pin"), + "The recorded inputs cannot be found or no longer match.", + ], + ], + ), + "", + "## By experiment", + "", + "Grouped worst-first. Each experiment links to its full provenance.", + "", + ] + groups: dict[str, list[Experiment]] = defaultdict(list) + for e in data.experiments: + groups[e.status].append(e) + for status in STATUS_ORDER_WORST_FIRST: + exps = groups.get(status) + if not exps: + continue + details = {e.status_detail for e in exps} + shared = len(details) == 1 + out += [f"### {chip(status)} {len(exps)}", ""] + if shared and (detail := details.pop()): + out += [esc(detail), ""] + rows = [] + for e in exps: + pin = e.manifest.get("provenance", {}).get("input", {}) + cur = next( + (i["current"] for i in data.inputs if i["partner"] == e.partner), "—" + ) + row = [ + f"[{e.name}](runs/{e.partner}.md#{anchor(e)})", + f"`{pin.get('version')}`", + f"`{cur}`", + ] + rows.append(row if shared else [*row, esc(e.status_detail)]) + headers = ["Experiment", "Built from", "Current input"] + out += [md_table(headers if shared else [*headers, "Detail"], rows), ""] + out += ["", source_note(data, "outputs/STATUS.md")] + return "\n".join(out) + + +def page_inputs(data: GlobiData) -> str: + """The current input bundle for each partner.""" + out = ["# Input bundles", ""] + if msg := unavailable(data): + return "\n".join([*out, msg]) + used = Counter( + e.manifest.get("provenance", {}).get("input", {}).get("partner") + for e in data.experiments + ) + out += [ + "Each partner's inputs are frozen, numbered bundles. Output records pin a", + "bundle by the sha256 of its `input-version.yml`, so a result can always be", + "traced to the exact files that produced it. See", + "[What makes up a region](reference/data-components.md) for what each file does.", + "", + ] + rows = [] + for p in data.inputs: + statuses = Counter(v.get("status") for v in p.get("versions", [])) + rows.append([ + f"[{p['partner']}]({TREE}/inputs/partners/{p['partner']})", + f"`{p.get('current')}`", + p.get("bundle_kind", ""), + ", ".join(f"{n} {s}" for s, n in statuses.items()), + used.get(p["partner"], 0) or "—", + short(p.get("bundle_sha256")), + ]) + out.append( + md_table( + [ + "Partner", + "Current", + "Kind", + "Versions", + "Experiments built on it", + "Bundle sha256", + ], + rows, + ) + ) + out += ["", source_note(data, "inputs/index.yml")] + return "\n".join(out) + + +def _feature_groups(features: list[dict[str, Any]]) -> list[list[str]]: + """Collapse runs of numbered features (shading_mask_00..47) into one row.""" + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for f in features: + groups[re.sub(r"_\d+$", "_*", f["name"])].append(f) + rows = [] + for name, fs in groups.items(): + f = fs[0] + label = f"`{name}` x{len(fs)}" if len(fs) > 1 else f"`{name}`" + rows.append([ + label, + f["source"], + f["kind"], + f.get("unit") or "", + f"`{f['default']}`" if f.get("default") is not None else "", + esc(f.get("description") or ""), + ]) + return rows + + +def page_surrogates(data: GlobiData) -> str: + """Every recorded surrogate version.""" + out = ["# Surrogate models", ""] + if msg := unavailable(data): + return "\n".join([*out, msg]) + out += [ + "A surrogate is a trained regressor that stands in for EnergyPlus: semantic", + "fields, weather and geometry go in, end-use energy intensity comes out. Each", + "is versioned under `outputs/surrogates//vNNN/` with its", + "`transforms.yml` (the model's input contract) kept in git and the model", + "itself in S3.", + "", + ] + if not data.surrogates: + out.append("No surrogate versions are recorded yet.") + for s in data.surrogates: + out += [ + f"## {s['partner']} `{s['surrogate_version']}` — {s.get('status', '')}", + "", + ] + if s.get("example"): + out += [ + '!!! warning "Illustrative example"', + f" {esc(s.get('summary', ''))}", + "", + ] + by_source = Counter(f["source"] for f in s.get("features", [])) + training = s.get("provenance", {}).get("training", {}) + model = s.get("model", {}) + out += [ + stat_tiles([ + ("Backend", model.get("backend", "—"), esc(model.get("format", ""))), + ( + "Features", + len(s.get("features", [])), + " · ".join(f"{n} {src}" for src, n in by_source.most_common()), + ), + ("Targets", len(s.get("targets", [])), "end uses, kWh/m²/yr"), + ( + "Training samples", + training.get("n_samples", "—"), + f"{training.get('n_iterations', '—')} iterations", + ), + ]), + "", + ] + metrics = (s.get("metrics") or {}).get("per_target") or {} + if metrics: + + def end_use(name: str) -> str: + """The last path segment of a target name.""" + return name.rsplit("/", 1)[-1] + + rows = sorted( + ((end_use(k), v.get("cvrmse") or 0.0) for k, v in metrics.items()), + key=lambda r: -r[1], + ) + out += [ + hbar_svg( + f"CV(RMSE) on held-out folds, by end use (lower is better; {metrics and s['metrics'].get('n_folds')} folds)", + [(n, v * 100) for n, v in rows], + fmt="{:.0f}", + unit="%", + ), + "", + md_table( + ["End use", "R²", "CV(RMSE)", "NMBE"], + [ + [end_use(k), v.get("r2"), v.get("cvrmse"), v.get("nmbe")] + for k, v in metrics.items() + ], + ), + "", + ] + out += [ + "**Weather options.** Weather is a model input, not a directory level, so one", + "surrogate covers current and future climate.", + "", + md_table( + ["Label", "Value", "File in the pinned input bundle"], + [ + [w.get("label"), f"`{w['value']}`", f"`{w.get('input_file')}`"] + for w in s.get("weather_options", []) + ], + ), + "", + '??? note "Features"', + "", + " " + + md_table( + ["Feature", "Source", "Kind", "Unit", "Default", "Notes"], + _feature_groups(s.get("features", [])), + ).replace("\n", "\n "), + "", + f"[Manifest]({BLOB}/{s['_dir']}/surrogate-version.yml) · " + f"[transforms.yml]({BLOB}/{s['_dir']}/transforms.yml)", + "", + ] + out.append(source_note(data, "outputs/surrogates")) + return "\n".join(out) + + +def rewrite_links(text: str, src: str, dest: str) -> str: + """Point a globi-data doc's relative links at pages here, or at GitHub.""" + src_dir = posixpath.dirname(src) + dest_dir = posixpath.dirname(dest) + targets = REFERENCE_DOCS | PAGE_FOR + + def fix(m: re.Match[str]) -> str: + label, url = m.group(1), m.group(2) + if re.match(r"^[a-z]+:|^#|^/", url): + return m.group(0) + path, _, frag = url.partition("#") + resolved = posixpath.normpath(posixpath.join(src_dir, path)) + if resolved in targets: + new = posixpath.relpath(targets[resolved], dest_dir) + else: + kind = ( + "tree" + if path.endswith("/") or "." not in posixpath.basename(resolved) + else "blob" + ) + new = f"{GITHUB}/{kind}/main/{resolved}" + return f"[{label}]({new}{'#' + frag if frag else ''})" + + return re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", fix, text) + + +def page_reference(data: GlobiData, src: str, dest: str) -> str: + """A globi-data doc with its links rewritten for this site.""" + title = posixpath.basename(src) + if data.root is None or not (data.root / src).exists(): + return f"# {title}\n\n" + ( + unavailable(data) or f"`{src}` is missing from globi-data." + ) + text = (data.root / src).read_text() + for pattern, replacement in PUBLIC_REWRITES.get(src, []): + text = re.sub(pattern, replacement, text, flags=re.S) + text = rewrite_links(text, src, dest) + return text.rstrip() + "\n\n" + source_note(data, src) + "\n" + + +# ---------------------------------------------------------------- mkdocs events + +_DATA: GlobiData | None = None +_INTERNAL: Path | None = None +INTERNAL_PREFIX = "internal/" +INTERNAL_BANNER = ( + '!!! danger "Internal: not for publication"\n' + " This page is built from private partner data and exists only in local\n" + " builds with `GLOBI_INTERNAL_DOCS` set. Do not share it outside the team.\n\n" +) + + +class InternalDeployError(PluginError): + """An attempt to deploy a build that includes internal pages.""" + + def __init__(self) -> None: + """Explain why the deploy was refused.""" + super().__init__( + "GLOBI_INTERNAL_DOCS is set: refusing to deploy a site with internal pages." + ) + + +def _internal_pages(root: Path) -> list[str]: + """Internal Markdown pages as site paths, index first.""" + pages = sorted(p.relative_to(root).as_posix() for p in root.rglob("*.md")) + pages.sort(key=lambda p: posixpath.basename(p) != "index.md") + return [INTERNAL_PREFIX + p for p in pages] + + +def _inject_partner_pages(items: list[Any], partners: list[str]) -> bool: + """Add one nav entry per partner to the section holding the catalog.""" + for item in items: + if not isinstance(item, dict): + continue + for value in item.values(): + if not isinstance(value, list): + continue + if "data/runs/index.md" in value: + value += [{p.replace("_", " "): f"data/runs/{p}.md"} for p in partners] + return True + if _inject_partner_pages(value, partners): + return True + return False + + +def on_config(config: Any) -> Any: + """Locate and load globi-data, and add partner pages to the nav.""" + global _DATA + config_dir = Path(config.config_file_path).parent + root = locate_root(config_dir) + if root is None: + message = "globi-data checkout not found; data pages will say so (set GLOBI_DATA_ROOT)" + if os.environ.get("GLOBI_DATA_REQUIRED") == "1": + raise SystemExit(message) + log.info(message) + else: + log.info("globi-data: reading %s", root) + _DATA = load(root) + + if config.nav: + _inject_partner_pages(config.nav, _DATA.partners) + + global _INTERNAL + if env := os.environ.get("GLOBI_INTERNAL_DOCS"): + if "gh-deploy" in sys.argv: + raise InternalDeployError + _INTERNAL = Path(env).expanduser().resolve() + log.warning( + "Including INTERNAL pages from %s: do not deploy this build", _INTERNAL + ) + if config.nav is not None: + config.nav.append({"Internal (not published)": _internal_pages(_INTERNAL)}) + return config + + +def on_files(files: Files, config: Any) -> Files: + """Add the generated pages to the site.""" + data = _DATA or load(None) + pages = { + "data/runs/index.md": page_runs_index(data), + "data/status.md": page_status(data), + "data/inputs.md": page_inputs(data), + "data/surrogates.md": page_surrogates(data), + } + pages |= {f"data/runs/{p}.md": page_partner(data, p) for p in data.partners} + pages |= { + dest: page_reference(data, src, dest) for src, dest in REFERENCE_DOCS.items() + } + for uri, content in pages.items(): + files.append(File.generated(config, uri, content=content)) + if _INTERNAL is not None: + for path in sorted(_INTERNAL.rglob("*")): + if path.is_file() and not path.name.startswith("."): + uri = INTERNAL_PREFIX + path.relative_to(_INTERNAL).as_posix() + files.append(File.generated(config, uri, abs_src_path=str(path))) + return files + + +class PrivateReferenceError(PluginError): + """A page names the private partner repository.""" + + def __init__(self, src_uri: str, line: int, match: str) -> None: + """Say where the reference is and how to fix it.""" + super().__init__( + f"{src_uri}:{line} mentions the private partner repository ({match!r}). " + "Remove it, or add a rewrite to PUBLIC_REWRITES." + ) + + +def on_page_markdown(markdown: str, page: Any, **_: Any) -> str: + """Refuse to publish a page that names the private partner repository.""" + if page.file.src_uri.startswith(INTERNAL_PREFIX): + return INTERNAL_BANNER + markdown + if m := PRIVATE_PATTERN.search(markdown): + line = markdown.count("\n", 0, m.start()) + 1 + raise PrivateReferenceError(page.file.src_uri, line, m.group(0)) + return markdown diff --git a/pyproject.toml b/pyproject.toml index 44de639..ff2b676 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "fiona>=1.9.5", # Additional Geospatial Tools "epinterface>=1.4.0", - "scythe-engine>=0.1.2", + "scythe-engine>=1.2.0", "ladybug-core>=0.44.29", "pydantic>=2.11,<3", "boto3>=1.40.49", @@ -46,6 +46,25 @@ visualization = [ "playwright>=1.40.0", ] +# ml = [ +# "lightgbm>=4.6.0", +# "xgboost>=3.2.0", +# "pytorch-tabular>=1.2.0", +# "torch>=2.5.0", +# "tensorboard>=2.20.0", +# "wandb>=0.25.0", +# ] + +ml-gpu = [ + "lightgbm>=4.6.0", + "xgboost>=3.2.0", + "numba>=0.63.1", + "pytorch-tabular>=1.2.0", + "torch>=2.5.0", + "tensorboard>=2.20.0", + "wandb>=0.25.0", +] + cli = [ "click>=8.1.7", "xlsxwriter>=3.2.9", @@ -78,13 +97,32 @@ docs = [ worker = "globi.worker.main:main" globi = "globi.tools.cli.main:cli" +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true + +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[[tool.uv.index]] +name = "pypi" +url = "https://pypi.org/simple" +explicit = true + [tool.uv.sources] -# scythe-engine = {git = "https://github.com/szvsw/scythe", branch = "feature/allow-optional-filerefs"} -# scythe-engine = {git = "https://github.com/szvsw/scythe", branch = "feature/update-hatchet"} +# PyTorch: CUDA 12.8 on Linux/Windows (where builds exist), PyPI (CPU) on macOS +torch = [ + { index = "pytorch-cu128", marker = "sys_platform != 'darwin'", extra = "ml-gpu" }, + { index = "pypi", marker = "sys_platform == 'darwin'", extra = "ml-gpu" }, +] +scythe-engine = {git = "https://github.com/szvsw/scythe", branch = "feature/to-threads"} # scythe-engine = {path = "../scythe", editable = true} # epinterface = {path = "../epinterface", editable = true} # epinterface = {path = "epinterface", editable = true} -# epinterface = {git = "https://github.com/szvsw/epinterface", branch = "feature/use-scene-context-for-geometry"} +epinterface = {git = "https://github.com/szvsw/epinterface", branch = "feature/onezone-converter"} [build-system] @@ -99,6 +137,10 @@ testpaths = ["tests"] python_files = "test_*.py" python_classes = "Test*" python_functions = "test_*" +markers = [ + "energyplus: runs real EnergyPlus simulations; opt in with --run-energyplus", + "slow: takes more than a few seconds", +] [tool.pyright] diff --git a/scripts/trainer_testing.py b/scripts/trainer_testing.py new file mode 100644 index 0000000..7151520 --- /dev/null +++ b/scripts/trainer_testing.py @@ -0,0 +1,345 @@ +"""Source geometry pipeline for GloBI project ml testing.""" + +from pathlib import Path +from typing import cast + +import boto3 +import pandas as pd +import yaml +from epinterface.sbem.fields.spec import CategoricalFieldSpec, SemanticModelFields +from scythe.experiments import BaseExperiment +from scythe.scatter_gather import RecursionMap +from scythe.settings import ScytheStorageSettings + +from globi.models.configs import ( + DeterministicGISPreprocessorConfig, + FileConfig, +) +from globi.models.surrogate.backends.xgb import ( + XGBBackend, + XGBModelConfig, + XGBTrainerConfig, +) +from globi.models.surrogate.pipeline import ( + ConvergenceThresholds, + ConvergenceThresholdsByTarget, + CrossValidationSpec, + FeatureConfigSpec, + IterationSpec, + ProgressiveTrainingSpec, + RegressionIOConfigSpec, + StratificationSpec, + TargetsConfigGlobSpec, +) +from globi.models.surrogate.samplers import ( + CategoricalSampler, + Priors, + ProductValuesSampler, + UnconditionalPrior, + UniformSampler, +) +from globi.pipelines import iterative_training, simulate_globi_building +from globi.pipelines.gis import preprocess_gis_file + + +def geometry_extraction( + config: DeterministicGISPreprocessorConfig, + file_config: FileConfig, +): + """Extract the geometry from the GIS file.""" + gdf, column_output_map = preprocess_gis_file(config, file_config) + + columns = { + "building_id": column_output_map.Building_ID_col, + "db_file": column_output_map.DB_File_col, + "semantic_fields_file": column_output_map.Semantic_Fields_File_col, + "component_map_file": column_output_map.Component_Map_File_col, + "epwzip_file": column_output_map.EPWZip_File_col, + "semantic_field_context": column_output_map.Semantic_Field_Context_col, + "neighbor_polys": column_output_map.Neighbor_Polys_col, + "neighbor_heights": column_output_map.Neighbor_Heights_col, + "neighbor_floors": column_output_map.Neighbor_Floors_col, + "rotated_rectangle": column_output_map.Rotated_Rectangle_col, + "long_edge_angle": column_output_map.Long_Edge_Angle_col, + "long_edge": column_output_map.Long_Edge_col, + "short_edge": column_output_map.Short_Edge_col, + "aspect_ratio": column_output_map.Aspect_Ratio_col, + "rotated_rectangle_area_ratio": column_output_map.Rotated_Rectangle_Area_Ratio_col, + "wwr": column_output_map.WWR_col, + "height": column_output_map.Height_col, + "num_floors": column_output_map.Num_Floors_col, + "f2f_height": column_output_map.F2F_Height_col, + "basement": column_output_map.Basement_col, + "attic": column_output_map.Attic_col, + "exposed_basement_frac": column_output_map.Exposed_Basement_Frac_col, + } + + columns_to_pop = [ + "semantic_field_context", + "f2f_height", + "height", + "basement", + "attic", + "exposed_basement_frac", + "wwr", + ] + for column in columns_to_pop: + columns.pop(column) + gdf = cast(pd.DataFrame, gdf[list(columns.values())]) + gdf = gdf.rename(columns={v: k for k, v in columns.items()}) + return gdf + + +def main(): + """Main function.""" + config = DeterministicGISPreprocessorConfig( + cart_crs="EPSG:3857", + min_building_area=10.0, + min_edge_length=3.0, + max_edge_length=1000.0, + neighbor_threshold=100.0, + f2f_height=3.0, + min_building_height=3.0, + max_building_height=300.0, + min_num_floors=1, + max_num_floors=125, + default_wwr=0.2, + default_num_floors=2, + default_basement="none", + default_attic="none", + default_exposed_basement_frac=0.25, + epw_query="source in ['tmyx']", + ) + + file_config = FileConfig( + gis_file=Path("tests/data/e2e/buildings.geojson"), + semantic_fields_file=Path("tests/data/e2e/semantic-fields.yml"), + component_map_file=Path("tests/data/e2e/component-map.yml"), + db_file=Path("tests/data/e2e/components-lib.db"), + epwzip_file=None, + ) + + df = geometry_extraction(config, file_config) + # TODO: MAJOR REPLACEMENT REQUIRED FOR THIS + settings = ScytheStorageSettings() + semantic_field_key = ( + f"{settings.BUCKET_PREFIX}/test-experiment-artifacts/semantic-fields.yml" + ) + component_map_key = ( + f"{settings.BUCKET_PREFIX}/test-experiment-artifacts/component-map.yml" + ) + db_key = f"{settings.BUCKET_PREFIX}/test-experiment-artifacts/components-lib.db" + semantic_fields_uri = f"s3://{settings.BUCKET}/{semantic_field_key}" + component_map_uri = f"s3://{settings.BUCKET}/{component_map_key}" + db_uri = f"s3://{settings.BUCKET}/{db_key}" + df["semantic_fields_file"] = semantic_fields_uri + df["component_map_file"] = component_map_uri + df["db_file"] = db_uri + df["rotated_rectangle"] = df["rotated_rectangle"].apply(lambda x: x.wkt) + df["neighbor_polys"] = df["neighbor_polys"].apply( + lambda x: [poly.wkt if poly is not None else None for poly in x] + ) + + s3 = boto3.client("s3") + s3.upload_file( + Filename=file_config.semantic_fields_file.as_posix(), + Bucket=settings.BUCKET, + Key=semantic_field_key, + ) + s3.upload_file( + Filename=file_config.component_map_file.as_posix(), + Bucket=settings.BUCKET, + Key=component_map_key, + ) + s3.upload_file( + Filename=file_config.db_file.as_posix(), Bucket=settings.BUCKET, Key=db_key + ) + + output_dir = Path("tests/data/training") + output_dir.mkdir(parents=True, exist_ok=True) + + outpath = output_dir / "context.parquet" + df.to_parquet(outpath) + + experiment_config = ProgressiveTrainingSpec( + context=outpath, + runnable=simulate_globi_building, + base_run_name="test-simulations", + convergence_criteria=ConvergenceThresholdsByTarget( + thresholds={ + "EnergyAndPeakAnnual/Energy/Raw/**": ConvergenceThresholds( + mae=3, + rmse=5, + mape=0.05, + r2=0.975, + cvrmse=0.05, + ), + # "EnergyAndPeak/Energy/Raw/**": ConvergenceThresholds( + # r2=0.9, + # ), + } + ), + regression_io_config=RegressionIOConfigSpec( + targets=TargetsConfigGlobSpec( + globs=[ + "EnergyAndPeakAnnual/*/Raw/**", + ], + normalization="min-max", + ), + features=FeatureConfigSpec( + exclude_columns=frozenset(["building_id"]), + cat_encoding="index", + ), + ), + ml_backend=XGBBackend( + hp=XGBModelConfig( + max_depth=7, + eta=0.1, + min_child_weight=None, + subsample=None, + colsample_bytree=None, + alpha=None, + lam=None, + gamma=None, + seed=42, + ), + trainer=XGBTrainerConfig( + num_boost_round=8000, + early_stopping_rounds=10, + verbose_eval=True, + ), + ), + stratification=StratificationSpec( + field="epwzip_file", + sampling="equal", + ), + samplers=make_priors(file_config.semantic_fields_file), + cross_val=CrossValidationSpec( + n_folds=5, + ), + iteration=IterationSpec( + n_per_iter=[500], + min_per_stratum=25, + max_iters=5, + recursion=RecursionMap( + factor=100, + max_depth=1, + ), + ), + storage_settings=settings, + experiment_id="placeholder", + sort_index=0, + ) + + exp = BaseExperiment( + runnable=iterative_training, + run_name=experiment_config.base_run_name, + storage_settings=settings, + ) + run, _ref = exp.allocate( + experiment_config, + version="bumpmajor", + ) + print(yaml.dump(run.model_dump(mode="json"), indent=2, sort_keys=False)) + # sample_spec = SampleSpec( + # parent=experiment_config, priors=experiment_config.samplers + # ) + # sample_df = sample_spec.populate_sample_df() + + +def make_priors(semantic_fields_file: Path): + """Make priors for the uninitiated model.""" + with open(semantic_fields_file) as f: + semantic_fields = SemanticModelFields.model_validate(yaml.safe_load(f)) + + categorical_semantic_fields = [ + field + for field in semantic_fields.Fields + if isinstance(field, CategoricalFieldSpec) + ] + # numeric_semantic_field_names = [ + # field for field in semantic_field_names if isinstance(field, NumericFieldSpec) + # ] + return Priors( + sampled_features={ + "height": UnconditionalPrior( + sampler=ProductValuesSampler( + features_to_multiply=["f2f_height", "num_floors"] + ) + ), + "f2f_height": UnconditionalPrior( + sampler=UniformSampler(min=2.5, max=4, round=None) + ), + "wwr": UnconditionalPrior( + sampler=UniformSampler(min=0.1, max=0.5, round=None) + ), + "basement": UnconditionalPrior( + sampler=CategoricalSampler( + values=[ + "none", + "unoccupied_unconditioned", + "unoccupied_conditioned", + "occupied_unconditioned", + "occupied_conditioned", + ], + weights=[0.5, 0.5 / 4, 0.5 / 4, 0.5 / 4, 0.5 / 4], + ) + ), + "attic": UnconditionalPrior( + sampler=CategoricalSampler( + values=[ + "none", + "unoccupied_unconditioned", + "unoccupied_conditioned", + "occupied_unconditioned", + "occupied_conditioned", + ], + weights=[0.5, 0.5 / 4, 0.5 / 4, 0.5 / 4, 0.5 / 4], + ) + ), + "exposed_basement_frac": UnconditionalPrior( + sampler=UniformSampler(min=0.1, max=0.4, round=None) + ), + **{ + f"semantic_field_{field.Name}": UnconditionalPrior( + sampler=CategoricalSampler( + values=field.Options, + weights=[1 / len(field.Options) for _ in field.Options], + ) + ) + for field in categorical_semantic_fields + }, + # TODO: add basement and attic priors + } + ) + + +if __name__ == "__main__": + main() + # instance = GloBIBuildingSpec( + # building_id="test-building", + # experiment_id="test-experiment", + # sort_index=0, + # db_file="test-db.db", + # semantic_fields_file="test-semantic-fields.yml", + # component_map_file="test-component-map.yml", + # epwzip_file="test-epwzip.epw", + # semantic_field_context={}, + # neighbor_polys=[], + # neighbor_heights=[], + # neighbor_floors=[], + # rotated_rectangle="test-rotated-rectangle.wkt", + # long_edge_angle=0, + # long_edge=10, + # short_edge=10, + # aspect_ratio=1, + # rotated_rectangle_area_ratio=100, + # wwr=0.2, + # height=10, + # num_floors=1, + # f2f_height=10, + # basement="none", + # attic="none", + # exposed_basement_frac=0.25, + # ) + # instance.log = lambda msg: print(msg) diff --git a/src/globi/allocate.py b/src/globi/allocate.py index c7691db..33d4763 100644 --- a/src/globi/allocate.py +++ b/src/globi/allocate.py @@ -1,28 +1,23 @@ """Allocate a GloBI experiment with Scythe.""" -import json import logging -import math from pathlib import Path import boto3 import geopandas as gpd -import numpy as np import yaml from epinterface.sbem.fields.spec import SemanticModelFields from epinterface.sbem.utils import check_model_existence from scythe.base import ExperimentInputSpec, ExperimentOutputSpec from scythe.experiments import BaseExperiment from scythe.scatter_gather import RecursionMap -from shapely import to_wkt -from tqdm import tqdm +from shapely import to_wkb +from globi.branching import calculate_branching_factor from globi.models.configs import GloBIExperimentSpec from globi.models.tasks import GloBIBuildingSpec from globi.pipelines import preprocess_gis_file, simulate_globi_building - -# TODO: TEST THIS!! - +from globi.pipelines.gis import build_building_specs s3_client = boto3.client("s3") @@ -57,52 +52,23 @@ def allocate_globi_experiment( scenario=config.scenario, ) - specs: list[GloBIBuildingSpec] = [] - if max_sims: buildings_gdf = buildings_gdf.sample(min(max_sims, len(buildings_gdf))) - for sort_index, (_, row) in tqdm( - enumerate(buildings_gdf.iterrows()), - total=len(buildings_gdf), - desc="Generating building specs from GIS:", - ): - row = row.to_dict() - globi_spec = GloBIBuildingSpec( - building_id=row[colmap.Building_ID_col], - experiment_id="placeholder", - sort_index=sort_index, - db_file=row[colmap.DB_File_col], - semantic_fields_file=config.file_config.semantic_fields_file, - component_map_file=config.file_config.component_map_file, - epwzip_file=row[colmap.EPWZip_File_col], - semantic_field_context=row[colmap.Semantic_Field_Context_col], - neighbor_polys=[to_wkt(poly) for poly in row[colmap.Neighbor_Polys_col]], - neighbor_heights=row[colmap.Neighbor_Heights_col], - neighbor_floors=row[colmap.Neighbor_Floors_col], - rotated_rectangle=to_wkt(row[colmap.Rotated_Rectangle_col]), - long_edge_angle=row[colmap.Long_Edge_Angle_col], - long_edge=row[colmap.Long_Edge_col], - short_edge=row[colmap.Short_Edge_col], - aspect_ratio=row[colmap.Aspect_Ratio_col], - rotated_rectangle_area_ratio=row[colmap.Rotated_Rectangle_Area_Ratio_col], - wwr=row[colmap.WWR_col], - height=row[colmap.Height_col], - num_floors=row[colmap.Num_Floors_col], - f2f_height=row[colmap.F2F_Height_col], - basement=row[colmap.Basement_col], - attic=row[colmap.Attic_col], - exposed_basement_frac=row[colmap.Exposed_Basement_Frac_col], - parent_experiment_spec=config, - ) - specs.append(globi_spec) + print(f"Generating building specs from GIS ({len(buildings_gdf)} buildings)...") + specs = build_building_specs( + buildings_gdf, + colmap, + config.file_config, + parent_experiment_spec=config, + ) if not specs: msg = "No specs provided" raise ValueError(msg) experiment = BaseExperiment[ExperimentInputSpec, ExperimentOutputSpec]( - experiment=simulate_globi_building, run_name=name + runnable=simulate_globi_building, run_name=name ) print(f"Submitting {len(buildings_gdf)} buildings for experiment {name}") min_branches_required, _, _ = calculate_branching_factor(specs) @@ -124,7 +90,7 @@ def allocate_globi_dryrun( max_tests: int | None = None, ): """Dry run the allocation of an experiment to estimate the cost.""" - from shapely import Polygon, to_wkt + from shapely import Polygon epwzip_file = epwzip_file or config.file_config.epwzip_file if epwzip_file is None: @@ -157,7 +123,7 @@ def allocate_globi_dryrun( neighbor_polys=[], neighbor_heights=[], neighbor_floors=[], - rotated_rectangle=to_wkt(basic_rectangle), + rotated_rectangle=to_wkb(basic_rectangle), long_edge_angle=0, long_edge=width, short_edge=width, @@ -182,7 +148,7 @@ def allocate_globi_dryrun( raise ValueError(msg) experiment = BaseExperiment[ExperimentInputSpec, ExperimentOutputSpec]( - experiment=simulate_globi_building, + runnable=simulate_globi_building, run_name=f"{config.name}/dryrun/{config.scenario}", ) @@ -197,32 +163,6 @@ def allocate_globi_dryrun( return run, ref -def calculate_branching_factor(specs: list[GloBIBuildingSpec]) -> tuple[int, int, int]: - """Calculate the branching factor for a given list of building specs. - - We do this by sampling 1k random buildings and checking the size of their serialized payloads. - - This is necessary because the async fanouts send all of the payloads for a branch over the wire at once. - """ - logger.info("Calculating branching factor...") - ixs = np.random.choice(len(specs), size=1000, replace=True) - total_bytes = 0 - for ix in ixs: - # check the file size of json.sumps - stringified = json.dumps(specs[ix].model_dump(mode="json"), indent=2) - nbytes = len(stringified.encode("utf-8")) - total_bytes += nbytes - avg_bytes = total_bytes / len(ixs) - max_bytes_MB = 3 # safety factor, 4MB is actual amx - max_bytes_B = max_bytes_MB * 1024 * 1024 - sims_per_branch = math.floor(max_bytes_B / avg_bytes) - min_branches_required = math.ceil(len(specs) / sims_per_branch) - logger.info(f"Avg payload size: {int(avg_bytes // 1024):0d} kB") - logger.info(f"Avg sims per branch: {sims_per_branch}") - logger.info(f"Min branches required: {min_branches_required}") - return min_branches_required, sims_per_branch, math.ceil(avg_bytes) - - if __name__ == "__main__": import logging diff --git a/src/globi/branching.py b/src/globi/branching.py new file mode 100644 index 0000000..91bdc63 --- /dev/null +++ b/src/globi/branching.py @@ -0,0 +1,47 @@ +"""Branching factor calculations.""" + +import json +import logging +import math + +import numpy as np +from scythe.base import ExperimentInputSpec + +logger = logging.getLogger(__name__) + + +# TODO: move this into scythe +def calculate_branching_factor[T: ExperimentInputSpec]( + specs: list[T], +) -> tuple[int, int, int]: + """Calculate the branching factor for a given list of building specs. + + We do this by sampling 1k random buildings and checking the size of their serialized payloads. + + This is necessary because the async fanouts send all of the payloads for a branch over the wire at once. + + Args: + specs (list[T]): The list of specs to calculate the branching factor for. + + Returns: + factor (int): The branching factor. + sims_per_branch (int): The number of simulations per branch. + avg_bytes (int): The average size of the serialized payload. + """ + logger.info("Calculating branching factor...") + ixs = np.random.choice(len(specs), size=1000, replace=True) + total_bytes = 0 + for ix in ixs: + # check the file size of json.sumps + stringified = json.dumps(specs[ix].model_dump(mode="json"), indent=2) + nbytes = len(stringified.encode("utf-8")) + total_bytes += nbytes + avg_bytes = total_bytes / len(ixs) + max_bytes_MB = 3 # safety factor, 4MB is actual amx + max_bytes_B = max_bytes_MB * 1024 * 1024 + sims_per_branch = math.floor(max_bytes_B / avg_bytes) + min_branches_required = math.ceil(len(specs) / sims_per_branch) + logger.info(f"Avg payload size: {int(avg_bytes // 1024):0d} kB") + logger.info(f"Avg sims per branch: {sims_per_branch}") + logger.info(f"Min branches required: {min_branches_required}") + return min_branches_required, sims_per_branch, math.ceil(avg_bytes) diff --git a/src/globi/gis/geometry.py b/src/globi/gis/geometry.py index a49fefa..bd535d0 100644 --- a/src/globi/gis/geometry.py +++ b/src/globi/gis/geometry.py @@ -7,9 +7,13 @@ import geopandas as gpd import numpy as np import pandas as pd +import pyproj from pydantic import BaseModel from scipy.spatial import cKDTree from shapely.geometry import Point, Polygon +from tqdm import tqdm + +tqdm.pandas() logger = logging.getLogger(__name__) @@ -33,7 +37,7 @@ def all_cols(self) -> list[str]: def inject_rotated_rectangles( - gdf: gpd.GeoDataFrame, cart_crs: str + gdf: gpd.GeoDataFrame, cart_crs: str | pyproj.CRS ) -> tuple[gpd.GeoDataFrame, InjectedGeometryColumnMap]: """Inject rotated rectangles into a GeoDataFrame. @@ -81,7 +85,7 @@ def inject_rotated_rectangles( ) a1 = cast( pd.DataFrame, - gdf[injected_column_map.Rotated_Rectangle_col].apply( + gdf[injected_column_map.Rotated_Rectangle_col].progress_apply( lambda x: pd.Series({ "x": x.exterior.coords[0][0], "y": x.exterior.coords[0][1], @@ -90,7 +94,7 @@ def inject_rotated_rectangles( ) b1 = cast( pd.DataFrame, - gdf[injected_column_map.Rotated_Rectangle_col].apply( + gdf[injected_column_map.Rotated_Rectangle_col].progress_apply( lambda x: pd.Series({ "x": x.exterior.coords[1][0], "y": x.exterior.coords[1][1], @@ -99,7 +103,7 @@ def inject_rotated_rectangles( ) a2 = cast( pd.DataFrame, - gdf[injected_column_map.Rotated_Rectangle_col].apply( + gdf[injected_column_map.Rotated_Rectangle_col].progress_apply( lambda x: pd.Series({ "x": x.exterior.coords[2][0], "y": x.exterior.coords[2][1], @@ -164,7 +168,7 @@ def inject_neighbor_ixs( log = log_fn or logger.info # 2. Compute centroids of the rectangles rotated_rectangles = gdf[injected_geometry_col_map.Rotated_Rectangle_col] - rectangle_centers = rotated_rectangles.apply(lambda x: x.centroid) + rectangle_centers = rotated_rectangles.progress_apply(lambda x: x.centroid) # 3. Extract centroid coordinates coords = np.array([ @@ -201,7 +205,7 @@ def check_row_for_ixs(row: gpd.GeoSeries): if remove_intersections: log("removing intersections") - gdf[injected_neighbor_column_map.Neighbor_Ixs_col] = gdf.apply( + gdf[injected_neighbor_column_map.Neighbor_Ixs_col] = gdf.progress_apply( check_row_for_ixs, axis=1 ) log("done removing intersections") @@ -293,14 +297,14 @@ def convert_neighbors( Returns: gdf (gpd.GeoDataFrame): The GeoDataFrame """ - neighbors = gdf[neighbor_col].apply( + neighbors = gdf[neighbor_col].progress_apply( lambda x: extract_neighbors_geo( gdf=gdf, neighbor_ixs=x, geometry_col=geometry_col, ) ) - neighbor_heights = gdf[neighbor_col].apply( + neighbor_heights = gdf[neighbor_col].progress_apply( lambda x: extract_neighbor_heights( gdf=gdf, neighbor_ixs=x, @@ -310,7 +314,7 @@ def convert_neighbors( ) gdf[neighbor_geo_out_col] = neighbors gdf[neighbor_heights_out_col] = neighbor_heights - gdf[neighbor_floors_out_col] = gdf[neighbor_heights_out_col].apply( + gdf[neighbor_floors_out_col] = gdf[neighbor_heights_out_col].progress_apply( lambda heights: ( [ int(h / neighbor_f2f_height) diff --git a/src/globi/gis/utils.py b/src/globi/gis/utils.py index d901fc0..2a97d9c 100644 --- a/src/globi/gis/utils.py +++ b/src/globi/gis/utils.py @@ -9,11 +9,13 @@ import geopandas as gpd import numpy as np import pandas as pd +import pyproj from epinterface.sbem.fields.spec import ( CategoricalFieldSpec, NumericFieldSpec, SemanticModelFields, ) +from tqdm import tqdm from globi.gis.errors import ( GISFileHasInvalidCategoricalSemanticFieldError, @@ -33,6 +35,8 @@ from globi.gis.weather import closest_epw from globi.type_utils import BasementAtticOccupationConditioningStatus +tqdm.pandas() + logger = logging.getLogger(__name__) @@ -41,7 +45,7 @@ def reproject_gdf( gdf: gpd.GeoDataFrame, cart_crs: str, log_fn: LogLikeFn | None = None -) -> gpd.GeoDataFrame: +) -> tuple[gpd.GeoDataFrame, pyproj.CRS | str]: """We want to have some safety to ensure there is no confusing behavior with the provided CRS. We want to ensure that either: @@ -68,6 +72,16 @@ def reproject_gdf( if not gdf.crs: raise GISFileHasNoCRSError() + original_cart_crs = pyproj.CRS.from_string(cart_crs) + estimated_utm_crs = gdf.estimate_utm_crs() + if original_cart_crs == "EPSG:3857" or not original_cart_crs.is_projected: + log( + "Specified cartographic CRS is EPSG:3857 or not projected, will use estimated UTM CRS for cartesian projection." + ) + final_cart_crs = estimated_utm_crs + else: + final_cart_crs = original_cart_crs + if gdf.crs == "EPSG:3857": log("Reprojecting gis file to EPSG:4326 from EPSG:3857.") gdf.to_crs("EPSG:4326", inplace=True) @@ -78,10 +92,10 @@ def reproject_gdf( raise GISFileHasUnexpectedCRSError(str(cart_crs), str(current_crs)) if current_crs != "EPSG:4326": - log("Projecting gis file to EPSG:4326 from {current_crs}.") + log(f"Projecting gis file to EPSG:4326 from {current_crs}.") gdf.to_crs("EPSG:4326", inplace=True) - return gdf + return gdf, final_cart_crs def rename_shp_cols( @@ -258,6 +272,7 @@ def drop_by_area( gdf: gpd.GeoDataFrame, area_col: str, min_area: float, + max_area: float, log_fn: LogLikeFn | None = None, ) -> tuple[gpd.GeoDataFrame, int]: """Drop features with an area less than the minimum area. @@ -266,6 +281,7 @@ def drop_by_area( gdf (gpd.GeoDataFrame): The input GeoDataFrame. area_col (str): The name of the area column. min_area (float): The minimum area. + max_area (float): The maximum area. log_fn (LogLikeFn | None): The function to use for logging. Returns: @@ -274,7 +290,9 @@ def drop_by_area( """ log = log_fn or logger.info log("Checking that features have an area greater than the minimum area...") - is_valid_area = cast(pd.Series, gdf[area_col] >= min_area) + is_valid_area = cast( + pd.Series, (gdf[area_col] >= min_area) & (gdf[area_col] <= max_area) + ) n_dropped = len(gdf) - is_valid_area.sum() gdf = cast(gpd.GeoDataFrame, gdf[is_valid_area]) if n_dropped > 0: @@ -650,7 +668,7 @@ def handle_epwzip( weather_file_col: str | None, assumed_epwzip: Path | str | None, epw_query: str | None, - cart_crs: str, + cart_crs: str | pyproj.CRS, log_fn: LogLikeFn | None = None, ) -> tuple[gpd.GeoDataFrame, str]: """Handle the epwzip column. @@ -710,7 +728,7 @@ def handle_epw_path(x: str) -> str: def inject_semantic_fields( gdf: gpd.GeoDataFrame, - semantic_fields: SemanticModelFields, + semantic_fields: SemanticModelFields | None, ) -> tuple[gpd.GeoDataFrame, str]: """Inject the semantic fields into the GeoDataFrame. @@ -723,11 +741,14 @@ def inject_semantic_fields( semantic_fields_file_col (str): The name of the semantic fields file column. """ semantic_fields_context_col = "GLOBI_SEMANTIC_FIELDS_CONTEXT" - gdf[semantic_fields_context_col] = gdf.apply( - lambda row: { - field_name: row[field_name] - for field_name in semantic_fields.semantic_field_names - }, - axis=1, - ) + if semantic_fields: + gdf[semantic_fields_context_col] = gdf.progress_apply( + lambda row: { + field_name: row[field_name] + for field_name in semantic_fields.semantic_field_names + }, + axis=1, + ) + else: + gdf[semantic_fields_context_col] = [{} for _ in range(len(gdf))] return gdf, semantic_fields_context_col diff --git a/src/globi/gis/weather.py b/src/globi/gis/weather.py index 4b32045..1f7f3bd 100644 --- a/src/globi/gis/weather.py +++ b/src/globi/gis/weather.py @@ -7,6 +7,7 @@ import geopandas as gpd import numpy as np +import pyproj from scipy.spatial import cKDTree logger = logging.getLogger(__name__) @@ -17,7 +18,7 @@ def closest_epw( query_pts: gpd.GeoSeries, source_filter: str | None = None, - crs: str | int = 3857, + crs: str | int | pyproj.CRS = 3857, distance_threshold_meters: int | None = 500_000, metadata: gpd.GeoDataFrame | None = None, log_fn: Callable | None = None, diff --git a/src/globi/models/configs.py b/src/globi/models/configs.py index e860c09..3b23eea 100644 --- a/src/globi/models/configs.py +++ b/src/globi/models/configs.py @@ -40,8 +40,6 @@ def does_dataframe_output(self) -> bool: class DeterministicGISPreprocessorConfig(BaseConfig): """Configuration for the GIS preprocessor.""" - # TODO: design decision - Separated out this config since this would be for deterministic elements primarily - cart_crs: str = Field( default="EPSG:3857", description="The cartesian CRS to project to.", @@ -50,7 +48,13 @@ class DeterministicGISPreprocessorConfig(BaseConfig): default=10.0, ge=1, le=1000, - description="The minimum area of a building to be included [m^2].", + description="The minimum area of a building's footprint to be included [m^2].", + ) + max_building_area: float = Field( + default=100_000.0, + ge=1, + le=1_000_000, + description="The maximum area of a building's footprint to be included [m^2].", ) min_edge_length: float = Field( default=3.0, @@ -127,6 +131,12 @@ class DeterministicGISPreprocessorConfig(BaseConfig): default_factory=lambda: "source in ['tmyx']", description="The EPW query filter for closest_epw.", ) + check_semantic_fields: bool = Field( + default=True, + description="Whether to check that the semantic fields columns are all present in the GIS file.", + ) + # TODO: add zoning calculator options + # TODO: add use fraction options class GISPreprocessorColumnMap(BaseConfig): @@ -154,6 +164,7 @@ class GISPreprocessorColumnMap(BaseConfig): Basement_col: str Attic_col: str Exposed_Basement_Frac_col: str + # TODO: add basement_use_fraction, attic_use_fraction, attic_height class FileConfig(BaseConfig): @@ -162,9 +173,7 @@ class FileConfig(BaseConfig): gis_file: Path = Field(..., description="The path to the local GIS file.") db_file: Path # these could be file refs? semantic_fields_file: Path # these could be file refs? - epwzip_file: ( - Path | str | None - ) # TODO: our gis to model conversion should handle EPW identification; see gis job submission in epengine + epwzip_file: Path | str | None component_map_file: Path diff --git a/src/globi/models/surrogate/__init__.py b/src/globi/models/surrogate/__init__.py new file mode 100644 index 0000000..d5affc9 --- /dev/null +++ b/src/globi/models/surrogate/__init__.py @@ -0,0 +1 @@ +"""Models used for the surrogate pipeline.""" diff --git a/src/globi/models/surrogate/backends/__init__.py b/src/globi/models/surrogate/backends/__init__.py new file mode 100644 index 0000000..2a5650b --- /dev/null +++ b/src/globi/models/surrogate/backends/__init__.py @@ -0,0 +1,31 @@ +"""Backend implementations for surrogate model training.""" + +import sys + +from globi.models.surrogate.backends.base import ( + SurrogateModelBackend, + TrainedModelWithArtifacts, + TrainingContext, + preload_torch_runtime, +) + +# On macOS, torch must be loaded before xgboost/lightgbm are *imported*, or torch ops +# deadlock on a second OpenMP runtime later on (see `preload_torch_runtime`). +if sys.platform == "darwin": + preload_torch_runtime() +from globi.models.surrogate.backends.lgb import LGBBackend +from globi.models.surrogate.backends.nn import NNBackend +from globi.models.surrogate.backends.xgb import XGBBackend + +MLBackend = XGBBackend | LGBBackend | NNBackend + +__all__ = [ + "LGBBackend", + "MLBackend", + "NNBackend", + "SurrogateModelBackend", + "TrainedModelWithArtifacts", + "TrainingContext", + "XGBBackend", + "preload_torch_runtime", +] diff --git a/src/globi/models/surrogate/backends/base.py b/src/globi/models/surrogate/backends/base.py new file mode 100644 index 0000000..8c47ac2 --- /dev/null +++ b/src/globi/models/surrogate/backends/base.py @@ -0,0 +1,175 @@ +"""Core backend abstractions for surrogate model training.""" + +import contextlib +import tempfile +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from pydantic import BaseModel, Field +from scythe.utils.filesys import FileReference, fetch_uri + +from globi.models.surrogate.transforms import PrepDataResult, Transformers, predict + + +@dataclass(frozen=True) +class TrainingContext: + """Runtime context shared by all model backends.""" + + prepped_data: PrepDataResult + tempdir: Path + + +@dataclass(frozen=True) +class TrainedModel: + """A trained model with a regressor and transforms.""" + + model_object: Any + transformers: Transformers + + +@dataclass(frozen=True) +class TrainedArtifacts: + """Artifacts for a trained model.""" + + regressor_path: Path + transforms_path: Path + + +@dataclass(frozen=True) +class TrainedModelWithArtifacts: + """Model artifacts emitted by backend training.""" + + trained_model: TrainedModel + artifacts: TrainedArtifacts + + +ML_MODEL_CACHE: dict[str, dict[FileReference, Any]] = {} +ML_TRANSFORMS_CACHE: dict[FileReference, Transformers] = {} + + +def preload_torch_runtime() -> None: + """Import torch (if installed) before any other OpenMP-backed library runs. + + xgboost / lightgbm and torch each ship their own OpenMP runtime; on macOS, if + torch's is loaded *after* one of the others has already run a parallel region, + torch ops (e.g. layer_norm) deadlock. Loading torch first lets the other + libraries share its runtime. Cheap once imported; a no-op without torch. + """ + with contextlib.suppress(ImportError): + import torch # noqa: F401 + + +class SurrogateModelBackend(BaseModel, ABC): + """Base interface for model-specific training and loading.""" + + ml_backend: Any = Field( + ..., + description="The discriminator for selecting a model backend.", + ) + + @abstractmethod + def train(self, context: TrainingContext) -> TrainedModel: + """Train the model and serialize artifacts.""" + + def train_and_save(self, context: TrainingContext) -> TrainedModelWithArtifacts: + """Train the model and save the artifacts.""" + preload_torch_runtime() + trained_model = self.train(context) + artifacts = TrainedArtifacts( + regressor_path=self.save_model(trained_model.model_object, context.tempdir), + transforms_path=self.save_transforms( + trained_model.transformers, context.tempdir + ), + ) + return TrainedModelWithArtifacts(trained_model, artifacts) + + @abstractmethod + def save_model(self, model_object: Any, output_dir: Path) -> Path: + """Serialize backend-specific model artifacts to disk.""" + + @classmethod + @abstractmethod + def load_model(cls, regressor_path: Path) -> Any: + """Load backend-specific model artifacts from disk.""" + + def load_model_from_cache(self, model_ref: FileReference) -> Any: + """Load a model from the cache.""" + preload_torch_runtime() + if self.ml_backend not in ML_MODEL_CACHE: + ML_MODEL_CACHE[self.ml_backend] = {} + if model_ref in ML_MODEL_CACHE[self.ml_backend]: + return ML_MODEL_CACHE[self.ml_backend][model_ref] + + if isinstance(model_ref, Path): + model = self.load_model(model_ref) + ML_MODEL_CACHE[self.ml_backend][model_ref] = model + return model + else: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + local_model_path = fetch_uri( + model_ref, + tmp_path + / Path("model.bin").with_suffix(Path(str(model_ref)).suffix), + ) + model = self.load_model(local_model_path) + ML_MODEL_CACHE[self.ml_backend][model_ref] = model + return model + + @classmethod + @abstractmethod + def make_raw_predict_fn( + cls, + model_object: Any, + ) -> Callable[[pd.DataFrame, list[str]], np.ndarray]: + """Bind a backend-specific raw prediction callable. + + The signature of the raw prediction callable is: + def raw_pred_fn(x: pd.DataFrame, col_order: list[str]) -> np.ndarray: + ... + """ + + def save_transforms(self, transformers: Transformers, output_dir: Path) -> Path: + """Save transforms in a backend-agnostic way.""" + transforms_path = output_dir / "transforms.yml" + with open(transforms_path, "w") as f: + yaml.dump( + transformers.model_dump(mode="json"), f, indent=2, sort_keys=False + ) + return transforms_path + + @classmethod + def load_transforms(cls, transforms_ref: FileReference) -> Transformers: + """Load transforms from a backend-agnostic way.""" + if transforms_ref in ML_TRANSFORMS_CACHE: + return ML_TRANSFORMS_CACHE[transforms_ref] + if isinstance(transforms_ref, Path): + with open(transforms_ref) as f: + transformers_yaml = yaml.safe_load(f) + transformers = Transformers.model_validate(transformers_yaml) + ML_TRANSFORMS_CACHE[transforms_ref] = transformers + return transformers + else: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) / "transforms.yml" + local_transforms_path = fetch_uri(transforms_ref, tmp_path) + with open(local_transforms_path) as f: + transformers_yaml = yaml.safe_load(f) + transformers = Transformers.model_validate(transformers_yaml) + ML_TRANSFORMS_CACHE[transforms_ref] = transformers + return transformers + + @classmethod + def build_predict_fn( + cls, *, model_object: Any, transformers: Transformers + ) -> Callable[[pd.DataFrame], pd.DataFrame]: + """Create a dataframe-level prediction callable from raw backend prediction.""" + preload_torch_runtime() + raw_pred = cls.make_raw_predict_fn(model_object) + return lambda x: predict(x, conf=transformers, pred_fn=raw_pred) diff --git a/src/globi/models/surrogate/backends/lgb.py b/src/globi/models/surrogate/backends/lgb.py new file mode 100644 index 0000000..2228b92 --- /dev/null +++ b/src/globi/models/surrogate/backends/lgb.py @@ -0,0 +1,195 @@ +"""LightGBM backend for surrogate training.""" + +import logging +import zipfile +from pathlib import Path +from typing import Any, Literal + +import numpy as np +import pandas as pd +from pydantic import BaseModel, Field + +from globi.models.surrogate.backends.base import ( + SurrogateModelBackend, + TrainedModel, + TrainingContext, +) + +logger = logging.getLogger(__name__) + + +class LGBTrainerConfig(BaseModel): + """The trainer hyperparameters for the lightgbm model.""" + + num_boost_round: int = Field( + default=4000, description="The number of boosting rounds." + ) + early_stopping_rounds: int = Field( + default=20, description="The number of boosting rounds to early stop." + ) + + +class LGBModelConfig(BaseModel): + """The model hyperparameters for the lightgbm model.""" + + objective: Literal["regression", "binary", "multiclass"] = Field( + default="regression", description="The objective function to use." + ) + metric: Literal["rmse"] = Field( + default="rmse", description="The metric to optimize." + ) + learning_rate: float = Field(default=0.1, description="The learning rate.") + num_leaves: int = Field(default=31, description="The number of leaves in the tree.") + max_depth: int = Field(default=-1, description="The maximum depth of the tree.") + min_data_in_leaf: int | None = Field( + default=None, + description="The minimum number of samples per leaf (lightgbm default: 20). Lower it for small training sets.", + ) + + @property + def param_dict(self) -> dict[str, Any]: + """The dictionary of parameters.""" + return self.model_dump(exclude_none=True) + + +class LGBBackend(SurrogateModelBackend): + """The parameters/backend for the lightgbm model.""" + + ml_backend: Literal["lgb"] = Field( + default="lgb", description="The type of model to use." + ) + hp: LGBModelConfig = Field( + default_factory=LGBModelConfig, + description="The hyperparameters for the model.", + ) + trainer: LGBTrainerConfig = Field( + default_factory=LGBTrainerConfig, + description="The trainer hyperparameters for the model.", + ) + + def train(self, context: TrainingContext) -> TrainedModel: + """Train lightgbm models and write model artifacts.""" + import lightgbm as lgb + + prep = context.prepped_data + col_order = prep.transformers.y.targets + models: dict[str, lgb.Booster] = {} + + logger.info("Training LightGBM model...") + for col in col_order: + lgb_train = lgb.Dataset( + prep.transformed.train.x, + prep.transformed.train.y.reset_index(drop=True)[col], + ) + lgb_test = lgb.Dataset( + prep.transformed.test.x, + prep.transformed.test.y.reset_index(drop=True)[col], + ) + + lgb_params = self.hp.param_dict + # TODO: inspect runtimes + # if torch.cuda.is_available(): + # lgb_params["device"] = "gpu" + + model = lgb.train( + lgb_params, + lgb_train, + num_boost_round=self.trainer.num_boost_round, + valid_sets=[lgb_test], + valid_names=["eval"], + callbacks=[lgb.early_stopping(self.trainer.early_stopping_rounds)], + ) + models[col] = model + + logger.info("Trained LightGBM model.") + return TrainedModel( + model_object=models, + transformers=prep.transformers, + ) + + def save_model(self, model_object: Any, output_dir: Path) -> Path: + """Serialize lightgbm models as a portable zip archive.""" + import lightgbm as lgb + import yaml + + if not isinstance(model_object, dict): + msg = f"Expected dict of lightgbm models, got {type(model_object)}" + raise TypeError(msg) + + model_root_path = output_dir / "lgb" + model_root_path.mkdir(parents=True, exist_ok=True) + model_paths: dict[str, str] = {} + for col, booster in model_object.items(): + if not isinstance(booster, lgb.Booster): + msg = f"Model for column {col} is not a Booster: {type(booster)}" + raise TypeError(msg) + relative_model_path = Path("models") / f"{col}.lgb" + model_path = model_root_path / relative_model_path + model_path.parent.mkdir(parents=True, exist_ok=True) + booster.save_model(model_path.as_posix()) + model_paths[col] = relative_model_path.as_posix() + + with open(model_root_path / "col_spec.yaml", "w") as f: + yaml.dump(model_paths, f, indent=2, sort_keys=False) + + model_output_zip_path = output_dir / "model.zip" + with zipfile.ZipFile(model_output_zip_path, "w") as zipf: + for path in model_root_path.glob("**/*"): + if path.is_file(): + zipf.write(path, path.relative_to(model_root_path)) + return model_output_zip_path + + @classmethod + def load_model(cls, regressor_path: Path) -> Any: + """Load lightgbm models from a zip archive.""" + import tempfile + + import lightgbm as lgb + import yaml + + models: dict[str, lgb.Booster] = {} + with tempfile.TemporaryDirectory() as tmp: + extract_root = Path(tmp) / "lgb" + extract_root.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(regressor_path, "r") as zipf: + zipf.extractall(extract_root) + + with open(extract_root / "col_spec.yaml") as f: + col_spec = yaml.safe_load(f) + if not isinstance(col_spec, dict): + msg = f"Expected a dict in col_spec.yaml, got {type(col_spec)}" + raise TypeError(msg) + + for col, rel_path in col_spec.items(): + model_path = extract_root / Path(str(rel_path)) + models[col] = lgb.Booster(model_file=model_path.as_posix()) + return models + + @classmethod + def make_raw_predict_fn(cls, model_object: Any): + """Create the raw lightgbm prediction callable.""" + return lambda x, col_order: lgb_pred(x, col_order, model=model_object) + + +def lgb_pred(x: pd.DataFrame, col_order: list[str], *, model) -> np.ndarray: + """Predict the targets for encoded features using lightgbm.""" + import lightgbm as lgb + + if not isinstance(model, dict): + msg = ( + f"Model is not a dictionary: {type(model)}. " + "LGB pred expects a dictionary of models per column." + ) + raise TypeError(msg) + + preds = {} + for col in col_order: + booster = model[col] + if not isinstance(booster, lgb.Booster): + msg = ( + f"Model for column {col} is not a lightgbm model: {type(booster)}. " + "LGB pred expects a dictionary of lightgbm models per column." + ) + raise TypeError(msg) + preds[col] = booster.predict(x.reset_index(drop=True)) + return np.stack(list(preds.values()), axis=1) diff --git a/src/globi/models/surrogate/backends/nn.py b/src/globi/models/surrogate/backends/nn.py new file mode 100644 index 0000000..a9ed853 --- /dev/null +++ b/src/globi/models/surrogate/backends/nn.py @@ -0,0 +1,538 @@ +"""PyTorch neural network backend for surrogate training.""" + +import copy +import gc +import logging +import warnings +from collections.abc import Callable +from pathlib import Path +from typing import Annotated, Any, Literal + +import numpy as np +import pandas as pd +from pydantic import BaseModel, Field + +from globi.models.surrogate.backends.base import ( + SurrogateModelBackend, + TrainedModel, + TrainingContext, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Activation helpers +# --------------------------------------------------------------------------- + +ACTIVATION_REGISTRY: dict[str, type] = {} + + +def _populate_activation_registry() -> None: + import torch.nn as nn + + ACTIVATION_REGISTRY.update({ + "relu": nn.ReLU, + "gelu": nn.GELU, + "silu": nn.SiLU, + "tanh": nn.Tanh, + }) + + +def _get_activation(name: str): + if not ACTIVATION_REGISTRY: + _populate_activation_registry() + cls = ACTIVATION_REGISTRY.get(name) + if cls is None: + msg = f"Unknown activation: {name!r}. Choose from {list(ACTIVATION_REGISTRY)}" + raise ValueError(msg) + return cls() + + +# --------------------------------------------------------------------------- +# Model hyper-parameters (architecture) +# --------------------------------------------------------------------------- + + +class NNModelConfig(BaseModel): + """Architectural hyperparameters for the MLP surrogate.""" + + activation: Literal["relu", "gelu", "silu", "tanh"] = Field( + default="silu", description="Activation function used in every block." + ) + layer_norm: bool = Field( + default=True, + description="Enable pre-norm LayerNorm in every block.", + ) + dropout: float | None = Field( + default=None, + description="Dropout probability applied in every block. None disables dropout.", + ) + hidden_dims: list[int] = Field( + default_factory=lambda: [256, 256, 256, 256], + description="Width of each hidden layer. Length determines depth.", + ) + init_batch_norm: bool = Field( + default=False, + description="Apply batch normalization to the input features.", + ) + quadratic_features: bool = Field( + default=False, + description="Prepend a quadratic polynomial feature expansion (outer product) to the MLP input.", + ) + + +# --------------------------------------------------------------------------- +# Optimizer configs (discriminated union) +# --------------------------------------------------------------------------- + + +class AdamOptimizerConfig(BaseModel): + """Configuration for the Adam optimizer.""" + + optimizer: Literal["adam"] = "adam" + lr: float = Field(default=1e-4, description="Learning rate.") + weight_decay: float = Field(default=0.0, description="Weight decay (L2 penalty).") + betas: tuple[float, float] = Field( + default=(0.9, 0.999), description="Adam beta coefficients." + ) + + def build(self, params): + """Instantiate an Adam optimizer for the given parameters.""" + import torch.optim as optim + + return optim.Adam( + params, lr=self.lr, weight_decay=self.weight_decay, betas=self.betas + ) + + +class SGDOptimizerConfig(BaseModel): + """Configuration for the SGD optimizer.""" + + optimizer: Literal["sgd"] = "sgd" + lr: float = Field(default=1e-2, description="Learning rate.") + momentum: float = Field(default=0.9, description="Momentum factor.") + weight_decay: float = Field(default=0.0, description="Weight decay (L2 penalty).") + + def build(self, params): + """Instantiate an SGD optimizer for the given parameters.""" + import torch.optim as optim + + return optim.SGD( + params, lr=self.lr, momentum=self.momentum, weight_decay=self.weight_decay + ) + + +OptimizerConfig = Annotated[ + AdamOptimizerConfig | SGDOptimizerConfig, + Field(discriminator="optimizer"), +] + +# --------------------------------------------------------------------------- +# LR scheduler configs (discriminated union) +# --------------------------------------------------------------------------- + + +class CosineAnnealingSchedulerConfig(BaseModel): + """Configuration for CosineAnnealingLR.""" + + scheduler: Literal["cosine_annealing"] = "cosine_annealing" + T_max: int | None = Field( + default=None, description="Max iterations. Defaults to n_epochs at runtime." + ) + eta_min: float = Field(default=0, description="Minimum learning rate.") + + def build(self, optimizer, *, n_epochs: int): + """Instantiate a CosineAnnealingLR scheduler.""" + import torch.optim.lr_scheduler as lr_scheduler + + return lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=self.T_max if self.T_max is not None else n_epochs, + eta_min=self.eta_min, + ) + + +class StepSchedulerConfig(BaseModel): + """Configuration for StepLR.""" + + scheduler: Literal["step"] = "step" + step_size: int = Field(default=50, description="Period of learning rate decay.") + gamma: float = Field(default=0.5, description="Multiplicative decay factor.") + + def build(self, optimizer, *, n_epochs: int): + """Instantiate a StepLR scheduler.""" + import torch.optim.lr_scheduler as lr_scheduler + + return lr_scheduler.StepLR( + optimizer, step_size=self.step_size, gamma=self.gamma + ) + + +class NoSchedulerConfig(BaseModel): + """Placeholder that disables LR scheduling.""" + + scheduler: Literal["none"] = "none" + + def build(self, optimizer, *, n_epochs: int): + """Return None (no scheduler).""" + return None + + +SchedulerConfig = Annotated[ + CosineAnnealingSchedulerConfig | StepSchedulerConfig | NoSchedulerConfig, + Field(discriminator="scheduler"), +] + +# --------------------------------------------------------------------------- +# Trainer hyper-parameters +# --------------------------------------------------------------------------- + + +class NNTrainerConfig(BaseModel): + """Training hyper-parameters for the neural network backend.""" + + epochs: int = Field(default=4000, description="Maximum number of training epochs.") + batch_size: int = Field(default=256, description="Mini-batch size.") + early_stopping_patience: int | None = Field( + default=20, + description="Epochs without validation improvement before stopping. None disables early stopping.", + ) + optimizer: OptimizerConfig = Field( + default_factory=AdamOptimizerConfig, + description="Optimizer configuration.", + ) + scheduler: SchedulerConfig = Field( + default_factory=CosineAnnealingSchedulerConfig, + description="Learning rate scheduler configuration.", + ) + + +# --------------------------------------------------------------------------- +# PyTorch modules +# --------------------------------------------------------------------------- + + +class ResidualMLPBlock: + """A single residual MLP block. + + Instantiated as an ``nn.Module`` at runtime so that ``torch`` is only + imported when actually needed. + """ + + @staticmethod + def create( + in_dim: int, + out_dim: int, + *, + activation: str, + layer_norm: bool, + dropout: float | None, + ): + """Build and return an ``nn.Module`` implementing one residual block.""" + import torch + import torch.nn as nn + + class _Block(nn.Module): + def __init__(self) -> None: + super().__init__() + self.norm = nn.LayerNorm(in_dim) if layer_norm else nn.Identity() + self.fc = nn.Linear(in_dim, out_dim) + self.act = _get_activation(activation) + self.drop = ( + nn.Dropout(dropout) if dropout is not None else nn.Identity() + ) + self.skip_proj = ( + nn.Linear(in_dim, out_dim) if in_dim != out_dim else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + residual = self.skip_proj(x) + h = self.norm(x) + h = self.fc(h) + h = self.act(h) + h = self.drop(h) + return residual + h + + return _Block() + + +class SurrogateMLP: + """Factory for the full MLP ``nn.Module``. + + Wrapping the ``nn.Module`` in a factory avoids a module-level ``torch`` + import, matching the lazy-import pattern of the other backends. + """ + + @staticmethod + def from_config( + n_features: int, + n_outputs: int, + config: NNModelConfig, + ): + """Construct the MLP from architectural config and runtime dimensions.""" + import torch + import torch.nn as nn + + use_quad = config.quadratic_features + init_batch_norm = config.init_batch_norm + effective_input_dim = n_features**2 if use_quad else n_features + dims = [effective_input_dim, *config.hidden_dims] + + class _MLP(nn.Module): + def __init__(self) -> None: + super().__init__() + self.quadratic_features = use_quad + self.init_batch_norm = init_batch_norm + self.bn = ( + nn.BatchNorm1d(effective_input_dim) + if init_batch_norm + else nn.Identity() + ) + blocks: list[nn.Module] = [] + for i in range(len(dims) - 1): + blocks.append( + ResidualMLPBlock.create( + dims[i], + dims[i + 1], + activation=config.activation, + layer_norm=config.layer_norm, + dropout=config.dropout, + ) + ) + self.blocks = nn.Sequential(*blocks) if blocks else nn.Identity() + self.head = nn.Linear(dims[-1], n_outputs) + self.n_features = n_features + self.n_outputs = n_outputs + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.quadratic_features: + x = (x.unsqueeze(-1) * x.unsqueeze(-2)).flatten(start_dim=-2) + x = self.bn(x) + return self.head(self.blocks(x)) + + return _MLP() + + +# --------------------------------------------------------------------------- +# Backend +# --------------------------------------------------------------------------- + + +class NNBackend(SurrogateModelBackend): + """PyTorch MLP backend for surrogate model training.""" + + ml_backend: Literal["nn"] = Field( + default="nn", description="The type of model to use." + ) + hp: NNModelConfig = Field( + default_factory=NNModelConfig, + description="The architectural hyperparameters for the model.", + ) + trainer: NNTrainerConfig = Field( + default_factory=NNTrainerConfig, + description="The training hyperparameters for the model.", + ) + + # ----- training -------------------------------------------------------- + + def train(self, context: TrainingContext) -> TrainedModel: + """Train a PyTorch MLP and return the best model.""" + import torch + from torch.utils.data import DataLoader, TensorDataset + + prep = context.prepped_data + + x_train_np = prep.transformed.train.x.reset_index(drop=True).to_numpy( + dtype=np.float32 + ) + y_train_np = prep.transformed.train.y.reset_index(drop=True).to_numpy( + dtype=np.float32 + ) + x_val_np = prep.transformed.test.x.reset_index(drop=True).to_numpy( + dtype=np.float32 + ) + y_val_np = prep.transformed.test.y.reset_index(drop=True).to_numpy( + dtype=np.float32 + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if not torch.cuda.is_available(): + warnings.warn("CUDA is not available, using CPU.", stacklevel=2) + + n_features = x_train_np.shape[1] + n_outputs = y_train_np.shape[1] + + model = SurrogateMLP.from_config(n_features, n_outputs, self.hp) + model = model.to(device) + + train_ds = TensorDataset( + torch.from_numpy(x_train_np), torch.from_numpy(y_train_np) + ) + val_ds = TensorDataset(torch.from_numpy(x_val_np), torch.from_numpy(y_val_np)) + train_loader = DataLoader( + train_ds, batch_size=self.trainer.batch_size, shuffle=True, drop_last=True + ) + val_loader = DataLoader( + val_ds, batch_size=self.trainer.batch_size, shuffle=False, drop_last=True + ) + + optimizer = self.trainer.optimizer.build(model.parameters()) + lr_scheduler = self.trainer.scheduler.build( + optimizer, n_epochs=self.trainer.epochs + ) + + loss_fn = torch.nn.MSELoss() + + best_val_loss = float("inf") + best_state: dict[str, Any] = {} + epochs_without_improvement = 0 + + logger.info( + f"Training NN ({n_features} -> {self.hp.hidden_dims} -> {n_outputs}) on {device}..." + ) + + train_loss_history = [] + val_loss_history = [] + for epoch in range(self.trainer.epochs): + # --- train ------------------------------------------------- + model.train() + train_loss_accum = 0.0 + n_train_batches = 0 + for xb, yb in train_loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + pred = model(xb) + loss = loss_fn(pred, yb) + loss.backward() + optimizer.step() + train_loss_accum += loss.item() + n_train_batches += 1 + + # --- validate ---------------------------------------------- + model.eval() + val_loss_accum = 0.0 + n_val_batches = 0 + with torch.no_grad(): + for xb, yb in val_loader: + xb, yb = xb.to(device), yb.to(device) + val_loss_accum += loss_fn(model(xb), yb).item() + n_val_batches += 1 + + avg_train = train_loss_accum / max(n_train_batches, 1) + avg_val = val_loss_accum / max(n_val_batches, 1) + train_loss_history.append(avg_train) + val_loss_history.append(avg_val) + + if lr_scheduler is not None: + lr_scheduler.step() + + if avg_val < best_val_loss: + best_val_loss = avg_val + best_state = copy.deepcopy(model.state_dict()) + epochs_without_improvement = 0 + else: + epochs_without_improvement += 1 + + print_every_n = 10 + if epoch % print_every_n == 0 or epoch == self.trainer.epochs - 1: + last_n_train = train_loss_history[-print_every_n:] + last_n_val = val_loss_history[-print_every_n:] + avg_last_n_train = sum(last_n_train) / len(last_n_train) + avg_last_n_val = sum(last_n_val) / len(last_n_val) + logger.info( + f" Epoch {epoch:>4d}/{self.trainer.epochs} " + f"train_mse={avg_last_n_train:.6f} val_mse={avg_last_n_val:.6f} " + f"best_val={best_val_loss:.6f}" + ) + + if ( + self.trainer.early_stopping_patience is not None + and epochs_without_improvement >= self.trainer.early_stopping_patience + ): + logger.info( + f" Early stopping at epoch {epoch} " + f"(no improvement for {self.trainer.early_stopping_patience} epochs)." + ) + break + + if best_state: + model.load_state_dict(best_state) + model.eval() + logger.info("Trained NN model.") + # clean up some gpu memory by deleting tensors and so on + del train_ds, val_ds, train_loader, val_loader + gc.collect() + torch.cuda.empty_cache() + + return TrainedModel( + model_object=model, + transformers=prep.transformers, + ) + + # ----- serialization --------------------------------------------------- + + def save_model(self, model_object: Any, output_dir: Path) -> Path: + """Serialize the MLP checkpoint (weights + architecture metadata).""" + import torch + + model_path = output_dir / "model.pt" + torch.save( + { + "state_dict": model_object.state_dict(), + "n_features": model_object.n_features, + "n_outputs": model_object.n_outputs, + "model_config": self.hp.model_dump(mode="json"), + }, + model_path, + ) + return model_path + + @classmethod + def load_model(cls, regressor_path: Path) -> Any: + """Load a checkpoint dict from disk.""" + import torch + + return torch.load(regressor_path, map_location="cpu", weights_only=False) + + @classmethod + def make_raw_predict_fn( + cls, + model_object: Any, + ) -> Callable[[pd.DataFrame, list[str]], np.ndarray]: + """Create the raw NN prediction callable.""" + import torch + from torch.utils.data import DataLoader, TensorDataset + + if isinstance(model_object, dict): + config = NNModelConfig(**model_object["model_config"]) + n_features: int = model_object["n_features"] + n_outputs: int = model_object["n_outputs"] + model = SurrogateMLP.from_config(n_features, n_outputs, config) + model.load_state_dict(model_object["state_dict"]) + else: + model = model_object + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model.to(device) + model.eval() + + def _predict(x: pd.DataFrame, col_order: list[str]) -> np.ndarray: + batch_size = 1024 + dataloader = DataLoader( + TensorDataset( + torch.from_numpy( + x.reset_index(drop=True).to_numpy(dtype=np.float32) + ), + ), + batch_size=batch_size, + shuffle=False, + ) + preds = [] + with torch.no_grad(): + for (xb,) in dataloader: + xb = xb.to(device) + pred = model(xb) + preds.append(pred.cpu().numpy()) + return np.concatenate(preds, axis=0) + + return _predict diff --git a/src/globi/models/surrogate/backends/xgb.py b/src/globi/models/surrogate/backends/xgb.py new file mode 100644 index 0000000..224c0f4 --- /dev/null +++ b/src/globi/models/surrogate/backends/xgb.py @@ -0,0 +1,158 @@ +"""XGBoost backend for surrogate training.""" + +import logging +import warnings +from pathlib import Path +from typing import Any, Literal + +import numpy as np +import pandas as pd +from pydantic import BaseModel, Field + +from globi.models.surrogate.backends.base import ( + SurrogateModelBackend, + TrainedModel, + TrainingContext, +) + +logger = logging.getLogger(__name__) + + +class XGBTrainerConfig(BaseModel): + """The trainer hyperparameters for the xgboost model.""" + + num_boost_round: int = Field( + default=4000, description="The number of boosting rounds." + ) + early_stopping_rounds: int = Field( + default=10, description="The number of boosting rounds to early stop." + ) + verbose_eval: bool = Field( + default=True, description="Whether to print verbose evaluation results." + ) + + +class XGBModelConfig(BaseModel): + """The model hyperparameters for the xgboost model.""" + + max_depth: int = Field(default=5, description="The maximum depth of the tree.") + eta: float = Field(default=0.1, description="The learning rate.") + min_child_weight: int | None = Field( + default=3, description="The minimum child weight." + ) + subsample: float | None = Field(default=None, description="The subsample rate.") + colsample_bytree: float | None = Field( + default=None, description="The column sample by tree rate." + ) + alpha: float | None = Field(default=None, description="The alpha parameter.") + lam: float | None = Field(default=None, description="The lambda parameter.") + gamma: float | None = Field(default=None, description="The gamma parameter.") + seed: int = Field( + default=42, description="The seed for the random number generator." + ) + + @property + def param_dict(self) -> dict[str, Any]: + """The dictionary of parameters.""" + params = { + "objective": "reg:squarederror", + "eval_metric": "rmse", + "tree_method": "auto", + "seed": self.seed, + **self.model_dump(exclude_none=True), + } + try: + import torch + + has_cuda = torch.cuda.is_available() + except ImportError: + has_cuda = False + if has_cuda: + params["device"] = "cuda" + else: + warnings.warn("CUDA is not available, using CPU.", stacklevel=3) + return params + + +class XGBBackend(SurrogateModelBackend): + """The parameters/backend for the xgboost model.""" + + ml_backend: Literal["xgb"] = Field( + default="xgb", description="The type of model to use." + ) + hp: XGBModelConfig = Field( + default_factory=XGBModelConfig, + description="The hyperparameters for the model.", + ) + trainer: XGBTrainerConfig = Field( + default_factory=XGBTrainerConfig, + description="The trainer hyperparameters for the model.", + ) + + def train(self, context: TrainingContext) -> TrainedModel: + """Train an xgboost model and write model artifacts.""" + import xgboost as xgb + + prep = context.prepped_data + train_dmat = xgb.DMatrix( + prep.transformed.train.x.reset_index(drop=True), + label=prep.transformed.train.y.reset_index(drop=True), + ) + test_dmat = xgb.DMatrix( + prep.transformed.test.x.reset_index(drop=True), + label=prep.transformed.test.y.reset_index(drop=True), + ) + + evals = [(train_dmat, "train"), (test_dmat, "eval")] + logger.info("Training XGBoost model...") + model = xgb.train( + self.hp.param_dict, + train_dmat, + num_boost_round=self.trainer.num_boost_round, + evals=evals, + early_stopping_rounds=self.trainer.early_stopping_rounds, + verbose_eval=self.trainer.verbose_eval, + ) + logger.info("Trained XGBoost model.") + + return TrainedModel( + model_object=model, + transformers=prep.transformers, + ) + + def save_model(self, model_object: Any, output_dir: Path) -> Path: + """Serialize an xgboost model.""" + import xgboost as xgb + + if not isinstance(model_object, xgb.Booster): + msg = f"Expected xgboost Booster, got {type(model_object)}" + raise TypeError(msg) + model_path = output_dir / "model.ubj" + model_object.save_model(model_path.as_posix()) + return model_path + + @classmethod + def load_model(cls, regressor_path: Path) -> Any: + """Load an xgboost model from disk.""" + import xgboost as xgb + + model = xgb.Booster() + model.load_model(regressor_path.as_posix()) + return model + + @classmethod + def make_raw_predict_fn(cls, model_object: Any): + """Create the raw xgboost prediction callable.""" + return lambda x, col_order: xgb_pred(x, col_order, model=model_object) + + +def xgb_pred(x: pd.DataFrame, col_order: list[str], *, model) -> np.ndarray: + """Predict the targets for encoded features using xgboost.""" + import xgboost as xgb + + if not isinstance(model, xgb.Booster): + msg = f"Model is not an xgboost model: {type(model)}" + raise TypeError(msg) + + dmat = xgb.DMatrix(x.reset_index(drop=True)) + return model.predict(dmat) diff --git a/src/globi/models/surrogate/dummy.py b/src/globi/models/surrogate/dummy.py new file mode 100644 index 0000000..88eaee3 --- /dev/null +++ b/src/globi/models/surrogate/dummy.py @@ -0,0 +1,124 @@ +"""Dummy simulation for testing.""" + +import math +from pathlib import Path +from typing import Literal, get_args + +import numpy as np +import pandas as pd +from scythe.base import ExperimentInputSpec, ExperimentOutputSpec +from scythe.registry import ExperimentRegistry + +StratificationOption = Literal["some", "other", "option", "another"] + + +class DummySimulationInput(ExperimentInputSpec): + """The input for the dummy simulation.""" + + x0: float + x1: float + x2: float + x3: float + stratification_field: StratificationOption + + @property + def encoded_stratification_field(self) -> float: + """Encode the stratification field as an integer.""" + return get_args(StratificationOption).index(self.stratification_field) / ( + len(get_args(StratificationOption)) + - (1 if len(get_args(StratificationOption)) > 1 else 0) + ) + + @property + def values(self) -> list[float]: + """Get the values of the input spec.""" + vals = self.model_dump( + exclude={ + "stratification_field", + "experiment_id", + "sort_index", + "workflow_run_id", + "root_workflow_run_id", + } + ) + x_vals = {k: v for k, v in vals.items() if k.startswith("x")} + return [*x_vals.values(), self.encoded_stratification_field] + + def n_inputs(self) -> int: + """Get the number of inputs.""" + return len(self.values) + + +class DummySimulationOutput(ExperimentOutputSpec): + """The output for the dummy simulation.""" + + y0: float + + +@ExperimentRegistry.Register( + description="A dummy simulation.", +) +def dummy_simulation( + input_spec: DummySimulationInput, tempdir: Path +) -> DummySimulationOutput: + """A dummy simulation.""" + n_inputs = input_spec.n_inputs() + n_outputs = 5 + problem = SimpleSyntheticProblem( + n_inputs, + n_outputs, + seed=42, + ) + y = problem.evaluate(np.array(input_spec.values)) + + main_result = pd.DataFrame({f"y{i}": [y[i]] for i in range(1, n_outputs)}) + main_result = main_result.set_index(input_spec.make_multiindex()) + main_result_neg = -main_result + main_result = pd.concat( + [main_result, main_result_neg], + axis=1, + keys=["positive", "negative"], + names=["sign"], + ) + return DummySimulationOutput( + y0=y[0], + dataframes={"main_result": main_result}, + ) + + +class SimpleSyntheticProblem: + """A simple synthetic problem.""" + + def __init__(self, n_inputs: int, n_outputs: int, seed: int): + """Initialize the simple synthetic problem.""" + self.n_inputs = n_inputs + self.n_outputs = n_outputs + rng = np.random.default_rng(seed) + + self.alpha = rng.normal(size=n_outputs) + self.beta = rng.normal(scale=0.8, size=(n_outputs, n_inputs)) + self.gamma = rng.normal(scale=0.4, size=(n_outputs, n_inputs)) + self.delta = rng.normal(scale=0.3, size=(n_outputs, max(0, n_inputs - 1))) + self.eta = rng.normal(scale=0.2, size=n_outputs) + self.sine_dim = rng.integers(0, n_inputs, size=n_outputs) + + def evaluate(self, x: np.ndarray) -> np.ndarray: + """Evaluate the simple synthetic problem.""" + x = np.asarray(x, dtype=float) + x = np.clip(x, 0.0, 1.0) + + linear = self.beta @ x + quad = self.gamma @ (x**2) + + if self.n_inputs > 1: + pairwise_terms = x[:-1] * x[1:] + pairwise = self.delta @ pairwise_terms + else: + pairwise = np.zeros(self.n_outputs) + + periodic = np.array([ + self.eta[j] * math.sin(2 * math.pi * x[self.sine_dim[j]]) + for j in range(self.n_outputs) + ]) + + return self.alpha + linear + quad + pairwise + periodic diff --git a/src/globi/models/surrogate/inference.py b/src/globi/models/surrogate/inference.py new file mode 100644 index 0000000..83eb4b6 --- /dev/null +++ b/src/globi/models/surrogate/inference.py @@ -0,0 +1,196 @@ +"""Applying a trained surrogate. + +A training run (``iterative_training`` on hatchet, or +``globi.validation.local_runner`` in-process) produces one regressor + transforms pair +per cross-validation fold. :class:`SurrogateEnsemble` is *the* surrogate: it holds +those fold models and predicts with their mean. + +Features are the same frame the trainer saw, i.e. the result index a simulation of the +building would carry (`ExperimentInputSpec.make_multiindex`: every scalar spec field +plus `computed_features`). :func:`features_from_specs` builds that frame from specs +alone, so a building can be predicted without ever being simulated. +""" + +import logging +from collections.abc import Iterable +from pathlib import Path +from typing import Self, cast + +import pandas as pd +import yaml +from pydantic import BaseModel, Field, TypeAdapter +from scythe.base import ExperimentInputSpec +from scythe.utils.filesys import FileReference + +from globi.models.surrogate.backends import MLBackend +from globi.models.surrogate.transforms import Transformers + +logger = logging.getLogger(__name__) + +#: Index column holding the conditioned floor area the intensity targets (kWh/m²) are +#: normalized by. +AREA_COLUMN = "feature.geometry.energy_model_conditioned_area" + +#: Prefix of the semantic-field feature columns (`feature.semantic.`). +SEMANTIC_PREFIX = "feature.semantic." + +_FILE_REFERENCE = TypeAdapter(FileReference) + + +def file_reference(ref: str | Path) -> FileReference: + """Validate a uri or path into scythe's `FileReference` (S3Url / HttpUrl / Path).""" + text = str(ref) + return _FILE_REFERENCE.validate_python(text if "://" in text else Path(text)) + + +def features_from_specs(specs: Iterable[ExperimentInputSpec]) -> pd.DataFrame: + """The feature frame a simulation of each spec would carry as its result index. + + One row per spec, columns = every index-able spec field plus `computed_features`, + exactly as `TrainFoldSpec` saw them (no simulation involved). + """ + rows = [ + spec.make_multiindex(n_rows=1, include_sort_subindex=False).to_frame( + index=False + ) + for spec in specs + ] + if not rows: + return pd.DataFrame() + return pd.concat(rows, axis=0, ignore_index=True) + + +class ReferencedMLBackend(BaseModel): + """A model backend with referenced regressor and transforms.""" + + regressor: FileReference + transforms: FileReference + ml_backend: MLBackend = Field( + ..., description="The ml backend for the model.", discriminator="ml_backend" + ) + + def load_transforms(self) -> Transformers: + """The (cached) transforms this model was trained with.""" + return self.ml_backend.load_transforms(self.transforms) + + def predict(self, features: pd.DataFrame) -> pd.DataFrame: + """Predict the targets for the given features. + + The result carries a fresh RangeIndex regardless of `features`' index. + """ + model = self.ml_backend.load_model_from_cache(self.regressor) + transforms = self.load_transforms() + pred_fn = self.ml_backend.build_predict_fn( + model_object=model, transformers=transforms + ) + return pred_fn(features) + + +class SurrogateEnsemble(BaseModel): + """The fold models of one training run; predictions are their mean.""" + + models: list[ReferencedMLBackend] = Field(..., min_length=1) + + # ---- construction ------------------------------------------------------ + + @classmethod + def from_pairs( + cls, pairs: Iterable[tuple[str | Path, str | Path]], backend: MLBackend + ) -> Self: + """Build from `(regressor, transforms)` references and the backend they use.""" + models = [ + ReferencedMLBackend( + regressor=file_reference(regressor), + transforms=file_reference(transforms), + ml_backend=backend, + ) + for regressor, transforms in pairs + ] + if not models: + msg = "No fold models given." + raise ValueError(msg) + return cls(models=models) + + @classmethod + def from_training_summary(cls, path: str | Path, backend: MLBackend) -> Self: + """From the `summary.yml` a local `run_local_iterative_training` writes.""" + summary = yaml.safe_load(Path(path).read_text()) + pairs = [(m["regressor"], m["transforms"]) for m in summary["fold_models"]] + ensemble = cls.from_pairs(pairs, backend) + logger.info("Loaded %d fold models from %s", len(ensemble.models), path) + return ensemble + + @classmethod + def from_file_refs(cls, path: str | Path, backend: MLBackend) -> Self: + """From the `result_file_refs` parquet (local or s3) of a hatchet training subrun. + + Its `regressor` / `transforms` columns hold one artifact pair per fold. + """ + refs = pd.read_parquet(str(path)) + missing = {"regressor", "transforms"} - set(refs.columns) + if missing: + msg = ( + f"{path} has no {sorted(missing)} column(s); expected a " + "result_file_refs parquet from a training subrun." + ) + raise ValueError(msg) + pairs = list(zip(refs["regressor"], refs["transforms"], strict=True)) + ensemble = cls.from_pairs(pairs, backend) + logger.info("Loaded %d fold models from %s", len(ensemble.models), path) + return ensemble + + @classmethod + def load(cls, source: str | Path, backend: MLBackend) -> Self: + """Dispatch on the source: a `summary.yml` or a `result_file_refs` parquet.""" + if str(source).endswith((".yml", ".yaml")): + return cls.from_training_summary(source, backend) + return cls.from_file_refs(source, backend) + + # ---- metadata ---------------------------------------------------------- + + @property + def transforms(self) -> Transformers: + """The transforms of the first fold (all folds share features and targets).""" + return self.models[0].load_transforms() + + @property + def targets(self) -> list[str]: + """The target columns the ensemble predicts.""" + return list(self.transforms.y.targets) + + @property + def features(self) -> list[str]: + """The feature columns the ensemble consumes.""" + return list(self.transforms.x.features) + + # ---- prediction -------------------------------------------------------- + + def predict_per_fold(self, features: pd.DataFrame) -> list[pd.DataFrame]: + """One prediction frame per fold model, each indexed like `features`.""" + x = features.reset_index(drop=True) + return [m.predict(x).set_index(features.index) for m in self.models] + + def predict(self, features: pd.DataFrame) -> pd.DataFrame: + """The ensemble (fold-mean) prediction, indexed like `features`.""" + per_fold = self.predict_per_fold(features) + return cast(pd.DataFrame, sum(per_fold) / len(per_fold)) + + def predict_specs(self, specs: Iterable[ExperimentInputSpec]) -> pd.DataFrame: + """Predict buildings straight from their specs, without simulating them.""" + return self.predict(features_from_specs(specs)) + + def unseen_categories(self, features: pd.DataFrame) -> dict[str, set]: + """Categorical values in `features` that no fold model was trained on. + + Predictions for such rows extrapolate: index encoding maps an unseen value to + whatever code the encoder assigns it. + """ + unseen: dict[str, set] = {} + for m in self.models: + for col, cats in m.load_transforms().x.cat_map.items(): + if col not in features.columns: + continue + extra = set(features[col].unique()) - set(cats) + if extra: + unseen.setdefault(col, set()).update(extra) + return unseen diff --git a/src/globi/models/surrogate/metrics.py b/src/globi/models/surrogate/metrics.py new file mode 100644 index 0000000..1a09cd8 --- /dev/null +++ b/src/globi/models/surrogate/metrics.py @@ -0,0 +1,84 @@ +"""Metric utilities for surrogate model evaluation.""" + +from typing import cast + +import numpy as np +import pandas as pd + + +def normalized_mean_bias_error( + preds: pd.DataFrame, targets: pd.DataFrame +) -> np.ndarray: + """Compute nMBE as mean residual divided by mean of true values. + + Residuals are defined as `true - predicted`. When the mean true value for a + target is exactly zero, a denominator of `1` is used so nMBE reproduces MBE. + + Rows are matched positionally (like the sklearn metrics), since predictions + typically carry a fresh RangeIndex while targets keep their feature MultiIndex. + """ + true = targets.to_numpy(dtype=float) + pred = preds[targets.columns].to_numpy(dtype=float) + mean_residual = (true - pred).mean(axis=0) + mean_true = true.mean(axis=0) + denominator = np.where(mean_true != 0, mean_true, 1.0) + return mean_residual / denominator + + +def compute_frame_metrics(preds: pd.DataFrame, targets: pd.DataFrame) -> pd.DataFrame: + """Compute per-target regression metrics for a prediction/target frame pair. + + Returns a frame indexed by `target` with columns `metric` in + {mae, rmse, r2, cvrmse, nmbe, mape}. + """ + from sklearn.metrics import ( + mean_absolute_error, + mean_absolute_percentage_error, + mean_squared_error, + r2_score, + ) + + mae = mean_absolute_error(targets, preds, multioutput="raw_values") + mse = mean_squared_error(targets, preds, multioutput="raw_values") + rmse = np.sqrt(mse) + r2 = r2_score(targets, preds, multioutput="raw_values") + cvrmse = rmse / np.abs(targets.mean(axis=0) + 1e-5) + nmbe = normalized_mean_bias_error(preds=preds, targets=targets) + mape = mean_absolute_percentage_error( + targets + 1e-5, + preds, + multioutput="raw_values", + ) + + metrics = pd.DataFrame( + { + "mae": mae, + "rmse": rmse, + "r2": r2, + "cvrmse": cvrmse, + "nmbe": nmbe, + "mape": mape, + }, + ) + metrics.columns.names = ["metric"] + metrics.index.names = ["target"] + + return metrics + + +def fold_test_averages(metrics: pd.DataFrame) -> pd.Series: + """Average the test-split metrics of a set of fold results over the folds. + + `metrics` is the frame returned by `TrainFoldSpec.compute_metrics` (global or + strata) concatenated over folds: index levels include `sort_index` (the fold) and + `iteration`; columns include a `split_segment` level. The result is a series + indexed by the remaining column levels plus `iteration`, as consumed by + `ConvergenceThresholdsByTarget.run`. + """ + return cast( + pd.Series, + metrics.xs("test", level="split_segment", axis=1) + .groupby(level="iteration") + .mean() + .unstack(), + ) diff --git a/src/globi/models/surrogate/outputs.py b/src/globi/models/surrogate/outputs.py new file mode 100644 index 0000000..11df318 --- /dev/null +++ b/src/globi/models/surrogate/outputs.py @@ -0,0 +1,58 @@ +"""Outputs for the surrogate model pipeline.""" + +from typing import Literal + +from pydantic import BaseModel +from scythe.experiments import ExperimentRun +from scythe.scatter_gather import ScatterGatherResult +from scythe.utils.filesys import S3Url + +from globi.models.surrogate.training import TrainWithCVSpec + + +class CombineResultsResult(BaseModel): + """The result of combining the results of the simulations.""" + + previous: ScatterGatherResult | None + incoming: ScatterGatherResult + combined: ScatterGatherResult + + +# TODO: This should perhaps go somewhere else since it is generally useful. +# (most likely into scythe itself) +class ExperimentRunWithRef(BaseModel): + """An experiment run with a workflow run id.""" + + run: ExperimentRun + workflow_run_id: str + + +class StartTrainingResult(BaseModel): + """The result of starting the training.""" + + training_spec: TrainWithCVSpec + experiment_run_with_ref: ExperimentRunWithRef + + +class TrainingEvaluationResult(BaseModel): + """The result of evaluating the training.""" + + converged: bool + # TODO: possibly get rid of this since we have nice combined outputs already. + metrics: dict + + +class RecursionTransition(BaseModel): + """The transition of the recursion.""" + + reasoning: Literal["max_depth", "converged"] | None + child_workflow_run_id: str | None + + +class FinalizeResult(BaseModel): + """The result of finalizing the training.""" + + reasoning: Literal["max_depth", "converged"] | None + data_uris: dict[str, S3Url] + metrics_uris: dict[str, S3Url] + experiment_ids: list[str] diff --git a/src/globi/models/surrogate/pipeline.py b/src/globi/models/surrogate/pipeline.py new file mode 100644 index 0000000..9c5b973 --- /dev/null +++ b/src/globi/models/surrogate/pipeline.py @@ -0,0 +1,405 @@ +"""Configs for the surrogate model pipeline.""" + +import fnmatch +import re +from functools import cached_property +from pathlib import Path +from typing import Literal, cast + +import numpy as np +import pandas as pd +from pydantic import BaseModel, Field +from scythe.base import ExperimentInputSpec +from scythe.experiments import SemVer, SerializableRunnable +from scythe.scatter_gather import RecursionMap, ScatterGatherResult +from scythe.utils.filesys import OptionalFileReference, S3Url + +from globi.models.surrogate.backends import MLBackend, XGBBackend +from globi.models.surrogate.samplers import Priors + + +class IterationSpec(BaseModel): + """The iteration spec.""" + + n_per_iter: int | list[int] = Field( + default=10_000, + description="The number of samples to generate per generation. If the current iteration exceeds the length of the list, the last element will be used.", + ) + min_per_stratum: int = Field( + default=100, description="The minimum number of samples per stratum." + ) + max_iters: int = Field( + default=100, + description="The maximum number of outer loop iterations to perform.", + ) + recursion: RecursionMap = Field( + default_factory=lambda: RecursionMap(factor=100, max_depth=1), + description="The recursion spec.", + ) + current_iter: int = Field( + default=0, + description="The index of the current training iteration within the outer loop.", + ) + + @property + def at_max_iters(self) -> bool: + """Whether the current iteration is the maximum number of iterations.""" + return self.current_iter + 1 >= self.max_iters + + @property + def n_per_gen_for_current_iter(self) -> int: + """The number of samples to generate for the current iteration.""" + if isinstance(self.n_per_iter, int): + return self.n_per_iter + return self.n_per_iter[min(self.current_iter, len(self.n_per_iter) - 1)] + + +class StratificationSpec(BaseModel): + """A spec for stratifying the data.""" + + field: str = Field( + default="feature.weather.file", description="The field to stratify by." + ) + sampling: Literal["equal", "error-weighted", "proportional"] = Field( + default="equal", + description="The sampling method to use over the strata.", + ) + aliases: list[str] = Field( + default_factory=lambda: ["epwzip_path", "epw_path"], + description="The alias to use for the stratum as a fallback.", + ) + + # TODO: consider allowing the stratification to be a compound with e.g. component_map_uri and semantic_fields_uri and database_uri + + +class CrossValidationSpec(BaseModel): + """The cross validation spec.""" + + n_folds: int = Field( + default=5, description="The number of folds for the entire parent task." + ) + + +class ConvergenceThresholds(BaseModel): + """The thresholds for convergence.""" + + mae: float | None = Field( + default=None, description="The maximum MAE for convergence." + ) + rmse: float | None = Field( + default=None, description="The maximum RMSE for convergence." + ) + mape: float | None = Field( + default=None, description="The maximum MAPE for convergence." + ) + r2: float | None = Field( + default=None, description="The minimum R2 for convergence." + ) + cvrmse: float | None = Field( + default=None, description="The maximum CV_RMSE for convergence." + ) + nmbe: float | None = Field( + default=None, + description=( + "The maximum normalized mean bias error for convergence. " + "This is treated as a minimization metric." + ), + ) + + def check_convergence(self, metrics: pd.Series, target: re.Pattern | None = None): + """Check if the metrics have converged. + + Note that this requires the metrics data frame to have the following shape: + + """ + # first, we select the data for the relevant targets: + if target is not None: + target_level = metrics.index.get_level_values("target") + # Interpret target as a regex and match + mask = cast(pd.Series, target_level.to_series().astype(str)).str.match( + target + ) + metrics = cast(pd.Series, metrics.loc[mask.values]) + + thresholds = pd.Series(self.model_dump(), name="metric") + + # first, we will select the appropriate threshold for each metric + comparators = thresholds.loc[metrics.index.get_level_values("metric")] + + # we can then copy over the index safely + comparators.index = metrics.index + + # we will ignore any thresholds that are not set or are NaN + comparators_are_na = comparators.isna() + + # next, we will flip the sign of the r2 metric since it is a maximization metric rather thin min + metrics = metrics * np.where( + metrics.index.get_level_values("metric") == "r2", -1, 1 + ) + comparators = comparators * np.where( + comparators.index.get_level_values("metric") == "r2", -1, 1 + ) + + # run the comparisons + comparison = metrics < comparators + comparison = comparison.loc[~comparators_are_na] + + return comparison + + +class ConvergenceThresholdsByTarget(BaseModel): + """The thresholds for convergence by target.""" + + thresholds: dict[str, ConvergenceThresholds] = Field( + default_factory=lambda: {"*": ConvergenceThresholds()}, + description="The thresholds for convergence by target.", + ) + + def make_comparisons(self, metrics: pd.Series) -> list[pd.Series]: + """Generate a list of all stratum/target/metric True/False comparisons.""" + return [ + self.thresholds[target].check_convergence( + metrics, re.compile(fnmatch.translate(target)) + ) + for target in self.thresholds + ] + + def combine_and_check_strata_and_targets(self, comparisons: list[pd.Series]): + """Combine the comparisons and aggregate first by targets then by strata.""" + comparison = pd.concat(comparisons, axis=0) + # now we will groupby the stratum (e.g. features.weather.file) + # and by the target (e.g. Electricity, Gas, etc.) + # we are converged if any of the metrics have converged for that target + # in that stratum + comparison_stratum_and_target = comparison.groupby( + level=[lev for lev in comparison.index.names if lev != "metric"] + ).any() # TODO: make it configurable such that instead of `any`, we can specify a count, i.e. at least 2 must be converged + + # then we will check that all targets have converged for each stratum + + # only levels left in multiindex should be stratum and target + + comparison_strata = comparison_stratum_and_target.groupby(level="stratum").all() + + # finally, we will check that all strata have converged + comparison_all = comparison_strata.all() + + return ( + comparison_all, + comparison_strata, + comparison_stratum_and_target, + comparison, + ) + + def run(self, metrics: pd.Series) -> tuple[bool, pd.Series, pd.Series, pd.Series]: + """Run the convergence criteria.""" + comparisons = self.make_comparisons(metrics) + return self.combine_and_check_strata_and_targets(comparisons) + + +class TargetsConfigBaseSpec(BaseModel): + """The base targets config spec.""" + + normalization: ( + Literal[ + "min-max", + "standard", + ] + | None + ) = Field(default="min-max", description="The normalization method to use.") + + +class TargetsConfigColumnSpec(TargetsConfigBaseSpec): + """The targets config spec.""" + + columns: list[str] = Field( + default_factory=list, description="The columns to use as targets." + ) + + +class TargetsConfigGlobSpec(TargetsConfigBaseSpec): + """The targets config spec.""" + + globs: list[str] = Field( + default_factory=list, description="The columns to use as targets." + ) + + +TargetsConfigSpec = TargetsConfigColumnSpec | TargetsConfigGlobSpec + + +class FeatureConfigSpec(BaseModel): + """The feature config spec.""" + + continuous_columns: frozenset[str] = Field( + default=frozenset(), description="The continuous columns to use as features." + ) + categorical_columns: frozenset[str] = Field( + default=frozenset(), description="The categorical columns to use as features." + ) + exclude_columns: frozenset[str] = Field( + default=frozenset(), + description="The columns to exclude from the features.", + ) + cont_cat_unicity_transition_threshold: int = Field( + default=10, + description="The threshold for the number of unique values to transition from continuous to categorical variable.", + ) + cat_encoding: Literal["index", "one-hot"] = Field( + default="index", + description="The encoding method to use for categorical columns.", + ) + cont_encoding: Literal["min-max", "standard"] | None = Field( + default=None, + description=( + "The encoding method to use for continuous columns. " + "Use 'standard' for standardization." + ), + ) + + +class RegressionIOConfigSpec(BaseModel): + """The input/output spec for a regression model.""" + + targets: TargetsConfigSpec = Field( + default_factory=TargetsConfigColumnSpec, description="The targets config spec." + ) + features: FeatureConfigSpec = Field( + default_factory=FeatureConfigSpec, + description="The features config spec.", + ) + + +class ProgressiveTrainingSpec(ExperimentInputSpec, SerializableRunnable): + """A spec for iteratively training an SBEM regression model.""" + + base_run_name: str = Field( + ..., + description="The base run name for the experiment.", + ) + convergence_criteria: ConvergenceThresholdsByTarget = Field( + default_factory=ConvergenceThresholdsByTarget, + description="The convergence criteria.", + ) + regression_io_config: RegressionIOConfigSpec = Field( + default_factory=RegressionIOConfigSpec, + description="The regression io config spec.", + ) + ml_backend: MLBackend = Field( + default_factory=XGBBackend, + description="The ml backend for the model.", + discriminator="ml_backend", + ) + stratification: StratificationSpec = Field( + default_factory=StratificationSpec, + description="The stratification spec.", + ) + samplers: Priors = Field( + ..., + description="The sampling spec.", + ) + cross_val: CrossValidationSpec = Field( + default_factory=CrossValidationSpec, + description="The cross validation spec.", + ) + iteration: IterationSpec = Field( + default_factory=IterationSpec, + description="The iteration spec.", + ) + context: OptionalFileReference = Field( + default=None, + description="The uri of the gis data to train on.", + ) + data_uris: ScatterGatherResult | None = Field( + default=None, + description="The uri of the previous simulation results to train on.", + ) + metrics_uris: list[ScatterGatherResult] = Field( + default_factory=list, + description="The uris of the iteration metrics from previous iterations.", + ) + previous_experiment_ids: list[str] = Field( + default_factory=list, + description="The ids of the previous experiments.", + ) + + def format_combined_output_key(self, key: str) -> str: + """Format the output key for a combined result file.""" + return f"{self.prefix}/combined/data/{key}.parquet" + + def format_combined_output_uri(self, key: str) -> S3Url: + """Format the output uri for a combined result file.""" + if self.storage_settings is None: + msg = "Storage settings are not set, so we can't construct a combined output uri." + raise ValueError(msg) + return S3Url( + f"s3://{self.storage_settings.BUCKET}/{self.format_combined_output_key(key)}" + ) + + def format_metrics_output_key(self, key: str) -> str: + """Format the output key for a metrics file.""" + return f"{self.prefix}/combined/metrics/{key}.parquet" + + def format_metrics_output_uri(self, key: str) -> S3Url: + """Format the output uri for a metrics file.""" + if self.storage_settings is None: + msg = "Storage settings are not set, so we can't construct a metrics output uri." + raise ValueError(msg) + return S3Url( + f"s3://{self.storage_settings.BUCKET}/{self.format_metrics_output_key(key)}" + ) + + def format_summary_manifest_key(self) -> str: + """Format the output key for a summary manifest file.""" + return f"{self.prefix}/summary.yml" + + def format_summary_manifest_uri(self) -> S3Url: + """Format the output uri for a summary manifest file.""" + if self.storage_settings is None: + msg = "Storage settings are not set, so we can't construct a summary manifest uri." + raise ValueError(msg) + return S3Url( + f"s3://{self.storage_settings.BUCKET}/{self.format_summary_manifest_key()}" + ) + + def subrun_name(self, subrun: Literal["sample", "train"]) -> str: + """Format the run name for a subrun.""" + return f"{self.experiment_id}/{subrun}" + + @property + def context_path(self) -> Path | None: + """The path to the gis data.""" + if self.context is None: + return None + if isinstance(self.context, Path): + return self.context + return self.fetch_uri(self.context) + + @cached_property + def context_data(self) -> pd.DataFrame | None: + """Load the gis data.""" + if self.context_path is None: + return None + return pd.read_parquet(self.context_path) + + @property + def current_version(self) -> SemVer: + """The current version.""" + vstr = [ + piece for piece in self.experiment_id.split("/") if piece.startswith("v") + ][-1] + return SemVer.FromString(vstr) + + +class StageSpec(BaseModel): + """A spec that is common to both the sample and train stages (and possibly others).""" + + parent: ProgressiveTrainingSpec = Field( + ..., + description="The parent spec.", + ) + + @cached_property + def random_generator(self) -> np.random.Generator: + """The random generator.""" + return np.random.default_rng(self.parent.iteration.current_iter) diff --git a/src/globi/models/surrogate/samplers.py b/src/globi/models/surrogate/samplers.py new file mode 100644 index 0000000..7ec463a --- /dev/null +++ b/src/globi/models/surrogate/samplers.py @@ -0,0 +1,970 @@ +"""Conditional Priors and Samplers. + +Ported from epengine/models/sampling.py with enhancements: +- Fixed NaN comparison bug in ConditionalPrior +- Added MultiColumnConditionalPrior for multi-column conditioning + without requiring ConcatenateFeaturesSampler intermediate columns +""" + +import gc +from abc import ABC, abstractmethod +from typing import Literal, cast + +import networkx as nx +import numpy as np +import pandas as pd +from pydantic import BaseModel, model_validator + +# TODO: Make sure that all of the samplers can be serialized and deserialized with proper discrimination, i.e. that they do not share identical field names. +# TODO: add support for keeping rows which already have values (i.e. do not overwrite values) + + +class SamplingError(Exception): + """A sampling error.""" + + pass + + +class Sampler(ABC): + """A sampler.""" + + @abstractmethod + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample features from a prior, which may depend on a context.""" + pass + + @property + @abstractmethod + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + pass + + +class UniformSampler(BaseModel, Sampler): + """A uniform sampler which generates values uniformly between a min and max value.""" + + min: float + max: float + round: Literal["ceil", "floor", "nearest"] | None = None + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample uniformly between a min and max value.""" + samples = generator.uniform(self.min, self.max, size=n) + if self.round == "ceil": + samples = np.ceil(samples) + elif self.round == "floor": + samples = np.floor(samples) + elif self.round == "nearest": + samples = np.round(samples) + return samples + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set() + + +class ClippedNormalSampler(BaseModel, Sampler): + """A clipped normal sampler which generates values from a normal distribution, clipped to a min and max value.""" + + mean: float + std: float + clip_min: float | None + clip_max: float | None + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample from a normal distribution, clipped to a min and max value.""" + clip_min = self.clip_min if self.clip_min is not None else -np.inf + clip_max = self.clip_max if self.clip_max is not None else np.inf + samples = generator.normal(self.mean, self.std, size=n).clip(clip_min, clip_max) + return samples + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set() + + +class FixedValueSampler(BaseModel): + """A fixed value sampler which generates a fixed value for all samples.""" + + value: float | str | int | bool + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample a fixed value.""" + return np.full(n, self.value) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set() + + +class CategoricalSampler(BaseModel): + """A categorical sampler which generates values from a categorical distribution.""" + + values: list[str] | list[float] | list[int] + weights: list[float] + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample from a categorical distribution.""" + return generator.choice(self.values, size=n, p=self.weights) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set() + + @model_validator(mode="after") + def check_values_and_weights(self): + """Check that the values and weights are the same length and normalized.""" + if len(self.values) != len(self.weights): + msg = "values and weights must be the same length" + raise ValueError(msg) + if not np.isclose(sum(self.weights), 1): + self.weights = [w / sum(self.weights) for w in self.weights] + return self + + +class CopySampler(BaseModel): + """A deterministic sampler which generates a copy of a feature in the provided context dataframe.""" + + feature_to_copy: str + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a copy of a feature.""" + if self.feature_to_copy not in context.columns: + msg = f"Feature to copy {self.feature_to_copy} not found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return context[self.feature_to_copy].to_numpy() + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_copy} + + +class AddValueSampler(BaseModel): + """A deterministic sampler which adds a value to a feature.""" + + feature_to_add_to: str + value_to_add: float + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a sum of a feature and a value.""" + if self.feature_to_add_to not in context.columns: + msg = f"Feature to add to {self.feature_to_add_to} not found in context dataframe." + raise SamplingError(msg) + return context[self.feature_to_add_to].to_numpy() + self.value_to_add + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_add_to} + + +class SumValuesSampler(BaseModel): + """A deterministic sampler which generates a sum of features.""" + + features_to_sum: list[str] + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a sum of features.""" + if not all(f in context.columns for f in self.features_to_sum): + msg = f"All features to sum {self.features_to_sum} must be found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return np.sum(context[self.features_to_sum].to_numpy(), axis=1) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set(self.features_to_sum) + + +class MultiplyValueSampler(BaseModel): + """A deterministic sampler which generates a product of a feature and a value.""" + + feature_to_multiply: str + value_to_multiply: float + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a multiply of a feature.""" + if self.feature_to_multiply not in context.columns: + msg = f"Feature to multiply {self.feature_to_multiply} not found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return context[self.feature_to_multiply].to_numpy() * self.value_to_multiply + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_multiply} + + +class ProductValuesSampler(BaseModel): + """A deterministic sampler which generates a product of features.""" + + features_to_multiply: list[str] + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a product of features.""" + if not all(f in context.columns for f in self.features_to_multiply): + msg = f"All features to multiply {self.features_to_multiply} must be found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return np.prod(context[self.features_to_multiply].to_numpy(), axis=1) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set(self.features_to_multiply) + + +class InvertSampler(BaseModel): + """A deterministic sampler which generates the multiplicative inverse of a feature.""" + + feature_to_invert: str + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute an invert of a feature.""" + if self.feature_to_invert not in context.columns: + msg = f"Feature to invert {self.feature_to_invert} not found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return 1 / context[self.feature_to_invert].to_numpy() + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_invert} + + +class PowerSampler(BaseModel): + """A deterministic sampler which generates a power of a feature.""" + + feature_to_power: str + power: float + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a power of a feature.""" + return context[self.feature_to_power].to_numpy() ** self.power + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_power} + + +class LogSampler(BaseModel): + """A deterministic sampler which generates a log of a feature.""" + + feature_to_log: str + base: float = np.e + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a log of a feature.""" + if self.feature_to_log not in context.columns: + msg = ( + f"Feature to log {self.feature_to_log} not found in context dataframe." + ) + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + return np.log(context[self.feature_to_log].to_numpy()) / np.log(self.base) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_log} + + +class RoundSampler(BaseModel): + """A deterministic sampler which applies ceil, floor, or nearest to a feature.""" + + feature_to_round: str + operation: Literal["ceil", "floor", "nearest"] + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Apply ceil, floor, or nearest to a feature.""" + if self.feature_to_round not in context.columns: + msg = f"Feature to round {self.feature_to_round} not found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + values = context[self.feature_to_round].to_numpy() + if self.operation == "ceil": + return np.ceil(values) + if self.operation == "floor": + return np.floor(values) + return np.round(values) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_round} + + +class ThresholdSampler(BaseModel): + """A deterministic sampler which generates a boolean value based on a threshold.""" + + feature_to_compare: str + threshold: float + operator: Literal["<", "<=", "==", ">=", ">"] + true_value: float | str | int | bool + false_value: float | str | int | bool + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compare a feature to a threshold.""" + if self.feature_to_compare not in context.columns: + msg = f"Feature to compare {self.feature_to_compare} not found in context dataframe." + raise SamplingError(msg) + source_values = context[self.feature_to_compare].to_numpy() + if self.operator == "<": + return np.where( + source_values < self.threshold, self.true_value, self.false_value + ) + elif self.operator == "<=": + return np.where( + source_values <= self.threshold, self.true_value, self.false_value + ) + elif self.operator == "==": + return np.where( + source_values == self.threshold, self.true_value, self.false_value + ) + elif self.operator == ">=": + return np.where( + source_values >= self.threshold, self.true_value, self.false_value + ) + elif self.operator == ">": + return np.where( + source_values > self.threshold, self.true_value, self.false_value + ) + else: + msg = f"Invalid operator {self.operator}." + raise SamplingError(msg) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.feature_to_compare} + + +class ColumnComparatorSampler(BaseModel): + """A deterministic sampler which compares one column to another column.""" + + left_feature: str + right_feature: str + operator: Literal["<", "<=", "==", ">=", ">"] + true_value: float | str | int | bool + false_value: float | str | int | bool + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compare one column to another column.""" + if self.left_feature not in context.columns: + msg = f"Left feature {self.left_feature} not found in context dataframe." + raise SamplingError(msg) + if self.right_feature not in context.columns: + msg = f"Right feature {self.right_feature} not found in context dataframe." + raise SamplingError(msg) + left_values = context[self.left_feature].to_numpy() + right_values = context[self.right_feature].to_numpy() + if self.operator == "<": + return np.where( + left_values < right_values, self.true_value, self.false_value + ) + elif self.operator == "<=": + return np.where( + left_values <= right_values, self.true_value, self.false_value + ) + elif self.operator == "==": + return np.where( + left_values == right_values, self.true_value, self.false_value + ) + elif self.operator == ">=": + return np.where( + left_values >= right_values, self.true_value, self.false_value + ) + elif self.operator == ">": + return np.where( + left_values > right_values, self.true_value, self.false_value + ) + else: + msg = f"Invalid operator {self.operator}." + raise SamplingError(msg) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.left_feature, self.right_feature} + + +class ConcatenateFeaturesSampler(BaseModel): + """A deterministic sampler which concatenates features. + + Retained for backward compatibility. Prefer MultiColumnConditionalPrior + for multi-column conditioning instead of creating intermediate compound key columns. + """ + + features_to_concatenate: list[str] + separator: str = ":" + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Compute a concatenation of features.""" + if not all(f in context.columns for f in self.features_to_concatenate): + msg = f"All features to concatenate {self.features_to_concatenate} must be found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + cols: pd.DataFrame = cast(pd.DataFrame, context[self.features_to_concatenate]) + return cols.astype(str).agg(self.separator.join, axis=1).to_numpy() + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set(self.features_to_concatenate) + + +PriorSampler = ( + UniformSampler + | ClippedNormalSampler + | FixedValueSampler + | CategoricalSampler + | CopySampler + | AddValueSampler + | SumValuesSampler + | MultiplyValueSampler + | ProductValuesSampler + | InvertSampler + | LogSampler + | RoundSampler + | ConcatenateFeaturesSampler + | PowerSampler + | ThresholdSampler + | ColumnComparatorSampler +) + + +# TODO: Rename this to MatchCondition +class ConditionalPriorCondition(BaseModel): + """A conditional prior condition.""" + + match_val: str | float | int | bool + sampler: PriorSampler + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample from a conditional prior condition.""" + return self.sampler.sample(context, n, generator) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return self.sampler.depends_on + + +# TODO: Rename this to MultiColumnMatchCondition +class MultiColumnCondition(BaseModel): + """A condition that matches on multiple source features simultaneously. + + Used with MultiColumnConditionalPrior to condition on combinations + of column values without creating intermediate compound key columns. + """ + + match_vals: tuple[str | float | int | bool, ...] + sampler: PriorSampler + + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample from this condition's sampler.""" + return self.sampler.sample(context, n, generator) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return self.sampler.depends_on + + +class PriorABC(ABC): + """A prior.""" + + @abstractmethod + def sample( + self, context: pd.DataFrame, n: int, generator: np.random.Generator + ) -> np.ndarray: + """Sample from a prior.""" + pass + + @property + @abstractmethod + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + pass + + +class ConditionalPrior(BaseModel, PriorABC): + """A conditional prior that selects a sampler based on a single source feature.""" + + source_feature: str + conditions: list[ConditionalPriorCondition] + fallback_prior: PriorSampler | None + + @staticmethod + def _sample_for_indices( + sampler: PriorSampler, + context: pd.DataFrame, + indices: np.ndarray, + generator: np.random.Generator, + sampler_label: str, + ) -> np.ndarray: + """Sample only for selected row indices and validate output length.""" + n_for_sampler = int(indices.size) + context_for_sampler = context.iloc[indices] + samples = sampler.sample(context_for_sampler, n_for_sampler, generator) + if len(samples) != n_for_sampler: + msg = ( + f"{sampler_label} returned an invalid number of samples: " + f"expected {n_for_sampler}, got {len(samples)}." + ) + raise SamplingError(msg) + return np.asarray(samples) + + @staticmethod + def _merge_samples( + final: np.ndarray | None, + n: int, + indices: np.ndarray, + samples: np.ndarray, + ) -> np.ndarray: + """Merge sampled values into the final array with dtype promotion if needed.""" + if final is None: + final = np.empty(n, dtype=samples.dtype) + else: + promoted_dtype = np.result_type(final.dtype, samples.dtype) + if promoted_dtype != final.dtype: + final = final.astype(promoted_dtype, copy=False) + + if final is None: + msg = "Internal error: final samples array was not initialized." + raise SamplingError(msg) + final[indices] = samples + return final + + def sample(self, context: pd.DataFrame, n: int, generator: np.random.Generator): + """Sample from a conditional prior.""" + if self.source_feature not in context.columns: + msg = ( + f"Source feature {self.source_feature} not found in context dataframe." + ) + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + + test_feature = context[self.source_feature].to_numpy() + + final: np.ndarray | None = None + + any_matched_mask = np.full(n, False) + for condition in self.conditions: + matched_indices = np.flatnonzero(test_feature == condition.match_val) + if matched_indices.size == 0: + continue + + samples_for_condition = self._sample_for_indices( + sampler=condition.sampler, + context=context, + indices=matched_indices, + generator=generator, + sampler_label="Conditional prior sampler", + ) + final = self._merge_samples( + final=final, n=n, indices=matched_indices, samples=samples_for_condition + ) + any_matched_mask[matched_indices] = True + + if self.fallback_prior is not None: + unmatched_indices = np.flatnonzero(~any_matched_mask) + if unmatched_indices.size > 0: + fallback_samples = self._sample_for_indices( + sampler=self.fallback_prior, + context=context, + indices=unmatched_indices, + generator=generator, + sampler_label="Fallback prior", + ) + final = self._merge_samples( + final=final, + n=n, + indices=unmatched_indices, + samples=fallback_samples, + ) + any_matched_mask[unmatched_indices] = True + + unmatched_mask = ~any_matched_mask + if unmatched_mask.any(): + unmatched_examples = test_feature[unmatched_mask][:5] + msg = ( + "No condition matched some rows and no fallback prior filled them for " + f"feature {self.source_feature}. Examples: {unmatched_examples}" + ) + raise SamplingError(msg) + + if final is None: + return np.empty(n) + + return final + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return {self.source_feature} | { + dependency for c in self.conditions for dependency in c.depends_on + } + + +class MultiColumnConditionalPrior(BaseModel, PriorABC): + """A conditional prior that selects a sampler based on multiple source features. + + This eliminates the need for ConcatenateFeaturesSampler + compound key columns. + Instead of creating an intermediate concatenated column and matching on strings, + this prior directly matches on tuples of column values. + + Example usage:: + + prior = MultiColumnConditionalPrior( + source_features=["Typology", "Age_bracket"], + conditions=[ + MultiColumnCondition( + match_vals=("SFH", "pre_1975"), + sampler=CategoricalSampler(values=[...], weights=[...]), + ), + MultiColumnCondition( + match_vals=("MFH", "post_2003"), + sampler=UniformSampler(min=0.5, max=1.0), + ), + ], + fallback_prior=CategoricalSampler(values=[...], weights=[...]), + ) + """ + + source_features: list[str] + conditions: list[MultiColumnCondition] + fallback_prior: PriorSampler | None + + @model_validator(mode="after") + def validate_condition_lengths(self): + """Ensure all conditions have match_vals aligned with source_features.""" + for i, c in enumerate(self.conditions): + if len(c.match_vals) != len(self.source_features): + msg = ( + f"Condition {i}: match_vals length {len(c.match_vals)} " + f"!= source_features length {len(self.source_features)}" + ) + raise ValueError(msg) + return self + + @staticmethod + def _matched_indices_for_condition( + source_values: list[np.ndarray], + match_vals: tuple[str | float | int | bool, ...], + n: int, + ) -> np.ndarray: + """Return row indices that match all source-feature values for a condition.""" + mask = np.full(n, True) + for feature_values, match_val in zip(source_values, match_vals, strict=True): + mask &= feature_values == match_val + return np.flatnonzero(mask) + + @staticmethod + def _unmatched_tuple_examples( + source_values: list[np.ndarray], unmatched_indices: np.ndarray + ) -> list[tuple[object, ...]]: + """Build a few unmatched source-feature tuples for error reporting.""" + return [ + tuple(feature_values[i] for feature_values in source_values) + for i in unmatched_indices[:5] + ] + + def sample(self, context: pd.DataFrame, n: int, generator: np.random.Generator): + """Sample from a multi-column conditional prior.""" + for f in self.source_features: + if f not in context.columns: + msg = f"Source feature {f} not found in context dataframe." + raise SamplingError(msg) + if len(context) != n: + msg = ( + f"Context dataframe must have {n} rows, but it has {len(context)} rows." + ) + raise SamplingError(msg) + + source_values = [context[f].to_numpy() for f in self.source_features] + final: np.ndarray | None = None + any_matched = np.full(n, False) + + for condition in self.conditions: + matched_indices = self._matched_indices_for_condition( + source_values=source_values, match_vals=condition.match_vals, n=n + ) + if matched_indices.size == 0: + continue + + samples = ConditionalPrior._sample_for_indices( + sampler=condition.sampler, + context=context, + indices=matched_indices, + generator=generator, + sampler_label="Multi-column conditional prior sampler", + ) + final = ConditionalPrior._merge_samples( + final=final, n=n, indices=matched_indices, samples=samples + ) + any_matched[matched_indices] = True + + if self.fallback_prior is not None: + unmatched_indices = np.flatnonzero(~any_matched) + if unmatched_indices.size > 0: + fallback_samples = ConditionalPrior._sample_for_indices( + sampler=self.fallback_prior, + context=context, + indices=unmatched_indices, + generator=generator, + sampler_label="Multi-column fallback prior", + ) + final = ConditionalPrior._merge_samples( + final=final, + n=n, + indices=unmatched_indices, + samples=fallback_samples, + ) + any_matched[unmatched_indices] = True + + unmatched_mask = ~any_matched + if unmatched_mask.any(): + unmatched_indices = np.flatnonzero(unmatched_mask) + unmatched_examples = self._unmatched_tuple_examples( + source_values=source_values, unmatched_indices=unmatched_indices + ) + msg = ( + "No condition matched some rows and no fallback prior filled them for " + f"features {self.source_features}. Examples of unmatched tuples: {unmatched_examples}" + ) + raise SamplingError(msg) + + if final is None: + return np.empty(n) + + return final + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return set(self.source_features) | { + dependency for c in self.conditions for dependency in c.sampler.depends_on + } + + +class UnconditionalPrior(BaseModel, PriorABC): + """An unconditional prior.""" + + sampler: PriorSampler + + def sample(self, context: pd.DataFrame, n: int, generator: np.random.Generator): + """Sample from an unconditional prior.""" + return self.sampler.sample(context, n, generator) + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return self.sampler.depends_on + + +Prior = UnconditionalPrior | ConditionalPrior | MultiColumnConditionalPrior + + +class Priors(BaseModel): + """A collection of priors defining a dependency graph for sampling.""" + + sampled_features: dict[str, Prior] + + def sample(self, context: pd.DataFrame, n: int, generator: np.random.Generator): + """Sample from all priors in dependency order.""" + working_df = context.copy(deep=True) + # TODO: Similarly, how do we ensure that there are no cycles in the dependency graph? + for i, feature in enumerate(self.topological_sort): + if (i + 1) % 20 == 0: + new_working_df = working_df.copy( + deep=True + ) # copy every 20 features to avoid fragmentation issues + del working_df + gc.collect() + working_df = new_working_df + if feature not in self.sampled_features and feature not in context.columns: + msg = f"Feature {feature} not found in sampled features or context dataframe." + raise SamplingError(msg) + if feature not in self.sampled_features: + if feature not in self.root_features: + msg = ( + f"Feature {feature} not found in root features but expected to." + ) + raise SamplingError(msg) + + # it's a context feature dependency, so we can skip over to the next + continue + + prior = self.sampled_features[feature] + + working_df.loc[:, feature] = prior.sample(working_df, n, generator) + if working_df.isna().any().any(): # pyright: ignore [reportAttributeAccessIssue] + cols_that_are_na = working_df.columns[working_df.isna().any()] + # TODO: allow na values eg in training? + msg = f"Working dataframe contains NaN values; possibly due to an unmatched value: {cols_that_are_na[:5]}{'...' if len(cols_that_are_na) > 5 else ''}" + raise SamplingError(msg) + return working_df + + @property + def depends_on(self) -> set[str]: + """The features that this sampler depends on.""" + return { + dependency + for prior in self.sampled_features.values() + for dependency in prior.depends_on + } + + @property + def dependency_graph(self) -> nx.DiGraph: + """Construct a dependency graph between columns in the context dataframe. + + Edges connect *from* the dependency *to* the dependent feature. + """ + g = nx.DiGraph() + for feature, prior in self.sampled_features.items(): + if prior.depends_on: + for dependency in prior.depends_on: + g.add_edge(dependency, feature) + # TODO: make sure that this is okay and that id does not cause problem siwth select_prior_tree_for_changed_features... + for feature in self.sampled_features: + if feature not in g.nodes: + g.add_node(feature) + + return g + + @property + def topological_sort(self) -> list[str]: + """The topological sort of the features.""" + return list(nx.topological_sort(self.dependency_graph)) + + @property + def root_features(self) -> set[str]: + """The features that have no dependencies.""" + return { + node + for node in self.dependency_graph.nodes + if self.dependency_graph.in_degree(node) == 0 + } + + def select_prior_tree_for_changed_features( + self, changed_features: set[str], resample_changed_features: bool = True + ) -> "Priors": + """Select the prior tree for the changed features. + + Returns a new Priors object with only the priors that are + downstream of the changed features. + + Args: + changed_features: The features that have changed. + resample_changed_features: Whether to resample the changed features + themselves (dependencies are always resampled). You probably want + this to be False, but for backwards compatibility it defaults to True. + + Returns: + A new Priors object with only the downstream priors. + """ + g = self.dependency_graph + all_changing_priors: set[str] = set() + for any_feature in self.root_features.union(set(self.sampled_features.keys())): + if any(f == any_feature for f in changed_features): + descendants = nx.descendants(g, any_feature) + + if any_feature in self.sampled_features and resample_changed_features: + all_changing_priors.add(any_feature) + + for dep in descendants: + if dep in self.sampled_features: + all_changing_priors.add(dep) + + return Priors( + sampled_features={ + f: p + for f, p in self.sampled_features.items() + if f in all_changing_priors + } + ) diff --git a/src/globi/models/surrogate/sampling.py b/src/globi/models/surrogate/sampling.py new file mode 100644 index 0000000..a287fb5 --- /dev/null +++ b/src/globi/models/surrogate/sampling.py @@ -0,0 +1,133 @@ +"""Models used for the training set sampling pipeline.""" + +from typing import cast + +import numpy as np +import pandas as pd +from pydantic import Field +from scythe.base import ExperimentInputSpec + +from globi.models.surrogate.pipeline import StageSpec +from globi.models.surrogate.samplers import Priors + + +class SampleSpec(StageSpec): + """A spec for the sampling stage of the progressive training.""" + + # TODO: add the ability to receive the last set of error metrics and use them to inform the sampling + priors: Priors = Field( + ..., + description="The priors to use for sampling.", + ) + + def stratified_selection(self) -> pd.DataFrame | None: + """Sample the gis data.""" + df = self.parent.context_data + if df is None: + return None + + stratification_field = self.parent.stratification.field + stratification_aliases = self.parent.stratification.aliases + + if stratification_field not in df.columns and not any( + alias in df.columns for alias in stratification_aliases + ): + msg = f"Stratification field {stratification_field} not found in gis data. Please check the field name and/or the aliases." + raise ValueError(msg) + + if stratification_field not in df.columns: + stratification_field = next( + alias for alias in stratification_aliases if alias in df.columns + ) + + strata = cast(list[str], df[stratification_field].unique().tolist()) + + if self.parent.stratification.sampling == "equal": + return self.sample_equally_by_stratum(df, strata, stratification_field) + elif self.parent.stratification.sampling == "error-weighted": + msg = "Error-weighted sampling is not yet implemented." + raise NotImplementedError(msg) + elif self.parent.stratification.sampling == "proportional": + msg = "Proportional sampling is not yet implemented." + raise NotImplementedError(msg) + else: + msg = f"Invalid sampling method: {self.parent.stratification.sampling}" + raise ValueError(msg) + + def sample_equally_by_stratum( + self, df: pd.DataFrame, strata: list[str], stratification_field: str + ) -> pd.DataFrame: + """Sample equally by stratum. + + This will break the dataframe up into n strata and ensure that each strata ends up with the same number of samples. + + Args: + df (pd.DataFrame): The dataframe to sample from. + strata (list[str]): The unique values of the strata. + stratification_field (str): The field to stratify the data by. + + Returns: + samples (pd.DataFrame): The sampled dataframe. + """ + stratum_dfs = { + stratum: df[df[stratification_field] == stratum] for stratum in strata + } + n_per_iter = self.parent.iteration.n_per_gen_for_current_iter + n_per_stratum = max( + n_per_iter // len(strata), + ( + self.parent.iteration.min_per_stratum + if self.parent.iteration.current_iter == 0 + else 0 + ), + ) + + # TODO: consider how we want to handle potentially having the same geometry appear in both + # the training and testing sets. + + # if any(len(stratum_df) < n_per_stratum for stratum_df in stratum_dfs.values()): + # msg = "There are not enough buildings in some strata to sample the desired number of buildings per stratum." + # # connsider making this a warning? + # raise ValueError(msg) + + sampled_strata = { + stratum: stratum_df.sample( + n=n_per_stratum, random_state=self.random_generator, replace=True + ) + for stratum, stratum_df in stratum_dfs.items() + } + combined = cast(pd.DataFrame, pd.concat(sampled_strata.values())) + if len(combined) < n_per_iter: + # This handles cases where, due to rounding, we do not end up with the desired number of samples. + remaining = n_per_iter - len(combined) + additional = df.sample( + n=remaining, random_state=self.random_generator, replace=True + ) + combined = cast(pd.DataFrame, pd.concat([combined, additional])) + + return combined + + # TODO: Add the ability to check the compatiblity of a sampling spec with an input_validator_type. + + def populate_sample_df(self) -> pd.DataFrame: + """Populate the sample dataframe with the priors.""" + base_df = self.stratified_selection() + if base_df is None: + base_df = pd.DataFrame() + # in case we needed more samples due to the strata min req + n_samples = max(self.parent.iteration.n_per_gen_for_current_iter, len(base_df)) + return self.priors.sample( + base_df, + n_samples, + self.random_generator, + ) + + def convert_to_specs( + self, df: pd.DataFrame, input_validator: type[ExperimentInputSpec] + ): + """Convert the sampled dataframe to a list of simulation specs.""" + df["experiment_id"] = "placeholder" + df["sort_index"] = np.arange(len(df)) + return [ + input_validator.model_validate(row) for row in df.to_dict(orient="records") + ] diff --git a/src/globi/models/surrogate/training.py b/src/globi/models/surrogate/training.py new file mode 100644 index 0000000..ffbf1fc --- /dev/null +++ b/src/globi/models/surrogate/training.py @@ -0,0 +1,693 @@ +"""Models used for the surrogate training pipeline.""" + +import fnmatch +import logging +import warnings +from collections.abc import Callable +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path +from typing import TYPE_CHECKING, Literal, cast + +import pandas as pd +from pydantic import Field +from scythe.base import ExperimentInputSpec, ExperimentOutputSpec +from scythe.scatter_gather import ScatterGatherResult +from scythe.utils.filesys import FileReference + +from globi.models.surrogate.backends import TrainedModelWithArtifacts +from globi.models.surrogate.backends.base import TrainingContext +from globi.models.surrogate.inference import ReferencedMLBackend +from globi.models.surrogate.metrics import compute_frame_metrics +from globi.models.surrogate.pipeline import ( + ProgressiveTrainingSpec, + StageSpec, + TargetsConfigColumnSpec, +) +from globi.models.surrogate.transforms import ( + DataPair, + IdentityScaler, + MinMaxScaler, + PrepDataResult, + StandardScaler, + TrainTestPair, + Transformers, + XTransformer, + YTransformer, + encode_inputs, +) + +if TYPE_CHECKING: + from mypy_boto3_s3.client import S3Client as S3ClientType +else: + S3ClientType = object + +logger = logging.getLogger(__name__) + + +EXCLUDED_COLUMNS = frozenset({ + "experiment_id", + "sort_index", + "workflow_run_id", + "root_workflow_run_id", +}) + + +def flatten_result_columns(df: pd.DataFrame) -> list[str]: + """Flatten a (possibly multi-level) column index into "a/b/c" strings. + + Integer levels (e.g. months) are zero-padded so the resulting names sort correctly. + """ + flat = df.columns.to_flat_index() + return [ + "/".join([str(c) if not isinstance(c, int) else f"{c:03d}" for c in col]) + if isinstance(col, tuple | list) + else str(col) + for col in flat + ] + + +def flatten_result_frames( + dfs: dict[str, pd.DataFrame], targets: list[str] +) -> pd.DataFrame: + """Select `targets` out of raw result frames, keyed and flattened as the trainer does. + + Column names follow the training convention `"/"` (see + `flatten_result_columns`), so they line up with `Transformers.y.targets`. The + row index (the feature MultiIndex) is preserved. + """ + parts: list[pd.DataFrame] = [] + for key, df in dfs.items(): + flat = df.copy() + flat.columns = pd.Index([f"{key}/{c}" for c in flatten_result_columns(df)]) + wanted = [c for c in flat.columns if c in targets] + if wanted: + parts.append(cast(pd.DataFrame, flat[wanted])) + if not parts: + msg = f"None of the targets {targets[:3]}... were found in result keys {list(dfs)}." + raise ValueError(msg) + out = pd.concat(parts, axis=1) + missing = [t for t in targets if t not in out.columns] + if missing: + msg = f"Result frames are missing targets: {missing[:5]}" + raise ValueError(msg) + return cast(pd.DataFrame, out[targets]) + + +@dataclass(frozen=True) +class TrainFoldRunResult: + """Trained artifacts and evaluation metrics for one fold.""" + + artifacts: TrainedModelWithArtifacts + global_metrics: pd.DataFrame + stratum_metrics: pd.DataFrame + + +class TrainFoldSpec(ExperimentInputSpec): + """Train an sbem model for a specific fold. + + The fold is determined by the sort_index, which does mean we need to know the n_folds. + + We will need to know: + - where the data is + - the desired stratification (e.g. feature.weather.file) + - how to divide the data into training and testing splits given the desired stratification + + The data uri should be assumed to have features in the index and targets in the columns. + + TODO: consider the potential for leakage when a stratum has few buildings! + + First, we will subdivide the data into its strata. + + Then for each stratum, we will create a train/test split according to the fold index. + + We wish to return validation metrics with the following hierarchy for the column index + - train/test ["split_segment"] + - loc1/loc2 ... ["stratum"] + - mae/rmse/r2/... ["metric"] + + Theoretically, we also might want to pass in normalization specifications for features and/or targets. + However, with xgb, this is less imperative. + """ + + data_uris: dict[str, FileReference] = Field( + ..., description="The uris (s3, http, or local paths) of the data to train on." + ) + parent: ProgressiveTrainingSpec = Field(..., description="The parent spec.") + + @cached_property + def combined_data(self) -> pd.DataFrame: + """Combines the data from the data uris into a single dataframe with a flattened column index.""" + all_dfs: dict[str, pd.DataFrame] = { + key: pd.read_parquet(str(uri)) for key, uri in self.data_uris.items() + } + + # We wiull only include dataframes which have valid targets in the training. + logger.info("Checking for valid targets in dataframes...") + dfs_to_use: dict[str, pd.DataFrame] = {} + for key, df in all_dfs.items(): + logger.info(f"Checking dataframe {key}...") + # TODO: use level names while constructing the sequential name? + new_columns = flatten_result_columns(df) + # we will only temporarily include the key prefix in the columns so we can perform the filtering check; + # it will get re-added later when concat the dataframes. + new_columns_with_prefix = [f"{key}/{col}" for col in new_columns] + df.columns = new_columns_with_prefix + viable_targets = self.valid_targets_in_df(df) + df.columns = new_columns + if viable_targets: + logger.info( + f"Including dataframe {key} with {len(viable_targets)} targets: {viable_targets}" + ) + dfs_to_use[key] = df + else: + logger.info( + f"Excluding dataframe {key} because it has no valid targets." + ) + + # TODO: consider how/if we want to handle dataframes with different indices. + if not all( + df.index.equals(next(iter(dfs_to_use.values())).index) + for df in dfs_to_use.values() + ): + msg = "The indices of the dataframes are not all equal. " + "This is not supported, since the features must be identical for all outputs.." + raise ValueError(msg) + + logger.info("Concatenating and shuffling dataframes...") + combined_df = pd.concat(dfs_to_use, axis=1) + combined_df.columns = combined_df.columns.to_flat_index() + combined_df.columns = ["/".join(col) for col in combined_df.columns] + shuffled_df = combined_df.sample(frac=1, random_state=42, replace=False) + logger.info(f"Shuffled dataframe has {len(shuffled_df)} rows.") + return shuffled_df + + @property + def data(self) -> pd.DataFrame: + """The combined data.""" + return self.combined_data + + @cached_property + def dparams(self) -> pd.DataFrame: + """The index of the data.""" + return self.data.index.to_frame() + + @cached_property + def all_feature_columns(self) -> frozenset[str]: + """The names of all columns.""" + init_cols = frozenset(self.dparams.columns) + is_exclusively_one_val = [ + col for col in init_cols if self.dparams[col].nunique() <= 1 + ] + all_cols = init_cols - frozenset(is_exclusively_one_val) + return all_cols + + @cached_property + def all_target_columns(self) -> frozenset[str]: + """The names of all columns.""" + return frozenset(self.data.columns) + + @cached_property + def continuous_columns(self) -> frozenset[str]: + """The continuous columns.""" + # TODO: add some logging calls here. + feature_conf = self.parent.regression_io_config.features + candidates = ( + self.all_feature_columns - feature_conf.exclude_columns - EXCLUDED_COLUMNS + ) + object_dype_columns = ( + self.dparams[candidates].select_dtypes(include=["object"]).columns.tolist() + ) + candidates = candidates - frozenset(object_dype_columns) + nunique_counts = cast(pd.Series, self.dparams[candidates].nunique()) + thresh = feature_conf.cont_cat_unicity_transition_threshold + passing_candidates = cast( + list[str], + cast(pd.Series, nunique_counts[nunique_counts > thresh]).index.tolist(), + ) + non_passing_candidates = cast( + list[str], + cast(pd.Series, nunique_counts[nunique_counts <= thresh]).index.tolist(), + ) + prespecified = feature_conf.continuous_columns + if prespecified: + skipped_candidates = frozenset(passing_candidates) - (prespecified) + possibly_not_continuous_candidats = ( + frozenset(non_passing_candidates) & prespecified + ) + if possibly_not_continuous_candidats: + warnings.warn( + f"The following columns were specified as continuous but have less than {thresh} unique values: {possibly_not_continuous_candidats}", + stacklevel=2, + ) + if skipped_candidates: + warnings.warn( + f"The following columns are likely continuous but are not included in the continuous columns: {skipped_candidates}", + stacklevel=2, + ) + return prespecified + return frozenset(passing_candidates) + + @cached_property + def categorical_columns(self) -> frozenset[str]: + """The categorical columns.""" + # TODO: add some logging calls here. + feature_conf = self.parent.regression_io_config.features + candidates = ( + self.all_feature_columns - feature_conf.exclude_columns - EXCLUDED_COLUMNS + ) + object_dtype_columns = ( + self.dparams[candidates].select_dtypes(include=["object"]).columns.tolist() + ) + non_obj_dtype_columns = candidates - frozenset(object_dtype_columns) + nunique_counts = cast(pd.Series, self.dparams[non_obj_dtype_columns].nunique()) + thresh = feature_conf.cont_cat_unicity_transition_threshold + passing_non_obj_dtype_candidates = cast( + list[str], + cast(pd.Series, nunique_counts[nunique_counts <= thresh]).index.tolist(), + ) + non_passing_non_obj_dtype_candidates = cast( + list[str], + cast(pd.Series, nunique_counts[nunique_counts > thresh]).index.tolist(), + ) + prespecified = feature_conf.categorical_columns + if prespecified: + skipped_candidates = frozenset(passing_non_obj_dtype_candidates) - ( + prespecified + ) + possibly_not_categorical_candidats = ( + frozenset(non_passing_non_obj_dtype_candidates) & prespecified + ) + if possibly_not_categorical_candidats: + warnings.warn( + f"The following columns were specified as categorical but have more than {thresh} unique values: {possibly_not_categorical_candidats}", + stacklevel=2, + ) + if skipped_candidates: + warnings.warn( + f"The following columns are likely categorical but are not included in the categorical columns: {skipped_candidates}", + stacklevel=2, + ) + return prespecified + return frozenset(passing_non_obj_dtype_candidates) | frozenset( + object_dtype_columns + ) + + @cached_property + def x_features(self) -> frozenset[str]: + """The all features.""" + return self.continuous_columns | self.categorical_columns + + @cached_property + def stratum_names(self) -> list[str]: + """The values of the stratification field.""" + return sorted(self.dparams[self.parent.stratification.field].unique().tolist()) + + @cached_property + def data_by_stratum(self) -> dict[str, pd.DataFrame]: + """Subdivide the data by the stratification field. + + We want 1/n_folds data in the test segment for each stratification option, + so we will need to compute train/test splits separately for each stratum. + + This would not be necessary if we knew that the strata always had equal representation, but + since we might use things like adaptive sampling or generating samples proportionally to the number of buildings in that stratum, + e.g. by population, then what *could* happen if we just did a random train/test split is that some strata might end up + entirely in the train set. + """ + return { + val: cast( + pd.DataFrame, + self.data[self.dparams[self.parent.stratification.field] == val], + ) + for val in self.stratum_names + } + + @cached_property + def train_test_split_by_fold_and_stratum(self) -> pd.DataFrame: + """Create the folds for the data. + + To do this, we will go to each stratum and use a strided step to + construct each fold, then assign the fold matching the sort_index + to the test split. We also recombine the strata since they are now + safely stratified. + """ + all_strata = [] + for val in self.stratum_names: + folds = [] + for i in range(self.parent.cross_val.n_folds): + fold = self.data_by_stratum[val].iloc[ + i :: self.parent.cross_val.n_folds + ] + folds.append(fold) + folds_df = pd.concat( + folds, + axis=0, + keys=[ + "test" if i == self.sort_index else "train" + for i in range(self.parent.cross_val.n_folds) + ], + names=["split_segment"], + ) + all_strata.append(folds_df) + return pd.concat(all_strata) + + @cached_property + def train_segment(self) -> tuple[pd.DataFrame, pd.DataFrame]: + """Get the training segment.""" + train_df = cast( + pd.DataFrame, + self.train_test_split_by_fold_and_stratum.xs( + "train", level="split_segment" + ), + ) + params = train_df.index.to_frame(index=False) + targets = train_df + return params, targets + + @cached_property + def test_segment(self) -> tuple[pd.DataFrame, pd.DataFrame]: + """Get the test segment.""" + test_df = cast( + pd.DataFrame, + self.train_test_split_by_fold_and_stratum.xs("test", level="split_segment"), + ) + params = test_df.index.to_frame(index=False) + targets = test_df + return params, targets + + def valid_targets_in_df(self, df: pd.DataFrame) -> list[str]: + """Get the valid targets in the dataframe.""" + if isinstance( + self.parent.regression_io_config.targets, TargetsConfigColumnSpec + ): + if self.parent.regression_io_config.targets.columns: + return [ + c + for c in df.columns + if c in self.parent.regression_io_config.targets.columns + ] + return sorted(df.columns.tolist()) + globs = self.parent.regression_io_config.targets.globs + if not globs: + return sorted(df.columns.tolist()) + viable_target_columns = [] + for col in df.columns: + if any(fnmatch.fnmatch(col, glob) for glob in globs): + viable_target_columns.append(col) + return sorted(viable_target_columns) + + @cached_property + def targets(self) -> list[str]: + """The list of regression targets.""" + logger.info("Determining targets...") + if isinstance( + self.parent.regression_io_config.targets, TargetsConfigColumnSpec + ): + final_targets = self.parent.regression_io_config.targets.columns or sorted( + self.all_target_columns + ) + else: + globs = self.parent.regression_io_config.targets.globs + if not globs: + final_targets = sorted(self.all_target_columns) + viable_target_columns = [] + for col in self.all_target_columns: + if any(fnmatch.fnmatch(col, glob) for glob in globs): + viable_target_columns.append(col) + final_targets = sorted(viable_target_columns) + logger.info( + f"Selected {len(final_targets)} / {len(self.all_target_columns)} targets." + ) + return final_targets + + @cached_property + def target_range(self) -> list[tuple[float, float]]: + """The range of the regression targets.""" + _, targets = self.train_segment + targets = targets[self.targets] + return [ + (float(targets[col].min() * 0.8), float(targets[col].max() * 1.2)) + for col in self.targets + ] + + def train(self, tempdir: Path): + """Train the model with hyperparameter-driven backend dispatch.""" + x_cat_encoding = self.parent.regression_io_config.features.cat_encoding + x_cont_encoding = self.parent.regression_io_config.features.cont_encoding + y_encoding = self.parent.regression_io_config.targets.normalization + data = self.prep_data( + x_cat_encoding=x_cat_encoding, + x_cont_encoding=x_cont_encoding, + y_encoding=y_encoding, + ) + + context = TrainingContext( + prepped_data=data, + tempdir=tempdir, + ) + artifacts = self.parent.ml_backend.train_and_save(context) + predict_fn = self.parent.ml_backend.build_predict_fn( + model_object=artifacts.trained_model.model_object, + transformers=artifacts.trained_model.transformers, + ) + global_metrics, stratum_metrics = self.evaluate( + fn=predict_fn, + selected=data.selected, + ) + return TrainFoldRunResult( + artifacts=artifacts, + global_metrics=global_metrics, + stratum_metrics=stratum_metrics, + ) + + def prep_data( + self, + *, + x_cat_encoding: Literal["index", "one-hot"], + x_cont_encoding: Literal["min-max", "standard"] | None, + y_encoding: Literal["min-max", "standard"] | None, + ) -> PrepDataResult: + """Prepare the data for training.""" + logger.info("Preparing data for training...") + x_train, y_train = self.train_segment + x_test, y_test = self.test_segment + + # Technically we are allowing some of our test-set features' categorical options + # through, but that's okay; we are assuming we exhaustively know the categorical options + # and this is not leakage. + cats = { + col: self.dparams[col].unique().tolist() for col in self.categorical_columns + } + + x_transformer = XTransformer( + features=sorted(self.x_features), + cat_map=cats, + cat_encoding=x_cat_encoding, + continuous_features=sorted(self.continuous_columns), + cont_encoding=x_cont_encoding, + cont_scaler=( + MinMaxScaler() + if x_cont_encoding == "min-max" + else ( + StandardScaler() + if x_cont_encoding == "standard" + else IdentityScaler() + ) + ), + ) + x_train_encoded = encode_inputs( + x_train, + conf=x_transformer, + fit_continuous=True, + ) + + x_test_encoded = encode_inputs( + x_test, + conf=x_transformer, + ) + scaler = ( + MinMaxScaler() + if y_encoding == "min-max" + else StandardScaler() + if y_encoding == "standard" + else IdentityScaler() + ) + y_transformer = YTransformer( + scaler=scaler, + targets=self.targets, + normalization=y_encoding, + ) + + # select the targets + logger.info("Selecting targets...") + y_train, y_test = ( + cast(pd.DataFrame, y_train.loc[:, y_transformer.targets]), + cast(pd.DataFrame, y_test.loc[:, y_transformer.targets]), + ) + logger.info("Selected targets.") + + logger.info(f"Scaling targets with {type(y_transformer.scaler).__name__}...") + y_train_scaled = y_transformer.scaler.fit_transform(y_train) + y_test_scaled = y_transformer.scaler.transform(y_test) + logger.info("Scaled targets.") + + transformers = Transformers( + x=x_transformer, + y=y_transformer, + ) + selected = TrainTestPair( + train=DataPair(x=x_train, y=y_train), + test=DataPair(x=x_test, y=y_test), + ) + transformed = TrainTestPair( + train=DataPair(x=x_train_encoded, y=y_train_scaled), + test=DataPair(x=x_test_encoded, y=y_test_scaled), + ) + return PrepDataResult( + selected=selected, + transformed=transformed, + transformers=transformers, + ) + + def evaluate( + self, + fn: Callable[[pd.DataFrame], pd.DataFrame], + selected: TrainTestPair, + ) -> tuple[pd.DataFrame, pd.DataFrame]: + """Evaluate a model on the train and test segments.""" + logger.info("Evaluating model on train and test segments...") + x_train = selected.train.x + x_test = selected.test.x + y_train = selected.train.y + y_test = selected.test.y + # predictions come back with a fresh RangeIndex; re-attach the feature + # index so per-stratum slicing (`xs`) works. + y_train_preds = fn(x_train).set_index(y_train.index) + y_test_preds = fn(x_test).set_index(y_test.index) + + # compute the metrics + global_train_metrics, stratum_train_metrics = self.compute_metrics( + y_train_preds, y_train + ) + global_test_metrics, stratum_test_metrics = self.compute_metrics( + y_test_preds, y_test + ) + + global_metrics = pd.concat( + [global_train_metrics, global_test_metrics], + axis=1, + keys=["train", "test"], + names=["split_segment"], + ) + stratum_metrics = pd.concat( + [stratum_train_metrics, stratum_test_metrics], + axis=1, + keys=["train", "test"], + names=["split_segment"], + ) + logger.info("Model evaluated on train and test segments.") + return global_metrics, stratum_metrics + + def compute_frame_metrics( + self, preds: pd.DataFrame, targets: pd.DataFrame + ) -> pd.DataFrame: + """Compute the metrics.""" + return compute_frame_metrics(preds, targets) + + def compute_metrics(self, preds: pd.DataFrame, targets: pd.DataFrame): + """Compute the metrics.""" + global_metrics = self.compute_frame_metrics(preds, targets) + stratum_metric_dfs = {} + names = [] + for stratum_name in self.stratum_names: + if ( + stratum_name + not in targets.index.get_level_values( + self.parent.stratification.field + ).unique() + ): + continue + names.append(stratum_name) + stratum_targets = cast( + pd.DataFrame, + targets.xs(stratum_name, level=self.parent.stratification.field), + ) + stratum_preds = cast( + pd.DataFrame, + preds.xs(stratum_name, level=self.parent.stratification.field), + ) + metrics = self.compute_frame_metrics(stratum_preds, stratum_targets) + stratum_metric_dfs[stratum_name] = metrics + + stratum_metrics = pd.concat( + stratum_metric_dfs, + axis=1, + keys=names, + names=["stratum"], + ) + global_metrics = ( + global_metrics.set_index( + pd.Index( + [self.sort_index] * len(global_metrics), + name="sort_index", + ), + append=True, + ) + .set_index( + pd.Index( + [self.parent.iteration.current_iter] * len(global_metrics), + name="iteration", + ), + append=True, + ) + .unstack(level="target") + ) + + stratum_metrics = ( + stratum_metrics.set_index( + pd.Index( + [self.sort_index] * len(stratum_metrics), + name="sort_index", + ), + append=True, + ) + .set_index( + pd.Index( + [self.parent.iteration.current_iter] * len(stratum_metrics), + name="iteration", + ), + append=True, + ) + .unstack(level="target") + ) + return global_metrics, stratum_metrics + + +class FoldResult(ReferencedMLBackend, ExperimentOutputSpec): + """The output for a fold.""" + + +class TrainWithCVSpec(StageSpec): + """Train an SBEM model using a scatter gather approach for cross-fold validation.""" + + data_uris: ScatterGatherResult = Field( + ..., + description="The uris of the data to train on.", + ) + + @property + def schedule(self) -> list[TrainFoldSpec]: + """Create the task schedule.""" + schedule = [] + + for i in range(self.parent.cross_val.n_folds): + schedule.append( + TrainFoldSpec( + experiment_id="placeholder", + sort_index=i, + data_uris=dict(self.data_uris.uris), + parent=self.parent, + ) + ) + return schedule diff --git a/src/globi/models/surrogate/transforms.py b/src/globi/models/surrogate/transforms.py new file mode 100644 index 0000000..de731ea --- /dev/null +++ b/src/globi/models/surrogate/transforms.py @@ -0,0 +1,317 @@ +"""Shared transform and prediction utilities for surrogate models.""" + +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import Annotated, Literal, cast + +import numpy as np +import pandas as pd +from pydantic import BaseModel, BeforeValidator, Field + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class DataPair: + """A pair of dataframes.""" + + x: pd.DataFrame + y: pd.DataFrame + + +@dataclass(frozen=True) +class TrainTestPair: + """A pair of train and test dataframes.""" + + train: DataPair + test: DataPair + + +class MinMaxScaler(BaseModel, arbitrary_types_allowed=True): + """The configuration for a min-max scaler.""" + + kind: Literal["min-max"] = "min-max" + mins_: dict[str, float] = Field(default_factory=dict) + maxs_: dict[str, float] = Field(default_factory=dict) + + @property + def mins(self) -> pd.Series: + """The mins.""" + return pd.Series(self.mins_, name="mins", dtype=float) + + @property + def maxs(self) -> pd.Series: + """The maxs.""" + return pd.Series(self.maxs_, name="maxs", dtype=float) + + def fit(self, y: pd.DataFrame) -> None: + """Fit the min-max scaler.""" + y_min = cast(pd.Series, y.min(axis=0)) + y_max = cast(pd.Series, y.max(axis=0)) + self.mins_ = y_min.to_dict() + self.maxs_ = y_max.to_dict() + + @property + def scale(self) -> pd.Series: + """The scale.""" + scale = self.maxs - self.mins + scale = scale.where(scale != 0, self.mins) + scale = scale.where(scale != 0, 1) + return scale + + def transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Transform the data.""" + return (y - self.mins) / self.scale + + def fit_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Fit and transform the data.""" + self.fit(y) + return self.transform(y) + + def inverse_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Inverse transform the data.""" + return y * self.scale + self.mins + + +class StandardScaler(BaseModel, arbitrary_types_allowed=True): + """The configuration for a standard scaler.""" + + kind: Literal["standard"] = "standard" + means_: dict[str, float] = Field(default_factory=dict) + stds_: dict[str, float] = Field(default_factory=dict) + + @property + def means(self) -> pd.Series: + """The means.""" + return pd.Series(self.means_, name="means", dtype=float) + + @property + def stds(self) -> pd.Series: + """The stds.""" + return pd.Series(self.stds_, name="stds", dtype=float) + + def fit(self, y: pd.DataFrame) -> None: + """Fit the standard scaler.""" + y_mean = cast(pd.Series, y.mean(axis=0)) + y_std = cast(pd.Series, y.std(axis=0)) + y_std = y_std.where(y_std != 0, 1) + self.means_ = y_mean.to_dict() + self.stds_ = y_std.to_dict() + + def transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Transform the data.""" + return (y - self.means) / self.stds + + def fit_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Fit and transform the data.""" + self.fit(y) + return self.transform(y) + + def inverse_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Inverse transform the data.""" + return y * self.stds + self.means + + +class IdentityScaler(BaseModel, frozen=True): + """A scaler that does nothing.""" + + kind: Literal["identity"] = "identity" + + def fit(self, y: pd.DataFrame) -> None: + """Fit the identity scaler.""" + + def transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Transform the data.""" + return y + + def fit_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Fit and transform the data.""" + self.fit(y) + return self.transform(y) + + def inverse_transform(self, y: pd.DataFrame) -> pd.DataFrame: + """Inverse transform the data.""" + return y + + +def _infer_legacy_scaler_kind(v: object) -> object: + """Add a `kind` tag to scaler payloads serialized before the tag existed.""" + if isinstance(v, dict) and "kind" not in v: + if "mins_" in v or "maxs_" in v: + return {**v, "kind": "min-max"} + if "means_" in v or "stds_" in v: + return {**v, "kind": "standard"} + if not v: + return {"kind": "identity"} + return v + + +Scaler = Annotated[ + Annotated[ + MinMaxScaler | StandardScaler | IdentityScaler, Field(discriminator="kind") + ], + BeforeValidator(_infer_legacy_scaler_kind), +] + + +class XTransformer(BaseModel, arbitrary_types_allowed=True, frozen=True): + """A transformer for the x features.""" + + features: list[str] + continuous_features: list[str] = Field(default_factory=list) + cont_scaler: Scaler = Field(default_factory=IdentityScaler) + cont_encoding: Literal["min-max", "standard"] | None + cat_map: dict[str, list[str | float | int]] + cat_encoding: Literal["index", "one-hot"] + + +class YTransformer(BaseModel, arbitrary_types_allowed=True, frozen=True): + """A transformer for the y features.""" + + scaler: Scaler + targets: list[str] + normalization: Literal["min-max", "standard"] | None + + +class Transformers(BaseModel, frozen=True): + """A pair of transformers.""" + + x: XTransformer + y: YTransformer + + +@dataclass(frozen=True) +class PrepDataResult: + """The result of preparing the data.""" + + selected: TrainTestPair + transformed: TrainTestPair + transformers: Transformers + + +def index_encode_categorical_columns( + df: pd.DataFrame, *, cats: dict[str, list[str | float | int]], do_copy: bool = False +) -> pd.DataFrame: + """Index encode the categorical columns.""" + if do_copy: + df = df.copy(deep=True) + for col in cats: + if col not in df.columns: + msg = f"Column {col} not found in dataframe." + raise ValueError(msg) + df[col] = pd.Categorical(df[col], categories=cats[col]).codes + return df + + +def one_hot_encode_categorical_columns( + df: pd.DataFrame, *, cats: dict[str, list[str | float | int]] +) -> pd.DataFrame: + """One-hot encode the categorical columns.""" + col_blocks: list[pd.DataFrame] = [] + regular_col_names = [c for c in df.columns if c not in cats] + col_blocks.append(df.loc[:, regular_col_names]) + for col in cats: + if col not in df.columns: + msg = f"Column {col} not found in dataframe." + raise ValueError(msg) + onehot_block = pd.get_dummies( + pd.Categorical(df[col], categories=cats[col]), prefix=col, prefix_sep="." + ).astype(float) + col_blocks.append(onehot_block) + new_df = pd.concat(col_blocks, axis=1) + + return new_df + + +def scale_continuous_columns( + df: pd.DataFrame, + *, + continuous_features: list[str], + scaler: MinMaxScaler | StandardScaler | IdentityScaler, + do_copy: bool = False, +) -> pd.DataFrame: + """Scale the continuous columns.""" + if not continuous_features: + return df + missing_cols = [col for col in continuous_features if col not in df.columns] + if missing_cols: + msg = f"Continuous columns not found in dataframe: {missing_cols}." + raise ValueError(msg) + if do_copy: + df = df.copy(deep=True) + continuous_df = cast(pd.DataFrame, df.loc[:, continuous_features]) + scaled_continuous_df = scaler.transform(continuous_df) + df.loc[:, continuous_features] = scaled_continuous_df.astype(float) + return df + + +def encode_inputs( + x: pd.DataFrame, + *, + conf: XTransformer, + fit_continuous: bool = False, + log: Callable[[str], None] = lambda x: logger.info(x), + do_copy: bool = False, + keep_index: bool = False, +) -> pd.DataFrame: + """Encode the inputs.""" + log(f"Selecting {len(conf.features)} features out of {len(x.columns)}...") + x_encoded = x.loc[:, conf.features] + log("Selected features.") + + log( + f"Encoding {len(conf.continuous_features)} continuous inputs with " + f"{conf.cont_encoding} encoding..." + ) + if fit_continuous and conf.continuous_features: + conf.cont_scaler.fit( + cast(pd.DataFrame, x_encoded.loc[:, conf.continuous_features]) + ) + x_encoded = scale_continuous_columns( + x_encoded, + continuous_features=conf.continuous_features, + scaler=conf.cont_scaler, + do_copy=do_copy, + ) + log("Encoded continuous inputs.") + + log(f"Encoding categorical inputs with {conf.cat_encoding} encoding...") + if conf.cat_encoding == "index": + x_encoded = index_encode_categorical_columns( + x_encoded, cats=conf.cat_map, do_copy=do_copy + ) + elif conf.cat_encoding == "one-hot": + x_encoded = one_hot_encode_categorical_columns(x_encoded, cats=conf.cat_map) + else: + raise NotImplementedError( + f"Unsupported categorical encoding: {conf.cat_encoding}" + ) + log("Encoded inputs.") + return ( + x_encoded.set_index(pd.MultiIndex.from_frame(x)) + if keep_index + else x_encoded.reset_index(drop=True) + ) + + +def predict[T: pd.DataFrame | np.ndarray]( + x: pd.DataFrame, + *, + conf: Transformers, + pred_fn: Callable[[pd.DataFrame, list[str]], T], +) -> pd.DataFrame: + """Predict the targets for the given features.""" + x_encoded = encode_inputs( + x, + conf=conf.x, + do_copy=False, + keep_index=False, + ) + preds = pred_fn(x_encoded.reset_index(drop=True), conf.y.targets) + preds = pd.DataFrame( + preds, columns=pd.Index(conf.y.targets) + ) # , index=x_encoded.index) + if conf.y.scaler: + preds = conf.y.scaler.inverse_transform(preds) + return preds diff --git a/src/globi/models/tasks.py b/src/globi/models/tasks.py index 699695a..54beee2 100644 --- a/src/globi/models/tasks.py +++ b/src/globi/models/tasks.py @@ -1,15 +1,26 @@ """Task models for the GloBI project.""" +import base64 import logging +import warnings +from collections.abc import Iterable from functools import cached_property from pathlib import Path -from typing import Literal +from typing import Any, Literal, cast import numpy as np from epinterface.geometry import compute_shading_mask -from pydantic import BaseModel, Field, model_validator +from pydantic import ( + BaseModel, + Field, + SerializationInfo, + field_serializer, + field_validator, + model_validator, +) from scythe.base import ExperimentInputSpec, ExperimentOutputSpec from scythe.utils.filesys import FileReference +from shapely import Polygon, from_wkb, from_wkt from globi.models.configs import GloBIExperimentSpec from globi.type_utils import ( @@ -97,7 +108,9 @@ def globi_spec(self) -> "GloBIBuildingSpec": neighbor_polys=[], neighbor_heights=[], neighbor_floors=[], - rotated_rectangle=f"Polygon ((0 0, {self.length} 0, {self.length} {self.width}, 0 {self.width}, 0 0))", + rotated_rectangle=from_wkt( + f"Polygon ((0 0, {self.length} 0, {self.length} {self.width}, 0 {self.width}, 0 0))" + ).wkb, long_edge_angle=0, long_edge=self.length, short_edge=self.width, @@ -132,7 +145,7 @@ class GloBIBuildingSpec(ExperimentInputSpec): ..., description="The semantic field values which will be used to compile the zone definition.", ) - neighbor_polys: list[str] = Field( + neighbor_polys: list[bytes] = Field( ..., description="The polygons of the neighboring buildings." ) neighbor_heights: list[float | int | None] = Field( @@ -141,7 +154,7 @@ class GloBIBuildingSpec(ExperimentInputSpec): neighbor_floors: list[float | int | None] = Field( ..., description="The number of floors of the neighboring buildings." ) - rotated_rectangle: str = Field( + rotated_rectangle: bytes = Field( ..., description="The rotated rectangle fitted around the base of the building." ) long_edge_angle: float = Field( @@ -156,6 +169,7 @@ class GloBIBuildingSpec(ExperimentInputSpec): aspect_ratio: float = Field( ..., description="The aspect ratio of the building footprint [unitless]." ) + # TODO: delete this entirely! rotated_rectangle_area_ratio: float = Field( ..., description="The ratio of the rotated rectangle footprint area to the building footprint area.", @@ -180,46 +194,123 @@ class GloBIBuildingSpec(ExperimentInputSpec): exposed_basement_frac: float = Field( ..., description="The fraction of the basement that is exposed to the air.", - gt=0, + ge=0, lt=1, ) + attic_use_fraction: float | None = Field( + default=None, + description="The use fraction of the attic.", + ge=0, + le=1, + ) + basement_use_fraction: float | None = Field( + default=None, + description="The use fraction of the basement.", + ge=0, + le=1, + ) + attic_height: float | None = Field( + default=None, + description="The height of the attic.", + ge=0, + ) parent_experiment_spec: GloBIExperimentSpec | None = Field( default=None, description="The parent experiment spec.", ) + zoning: Literal["by_storey", "core/perim", "one", "auto"] = Field( + default="auto", + description="The zoning strategy to use for the building. If `auto,` chooses between `by_storey` and `core/perim` based off edge lengths.", + ) + + @staticmethod + def _coerce_geo_bytes(value: Any) -> bytes: + """Coerce a geometry value (Polygon, WKT str, base64 str/bytes, or raw WKB) to WKB bytes.""" + if isinstance(value, Polygon): + return value.wkb + if isinstance(value, str): + try: + return from_wkt(value).wkb + except Exception: + return base64.b64decode(value) + if isinstance(value, bytes): + try: + return base64.b64decode(value, validate=True) + except Exception: + return value + return value + + @field_validator("rotated_rectangle", mode="before") + def validate_rotated_rectangle(cls, value: Any) -> bytes: + """Validate the rotated rectangle.""" + return cls._coerce_geo_bytes(value) + + @field_validator("neighbor_polys", mode="before") + def validate_neighbor_polys(cls, value: Any) -> list[bytes]: + """Validate the neighbor polygons. + + After a Parquet roundtrip, list columns come back as numpy arrays, + not Python lists — so we must accept any iterable. + """ + if isinstance(value, Iterable) and not isinstance(value, str | bytes): + return [cls._coerce_geo_bytes(poly) for poly in value] + return value # type: ignore[return-value] + + @field_serializer("rotated_rectangle") + def serialize_rotated_rectangle( + self, value: bytes, _info: SerializationInfo + ) -> bytes | str: + """Serialize WKB bytes as base64 for JSON mode.""" + if _info.mode == "json": + return base64.b64encode(value).decode("ascii") + return value + + @field_serializer("neighbor_polys") + def serialize_neighbor_polys( + self, value: list[bytes], _info: SerializationInfo + ) -> list[bytes] | list[str]: + """Serialize WKB bytes as base64 for JSON mode.""" + if _info.mode == "json": + return [base64.b64encode(v).decode("ascii") for v in value] + return value + + @cached_property + def shading_mask(self) -> np.ndarray: + """The shading mask for the building.""" + return compute_shading_mask( + cast(Polygon, from_wkb(self.rotated_rectangle)), + neighbors=[cast(Polygon, from_wkb(poly)) for poly in self.neighbor_polys], + neighbor_heights=self.neighbor_heights, + azimuthal_angle=2 * np.pi / 48, + ) @property - def feature_dict(self) -> dict[str, str | int | float]: + def computed_features(self) -> dict[str, str | int | float]: """Return a dictionary of features which will be available to ML algos.""" + perimeter = 2 * (self.long_edge + self.short_edge) + perim_to_area = perimeter / (self.long_edge * self.short_edge) features: dict[str, str | int | float] = { - "feature.geometry.long_edge": self.long_edge, - "feature.geometry.short_edge": self.short_edge, - "feature.geometry.orientation": self.long_edge_angle, - "feature.geometry.orientation.cos": np.cos(self.long_edge_angle), - "feature.geometry.orientation.sin": np.sin(self.long_edge_angle), - "feature.geometry.aspect_ratio": self.aspect_ratio, - "feature.geometry.wwr": self.wwr, - "feature.geometry.num_floors": self.num_floors, - "feature.geometry.f2f_height": self.f2f_height, + # "feature.geometry.long_edge": self.long_edge, + # "feature.geometry.short_edge": self.short_edge, + # "feature.geometry.orientation": self.long_edge_angle, + # "feature.geometry.orientation.cos": np.cos(self.long_edge_angle), + # "feature.geometry.orientation.sin": np.sin(self.long_edge_angle), + # "feature.geometry.aspect_ratio": self.aspect_ratio, + # "feature.geometry.wwr": self.wwr, + # "feature.geometry.num_floors": self.num_floors, + # "feature.geometry.f2f_height": self.f2f_height, # "feature.geometry.fp_area": self.fp_area, - "feature.geometry.zoning": self.use_core_perim_zoning, + "feature.geometry.perimeter": perimeter, + "feature.geometry.perim_to_area": perim_to_area, + "feature.geometry.zoning": self.geometry_zoning, "feature.geometry.energy_model_conditioned_area": self.energy_model_conditioned_area, "feature.geometry.energy_model_occupied_area": self.energy_model_occupied_area, - "feature.geometry.attic_height": self.attic_height or 0, - "feature.geometry.exposed_basement_frac": self.exposed_basement_frac, + # "feature.geometry.attic_height": self.attic_height or 0, + # "feature.geometry.exposed_basement_frac": self.exposed_basement_frac, } - # TODO: consider passing in - # neighbors directly to Model.geometry, letting model perform neighbor - # insertion directly rather than via a callback, - # and then let shading mask become a computed property of the model.geometry. - shading_mask = compute_shading_mask( - self.rotated_rectangle, - neighbors=self.neighbor_polys, - neighbor_heights=self.neighbor_heights, - azimuthal_angle=2 * np.pi / 48, - ) + shading_mask = self.shading_mask shading_mask_values = { f"feature.geometry.shading_mask_{i:02d}": val for i, val in enumerate(shading_mask.tolist()) @@ -233,7 +324,7 @@ def feature_dict(self) -> dict[str, str | int | float]: for feature_name, feature_value in self.semantic_field_context.items() }) - features["feature.weather.file"] = self.epwzip_path.stem + # features["feature.weather.file"] = self.epwzip_path.stem # conditional features are derived from the static and semantic features, # and may be subject to things like conditional sampling, estimation etc. @@ -249,7 +340,7 @@ def feature_dict(self) -> dict[str, str | int | float]: "Yes" if self.basement_is_conditioned else "No" ) features["feature.extra_spaces.basement.use_fraction"] = ( - self.basement_use_fraction + self.basement_use_fraction or 0 ) features["feature.extra_spaces.attic.exists"] = ( "Yes" if self.has_attic else "No" @@ -260,17 +351,34 @@ def feature_dict(self) -> dict[str, str | int | float]: features["feature.extra_spaces.attic.conditioned"] = ( "Yes" if self.attic_is_conditioned else "No" ) - features["feature.extra_spaces.attic.use_fraction"] = self.attic_use_fraction + features["feature.extra_spaces.attic.use_fraction"] = ( + self.attic_use_fraction or 0 + ) return features - # TODO: use the scythe automatic referencing for these paths - FileReference class from scythe.utils.files - # choose a local file and direclty use the 'Path' for this - # self scythe - fetch uri - # input_sepc.weather_file - # everything gets a tempdir + @model_validator(mode="before") + def validate_semantic_field_context(cls, values: dict[str, Any]): + """Fold flat `semantic_field_` keys (e.g. from a sampled dataframe row) into the context. - # + The flat keys are removed afterwards: the spec allows extra fields, and leaving + them in place would duplicate `feature.semantic.` in the result index. + """ + if not isinstance(values, dict): + return values + values = dict(values) + flat_keys = [ + k + for k in values + if k.startswith("semantic_field_") and k != "semantic_field_context" + ] + additional_semantic_fields = { + k.replace("semantic_field_", ""): values.pop(k) for k in flat_keys + } + if "semantic_field_context" not in values: + values["semantic_field_context"] = {} + values["semantic_field_context"].update(additional_semantic_fields) + return values @cached_property def db_path(self) -> Path: @@ -317,10 +425,30 @@ def component_map(self) -> Path: return self.fetch_uri(self.component_map_file) @property - def use_core_perim_zoning(self) -> Literal["by_storey", "core/perim"]: - """Whether to use the core perimeter for the simulation.""" - use_core_perim = self.long_edge > 15 and self.short_edge > 15 - return "core/perim" if use_core_perim else "by_storey" + def perim_depth(self) -> float: + """The depth of the perimeter in meters.""" + return 4.57 # 15 feet + + @property + def min_core_depth(self) -> float: + """The minimum depth of the core in meters.""" + return 1 # 3ft + + @property + def geometry_zoning(self) -> Literal["by_storey", "core/perim"]: + """Whether to use the core perimeter for the simulation. + + The shorter of the two edges must be long enough such that two perimeter zones and a core zone can fit. + . + """ + min_width = 2 * self.perim_depth + self.min_core_depth + + if self.zoning == "auto": + use_core_perim = self.long_edge > min_width and self.short_edge > min_width + return "core/perim" if use_core_perim else "by_storey" + if self.zoning == "one": + return "by_storey" + return self.zoning @property def basement_is_occupied(self) -> bool: @@ -342,37 +470,63 @@ def attic_is_conditioned(self) -> bool: """Whether the attic is conditioned.""" return self.attic in ConditionedOptions - @cached_property - def basement_use_fraction(self) -> float: + @model_validator(mode="after") + def basement_use_fraction_validator(self): """The use fraction of the basement.""" if not self.basement_is_occupied: - return 0 - return np.random.uniform(0.2, 0.6) + self.basement_use_fraction = ( + 0 # TODO: previously this was allowed to be None + ) + return self + if self.basement_use_fraction is not None: + return self + self.basement_use_fraction = np.random.uniform(0.2, 0.6) + return self - @cached_property - def attic_use_fraction(self) -> float: + @model_validator(mode="after") + def attic_use_fraction_validator(self): """The use fraction of the attic.""" if not self.attic_is_occupied: - return 0 - # TODO: use sampling as a fallback value when a default is not provided rather - # than always sampling. - return np.random.uniform(0.2, 0.6) + self.attic_use_fraction = 0 # TODO: previously this was allowed to be None + return self + if self.attic_use_fraction is not None: + return self + self.attic_use_fraction = np.random.uniform(0.2, 0.6) + return self - @cached_property + @property def has_basement(self) -> bool: """Whether the building has a basement.""" return self.basement != "none" - @cached_property + @property def has_attic(self) -> bool: """Whether the building has an attic.""" return self.attic != "none" - @cached_property - def attic_height(self) -> float | None: + @model_validator(mode="after") + def attic_height_validator(self): """The height of the attic.""" + if self.short_edge > 18 and self.attic != "none": + warnings.warn( + f"Short edge is too large: {self.short_edge:0.2f}m > 18m. Removing attic.", + stacklevel=3, + ) + self.attic = "none" + self.attic_height = 0 + return self + if not self.has_attic: - return None + self.attic_height = 0 # TODO: previously this was allowed to be None + return self + if self.attic_height is not None: + if self.attic_height > self.f2f_height * 2.5: + msg = f"Attic height is too large: {self.attic_height} > {self.f2f_height * 2.5:0.1f}. Setting to 2x f2f height." + warnings.warn(msg, stacklevel=2) + self.attic_height = self.f2f_height * 2 + + return self + # TODO: some VERY wonky numbers can come out if the building is too large. min_occupied_or_conditioned_rise_over_run = 6 / 12 max_occupied_or_conditioned_rise_over_run = 9 / 12 min_unoccupied_and_unconditioned_rise_over_run = 4 / 12 @@ -396,9 +550,12 @@ def attic_height(self) -> float | None: attic_height = None attempts -= 1 if attic_height is None: - msg = "Failed to sample valid attic height (must be less than 2.5x f2f height)." - raise ValueError(msg) - return attic_height + msg = "Failed to sample valid attic height (must be less than 2.tx f2f height). Setting to 2x f2f height." + warnings.warn(msg, stacklevel=2) + self.attic_height = self.f2f_height * 2 + return self + self.attic_height = attic_height + return self @property def n_conditioned_floors(self) -> int: diff --git a/src/globi/pipelines/__init__.py b/src/globi/pipelines/__init__.py new file mode 100644 index 0000000..9c61f28 --- /dev/null +++ b/src/globi/pipelines/__init__.py @@ -0,0 +1,13 @@ +"""Pipelines for the GloBI project.""" + +from globi.models.surrogate.dummy import dummy_simulation +from globi.pipelines.gis import preprocess_gis_file +from globi.pipelines.simulations import simulate_globi_building +from globi.pipelines.training import iterative_training + +__all__ = [ + "dummy_simulation", + "iterative_training", + "preprocess_gis_file", + "simulate_globi_building", +] diff --git a/src/globi/pipelines.py b/src/globi/pipelines/gis.py similarity index 56% rename from src/globi/pipelines.py rename to src/globi/pipelines/gis.py index 24ddd58..162b21c 100644 --- a/src/globi/pipelines.py +++ b/src/globi/pipelines/gis.py @@ -1,27 +1,14 @@ -"""Experiment configuration for building builder simulations.""" +"""GIS processing pipelines for the GloBI project.""" import logging from pathlib import Path from typing import cast import geopandas as gpd -import numpy as np import pandas as pd import yaml -from epinterface.geometry import ( - SceneContext, - ShoeboxGeometry, -) -from epinterface.sbem.builder import ( - AtticAssumptions, - BasementAssumptions, - Model, - construct_zone_def, -) from epinterface.sbem.fields.spec import SemanticModelFields -from scythe.registry import ExperimentRegistry -from scythe.utils.filesys import FileReference -from shapely import Polygon, from_wkt +from shapely import to_wkb from globi.gis.errors import SemanticFieldsFileHasNoBuildingIDColumnError from globi.gis.geometry import ( @@ -52,205 +39,22 @@ DeterministicGISPreprocessorConfig, FileConfig, GISPreprocessorColumnMap, + GloBIExperimentSpec, ) -from globi.models.tasks import GloBIBuildingSpec, GloBIOutputSpec +from globi.models.tasks import GloBIBuildingSpec logger = logging.getLogger(__name__) -INDEX_COLS_TO_KEEP: list[str] = [ - "feature.geometry.long_edge", - "feature.geometry.short_edge", - "feature.geometry.orientation", - "feature.geometry.num_floors", - "feature.geometry.energy_model_conditioned_area", - "feature.geometry.energy_model_occupied_area", - "feature.semantic.Typology", - "feature.semantic.Age_bracket", - "feature.semantic.Region", - "feature.weather.file", - "feature.geometry.wwr", - "feature.geometry.f2f_height", - "feature.geometry.attic_height", -] - - -def simulate_globi_building_pipeline( - input_spec: GloBIBuildingSpec, - tempdir: Path, -) -> GloBIOutputSpec: - """Simulate a GlobiSpec building and return energy and peak results. - - Args: - input_spec: The input specification containing building parameters and file URIs - tempdir: Temporary directory for intermediate files - Returns: - Output specification containing a DataFrame with MultiIndex: - - Top level: Measurement type (Energy, Peak) - - Feature levels from input specification - """ - spec = input_spec - log = logger.info - zone_def = construct_zone_def( - component_map_path=spec.component_map, - db_path=spec.db_path, - semantic_field_context=spec.semantic_field_context, - ) - model = Model( - Weather=spec.epwzip_path, - Zone=zone_def, - Basement=BasementAssumptions( - Conditioned=spec.basement_is_conditioned, - UseFraction=spec.basement_use_fraction - if spec.basement_is_occupied - else None, - ), - Attic=AtticAssumptions( - Conditioned=spec.attic_is_conditioned, - UseFraction=spec.attic_use_fraction if spec.attic_is_occupied else None, - ), - geometry=ShoeboxGeometry( - x=0, - y=0, - w=spec.long_edge, - d=spec.short_edge, - h=spec.f2f_height, - wwr=spec.wwr, - num_stories=spec.num_floors, - basement=spec.has_basement, - zoning=spec.use_core_perim_zoning, - roof_height=spec.attic_height, - exposed_basement_frac=spec.exposed_basement_frac, - scene_context=SceneContext( - building=cast(Polygon, from_wkt(spec.rotated_rectangle)), - neighbors=[ - cast(Polygon, from_wkt(poly)) for poly in spec.neighbor_polys - ], - neighbor_heights=[ - float(h) if h is not None else 0 for h in spec.neighbor_heights - ], - orientation=spec.long_edge_angle, - ), - ), - ) - - log("Building and running model...") - overheating_config = ( - spec.parent_experiment_spec.overheating_config - if spec.parent_experiment_spec - else None - ) - run_result = model.run( - eplus_parent_dir=tempdir, - overheating_config=overheating_config, - ) - # Validate conditioned area - if not np.allclose( - model.total_conditioned_area, spec.energy_model_conditioned_area - ): - msg = ( - f"Total conditioned area mismatch: " - f"{model.total_conditioned_area} != {spec.energy_model_conditioned_area}" - ) +def _scenario_field_name(semantic_field_names: list[str]) -> str: + """Return the semantic field name that should receive scenario overrides.""" + matches = [name for name in semantic_field_names if name.lower() == "scenario"] + if not matches: + return "scenario" + if len(matches) > 1: + msg = f"Multiple scenario semantic fields found: {matches}" raise ValueError(msg) - - # Results Post-processing - # TODO: consider if we actually want all t he columns we are including. - feature_index = spec.make_multiindex( - n_rows=1, additional_index_data=spec.feature_dict - ) - results = run_result.energy_and_peak.to_frame().T.set_index(feature_index) - - dfs: dict[str, pd.DataFrame] = { - "EnergyAndPeak": results, - } - if run_result.overheating_results is not None: - # TODO: add feature dict to overheating df indices? Or instead of a full feature df, just add a single column with the building id? - edh = run_result.overheating_results.edh - old_ix = edh.index - feature_index = spec.make_multiindex( - n_rows=len(edh), include_sort_subindex=False - ) - edh.index = feature_index - edh = edh.set_index(old_ix, append=True) - dfs["ExceedanceDegreeHours"] = edh - - basic_oh = run_result.overheating_results.basic_oh - old_ix = basic_oh.index - feature_index = spec.make_multiindex( - n_rows=len(basic_oh), include_sort_subindex=False - ) - basic_oh.index = feature_index - basic_oh = basic_oh.set_index(old_ix, append=True) - dfs["BasicOverheating"] = basic_oh - - heat_index_categories = run_result.overheating_results.hi - old_ix = heat_index_categories.index - feature_index = spec.make_multiindex( - n_rows=len(heat_index_categories), include_sort_subindex=False - ) - heat_index_categories.index = feature_index - heat_index_categories = heat_index_categories.set_index(old_ix, append=True) - dfs["HeatIndexCategories"] = heat_index_categories - - consecutive_e_zone = run_result.overheating_results.consecutive_e_zone - # may be zero if no streaks found in any zones - if len(consecutive_e_zone) > 0: - old_ix = consecutive_e_zone.index - feature_index = spec.make_multiindex( - n_rows=len(consecutive_e_zone), include_sort_subindex=False - ) - consecutive_e_zone.index = feature_index - consecutive_e_zone = consecutive_e_zone.set_index(old_ix, append=True) - dfs["ConsecutiveExceedances"] = consecutive_e_zone - - hourly_data_outpath: FileReference | None = None - - if spec.parent_experiment_spec and spec.parent_experiment_spec.hourly_data_config: - hourly_df = run_result.sql.timeseries_by_name( - spec.parent_experiment_spec.hourly_data_config.data, - reporting_frequency="Hourly", - ) - hourly_df.index.names = ["Timestep"] - hourly_df.columns.names = ["Trash", "Group", "Meter"] - hourly_df: pd.DataFrame = cast( - pd.DataFrame, - hourly_df.droplevel("Trash", axis=1) - .stack(level="Group", future_stack=True) - .unstack(level="Timestep"), - ) - hourly_multiindex = spec.make_multiindex( - n_rows=len(hourly_df), include_sort_subindex=False - ) - old_ix = hourly_df.index - hourly_df.index = hourly_multiindex - hourly_df = hourly_df.set_index(old_ix, append=True) - - if spec.parent_experiment_spec.hourly_data_config.does_dataframe_output: - for meter_name in hourly_df.columns.get_level_values("Meter").unique(): - variable_df = hourly_df.xs(meter_name, level="Meter", axis=1) - dataframe_key = f"HourlyData.{meter_name.replace(' ', '')}" - dfs[dataframe_key] = variable_df - if spec.parent_experiment_spec.hourly_data_config.does_file_output: - hourly_data_outpath = tempdir / "outputs_hourly_data.pq" - hourly_df.to_parquet(hourly_data_outpath) - - return GloBIOutputSpec( - dataframes=dfs, - hourly_data=hourly_data_outpath, - ) - - -@ExperimentRegistry.Register(retries=2, schedule_timeout="10h", execution_timeout="30m") -def simulate_globi_building( - input_spec: GloBIBuildingSpec, tempdir: Path -) -> GloBIOutputSpec: - """Simulate a GlobiSpec building and return monthly energy and peak results. - - NB: this is separated from the pipeline above so the pipeline can still be used as a - local invocation without *too* much difficulty. - """ - return simulate_globi_building_pipeline(input_spec, tempdir) + return matches[0] def preprocess_gis_file( @@ -296,33 +100,45 @@ def preprocess_gis_file( if semantic_fields.Building_ID_col is None: raise SemanticFieldsFileHasNoBuildingIDColumnError() - gdf = cast(gpd.GeoDataFrame, gpd.read_file(gis_fp)) + gdf = ( + cast(gpd.GeoDataFrame, gpd.read_parquet(gis_fp)) + if gis_fp.suffix in [".pq", ".parquet"] + else cast(gpd.GeoDataFrame, gpd.read_file(gis_fp)) + ) validate_has_rows(gdf) # Check that the current CRS is WGS84 or the cart one, convert to WGS84 early and use throughout - gdf = reproject_gdf(gdf, config.cart_crs) + gdf, estimated_utm_crs = reproject_gdf(gdf, config.cart_crs) - required_col_names = semantic_fields.field_names + required_col_names_semantic = semantic_fields.semantic_field_names + required_col_names_rich = semantic_fields.rich_field_names # We need to deal with the fact that shapefiles will trucnate the column # name to 10 characters, but users might not realize this when they # export from e.g. ArcGIS. - gdf = rename_shp_cols(gdf, required_col_names, log_fn=logger.info) + gdf = rename_shp_cols(gdf, required_col_names_rich, log_fn=logger.info) + gdf = rename_shp_cols( + gdf, + [c for c in required_col_names_semantic if c is not None], + log_fn=logger.info, + ) if scenario is not None: - gdf["scenario"] = scenario + gdf[_scenario_field_name(required_col_names_semantic)] = scenario # We want to run a consistency check to make sure that the requested semantic fields # are actually in the GDF after we have dealt with appropriate renaming. # We also should run a consistency check to make sure that every cell value that is listed as a # semantic field is actually one of the expected values. - check_for_column_existence(gdf, required_col_names, log_fn=logger.info) - validate_semantic_field_compatibility( - gdf, - semantic_fields, - missing_ok=False, - log_fn=logger.info, - ) + if config.check_semantic_fields: + check_for_column_existence(gdf, required_col_names_semantic, log_fn=logger.info) + validate_semantic_field_compatibility( + gdf, + semantic_fields, + missing_ok=False, + log_fn=logger.info, + ) + check_for_column_existence(gdf, required_col_names_rich, log_fn=logger.info) # If the building ID column is not provided or partial, we will inject uuids gdf, semantic_fields.Building_ID_col = check_building_ids( @@ -405,12 +221,13 @@ def preprocess_gis_file( validate_has_rows(gdf) logger.info("injecting rotated rectangles") gdf, injected_geometry_column_map = inject_rotated_rectangles( - gdf, cart_crs=config.cart_crs + gdf, cart_crs=estimated_utm_crs ) gdf, n_dropped_by_area = drop_by_area( gdf, area_col=injected_geometry_column_map.Footprint_Area_col, min_area=config.min_building_area, + max_area=config.max_building_area, log_fn=logger.info, ) validate_has_rows(gdf) @@ -463,7 +280,9 @@ def preprocess_gis_file( ) # Construct a dictionary of the semantic field values for each building. - gdf, semantic_fields_context_col = inject_semantic_fields(gdf, semantic_fields) + gdf, semantic_fields_context_col = inject_semantic_fields( + gdf, semantic_fields if config.check_semantic_fields else None + ) # EPW FILE HANDLING gdf, semantic_fields.Weather_File_col = handle_epwzip( @@ -471,7 +290,7 @@ def preprocess_gis_file( weather_file_col=semantic_fields.Weather_File_col, assumed_epwzip=file_config.epwzip_file, epw_query=config.epw_query, - cart_crs=config.cart_crs, + cart_crs=estimated_utm_crs, log_fn=logger.info, ) @@ -524,15 +343,57 @@ def preprocess_gis_file( return gdf, column_output_map -if __name__ == "__main__": - import tempfile +def build_building_specs( + gdf: gpd.GeoDataFrame | pd.DataFrame, + colmap: GISPreprocessorColumnMap, + file_config: FileConfig, + parent_experiment_spec: GloBIExperimentSpec | None = None, +) -> list[GloBIBuildingSpec]: + """Convert a preprocessed GIS frame into one simulation spec per building. - from globi.models.tasks import MinimalBuildingSpec + Args: + gdf: The frame returned by `preprocess_gis_file`. + colmap: The column map returned alongside it. + file_config: Source of the semantic fields / component map file references. + parent_experiment_spec: Optional experiment spec (enables overheating/hourly outputs). - with tempfile.TemporaryDirectory() as tempdir: - with open("inputs/building.yml") as f: - input_spec = MinimalBuildingSpec.model_validate(yaml.safe_load(f)) - o = simulate_globi_building_pipeline( - input_spec=input_spec.globi_spec, - tempdir=Path(tempdir), + Returns: + specs: One `GloBIBuildingSpec` per row, with `sort_index` set positionally. + """ + specs: list[GloBIBuildingSpec] = [] + for sort_index, (_, row) in enumerate(gdf.iterrows()): + row = row.to_dict() + specs.append( + GloBIBuildingSpec( + building_id=row[colmap.Building_ID_col], + experiment_id="placeholder", + sort_index=sort_index, + db_file=row[colmap.DB_File_col], + semantic_fields_file=file_config.semantic_fields_file, + component_map_file=file_config.component_map_file, + epwzip_file=row[colmap.EPWZip_File_col], + semantic_field_context=row[colmap.Semantic_Field_Context_col], + neighbor_polys=[ + to_wkb(poly) for poly in row[colmap.Neighbor_Polys_col] + ], + neighbor_heights=row[colmap.Neighbor_Heights_col], + neighbor_floors=row[colmap.Neighbor_Floors_col], + rotated_rectangle=to_wkb(row[colmap.Rotated_Rectangle_col]), + long_edge_angle=row[colmap.Long_Edge_Angle_col], + long_edge=row[colmap.Long_Edge_col], + short_edge=row[colmap.Short_Edge_col], + aspect_ratio=row[colmap.Aspect_Ratio_col], + rotated_rectangle_area_ratio=row[ + colmap.Rotated_Rectangle_Area_Ratio_col + ], + wwr=row[colmap.WWR_col], + height=row[colmap.Height_col], + num_floors=row[colmap.Num_Floors_col], + f2f_height=row[colmap.F2F_Height_col], + basement=row[colmap.Basement_col], + attic=row[colmap.Attic_col], + exposed_basement_frac=row[colmap.Exposed_Basement_Frac_col], + parent_experiment_spec=parent_experiment_spec, + ) ) + return specs diff --git a/src/globi/pipelines/simulations.py b/src/globi/pipelines/simulations.py new file mode 100644 index 0000000..3546936 --- /dev/null +++ b/src/globi/pipelines/simulations.py @@ -0,0 +1,406 @@ +"""Experiment configuration for building builder simulations.""" + +import logging +from datetime import timedelta +from pathlib import Path +from typing import cast + +import numpy as np +import pandas as pd +import yaml +from epinterface.geometry import ( + SceneContext, + ShoeboxGeometry, +) +from epinterface.sbem.builder import ( + AtticAssumptions, + BasementAssumptions, + Model, + construct_zone_def, +) +from scythe.registry import ExperimentRegistry +from scythe.utils.filesys import FileReference +from shapely import Polygon, from_wkb + +from globi.models.tasks import GloBIBuildingSpec, GloBIOutputSpec + +logger = logging.getLogger(__name__) + + +def simulate_globi_building_pipeline( # noqa: C901 + input_spec: GloBIBuildingSpec, + tempdir: Path, +) -> GloBIOutputSpec: + """Simulate a GlobiSpec building and return energy and peak results. + + Args: + input_spec: The input specification containing building parameters and file URIs + tempdir: Temporary directory for intermediate files + Returns: + Output specification containing a DataFrame with MultiIndex: + - Top level: Measurement type (Energy, Peak) + - Feature levels from input specification + """ + spec = input_spec + log = logger.info + zone_def = construct_zone_def( + component_map_path=spec.component_map, + db_path=spec.db_path, + semantic_field_context=spec.semantic_field_context, + ) + model = Model( + Weather=spec.epwzip_path, + Zone=zone_def, + Basement=BasementAssumptions( + Conditioned=spec.basement_is_conditioned, + UseFraction=spec.basement_use_fraction + if spec.basement_is_occupied + else None, + ), + Attic=AtticAssumptions( + Conditioned=spec.attic_is_conditioned, + UseFraction=spec.attic_use_fraction if spec.attic_is_occupied else None, + ), + geometry=ShoeboxGeometry( + x=0, + y=0, + w=spec.long_edge, + d=spec.short_edge, + h=spec.f2f_height, + wwr=spec.wwr, + num_stories=spec.num_floors, + basement=spec.has_basement, + perim_depth=spec.perim_depth, + zoning=spec.geometry_zoning, + roof_height=spec.attic_height or None, + exposed_basement_frac=spec.exposed_basement_frac, + scene_context=SceneContext( + building=cast(Polygon, from_wkb(spec.rotated_rectangle)), + neighbors=[ + cast(Polygon, from_wkb(poly)) for poly in spec.neighbor_polys + ], + neighbor_heights=[ + float(h) if h is not None else 0 for h in spec.neighbor_heights + ], + orientation=spec.long_edge_angle, + ), + ), + ) + + single_zone_mode = spec.zoning == "one" + if single_zone_mode: + model = model.convert_to_onezone() + + # if single_zone_mode: + # model = Model( + # Weather=spec.epwzip_path, + # Zone=zone_def, + # Basement=BasementAssumptions( + # Conditioned=False, + # UseFraction=None, + # ), + # Attic=AtticAssumptions( + # Conditioned=False, + # UseFraction=None, + # ), + # geometry=ShoeboxGeometry( + # x=0, + # y=0, + # w=spec.long_edge, + # d=spec.short_edge, + # h=( + # spec.f2f_height * spec.num_floors + # + (spec.f2f_height if spec.basement_is_conditioned else 0) + # + (spec.f2f_height / 2 if spec.attic_is_conditioned else 0) + # ), + # wwr=spec.wwr, + # num_stories=1, + # basement=False, + # zoning=spec.geometry_zoning, + # roof_height=None, + # exposed_basement_frac=0, + # scene_context=SceneContext( + # building=cast(Polygon, from_wkt(spec.rotated_rectangle)), + # neighbors=[ + # cast(Polygon, from_wkt(poly)) for poly in spec.neighbor_polys + # ], + # neighbor_heights=[ + # float(h) if h is not None else 0 for h in spec.neighbor_heights + # ], + # orientation=spec.long_edge_angle, + # ), + # ), + # ) + + # old_pd = model.Zone.Operations.SpaceUse.Occupancy.PeopleDensity + # old_epd = model.Zone.Operations.SpaceUse.Equipment.PowerDensity + # old_lpd = model.Zone.Operations.SpaceUse.Lighting.PowerDensity + # model.Zone.Operations.SpaceUse.Occupancy.PeopleDensity = ( + # old_pd * spec.num_floors + # + old_pd + # * ((spec.basement_use_fraction or 0) if spec.basement_is_occupied else 0) + # + old_pd * ((spec.attic_use_fraction or 0) if spec.attic_is_occupied else 0) + # ) + # model.Zone.Operations.SpaceUse.Equipment.PowerDensity = ( + # old_epd * spec.num_floors + # + old_epd + # * ((spec.basement_use_fraction or 0) if spec.basement_is_occupied else 0) + # + old_epd + # * ((spec.attic_use_fraction or 0) if spec.attic_is_occupied else 0) + # ) + # model.Zone.Operations.SpaceUse.Lighting.PowerDensity = ( + # old_lpd * spec.num_floors + # + old_lpd + # * ((spec.basement_use_fraction or 0) if spec.basement_is_occupied else 0) + # + old_lpd + # * ((spec.attic_use_fraction or 0) if spec.attic_is_occupied else 0) + # ) + # old_vdot = model.Zone.Operations.HVAC.Ventilation.FreshAirPerFloorArea + # model.Zone.Operations.HVAC.Ventilation.FreshAirPerFloorArea = ( + # old_vdot * spec.num_floors + # + (old_vdot if spec.basement_is_conditioned else 0) + # + (old_vdot if spec.attic_is_conditioned else 0) + # ) + # # infiltration is area weighted + # old_infil = model.Zone.Envelope.Infiltration.AirChangesPerHour + # old_infil_attic = model.Zone.Envelope.AtticInfiltration.AirChangesPerHour + # old_infil_basement = model.Zone.Envelope.BasementInfiltration.AirChangesPerHour + # base_weight = spec.num_floors + # attic_weight = 1 if spec.has_attic else 0 + # basement_weight = 1 if spec.has_basement else 0 + # total_weight = base_weight + attic_weight + basement_weight + # base_weight = base_weight / total_weight + # attic_weight = attic_weight / total_weight + # basement_weight = basement_weight / total_weight + # model.Zone.Envelope.Infiltration.AirChangesPerHour = ( + # old_infil * base_weight + # + old_infil_attic * attic_weight + # + old_infil_basement * basement_weight + # ) + # else: + # model = Model( + # Weather=spec.epwzip_path, + # Zone=zone_def, + # Basement=BasementAssumptions( + # Conditioned=spec.basement_is_conditioned, + # UseFraction=spec.basement_use_fraction + # if spec.basement_is_occupied + # else None, + # ), + # Attic=AtticAssumptions( + # Conditioned=spec.attic_is_conditioned, + # UseFraction=spec.attic_use_fraction if spec.attic_is_occupied else None, + # ), + # geometry=ShoeboxGeometry( + # x=0, + # y=0, + # w=spec.long_edge, + # d=spec.short_edge, + # h=spec.f2f_height, + # wwr=spec.wwr, + # num_stories=spec.num_floors, + # basement=spec.has_basement, + # perim_depth=4.57, + # zoning=spec.geometry_zoning, + # roof_height=spec.attic_height or None, + # exposed_basement_frac=spec.exposed_basement_frac, + # scene_context=SceneContext( + # building=cast(Polygon, from_wkt(spec.rotated_rectangle)), + # neighbors=[ + # cast(Polygon, from_wkt(poly)) for poly in spec.neighbor_polys + # ], + # neighbor_heights=[ + # float(h) if h is not None else 0 for h in spec.neighbor_heights + # ], + # orientation=spec.long_edge_angle, + # ), + # ), + # ) + + log("Building and running model...") + overheating_config = ( + spec.parent_experiment_spec.overheating_config + if spec.parent_experiment_spec + else None + ) + run_result = model.run( + eplus_parent_dir=tempdir, + overheating_config=overheating_config, + ) + # Validate conditioned area + if ( + not np.allclose( + model.total_conditioned_area, spec.energy_model_conditioned_area + ) + and not single_zone_mode + ): + msg = ( + f"Total conditioned area mismatch: " + f"{model.total_conditioned_area} != {spec.energy_model_conditioned_area}" + ) + raise ValueError(msg) + if ( + not np.allclose(model.total_conditioned_area, spec.energy_model_footprint_area) + and single_zone_mode + ): + msg = ( + f"Total conditioned area mismatch: " + f"{model.total_conditioned_area} != {spec.energy_model_footprint_area}" + ) + raise ValueError(msg) + + # Results Post-processing + # TODO: consider if we actually want all t he columns we are including. + feature_index = spec.make_multiindex(n_rows=1) + results = run_result.energy_and_peak.to_frame().T.set_index(feature_index) + + energy = results["Energy"] + energy_annual = ( + energy.T.groupby( + level=[level for level in energy.columns.names if level != "Month"] + ) + .sum() + .T + ) + peak = results["Peak"] + peak_annual = ( + peak.T.groupby( + level=[level for level in peak.columns.names if level != "Month"] + ) + .max() + .T + ) + EnergyAndPeakAnnual = cast( + pd.DataFrame, + pd.concat( + [energy_annual, peak_annual], + axis=1, + keys=["Energy", "Peak"], + names=results.columns.names[:-1], + ), + ) + if single_zone_mode: + results = ( + results + * spec.energy_model_footprint_area + / spec.energy_model_conditioned_area + ) + EnergyAndPeakAnnual = ( + EnergyAndPeakAnnual + * spec.energy_model_footprint_area + / spec.energy_model_conditioned_area + ) + + dfs: dict[str, pd.DataFrame] = { + "EnergyAndPeak": results, + "EnergyAndPeakAnnual": EnergyAndPeakAnnual, + } + if run_result.overheating_results is not None: + if single_zone_mode: + raise NotImplementedError( + "Overheating results not supported for single zone mode" + ) + # TODO: add feature dict to overheating df indices? Or instead of a full feature df, just add a single column with the building id? + edh = run_result.overheating_results.edh + old_ix = edh.index + feature_index = spec.make_multiindex( + n_rows=len(edh), include_sort_subindex=False + ) + edh.index = feature_index + edh = edh.set_index(old_ix, append=True) + dfs["ExceedanceDegreeHours"] = edh + + basic_oh = run_result.overheating_results.basic_oh + old_ix = basic_oh.index + feature_index = spec.make_multiindex( + n_rows=len(basic_oh), include_sort_subindex=False + ) + basic_oh.index = feature_index + basic_oh = basic_oh.set_index(old_ix, append=True) + dfs["BasicOverheating"] = basic_oh + + heat_index_categories = run_result.overheating_results.hi + old_ix = heat_index_categories.index + feature_index = spec.make_multiindex( + n_rows=len(heat_index_categories), include_sort_subindex=False + ) + heat_index_categories.index = feature_index + heat_index_categories = heat_index_categories.set_index(old_ix, append=True) + dfs["HeatIndexCategories"] = heat_index_categories + + consecutive_e_zone = run_result.overheating_results.consecutive_e_zone + # may be zero if no streaks found in any zones + if len(consecutive_e_zone) > 0: + old_ix = consecutive_e_zone.index + feature_index = spec.make_multiindex( + n_rows=len(consecutive_e_zone), include_sort_subindex=False + ) + consecutive_e_zone.index = feature_index + consecutive_e_zone = consecutive_e_zone.set_index(old_ix, append=True) + dfs["ConsecutiveExceedances"] = consecutive_e_zone + + hourly_data_outpath: FileReference | None = None + + if spec.parent_experiment_spec and spec.parent_experiment_spec.hourly_data_config: + hourly_df = run_result.sql.timeseries_by_name( + spec.parent_experiment_spec.hourly_data_config.data, + reporting_frequency="Hourly", + ) + hourly_df.index.names = ["Timestep"] + hourly_df.columns.names = ["Trash", "Group", "Meter"] + hourly_df: pd.DataFrame = cast( + pd.DataFrame, + hourly_df.droplevel("Trash", axis=1) + .stack(level="Group", future_stack=True) + .unstack(level="Timestep"), + ) + hourly_multiindex = spec.make_multiindex( + n_rows=len(hourly_df), include_sort_subindex=False + ) + old_ix = hourly_df.index + hourly_df.index = hourly_multiindex + hourly_df = hourly_df.set_index(old_ix, append=True) + + if spec.parent_experiment_spec.hourly_data_config.does_dataframe_output: + for meter_name in hourly_df.columns.get_level_values("Meter").unique(): + variable_df = hourly_df.xs(meter_name, level="Meter", axis=1) + dataframe_key = f"HourlyData.{meter_name.replace(' ', '')}" + dfs[dataframe_key] = variable_df + if spec.parent_experiment_spec.hourly_data_config.does_file_output: + hourly_data_outpath = tempdir / "outputs_hourly_data.pq" + hourly_df.to_parquet(hourly_data_outpath) + + return GloBIOutputSpec( + dataframes=dfs, + hourly_data=hourly_data_outpath, + ) + + +@ExperimentRegistry.Register( + retries=2, + schedule_timeout=timedelta(hours=168), + execution_timeout=timedelta(minutes=30), +) +def simulate_globi_building( + input_spec: GloBIBuildingSpec, tempdir: Path +) -> GloBIOutputSpec: + """Simulate a GlobiSpec building and return monthly energy and peak results. + + NB: this is separated from the pipeline above so the pipeline can still be used as a + local invocation without *too* much difficulty. + """ + return simulate_globi_building_pipeline(input_spec, tempdir) + + +if __name__ == "__main__": + import tempfile + + from globi.models.tasks import MinimalBuildingSpec + + with tempfile.TemporaryDirectory() as tempdir: + with open("inputs/building.yml") as f: + input_spec = MinimalBuildingSpec.model_validate(yaml.safe_load(f)) + o = simulate_globi_building_pipeline( + input_spec=input_spec.globi_spec, + tempdir=Path(tempdir), + ) diff --git a/src/globi/pipelines/training.py b/src/globi/pipelines/training.py new file mode 100644 index 0000000..b9c158b --- /dev/null +++ b/src/globi/pipelines/training.py @@ -0,0 +1,473 @@ +"""The training pipeline.""" + +import asyncio +import logging +import tempfile +from datetime import timedelta +from pathlib import Path +from typing import cast + +import boto3 +import pandas as pd +import yaml +from hatchet_sdk import Context +from scythe.base import ExperimentInputSpec +from scythe.experiments import ( + BaseExperiment, +) +from scythe.hatchet import hatchet +from scythe.registry import ExperimentRegistry +from scythe.scatter_gather import RecursionMap, ScatterGatherResult, scatter_gather +from scythe.settings import ScytheStorageSettings +from scythe.utils.filesys import S3Url +from scythe.worker import ScytheWorkerLabel + +from globi.branching import calculate_branching_factor +from globi.models.surrogate.metrics import fold_test_averages +from globi.models.surrogate.outputs import ( + CombineResultsResult, + ExperimentRunWithRef, + FinalizeResult, + RecursionTransition, + StartTrainingResult, + TrainingEvaluationResult, +) +from globi.models.surrogate.pipeline import ProgressiveTrainingSpec +from globi.models.surrogate.sampling import SampleSpec +from globi.models.surrogate.training import ( + FoldResult, + TrainFoldSpec, + TrainWithCVSpec, +) + +logger = logging.getLogger(__name__) + + +def _require_training_metrics(results: ScatterGatherResult) -> None: + """Raise a descriptive error when the training gather produced no metrics. + + If every fold fails, the gather only returns its error frame, so the + `strata` / `global` keys are missing. + """ + missing = [k for k in ("strata", "global") if k not in results.uris] + if not missing: + return + msg = f"Training produced no {'/'.join(missing)} metrics; all folds likely failed." + errors_uri = results.uris.get("errors") + if errors_uri is not None: + try: + errors = pd.read_parquet(str(errors_uri)) + except Exception as e: + msg += f" Could not read error frame at {errors_uri}: {e}" + else: + sample = errors["msg"].head(3).tolist() if "msg" in errors else [] + msg += f" {len(errors)} fold error(s) at {errors_uri}; first: {sample}" + raise RuntimeError(msg) + + +@ExperimentRegistry.Register( + description="Train a regressor with cross-fold validation.", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(hours=1), + desired_worker_labels=ScytheWorkerLabel.HAS_GPU.worker_label, +) +def train_regressor_with_cv_fold( + input_spec: TrainFoldSpec, tempdir: Path +) -> FoldResult: + """Train a regressor with cross-fold validation.""" + trained = input_spec.train(tempdir) + return FoldResult( + ml_backend=input_spec.parent.ml_backend, + regressor=trained.artifacts.artifacts.regressor_path, + transforms=trained.artifacts.artifacts.transforms_path, + dataframes={ + "global": trained.global_metrics, + "strata": trained.stratum_metrics, + }, + ) + + +iterative_training = hatchet.workflow( + name="iterative_training", + description="Sample a collection of buliding simulations to then simulate and train a surrogate model.", + input_validator=ProgressiveTrainingSpec, +) + + +@iterative_training.task( + name="iterative_training.create_simulations", + schedule_timeout=timedelta(minutes=30), + execution_timeout=timedelta(minutes=30), +) +async def create_simulations( + spec: ProgressiveTrainingSpec, context: Context +) -> ExperimentRunWithRef: + """Create the simulations.""" + logger.info("Refreshing timeout with an additional 120 minutes...") + context.refresh_timeout("120m") + logger.info("Timeout refreshed.") + return await asyncio.to_thread(_create_simulations, spec, context) + + +def _create_simulations( + spec: ProgressiveTrainingSpec, context: Context +) -> ExperimentRunWithRef: + """Create the simulations.""" + # STEP 1: Generate the training samples, allocate simulations + logger.info("Generating training samples...") + sample_spec = SampleSpec(parent=spec, priors=spec.samplers) + sample_df = sample_spec.populate_sample_df() + logger.info("Training samples generated.") + + # TODO: we shouldn't have to cast here, but the typing on `runnable` is not working as expected. + input_validator = cast( + type[ExperimentInputSpec], spec.runnable.input_validator_type + ) + logger.info("Converting training samples to specs...") + specs = sample_spec.convert_to_specs(sample_df, input_validator) + logger.info("Training samples converted to specs.") + + # STEP 2: Simulate the simulations using scythe + run_name = spec.subrun_name("sample") + + exp = BaseExperiment( + runnable=spec.runnable, + run_name=run_name, + storage_settings=spec.storage_settings or ScytheStorageSettings(), + ) + + factor, *_ = calculate_branching_factor(specs) + recursion_map = spec.iteration.recursion.model_copy( + deep=True, update={"factor": factor, "max_depth": 1} + ) + if recursion_map.factor == 1: + recursion_map = recursion_map.model_copy(deep=True, update={"max_depth": 0}) + + logger.info("Allocating simulations...") + run, ref = exp.allocate( + specs, + version="bumpmajor", + recursion_map=recursion_map, + ) + logger.info("Simulations allocated.") + + run_name = run.versioned_experiment.base_experiment.run_name + if not run_name: + msg = "Run name is required." + raise ValueError(msg) + + return ExperimentRunWithRef( + run=run, + workflow_run_id=ref.workflow_run_id, + ) + + +@iterative_training.task( + name="iterative_training.await_simulations", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(hours=5), + parents=[create_simulations], +) +async def await_simulations( + spec: ProgressiveTrainingSpec, context: Context +) -> ScatterGatherResult: + """Await the simulations.""" + parent_output = context.task_output(create_simulations) + workflow_run_id = parent_output.workflow_run_id + logger.info("Awaiting simulations...") + results = await scatter_gather.aio_get_result(workflow_run_id) + logger.info("Simulations completed.") + + return results + + +@iterative_training.task( + name="iterative_training.combine_results", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(hours=1), + parents=[await_simulations], +) +def combine_results( + spec: ProgressiveTrainingSpec, context: Context +) -> CombineResultsResult: + """Combine the results of the simulations. + + Specifically, this step is responsible for combining the results of the simulations + of the previous iteration(s) with the results of the current iteration. In other words, + this is where we grow our simulation cache. + """ + # TODO: major consider how we handle beyond-memory scale scenarios. + # i.e. we probably need to refactor to allow lists of files that only the + # main worker is responsible for combining. + results = context.task_output(await_simulations) + combined_results: dict[str, S3Url] = {} + + # TODO: in the old version, w removed constant columns from the data, i.e.: + # is_constant = (df.max(axis=0) - df.min(axis=0)).abs() < 1e-5 + # df = df.loc[:, ~is_constant] + # Should this sort of data cleaning be done here, or should it be done in the training task? + # also, should we make sure to remove NaN? + + if spec.data_uris: + logger.info("Combining results from previous iterations...") + shared_keys = set(spec.data_uris.uris.keys()) & set(results.uris.keys()) + old_keys_only = set(spec.data_uris.uris.keys()) - shared_keys + new_keys_only = set(results.uris.keys()) - shared_keys + # TODO: consider copying these over to the `combined` folder anyways. + for key in old_keys_only: + combined_results[key] = spec.data_uris.uris[key] + for key in new_keys_only: + combined_results[key] = results.uris[key] + # TODO: refactor to use a threadpool executor? + # For memory reasons, it might be a good idea to stay single threaded here. + for key in shared_keys: + logger.info(f"Combining results for key {key}...") + old_df = pd.read_parquet(str(spec.data_uris.uris[key])) + new_df = pd.read_parquet(str(results.uris[key])) + combined_df = pd.concat([old_df, new_df], axis=0) + uri = spec.format_combined_output_uri(key) + combined_df.to_parquet(str(uri)) + logger.info(f"Results for key {key} combined and saved to s3.") + combined_results[key] = uri + + else: + # TODO: consider copying these over to the `combined` folder anyways. + logger.info( + "No previous iterations to combine results from, so using results from current iteration." + ) + combined_results = results.uris + + return CombineResultsResult( + incoming=results, + previous=spec.data_uris, + combined=ScatterGatherResult(uris=combined_results), + ) + + +@iterative_training.task( + name="iterative_training.start_training", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(hours=1), + parents=[combine_results], +) +def start_training( + spec: ProgressiveTrainingSpec, context: Context +) -> StartTrainingResult: + """Start the training.""" + results = context.task_output(combine_results) + + train_spec = TrainWithCVSpec( + parent=spec, + data_uris=results.combined, + ) + + # Alternatively, one task per fold-column combination? + specs = train_spec.schedule + + logger.info("Scheduling training...") + run_name = spec.subrun_name("train") + exp = BaseExperiment( + runnable=train_regressor_with_cv_fold, + run_name=run_name, + storage_settings=spec.storage_settings or ScytheStorageSettings(), + ) + run, ref = exp.allocate( + specs, + version="bumpmajor", # There is normally only ever one training round per parent minor version, except during replays etc + recursion_map=RecursionMap( + factor=2, + max_depth=0, + ), + ) + logger.info("Training scheduled.") + + if not run.versioned_experiment.base_experiment.run_name: + msg = "Run name is required." + raise ValueError(msg) + + return StartTrainingResult( + training_spec=train_spec, + experiment_run_with_ref=ExperimentRunWithRef( + run=run, + workflow_run_id=ref.workflow_run_id, + ), + ) + + +@iterative_training.task( + name="iterative_training.await_training", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(hours=5), + parents=[start_training], +) +async def await_training( + spec: ProgressiveTrainingSpec, context: Context +) -> ScatterGatherResult: + """Await the training.""" + parent_output = context.task_output(start_training) + workflow_run_id = parent_output.experiment_run_with_ref.workflow_run_id + logger.info("Awaiting training...") + results = await scatter_gather.aio_get_result(workflow_run_id) + logger.info("Training completed.") + + return results + + +@iterative_training.task( + name="iterative_training.evaluate_training", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(minutes=5), + parents=[await_training], +) +def evaluate_training( + spec: ProgressiveTrainingSpec, context: Context +) -> TrainingEvaluationResult: + """Evaluate the training.""" + results_output = context.task_output(await_training) + _require_training_metrics(results_output) + strata_uri = results_output.uris["strata"] + globals_uri = results_output.uris["global"] + logger.info("Reading strata results from s3...") + results = pd.read_parquet(str(strata_uri)) + logger.info("Strata results read from s3.") + logger.info("Reading global results from s3...") + results_globals = pd.read_parquet(str(globals_uri)) + logger.info("Global results read from s3.") + + fold_averages = fold_test_averages(results) + global_averages = fold_test_averages(results_globals) + + logger.info("Running convergence criteria...") + ( + convergence_all, + _convergence_monitor_segment, + _convergence_monitor_segment_and_target, + _convergence, + ) = spec.convergence_criteria.run(fold_averages) + logger.info("Convergence criteria run.") + + return TrainingEvaluationResult( + converged=convergence_all, + # TODO: maybe we should change/improve what gets logged here? + metrics={ + "global_averages": global_averages.reset_index().to_dict(orient="records"), + }, + ) + + +@iterative_training.task( + name="iterative_training.transition_recursion", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(minutes=5), + parents=[evaluate_training, combine_results, await_training], +) +def transition_recursion( + spec: ProgressiveTrainingSpec, context: Context +) -> RecursionTransition: + """Transition the recursion.""" + results = context.task_output(evaluate_training) + if results.converged: + logger.info("Converged! Time to wrap up... no more recursion.") + return RecursionTransition(reasoning="converged", child_workflow_run_id=None) + if spec.iteration.at_max_iters: + logger.info( + "Not converged, but we're at the max number of iterations. Time to wrap up... no more recursion." + ) + return RecursionTransition(reasoning="max_depth", child_workflow_run_id=None) + + await_training_output = context.task_output(await_training) + # start_training_output = context.task_output(start_training) + combine_results_output = context.task_output(combine_results) + + logger.info( + "Not converged, but we have more iterations to try. Time to continue recursion..." + ) + next_spec = spec.model_copy(deep=True) + next_spec.iteration.current_iter += 1 + next_spec.data_uris = combine_results_output.combined + next_spec.metrics_uris.append(await_training_output) + next_spec.previous_experiment_ids.append(spec.experiment_id) + exp = BaseExperiment( + runnable=iterative_training, + run_name=f"{next_spec.base_run_name}", + storage_settings=spec.storage_settings or ScytheStorageSettings(), + ) + # manually bump minor here to avoid race conditions between e.g. simultaneously running v29.2.0 and v30.1.0... pretty sure the error only happens when they finish in the exact same second, but... it happened once so. + _run, ref = exp.allocate( + next_spec, + version=spec.current_version.next_minor_version(), + recursion_map=None, + ) + logger.info("Recursion transitioned.") + return RecursionTransition( + reasoning=None, child_workflow_run_id=ref.workflow_run_id + ) + + +@iterative_training.task( + name="iterative_training.finalize", + schedule_timeout=timedelta(hours=5), + execution_timeout=timedelta(minutes=30), + parents=[transition_recursion, await_training, combine_results], +) +def finalize(spec: ProgressiveTrainingSpec, context: Context) -> FinalizeResult: + """Run when training has exited the loop (converged, max depth, or other reason). Saves final models and artifacts.""" + # TODO: save the final model? + transition = context.task_output(transition_recursion) + logger.info(f"Training finished. Finalizing: {transition.reasoning}") + + logger.info("Fetching metrics from all iterations...") + await_training_output = context.task_output(await_training) + metrics_uris = [*spec.metrics_uris, await_training_output] + metrics_by_key: dict[str, list[pd.DataFrame]] = {} + for i, metrics_uri in enumerate(metrics_uris): + logger.info(f"\tFetching metrics from iteration {i}...") + for key in metrics_uri.uris: + logger.info(f"\t\tFetching metrics for key {key} from iteration {i}...") + if key not in metrics_by_key: + metrics_by_key[key] = [] + metrics_by_key[key].append(pd.read_parquet(str(metrics_uri.uris[key]))) + logger.info("Combining metrics from all iterations...") + combined_metrics = { + key: pd.concat(metrics, axis=0) for key, metrics in metrics_by_key.items() + } + combined_metrics_uris = { + key: spec.format_metrics_output_uri(key) for key in combined_metrics + } + logger.info("Saving combined metrics to s3...") + for key, metrics in combined_metrics.items(): + logger.info(f"\tSaving metrics for key {key} to s3...") + metrics.to_parquet(str(combined_metrics_uris[key])) + logger.info("Final metrics saved to s3.") + + # Get the simulation data outputs from all steps and this step + combine_results_output = context.task_output(combine_results) + + # Get the experiment ids from all steps and this step + experiment_ids = [*spec.previous_experiment_ids, spec.experiment_id] + + # TODO: save final models, or return them a little more directly? + # Also, need to return Transformers object somehow. + + result = FinalizeResult( + reasoning=transition.reasoning, + data_uris=combine_results_output.combined.uris, + metrics_uris=combined_metrics_uris, + experiment_ids=experiment_ids, + ) + + s3_client = boto3.client("s3") + summary_manifest_uri = spec.format_summary_manifest_key() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "summary.yml" + with open(temp_path, "w") as f: + yaml.dump(result.model_dump(mode="json"), f, indent=2, sort_keys=False) + if spec.storage_settings is None: + msg = ( + "Storage settings are not set, so we can't upload the summary manifest." + ) + raise ValueError(msg) + s3_client.upload_file( + temp_path.as_posix(), spec.storage_settings.BUCKET, summary_manifest_uri + ) + return result diff --git a/src/globi/tools/cli/main.py b/src/globi/tools/cli/main.py index e36e447..096da8b 100644 --- a/src/globi/tools/cli/main.py +++ b/src/globi/tools/cli/main.py @@ -102,6 +102,33 @@ def manifest( ) +@submit.command() +@click.option( + "--path", + type=click.Path(exists=True), + help="The path to the manifest file which will be used to configure the experiment.", + prompt="Manifest file path (.yml)", +) +def surrogate(path): + """Submit a GloBI surrogate experiment.""" + from scythe.experiments import BaseExperiment + + from globi.models.surrogate.pipeline import ProgressiveTrainingSpec + from globi.pipelines.training import iterative_training + + with open(path) as f: + manifest = yaml.safe_load(f) + + config = ProgressiveTrainingSpec.model_validate(manifest) + + exp = BaseExperiment(runnable=iterative_training, run_name=config.base_run_name) + run, _ref = exp.allocate( + config, + version="bumpmajor", + ) + print(yaml.dump(run.model_dump(mode="json"), indent=2, sort_keys=False)) + + @cli.command() @click.option( "--config", @@ -127,7 +154,7 @@ def simulate( import pandas as pd from globi.models.tasks import MinimalBuildingSpec - from globi.pipelines import simulate_globi_building_pipeline + from globi.pipelines.simulations import simulate_globi_building_pipeline if isinstance(config, str): config = Path(config) @@ -315,6 +342,181 @@ def e2e( sys.exit(1) +_backend_option = click.option( + "--backend", + "backends", + multiple=True, + type=click.Choice(["xgb", "lgb", "nn"]), + help="Restrict to these backends (default: every backend in the config).", +) +_workers_option = click.option( + "--workers", + type=int, + default=None, + help="Override the number of simulation worker processes.", +) + + +@tests.command("surrogate-local") +@click.option( + "--config", + type=click.Path(exists=True, dir_okay=False), + default="tests/data/surrogate_local/validation-dummy.yml", + show_default=True, + help="LocalValidationConfig yaml.", +) +@_backend_option +@_workers_option +@click.option("--max-iters", type=int, default=None, help="Override max_iters.") +@click.option("--n-per-iter", type=int, default=None, help="Override n_per_iter.") +def surrogate_local( + config: str, + backends: tuple[str, ...], + workers: int | None, + max_iters: int | None, + n_per_iter: int | None, +): + """Train surrogate(s) locally: sample -> simulate -> train folds -> evaluate. + + Runs entirely in-process (no Hatchet, no S3). Outputs go to `/training//`. + """ + import logging + + from globi.validation.harness import load_config, run_training + + logging.basicConfig(level=logging.INFO) + conf = load_config(Path(config)) + overrides = { + k: v + for k, v in { + "n_workers": workers, + "max_iters": max_iters, + "n_per_iter": n_per_iter, + }.items() + if v is not None + } + conf = conf.model_copy(update=overrides) + + results = run_training(conf, list(backends) or None) + for name, result in results.items(): + print("=" * 80) + print( + f"{name}: stopped after {len(result.iterations)} iteration(s) " + f"({result.reasoning}); summary at {result.summary_path}" + ) + for i, it in enumerate(result.iterations): + print(f"--- iteration {i} test metrics (fold-averaged) ---") + print(it.test_metrics_table.to_string(float_format=lambda x: f"{x:.4g}")) + + +@tests.command("surrogate-validate") +@click.option( + "--config", + type=click.Path(exists=True, dir_okay=False), + default="tests/data/surrogate_local/validation-dummy.yml", + show_default=True, + help="LocalValidationConfig yaml (same one used for `surrogate-local`).", +) +@_backend_option +@_workers_option +def surrogate_validate( + config: str, + backends: tuple[str, ...], + workers: int | None, +): + """Validate trained surrogate(s) against deterministic EnergyPlus simulations. + + Simulates the buildings held out by `surrogate-local` exactly as the GIS data + defines them (no priors), predicts them with each trained surrogate and writes + per-building and stock-level comparisons to `/validation//` + (see `report.md`). The hold-out is pinned in `/holdout.json`; to change + `n_test` or the seed, train into a new workdir. + """ + import logging + + from globi.validation.harness import ( + load_config, + run_validations, + summarize_validations, + ) + + logging.basicConfig(level=logging.INFO) + conf = load_config(Path(config)) + if workers is not None: + conf = conf.model_copy(update={"n_workers": workers}) + + results = run_validations(conf, list(backends) or None) + print("=" * 80) + print(summarize_validations(results).to_string(float_format=lambda x: f"{x:.4g}")) + for name, result in results.items(): + print(f"{name}: report at {result.report_path}") + + +@tests.command("surrogate-stack-manifest") +@click.option( + "--config", + type=click.Path(exists=True, dir_okay=False), + default="tests/data/surrogate_local/validation-e2e.yml", + show_default=True, + help="LocalValidationConfig yaml (energyplus mode).", +) +@click.option( + "--out", + type=click.Path(dir_okay=False), + default="outputs/surrogate-stack/training.yml", + show_default=True, + help="Where to write the ProgressiveTrainingSpec manifest.", +) +@click.option( + "--backend", + "backend_name", + type=click.Choice(["xgb", "lgb", "nn"]), + default=None, + help="Which configured backend to use (default: the first).", +) +def surrogate_stack_manifest(config: str, out: str, backend_name: str | None): + """Write a small-count training manifest for the docker + hatchet stack. + + Uploads the context/db/semantic-fields/component-map to the configured scythe + bucket (e.g. localstack via `make engine`) and writes a `ProgressiveTrainingSpec` + yaml that can be submitted with `globi submit surrogate --path `. + """ + import logging + + import pandas as pd + from scythe.settings import ScytheStorageSettings + + from globi.models.configs import GloBIExperimentSpec + from globi.validation.context import upload_context_artifacts + from globi.validation.harness import build_spec, load_config + + logging.basicConfig(level=logging.INFO) + conf = load_config(Path(config)) + if conf.mode != "energyplus" or conf.manifest is None: + msg = ( + "surrogate-stack-manifest needs an energyplus-mode config with a manifest." + ) + raise click.UsageError(msg) + backend = conf.selected_backends([backend_name] if backend_name else None)[0] + manifest = GloBIExperimentSpec.from_manifest(conf.manifest) + storage = ScytheStorageSettings() + + # builds (or reuses) the hold-out set and the local training context + spec = build_spec(conf, backend) + out_path = Path(out) + out_path.parent.mkdir(parents=True, exist_ok=True) + _, context_uri = upload_context_artifacts( + pd.read_parquet(conf.context_path), + manifest.file_config, + storage, + context_path=out_path.parent / "context.parquet", + ) + spec = spec.model_copy(update={"context": context_uri, "storage_settings": storage}) + with open(out_path, "w") as f: + yaml.dump(spec.model_dump(mode="json"), f, indent=2, sort_keys=False) + print(f"Wrote {out_path}. Submit with:\n globi submit surrogate --path {out_path}") + + @cli.group() def get(): """Get a GloBI experiment from different sources.""" @@ -371,7 +573,7 @@ def experiment( s3_client: S3Client = boto3.client("s3") s3_settings = ScytheStorageSettings() - exp = BaseExperiment(experiment=simulate_globi_building, run_name=run_name) + exp = BaseExperiment(runnable=simulate_globi_building, run_name=run_name) if not version: exp_version = exp.latest_version(s3_client, from_cache=False) diff --git a/src/globi/tools/visualization/data_sources.py b/src/globi/tools/visualization/data_sources.py index 8397258..4c8bace 100644 --- a/src/globi/tools/visualization/data_sources.py +++ b/src/globi/tools/visualization/data_sources.py @@ -262,7 +262,7 @@ def load_run_data(self, run_id: str) -> pd.DataFrame: s3_client = self.client s3_settings = ScytheStorageSettings() exp = BaseExperiment( - experiment=simulate_globi_building, + runnable=simulate_globi_building, run_name=run_id, ) diff --git a/src/globi/validation/__init__.py b/src/globi/validation/__init__.py new file mode 100644 index 0000000..bc0a532 --- /dev/null +++ b/src/globi/validation/__init__.py @@ -0,0 +1,17 @@ +"""Local (in-process) surrogate training harness and EnergyPlus validation. + +This package mirrors the stages of the hatchet ``iterative_training`` workflow +(sample -> simulate -> combine -> train per fold -> evaluate -> recurse) as plain +functions that run on a single machine with no Hatchet server or S3 bucket, and adds +tooling for comparing a trained surrogate against deterministic EnergyPlus runs. + +Importing this package populates offline-safe Hatchet client env vars (only where +unset), because the surrogate models transitively construct a Hatchet client at +import time. +""" + +from globi.validation.env import ensure_local_hatchet_env + +ensure_local_hatchet_env(strict=False) + +__all__ = ["ensure_local_hatchet_env"] diff --git a/src/globi/validation/config.py b/src/globi/validation/config.py new file mode 100644 index 0000000..5bfe52e --- /dev/null +++ b/src/globi/validation/config.py @@ -0,0 +1,123 @@ +"""Configuration for a local surrogate training + validation run.""" + +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from globi.models.base import BaseConfig +from globi.models.surrogate.backends import MLBackend +from globi.models.surrogate.pipeline import ConvergenceThresholds +from globi.validation.context import ( + DEFAULT_EXCLUDED_FEATURE_COLUMNS, + DEFAULT_TARGET_GLOBS, +) + + +class LocalValidationConfig(BaseConfig): + """Everything needed to train small surrogates locally and validate them. + + Two modes: + + - ``energyplus``: sample buildings from the GIS data in ``manifest``, fill in + semantic fields / geometry knobs from priors, simulate with EnergyPlus, train; + validate against a held-out set of buildings simulated exactly as the GIS data + defines them. + - ``dummy``: the same loop on the analytic ``dummy_simulation`` runnable + (``dummy_training_spec``), which runs in seconds and needs no EnergyPlus. + """ + + mode: Literal["energyplus", "dummy"] = Field( + default="energyplus", description="Which simulator drives the loop." + ) + workdir: Path = Field( + default=Path("outputs/surrogate-local"), + description="Root output directory (training/, validation/, sim_cache/ ...).", + ) + seed: int = Field( + default=0, description="Seed for sampling and hold-out selection." + ) + n_workers: int = Field( + default=1, ge=1, description="Worker processes for simulation (1 = in-process)." + ) + backends: list[MLBackend] = Field( + ..., + description="ML backends to train, one surrogate per entry (discriminated on `ml_backend`).", + ) + + # ---- energyplus mode ------------------------------------------------- + manifest: Path | None = Field( + default=None, + description="GloBIExperimentSpec yaml describing the building stock (energyplus mode).", + ) + n_per_iter: int | list[int] = Field( + default=32, description="Training samples per iteration." + ) + min_per_stratum: int = Field(default=8, description="Min samples per stratum.") + max_iters: int = Field(default=1, ge=1, description="Max outer iterations.") + n_folds: int = Field(default=4, ge=2, description="Cross-validation folds.") + targets_globs: list[str] = Field( + default_factory=lambda: list(DEFAULT_TARGET_GLOBS), + description="Globs selecting regression targets from the flattened result columns.", + ) + exclude_columns: list[str] = Field( + default_factory=lambda: sorted(DEFAULT_EXCLUDED_FEATURE_COLUMNS), + description="Index columns that must not become features.", + ) + thresholds: dict[str, ConvergenceThresholds] = Field( + default_factory=lambda: {"*": ConvergenceThresholds(r2=0.95)}, + description="Convergence thresholds by target glob.", + ) + fixed_basement_attic: bool = Field( + default=True, + description="Pin basement/attic priors to 'none' (deterministic spec validators).", + ) + n_test: int = Field( + default=8, ge=1, description="Held-out deterministic buildings to validate on." + ) + + # ---- dummy mode ------------------------------------------------------ + dummy_training_spec: Path = Field( + default=Path("tests/data/dummy_training/training.yml"), + description="ProgressiveTrainingSpec yaml for the dummy runnable (dummy mode).", + ) + + @property + def training_dir(self) -> Path: + """Where per-backend training runs live.""" + return self.workdir / "training" + + @property + def validation_dir(self) -> Path: + """Where per-backend validation outputs live.""" + return self.workdir / "validation" + + @property + def sim_cache_dir(self) -> Path: + """Simulation cache shared by training and validation.""" + return self.workdir / "sim_cache" + + @property + def holdout_path(self) -> Path: + """The held-out building ids plus the fingerprint of the settings that chose them.""" + return self.workdir / "holdout.json" + + @property + def context_path(self) -> Path: + """The training context parquet.""" + return self.workdir / "context.parquet" + + def backend_dir(self, backend: MLBackend) -> Path: + """Training output dir for a backend.""" + return self.training_dir / backend.ml_backend + + def selected_backends(self, names: list[str] | None) -> list[MLBackend]: + """Filter configured backends by name (all if `names` is empty/None).""" + if not names: + return list(self.backends) + chosen = [b for b in self.backends if b.ml_backend in names] + missing = set(names) - {b.ml_backend for b in chosen} + if missing: + msg = f"Backends {sorted(missing)} are not configured; available: {[b.ml_backend for b in self.backends]}" + raise ValueError(msg) + return chosen diff --git a/src/globi/validation/context.py b/src/globi/validation/context.py new file mode 100644 index 0000000..1620475 --- /dev/null +++ b/src/globi/validation/context.py @@ -0,0 +1,321 @@ +"""Build the GIS context, priors and training spec for a local surrogate run. + +The training *context* is the deterministic geometry of a building stock (footprints, +neighbours, floors, weather file) with the prior-sampled attributes removed. The +sampler (`globi.models.surrogate.samplers.Priors`) then fills in semantic fields, +window-to-wall ratio, floor-to-floor height, etc. for each training sample. +""" + +import logging +from collections.abc import Iterable +from pathlib import Path +from typing import cast + +import geopandas as gpd +import pandas as pd +import yaml +from epinterface.sbem.fields.spec import ( + CategoricalFieldSpec, + NumericFieldSpec, + SemanticModelFields, +) +from scythe.scatter_gather import RecursionMap +from scythe.settings import ScytheStorageSettings +from scythe.utils.filesys import S3Url + +from globi.models.configs import ( + DeterministicGISPreprocessorConfig, + FileConfig, + GISPreprocessorColumnMap, + GloBIExperimentSpec, +) +from globi.models.surrogate.backends import MLBackend, XGBBackend +from globi.models.surrogate.pipeline import ( + ConvergenceThresholds, + ConvergenceThresholdsByTarget, + CrossValidationSpec, + FeatureConfigSpec, + IterationSpec, + ProgressiveTrainingSpec, + RegressionIOConfigSpec, + StratificationSpec, + TargetsConfigGlobSpec, +) +from globi.models.surrogate.samplers import ( + CategoricalSampler, + Priors, + ProductValuesSampler, + UnconditionalPrior, + UniformSampler, +) +from globi.pipelines.gis import preprocess_gis_file + +logger = logging.getLogger(__name__) + +# Columns of the preprocessed GIS frame that become GloBIBuildingSpec fields, keyed by +# the spec field name. Values are attribute names on GISPreprocessorColumnMap. +_CONTEXT_COLUMNS: dict[str, str] = { + "building_id": "Building_ID_col", + "db_file": "DB_File_col", + "semantic_fields_file": "Semantic_Fields_File_col", + "component_map_file": "Component_Map_File_col", + "epwzip_file": "EPWZip_File_col", + "neighbor_polys": "Neighbor_Polys_col", + "neighbor_heights": "Neighbor_Heights_col", + "neighbor_floors": "Neighbor_Floors_col", + "rotated_rectangle": "Rotated_Rectangle_col", + "long_edge_angle": "Long_Edge_Angle_col", + "long_edge": "Long_Edge_col", + "short_edge": "Short_Edge_col", + "aspect_ratio": "Aspect_Ratio_col", + "rotated_rectangle_area_ratio": "Rotated_Rectangle_Area_Ratio_col", + "num_floors": "Num_Floors_col", +} + +FILE_COLUMNS = ("db_file", "semantic_fields_file", "component_map_file") + +#: Index columns that must never become surrogate features: unique per building +#: (`building_id`, `rotated_rectangle`) or unused by the simulator (`height`, which the +#: model derives from `f2f_height * num_floors`). +DEFAULT_EXCLUDED_FEATURE_COLUMNS = frozenset({ + "building_id", + "rotated_rectangle", + "height", +}) + +DEFAULT_TARGET_GLOBS = ("EnergyAndPeakAnnual/Energy/Raw/*",) + +BASEMENT_ATTIC_OPTIONS = [ + "none", + "unoccupied_unconditioned", + "unoccupied_conditioned", + "occupied_unconditioned", + "occupied_conditioned", +] + + +def build_training_context( + config: DeterministicGISPreprocessorConfig, + file_config: FileConfig, + *, + scenario: str | None = None, + exclude_building_ids: Iterable[str] | None = None, + file_refs: dict[str, str] | None = None, + preprocessed: tuple[gpd.GeoDataFrame, GISPreprocessorColumnMap] | None = None, +) -> pd.DataFrame: + """Preprocess a GIS file into a parquet-serializable training context. + + Args: + config: GIS preprocessor config. + file_config: File config (GIS, db, semantic fields, component map, epw). + scenario: Optional scenario override passed to the preprocessor. + exclude_building_ids: Buildings to hold out (e.g. the validation set). + file_refs: Optional overrides for the `db_file` / `semantic_fields_file` / + `component_map_file` columns (e.g. s3 uris for a remote run). + preprocessed: An already preprocessed `(gdf, colmap)` pair to use instead of + running `preprocess_gis_file` again. + + Returns: + A frame with one row per building whose columns are `GloBIBuildingSpec` field + names. Prior-sampled fields (semantic fields, wwr, f2f_height, basement, attic, + exposed_basement_frac, height) are intentionally absent. + """ + gdf, colmap = preprocessed or preprocess_gis_file( + config, file_config, scenario=scenario + ) + columns = {field: getattr(colmap, attr) for field, attr in _CONTEXT_COLUMNS.items()} + df = cast(pd.DataFrame, pd.DataFrame(gdf[list(columns.values())])) + df = df.rename(columns={v: k for k, v in columns.items()}) + + if exclude_building_ids is not None: + excluded = set(exclude_building_ids) + before = len(df) + df = cast(pd.DataFrame, df[~df["building_id"].isin(list(excluded))]) + logger.info("Held out %d buildings from training context.", before - len(df)) + + # geometries and file references must be plain strings to round-trip via parquet + df["rotated_rectangle"] = df["rotated_rectangle"].apply(lambda g: g.wkt) + df["neighbor_polys"] = df["neighbor_polys"].apply( + lambda polys: [poly.wkt if poly is not None else None for poly in polys] + ) + for col in (*FILE_COLUMNS, "epwzip_file"): + df[col] = df[col].astype(str) + for col, ref in (file_refs or {}).items(): + df[col] = ref + + return df.reset_index(drop=True) + + +def default_priors_from_semantic_fields( + semantic_fields_file: Path, + *, + fixed_basement_attic: bool = True, + f2f_height: tuple[float, float] = (2.5, 4.0), + wwr: tuple[float, float] = (0.1, 0.5), + exposed_basement_frac: tuple[float, float] = (0.1, 0.4), +) -> Priors: + """Uniform priors over every semantic field plus basic geometry knobs. + + Categorical fields are sampled uniformly over their options; numeric fields + uniformly over their `[Min, Max]` range. + + Args: + semantic_fields_file: The semantic fields yaml. + fixed_basement_attic: Pin basement/attic to "none" (keeps the spec validators + deterministic and shrinks the prior space for small samples). + f2f_height: Uniform range for floor-to-floor height [m]. + wwr: Uniform range for window-to-wall ratio. + exposed_basement_frac: Uniform range for the exposed basement fraction. + """ + with open(semantic_fields_file) as f: + semantic_fields = SemanticModelFields.model_validate(yaml.safe_load(f)) + semantic_priors: dict[str, UnconditionalPrior] = {} + for field in semantic_fields.Fields: + if isinstance(field, CategoricalFieldSpec): + sampler = CategoricalSampler( + values=list(field.Options), + weights=[1 / len(field.Options)] * len(field.Options), + ) + elif isinstance(field, NumericFieldSpec): + sampler = UniformSampler(min=field.Min, max=field.Max) + else: + msg = f"No default prior for semantic field {field.Name!r} of type {type(field).__name__}." + raise NotImplementedError(msg) + semantic_priors[f"semantic_field_{field.Name}"] = UnconditionalPrior( + sampler=sampler + ) + + if fixed_basement_attic: + basement_attic_sampler = CategoricalSampler(values=["none"], weights=[1.0]) + else: + n_other = len(BASEMENT_ATTIC_OPTIONS) - 1 + basement_attic_sampler = CategoricalSampler( + values=BASEMENT_ATTIC_OPTIONS, + weights=[0.5, *[0.5 / n_other] * n_other], + ) + + return Priors( + sampled_features={ + "f2f_height": UnconditionalPrior( + sampler=UniformSampler(min=f2f_height[0], max=f2f_height[1]) + ), + "height": UnconditionalPrior( + sampler=ProductValuesSampler( + features_to_multiply=["f2f_height", "num_floors"] + ) + ), + "wwr": UnconditionalPrior(sampler=UniformSampler(min=wwr[0], max=wwr[1])), + "basement": UnconditionalPrior(sampler=basement_attic_sampler), + "attic": UnconditionalPrior(sampler=basement_attic_sampler), + "exposed_basement_frac": UnconditionalPrior( + sampler=UniformSampler( + min=exposed_basement_frac[0], max=exposed_basement_frac[1] + ) + ), + **semantic_priors, + } + ) + + +def build_training_spec( + manifest: GloBIExperimentSpec, + *, + context_path: Path | S3Url, + ml_backend: MLBackend | None = None, + priors: Priors | None = None, + n_per_iter: int | list[int] = 32, + min_per_stratum: int = 8, + max_iters: int = 1, + n_folds: int = 4, + targets_globs: Iterable[str] = DEFAULT_TARGET_GLOBS, + exclude_columns: Iterable[str] = DEFAULT_EXCLUDED_FEATURE_COLUMNS, + thresholds: dict[str, ConvergenceThresholds] | None = None, + base_run_name: str = "local-surrogate", + storage_settings: ScytheStorageSettings | None = None, +) -> ProgressiveTrainingSpec: + """Assemble a `ProgressiveTrainingSpec` that simulates real GloBI buildings. + + The spec is runnable in-process via `globi.validation.local_runner` when + `storage_settings` is None, or submitted to hatchet via `globi submit surrogate` + when the context and file references live on s3. + """ + # imported lazily: registers the runnable and needs the hatchet env + from globi.pipelines.simulations import simulate_globi_building + + return ProgressiveTrainingSpec( + experiment_id="placeholder", + sort_index=0, + storage_settings=storage_settings, + runnable=simulate_globi_building, + base_run_name=base_run_name, + context=context_path, + convergence_criteria=ConvergenceThresholdsByTarget( + thresholds=thresholds or {"*": ConvergenceThresholds(r2=0.95)} + ), + regression_io_config=RegressionIOConfigSpec( + targets=TargetsConfigGlobSpec( + globs=list(targets_globs), normalization="min-max" + ), + features=FeatureConfigSpec( + exclude_columns=frozenset(exclude_columns), + cat_encoding="index", + ), + ), + ml_backend=ml_backend or XGBBackend(), + stratification=StratificationSpec(field="epwzip_file", sampling="equal"), + samplers=priors + or default_priors_from_semantic_fields( + manifest.file_config.semantic_fields_file + ), + cross_val=CrossValidationSpec(n_folds=n_folds), + iteration=IterationSpec( + n_per_iter=n_per_iter, + min_per_stratum=min_per_stratum, + max_iters=max_iters, + recursion=RecursionMap(factor=100, max_depth=1), + ), + ) + + +def upload_context_artifacts( + context: pd.DataFrame, + file_config: FileConfig, + storage_settings: ScytheStorageSettings, + *, + key_prefix: str = "local-surrogate-artifacts", + context_path: Path, +) -> tuple[pd.DataFrame, S3Url]: + """Upload the db / semantic fields / component map and context parquet to s3. + + Used to hand a locally-built context to the hatchet workflow (e.g. the docker + + hatchet-lite + localstack stack). Returns the context with s3 file references + substituted in, and the s3 uri of the uploaded context parquet. + """ + import boto3 + + s3 = boto3.client("s3") + prefix = f"{storage_settings.BUCKET_PREFIX}/{key_prefix}" + local_files = { + "db_file": file_config.db_file, + "semantic_fields_file": file_config.semantic_fields_file, + "component_map_file": file_config.component_map_file, + } + context = context.copy() + for col, local_path in local_files.items(): + key = f"{prefix}/{Path(local_path).name}" + s3.upload_file( + Filename=Path(local_path).as_posix(), + Bucket=storage_settings.BUCKET, + Key=key, + ) + context[col] = f"s3://{storage_settings.BUCKET}/{key}" + + context_path.parent.mkdir(parents=True, exist_ok=True) + context.to_parquet(context_path) + context_key = f"{prefix}/{context_path.name}" + s3.upload_file( + Filename=context_path.as_posix(), + Bucket=storage_settings.BUCKET, + Key=context_key, + ) + return context, S3Url(f"s3://{storage_settings.BUCKET}/{context_key}") diff --git a/src/globi/validation/env.py b/src/globi/validation/env.py new file mode 100644 index 0000000..7773472 --- /dev/null +++ b/src/globi/validation/env.py @@ -0,0 +1,67 @@ +"""Environment helpers for running globi pipelines without a Hatchet server. + +Importing ``globi.pipelines`` (directly or transitively) constructs a +``hatchet_sdk.Hatchet`` client at import time, which validates ``HATCHET_CLIENT_TOKEN`` +as a JWT and reads the broadcast address from its claims. No network connection is +made until a workflow is actually submitted, so a syntactically valid token is enough +for local, in-process execution. +""" + +import os +from pathlib import Path + +HATCHET_ENV_EXAMPLE_FILE = ".env.local.host.hatchet.example" + +_LOCAL_DEFAULTS = { + "HATCHET_CLIENT_TLS_STRATEGY": "none", + "HATCHET_CLIENT_HOST_PORT": "localhost:7077", +} + + +def find_repo_root(start: Path | None = None) -> Path | None: + """Walk upwards from `start` (default: cwd) looking for the hatchet example env file.""" + start = (start or Path.cwd()).resolve() + for candidate in [start, *start.parents]: + if (candidate / HATCHET_ENV_EXAMPLE_FILE).exists(): + return candidate + # fall back to the package's own location (installed in editable mode) + pkg_root = Path(__file__).resolve().parents[3] + if (pkg_root / HATCHET_ENV_EXAMPLE_FILE).exists(): + return pkg_root + return None + + +def _parse_env_file(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip().strip('"').strip("'") + return values + + +def ensure_local_hatchet_env( + repo_root: Path | None = None, *, strict: bool = True +) -> None: + """Populate the Hatchet client env vars with offline-safe defaults if unset. + + Existing values (e.g. from `make cli-native`'s `--env-file`s) are never overwritten. + The token is read from `.env.local.host.hatchet.example` in the repo root. + + Args: + repo_root: Where to look for the example env file (default: search upwards). + strict: Raise if no token could be found; otherwise leave the env untouched + and let the Hatchet client report the problem on import. + """ + root = repo_root or find_repo_root() + file_values = _parse_env_file(root / HATCHET_ENV_EXAMPLE_FILE) if root else {} + for key, value in {**_LOCAL_DEFAULTS, **file_values}.items(): + os.environ.setdefault(key, value) + if strict and "HATCHET_CLIENT_TOKEN" not in os.environ: + msg = ( + "HATCHET_CLIENT_TOKEN is not set and no " + f"{HATCHET_ENV_EXAMPLE_FILE} could be found to read a default from." + ) + raise RuntimeError(msg) diff --git a/src/globi/validation/harness.py b/src/globi/validation/harness.py new file mode 100644 index 0000000..740e1f3 --- /dev/null +++ b/src/globi/validation/harness.py @@ -0,0 +1,332 @@ +"""Orchestrate a configured local training + validation run. + +This is what the ``globi tests surrogate-local`` / ``surrogate-validate`` commands +call; it glues :mod:`config`, :mod:`context`, :mod:`local_runner` and +:mod:`validate` together and decides the on-disk layout:: + + / + holdout.json buildings reserved for validation (energyplus) + + the fingerprint of the settings that chose them + context.parquet training context (energyplus) + sim_cache/ per-spec simulation cache (shared) + training// run_local_iterative_training output + training_spec.yml + validation// validate_surrogate output +""" + +import hashlib +import json +import logging +from pathlib import Path +from typing import Any, cast + +import geopandas as gpd +import pandas as pd +import yaml +from scythe.base import ExperimentInputSpec + +from globi.models.configs import GISPreprocessorColumnMap, GloBIExperimentSpec +from globi.models.surrogate.backends import MLBackend +from globi.models.surrogate.inference import SurrogateEnsemble +from globi.models.surrogate.pipeline import IterationSpec, ProgressiveTrainingSpec +from globi.pipelines.gis import preprocess_gis_file +from globi.validation.config import LocalValidationConfig +from globi.validation.context import ( + build_training_context, + build_training_spec, + default_priors_from_semantic_fields, +) +from globi.validation.env import ensure_local_hatchet_env +from globi.validation.local_runner import ( + LocalIterativeResult, + run_local_iterative_training, + sample_training_specs, + simulate_specs_locally, +) +from globi.validation.validate import ( + ValidationResult, + deterministic_specs, + validate_surrogate, +) + +logger = logging.getLogger(__name__) + +#: iteration offset used to draw the dummy-mode "ground truth" from a different seed +DUMMY_HOLDOUT_ITER = 1000 + + +def load_config(path: Path) -> LocalValidationConfig: + """Load a `LocalValidationConfig` yaml and prepare the process for local execution.""" + ensure_local_hatchet_env() + import globi.pipelines # noqa: F401 - registers runnables + + return LocalValidationConfig.from_manifest(path) + + +# --------------------------------------------------------------------------- # +# Spec construction +# --------------------------------------------------------------------------- # + + +def _load_manifest(config: LocalValidationConfig) -> GloBIExperimentSpec: + if config.manifest is None: + msg = "`manifest` is required in energyplus mode." + raise ValueError(msg) + return GloBIExperimentSpec.from_manifest(config.manifest) + + +# preprocessed GIS frames by manifest path: the hold-out, the training context and the +# ground truth all need the same frame, so preprocess once per process. +_PREPROCESSED: dict[str, tuple[gpd.GeoDataFrame, GISPreprocessorColumnMap]] = {} + + +def _preprocessed( + config: LocalValidationConfig, manifest: GloBIExperimentSpec +) -> tuple[gpd.GeoDataFrame, GISPreprocessorColumnMap]: + key = str(config.manifest) + if key not in _PREPROCESSED: + _PREPROCESSED[key] = preprocess_gis_file( + manifest.gis_preprocessor_config, + manifest.file_config, + scenario=manifest.scenario, + ) + return _PREPROCESSED[key] + + +def _sha1_of_files(*paths: Path) -> str: + h = hashlib.sha1() # noqa: S324 - not used for security + for path in paths: + h.update(path.read_bytes()) + return h.hexdigest() + + +def _holdout_fingerprint( + config: LocalValidationConfig, manifest: GloBIExperimentSpec +) -> dict[str, Any]: + """Everything that determines which buildings get held out (and the context).""" + return { + "seed": config.seed, + "n_test": config.n_test, + "manifest": str(config.manifest), + "scenario": manifest.scenario, + "inputs_sha1": _sha1_of_files( + manifest.file_config.gis_file, manifest.file_config.semantic_fields_file + ), + } + + +def _holdout_ids( + config: LocalValidationConfig, manifest: GloBIExperimentSpec +) -> list[str]: + """Pick (or reload) the buildings reserved for validation. + + The hold-out is chosen once, when the training context is built, and pinned in + `holdout.json` together with the settings that chose it. Re-choosing it later + (e.g. after changing `n_test` or the GIS file) would let buildings that are in the + training context leak into the validation set, so a mismatch is an error rather than a + rebuild. + """ + fingerprint = _holdout_fingerprint(config, manifest) + if config.holdout_path.exists(): + stored = json.loads(config.holdout_path.read_text()) + stale = { + k: (stored.get(k), v) for k, v in fingerprint.items() if stored.get(k) != v + } + if stale: + details = ", ".join( + f"{k}: {old!r} -> {new!r}" for k, (old, new) in stale.items() + ) + msg = ( + f"{config.holdout_path} was built with different settings ({details}); " + "use a new `workdir` (or delete this one) rather than mixing hold-outs." + ) + raise ValueError(msg) + return stored["ids"] + + _, ids = deterministic_specs( + manifest, + n=config.n_test, + seed=config.seed, + preprocessed=_preprocessed(config, manifest), + ) + config.workdir.mkdir(parents=True, exist_ok=True) + config.holdout_path.write_text(json.dumps({**fingerprint, "ids": ids}, indent=2)) + logger.info("Held out %d buildings for validation.", len(ids)) + return ids + + +def build_spec( + config: LocalValidationConfig, backend: MLBackend +) -> ProgressiveTrainingSpec: + """Build the training spec for one backend according to `config.mode`.""" + if config.mode == "dummy": + with open(config.dummy_training_spec) as f: + base = ProgressiveTrainingSpec.model_validate({ + **yaml.safe_load(f), + "storage_settings": None, + }) + return base.model_copy( + update={ + "ml_backend": backend, + "iteration": IterationSpec( + n_per_iter=config.n_per_iter, + min_per_stratum=config.min_per_stratum, + max_iters=config.max_iters, + recursion=base.iteration.recursion, + ), + "cross_val": base.cross_val.model_copy( + update={"n_folds": config.n_folds} + ), + } + ) + + manifest = _load_manifest(config) + holdout = _holdout_ids(config, manifest) + if not config.context_path.exists(): + context = build_training_context( + manifest.gis_preprocessor_config, + manifest.file_config, + scenario=manifest.scenario, + exclude_building_ids=holdout, + preprocessed=_preprocessed(config, manifest), + ) + context.to_parquet(config.context_path) + logger.info( + "Wrote training context (%d buildings) to %s", + len(context), + config.context_path, + ) + return build_training_spec( + manifest, + context_path=config.context_path, + ml_backend=backend, + priors=default_priors_from_semantic_fields( + manifest.file_config.semantic_fields_file, + fixed_basement_attic=config.fixed_basement_attic, + ), + n_per_iter=config.n_per_iter, + min_per_stratum=config.min_per_stratum, + max_iters=config.max_iters, + n_folds=config.n_folds, + targets_globs=config.targets_globs, + exclude_columns=config.exclude_columns, + thresholds=config.thresholds, + base_run_name=f"local-surrogate-{backend.ml_backend}", + ) + + +# --------------------------------------------------------------------------- # +# Training +# --------------------------------------------------------------------------- # + + +def run_training( + config: LocalValidationConfig, backend_names: list[str] | None = None +) -> dict[str, LocalIterativeResult]: + """Train one surrogate per selected backend; returns results keyed by backend name.""" + results: dict[str, LocalIterativeResult] = {} + for backend in config.selected_backends(backend_names): + spec = build_spec(config, backend) + out_dir = config.backend_dir(backend) + out_dir.mkdir(parents=True, exist_ok=True) + with open(out_dir / "training_spec.yml", "w") as f: + yaml.dump(spec.model_dump(mode="json"), f, indent=2, sort_keys=False) + logger.info("Training %s surrogate -> %s", backend.ml_backend, out_dir) + results[backend.ml_backend] = run_local_iterative_training( + spec, + out_dir, + n_workers=config.n_workers, + seed=config.seed, + sim_cache_dir=config.sim_cache_dir, + ) + return results + + +def load_trained_ensemble( + config: LocalValidationConfig, backend: MLBackend +) -> SurrogateEnsemble: + """Reload the surrogate a previous `run_training` wrote for `backend`.""" + summary_path = config.backend_dir(backend) / "summary.yml" + if not summary_path.exists(): + msg = f"No trained {backend.ml_backend} surrogate at {summary_path}; run `surrogate-local` first." + raise FileNotFoundError(msg) + return SurrogateEnsemble.from_training_summary(summary_path, backend) + + +# --------------------------------------------------------------------------- # +# Validation +# --------------------------------------------------------------------------- # + + +def ground_truth_specs( + config: LocalValidationConfig, +) -> tuple[Any, list[ExperimentInputSpec]]: + """The runnable and specs for the deterministic (ground truth) set.""" + if config.mode == "dummy": + # a fresh draw from the priors with a seed no training iteration uses + spec = build_spec(config, config.backends[0]).model_copy( + update={ + "iteration": IterationSpec( + n_per_iter=config.n_test, + min_per_stratum=1, + current_iter=DUMMY_HOLDOUT_ITER, + ) + } + ) + return spec.runnable, sample_training_specs(spec, seed=config.seed) + + from globi.pipelines.simulations import simulate_globi_building + + manifest = _load_manifest(config) + holdout = _holdout_ids(config, manifest) + specs, _ = deterministic_specs( + manifest, + n=None, + building_ids=holdout, + preprocessed=_preprocessed(config, manifest), + ) + return simulate_globi_building, cast(list[ExperimentInputSpec], specs) + + +def simulate_ground_truth(config: LocalValidationConfig) -> dict[str, pd.DataFrame]: + """Simulate (or load from cache) the deterministic ground-truth set.""" + runnable, specs = ground_truth_specs(config) + logger.info("Simulating %d ground-truth buildings...", len(specs)) + return simulate_specs_locally( + runnable, specs, cache_dir=config.sim_cache_dir, n_workers=config.n_workers + ) + + +def run_validations( + config: LocalValidationConfig, backend_names: list[str] | None = None +) -> dict[str, ValidationResult]: + """Validate every selected (already trained) backend against the ground truth.""" + ground_truth = simulate_ground_truth(config) + results: dict[str, ValidationResult] = {} + for backend in config.selected_backends(backend_names): + ensemble = load_trained_ensemble(config, backend) + out_dir = config.validation_dir / backend.ml_backend + logger.info("Validating %s surrogate -> %s", backend.ml_backend, out_dir) + results[backend.ml_backend] = validate_surrogate( + ensemble, ground_truth, out_dir=out_dir + ) + return results + + +def summarize_validations(results: dict[str, ValidationResult]) -> pd.DataFrame: + """One row per (backend, target) with the headline metrics and stock error.""" + rows = [] + for name, r in results.items(): + g = r.metrics_global["all"] + stock = r.aggregate.loc[("all", "all")] + for target in g.index: + rows.append({ + "backend": name, + "target": target, + "r2": g.loc[target, "r2"], + "cvrmse": g.loc[target, "cvrmse"], + "nmbe": g.loc[target, "nmbe"], + "mae": g.loc[target, "mae"], + "stock_pct_error": stock.loc[target, "pct_error"], + }) + return pd.DataFrame(rows).set_index(["backend", "target"]) diff --git a/src/globi/validation/local_runner.py b/src/globi/validation/local_runner.py new file mode 100644 index 0000000..e3808cc --- /dev/null +++ b/src/globi/validation/local_runner.py @@ -0,0 +1,476 @@ +"""Run the surrogate training pipeline in-process, without Hatchet or S3. + +Each function mirrors one task of the ``iterative_training`` hatchet workflow in +``globi.pipelines.training``: + +========================== ================================== +hatchet task local equivalent +========================== ================================== +create_simulations :func:`sample_training_specs` + :func:`simulate_specs_locally` +await_simulations (synchronous) +combine_results :func:`combine_with_previous` +start/await_training :func:`train_folds_locally` +evaluate_training :func:`train_folds_locally` (fold averages + convergence) +transition_recursion :func:`run_local_iterative_training` (the loop) +finalize :func:`run_local_iterative_training` (``summary.yml``) +========================== ================================== +""" + +import hashlib +import json +import logging +import multiprocessing +from concurrent.futures import ProcessPoolExecutor, as_completed +from dataclasses import dataclass, field +from functools import cache +from pathlib import Path +from typing import Any, Literal, cast + +import numpy as np +import pandas as pd +import yaml +from scythe.base import ExperimentInputSpec +from scythe.utils.results import transpose_dataframe_dict +from tqdm import tqdm + +from globi.models.surrogate.inference import ReferencedMLBackend, SurrogateEnsemble +from globi.models.surrogate.metrics import fold_test_averages +from globi.models.surrogate.pipeline import ProgressiveTrainingSpec +from globi.models.surrogate.sampling import SampleSpec +from globi.models.surrogate.training import TrainFoldSpec +from globi.validation.env import ensure_local_hatchet_env + +logger = logging.getLogger(__name__) + +# fields that vary between otherwise-identical specs and must not affect the cache key +_CACHE_KEY_EXCLUDED_FIELDS = frozenset({ + "experiment_id", + "sort_index", + "workflow_run_id", + "root_workflow_run_id", + "storage_settings", +}) + +# spec fields that reference input files: keyed by their *contents*, so editing the +# component library / semantic fields / component map invalidates cached results. +_CACHE_KEY_FILE_FIELDS = ( + "db_file", + "semantic_fields_file", + "component_map_file", + "epwzip_file", +) + + +# --------------------------------------------------------------------------- # +# Simulation +# --------------------------------------------------------------------------- # + + +@cache +def _file_sha1(path: str, size: int, mtime_ns: int) -> str: + """Content hash of a local file, memoised on (path, size, mtime).""" + return hashlib.sha1(Path(path).read_bytes()).hexdigest() # noqa: S324 - not used for security + + +def _file_fingerprint(ref: object) -> object: + """Replace a local file path by a content hash; leave urls / missing paths as-is.""" + if not isinstance(ref, str): + return ref + path = Path(ref) + if not path.is_file(): + return ref + st = path.stat() + return f"sha1:{_file_sha1(str(path.resolve()), st.st_size, st.st_mtime_ns)}" + + +def spec_cache_key(spec: ExperimentInputSpec) -> str: + """A content hash of a spec, independent of its position in an experiment. + + Local input files are hashed by content, not path. + """ + payload = spec.model_dump(mode="json", exclude=set(_CACHE_KEY_EXCLUDED_FIELDS)) + for name in _CACHE_KEY_FILE_FIELDS: + if name in payload: + payload[name] = _file_fingerprint(payload[name]) + return hashlib.sha1( # noqa: S324 - not used for security + json.dumps(payload, sort_keys=True, default=str).encode() + ).hexdigest() + + +def run_spec_locally( + runnable_name: str, spec_json: dict[str, Any] +) -> dict[str, pd.DataFrame]: + """Execute one registered experiment in this process and return its dataframes. + + Safe to call from a freshly spawned worker process: it sets up the hatchet env, + imports the pipelines (which registers the runnables), and re-validates the spec. + """ + ensure_local_hatchet_env() + from scythe.registry import ExperimentRegistry + + import globi.pipelines # noqa: F401 - registers runnables + + runnable = ExperimentRegistry.get_runnable(runnable_name) + spec = runnable.input_validator_type.model_validate(spec_json) # pyright: ignore [reportAttributeAccessIssue] + output = runnable.mock_run(input=spec) # pyright: ignore [reportAttributeAccessIssue] + return {k: v for k, v in output.dataframes.items() if isinstance(v, pd.DataFrame)} + + +def _cache_paths(cache_dir: Path, runnable_name: str, key: str) -> Path: + return cache_dir / runnable_name / key + + +def _read_cached(spec_dir: Path) -> dict[str, pd.DataFrame] | None: + manifest = spec_dir / "keys.json" + if not manifest.exists(): + return None + keys: list[str] = json.loads(manifest.read_text()) + return {k: pd.read_parquet(spec_dir / f"{k}.parquet") for k in keys} + + +def _write_cached(spec_dir: Path, dfs: dict[str, pd.DataFrame]) -> None: + spec_dir.mkdir(parents=True, exist_ok=True) + for k, df in dfs.items(): + df.to_parquet(spec_dir / f"{k}.parquet") + (spec_dir / "keys.json").write_text(json.dumps(sorted(dfs))) + + +def simulate_specs_locally( + runnable: Any, + specs: list[ExperimentInputSpec], + *, + cache_dir: Path, + n_workers: int = 1, +) -> dict[str, pd.DataFrame]: + """Simulate specs in-process (optionally in parallel) with a per-spec parquet cache. + + Args: + runnable: A scythe-registered standalone (e.g. `simulate_globi_building`). + specs: The input specs. Order is preserved in the gathered output. + cache_dir: Root of the cache; results live at `cache_dir///`. + n_workers: Number of worker processes for uncached specs (1 = in-process). + + Returns: + The per-key concatenation of every spec's result frames, exactly as scythe's + gather would produce them. + """ + runnable_name: str = runnable.name + keys = [spec_cache_key(spec) for spec in specs] + results: dict[int, dict[str, pd.DataFrame]] = {} + pending: list[int] = [] + for i, key in enumerate(keys): + cached = _read_cached(_cache_paths(cache_dir, runnable_name, key)) + if cached is not None: + results[i] = cached + else: + pending.append(i) + logger.info( + "%d/%d specs cached; simulating %d.", len(results), len(specs), len(pending) + ) + + def _run_and_cache(i: int) -> dict[str, pd.DataFrame]: + dfs = run_spec_locally(runnable_name, specs[i].model_dump(mode="json")) + _write_cached(_cache_paths(cache_dir, runnable_name, keys[i]), dfs) + return dfs + + if pending: + # The first miss runs in-process so shared caches (e.g. the downloaded EPW) + # are warm before fanning out. + first, *rest = pending + results[first] = _run_and_cache(first) + if n_workers <= 1 or not rest: + for i in tqdm(rest, desc=f"Simulating ({runnable_name})"): + results[i] = _run_and_cache(i) + else: + ctx = multiprocessing.get_context("spawn") + with ProcessPoolExecutor(max_workers=n_workers, mp_context=ctx) as pool: + futures = { + pool.submit( + run_spec_locally, + runnable_name, + specs[i].model_dump(mode="json"), + ): i + for i in rest + } + for fut in tqdm( + as_completed(futures), + total=len(futures), + desc=f"Simulating ({runnable_name}, {n_workers} workers)", + ): + i = futures[fut] + dfs = fut.result() + _write_cached(_cache_paths(cache_dir, runnable_name, keys[i]), dfs) + results[i] = dfs + + ordered = [results[i] for i in range(len(specs))] + return transpose_dataframe_dict(ordered) + + +# --------------------------------------------------------------------------- # +# Sampling / combining +# --------------------------------------------------------------------------- # + + +def sample_training_specs( + spec: ProgressiveTrainingSpec, *, seed: int | None = None +) -> list[ExperimentInputSpec]: + """Draw this iteration's training samples and convert them to simulation specs. + + Mirrors step 1 of `iterative_training.create_simulations`. The sampler's own RNG + is seeded by `spec.iteration.current_iter`; `seed` additionally pins numpy's global + RNG, which some `GloBIBuildingSpec` validators draw from. + """ + sample_spec = SampleSpec(parent=spec, priors=spec.samplers) + sample_df = sample_spec.populate_sample_df() + if seed is not None: + np.random.seed(seed + spec.iteration.current_iter) + input_validator = cast( + type[ExperimentInputSpec], spec.runnable.input_validator_type + ) + return sample_spec.convert_to_specs(sample_df, input_validator) + + +def combine_with_previous( + previous: dict[str, Path] | None, + incoming: dict[str, pd.DataFrame], + out_dir: Path, +) -> dict[str, Path]: + """Grow the simulation cache: concat new results onto previous iterations' parquets. + + Mirrors `iterative_training.combine_results`. + """ + out_dir.mkdir(parents=True, exist_ok=True) + combined: dict[str, Path] = {} + previous = previous or {} + for key in set(previous) | set(incoming): + parts: list[pd.DataFrame] = [] + if key in previous: + parts.append(pd.read_parquet(previous[key])) + if key in incoming: + parts.append(incoming[key]) + df = pd.concat(parts, axis=0) if len(parts) > 1 else parts[0] + path = out_dir / f"{key}.parquet" + df.to_parquet(path) + combined[key] = path + return combined + + +# --------------------------------------------------------------------------- # +# Training +# --------------------------------------------------------------------------- # + + +@dataclass +class LocalTrainingResult: + """Everything one training round produces, with artifacts on local disk.""" + + fold_models: list[ReferencedMLBackend] + global_metrics: pd.DataFrame + strata_metrics: pd.DataFrame + fold_averages: pd.Series + global_averages: pd.Series + converged: bool + data_paths: dict[str, Path] + + @property + def test_metrics_table(self) -> pd.DataFrame: + """Fold-averaged test metrics as a (target x metric) table.""" + s = self.global_averages + table = s.unstack(level="metric") if "metric" in s.index.names else s.to_frame() + return cast(pd.DataFrame, table) + + +def train_folds_locally( + spec: ProgressiveTrainingSpec, + data_paths: dict[str, Path], + models_dir: Path, +) -> LocalTrainingResult: + """Train one model per cross-validation fold and evaluate convergence. + + Mirrors `start_training` + `await_training` + `evaluate_training`. + """ + fold_models: list[ReferencedMLBackend] = [] + global_frames: list[pd.DataFrame] = [] + strata_frames: list[pd.DataFrame] = [] + for i in range(spec.cross_val.n_folds): + fold_dir = models_dir / f"fold_{i}" + fold_dir.mkdir(parents=True, exist_ok=True) + logger.info("Training fold %d/%d...", i + 1, spec.cross_val.n_folds) + fold_spec = TrainFoldSpec( + experiment_id="placeholder", + sort_index=i, + data_uris=dict(data_paths), + parent=spec, + ) + trained = fold_spec.train(fold_dir) + fold_models.append( + ReferencedMLBackend( + regressor=trained.artifacts.artifacts.regressor_path, + transforms=trained.artifacts.artifacts.transforms_path, + ml_backend=spec.ml_backend, + ) + ) + global_frames.append(trained.global_metrics) + strata_frames.append(trained.stratum_metrics) + + global_metrics = pd.concat(global_frames, axis=0) + strata_metrics = pd.concat(strata_frames, axis=0) + fold_averages = fold_test_averages(strata_metrics) + global_averages = fold_test_averages(global_metrics) + converged, *_ = spec.convergence_criteria.run(fold_averages) + + return LocalTrainingResult( + fold_models=fold_models, + global_metrics=global_metrics, + strata_metrics=strata_metrics, + fold_averages=fold_averages, + global_averages=global_averages, + converged=bool(converged), + data_paths=data_paths, + ) + + +# --------------------------------------------------------------------------- # +# The outer loop +# --------------------------------------------------------------------------- # + +StopReason = Literal["converged", "max_depth"] + + +@dataclass +class LocalIterativeResult: + """The outcome of a local iterative training run.""" + + reasoning: StopReason + iterations: list[LocalTrainingResult] = field(default_factory=list) + workdir: Path = Path() + + @property + def final(self) -> LocalTrainingResult: + """The last training round.""" + return self.iterations[-1] + + @property + def fold_models(self) -> list[ReferencedMLBackend]: + """The fold models from the last training round.""" + return self.final.fold_models + + @property + def ensemble(self) -> SurrogateEnsemble: + """The trained surrogate (the last round's fold models).""" + return SurrogateEnsemble(models=self.fold_models) + + @property + def summary_path(self) -> Path: + """Where `summary.yml` was written.""" + return self.workdir / "summary.yml" + + +def run_local_iterative_training( + spec: ProgressiveTrainingSpec, + workdir: Path, + *, + n_workers: int = 1, + seed: int | None = 0, + sim_cache_dir: Path | None = None, +) -> LocalIterativeResult: + """Run the whole sample -> simulate -> train -> evaluate loop locally. + + Layout of `workdir`:: + + iter_000/ + specs.parquet the sampled training specs (this iteration only) + data/.parquet combined simulation results (all iterations so far) + models/fold_/ regressor + transforms.yml per fold + metrics/{global,strata}.parquet + summary.yml + + Args: + spec: The training spec; `iteration.current_iter` is the starting iteration. + workdir: Output directory. + n_workers: Worker processes for simulation. + seed: Seed for numpy's global RNG (see `sample_training_specs`). + sim_cache_dir: Per-spec simulation cache (default: `workdir / "sim_cache"`). + """ + workdir.mkdir(parents=True, exist_ok=True) + cache_dir = sim_cache_dir or (workdir / "sim_cache") + result = LocalIterativeResult(reasoning="max_depth", workdir=workdir) + previous_data: dict[str, Path] | None = None + all_global: list[pd.DataFrame] = [] + all_strata: list[pd.DataFrame] = [] + + current = spec.iteration.current_iter + while True: + iter_spec = spec.model_copy( + update={ + "iteration": spec.iteration.model_copy(update={"current_iter": current}) + } + ) + iter_dir = workdir / f"iter_{current:03d}" + logger.info("=== Iteration %d ===", current) + + iter_dir.mkdir(parents=True, exist_ok=True) + specs = sample_training_specs(iter_spec, seed=seed) + pd.DataFrame([s.model_dump(mode="json") for s in specs]).to_parquet( + iter_dir / "specs.parquet" + ) + incoming = simulate_specs_locally( + iter_spec.runnable, specs, cache_dir=cache_dir, n_workers=n_workers + ) + data_paths = combine_with_previous(previous_data, incoming, iter_dir / "data") + + training = train_folds_locally(iter_spec, data_paths, iter_dir / "models") + metrics_dir = iter_dir / "metrics" + metrics_dir.mkdir(exist_ok=True) + training.global_metrics.to_parquet(metrics_dir / "global.parquet") + training.strata_metrics.to_parquet(metrics_dir / "strata.parquet") + all_global.append(training.global_metrics) + all_strata.append(training.strata_metrics) + result.iterations.append(training) + logger.info( + "Iteration %d test metrics:\n%s", current, training.test_metrics_table + ) + + if training.converged: + result.reasoning = "converged" + break + if iter_spec.iteration.at_max_iters: + result.reasoning = "max_depth" + break + previous_data = data_paths + current += 1 + + # mirrors `finalize`: combined metrics across iterations + a summary manifest + combined_dir = workdir / "combined_metrics" + combined_dir.mkdir(exist_ok=True) + pd.concat(all_global, axis=0).to_parquet(combined_dir / "global.parquet") + pd.concat(all_strata, axis=0).to_parquet(combined_dir / "strata.parquet") + summary = { + "reasoning": result.reasoning, + "ml_backend": spec.ml_backend.ml_backend, + "n_iterations": len(result.iterations), + "data_paths": {k: str(v) for k, v in result.final.data_paths.items()}, + "fold_models": [ + {"regressor": str(m.regressor), "transforms": str(m.transforms)} + for m in result.fold_models + ], + "test_metrics_by_iteration": [ + { + "iteration": i, + "converged": it.converged, + "metrics": _series_to_records(it.global_averages), + } + for i, it in enumerate(result.iterations) + ], + } + with open(result.summary_path, "w") as f: + yaml.dump(summary, f, indent=2, sort_keys=False) + return result + + +def _series_to_records(s: pd.Series) -> list[dict[str, Any]]: + names = [str(n) for n in s.index.names] + return [ + {**dict(zip(names, cast(tuple, idx), strict=True)), "value": float(v)} + for idx, v in s.items() + ] diff --git a/src/globi/validation/validate.py b/src/globi/validation/validate.py new file mode 100644 index 0000000..3e43285 --- /dev/null +++ b/src/globi/validation/validate.py @@ -0,0 +1,363 @@ +"""Validate a trained surrogate against deterministic (EnergyPlus) ground truth. + +The ground truth is the output of the standard GloBI run: buildings whose semantic +fields are fully defined in the GIS data (no priors), simulated with EnergyPlus. The +surrogate (`globi.models.surrogate.inference.SurrogateEnsemble`) predicts from the +feature index those simulations carry, and this module compares the two per building +and at stock level. Only the comparison lives here; how to apply a surrogate is +defined on the inference side. +""" + +import logging +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, cast + +import geopandas as gpd +import numpy as np +import pandas as pd + +from globi.models.configs import GISPreprocessorColumnMap, GloBIExperimentSpec +from globi.models.surrogate.inference import ( + AREA_COLUMN, + SEMANTIC_PREFIX, + SurrogateEnsemble, +) +from globi.models.surrogate.metrics import compute_frame_metrics +from globi.models.surrogate.training import flatten_result_frames +from globi.models.tasks import GloBIBuildingSpec +from globi.pipelines.gis import build_building_specs, preprocess_gis_file + +logger = logging.getLogger(__name__) + + +def deterministic_specs( + manifest: GloBIExperimentSpec, + *, + n: int | None, + seed: int = 0, + building_ids: list[str] | None = None, + preprocessed: tuple[gpd.GeoDataFrame, GISPreprocessorColumnMap] | None = None, +) -> tuple[list[GloBIBuildingSpec], list[str]]: + """Build simulation specs for buildings exactly as the GIS data defines them. + + Overheating / hourly outputs are disabled so the result frames match what the + training samples produce. + + Args: + manifest: The experiment manifest (GIS, db, semantic fields, ...). + n: Number of buildings to sample (None = all). + seed: Sampling seed. + building_ids: Use exactly these buildings instead of sampling. + preprocessed: An already preprocessed `(gdf, colmap)` pair to use instead of + running `preprocess_gis_file` again. + + Returns: + The specs and the ids of the buildings they describe. + """ + gdf, colmap = preprocessed or preprocess_gis_file( + manifest.gis_preprocessor_config, + manifest.file_config, + scenario=manifest.scenario, + ) + if building_ids is not None: + mask = gdf[colmap.Building_ID_col].isin(list(building_ids)) + gdf = cast(gpd.GeoDataFrame, gdf[mask]) + elif n is not None: + gdf = cast(gpd.GeoDataFrame, gdf.sample(n=min(n, len(gdf)), random_state=seed)) + parent = manifest.model_copy( + update={"overheating_config": None, "hourly_data_config": None} + ) + specs = build_building_specs( + gdf, colmap, manifest.file_config, parent_experiment_spec=parent + ) + return specs, [s.building_id for s in specs] + + +# --------------------------------------------------------------------------- # +# Validation +# --------------------------------------------------------------------------- # + + +@dataclass +class ValidationResult: + """All validation outputs (also written to `out_dir`).""" + + per_building: pd.DataFrame + metrics_global: pd.DataFrame + metrics_by_group: pd.DataFrame + aggregate: pd.DataFrame + metrics_per_fold: pd.DataFrame + unseen: dict[str, set] + out_dir: Path + + @property + def report_path(self) -> Path: + """Path to the markdown report.""" + return self.out_dir / "report.md" + + +def _group_columns(features: pd.DataFrame) -> list[str]: + cols = [c for c in features.columns if c.startswith(SEMANTIC_PREFIX)] + if "num_floors" in features.columns: + cols.append("num_floors") + return [c for c in cols if features[c].nunique() > 1] + + +def _stock_aggregate( + truth: pd.DataFrame, pred: pd.DataFrame, area: pd.Series +) -> pd.DataFrame: + """Area-weighted totals (kWh) of intensity targets (kWh/m2), true vs predicted.""" + true_total = truth.mul(area.to_numpy(), axis=0).sum(axis=0) + pred_total = pred.mul(area.to_numpy(), axis=0).sum(axis=0) + out = pd.DataFrame({"true_total": true_total, "pred_total": pred_total}) + out.insert(0, "n", len(truth)) + out["error"] = out["pred_total"] - out["true_total"] + out["pct_error"] = out["error"] / out["true_total"].where(out["true_total"] != 0, 1) + out.index.name = "target" + return out + + +_GROUP_LEVELS = ["group_column", "group_value", "target"] + + +def _concat_named( + frames: Mapping[Any, pd.DataFrame], *, axis: Literal[0, 1], names: list[str] +) -> pd.DataFrame: + """`pd.concat` with a keyed outer level, typed loosely to satisfy pandas-stubs.""" + return cast( + pd.DataFrame, + pd.concat(frames, axis=axis, names=names), # pyright: ignore [reportCallIssue, reportArgumentType] + ) + + +def _per_building_frame( + features: pd.DataFrame, + truth: pd.DataFrame, + pred: pd.DataFrame, + has_unseen: pd.Series, + *, + id_cols: list[str], +) -> pd.DataFrame: + """Flat per-building frame: id/grouping columns + true/pred/err per target.""" + out = pd.DataFrame(features[[c for c in id_cols if c in features]]) + out["has_unseen_category"] = has_unseen.to_numpy() + for t in truth.columns: + out[f"{t}/true"] = truth[t].to_numpy() + out[f"{t}/pred"] = pred[t].to_numpy() + out[f"{t}/abs_err"] = (pred[t] - truth[t]).abs().to_numpy() + out[f"{t}/pct_err"] = ( + (pred[t] - truth[t]) / truth[t].where(truth[t] != 0, np.nan) + ).to_numpy() + return out + + +def _global_metrics( + truth: pd.DataFrame, pred: pd.DataFrame, has_unseen: pd.Series +) -> pd.DataFrame: + """Ensemble metrics on all buildings and, if relevant, on the seen-category subset.""" + subsets = {"all": compute_frame_metrics(pred, truth)} + if has_unseen.any() and (~has_unseen).sum() >= 2: + seen = cast(np.ndarray, (~has_unseen).to_numpy()) + subsets["seen_only"] = compute_frame_metrics( + cast(pd.DataFrame, pred[seen]), cast(pd.DataFrame, truth[seen]) + ) + return _concat_named(subsets, axis=1, names=["subset", "metric"]) + + +def _group_metrics( + features: pd.DataFrame, truth: pd.DataFrame, pred: pd.DataFrame, groups: list[str] +) -> pd.DataFrame: + """Metrics per value of each grouping column (groups with <2 buildings skipped).""" + frames: dict[tuple[str, str], pd.DataFrame] = {} + for col in groups: + for val, idx in features.groupby(col).groups.items(): + if len(idx) < 2: + continue + frame = compute_frame_metrics(pred.loc[idx], truth.loc[idx]) + frame.insert(0, "n", len(idx)) + frames[(col, str(val))] = frame + if not frames: + return pd.DataFrame() + return _concat_named(frames, axis=0, names=_GROUP_LEVELS).sort_index() + + +def _group_aggregates( + features: pd.DataFrame, + truth: pd.DataFrame, + pred: pd.DataFrame, + area: pd.Series, + groups: list[str], +) -> pd.DataFrame: + """Area-weighted stock totals overall (`("all", "all")`) and per group value.""" + frames = {("all", "all"): _stock_aggregate(truth, pred, area)} + for col in groups: + for val, idx in features.groupby(col).groups.items(): + frames[(col, str(val))] = _stock_aggregate( + truth.loc[idx], pred.loc[idx], area.loc[idx] + ) + return _concat_named(frames, axis=0, names=_GROUP_LEVELS).sort_index() + + +def validate_surrogate( + ensemble: SurrogateEnsemble, + ground_truth: dict[str, pd.DataFrame], + *, + out_dir: Path, + area_column: str = AREA_COLUMN, + group_columns: list[str] | None = None, +) -> ValidationResult: + """Compare surrogate predictions to EnergyPlus results for the same buildings. + + Args: + ensemble: The trained surrogate (per-fold metrics are reported alongside the + ensemble's). + ground_truth: Raw simulation result frames keyed like the training data + (e.g. `{"EnergyAndPeakAnnual": df}`), feature MultiIndex on the rows. + out_dir: Where to write parquet outputs and `report.md`. + area_column: Index column used to area-weight the stock aggregates. + group_columns: Index columns to break metrics down by (default: every + `feature.semantic.*` column with >1 value, plus `num_floors`). + """ + out_dir.mkdir(parents=True, exist_ok=True) + truth = flatten_result_frames(ground_truth, ensemble.targets) + features = truth.index.to_frame(index=False) + truth = truth.reset_index(drop=True) + + per_fold = ensemble.predict_per_fold(features) + pred = cast(pd.DataFrame, sum(per_fold) / len(per_fold)) + unseen = ensemble.unseen_categories(features) + has_unseen = pd.Series(False, index=features.index) + for col, extra in unseen.items(): + has_unseen |= features[col].isin(list(extra)) + if unseen: + logger.warning( + "%d/%d buildings have categorical values unseen in training: %s", + int(has_unseen.sum()), + len(features), + {k: sorted(map(str, v)) for k, v in unseen.items()}, + ) + + groups = group_columns or _group_columns(features) + if area_column in features.columns: + area = cast(pd.Series, features[area_column].astype(float)) + else: + logger.warning("Area column %s not found; aggregating unweighted.", area_column) + area = pd.Series(1.0, index=features.index) + + fold_frames = { + f"fold_{i}": compute_frame_metrics(p, truth) for i, p in enumerate(per_fold) + } + fold_frames["ensemble"] = compute_frame_metrics(pred, truth) + + result = ValidationResult( + per_building=_per_building_frame( + features, + truth, + pred, + has_unseen, + id_cols=["building_id", area_column, *groups], + ), + metrics_global=_global_metrics(truth, pred, has_unseen), + metrics_by_group=_group_metrics(features, truth, pred, groups), + aggregate=_group_aggregates(features, truth, pred, area, groups), + metrics_per_fold=pd.concat(fold_frames, axis=1, names=["model", "metric"]), + unseen=unseen, + out_dir=out_dir, + ) + _write_outputs(result, pred, truth, features) + return result + + +def _write_outputs( + result: ValidationResult, + pred: pd.DataFrame, + truth: pd.DataFrame, + features: pd.DataFrame, +) -> None: + out = result.out_dir + pd.concat({"true": truth, "pred": pred}, axis=1).set_index( + pd.MultiIndex.from_frame(features) + ).to_parquet(out / "predictions.parquet") + result.per_building.to_parquet(out / "per_building.parquet") + result.metrics_global.to_parquet(out / "metrics_global.parquet") + result.metrics_per_fold.to_parquet(out / "metrics_per_fold.parquet") + if not result.metrics_by_group.empty: + result.metrics_by_group.to_parquet(out / "metrics_by_group.parquet") + result.aggregate.to_parquet(out / "aggregate.parquet") + result.report_path.write_text(render_report(result)) + + +def _md(df: pd.DataFrame | pd.Series) -> str: + """Markdown table, falling back to plain text if `tabulate` is unavailable.""" + try: + return str(df.to_markdown(floatfmt=".4g")) + except ImportError: + return "```\n" + df.to_string() + "\n```" + + +def render_report(result: ValidationResult) -> str: + """Render a markdown summary of the validation.""" + n = len(result.per_building) + lines = [ + "# Surrogate validation report", + "", + f"Buildings: **{n}**; targets: **{len(result.metrics_global)}**; " + f"fold models: **{result.metrics_per_fold.columns.get_level_values('model').nunique() - 1}**", + "", + ] + if result.unseen: + lines += [ + "> **Warning**: some buildings have categorical values never seen in " + f"training ({int(result.per_building['has_unseen_category'].sum())}/{n}):", + "", + *[ + f"> - `{col}`: {sorted(map(str, vals))}" + for col, vals in result.unseen.items() + ], + "", + ] + lines += [ + "## Global metrics (ensemble, all buildings)", + "", + _md(result.metrics_global["all"]), + "", + ] + if "seen_only" in result.metrics_global.columns.get_level_values("subset"): + lines += [ + "## Global metrics (buildings with seen categories only)", + "", + _md(result.metrics_global["seen_only"]), + "", + ] + stock = result.aggregate.loc[("all", "all")] + lines += [ + "## Stock aggregate (area-weighted totals)", + "", + _md(stock), + "", + "## Per-fold vs ensemble (r2 / cvrmse)", + "", + _md(result.metrics_per_fold.xs("r2", level="metric", axis=1)), + "", + _md(result.metrics_per_fold.xs("cvrmse", level="metric", axis=1)), + "", + ] + if not result.metrics_by_group.empty: + lines += ["## Metrics by group (n, r2, cvrmse, nmbe)", ""] + by_group = result.metrics_by_group[["n", "r2", "cvrmse", "nmbe"]] + lines += [_md(by_group), ""] + lines += ["## Stock aggregate by group", ""] + grouped = result.aggregate.drop(index=("all", "all"), errors="ignore") + lines += [_md(grouped), ""] + lines += [ + "## Files", + "", + "- `predictions.parquet`: true/pred per target, feature MultiIndex", + "- `per_building.parquet`: flat per-building errors with grouping columns", + "- `metrics_global.parquet`, `metrics_per_fold.parquet`, `metrics_by_group.parquet`", + "- `aggregate.parquet`: area-weighted stock totals overall and per group", + "", + ] + return "\n".join(lines) diff --git a/src/globi/worker/Dockerfile b/src/globi/worker/Dockerfile index 321b76b..e4f1409 100644 --- a/src/globi/worker/Dockerfile +++ b/src/globi/worker/Dockerfile @@ -93,12 +93,13 @@ RUN EP_VERSION_DASH=$(echo "${EP_VERSION}" | tr '.' '-') && \ WORKDIR /code COPY uv.lock pyproject.toml README.md /code/ -RUN uv sync --locked --no-install-project --extra cli +# TODO: only insttall ml for certain containers by passing in a flag to the docker build command +RUN uv sync --locked --no-install-project --extra cli --extra ml-gpu RUN uv run epi prisma generate COPY src /code/src/ -RUN uv sync --locked --extra cli +RUN uv sync --locked --extra cli --extra ml-gpu CMD [ "uv", "run", "src/globi/worker/main.py" ] diff --git a/src/globi/worker/main.py b/src/globi/worker/main.py index 4a38a4a..d97ac73 100644 --- a/src/globi/worker/main.py +++ b/src/globi/worker/main.py @@ -1,16 +1,22 @@ """Worker main script.""" +import logging + from scythe.worker import ScytheWorkerConfig from globi.pipelines import * # noqa: F403 +from globi.pipelines import iterative_training conf = ScytheWorkerConfig() def main(): - """Main function for the worker.""" - conf.start() + """Start the worker.""" + logging.basicConfig(level=logging.INFO) + # only orchestrator (fan) workers should pick up iterative_training steps; leaf + # workers would otherwise hold a slot while e.g. await_simulations waits. + conf.start(additional_workflows=[iterative_training] if conf.DOES_FAN else []) if __name__ == "__main__": - conf.start() + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a01d628 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,63 @@ +"""Shared pytest configuration for globi. + +Importing anything under ``globi.pipelines`` builds a Hatchet client at import time, +so we populate offline-safe env defaults before any test module is collected. +""" + +from pathlib import Path + +import pytest + +from globi.validation.env import ensure_local_hatchet_env + +REPO_ROOT = Path(__file__).resolve().parents[1] + +ensure_local_hatchet_env(REPO_ROOT) + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Register the opt-in flag for EnergyPlus-backed tests.""" + parser.addoption( + "--run-energyplus", + action="store_true", + default=False, + help="Run tests that execute real EnergyPlus simulations (slow).", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Skip `energyplus`-marked tests unless explicitly requested.""" + if config.getoption("--run-energyplus"): + return + skip = pytest.mark.skip(reason="needs --run-energyplus") + for item in items: + if "energyplus" in item.keywords: + item.add_marker(skip) + + +@pytest.fixture(scope="session") +def repo_root() -> Path: + """The repository root.""" + return REPO_ROOT + + +@pytest.fixture(scope="session") +def e2e_data_dir(repo_root: Path) -> Path: + """The e2e fixture directory.""" + return repo_root / "tests" / "data" / "e2e" + + +_BACKEND_MODULES = {"xgb": "xgboost", "lgb": "lightgbm", "nn": "torch"} + + +@pytest.fixture(params=["xgb", "lgb", "nn"]) +def backend_name(request: pytest.FixtureRequest) -> str: + """Parametrize over ML backends, skipping any whose library is not installed.""" + name: str = request.param + pytest.importorskip(_BACKEND_MODULES[name]) + if name == "xgb": + # XGBModelConfig.param_dict imports torch to probe for CUDA. + pytest.importorskip("torch") + return name diff --git a/tests/data/dummy_training/context.parquet b/tests/data/dummy_training/context.parquet new file mode 100644 index 0000000..c1c14bf Binary files /dev/null and b/tests/data/dummy_training/context.parquet differ diff --git a/tests/data/dummy_training/training.yml b/tests/data/dummy_training/training.yml new file mode 100644 index 0000000..52ea709 --- /dev/null +++ b/tests/data/dummy_training/training.yml @@ -0,0 +1,68 @@ +runnable: scythe_experiment_dummy_simulation +experiment_id: placeholder +sort_index: 0 +workflow_run_id: null +root_workflow_run_id: null +storage_settings: + BUCKET: ml-for-bem + BUCKET_PREFIX: globi-testing +base_run_name: test-experiment +convergence_criteria: + thresholds: + "*": + mae: null + rmse: null + mape: null + r2: 0.975 + cvrmse: null + nmbe: null +regression_io_config: + targets: + columns: [] + normalization: min-max + features: + continuous_columns: [] + categorical_columns: [] + exclude_columns: [] + cont_cat_unicity_transition_threshold: 10 +ml_backend: + trainer: + num_boost_round: 4000 + early_stopping_rounds: 20 + ml_backend: lgb +stratification: + field: stratification_field + sampling: equal +samplers: + sampled_features: + x0: + sampler: + min: 0 + max: 1 + x1: + sampler: + min: 0 + max: 1 + x2: + sampler: + min: 0 + max: 1 + x3: + sampler: + min: 0 + max: 1 +cross_val: + n_folds: 5 +iteration: + n_per_iter: 100 + min_per_stratum: 25 + max_iters: 3 + recursion: + path: null + factor: 100 + max_depth: 1 + current_iter: 0 +context: tests/data/dummy_training/context.parquet +data_uris: null +metrics_uris: [] +previous_experiment_ids: [] diff --git a/tests/data/e2e/artifacts.yml b/tests/data/e2e/artifacts.yml index 5e0b6c6..b111507 100644 --- a/tests/data/e2e/artifacts.yml +++ b/tests/data/e2e/artifacts.yml @@ -1,4 +1,4 @@ -gis_file: tests/data/e2e/buildings.parquet +gis_file: tests/data/e2e/buildings.geojson db_file: tests/data/e2e/components-lib.db semantic_fields_file: tests/data/e2e/semantic-fields.yml # TODO: add local epwzip file diff --git a/tests/data/e2e/buildings.parquet b/tests/data/e2e/buildings.geojson similarity index 100% rename from tests/data/e2e/buildings.parquet rename to tests/data/e2e/buildings.geojson diff --git a/tests/data/surrogate_local/validation-dummy.yml b/tests/data/surrogate_local/validation-dummy.yml new file mode 100644 index 0000000..04d7d91 --- /dev/null +++ b/tests/data/surrogate_local/validation-dummy.yml @@ -0,0 +1,38 @@ +# Fast, EnergyPlus-free exercise of the whole local surrogate loop + validation using the +# analytic `dummy_simulation` runnable. Runs in well under a minute for all backends. +# +# uv run globi tests surrogate-local --config tests/data/surrogate_local/validation-dummy.yml +# uv run globi tests surrogate-validate --config tests/data/surrogate_local/validation-dummy.yml +mode: dummy +dummy_training_spec: tests/data/dummy_training/training.yml +workdir: outputs/surrogate-dummy +seed: 0 +n_workers: 1 + +n_per_iter: 60 +min_per_stratum: 15 +max_iters: 2 +n_folds: 3 +n_test: 40 + +# Small-data settings. NB: the NN trainer drops incomplete batches, so batch_size must +# be smaller than the number of rows in a single fold. +backends: + - ml_backend: xgb + trainer: + num_boost_round: 200 + early_stopping_rounds: 20 + verbose_eval: false + - ml_backend: lgb + hp: + min_data_in_leaf: 3 + trainer: + num_boost_round: 200 + early_stopping_rounds: 20 + - ml_backend: nn + hp: + hidden_dims: [32, 32] + trainer: + epochs: 60 + batch_size: 8 + early_stopping_patience: 15 diff --git a/tests/data/surrogate_local/validation-e2e.yml b/tests/data/surrogate_local/validation-e2e.yml new file mode 100644 index 0000000..e963e76 --- /dev/null +++ b/tests/data/surrogate_local/validation-e2e.yml @@ -0,0 +1,61 @@ +# Small EnergyPlus-backed surrogate run on the e2e test region (1000 buildings, one EPW). +# +# Training samples are drawn from the GIS geometry with semantic fields / wwr / f2f +# height sampled from priors; the validation set is `n_test` held-out buildings simulated +# exactly as the GIS data defines them (the standard deterministic globi run). +# +# This is a smoke test of the plumbing (feature index alignment, predict path), not a +# validation: at 16 + 4 simulations the metrics are noise. Hold-out validation that +# means anything is an at-scale run on the hatchet stack. +# +# uv run globi tests surrogate-local --config tests/data/surrogate_local/validation-e2e.yml --workers 4 +# uv run globi tests surrogate-validate --config tests/data/surrogate_local/validation-e2e.yml --workers 4 +# +# Budget: (n_per_iter * max_iters + n_test) EnergyPlus runs; a shoebox takes ~1-5 s on a laptop +# (the smoke profile below finishes in ~30 s with --workers 4). +mode: energyplus +manifest: tests/data/e2e/manifest.yml +workdir: outputs/surrogate-e2e +seed: 0 +n_workers: 1 + +# 16 training + 4 hold-out simulations (the opt-in pytest test uses this profile). +# NB: the hold-out is pinned in /holdout.json at training time; change +# `n_test` / `seed` only together with `workdir`. +n_per_iter: [16] +min_per_stratum: 16 +max_iters: 1 +n_folds: 4 +n_test: 4 + +targets_globs: + - "EnergyAndPeakAnnual/Energy/Raw/*" +# never let these index columns become features (unique per building / unused by the model) +exclude_columns: [building_id, rotated_rectangle, height] +thresholds: + "*": + r2: 0.9 +fixed_basement_attic: true + +backends: + - ml_backend: xgb + hp: + max_depth: 4 + min_child_weight: 1 + trainer: + num_boost_round: 300 + early_stopping_rounds: 20 + verbose_eval: false + # - ml_backend: lgb + # hp: + # min_data_in_leaf: 3 + # trainer: + # num_boost_round: 300 + # early_stopping_rounds: 20 + # - ml_backend: nn + # hp: + # hidden_dims: [64, 64] + # trainer: + # epochs: 200 + # batch_size: 4 + # early_stopping_patience: 30 diff --git a/tests/test_conditional_priors.py b/tests/test_conditional_priors.py new file mode 100644 index 0000000..d61db97 --- /dev/null +++ b/tests/test_conditional_priors.py @@ -0,0 +1,242 @@ +"""Smoke tests for conditional prior sampling behavior. + +Run with: + python scripts/test_conditional_priors.py +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from globi.models.surrogate.samplers import ( + ConditionalPrior, + ConditionalPriorCondition, + CopySampler, + MultiColumnCondition, + MultiColumnConditionalPrior, + SamplingError, +) + + +def _require(condition: bool, message: str) -> None: + """Raise AssertionError with message when condition is not met.""" + if not condition: + raise AssertionError(message) + + +def _track_copy_sampler_calls( + prior: Any, context: pd.DataFrame +) -> list[dict[str, Any]]: + """Execute prior.sample while recording CopySampler call metadata.""" + calls: list[dict[str, Any]] = [] + original_sample = CopySampler.sample + + def tracking_sample( + self: CopySampler, + subset_context: pd.DataFrame, + n: int, + generator: np.random.Generator, + ) -> np.ndarray: + calls.append({ + "feature_to_copy": self.feature_to_copy, + "n": n, + "rows": len(subset_context), + "kind_values": tuple(subset_context["kind"].tolist()), + "stage_values": tuple(subset_context["stage"].tolist()), + }) + return original_sample(self, subset_context, n, generator) + + with patch.object( + CopySampler, "sample", autospec=True, side_effect=tracking_sample + ): + _ = prior.sample(context, len(context), np.random.default_rng(0)) + + return calls + + +def _single_call(calls: list[dict[str, Any]], feature_to_copy: str) -> dict[str, Any]: + matching_calls = [c for c in calls if c["feature_to_copy"] == feature_to_copy] + _require( + len(matching_calls) == 1, + f"Expected exactly one call for {feature_to_copy}, got {len(matching_calls)}", + ) + return matching_calls[0] + + +def test_conditional_prior_subset_sampling() -> None: + """ConditionalPrior samples only matched rows for each condition.""" + context = pd.DataFrame({ + "kind": ["A", "A", "B", "C", "B", "A"], + "stage": [1, 1, 2, 3, 2, 1], + "value_for_a": [10, 11, 12, 13, 14, 15], + "value_for_b": [20, 21, 22, 23, 24, 25], + "value_fallback": [30, 31, 32, 33, 34, 35], + }) + + prior = ConditionalPrior( + source_feature="kind", + conditions=[ + ConditionalPriorCondition( + match_val="A", sampler=CopySampler(feature_to_copy="value_for_a") + ), + ConditionalPriorCondition( + match_val="B", sampler=CopySampler(feature_to_copy="value_for_b") + ), + ], + fallback_prior=CopySampler(feature_to_copy="value_fallback"), + ) + + out = prior.sample(context, len(context), np.random.default_rng(42)) + expected = np.array([10, 11, 22, 33, 24, 15]) + _require(np.array_equal(out, expected), f"Unexpected output: {out}") + + calls = _track_copy_sampler_calls(prior, context) + call_a = _single_call(calls, "value_for_a") + call_b = _single_call(calls, "value_for_b") + call_fallback = _single_call(calls, "value_fallback") + + _require( + call_a["n"] == 3 and call_a["rows"] == 3, "Expected A sampler to get 3 rows" + ) + _require( + call_b["n"] == 2 and call_b["rows"] == 2, "Expected B sampler to get 2 rows" + ) + _require( + call_fallback["n"] == 1 and call_fallback["rows"] == 1, + "Expected fallback sampler to get 1 row", + ) + _require(set(call_a["kind_values"]) == {"A"}, "A sampler saw unexpected kinds") + _require(set(call_b["kind_values"]) == {"B"}, "B sampler saw unexpected kinds") + _require( + set(call_fallback["kind_values"]) == {"C"}, + "Fallback sampler saw unexpected kinds", + ) + + +def test_multicolumn_conditional_prior_subset_sampling() -> None: + """MultiColumnConditionalPrior samples only matched tuples per condition.""" + context = pd.DataFrame({ + "kind": ["A", "A", "B", "C", "B", "A", "B"], + "stage": [1, 2, 2, 3, 2, 1, 1], + "value_for_a1": [100, 101, 102, 103, 104, 105, 106], + "value_for_b2": [200, 201, 202, 203, 204, 205, 206], + "value_fallback": [300, 301, 302, 303, 304, 305, 306], + }) + + prior = MultiColumnConditionalPrior( + source_features=["kind", "stage"], + conditions=[ + MultiColumnCondition( + match_vals=("A", 1), sampler=CopySampler(feature_to_copy="value_for_a1") + ), + MultiColumnCondition( + match_vals=("B", 2), sampler=CopySampler(feature_to_copy="value_for_b2") + ), + ], + fallback_prior=CopySampler(feature_to_copy="value_fallback"), + ) + + out = prior.sample(context, len(context), np.random.default_rng(123)) + expected = np.array([100, 301, 202, 303, 204, 105, 306]) + _require(np.array_equal(out, expected), f"Unexpected output: {out}") + + calls = _track_copy_sampler_calls(prior, context) + call_a1 = _single_call(calls, "value_for_a1") + call_b2 = _single_call(calls, "value_for_b2") + call_fallback = _single_call(calls, "value_fallback") + + _require( + call_a1["n"] == 2 and call_a1["rows"] == 2, + "Expected (A,1) sampler to get 2 rows", + ) + _require( + call_b2["n"] == 2 and call_b2["rows"] == 2, + "Expected (B,2) sampler to get 2 rows", + ) + _require( + call_fallback["n"] == 3 and call_fallback["rows"] == 3, + "Expected multi fallback sampler to get 3 rows", + ) + _require( + set(zip(call_a1["kind_values"], call_a1["stage_values"], strict=True)) + == {("A", 1)}, + "(A,1) sampler saw unexpected tuples", + ) + _require( + set(zip(call_b2["kind_values"], call_b2["stage_values"], strict=True)) + == {("B", 2)}, + "(B,2) sampler saw unexpected tuples", + ) + _require( + set( + zip( + call_fallback["kind_values"], call_fallback["stage_values"], strict=True + ) + ) + == {("A", 2), ("C", 3), ("B", 1)}, + "Multi fallback sampler saw unexpected tuples", + ) + + +def test_unmatched_without_fallback_raises() -> None: + """Both conditional prior variants raise when unmatched rows remain.""" + context = pd.DataFrame({ + "kind": ["A", "C"], + "stage": [1, 2], + "value_for_a": [1, 2], + }) + + single = ConditionalPrior( + source_feature="kind", + conditions=[ + ConditionalPriorCondition( + match_val="A", sampler=CopySampler(feature_to_copy="value_for_a") + ) + ], + fallback_prior=None, + ) + multi = MultiColumnConditionalPrior( + source_features=["kind", "stage"], + conditions=[ + MultiColumnCondition( + match_vals=("A", 1), sampler=CopySampler(feature_to_copy="value_for_a") + ) + ], + fallback_prior=None, + ) + + try: + single.sample(context, len(context), np.random.default_rng(7)) + msg = "Expected ConditionalPrior to raise SamplingError." + raise AssertionError(msg) + except SamplingError: + pass + + try: + multi.sample(context, len(context), np.random.default_rng(7)) + msg = "Expected MultiColumnConditionalPrior to raise SamplingError." + raise AssertionError(msg) + except SamplingError: + pass + + +def main() -> None: + """Run all smoke tests.""" + test_conditional_prior_subset_sampling() + print("PASS: ConditionalPrior subset sampling") + + test_multicolumn_conditional_prior_subset_sampling() + print("PASS: MultiColumnConditionalPrior subset sampling") + + test_unmatched_without_fallback_raises() + print("PASS: Unmatched rows raise without fallback") + + print("All conditional prior smoke tests passed.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_gis_scenario_override.py b/tests/test_gis_scenario_override.py new file mode 100644 index 0000000..181360e --- /dev/null +++ b/tests/test_gis_scenario_override.py @@ -0,0 +1,30 @@ +"""Regression tests for GIS scenario overrides.""" + +from pathlib import Path + +from globi.models.configs import DeterministicGISPreprocessorConfig, FileConfig +from globi.pipelines.gis import preprocess_gis_file + +DATA_DIR = Path(__file__).parent / "data" / "e2e" + + +def test_preprocess_scenario_override_uses_semantic_field_casing() -> None: + """Scenario overrides are written to uppercase semantic fields.""" + file_config = FileConfig( + gis_file=DATA_DIR / "buildings.geojson", + db_file=DATA_DIR / "components-lib.db", + semantic_fields_file=DATA_DIR / "semantic-fields.yml", + epwzip_file="https://example.com/weather.zip", + component_map_file=DATA_DIR / "component-map.yml", + ) + config = DeterministicGISPreprocessorConfig.from_(DATA_DIR / "gis-preprocessor.yml") + + gdf, colmap = preprocess_gis_file( + config, + file_config, + scenario="Retrofit", + ) + + contexts = gdf[colmap.Semantic_Field_Context_col] + assert set(gdf["Scenario"].unique()) == {"Retrofit"} + assert {context["Scenario"] for context in contexts} == {"Retrofit"} diff --git a/tests/test_surrogate_transforms.py b/tests/test_surrogate_transforms.py new file mode 100644 index 0000000..576f1c1 --- /dev/null +++ b/tests/test_surrogate_transforms.py @@ -0,0 +1,116 @@ +"""Tests for surrogate feature transforms.""" + +import pandas as pd + +from globi.models.surrogate.transforms import ( + IdentityScaler, + MinMaxScaler, + StandardScaler, + Transformers, + XTransformer, + YTransformer, + encode_inputs, +) + + +def _base_input_frame() -> pd.DataFrame: + return pd.DataFrame({ + "cont_a": [1.0, 2.0, 4.0], + "cont_b": [10.0, 20.0, 40.0], + "cat": ["x", "y", "x"], + }) + + +def test_encode_inputs_with_min_max_continuous_encoding() -> None: + """Continuous features are fit on train and transformed to [0, 1].""" + x = _base_input_frame() + conf = XTransformer( + features=["cont_a", "cont_b", "cat"], + continuous_features=["cont_a", "cont_b"], + cont_scaler=MinMaxScaler(), + cont_encoding="min-max", + cat_map={"cat": ["x", "y"]}, + cat_encoding="index", + ) + + encoded = encode_inputs(x, conf=conf, fit_continuous=True) + + # continuous ranges should be min-max normalized on train-fit + assert encoded["cont_a"].tolist() == [0.0, 1.0 / 3.0, 1.0] + assert encoded["cont_b"].tolist() == [0.0, 1.0 / 3.0, 1.0] + # categorical feature still encoded + assert encoded["cat"].tolist() == [0, 1, 0] + assert conf.cont_scaler.mins_ == {"cont_a": 1.0, "cont_b": 10.0} + assert conf.cont_scaler.maxs_ == {"cont_a": 4.0, "cont_b": 40.0} + + +def test_encode_inputs_with_standard_continuous_encoding() -> None: + """Continuous features can be standardized with standard mode.""" + x = _base_input_frame() + conf = XTransformer( + features=["cont_a", "cont_b", "cat"], + continuous_features=["cont_a", "cont_b"], + cont_scaler=StandardScaler(), + cont_encoding="standard", + cat_map={"cat": ["x", "y"]}, + cat_encoding="index", + ) + + encoded = encode_inputs(x, conf=conf, fit_continuous=True) + # means are approximately zero after fitting/transforming same frame + assert abs(float(encoded["cont_a"].mean())) < 1e-12 + assert abs(float(encoded["cont_b"].mean())) < 1e-12 + assert encoded["cat"].tolist() == [0, 1, 0] + assert conf.cont_scaler.means_ == {"cont_a": 7.0 / 3.0, "cont_b": 70.0 / 3.0} + + +def test_encode_inputs_applies_fitted_continuous_scaler_to_new_data() -> None: + """Test frames should use previously fit scaler parameters.""" + train = _base_input_frame() + test = pd.DataFrame({ + "cont_a": [1.0, 2.5, 4.0], + "cont_b": [10.0, 25.0, 40.0], + "cat": ["y", "x", "y"], + }) + conf = XTransformer( + features=["cont_a", "cont_b", "cat"], + continuous_features=["cont_a", "cont_b"], + cont_scaler=MinMaxScaler(), + cont_encoding="min-max", + cat_map={"cat": ["x", "y"]}, + cat_encoding="index", + ) + + _ = encode_inputs(train, conf=conf, fit_continuous=True) + encoded_test = encode_inputs(test, conf=conf) + + assert encoded_test["cont_a"].tolist() == [0.0, 0.5, 1.0] + assert encoded_test["cont_b"].tolist() == [0.0, 0.5, 1.0] + assert encoded_test["cat"].tolist() == [1, 0, 1] + + +def test_transformers_model_dump_and_validate_keeps_cont_transformer() -> None: + """Continuous feature transformer is serialized in transforms payload.""" + transformers = Transformers( + x=XTransformer( + features=["cont_a", "cat"], + continuous_features=["cont_a"], + cont_scaler=MinMaxScaler(mins_={"cont_a": 1.0}, maxs_={"cont_a": 5.0}), + cont_encoding="min-max", + cat_map={"cat": ["x", "y"]}, + cat_encoding="one-hot", + ), + y=YTransformer( + scaler=IdentityScaler(), + targets=["target"], + normalization=None, + ), + ) + + dumped = transformers.model_dump(mode="json") + roundtrip = Transformers.model_validate(dumped) + + assert roundtrip.x.cont_encoding == "min-max" + assert isinstance(roundtrip.x.cont_scaler, MinMaxScaler) + assert roundtrip.x.cont_scaler.mins_ == {"cont_a": 1.0} + assert roundtrip.x.cont_scaler.maxs_ == {"cont_a": 5.0} diff --git a/uv.lock b/uv.lock index 73b2575..2a55a66 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,24 @@ version = 1 revision = 3 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version == '3.13.*'", - "python_full_version < '3.13'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version < '3.13' and sys_platform == 'darwin'", +] + +[[package]] +name = "absl-py" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4", size = 116543, upload-time = "2026-01-28T10:17:05.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, ] [[package]] @@ -148,6 +163,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } + [[package]] name = "anyio" version = "4.11.0" @@ -899,6 +920,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/c6/c71e82e041c95ffe6a92ac707785500aa2a515a4339c2c7dd67e3c449249/cramjam-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:028400d699442d40dbda02f74158c73d05cb76587a12490d0bfedd958fd49188", size = 1713108, upload-time = "2025-07-27T21:24:10.147Z" }, ] +[[package]] +name = "cuda-bindings" +version = "12.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c2/65bfd79292b8ff18be4dd7f7442cea37bcbc1a228c1886f1dea515c45b67/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:694ba35023846625ef471257e6b5a4bc8af690f961d197d77d34b1d1db393f56", size = 11760260, upload-time = "2025-10-21T14:51:40.79Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c1/dabe88f52c3e3760d861401bb994df08f672ec893b8f7592dc91626adcf3/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fda147a344e8eaeca0c6ff113d2851ffca8f7dfc0a6c932374ee5c47caa649c8", size = 12151019, upload-time = "2025-10-21T14:51:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/05/8b/b4b2d1c7775fa403b64333e720cfcfccef8dcb9cdeb99947061ca5a77628/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf8bfaedc238f3b115d957d1fd6562b7e8435ba57f6d0e2f87d0e7149ccb2da5", size = 11570071, upload-time = "2025-10-21T14:51:47.472Z" }, + { url = "https://files.pythonhosted.org/packages/63/56/e465c31dc9111be3441a9ba7df1941fe98f4aa6e71e8788a3fb4534ce24d/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:32bdc5a76906be4c61eb98f546a6786c5773a881f3b166486449b5d141e4a39f", size = 11906628, upload-time = "2025-10-21T14:51:49.905Z" }, + { url = "https://files.pythonhosted.org/packages/ec/07/6aff13bc1e977e35aaa6b22f52b172e2890c608c6db22438cf7ed2bf43a6/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3adf4958dcf68ae7801a59b73fb00a8b37f8d0595060d66ceae111b1002de38d", size = 11566797, upload-time = "2025-10-21T14:51:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/1e6be415e37478070aeeee5884c2022713c1ecc735e6d82d744de0252eee/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56e0043c457a99ac473ddc926fe0dc4046694d99caef633e92601ab52cbe17eb", size = 11925991, upload-time = "2025-10-21T14:51:56.535Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b5/96a6696e20c4ffd2b327f54c7d0fde2259bdb998d045c25d5dedbbe30290/cuda_bindings-12.9.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f53a7f453d4b2643d8663d036bafe29b5ba89eb904c133180f295df6dc151e5", size = 11624530, upload-time = "2025-10-21T14:52:01.539Z" }, + { url = "https://files.pythonhosted.org/packages/d1/af/6dfd8f2ed90b1d4719bc053ff8940e494640fe4212dc3dd72f383e4992da/cuda_bindings-12.9.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b72ee72a9cc1b531db31eebaaee5c69a8ec3500e32c6933f2d3b15297b53686", size = 11922703, upload-time = "2025-10-21T14:52:03.585Z" }, + { url = "https://files.pythonhosted.org/packages/39/73/d2fc40c043bac699c3880bf88d3cebe9d88410cd043795382826c93a89f0/cuda_bindings-12.9.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20f2699d61d724de3eb3f3369d57e2b245f93085cab44fd37c3bea036cea1a6f", size = 11565056, upload-time = "2025-10-21T14:52:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/6c/19/90ac264acc00f6df8a49378eedec9fd2db3061bf9263bf9f39fd3d8377c3/cuda_bindings-12.9.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d80bffc357df9988dca279734bc9674c3934a654cab10cadeed27ce17d8635ee", size = 11924658, upload-time = "2025-10-21T14:52:10.411Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/02/59a5bc738a09def0b49aea0e460bdf97f65206d0d041246147cf6207e69c/cuda_pathfinder-1.4.1-py3-none-any.whl", hash = "sha256:40793006082de88e0950753655e55558a446bed9a7d9d0bcb48b2506d50ed82a", size = 43903, upload-time = "2026-03-06T21:05:24.372Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -956,6 +1005,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "einops" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, +] + [[package]] name = "energy-pandas" version = "0.4.1" @@ -977,8 +1035,8 @@ wheels = [ [[package]] name = "epinterface" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } +version = "1.5.0" +source = { git = "https://github.com/szvsw/epinterface?branch=feature%2Fonezone-converter#792434ef3cb8bcd160ce2522d57dbd8ae28611e8" } dependencies = [ { name = "archetypal" }, { name = "click" }, @@ -992,10 +1050,6 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pythermalcomfort" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b8/45/163ac4e8fcd92d94cb223fad8a66a17b7f408e1c0964ab8ff49af500f1b8/epinterface-1.4.0.tar.gz", hash = "sha256:4c880d917cc0aed3cb49cdfcee1f255bd6bfcaeab381a903d48310af6499cd06", size = 5981334, upload-time = "2026-03-06T14:55:08.308Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/79/84f389e491fff2e701de506f1034c81f41b7c28cea2d8ab6f49e172e2c2c/epinterface-1.4.0-py3-none-any.whl", hash = "sha256:6fabc96b9fa6cfb6d3609c5a06ae6071e5ad3a3519123a3f3f29dd991178cb4f", size = 883207, upload-time = "2026-03-06T14:55:06.55Z" }, -] [[package]] name = "eppy" @@ -1292,6 +1346,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, ] +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + [[package]] name = "future" version = "1.0.0" @@ -1379,6 +1438,16 @@ cli = [ { name = "click" }, { name = "xlsxwriter" }, ] +ml-gpu = [ + { name = "lightgbm" }, + { name = "numba" }, + { name = "pytorch-tabular" }, + { name = "tensorboard" }, + { name = "torch", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin'" }, + { name = "wandb" }, + { name = "xgboost" }, +] visualization = [ { name = "bokeh" }, { name = "folium" }, @@ -1416,28 +1485,36 @@ requires-dist = [ { name = "bokeh", marker = "extra == 'visualization'", specifier = ">=3.8.0" }, { name = "boto3", specifier = ">=1.40.49" }, { name = "click", marker = "extra == 'cli'", specifier = ">=8.1.7" }, - { name = "epinterface", specifier = ">=1.4.0" }, + { name = "epinterface", git = "https://github.com/szvsw/epinterface?branch=feature%2Fonezone-converter" }, { name = "fiona", specifier = ">=1.9.5" }, { name = "folium", marker = "extra == 'visualization'", specifier = ">=0.15.0" }, { name = "geopandas", specifier = ">=0.14.0" }, { name = "ladybug-core", specifier = ">=0.44.29" }, + { name = "lightgbm", marker = "extra == 'ml-gpu'", specifier = ">=4.6.0" }, { name = "matplotlib", marker = "extra == 'visualization'", specifier = ">=3.8.0" }, + { name = "numba", marker = "extra == 'ml-gpu'", specifier = ">=0.63.1" }, { name = "numpy", specifier = ">=1.26.0" }, { name = "pandas", specifier = ">=2.1.0" }, { name = "playwright", marker = "extra == 'visualization'", specifier = ">=1.40.0" }, { name = "plotly", marker = "extra == 'visualization'", specifier = ">=5.18.0" }, { name = "pydantic", specifier = ">=2.11,<3" }, { name = "pyproj", specifier = ">=3.6.0" }, + { name = "pytorch-tabular", marker = "extra == 'ml-gpu'", specifier = ">=1.2.0" }, { name = "rasterio", marker = "extra == 'visualization'", specifier = ">=1.3.9" }, { name = "scikit-learn", specifier = ">=1.3.0" }, { name = "scipy", specifier = ">=1.11.0,<1.15" }, - { name = "scythe-engine", specifier = ">=0.1.2" }, + { name = "scythe-engine", git = "https://github.com/szvsw/scythe?branch=feature%2Fto-threads" }, { name = "seaborn", marker = "extra == 'visualization'", specifier = ">=0.13.0" }, { name = "shapely", specifier = ">=2.0.0" }, { name = "streamlit", marker = "extra == 'visualization'", specifier = ">=1.28.0" }, + { name = "tensorboard", marker = "extra == 'ml-gpu'", specifier = ">=2.20.0" }, + { name = "torch", marker = "sys_platform == 'darwin' and extra == 'ml-gpu'", specifier = ">=2.5.0", index = "https://pypi.org/simple", conflict = { package = "globi", extra = "ml-gpu" } }, + { name = "torch", marker = "sys_platform != 'darwin' and extra == 'ml-gpu'", specifier = ">=2.5.0", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "globi", extra = "ml-gpu" } }, + { name = "wandb", marker = "extra == 'ml-gpu'", specifier = ">=0.25.0" }, + { name = "xgboost", marker = "extra == 'ml-gpu'", specifier = ">=3.2.0" }, { name = "xlsxwriter", marker = "extra == 'cli'", specifier = ">=3.2.9" }, ] -provides-extras = ["visualization", "cli"] +provides-extras = ["visualization", "ml-gpu", "cli"] [package.metadata.requires-dev] dev = [ @@ -1976,7 +2053,7 @@ dependencies = [ { name = "nbformat" }, { name = "packaging" }, { name = "prometheus-client" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux'" }, { name = "pyzmq" }, { name = "send2trash" }, { name = "terminado" }, @@ -1994,7 +2071,7 @@ name = "jupyter-server-terminals" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux'" }, { name = "terminado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } @@ -2156,6 +2233,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/52/7b8421a8ace22a17ae77dd9a8367e916364ed8be72502cb744805f06d6ac/ladybug_geometry-1.34.14-py3-none-any.whl", hash = "sha256:af91ee9285333ca1ddfaf439530306dff7f0a891cae40d4dc5491f139fcf7d36", size = 198221, upload-time = "2025-11-07T04:16:46.986Z" }, ] +[[package]] +name = "lightgbm" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/0b/a2e9f5c5da7ef047cc60cef37f86185088845e8433e54d2e7ed439cce8a3/lightgbm-4.6.0.tar.gz", hash = "sha256:cb1c59720eb569389c0ba74d14f52351b573af489f230032a1c9f314f8bab7fe", size = 1703705, upload-time = "2025-02-15T04:03:03.111Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/75/cffc9962cca296bc5536896b7e65b4a7cdeb8db208e71b9c0133c08f8f7e/lightgbm-4.6.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:b7a393de8a334d5c8e490df91270f0763f83f959574d504c7ccb9eee4aef70ed", size = 2010151, upload-time = "2025-02-15T04:02:50.961Z" }, + { url = "https://files.pythonhosted.org/packages/21/1b/550ee378512b78847930f5d74228ca1fdba2a7fbdeaac9aeccc085b0e257/lightgbm-4.6.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:2dafd98d4e02b844ceb0b61450a660681076b1ea6c7adb8c566dfd66832aafad", size = 1592172, upload-time = "2025-02-15T04:02:53.937Z" }, + { url = "https://files.pythonhosted.org/packages/64/41/4fbde2c3d29e25ee7c41d87df2f2e5eda65b431ee154d4d462c31041846c/lightgbm-4.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4d68712bbd2b57a0b14390cbf9376c1d5ed773fa2e71e099cac588703b590336", size = 3454567, upload-time = "2025-02-15T04:02:56.443Z" }, + { url = "https://files.pythonhosted.org/packages/42/86/dabda8fbcb1b00bcfb0003c3776e8ade1aa7b413dff0a2c08f457dace22f/lightgbm-4.6.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cb19b5afea55b5b61cbb2131095f50538bd608a00655f23ad5d25ae3e3bf1c8d", size = 3569831, upload-time = "2025-02-15T04:02:58.925Z" }, + { url = "https://files.pythonhosted.org/packages/5e/23/f8b28ca248bb629b9e08f877dd2965d1994e1674a03d67cd10c5246da248/lightgbm-4.6.0-py3-none-win_amd64.whl", hash = "sha256:37089ee95664b6550a7189d887dbf098e3eadab03537e411f52c63c121e3ba4b", size = 1451509, upload-time = "2025-02-15T04:03:01.515Z" }, +] + +[[package]] +name = "lightning-utilities" +version = "0.15.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/45/7fa8f56b17dc0f0a41ec70dd307ecd6787254483549843bef4c30ab5adce/lightning_utilities-0.15.3.tar.gz", hash = "sha256:792ae0204c79f6859721ac7f386c237a33b0ed06ba775009cb894e010a842033", size = 33553, upload-time = "2026-02-22T14:48:53.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl", hash = "sha256:6c55f1bee70084a1cbeaa41ada96e4b3a0fea5909e844dd335bd80f5a73c5f91", size = 31906, upload-time = "2026-02-22T14:48:52.488Z" }, +] + [[package]] name = "littleutils" version = "0.2.4" @@ -2236,6 +2343,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/1b/6ef961f543593969d25b2afe57a3564200280528caa9bd1082eecdd7b3bc/markdown-3.10.1-py3-none-any.whl", hash = "sha256:867d788939fe33e4b736426f5b9f651ad0c0ae0ecf89df0ca5d1176c70812fe3", size = 107684, upload-time = "2026-01-21T18:09:27.203Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -2365,6 +2484,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mergedeep" version = "1.3.4" @@ -2515,6 +2643,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/5c/2597cef67b6947b15c47f8dba967a0baf19fbdfdc86f6e4a8ba7af8b581a/mkdocstrings_python-1.19.0-py3-none-any.whl", hash = "sha256:395c1032af8f005234170575cc0c5d4d20980846623b623b35594281be4a3059", size = 143417, upload-time = "2025-11-10T13:30:54.164Z" }, ] +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + [[package]] name = "msgpack" version = "1.1.2" @@ -2992,6 +3129,168 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, ] +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/1c/857979db0ef194ca5e21478a0612bcdbbe59458d7694361882279947b349/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a", size = 322400625, upload-time = "2025-06-26T04:11:04.496Z" }, + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/6a/03aa43cc9bd3ad91553a88b5f6fb25ed6a3752ae86ce2180221962bc2aa5/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b48363fc6964dede448029434c6abed6c5e37f823cb43c3bcde7ecfc0457e15", size = 138936938, upload-time = "2025-09-06T00:32:05.589Z" }, + { url = "https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:042f2500f24c021db8a06c5eec2539027d57460e1c1a762055a6554f72c369bd", size = 139103095, upload-time = "2025-09-06T00:32:31.266Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, +] + +[[package]] +name = "omegaconf" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, +] + [[package]] name = "openpyxl" version = "3.1.5" @@ -3867,6 +4166,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/e5/fecf13f06e5e5f67e8837d777d1bc43fac0ed2b77a676804df5c34744727/python_json_logger-4.0.0-py3-none-any.whl", hash = "sha256:af09c9daf6a813aa4cc7180395f50f2a9e5fa056034c9953aec92e381c5ba1e2", size = 15548, upload-time = "2025-10-06T04:15:17.553Z" }, ] +[[package]] +name = "pytorch-lightning" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fsspec", extra = ["http"] }, + { name = "lightning-utilities" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "torch", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin'" }, + { name = "torchmetrics" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/ac/ebd5f6f58691cbd4f73836e43e1727f3814311b960c41f88e259606ca2b2/pytorch_lightning-2.6.1.tar.gz", hash = "sha256:ba08f8901cf226fcca473046ad9346f414e99117762dc869c76e650d5b3d7bdc", size = 665563, upload-time = "2026-01-30T14:59:11.636Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl", hash = "sha256:1f8118567ec829e3055f16cf1aa320883a86a47c836951bfd9dcfa34ec7ffd59", size = 857273, upload-time = "2026-01-30T14:59:10.141Z" }, +] + +[[package]] +name = "pytorch-tabular" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, + { name = "numpy" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "pytorch-lightning" }, + { name = "rich" }, + { name = "scikit-base" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "torch", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin'" }, + { name = "torchmetrics" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/f2/823de16d6a461504f4ed8e4a555d6ce356e5f81e6525d95e2b64895ec94f/pytorch_tabular-1.2.0.tar.gz", hash = "sha256:1b96b576eb3de443840b313d0b298293eaf83dcfdbba53ed8974b76d1351b821", size = 2312825, upload-time = "2026-01-26T21:48:22.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/c9/1e01c682e2ad7132bc1943d8d367c96f241bf85679e76d66eb0c4e4cbde9/pytorch_tabular-1.2.0-py3-none-any.whl", hash = "sha256:0a59f8a2304856b3d1e905f7b66153ebc65df1a6a017f2c8a13a29f62dc95b26", size = 165800, upload-time = "2026-01-26T21:48:21.195Z" }, +] + [[package]] name = "pytz" version = "2025.2" @@ -4045,6 +4387,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + [[package]] name = "rpds-py" version = "0.28.0" @@ -4164,6 +4519,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] +[[package]] +name = "scikit-base" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/a8/610f99f01f326178b8a7347db2ede654b42548e9697b516480cc081e344d/scikit_base-0.13.1.tar.gz", hash = "sha256:169e5427233f7237b38c7d858bf07b8a86bbf59feccf0708e26dad4ac312c593", size = 134482, upload-time = "2026-01-25T11:31:38.814Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/55/c20d8319aab037e11f1d6403b6102d1041694abe24a3aa4a1e27f2cdb9f2/scikit_base-0.13.1-py3-none-any.whl", hash = "sha256:1aca86759435fd2d32d83a526ce11095119c0745e4e5dd91f2e5820023ca8e39", size = 159779, upload-time = "2026-01-25T11:31:36.759Z" }, +] + [[package]] name = "scikit-learn" version = "1.7.2" @@ -4227,8 +4591,8 @@ wheels = [ [[package]] name = "scythe-engine" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } +version = "1.3.1" +source = { git = "https://github.com/szvsw/scythe?branch=feature%2Fto-threads#97312183ceaf8b4c5229b7c2cb41ec8fedd89bdc" } dependencies = [ { name = "boto3" }, { name = "fastparquet" }, @@ -4241,10 +4605,6 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/00/061a3e1301b03b3b1c6817ea5db19cc62c0448c02c504e391d49273451c2/scythe_engine-0.1.2.tar.gz", hash = "sha256:a53c49a8a8700f1dfd7a61f4868898289c1d3751b42ca767369faf7a3c08dc5e", size = 225628, upload-time = "2026-02-12T15:53:02.416Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/69/1cfac0fe0aa049d335f2ff6a3aeef32cc7893551ffe831e4d78ccde50b7b/scythe_engine-0.1.2-py3-none-any.whl", hash = "sha256:b2dd6924c0b26a1dfe9a68e9f6b028b77a944263849c82d41a28e635baf899d8", size = 33195, upload-time = "2026-02-12T15:53:00.827Z" }, -] [[package]] name = "seaborn" @@ -4269,6 +4629,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/78/504fdd027da3b84ff1aecd9f6957e65f35134534ccc6da8628eb71e76d3f/send2trash-2.1.0-py3-none-any.whl", hash = "sha256:0da2f112e6d6bb22de6aa6daa7e144831a4febf2a87261451c4ad849fe9a873c", size = 17610, upload-time = "2026-01-14T06:27:35.218Z" }, ] +[[package]] +name = "sentry-sdk" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/e9/2e3a46c304e7fa21eaa70612f60354e32699c7102eb961f67448e222ad7c/sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b", size = 413813, upload-time = "2026-03-02T15:12:41.355Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de", size = 439198, upload-time = "2026-03-02T15:12:39.546Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" @@ -4429,6 +4802,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/60/868371b6482ccd9ef423c6f62650066cf8271fdb2ee84f192695ad6b7a96/streamlit-1.51.0-py3-none-any.whl", hash = "sha256:4008b029f71401ce54946bb09a6a3e36f4f7652cbb48db701224557738cfda38", size = 10171702, upload-time = "2025-10-29T17:07:35.97Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + [[package]] name = "tables" version = "3.10.2" @@ -4473,13 +4858,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, ] +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools" }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598, upload-time = "2023-10-23T21:23:33.714Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, +] + [[package]] name = "terminado" version = "0.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ptyprocess", marker = "os_name != 'nt'" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux'" }, { name = "tornado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } @@ -4573,6 +4988,106 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, ] +[[package]] +name = "torch" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version < '3.13' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "filelock", marker = "sys_platform == 'darwin'" }, + { name = "fsspec", marker = "sys_platform == 'darwin'" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "networkx", marker = "sys_platform == 'darwin'" }, + { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "sympy", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, + { url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" }, + { url = "https://files.pythonhosted.org/packages/c9/5c/dee910b87c4d5c0fcb41b50839ae04df87c1cfc663cf1b5fca7ea565eeaa/torch-2.10.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:6d3707a61863d1c4d6ebba7be4ca320f42b869ee657e9b2c21c736bf17000294", size = 79498198, upload-time = "2026-01-21T16:24:34.704Z" }, + { url = "https://files.pythonhosted.org/packages/1a/0b/39929b148f4824bc3ad6f9f72a29d4ad865bcf7ebfc2fa67584773e083d2/torch-2.10.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:3202429f58309b9fa96a614885eace4b7995729f44beb54d3e4a47773649d382", size = 79851305, upload-time = "2026-01-21T16:24:09.209Z" }, + { url = "https://files.pythonhosted.org/packages/0e/13/e76b4d9c160e89fff48bf16b449ea324bda84745d2ab30294c37c2434c0d/torch-2.10.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:cdf2a523d699b70d613243211ecaac14fe9c5df8a0b0a9c02add60fb2a413e0f", size = 79498248, upload-time = "2026-01-21T16:23:09.315Z" }, + { url = "https://files.pythonhosted.org/packages/4f/93/716b5ac0155f1be70ed81bacc21269c3ece8dba0c249b9994094110bfc51/torch-2.10.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:bf0d9ff448b0218e0433aeb198805192346c4fd659c852370d5cc245f602a06a", size = 79464992, upload-time = "2026-01-21T16:23:05.162Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/71994e7d0d5238393df9732fdab607e37e2b56d26a746cb59fdb415f8966/torch-2.10.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f5ab4ba32383061be0fb74bda772d470140a12c1c3b58a0cfbf3dae94d164c28", size = 79850324, upload-time = "2026-01-21T16:22:09.494Z" }, +] + +[[package]] +name = "torch" +version = "2.10.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "filelock", marker = "sys_platform != 'darwin'" }, + { name = "fsspec", marker = "sys_platform != 'darwin'" }, + { name = "jinja2", marker = "sys_platform != 'darwin'" }, + { name = "networkx", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, + { name = "sympy", marker = "sys_platform != 'darwin'" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6f09cdf2415516be028ae82e6b985bcfc3eac37bc52ab401142689f6224516ca", upload-time = "2026-01-21T15:22:03Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:628e89bd5110ced7debee2a57c69959725b7fbc64eab81a39dd70e46c7e28ba5", upload-time = "2026-01-21T15:22:11Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:fbde8f6a9ec8c76979a0d14df21c10b9e5cab6f0d106a73ca73e2179bc597cae", upload-time = "2026-01-21T15:22:17Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:bdbcc703382f948e951c063448c9406bf38ce66c41dd698d9e2733fcf96c037a", upload-time = "2026-01-21T15:22:29Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:7b4bd23ed63de97456fcc81c26fea9f02ee02ce1112111c4dac0d8cfe574b23e", upload-time = "2026-01-21T15:22:51Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313-win_amd64.whl", hash = "sha256:4d1b0b49c54223c7c04050b49eac141d77b6edbc34aea1dfc74a6fdb661baa8c", upload-time = "2026-01-21T15:22:54Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:f1f8b840c64b645a4bc61a393db48effb9c92b2dc26c8373873911f0750d1ea7", upload-time = "2026-01-21T15:23:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:23f58258012bcf1c349cb22af387e33aadca7f83ea617b080e774eb41e4fe8ff", upload-time = "2026-01-21T15:23:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp313-cp313t-win_amd64.whl", hash = "sha256:01b216e097b17a5277cfb47c383cdcacf06abeadcb0daca0c76b59e72854c3b6", upload-time = "2026-01-21T15:23:53Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c42377bc2607e3e1c60da71b792fb507c3938c87fd6edab8b21c59c91473c36d", upload-time = "2026-01-21T15:23:56Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:37d71feea068776855686a1512058df3f19f6f040a151f055aa746601678744f", upload-time = "2026-01-21T15:24:08Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314-win_amd64.whl", hash = "sha256:c57017ca29e62271e362fdeee7d20070e254755a5148b30b553d8a10fc83c7ef", upload-time = "2026-01-21T15:24:10Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:777461f50b2daf77e4bdd8e2ad34bdfc5a993bf1bdf2ab9ef39f5edfe4e9c12b", upload-time = "2026-01-21T15:24:20Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7bcba6a7c5f0987a13298b1ca843155dcceceac758fa3c7ccd5c7af4059a1080", upload-time = "2026-01-21T15:24:44Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp314-cp314t-win_amd64.whl", hash = "sha256:70d89143c956389d4806cb4e5fe0b1129fe0db280e1073288d17fa76c101cba4", upload-time = "2026-01-21T15:24:46Z" }, +] + +[[package]] +name = "torchmetrics" +version = "1.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lightning-utilities" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "torch", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/2e/48a887a59ecc4a10ce9e8b35b3e3c5cef29d902c4eac143378526e7485cb/torchmetrics-1.8.2.tar.gz", hash = "sha256:cf64a901036bf107f17a524009eea7781c9c5315d130713aeca5747a686fe7a5", size = 580679, upload-time = "2025-09-03T14:00:54.077Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl", hash = "sha256:08382fd96b923e39e904c4d570f3d49e2cc71ccabd2a94e0f895d1f0dac86242", size = 983161, upload-time = "2025-09-03T14:00:51.921Z" }, +] + [[package]] name = "tornado" version = "6.5.2" @@ -4625,6 +5140,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/7a/f38385f1b2d5f54221baf1db3d6371dc6eef8041d95abff39576c694e9d9/transforms3d-0.4.2-py3-none-any.whl", hash = "sha256:1c70399d9e9473ecc23311fd947f727f7c69ed0b063244828c383aa1aefa5941", size = 1376759, upload-time = "2024-06-20T11:09:19.43Z" }, ] +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, + { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, + { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, +] + [[package]] name = "tsam" version = "2.3.9" @@ -4726,6 +5258,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] +[[package]] +name = "wandb" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/60/d94952549920469524b689479c864c692ca47eca4b8c2fe3389b64a58778/wandb-0.25.0.tar.gz", hash = "sha256:45840495a288e34245d69d07b5a0b449220fbc5b032e6b51c4f92ec9026d2ad1", size = 43951335, upload-time = "2026-02-13T00:17:45.515Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/7d/0c131db3ec9deaabbd32263d90863cbfbe07659527e11c35a5c738cecdc5/wandb-0.25.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:5eecb3c7b5e60d1acfa4b056bfbaa0b79a482566a9db58c9f99724b3862bc8e5", size = 23287536, upload-time = "2026-02-13T00:17:20.265Z" }, + { url = "https://files.pythonhosted.org/packages/c3/95/31bb7f76a966ec87495e5a72ac7570685be162494c41757ac871768dbc4f/wandb-0.25.0-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:daeedaadb183dc466e634fba90ab2bab1d4e93000912be0dee95065a0624a3fd", size = 25196062, upload-time = "2026-02-13T00:17:23.356Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/258cdedbf30cebc692198a774cf0ef945b7ed98ee64bdaf62621281c95d8/wandb-0.25.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:5e0127dbcef13eea48f4b84268da7004d34d3120ebc7b2fa9cefb72b49dbb825", size = 22799744, upload-time = "2026-02-13T00:17:26.437Z" }, + { url = "https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6c4c38077836f9b7569a35b0e1dcf1f0c43616fcd936d182f475edbfea063665", size = 25262839, upload-time = "2026-02-13T00:17:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/cb2d1c7143f534544147fb53fe87944508b8cb9a058bc5b6f8a94adbee15/wandb-0.25.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6edd8948d305cb73745bf564b807bd73da2ccbd47c548196b8a362f7df40aed8", size = 22853714, upload-time = "2026-02-13T00:17:31.68Z" }, + { url = "https://files.pythonhosted.org/packages/d7/94/68163f70c1669edcf130822aaaea782d8198b5df74443eca0085ec596774/wandb-0.25.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ada6f08629bb014ad6e0a19d5dec478cdaa116431baa3f0a4bf4ab8d9893611f", size = 25358037, upload-time = "2026-02-13T00:17:34.676Z" }, + { url = "https://files.pythonhosted.org/packages/cc/fb/9578eed2c01b2fc6c8b693da110aa9c73a33d7bb556480f5cfc42e48c94e/wandb-0.25.0-py3-none-win32.whl", hash = "sha256:020b42ca4d76e347709d65f59b30d4623a115edc28f462af1c92681cb17eae7c", size = 24604118, upload-time = "2026-02-13T00:17:37.641Z" }, + { url = "https://files.pythonhosted.org/packages/25/97/460f6cb738aaa39b4eb2e6b4c630b2ae4321cdd70a79d5955ea75a878981/wandb-0.25.0-py3-none-win_amd64.whl", hash = "sha256:78307ac0b328f2dc334c8607bec772851215584b62c439eb320c4af4fb077a00", size = 24604122, upload-time = "2026-02-13T00:17:39.991Z" }, + { url = "https://files.pythonhosted.org/packages/27/6c/5847b4dda1dfd52630dac08711d4348c69ed657f0698fc2d949c7f7a6622/wandb-0.25.0-py3-none-win_arm64.whl", hash = "sha256:c6174401fd6fb726295e98d57b4231c100eca96bd17de51bfc64038a57230aaf", size = 21785298, upload-time = "2026-02-13T00:17:42.475Z" }, +] + [[package]] name = "watchdog" version = "6.0.0" @@ -4777,6 +5338,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, ] +[[package]] +name = "werkzeug" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/f1/ee81806690a87dab5f5653c1f146c92bc066d7f4cebc603ef88eb9e13957/werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", size = 864736, upload-time = "2026-02-19T15:17:18.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, +] + [[package]] name = "widgetsnbextension" version = "4.0.15" @@ -4786,6 +5359,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, ] +[[package]] +name = "xgboost" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "nvidia-nccl-cu12", marker = "sys_platform == 'linux'" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/bb/1eb0242409d22db725d7a88088e6cfd6556829fb0736f9ff69aa9f1e9455/xgboost-3.2.0.tar.gz", hash = "sha256:99b0e9a2a64896cdaf509c5e46372d336c692406646d20f2af505003c0c5d70d", size = 1263936, upload-time = "2026-02-10T11:03:05.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/49/6e4cdd877c24adf56cb3586bc96d93d4dcd780b5ea1efb32e1ee0de08bae/xgboost-3.2.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:2f661966d3e322536d9c448090a870fcba1e32ee5760c10b7c46bac7a342079a", size = 2507014, upload-time = "2026-02-10T10:50:57.44Z" }, + { url = "https://files.pythonhosted.org/packages/93/f1/c09ef1add609453aa3ba5bafcd0d1c1a805c1263c0b60138ec968f8ec296/xgboost-3.2.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:eabbd40d474b8dbf6cb3536325f9150b9e6f0db32d18de9914fb3227d0bef5b7", size = 2328527, upload-time = "2026-02-10T10:51:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/96/9f/d9914a7b8df842832850b1a18e5f47aaa071c217cdd1da2ae9deb291018b/xgboost-3.2.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:852eabc6d3b3702a59bf78dbfdcd1cb9c4d3a3b6e5ed1f8781d8b9512354fdd2", size = 131100954, upload-time = "2026-02-10T11:02:42.704Z" }, + { url = "https://files.pythonhosted.org/packages/79/98/679de17c2caa4fd3b0b4386ecf7377301702cb0afb22930a07c142fcb1d8/xgboost-3.2.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:99b4a6bbcb47212fec5cf5fbe12347215f073c08967431b0122cfbd1ee70312c", size = 131748579, upload-time = "2026-02-10T10:54:40.424Z" }, + { url = "https://files.pythonhosted.org/packages/1f/3d/1661dd114a914a67e3f7ab66fa1382e7599c2a8c340f314ad30a3e2b4d08/xgboost-3.2.0-py3-none-win_amd64.whl", hash = "sha256:0d169736fd836fc13646c7ab787167b3a8110351c2c6bc770c755ee1618f0442", size = 101681668, upload-time = "2026-02-10T10:59:31.202Z" }, +] + [[package]] name = "xlsxwriter" version = "3.2.9"