From 60374de342bfa45889f90ad32ac698f8c7e2ab26 Mon Sep 17 00:00:00 2001 From: lglsj666 Date: Thu, 22 Oct 2026 00:46:12 -0400 Subject: [PATCH 01/15] Add RCA research page and update member profiles --- assets/css/main.css | 7 +- content/_index.md | 2 +- content/people/chenkai-zhang.md | 2 +- content/people/hans-arno-jacobsen.md | 2 +- content/people/mengli-duan.md | 2 +- content/people/michalis-bachras.md | 2 +- content/people/rev-liu.md | 12 +-- content/people/yifang-tian.md | 2 +- content/people/yifeng-he.md | 1 + content/people/yiran-li.md | 2 +- ...for-message-level-log-anomaly-detection.md | 2 +- ...c-workflows-elevate-root-cause-analysis.md | 2 +- ...-and-incident-response-in-microservices.md | 2 +- ...ogram-repair-for-microservice-incidents.md | 2 +- ...ts-with-high-fidelity-failure-scenarios.md | 2 +- ...ocalizing-runtime-faults-from-telemetry.md | 2 +- content/research/_index.md | 2 +- content/research/data-management/_index.md | 5 ++ .../distributed-machine-learning/_index.md | 4 + content/research/rca-aiops/_index.md | 82 ++++++++++++++++++ hugo.yaml | 6 +- layouts/index.html | 2 +- static/images/people/ruiwu-liu.jpg | Bin 0 -> 110588 bytes 23 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 content/research/rca-aiops/_index.md create mode 100644 static/images/people/ruiwu-liu.jpg diff --git a/assets/css/main.css b/assets/css/main.css index e79efbb..91dd9e4 100644 --- a/assets/css/main.css +++ b/assets/css/main.css @@ -12,6 +12,7 @@ --accent: #86a7d8; --copper: #9e6339; --sage: #54724b; + --teal: #23746e; --shadow: 0 18px 42px rgba(13, 31, 60, 0.08); --shadow-soft: 0 10px 24px rgba(13, 31, 60, 0.06); --radius: 1.35rem; @@ -1031,7 +1032,7 @@ a.chip { } .card-grid-wide { - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(2, minmax(0, 1fr)); } .accent-copper { @@ -1046,6 +1047,10 @@ a.chip { border-top: 4px solid var(--primary); } +.accent-teal { + border-top: 4px solid var(--teal); +} + .person-card { display: flex; flex-direction: column; diff --git a/content/_index.md b/content/_index.md index 2e80603..11c4639 100644 --- a/content/_index.md +++ b/content/_index.md @@ -1,7 +1,7 @@ +++ title = "Home" hero_subtitle = "The Middleware Systems Research Group at the University of Toronto" -hero_tagline = "Systems research that spans data, learning, and quantum computing" +hero_tagline = "Systems research that spans data, learning, quantum computing, and reliability" +++ Traditionally, the “M” in MSRG stood for “middleware”—the Middleware Systems Research Group at the University of Toronto. But don’t worry, “middleware is literally everywhere.” Over the years, that “M” has evolved to mean “mobile,” “microservices,” “machine learning,” and, last but not least, “matter,” as in entangled particles in space-time. What *more* could you ask for? diff --git a/content/people/chenkai-zhang.md b/content/people/chenkai-zhang.md index 17a2e48..ddb1d92 100644 --- a/content/people/chenkai-zhang.md +++ b/content/people/chenkai-zhang.md @@ -9,6 +9,6 @@ email = "zhangchenkai.zhang@mail.utoronto.ca" interests = ["AI Agent Reliability", "Fault Diagnosis"] author_names = ["Chenkai Zhang"] portrait = "" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] linkedin = "https://www.linkedin.com/in/chenkaizhang1/" +++ diff --git a/content/people/hans-arno-jacobsen.md b/content/people/hans-arno-jacobsen.md index e7e6a45..b1015bf 100644 --- a/content/people/hans-arno-jacobsen.md +++ b/content/people/hans-arno-jacobsen.md @@ -5,7 +5,7 @@ role = "Professor" status = "current" start_date = "2000-01" location = "Toronto, Canada" -research = ["quantum-computing-systems", "distributed-machine-learning", "data-management"] +research = ["quantum-computing-systems", "distributed-machine-learning", "data-management", "rca-aiops"] email = "jacobsen@eecg.utoronto.ca" homepage = "https://www.eecg.toronto.edu/~jacobsen" orcid = "https://orcid.org/0000-0003-0813-0101" diff --git a/content/people/mengli-duan.md b/content/people/mengli-duan.md index d6757d0..66a1de8 100644 --- a/content/people/mengli-duan.md +++ b/content/people/mengli-duan.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2026-09" location = "Toronto, Canada" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] email = "mengli.duan@mail.utoronto.ca" interests = ["Machine Learning Systems"] author_names = ["Mengli Duan", "Dawn (Mengli) Duan", "Dawn Duan"] diff --git a/content/people/michalis-bachras.md b/content/people/michalis-bachras.md index 689255b..25bac30 100644 --- a/content/people/michalis-bachras.md +++ b/content/people/michalis-bachras.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2023-09" location = "Toronto, Canada" -research = ["data-management"] +research = ["data-management", "rca-aiops"] interests = ["Data Management", "Distributed Systems"] portrait = "/images/people/michalis-bachras.jpg" linkedin = "https://www.linkedin.com/in/michalis-bachras/" diff --git a/content/people/rev-liu.md b/content/people/rev-liu.md index 57dda5d..6e41bf0 100644 --- a/content/people/rev-liu.md +++ b/content/people/rev-liu.md @@ -1,12 +1,14 @@ +++ -name = "Rev Liu" +name = "Ruiwu Liu" slug = "rev-liu" -role = "MSc Student" +role = "PhD Student" status = "current" -start_date = "2025-09" +start_date = "2026-09" location = "Toronto, Canada" +research = ["distributed-machine-learning", "rca-aiops"] email = "rev.liu@mail.utoronto.ca" -interests = [] +interests = ["VLM", "RL", "Artificial intelligence in IT operations"] author_names = [] -portrait = "" +portrait = "/images/people/ruiwu-liu.jpg" +linkedin = "https://www.linkedin.com/in/ruiwu-liu-b1534827b/" +++ diff --git a/content/people/yifang-tian.md b/content/people/yifang-tian.md index ecbcab4..e7c96a1 100644 --- a/content/people/yifang-tian.md +++ b/content/people/yifang-tian.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2022-01" location = "Toronto, Canada" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] interests = ["Machine Learning Systems", "Distributed Systems"] portrait = "/images/people/yifang-tian.jpg" email = "yifang.tian@mail.utoronto.ca" diff --git a/content/people/yifeng-he.md b/content/people/yifeng-he.md index 57fa204..3d6a69e 100644 --- a/content/people/yifeng-he.md +++ b/content/people/yifeng-he.md @@ -5,6 +5,7 @@ role = "MSc Student" status = "current" start_date = "2025-09" location = "Toronto, Canada" +research = ["rca-aiops"] email = "yifeng.he@mail.utoronto.ca" interests = [] author_names = ["Yifeng He", "Yifeng (Jason) He"] diff --git a/content/people/yiran-li.md b/content/people/yiran-li.md index 50cb7af..912eaa0 100644 --- a/content/people/yiran-li.md +++ b/content/people/yiran-li.md @@ -5,7 +5,7 @@ role = "Post-Doc" status = "current" start_date = "2025-11" location = "Toronto, Canada" -research = ["quantum-computing-systems", "distributed-machine-learning", "data-management"] +research = ["quantum-computing-systems", "distributed-machine-learning", "data-management", "rca-aiops"] interests = ["Graph Analytics", "Graph Learning", "Quantum Data Management", "AI Agent Reliability"] email = "" homepage = "https://cyanidecentral.github.io/" diff --git a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md index 0058553..1bb466b 100644 --- a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md +++ b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Huanchi Wang", "Zihang Huang", "Yifang Tian", "Kristina Dzeparoska", "Hans-Arno Jacobsen", "Alberto Leon-Garcia"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2605.22779" abstract = "Production systems generate millions of log lines daily, yet most anomaly detectors operate at the session or window-level, flagging groups of lines rather than identifying the specific message responsible. This coarse granularity forces operators to inspect many routine lines per alert. Message-level detection offers finer granularity, but remains challenging. A single event template may correspond to both normal and anomalous messages, failures arise from heterogeneous subsystems, and line-level labeling at scale is impractical. Although large language models (LLMs) can reason over log semantics, applying them to every line is too costly for continuous monitoring. We present FAME (Failure-Aware Mixture-of-Experts), a label-efficient message-level mixture-of-experts framework that uses an LLM only once offline. We annotate at most K labeled lines per template to derive binary normal/anomaly indicators and representative examples. The LLM proposes a partition of templates into failure domains, and a certification step validates the proposal before training. FAME trains a lightweight router and domain experts that run on-premise and output anomaly predictions and failure-domain labels. On BGL, FAME achieves F1 = 98.16 at K = 100 reducing annotation effort by 76x and detects 97.7% of anomalies from unseen EventIDs. On Thunderbird, FAME reaches F1 = 99.95 with perfect recall." +++ diff --git a/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md b/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md index a643ed0..301fa0a 100644 --- a/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md +++ b/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md @@ -4,7 +4,7 @@ year = 2025 authors = ["Yifang Tian", "Yaming Liu", "Zichun Chong", "Zihang Huang", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning", "data-management"] +research = ["distributed-machine-learning", "data-management", "rca-aiops"] external_url = "https://arxiv.org/abs/2508.12472" abstract = "Root cause analysis (RCA) in microservice systems is challenging, requiring on-call engineers to rapidly diagnose failures across heterogeneous telemetry such as metrics, logs, and traces. Traditional RCA methods often focus on single modalities or merely rank suspect services, falling short of providing actionable diagnostic insights with remediation guidance. This paper introduces GALA, a novel multi-modal framework that combines statistical causal inference with LLM-driven iterative reasoning for enhanced RCA. Evaluated on an open-source benchmark, GALA achieves substantial improvements over state-of-the-art methods of up to 42.22% accuracy. Our novel human-guided LLM evaluation score shows GALA generates significantly more causally sound and actionable diagnostic outputs than existing methods. Through comprehensive experiments and a case study, we show that GALA bridges the gap between automated failure diagnosis and practical incident resolution by providing both accurate root cause identification and human-interpretable remediation guidance." +++ diff --git a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md index 93bedfc..45c5f22 100644 --- a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md +++ b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Yifang Tian", "Yaming Liu", "Zichun Chong", "Zihang Huang", "Yiran Li", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning", "data-management"] +research = ["distributed-machine-learning", "data-management", "rca-aiops"] external_url = "https://arxiv.org/abs/2608.08968" abstract = "Microservice root cause analysis (RCA) requires correlating failures across heterogeneous telemetry within complex service dependency graphs. Existing methods often rely on a single telemetry modality; recent LLM-based approaches can suffer from unconstrained exploration and hallucination; and most systems stop at fault ranking without producing actionable incident response. We present GALA+, a graph-augmented LLM agentic framework centered on graph-guided investigation, which uses service dependencies to bound exploration and refine diagnosis through localized multi-modal evidence. For initial hypothesis generation, GALA+ combines complementary telemetry signals with STRIX, a novel trace- and graph-structure-aware scoring module. GALA+ then produces ranked diagnoses, incident summaries, and stratified action recommendations. We further introduce SURE-Score, a human-guided evaluation framework co-developed with industry SRE experts for assessing RCA-specific output quality beyond conventional text similarity metrics. On two microservice benchmarks, GALA+ consistently achieves the strongest overall results, surpassing the best LLM-based baseline by more than 25 percentage points in AC@1, while also receiving the highest ratings from both SURE-Score and independent human SRE evaluation." +++ diff --git a/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md b/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md index 6ad8762..b87d2a2 100644 --- a/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md +++ b/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Yuanchen Gao", "Yifang Tian", "Yiran Li", "Charles Zhang", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["data-management"] +research = ["data-management", "rca-aiops"] external_url = "https://arxiv.org/abs/2608.17018" abstract = "Microservice failures are often diagnosed from operational telemetry. However, automated program repair systems usually start from issue reports, localized code context, or failing tests. This mismatch leaves a gap between telemetry-based diagnosis and patch generation. We present ORCA, an observability-grounded APR pipeline for microservice incidents. ORCA first distills the differences in paired failure and reference telemetry into a fault signature, then uses the signature to identify candidate code and deployment-configuration locations. Repair graph agents and an Exploration agent generate unified-diff patch candidates from these locations. ORCA evaluates generated patches with a Telemetry-Grounded Patch Verifier that separates patch validity, syntactic and semantic correctness, test-oracle integrity, and telemetry replay. On a 575-case benchmark, ORCA outperforms all evaluated baselines in terms of cost-effectiveness. Results show that operational telemetry can be transformed from diagnostic evidence into actionable repair context: paired telemetry supports repair-oriented localization, while repair graph agents convert localized code and configuration evidence into constrained patch-generation context for the LLM. Telemetry-grounded verification then exposes repair outcomes that issue- or test-only evaluation would miss." +++ diff --git a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md index e953352..66c447d 100644 --- a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md +++ b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Jackson Clark", "Yiming Su", "Saad Mohammad Rafid Pial", "Yifang Tian", "Lily Gniedziejko", "Hans-Arno Jacobsen", "Yinfang Chen", "Tianyin Xu"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2605.07161" abstract = "AI agents are increasingly used to diagnose and mitigate failures in production systems, known as agentic Site Reliability Engineering (SRE). Current SRE benchmarks are limited to oversimplistic SRE tasks and are unfortunately hard to extend due to bespoke designs. We present SREGym, a high-fidelity benchmark for SRE agents. SREGym exposes a live system environment built atop real-world cloud-native system stacks, where high-fidelity failure scenarios are simulated through fault injectors. SREGym models the complexity of production environments by simulating (1) a wide range of faults at different layers, (2) various ambient noises, and (3) diverse failure modes such as metastable failures and correlated failures. SREGym is architected as a modular, extensible framework that orchestrates fault and noise injectors across stacks. SREGym currently includes 90 realistic, challenging SRE problems. We use SREGym to evaluate frontier agents and show that their capabilities varies significantly in addressing different kinds of failures, with up to 40% differences in end-to-end results. SREGym is actively maintained as an open-source project and has been used by researchers and practitioners." +++ diff --git a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md index 99e1ca3..dcc60d9 100644 --- a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md +++ b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Chenkai Zhang", "Yiran Li", "Yifang Tian", "Michalis Bachras", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning"] +research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2608.14680" abstract = "Reliability in LLM-based agentic systems is a property of the whole execution (its tool calls, model calls, guardrails, and inter-agent messages), not of the final answer alone, yet evaluating only task outcomes reveals little about how or why a run fails. We present AGENTCHAOSBENCH, a benchmark for detecting and localizing runtime faults in agentic systems from their execution telemetry. We run five heterogeneous applications that coordinate agents over the Agent-to-Agent protocol and call tools through the Model Context Protocol, and inject ten types of operational fault (unavailable or slow tools, corrupted or oversized responses, and delayed, looped, or misrouted delegations and bypassed guardrails) at their tool, model, guardrail, and inter-agent boundaries, alongside a no-fault control. The resulting dataset contains 275 sanitized traces: 250 faulty executions spanning ten fault types and 25 no-fault controls. Each faulty trace is aligned with the no-fault execution of the same input; fault-type labels and, where applicable, location labels are held out from diagnosis. On structured single-trace inputs, a first set of zero-shot LLM baselines shows the task is far from solved: local detectors up to 14B parameters reach only 13.6-19.2% top-1 fault-type accuracy and the frontier DeepSeek-v4-pro only 24.8%, while jointly identifying the fault type and its location tops out at 22%; reference-dependent faults (above all a bypassed guardrail) stay near-unsolved from a single trace. An aligned reference improves selected relative faults but does not resolve guardrail bypass. The held-out labels and compact prediction format support reproducible comparison of LLM-based and non-LLM diagnosis methods." +++ diff --git a/content/research/_index.md b/content/research/_index.md index 8c2aea3..b1a42e8 100644 --- a/content/research/_index.md +++ b/content/research/_index.md @@ -1,5 +1,5 @@ +++ title = "Research" layout = "overview" -summary = "Three active research themes connect MSRG's work in systems, AI infrastructure, and quantum computing." +summary = "Four active research themes connect MSRG's work in data management, distributed machine learning, quantum computing, and reliable systems." +++ diff --git a/content/research/data-management/_index.md b/content/research/data-management/_index.md index 16b1d8d..88c67a9 100644 --- a/content/research/data-management/_index.md +++ b/content/research/data-management/_index.md @@ -20,6 +20,11 @@ can repeat computations and produce the same intermediate data. Our approach detects these overlaps online and coordinates execution so queries can share work and improve throughput. +Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research examines the +operational reliability of cloud-native systems, using metrics, logs, traces, +and service dependencies to support fault diagnosis, incident response, and +automated repair. + ## Blockchain, Distributed Ledger Technology & Consensus We develop consensus algorithms, fault-tolerance mechanisms, and consistency diff --git a/content/research/distributed-machine-learning/_index.md b/content/research/distributed-machine-learning/_index.md index 0047f4a..2e34070 100644 --- a/content/research/distributed-machine-learning/_index.md +++ b/content/research/distributed-machine-learning/_index.md @@ -33,6 +33,10 @@ Our distributed graph systems target graphs containing billions of vertices and edges. We also optimize the training and inference stages of graph neural networks. +Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research connects these +interests with learning-based anomaly detection, LLM agents for failure +diagnosis, and the reliability of agent executions. + ## Current directions - Resource allocation and scheduling for distributed training and inference diff --git a/content/research/rca-aiops/_index.md b/content/research/rca-aiops/_index.md new file mode 100644 index 0000000..d032169 --- /dev/null +++ b/content/research/rca-aiops/_index.md @@ -0,0 +1,82 @@ ++++ +title = "Root Cause Analysis & AIOps" +slug = "rca-aiops" +layout = "area" +weight = 40 +accent = "teal" +question = "How can we turn operational telemetry into reliable diagnosis and actionable incident response?" +summary = "Root cause analysis, anomaly detection, and telemetry-grounded repair for reliable microservices and AI agents." +blurb = """MSRG develops methods for **root cause analysis** and **AI for IT operations (AIOps)** in distributed systems. We combine **metrics, logs, and traces** with service dependency graphs and LLM agents to investigate failures and produce actionable diagnoses. Our work connects anomaly detection, incident response, automated repair, and realistic evaluation, with a focus on microservices and the reliability of AI agent executions.""" +# Research descriptions are based on the publication abstracts linked below. ++++ + +## From telemetry to root causes + +A failure in one service can surface as errors, slowdowns, or unusual resource +usage elsewhere. Root cause analysis (RCA) connects these symptoms to the +underlying fault. Doing so requires reasoning across heterogeneous telemetry +and the dependencies between services. + +We study how AI for IT operations (AIOps) can help engineers move from detecting +an anomaly to understanding a failure and deciding what to do next. Our research +brings together statistical analysis, graph structure, and LLM agents, with an +emphasis on diagnoses and repair decisions grounded in operational evidence. + +## Graph-guided diagnosis and incident response + +[GALA](/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis/) +combines statistical causal inference with iterative LLM reasoning over metrics, +logs, and traces. It investigates both root cause identification and +human-readable remediation guidance. + +[GALA+](/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices/) +uses service dependencies to guide investigation and focus agents on local, +multimodal evidence. Its outputs include ranked diagnoses, incident summaries, +and action recommendations. The work also introduces SURE-Score, developed +with industry site reliability engineers to assess diagnostic quality beyond +text similarity. + +## Message-level anomaly detection + +Effective investigation starts with finding useful signals in large volumes of +telemetry. +[FAME](/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection/) +studies anomaly detection at the level of individual log messages. It uses an +LLM during offline preparation to help organize failure domains, then trains +a lightweight router and specialized experts for online detection with limited +labeling effort. + +## Telemetry-grounded repair + +Diagnosis can also provide context for changing code and configuration. +[ORCA](/publications/orca-observability-grounded-program-repair-for-microservice-incidents/) +compares failure and reference telemetry to build a fault signature, locate +candidate repair sites, and guide patch generation. Its verification process +examines patch correctness, test integrity, and telemetry replay to connect +automated repair with the incident that motivated it. + +## Reliable agents and realistic evaluation + +AI agents are both tools for incident response and systems whose own failures +need diagnosis. +[When Agentic Executions Fail](/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry/) +introduces AGENTCHAOSBENCH to study fault detection and localization from agent +execution telemetry, including faults at tool, model, guardrail, and inter-agent +boundaries. + +[SREGym](/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios/) +evaluates SRE agents in live cloud-native environments with injected faults and +ambient noise. These complementary settings help us examine whether an agent +can identify a fault, locate its origin, and support incident resolution under +realistic operating conditions. + +## Current directions + +- Correlating metrics, logs, and traces across service dependencies +- Graph-guided and evidence-grounded LLM investigation +- Connecting root cause diagnosis to remediation and automated repair +- Evaluating diagnostic quality, agent reliability, and operational usefulness + +This research connects our work in +[Data Management](/research/data-management/) and +[Distributed Machine Learning](/research/distributed-machine-learning/). diff --git a/hugo.yaml b/hugo.yaml index f91bcd7..2ac0917 100644 --- a/hugo.yaml +++ b/hugo.yaml @@ -42,7 +42,7 @@ permalinks: data-sets: /data-sets/:slug/ params: - description: Systems research at the University of Toronto spanning data management, distributed machine learning, and quantum computing systems. + description: Systems research at the University of Toronto spanning data management, distributed machine learning, quantum computing systems, and root cause analysis and AIOps. # Public contact: https://www.qscc.ca/open-positions/ contact_email: jacobsen@eecg.utoronto.ca contact_phone: "+1 416 946 7586" @@ -79,6 +79,10 @@ menu: parent: research pageRef: /research/data-management weight: 23 + - name: Root Cause Analysis & AIOps + parent: research + pageRef: /research/rca-aiops + weight: 24 - identifier: people name: People pageRef: /people diff --git a/layouts/index.html b/layouts/index.html index b7f343f..09ec66f 100644 --- a/layouts/index.html +++ b/layouts/index.html @@ -74,7 +74,7 @@

News

Research Themes

-

Three active tracks with room for overlap

+

Four active tracks with room for overlap

{{ range $researchSection.Pages.ByWeight }} diff --git a/static/images/people/ruiwu-liu.jpg b/static/images/people/ruiwu-liu.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a83232ac23f449c85794de06892404ef8bdf46c GIT binary patch literal 110588 zcmeFZRZtz#`z?45b`so!>%k?#-Gf68?tXB0cLD)|9o*eDxJz({;K7~X!JQ2E-l>_Y zdAKz*Pyajf+g)G3bl0w4yH|Jb?^|p4zombh08BY4St$Sz2mk=zHo(6XfCK=9h=_!U z0761SdIti%L%~KtK}JTw$G}9xCc-BsBE%;oAfceAA|azCBOs*y1g2$VVqs+=refz} zW9Fh~W?}wsod7}a-l4ol!9zj8VqG?p_Xhai8xRg20TBuG?maTf+X;=B05~8#JRAZ%A|e99+u43^#{mdfh}e{DVn{g3 z#vm#uT=pPX-aBgX+MjqTzy5+bOq_$?BjbM{AS9xprK4wH42!Q_| zu-^9n1K9rs7uFjtI0OWE1kiuO1%&f>3wSI9L`pU!Y%yh!u@epzd(b;vaadmM&-c_E zDu3}zoPQzXgE_ZoF8&*||Bmec9(&>S7%8~%w|HS`f48(Zfm^|TUKEWzj!&Bq``+7p|E~w8hTjo17DqBPj(5?v8 zt_onr#1Lez!wIf6-3;I_ciaEApJf!y5u0|KGdzv7$(P@HoP*mR&xzS@VC$G``VYXS z-TgPQBjDn}AI2IG%nUu!x)3D|h~K#S)lC%Uzy%s(-4K^1k9Op^U!UBEta*l<)DHT) zQ0K=BwQ#}6BN{vO6%i)<-h;)cWCD3=6--|o>EUOLW0hrzv{~^DK4FPbi=^i5&3(-Q zJ0|@D_=em^UYts7L&ELcD|82t4~l2Byn|Novi&Knc|Pe*Ih7s(QL>JL06ixqHH*}^ zz*C-R&oAP-J6)tMkCBDH5CXCXpP2pu1Rb6I17r6LAYR)SUQhS?-RiG6986fp@c=HN z=6-rx7`OhKJuf!LSgX+A5|}7iEP92PgRkH>C`tfgFToNybeN0lti5yy-fP)*OKo!m z3k`PfqNgT;S?NI7({1#pgH7#n<>=j{bfA3#1Id=!dDdJBKv&i5#QOHz%eq0fET^W*ISkDS=dhf?*0E@n z9l2ESMfhdOUB<|x$If3|;L+v%s4gmTX^)hkgMMR~+4dUuP2_vu<|Ckx&0y8DyqgpL zP^(Qi!B(>EC-R_OHSWb}*?no6dMUNnGf6OXG4nbX!-Z%S_}dJL#2Lla>F8a>v_T)g zy~c!RQgLgg<>lvQIkQFz_J)$2`}aaqd;0FMZ6En<{|UA*v8fcQQ;Z|fo$gy7QdhV! z4Rc9)ar`m!!_j_QA+GfWgBTkLd#1O>8%GSM+XCA5cxF~mBc37lydLsXXmrU!ApgZo zWN*_2YZfiuS)MjqCN(_lSf^=GS+Adn0Wav96KE32uPlm)`~T-ybc-^JK~(Z{fj~N) zzu)Ad5eaB~x**H6xD4&haa<+JE46WK$kYo@@^g86HUNj(^WHzmRh8v=qY3ZIcA~O5 z^k@sJmwr=?d61ZqY|1nvyb3yF%xT~qe7{bLwf4FbX!G4=xaOMxxZe@Y-!wmNunN1R z0RgclB30~)e&%G*F~TSk(%7Fnu$hju4r6ARo+McrRb`r5Q<`=-=vVh(>r*%W>fq6j zGp3wuZilsmm7tANy*Kf@`9?o9 z*89?{u}bQsyrGvSAvPt~;Qpp;yPe+^jitEA!FAlx|CcSn&K!KsOrQ|Jb7onDEq9KD zmEDN}o;+eQfM~G&!ambfPkTd;P`o=m1tfe=$6GBHh(I0O^x+TxxUnM)8l@SzRjk zF3CG_022SkfzH*GIeqz*jfR0N1fQQ_%k7ejvYl{EFA-5R0j}t9W!F zfM8BFV{)F4l^qA?m%q(-$}ynn1FrGL)A5-8;ept7+ud|^_F=UKua%dbg`%K*E_rS6 z!}?xQ_hB*LGehyw0^uEd-RU2#c*wqIWky)MrgCrSrb*S(a_89}gxZgUV^69rllh~k z_PJEc%i0-ecWRk0Z0otnbY5U+5v-Z;16G{c?e(#Cx<;N?t_@rGS(-R6Uub=-uhVdK zolPunN@r?A)Vk3XN0@N8jtO5*qTx%-2{FA(=V%_7T)~33;zJFtf#&az@keZf+;}T7 z{{Ydhl*}JpM~q|6R@>n2-AEn-*K9JH!)A2PsU&6QCovNOFb7z|%xhN}^Rr))+#t^5P4mTZ>=Pi5v zysfsjs#d)d`|aVZ3i?l+qKX8rdH_h1wMJ*^67mqj}{YqBfmYi~M z@CMx6UASeP{R7ZT=@N=Xv6Jn|{1I+!Hr{U8Z2cU^Rn%9?7aq&#;3!S<8rBhMV^x`z zX|Fi^j(0828%y�Eeo7gF1WfX!k>Tb130F<|6D~rMD}U= zSm9qzT1Z0g^nWbf4UtEC8;1FV6Hy$>0)J)WOA3Swe_MK-29<2knRtE+oAS@jFLL6A3%R|LBB)~|9>=}QYX&RnOSj+ zSB7cx7_ip`39C+Z0$h_ad@7fMp+mXD+mx3KXDte-sAw{X?pN8zNND@;7bZ4&(z(&8F?+wu!yew5KeDqa`Q^&E^ zWB&kO*PjLCM#1Jgp*>Ufh|W6Mb-bGmZl z^1gCyljCUG7xx@8vKB37RY0LC-TP)g2a~cV_3b?ZHCvZfZq>=!s?~UKTD40g4(W`x4Mm6C*79__we*(K9TNI6SY zXPaquUFu)`4oJGY8!dFjjWcK#{{hTOn2YfTPdFakE6Ab2`|~9}MWF>wmspu0bFoDf zbain$8!Y{~`0(x5?{LB+ZT+1D6leLtWw(g+rDnQhO&%aG=2#7vo4Iq!N;yg{Ap^udRy+K7x#x9 z&t(E&%LsAD+7oyI1yo6L9wKCz#D%Gj;3_)WVd+D#9Y6V~z5kHrjD_cK@3b6;zoo4h zyG(9ssTWGRvYs-kdbQ?%#`(}3;IrHar#ldqav(EJMZ1pf4Lh^RndfTihN*93F-w@x z@5EjdBZhXUwL}MzsXB!W>qPFeSvY;o90wC>kT{*IU(4fHr$e zuySi0YVE{6(ePGzfN zU?n~K%aK9y=6?X)^kka=+I@j`P0>EOJwdFEV>Jck)d4G;Rh~JN^xqj2Y){?M-snM9 zEH^>;&e)?e54xAD7>0f6e{BjaOSD-{o&+J&=utj9q)df>tLu_Iuw}-P(l^>2J_;nY zV`cAD0iiRt&$3R%V@^)|UQQ$Q(hQ^2J1wW@*^TQ~I+^@oNjAK$#`MS8JB?oEg?ahr zilXv%-o9mWj@rRz(c!`c!9GMqTLD;Tbo$9Nd;b8_yTp)W-YFAcVQQ@VsWSD!oMmj#1&Le_K9KkQf1 z0r8Qd&cK_Z{|}cmtavN^Acnz{+)JYqQM=c;zzUXr$AC)5g7sEApMizsiw37*?mNEY z>uL^zix9aJ5A4=KAIFgV{m%zP;D+ z_*>t90FrN3w>_0CDi*V6A=Vlb%q_bdMy5{F?v=+mAN1CwYDL4#J1CY4$m0;Ekz%U0 zQ((Uf8e?XK5_kzuCydE4CrDzGWash}WLJcSI@MB-_+xbRPm>1~!OIHCO)J${TM73n zV$GhnxuCw%vD;eKsj23f!dP{mj{;*TPuFa5e+UBoIHsRstTTZ@hsg$Mhw0H(`&feK z>}SKo99QH{r=lP7{66*Sm}c!yjbaA2+?~sF=b}R0w{gybR*toSw=U`LSurLS0$G|$ zI9k9_=xb#OhNFMPe97$-hbbquqRMY*VQU~_psEbiZdulBul{XHPQ~ZDKxQd1>Fvi@ zhmDy?5)&56vghSR=X5Azeq{a05Q0O#3cSP@EWO@&ew;^X#g|1sN6SWM`eYw5%+lx+ zNhB6kHlP$y#gi@&f?$go$1Z-e>1PhiehNo(I_p|6Bqw34VK9ZYZQn!SXER-tQq~oLi z+>&-8EGMNU7ez|?a$YwVcSPyaT4b!3xo~h;3M^!R0(Yz(a9bKqGx%%1OfO7OWpG8$ z$3^tJ17)18kZRDWN+0cT59k1tTux@p3K9Qe?HXry5F$#1!kG|D&nMumjgUTTa4-5T}}U~syT zZka8q!MKa_bEpXM!@P;hUJKd%sbW`?CCzd-_iWrt_^I|WL!{`+gpJp+o7F6HhtgNA z#sKd5zhlHwsA)nkh`LjcI3GvX{Y9VsZ5Qw_0}s0K+-D4vr9r(N*YjL&ITl1Z8W)x3k?T zp_ZY`Tr^8E-O`VogGu^dOutj_qY4Rete3R^=#Cu4#XoNLWDWNOquL9)f;SBaGI`xMj6JgbuQL$p#Kdwr@C0X)11QU{BaMp>-9$ z>h^O#gcL@C#&Y~60p2QXF*~ELB=%^}NAoTzZSfPrM1P%lt?H{r?xDfKv%5 ziEyEm5#wH7PXtKM0Ds5EcWIRNF7PN9v$|Vx32+V2*}E~s9A9>=V+-SQXdw;L{Ug+= z2+&y+l*dF`|4rn6{MFXt$Orjq?5gZ27F^XxDXJF;L3zc4XkCO>r!;95M`{zkzZ{v= za;Ki1X+?>}J@%(+$vUH3K`lU!T-Rdc2O#l#DKGnt^*UjynMz~Ai7^9ReT1%K~ zvi`^etw}Dg>>P4AR@bp*V$2q@Qee3?7DSZ|1_6k5Otr*hb8G#Y8bAsDevk%SxA0cHF@6=kW!rAChws?fI|q znAz}&o1PmrJ<=$^TaxlLE5KUM>q7S5uw?uQT+*S0L$yKy$k-%9&{_;%(V%A7`AF`2 zGSyqRp|M3Sf))xUC~O9Hxiqzg`o4**zyQJ#Ca>C0OS8rg$I; z72VMi7)lT*dVmK;SEXwuD3WLsCD@X-Rw~49dEgNH9CkWxpz~a@v7R+Uy5bK}J(hvh zWZn7eq?@CD(hd4$*%GOkL{X!vsyq4DY-s$tHZNWsVN*TLPP_>)d{ZbiY}yj9z?6BA z)rgTeuAPv}TlJoWpss78o8lkfW5Y`u|rxlJE~CTu?X3nux2mV@&E zTg%QYXY%MQpTicp&!Gady{Lf_g>)a(X{?swcEH-kdBCfbWuie?Rgpj%`BV9(@ee-^ z?KuCl8Oc!J<1$=qoKnJyh)=KMB9>|^mYaxC-v0pEiU6?^?a~(0HnT}y_=UC%n+cWu zk%m8`?|!&f{Md6E-F@8dI3Ew+sI{6Zj^5mHT)ty>|p5n^P2D}6+DEvEjJbxUjYQjs3g&*W8rzdR^8igO0i z>1M4UZ7k(u-jeXQzy+3dXR4N0HT-!dLvhh41>{H4|1TWD%1l!0vdAwHyGjT_fxN{I zab4V)U5y4zR=z(Lao}EaO$yUpn?pXPftopicnf)G`M&3^?G#5gNOx*5^=i6!nX0QF z(%A9+*=wM~^lw@hW?^Y0@dACe6>F^`apwji+aJRdj@rIwP@RQ@COagl{`mkG$RB{~ zxv<=m^yxI6@)BG^Cvpm0?`YS!P`|D4Jm2R&@=$;MTH|zSlNuFE2?y*i^ zvCpI584Ca7p;|T*Nuc~1Odxn%c6X?EE#c6-qgA1sFx1fuxB_Y!@o-&vlcE0o-V&tFQ%`&rP=&h3Z z$_haW3s~Lsp0n@i9P++rN_D+Z4;W=0 zH{8*kz&mN>4kh5LU>j&LgTObUk@G6wPa~Q~nPM$=eyN4ayb+vM4nyLfPn~`+lX+-+ z#Z4VO#$V5jcaixOKZ<7cd$7(cB8Cknxz<*^pgdEOT5V8>V*4Y>c+y`L;?Y;mv!xR6Lp~a~F1<_leYvx*?*$SQM_h5Y^WF{VM#;kj~Uq3~n=gZUx(_Z1)IS zt@vibp@flo%GmO!Znkq(sZk0ybDHUk!^k4RisljJ_>$y`_e3WNS);hBqCGO)Zo`At z6&1s>@KJN+k(B1tK?CwANO~V}&vWyIlsdG|B-LF9W;QLTYAX0YnNtJ}h;bm!!2{qz zC%|LSfijywT;G0`&^7-cZ{TzJvox|Pb*7_rCDxAe&wqgA5OQ(Lma5$P&nQj)W(Jbt zCZn98{(2e(y65Zd0Wa$BAR>+^PYrPSd$v$U>>;DIQ>Df(+0li_LCIMoJLQp~B5nmS zzTve~wX_T=vx)8`Tu%a3HLXMwki<;0J?41iTI+Vby%B|DC!#VwMZ!U}U)x+Phagb2 zhX9YrLxq2CVVm2zYA%NHE10!Qr|fAI-Fm*GOR})rV}IzXoGtdcN8}qX=R;lPudrfj z3z_!t<+G$OyG9v_x92(b)@Ny6REHj(K{~WbmFLr}PD7naGj@c`W?sLIqtisem(RDj zD{E3(wluTb=DKdY)a_&)vxHgR<=XoG<|fi(`oa;euJ_MX_i>nJXorV-4LEj^b~&E} zLLm+T6m1*=X2BQPH{9^WE$y)9`>$Pk9wPM7(gYyB=Rd!U+9Daw=5GgM@?L)QglIS2 zAswyBz5H46*wU9ua6#<)Cg}VRFg8K~TO^e}7A0cv4QTpk5#o`;?5Vg@Q*%{xf%se^ zS=a0XVX!;+slf~FS#7z)Lx;Iw<2$9fOE)}szJE)Sj~@F|Qd}D9q^aEm+8$ATf<&X~ z8+mG*nuFAn>8xm%tFn6j=H701GWBXEr)P!{!eH_1TrwPitA6LcixE0ieUmG1WlJ^T zvy&)txf>zmo6y%3$1g-HU!1N@H=UMC(HCV}@fTr-_~X^Sw;b61CV*$p(|h&dYULT9 zC|PEC-tdoWBLq8#>ARey>N))&Rklj6gkjHV)6AIncL;xNypIBCw^!tv?Z#93+a7na zA##8-yJA@u$`-SgW|5BzMNMRgbydcUcCjN(YQlNFJa9UDHII=!E#CX78+M|jvWF% z&DCDi^U|H*uS(DKE8RA$k4rxY-xFmrz9729+p0{p@8=dO(#N}65zc>L3Cd2pE^qcg zo$>un;!FR)d#KFWW!xt`Uuq9-4sVVlcy!n+h2!=s5aK;oW@;zF%5+&s>&$BQ&u*kr z+^a07mRe^^SUD8Mf~9dxXepJq2px4fiVX;6hb-iUCuu8TEzS8zu{WfS`KW!fgd_)2 zlly1qvI!k_b{_tE&P`A-;e*RxGDOOJfDXxJQ*f8&LX9HCSl5IYR(LR-2IP)#@{Yw> zJ;@qS&5^m+2Maxe0Yfc?MtOq){L<#+41Bpr9N0fK^UB=4Oigd{iOmPEW{Mc5CV5F2 zP(}BoM32SX?LRb%j?3H86)6IL4l)4Qr4GUlsJVnX%A?-vFYW#VgbEJVO1(RtQ&apg z)QPT_okVk5YvH}T<`v|G$}b!H$XDRG6_{tw{kG*m29W)eXELp7Y>?o`!y zRhCW}6Sc*ay2MGnD2rCSb(ub0Y#)glWn;ZekuLQ_%~(>0G9Db=@^M2b^o#kG*}-JKsQzm!`n)ecSt zI%Nl^j?vy&5nD~73#hb==m!rLsEqIA6*Y;|ik=4Z=KYY2 z<#5&24)`9TG=aa;`oaB7{1vXS{bgg;-Ls^RkbB?b=;Ym05n@Wm@LGZ7gSs?dxm9Zs zd-Bt_$RE*!?061|473#=0A4mD_SRDCp+(s)C=XBdOi8FJhBM5BG=K9)sTc>H7TB)91seuubmi ziUA2h=GwrQN20c;rgm-)WqD^aCPD~IibG){4QFUCYiUACm6Z^Qdm*a4@VPIy+_s$< zY^aQE8YlUJU^0TLytV%jd|A{#%>sMuwIHt!2G(k5Vo}n^wbN1MO+=H(}2c=2q!ft8&Q+7>P+r1xwHpg<1F&PTc}G z%JePY_K020vDlVC=P%x-i(*#U9Yd~R5F7Q{`r8tcflBdcm3 z$x6ByxD*vR|CTXzFYC;kmU5BOuMR}Fnm49mLAz}It!$%c*%SOD@iXGKEk zA%u{!k-V>A z;<5G8Y2y%ot=J%xcH%l08x6$Ts^T`7fHyjN6SNEpdl_5rqTH7FO-G4_oQu^H3@1O%7` zc6t_h@Nj`V&#Kjjj^#9oRTb9S!x>JVJ41GzXkHV0r~Nbd*-BdOg|7?oG)%Ui83WpX zK0hv~o+m|ss$?bh+QApwRkASOxj1elPD~?Bi<$U6$U_dvfHFTC zYPNY-ey76tS&antsA`&Z36JTtH1C1NnFpe2R}zQ`eh=P6JxI@_#R2=c6^$<_xdE~f z`PVcw9+p_G0P_-o7^<`B8DnaFWn=UYp=__R=#D~Y0>-Hh25dtMg%#*3F8lC-0%|S_ zrYf4DbVcBAqx!AK%jLDyU`HW_rQ{Rl*^WT?hLasG9GUkh(-@s!u-Z|tORCcNoTUVU z7E+T77IufQw1f@3Z_22n3m=xA*NY1qzDhav*wI~;?M73#0@Xr=gv!`wD9NoLNg38; z{3;`;L&yi`u2mohZr}mh(x!21d3^Jy^0T##V->c);up{PQf~I|)*`dS^I{+!!>f6v zQYHauO=raq<8`Ecd_xOa)pWUfV`;ML4_=#d>o%2Gee~*Py#D~mG5cB;`XjAc-jAvd zujbl3e&>f`GW8$M!7_Lrw69$aLu4Ac2l<2UGs86`CX&ROVF+rfGWl!MnF&`GbLp6A z*6SuK-$_f6c3$ytT?oxA$URE9)WPl{V`fi7d?oqzgSRW#H^oeP*;c+|UxPV9mpKmA ziyc^9A^t*#zg8($@i?aRJAmgZmCOM%#N;RDLB-rwq*$2M3pY2V%N+YuG12RV>9lrL zOlBFgjdYKt9`>t8(qwbhwCvm9(Ax@p=yQ1?>@CqOgQoCMnmw9lwery)pPLh(;8W;$uKJ`E7Z@B zZe|bHx6dVHm2oAb%bfh>Wu%lV<6ee?EjE+MP4`E))DH>UQpLZi33 z*~E(V?8*>JtL*@l9mC4;O{w8HeY1~y-37sv?OSJuL5Cwd3?F*o<(g1U!c-K#_mMP7 ziQQB&MVm@&w*HuGr3Tkb3tZ5N5-s|UPfaKMJomfW zdP=3WLtX>b{S!I&CcOJWx88+5e8Oe*`VJ?Mi-n?8`EGb)&4A;4JqSl-R$9*Oe_|K@ zuVMeeBdM#j9G?heR9HYSs@DBU{S*D6gz$mtweT-JoRE!yuARtautv1>1?;DA=3<;@ z%}|Ppj$Ghg*V%0k#S?lXbfAk<#*QxRFe-(Qf~uL`Z|N%N51@v$t;1@yJ!{<0l1P)L ztJCX>nirg{#VxPlkk(F9x1l2~uu>)6o~ahoD-gVjtV%m@90~nlP!L(ggtHL|a2e}} z>+qRlzQ4OGj@K=LgS9-)VzSY#%w!QI@j3J+eYnpkqBMat0vHf8;Ig7i;A^}s?OJaS zMNPg~0$9==yv0$vad!qN&{nr$T{raNUIm1kN^m6`6IyCumzjK-?;?1Xqy=bVGR{!| zJQoGys9>+6^~2tnM$F0W{wtZ>v>wq&cB|R~`h!59Sz1RkL?8c#VvVHMP4wVEc#08V z7@~HV=1OlcA~&(ofxDC+JuI2KhtR^W7DZiqsCGB4lTh|djDp`>i~n-);PHY<{Xb^KrlsaTmqLBnYi5+nnf z1e1^|=J@E%7)2)BU8xgM;yR8Aeztu)3N>aO%|2B6bDagtT5$*(Z1`M_NiT*s66+F) zE|2r}yA>s_y?ySC&|DZL`U$P)4or!&e@{F0qpe42s})LZwk^)=Ob#`{n#7x{)WDiT zpC?pEQjt(*@$QHSqA&yZr#H{9=adjHhq;x7-mX1Hx(cV&4T}=c%r_Iq4@H=iL+$S?Wc(Nqi6o#-Jmg6tCm?J#p!Afpxs!Kz+ z?Q^Pk(v;Q!(jC?%5|bPYVkxrE%$2-vU*ZopJP$fFBFw(zRVMP3$x5}EuzBl&`7K@S zK-2DVh_f{@r&j(IU_ACwm3F`M+ey2QU3tH62$%Tfmbo#m2qXN+4HwN$8YSu_X%7^1 z?vJOkj<31e!uxt>EmIvjjb}u&xJMnNkq0vmAK_@4BT&O5%IrE!axZc4&i=@-mNr*@ zODZC#7>%3mZu{;hOZIixy6IHp)1<_E7R}p^Up^Sl7wSSseSfC1ZAgzMiRm8l2TJ0Mht5? zoe=a&zNfyB6|T=h!-Y@(03XK%$gQD3umya2J)s2cAUUhB;cKU+lHee!d$l6mk*Xjj zzeaTA;Islni7~VM{Z5fq9eI0VGeVR}YUC`#q#whNRzdU|@Hdcy|BY^v?RBmsEJq}} zq{^z#8G$~M-9_)4WuBpyB}a~f%9of%US?qx~FgSz!x_4otrootM$er~Sd ztNH|U+7-scgmu-EY#xHv?Su52ZYEJ|?LlO)0{~{?F^^>nl_QxH4ueh3NA61Tmh_mv z?%Gj0S>U0Ocu8NhvF5p57j?AD6PbDENDr*l zc`y5~U0RY$9~$A9SL7}Yq&p7w@qX##50(2R(ZrY!rF02rUIvs-I5%CMX7c1AwL{Q) zPSIW~Qg8bR-?!FYVMPH~AC*8Zc&jR&vC^|u+X=%>AvT42^&}y&L)TTO9y(rht(}hH z$ybA^vDC(Zz`(m3u(u}s)|&W{kR!he?ta@!CqH4$tc(=|1UWc*?1rWp_d)+x#%&_05AAiYy}{mYuYuyI$TCO9{*%yrYW~VZ72s zd7?kxZ*S8;N&b{)!rcQRm@)}S(r&t7IXxyk@M5vc3vYVqW-}K=0F_Y(Lc~N#&&tY5 zBdV(QRvaJn$EH$ZD*co2W5HIK4Ur0!zA5G-&fYnbEkjnPXO)#m7HwIdXYK995B9`; z;5Qc9(YQUg=ZKauE04#|Hawg7fuh92&iPr&d0^KvdQ4?WT)=Z%H^CUw^JpF@*seYIR?|}Q+PAIOV7`oZuaT<_plbYixoF)i_J^^&c;hx&Rz`A? zp{Oa{r)jx@GqE3Bj~-S4_%TuxBXr<{oVY_!h#=s(Q^ZRF&$3V$4cwg7i!yT0eV5km zo(lj8F?>7l=U1r(I*Im>WZsql8zG&aFp{$;k_d8<=xJ&p$QT9P?epm? zE2C9pzg*eKxl;KDe=i+?5n?J)vmIQkKtyqHdsEPWmbQW?){e+%ASq9h9sb-}%e+=^ zW_?&0c#UaqMMTacap<@%{_Kg_l@Ml38XKn34RJvbc~WeKvAaQjIN0 zzk9<8M;9rRBt!AMnq&OAnHRbj#u#8&B#EoYobd&&Mkncxazy#^Z%Y|-BtUlcH`)uM zJ8YMVMqLe5^)Ycgw}#m}U4bX*N9myV9cLJ2bcXwgON>x7Z~qp133HZ<%8GQSj0hw8 zPLZM?iR14x>4%(>vZ6VD!thGq z^PNh{2owTUsWp31WR<)tYPVQOqi!gyk6E*d{~mkUoh8D&vu7xm#wr+f8oO@`6)HzT zw95KDs2k<&&*j^IUpjtz!0qQnw|J=XJ6~+~JF}v*e7&n&_PAeO>S()WHABU|{={v> z(7J#+$f3wEa!KlY2P|8r)V536%q)H+EycQw>XQhixmtmO7Ro4nl_3)TEc!%>rF^b(xKUCy^ZcJQna(Vt*2yaw%R?Y;$IM| zZxDhSxMwHoeJ5l$qoE^b^+pa_Q=IPU5yZ67xWpn)+(fMfM-F#TPiPLl=TbHu!JZUs zqJWYG?&rZmVx$;{l3G0xcHK_Z9{Lzg4>e>sE_#Y&xnrz8AVR9x5;Lc%%#~*0f0t*D zifvwKsCPk}lt0ll-R8n!rD`eTn-aetc;{}9d%qanoBA#2tFPOppY>@4UUM?_GxVn1 zdXxa$QLOkSFLbO%bchq1XDsQbh@14W!ex(m|J8C~vb*1}oJ92O8-#Dxz&EL!_OuXZ zE|QweG?j3qRO=w7I0tn`npQ2)lF%hk7}9yC=(fzA!ZjFA=S97I2S>lT_RakWvURK% zWoa5+cm|O>;w1i)J7d4Rr6HyDD&?^G58$!-55S1vwxcrBAn>b>yg^?_tkKLr*jzC4 zuIG=9_{{5u?BA2-U(L(*ZiJ7#eD6Ae)YY;h&m0C`1-ffT%b|rB4THNstaNRSJWTwM zm`WBM_U3C`*N6tR3-lxqirnGQagE&duF{R%ynP@$IXEF7QY`XyaJ@v@1KAo#oc${N zIwY-xOK$t46${yungYwNSi$gwA}2MfdK7aQp{_i&jfrC(j;9Dt{SSbUu(+Lgh?qWl z0%y8@V`$YVO0?Ma=%A<=s?nC{nx;kyd~8vOn7<`R_~O-3R0p1qAIYjK3YxiZ{wuqn7_;_^?9ADi3%ve(A0zLrlxOS@-8Cxhl&wY?5T=&-8PBbz zR6;C0Me%rR=+7ZaNXC7^we*2acQwB)4-q(;ALr8A2LA_1(Nm!r&TH2vEw#6VPsH;H zdF>(~JbiDsE+Sh1&xLVtEQFeT+j@&8Tb3Z%ZXz2I5iET4Is0#ejIs~1m17hxYQo&3 zs^CI6+ayM(Wb!WTt_(0zw8jb&Q~1m-yfdxZ2&iKr5)yjN946qnM)2AL7eJ*wS2&qrFnW)9q!%P@aJHW0w1WDOCFhFoCIYZw`MEI#&2C zh6Ektj!FjAEL5yW9!zwcyQ%gZu*eP<0X3L>JjHgXGrcub2k$<_a()jzzQ${G9b*pC zCRGv6t;YCK>b~+4$HB%SA~WQmFrAtyhjaRV5A37e`rv=Qqtw{P?)9l@|4j7NVBX%4 zeG43GzA&ll5McYL>OcM$Q8ApD)MZxWOQYro26Ggd&95W0J^g5t2P`><>a2}23-4L5 zc(|R(XF@NxQB&AD`*?uVU^r{qdbM|FCFACYOUO7#s>oXQ{x0%v>~@L;jG3-gHY;3B z&3$y^-emJ}3sY{+Kcug`bHp^#(5XImTf)?>W%D%K@qfjwcxV5v;tbT%J^i+y8k?BC zURvU@Ta>GXksfE04CszR5TSF$XN01UUA8YvX;pJf~h z+r;)`GLc1P0#9AmI6!*quU7=3>*U573&C%49wRF{H;03;m61AeMox*`V5j6N?!sVY z6rGAwly9TBjjNTC{VMOlu|`^gxWqUk5Fv)IT$OeNa$O~1k9Ap?&3CF6WoZuu6_8n1 zGfdQ5z;Iz}A)y2>irq5huSuLD3IDP;T8(2@UpTDZ!%)-XWuht)&7cWlITqMtgEp}6y3Ww3D4ELVr2Xume6N#cm zPz6yv%5*2)k#ZP55>W9gs0g%Q!oG7=e7YaVBIYPYIVi0VIo}jo6tsIKPUQ-EW+Gz7 z2gV3oXi^;@lU$A9;<~pf+%DXz$d^$IQ}=oYx&_;zf+qKgqXw6&S#;fT8pEGQj+|(8Mv5zKrw07pD`BY2@I$ zevac0Xf}m)m6-nkH7))*k~bo9mni3?*51$I?dlX#W2bf{q(eKJC%zDx>+g2Oy~#>_ zgAcOi#;Z*v1wVy$3bJWv8uYH6%*G&D*D3o%Qr9`cN*D|IQZ3?^`Rp7 zG|pJc9;|DO0UaJZEAzgXsK5$j8rCJL-RL zN}C$q-}& z{)f?~6izq3#5ovnP1i}%yVq_mx6<$xA5o3OMMc`utc6W7d87xg$jW{ZEg2zb3_4SJ zsEp-)Db?z?L=MrLlQV~f4J=)6nVAxO;f@ilL>cfRet*Tb`vSSwR4mu7Tj{MlnV@zh z3rSG#u9sM8sA95n2M4sza>XY%b0sRY>vL&2kH0%P+G@0L)L-aN#GHZ+m zT3CLhk-v2fo*;vCf&&xJBbzDdQ?YfEv>}Qi=`mWq74Pn74{7*@8f~i@;&YWUqAqb_ zjUn^4XpO!6m&0r^ea*NJUk?kiZTN@~?3PfSrHxft8jLk~hbEuxrwqO4iCn}AWjQCp zhTcHf?2azh{l()2wF&`lI9o?<&F+mO7sjcZM>|N=1%`{0aIj9%WtFpKPIakzEh1>m zkTkZSLO4QGWo0W0et#z4d*NRyUfM|tVzg_2#4%K~GsYOzO>=stsz~KiP?OJS4~@ZNA#;h@P}S#@5+HtC zgwG3;{&;*{MJz;j!Z=!cX1I{w9bv$Rv8MU7Hpxg*-Iw@N*mbYSv^U$M+4k;n!8D$F z)i%PE#@opGweU$}ZFIsLv9abX_Y23f^b+!P!*upKjh4hirSlB$@lE-DtzgP0mWB~z za6$w&m1ng7#j}kuvt1MmwQVFPxyOAsj?Rz6c-?p~e@| zcaO%Zj+`j8kzR0-5c;QkCL=qDKws}?QzM=4RhRE%2`Q+EpOdG~Uj1d5aNYk*q{>oH zx46Lj_I7lFL;}d;TfH`NOI~Gv_$aQOZyHf#kfw+F1h0LCpKr8sh%Jrz8&lmSUhC~5 za&N0L{Dip_D?jBPScvJGEK30+%YjQ#Gh=Y&NDdifbFGzDqft|gh(X;Rq2xXeBms=$ z84w;*#rDx=f82l%Te_IPJFoL)#dO_7z)U}VJ%r~?wlQ-$jbQ|vK94H<=&I>$vtO&5 ze$itkjlfq6K^AIg;w6!L^4v@R0Rk5qvohl-VGqfWtf@Oc(3a8~XrFi9cknQK{n`b@ zFu9@oo*W9OAP1u#G^#tW6U@<`uY2DL&$kPO%c|q-L&tC)(@T8pqY-8~0?MBX9%fZ= zBE}4>ABa&pO=7$C7Sz~A94|7VLNB4NDerf+^~UP=SMKLo_$Sg;2;Co?O376h8dTUF zDLnK2D*dHYwPrDx)KFLCDsHQb&wirv4K7rhST=1m1o$8|gtTVP2Md_+@p4+|oss%VJ&s03tqoZ<8lQDU9pX+5Me#zKFfnNY%IH@&6lhUlrBnx9uC80>ui& zodSVEaVhTZF2#Z+I6(>&iWCZI2*C;jcXuyRC?&YNwG^jV@lv4wn|;o>XWWN-U-!Nb zYh>hm$wSsR*P8QZAxVWo4L>aP-^?%PuljU3fM-;S1r^8|Ds}X@O#17wtB!9&%-$t2 zhD{s9k*;kv>x_RbHRlk+d4g&mZe!1P0?gATZM_D}JfKU6XBmmqStLYzH5E1x@4IP< z2rA0#8;_249kH0BBA96LhGpzcrnL;i?KU?H;rMnH1$|1svn}k}RX#5M`kl1LorzHM ztuAfj3|HxngW&lPv`o0?qH^MPmI5~cmycDrWK+-;JA8NXPpA}#d+%>F# zs}lV?{vZ!BcBj}BE1FPbrn9qkHz-$jYdX$i_Cc@RI>mGE6Ha}rP@ZiV4^HE3mykqq zI(>n)ujE$)^|79>Ri$ z-n7eBnGI^TA=L!L7)9)pDPUZoA|Vez_?6Jsd`IuJXhm%;;luVK#~AYOD?HPeT{l+M zzWUfDXOidCL4(QAi`=q(NNiHc{*=YZ-`U_|{dQyuxy;qZ&ZnC?mFZ`#Eln23{2buO z9T*n;W-XX=R!Qt~op(EKuRYhKhtzjC&P}UE z)MS%u9Ksw~{zYXhDK3n73KLOt`~QW+jtyjR`v>4!>mGGBh`3xAUt3kB>*(lXwMX(5 zE}ootnO?3E{sRc_&m>b}Za=kgLV2a+AM9$sd$Jy-iSoS|8f>h~e*lUT#J!H%yxu&k zJt=YmBYu3&XXU*^S zBT0q6bFUoR9!m~>x#@F0K6u`3G% z48K++v5^P?+X=`7QZ=xFgXp=uGwuTOv2lloPlrpjdYiC}#Ek?ANP_Qq@ok`MQ@D!S z*+iWfTU~JLubkT>$b}k3Afhx$quLmnO^+fZFoIb8SQ1v}s(y~=8@Z4+YDmjIf-K6N zhpf!S=)i>z%^cde9m|JLt;lBBLz6GK$r|IcJ1h%TgC31q zGi_^$C47j1*KEsA1lFW~^?WJqYy|@17k7;S7}0z~%fms=;QGF7W%rPKR()rot%FukAkiU;3CfZQIHKBRIQ?tdFmRbJb^BInowFZQHe8klo* z&nh7bOq-O8AVO`<)n-?!(by|KxT{uw35H`1rf{yTV3MzdCSF8&6~!4VA{aa?iFG*=&GY75~&0rI?E&(Ra=#^a3raGfyW-VbelawZ0Ga$P9~ zKU-}*IvdU_43A$ut0_12{;__+0;wqP&nQr}X=;F;!k#!w?n6J#0iR}hds5QTn=#3q z4MMV1XJiJgKO}u})Z>sP54?9b0Qa5Xt!6i(l|rGBiBM~+@Lc{ayjTl_dI>i_$y@+X z)Aep%5z&Rp^uK;y)8bbYZS~gWG5c9>1&XyeLb7_C6l!klUep-bkqCJE^1a%0Ov&er z?_yta)#j{)edanIKScoU`P6tcJ?|7i=P=#WA}7P?=GRC5 zzosjrtbOGN2Hb<$KJVmcMtz9F5R#0EM-0O~a8?;n_RLo3E)Sh6?h@0`U>7f{Lekv2 ze*gb=SWcp#iqMIu*{ zR38aHGRV(1wdC&&59T!M83>fzn0J3iM4Xh+OxT~{xRVxsy`HByhpG!A(Rt_jBaX!w zHAC@Bn+#<)Wg6OPze@Alcvs%aegb3&Ux%<0qUYC~yl+&6zf8W(qvbM=I!X=X!mv0CERPX1L}P@n1yKu@*qc7N_yh$(&~O591xv-InJyT^;vb^x=vVi*)Zjm$d6d89K%tp8pD+VQJ& zz4|6wJauvEndj9s5E@=B;I+Q5W6Q&I+XkRp2?|vG8FU7vHothd%1tAVY>zfW zdgB5QzGXI8B*9St1VuXw#LVbx#j(*~tK9m_85!6htAD;33+|aBa@D4BivgPWwnoV% zkiil$f1rc(WJ6vo&Wds!s98+0Ykbi8S=r&Sfo0u_a*GFaks6Na`W`sGmzxp1%wHciX1K!)5BM)hZ#eo(z&5< z9_EaFz6jNT?I+f)pAs&Ua3!92qSw^3HzS#1^By;T$CxEum)*WW`)=2j$yR-yt=^jn z)C!PxN2AF;2JyQ7_RV(bf=mRoGVH*Aw38kw?R5v)QqS?9Oj%)S>vqAAPXU=tcH6Lhd-c3NPyh>;B=RIDQb~2 zI?-&Xc5@;ylj_z*jN>F_=$g0x&v8u)GeinZ zJv&YhjHY3Ly;JWL9Tt~1O|KWuObQ1l`VO|%TkaUBtE1&YqPe)`>kpBk{enb~KfbnE z)h+8g`%R6yRf)NJ87y`?w!UZh)}9io?FgPe{upkseTUSrBc_~H*uRb~4w?vD6|N*z z=ROn5x%tlU7Hg%>&h}BUle%MBa)?V~*BT%1eeq2N4|JrVb%0KGT)BSi_h06}a}8*l zi&8o?Kz5_ z_@vHRn@lEkklCA>hAWV`Uv*tb@P@3Tj4PQURde7kt#u$M;rlj3gu*!w8a1u@NkHyq;=8gLN2|yf^~8?h{oO)dGDpj+;g(_#fUGo8>5$>Vj%Lw0|AOS= zP2wP?oloL7S^rY>TN8REduSwWeqoyZrq_^))mH2GdM`DNz<^8QO>N*F4hynMNNFsx zzBBPaFY4)%1X*U|TM;WmYtMw&#UniG9>j0H=0d>7skJ6p?UCj(Ic2P_DaJLQ+WT07 zP?1YTBt+GN%xO}JaL~spFyRF)C=!;&VU{lW1B^w0#CtNu={r+<6&Qc*%!p8HZ%qgGMiR1xT%G583jaE7yG-F% zwu_<8TWk1*Pzvd3sGI7&o2OKudSUgda=mg?<+0jShzpW}_5n|H;3uyEytH@~^&B5M z(9F_3Sz_9*q*M#d{j9a)<{!yP zr>BGEwan?p*OksSr9rzZ9Cj7ur_ERY094)+w`fdGnKhg32B*mLZg<>=7z=EH-r%o~ zMC1I#fHYL^T!8Gox0%)~@l}EQL+yN>_5(Wh<3BDndRdIk7O+JUKLr;rYtjZy;p4&P zq0HohOjVBO>{Fko=2bu3XvvoMM1oLrivmH9N1iq{j|MJt$({V!f3Y!|momkIZaQ{w zs~rLDxDu>1>nqdoHD+H&h#OGqI<2iZ8W6I7>Wrg)Wk-isnKoVyCid*D;Sd2T>~OWM zgq$!{7B`$F6iambw|$hNXmqe_A8^(7c!jg&dteQ;*-L$H5Idz4lly~x@E0`YvbV(A zP&CVt+3JShij57{A1Ea5M|k_zw#BqG=&t%7fCM*nkVr$0Y+brEtE2u!`MXdCZ{=^z zR@A(VTz#v%!(w_$6>QJxD&h;9i*PqF!+JcOBQlV|u_&cBJzYEQZf!r99iMPZQ&d*MY0;^Z&TOey}c= z*SdT~qc^pIej_a!S4JEv`6Xo)kjpoIB=ano#PjjT&tM<5MYp5vq@l|pV)#O}+1Wwc z$k@uj7!3c?*~68|1SvU*Q3J^ef7|`SH!s)=A56^PqWOD^BX6~)s}vfRd>o761H!)& zZncWo>K0B`Fzhau{WPpfO3vgTEOU5uo!!{ayxIw+#7ra7Qw84QbHbdKj-J=e90##W z8c)ad%4@a1G}Y#eWHo$*r>+MzJBTn;rOp38C+7dcoALis=EJmwQNB9-3Uu@>M?l;u z^_DV+WE@bUQ7F-E=OF58u8r~l?e>iw`epA7zgv1!t!p8WruAdobnOOLmXyGH;NFz^ z3G2IpPv8 zCQe{?l1SdFgUaLZ98MI))63wSjaqDK0~6XVmwOG(P|O7+qzK`XWnI=rttS6?gOE-MfSbznqZBG0j{|N_UR%Ob5TxlS6bfUtCohJU|rFfO4dl`vow`5}C>6 zz{xKGq|8LH3w_)Y#lwMRIkOqgAYs)Vph2pJQpGJ;o|n2_suNbE0(lpRiC1!SbQ{)! z04n*S0PjcfV-#A%XMjD15IjGTD6mWyENhtKAHckJ8_qv~8U-}FeL#t7@1Y)H7*OH5 zl01XEz4X{V)c6`@1FF%8J;wC|#tI!6dEw5yyU8aS(BLY1J;6JxrdTS-9+~Q5GxUT- zh$X#SahsQ-?H<;B{btBIXl(s~S3aA}m`SFc<<7 zW@b8nIyibWxf5yP&HjkdS$+Jo+XJ7w>nI;+huiIsiNq9Jixd+rg|4uN3*_kzC4aYR ztg4FR!<}=GfI4;UnRfzol+iTsJ>~EpQ+9e`FHhu9B~5|YN6&^wy(?I}?nSSgIJAE| zR%8Vd^}Qcq5UWgsJ2PY#$7L61AMfiL`hnJCUVYYC)D7^U5oTRaIj}Sb-MH>NYgz-{ znvIxCt$8@4XTBeb0G0j(!;~yE3KCy^OADB)zeN-HyqtICr(+|pmuqox35`gAIf!vK zBrnUDSfCXv*$^t4WTVKf7Mo^OWU7mb-D%QLrSD6$CSK%G=zPX~VVBljq9M$w$z_Cz zi^}Dbgt3G-uPTn*?H={i{x3J@m@hVxse2T<dAOIs*mY$l1;|y%0|Xn2RrRa?tHBm=u<5buUl5> zYKc791Hc)H%Gp?_unQ&@lc}d~ftSO3G3`N?!qBBH0-Tk$Wf{h2u1#~Ch@FO=m6{Zs z&p%(|U^I4)k3oj9gyUm@H;*U&0k9h=B{~yF$(6xAeHzFb-(FLc=iv5T5;};4ZC%dw zG&i--ynOf3W7+-~%MGEg10bv|95jewQYm?1iA+cFte+)bTfFHCF~+HBOvdqu!iqZ^ zu&NGwX>syI+)v}fB1yY#Sd}A+eG|q*W&E$X=Ko1#CHc<~j-qEGD4VW=>G`<4rR)vo zC*MN4j$5=P6m9h3Vhb;IV&sDerHx5}Y|NwFMJ9k5BBnR;AQ zaa9|Op4~B^DcG%8TFK1jVKmovh6G}Ga5yDUJ@cuj3Dng1#{T8s*>JO~4J{Nqb*+;=H(lidfp|Y!rB6x7{ss0l2+z0aSh`ERa z1F{-T5=mqeX$!BK#F=qS-+_(_F^%y`F0~>qDfsJ~rDJ1tTC^KB7M7##%Wo_h^To8H zo1F}j#q7`GoKp18(J!G!krTf9M9^b}T4ByXNQ%r%q#ng?##daUY~v{Vxk&t%#@K?* zoxBD`Gf22Gr<;v&luNo7_@|Hp#Zk#B3^{{k!bU(9n+G$<%qQ%~e<3D>$ZEomq+pgd z;@vG>d5j_XQh1hFrMrNM3~+$Tj_IBZA`1A#K@Z>(!Ylb-yy1k0dT4bmz4wWLVb1kCaC~3`c&v`mtr2M4v`{z zvC|uDh;Sy0)koFs(CM~Z9=cbBy*J0E@0_6r=BstGs>(dXiv)SX4;0XTudV!UEH#FK>=Q7&j|zErAig$U@q+jURp8;A+Kiras2>U0M~e%P700Kv+WdPubkE zW3dSGt>Ng4lMmgp)&1#guhQ6{vF6miD zW2y@i&A>;+MrPuxops%I+SZE+C8i|NwS=d~CdV!AOa9CLzgIn*%bsT`4qr`(=*5Dm z1xwOJu{z6^bkA5C3Xn z*_SKFW;>q)fLlV6tPAG}l?3?%Myx)Ck*b3KHBov@MgQN%bfO0TF-QoDOl+4~^wS-% z!`<0E3qz`-Wk_TAyh(6qn`aB%8eQ`L3o=EhsIea3pIsd&u*m%^*+iKmhi0FA1f72C z7`B7vx(B_Q%e^yfZ|{)KRno@2RSONr?%1=}zca-zlzRA1Z5VI1SrWM7yEMn9?MemM zSigg%Y8bJre}g#r+Mi2ReBl$^ic}!>eq{P>@cDXWK*I8UL$bV7t7k z1rl4>-HJAzGbw_XPa~dqrP&Z0j%9On2v;NgVE>jZqzPBKzJfwNsF}3oIk?#b^+YY( zxSqTWAOXBizB=b>P20Q>_ZP-f#9xTDonewuLW3^cYLgR%j#FrB0cSK5ne0(Za z6U-*ghys^{xnZ;FN@D4ur451)6bP40H>+%hK#4wwX4r73;U{dL5R(L|<68P~!UN;% zZJ22cNXiiAm)#~x5`#p5=_KIv*$6P=@#3<3Z{Z} z7&yTCKc-U?ra(MB=0MuIl}F{Uu=U$qK5FWUA^;y z(bx8Uu~k)<+Qi!Ud2;fS*0k35Q+#gLQ2Qxm^$NF}#f9@*-g^B=^pH&QcZ1g~h6|q# zHMc(s?e=ZA-2~y-*hW{42XH#nc_mXzi`%5SVuaxCbYd&L^)%mL1>KeJEDB-3IIDk6 zzW+a{QDj;GP?~;b7IxQ%!Ihk(k7s`a4e2n_)~DR%lM2$gS3hsU?UqM%R!!A3>+-_6 zUMJ6ez7Eeeb88Du)-zB3dKSHphof=fskEWaSkGSJ@(;k0JMd448OP&j?qqe``HuKN zDYX8cfsSCOb&I0vQ#vtrepw|{CUqE4I1*$Kk_bVnF%T8IqyX43Ud8%mOPK}|M)c1g zHeS3G>R}I$FKbUZ= zMYFyyR8K0b&mmj>po<>SQzz)qb4x?zdWP>ZqVPvL53PxY@9_fo|o$)|+$ zJ&e_OXlLQGw8oQG;0oIQnG97eb*dTQ~K6YUm)q4KXwQ{&A3xMQqv*UD9C+`uVz=FAfWI?#Wy=?vGaI zECh|cYvU!DsgzNXz;AX{WbmU6gTSj54^aC$dv0;H)47*kgY58R=t^XSpHwA9Ao{4! zr`D7%jM2%gyJS=U<@jAz;EN85Cxo(1D;*ml^M`b`pfq~^kr6+_J%U9ljIjfqzzHUr zqbOD}{d_bs`-&)p{_xFoJh8WXf;_yTt3%!nZ*~L73-^O#POc&X$=z` zD?aM^v{JRv{XJZtW{Z<)t9z#11(}`v6gUmamOH4h4p#*K0jL?fU8lEJtDQDhP^#yi z6zAp^cCO1(GB_`D?QqCk)rtRFNOOccwRU+oO=QQy)uvPCTid3qL76IvaS?S2g); zeXL2i-i({>^70-ioD_Q^@suOdU}RG#8DonYZ1~!$+Jpo;RKWuW4l(^4qSK*w#L7mc zg}W4wX!x-rV{np;URJ;3^@Ul?w|d%Lz5m=|(H1>&>)VcsR*662Z~UPju2>s8vECMm zAUHJPXQPN*tG}-b8DWYmzLMpm8`_U*C~)@vi0e;~2es}DEFZKgaLUI*18T&uj3tM~ zCYGm=4T<3p|3u=Hg(8(c+?y%|5*!1FDDzSkPh}i5aN``Dx82jjuJc{o~x zLX1x6HY1pISw#a%287i~!9T#!FyhS!rYK;@4E{LVxDY8C2+kDU$;YE+r7eX8Lf+Aq z@syH+F9F3fhHAh#AvK`eIDnO^G*T5H3R7s_QV<2S7Y|6%KNeP@YL3cLAYt82XEaJt zhv2K2qXhA!Ka0YfW7T|7YS`JTVwGy5D8fw?7&^cPXU|w83Kn~kJ(v0pAEp3a8fR-1 zC#VrbT!}CX8MNV0vkHB3DfDnG1`Vk6vm=LI1($y(Dq}BQjor;+k2O|Q6%O(*qf{8s z;9x2z!htGK=Ri%1nJTx7$Oe=Skh(a0LSMEP^3#c0JX1=PIUVAuSou6j@)tbswZ6j5 z3Je+=5rq|cf}L^WR_q!(-uyRCPYuFU9o7`E*qrvJY65POPW2pzu4@B#3BL6YIm-T7 zRIwc9w6|FWqIU;7&1{A;J&VeGU*WA+;oUm&=T5F3SPCl)>HUD4>sU^5I@RO)fC0QO zU21gl|H0fJDN914Mt@H{&{3;tIaI${ zZwt`vT6F#qrlvMWmGz=oopo3CcM3uK0cwl?=~SBK&PsPpr zD@0%8YBgwa10Ck*FXJ+r?UNsiD*WLoEKhR#m&x}3?q&Yx<^Ol5BxZ7WYK5775{Y9t z`dPz`A(3B@#Z7%%XCK&&YLsJmee1y7Q-)Jf$9@)A#JI`8$3bkZnA6B^pSzZOd-!{? z@pYC;CkZ+^xH0K`klAmEc?)(XPrf?nEiCYurge@N3>%9J~^ z%_t?}8=L{>7fbVn_&*R)nG_T%vJfW&Yp!}%6@wt!in%vkvxEl&y7ay&8cs>bidj?+ z+2~9m0L$nrG5sS>_?zX5P~(^MKDaP^UD{J45QiAO?|_U11VF$|^-CP3$~gDVfG;k@ z=!F&3e-qVZAf4DdE&x`gR%*A8)LBz3$UyKI=;E>aOEJzyQ1uPS7;p&yOZ>oks%$A# z*o=a}W&BJe_)EERgBBK5bd7wAuqqQ!9~(&uget@^0Yi<0924dMN5&DGC@+}Q27G;xb zUOUs;L%Avrup{X`Nx;n{gyZ7bdcb$P$j7Wu!z6vyXhmw0LXwTyXd{XXBl_g`L8VD3 zoeS+;H;N}O#ON63#vo5+=HLkt<6`xAGUvQp`5@ouBpqK=_`R=2a6aKHsDUuljB{9w zZZk4%WV6sW8r67@iZYCb;hQfHLn?e}C%#*CPt9AV#&R!22zz^Ge6?gaS(Fag%a`;V z{j+GXb&7wNbBKKvn9@Sb?~M-~XT0i_&^#aNOrHOk_B^C~W;w;kB6){N((K@Wu*&NmP7^}0Fc)FDWuQIk4S(8lSLg=?js0>Xsn*SABO3x$uU$zJaZuwJK7!vKjeW?+YP#F+pzOm{= zeAjZ=?b#$`vtEo$IzauZv86k)hn?E zX*f1^5 zJ%m9F#zHq1>HCrUqQ)Td6%FkPB%#{cYb<)#wDgZ#||8YT*(N0J;)cbiTuRe(poiqsY6Q73ltO0JV$Hn2{7);Wcr#1O{2sglL?rlm4 z1(LT^!#8b6Z2AetXq26B=H~dSy1MGU)I`|XO0AW;qL?gL>QxW}Cb97FLfx1(q=) zilD3Tc#u>dp1yv0b&dcXhYxP)6J%{9Lciyw?^P{&_*o9e-1a-QB!@@ukYmu{)5RWe z1&ZJH)U3;y)A^(_2(ZRZKw7z}pyr@{=t zMMcPTLhi2W)Ec`G*YnSGN60|Bl%xKJs)pGAQuX~$K~XG>{?BBoihVM!s8L{c*zD8? z3cT502<@y;dUG7f_g-YU72S1aQs`vzo(Go>4`NpdyWvnZxXekiB@$)?>)AmMm8dufqOv=>h;AM z!}gUSLB+yz_Em4nI&2n7Dj#xbKsQ4GqKQ_cnPahHm`jS(rVLp{b*VSDM&a@!&Mg_! zDq6oy!y#L&oAPJS7Dsg8d3Z~MJ1t^FWD04qptJm>pDx%`9cpb5Q4_h+^juGb?GIUr zZOp`@EBO+q<>o-KLbO8FXdiVZMAP0~Nn&(kK`n`Ew_1Ic8vA_S;pHq(FoP}*-8cv1 zRM!jzk92~Tu#iDPU&TeX5$%s*YT-GKNPV^4NXn$X(x<}%d=gY+z9{?Xm@?pS`2=`h zL??*T5ap|#;P^2Gwi~=3lPI%6Ehrs>b-f7F{0<1g=V! z7yxd{?A!B=W7rXw9aI(sj@kIY{F&X17>6Vs9Q@)5FrdrAThRZ93oW@c+4 zS?(XWyq(HK?ij}cEIfD%xYZD|oGS|P^eRM(ulM)J*EwKNex zsh?_nsURID6lr8IbNOZMOwU1=_`<++5MOdP)hGh6n>FW>6W$G)1RQdn=ShJnzd9Kx5%6F-*-rYevkV5mKEsR{md3Pbj}4J! zQ;9m5G{(aPf6JOH-(B_(?sH-J<>$+bb+Ud!S5)Lv<2dFsSJukZBg%}U=Q4h!gp&Ih zgiPp_KspFN$8-D3*z_Jxw*tZJN>xw1w`kT*jAi`YZ@NEsEG6!$%wJ8PTVtarSTPs@ ztWfD+ZP5SBLH=i|`frXU;sqi{pYS%QIlvL~{LcWsxjaqUFGw@uZ;K6LK4s&iM1JeS zv9qjgkv`l_NT%^KVb;Li!0>qWX}{@K*bO^)WFiPNbM;T`8S}n$d4GlS*Q5r{;9WRB zY-e2J5XsBwYV$M7>_?e9(P2_ERpKKDsa-Ze@`4^g81=CK%4&v_#9J*@_Sf6z*+~ah zA5{p|=>lcLTT+<6izW;6S)m1IVhCECMckfh2y0amNT1~yTFr>d`xo&R)wm3vYcY5e zLGHtdlSb_^WryZyk&ve2uTpJq2OLzfVan$-wB4rrxuS+a#Z`qPpYg%{sW(%C_A_Qn zxuq&(VrpBRnBHV65$`z(7axa7Jm>(Gsi&)!v5G>8RbmM(B$Swoo=Xi#*6tLuB_%_? zn7Fl5^IjfkfQHL}1A<0gyAjVIE`N;f9W(HR$ZU1Bil}KhN_gIwK5G1OroWE3vJo6A ztBUe9brjIcJk+<<#0dVe5nvPeY<;cv3_=8>*w%SGE{!*QpsL)H3wNuh3@Yg@7M%(n zcuL1AI{rQo+#5Ndfv@_M_Jg7g%7^O50PdQqjd25}`x4SoSTn{K(SoAf-$iYUYiv2GYQozK() zi=-wY0AY=Mpe)!h))(f>q$KN^uDw3BG)U^Q+&TpnNSwg2 zeN)Pe3Ab2}9ZrPhK|y$-W0xxgx_Vv;P4I<`dnLv}A=vVD3Z?4XQ5d$mQ!wm8R^jXO^jL+=*Wr zOGmxnVLxWq%v#Inh8)mut%ma|hiZlnUy(~e=#ncGOcuhXIO`bA23D}-^p!0aX-(<& z-AlQjk6e${k5!L$KxZW%+&iX^#vdB(R%|MDu||~>GhW=(zf82f8nv)Kn6cxTBPjE)j*V0G4F+SzTh^hNuRqK zxubZbPTlguf3A5rUsJ%aFte+G!+A_kKH^U(i&g=+v}=XKonGt+t!gup7yL>jda%X% zDXR3jD+ovDH@n&hcK=B%j?~bG7`A$i#VeWpVHsVj>1^`v11GYT`_z;BZB`kNNDExb zSHmqieER<|n*HML#zhpuc;8_tnu7m%phuP$ct829tJ-Uj?y3S@Y(T)UY2iG3&*8sh z*4beM{~i<-{2*d0wDlX*)*i$3rn8C4gM;y1A{ecIlUNd3`kChXV+CG{3VH?C{XOVL z`^7p-&g&P5vJ}HJ=fjZ*joJI$%6xmGk5-ZuXvNLUAya)D)O34;$*j@Vgj%vD9f{r7 zNZ;XWsm&&8#6JL3aiI!)w^j5soinj>Zow%uhy-dCc-$(22nwLiPH-Icxu2lAoPa#E ze`mzdz8J79>xaG$1wKqC9$33!%QlCG?{lBwpa!zOw%rR$c${5gH}lgt5f;@~@^-ZP z{vCH)eKUT<$L0U^vs-6~Y0n0Io%_T!bBiyHthyxK+)#Ob;c}!q-=eaFCklejR++S4n>VWL!YwP&@m9mmw1XAZm{D z@JMmco1LvuX55J%b?`Mg<=Qb|W}JJX#Rl&=!s`_??sP!v>T`M{8;36FkX|)am7XqLlntnyyr6 zv{!V^F}J<9WHW-H6*X$)f}Fyk2H(9dDiLtQ7S5g#>aBRJc0UDxhdAbjpK3wu8C{U| z1T_E|0MAe<4Q=^YRK^oB2fYabu~&9eGcp-J=dk$d2NWjJSmm7AW%7JPvDWSL5=>lU{ZPZ;#8t8wiql-s`@EI%|wlrdn^UsOpXSzF0=6rWEeZ^ZSx@-59< z3Pj$<7`D%0G`41|pARy(aZFXwa6o*>Bc1Zo5=AI_);O zH4iz_?adTS5K>FsZJP)Pv6erV-pyD()W^VlGDOL7*+{Z(vw*8A5tihNkz1eUsiB>| zUg~r|p-(N0@~o-Swo_|`7h1FuE9Sj|AYz@ zIL;)9s{z^DOeX6bUiD=9aep|vh`Hk(Zj_>mDribBo0YA~{AJbLKqv7bfVXu~P&i?k zCE)sw`EkHOS>aKvTUYYd2(pf7Nk;I7;H*@0&SItB_({$;%Q54i!EOj|(%f!-W8!dg z!Xs|C0m~}Z{|s63|DKIG*`>In3;cT`Puj9!D>ldzoAe{+$>*93v0*~rQh>~gLd=?dmE{s!y9Z_`oW^Be zs6Y4(EY^Em|AAFT|8Cu)eqQLd=$;rd{m#|Rc95$=Me!B7)-szwZkakoTy@{T3x~&ye*#`YtVwcT8ttv$VYxyNZ64{1SV&cJYzFQ8(s^M#Yn_+ zRG)D@5_Kg&IeNSl)70HWO5$hor_dOrr9nl6TNiE{B9F~nGYBW-qnd0Qmpg@zhku4* znI%21^7W}=15FbeURBa8+I8Pf0v;x>J54JbM_1L`c3lL*04zT&ne|s|i&KJ5aIXH| z?gMIcM(KjyGt3xv`f z-WuSUAdE-Dw?pzmIvA0wN)q}l>Sad2bd8ZQmfgJqNu2Y_s+OQPRVRkRkaw2!z4}6) z4wzO2d@=cZ3YFTpH`z{cm$<(DplkZJ&Eo<6{WtSGXIOAK$>h^hX+4iR4$yn4KiB!h z6_t}YpCM>cK?e3(^{_Xf&27QsA=bb0TNWx0<&_i3%KHeB-zQAG$K3@=IJ#{dr+=nI z+`d!%17LaavT-B&Jn6UpxYf(7e7yLS-_an4!8VX(6>%5Igqi7G89ebf&s&49JcwLt)1r`ec#WMu1a3%S#VQm*}0RHEzW+cZ(hA+3^ryO zXlEE?!E|d0kTp(qeOHa4x6s1oV@?)wiQUhzo3q3Rkwv(lv5Ii^mW2QtDh$<61Y+`u+LD*k+I2lM5HQ>=dL z7&U!e`wS<-?pX*NlL_Qzt=oLxVH<2auOx2Ti=Qs#V6xdx;sY7US?5*Lk)Cb+#Vg;b zHw-YqX@F^YZR!fR0Yrc3aHWI+NWR4ug0U~)e`Z}XR4`Z5UipEQ!lZYVM2swZ3X~(i zg|fGh1Y4#jC4AlLhEs`F2utGSA$VY8ty)MFFGEGYW%Y>AU00qB3|C)t6Swc}C;6A9 zE3xYxW^4W2OqGvZDIYm;ic2w6M&BmqboPJ~N26Cu!I5R81)O)qpdECMeqDXR>DSQA z?~XUm6{wZWEI7_VcI$&TiTda6Io(g}+GAf_dH?L-{uY68;n@}$-(DT(*s3r@!E`Mh zQXtLMOK2QVOopf#wlbs5Bf4pI)h*VLId2>ruYja{GZ~B=|5`l+AKY5UmIaBMhYlyj zzQyKkUO{oLmZ8?CLN9LT#i#NG8g=v2U&B^xv02CFnp0zKs zPjG0$+Ppre3Cn_?1tUsXKf05Xq9d0_cH1UL1MsVD3TJzfd=*40|3cR$1T13C4MfyuhZC_H) z{M3%wn?>z;(QJd({P;>UhYe{(qBI8~5d!BT=Ig?G1Xe{+MQ63mzQPu5kHllF_TO+w z_t&Zf-3&8tmGWV)H5F06T2F8T8@v)PAXa=LI?xM$5qN}=RCq4k9MkAEx6jEf(^o&{ zpgg+IB=gZP2@pK+gT@nNAT%af6Sem4DTLi$zR?vj`PBMG>jN4(m2~(jUroC!@e?@N zQb8#KqBw&jvchP5c?vV4{@tkY|Ks;ax?S)w-qFUPd(2bNeXNj{VZVYKqz>1x-W!znD9(s3yN|-G|;mG(mb72q;p5bSY9oq$YF$ zktU%@uPQ`}7;5N5IwTM}(ghU304kw|Dj-rt1f+|g|C4>%I2U`L%kSG48SlutSs5c^ z&3DZ?pZR;>vT&sS#EV zDDqgvb(-6hLD%!MKJ|`!b!&O^pl37;{?io zPnFqOZ`jH58f-dqpP6Z#Tp%%h->WY%F49l^19*@YT~*UaT-UoMS@1AaEpyhY#`})Y z7Y9;7;_vv;AZsHwTw^K#6p(p3dflvHy0ZKAxwtnly1O$Vo~FIChHynnL4|z~A4z4G zn|`)p(;o7rR{`o@5*3-L&2xpI%QHI)*1YtSfuO{gb5rpVc6e@!@5z?|T&78(t$5T! z=Ln7Jw2?Y}Jk%m~v^7mH#Xzf2-Y%Q2#g|WJK|zN9qA50Q02gb*_7}f?j|`H}{3Cs& zMq8CUpnM~>>usfyxHZS4)AyBO+>7WiE}ZQSt9yB>b$pub+Op8$w#`b5JYeE|3io$I zmETi|KddIb#f<*`H3I&slWg(P|Cl+XrSR9$@T>lS&e{i-+Mm1Wu6#k{$cJw!{Lr!| z1c~IoyQYladjBKUpVB)Y>I!Z8U}hGuW8SUt<5BiY!jC6Rr#OEFJSy&OH{a{E$kjJm z_MaPH{t>WIe`8Lyoi>$Hp{KP9DSWPW5Lxl{AK3f+Tv2ms5_IJfA*pKkAvaf zCn|$m);rqSvnLE^9)!~St%_+~eEd%+v!l$-eyrx=+6M+3k93yu^()d1{@QA(t_kT( zG!R5|7_gsWIRXm&wN+0T&|`5K!<2XMG=X}2Efu+Liz4YVO5hNC+8iu{J;Gn#_+`mF zTSVHT4j+o5Fim_Nnn+w@Kr&X<2SgP95UBEdj5aXq@pB%*Y%ACIc?`^IT1QF&tJdOj zBx|4Y<-%{#^4jQj=^fS#cIMcEQ=)zlU$nb16}P1pX3c8OO3aln`jftxBu<;tn1e>f z8I_@qrgar6nz#9U=)3hp8{~40`2NwUt$xS3;Ikah9Cc07!A}PAE-o#D9@!Iyd!t<9H+N$6!Dtw z&wK-1ZFMZt+%7)yal2=Bbi~fgfzb4;x6OfJj(0u@z3zdYD$P=ck)&x38W;!_jtbXv z+|G=asd9o~E2>8Y_qs&&K@CD-G#eEW#&4yblcLQT2iG0dwkmha=mt$nuKdbGNF;Io z`I?;ICtvXJpfbSr>t>TIkJDVQlTOOtNBfm?@4srdb;<2~&n%@a9*lLY#X89C{F&84 z6d>%S`jmGQ@9TD2UwrGZeK*;#PPgKoGG{h?^=${q^SV>0WILu&~`s z=XC~G;hzUUGd@L#=v6$%vq18t;I={k%_Xzw$~v2l7Mqm(@_EwCL_KYuzyQRK!J6L# z|0Qp(5O&jq%shA_#H&nr`1XS@V^g0ZPS0+p<#JOvu{$mBOALJHCnS#uxM4Tw^LoMl zx2Q43?rU=?-PGZfqldq#=$iVhnaliVL2=hJ=_lvc5M6uG(g{fhF99~#nEiAw+hYHr zO$|$UJ)dZ!=aig)Jaj3m9FOFiu5jhBPx_UjIl?9%A@K~I$(m_*_%qHr>uvb-liqC| z1I?3FL3Ni#Kf@7@pDVBVh_~+xY!~(nF))Y z@+h%4aLO~~1tp1e{&$UB|7ZJxNWVy0D|gflN-H=UX$Ts@pb5cj)JjE>i6C^x!u}xx z?LPnp{^F0Q@(m8o`P!?Wtsg1bEBPFqysJ(5TXV2Vvwd$#*of;#ZE>Vlme2#++*c8G zoi<)4f(4eZ#>L@&+M3c)6tvEEk#rrwc1W_c#-btT-SavZ5~jzegZT@wV7jN7%O_MR z&3E`g4_E8ailD_7y3f!!lCSMUOlujMhPl%3wv|$VnO2H(2$%WAv2+87pCXKsQ-J|^ zOu@-=bWm|tc^$qgiB)xGe+JQ%jt8b1ywt1wfR(Z$a|V!@Yp56Q1J4OQZcZn!=VR5T zL}8FZ!*0K`)U_0wJ4w%ae^R+SuOvnNV@JCe5}6fK}B>z+Zx#db8l$RYcN5 zi*|pj-Rc(ngk1-vdQ{$K3Oy~+SMS%T;xdGaaj(BF?kkzQH0<5(8qKx%cBEB5Hp2I? z{NbO};ujluE=-)vt!crPyx|&em864re`>6yZe;G3(Jc8fu-DEf-tWIX# zq@{dC>Sz_pB&=GN-{zG^8~K!VY(Cnuv7Qxupsy(^5Ts9ZRg<}v(jieWo=wYYwV}(0 zGZ>G(fV`9*xYys))KLTex=*Q`UCYesj|x^Q89zrb;)c08?|Mv)ESzY?D?1+=j3=|A zJBFw;)DXP#ncr=IL7GgTcOUv`KKIG(9`2| zthL>ePHLGXO<=Ra4&S}BKD_OT?kqQweHDu|{}}jQiu-F*ah*9JdTBYB1`t+*wzFiV zJ3g*X+-~=(XQ;c&BBg00SqJ?z{x$PYb!1U!Q7h0`o=A0i8;pLxGsG>-8!h`>oVk|g z?g8@i2UVh+z~CYcaGjfBeWUEGd-#OFa-}UsBvb@l#9?0}+Dxr8ph~iPOs_r?Wd5@{ zs!k+@y}mlb)hF-WDeeXPh5;o#F7~RCeoUk~)1i$n_9|RcxA8w~Mj02uyT=*mIK(zC z6wOhufb(jDXPfAjPVK17)2CVDI@y>00kBaw>91mJ`~c#~QIS}L_c?jc=WobUsE$j} z5I;gJ3Hd>ToR(F4GCapxhP}}SFylU|yWADCi@cQ)vCRB3*QMGnAIPO*F*;p~drfsv zuLu|68;NkrmV;Yzj+wt`qV8qzGHjPDCEC))n!eKvuvApt`K;0L`I{G$FPBctJ)s|H zvUjj#5QU9Gdq z&k!HZ-t`?(!UFq~CpCs2)1w8SyhOX@l+sEk60Uy*x#md@)YM*2wANSyy|ueZrPmam;WrG`3%#F#fi*u_O( z-y5zESn901!}Dm*`e+1(7rU6avSqBT{JUTu&UxOyXUudy!LYRRA2Hv!f9LA`zS>1^ z@>70wbvucxfhHG@(6!>BnOw1IoeQU@{{SX>%)Va8=A4w|+_?TsZ#6BT=zU1FB4%eA zBUe~4$fxhg{A`hVfNGb{%sRObErtJ=E8qTqYCEg?a>KZ@SeY+d)+RckxE(kBbE2sf z&E|dXZ@BvGTuWh*O&-*c&)mCDs-Zjf#+SM%UJb`94TBtFMdaC9@mn>?>F-lF zvE7%IjN65S(4O#WrrVzb?l%6}DSc~f4qbEH4EqOAUngCL+BO$=yC4ZhFI%~&%ydUZ z+m~E*Ne`Xu!@#A55hxw|J9-Zkz~2E5h*hh~2RZ{%0CPs181Oj_PPENLglcUgf0PdN&_bzVDA-B+4GJ-`c2vcdP{&*IW@9TE)pYi>UJ8Wx(4+ zRSXBu2r@oFa=7>SlwdcsM}in2P`x<*s%h3)1 z@N$3&5uj(u9$?W(q9}utmnS%BV|KR9*PfY5f1;XrnJk!n4aDEf?jpHmg4ytx8IA1(C;EQDyK%4`W-g-QKf6t+qB&(pk=nzuC?`AR&vT29_<#{{@L<>4$< zV)5@$y#fD=eU)-W3w263ZvATT6+`;p6kKD6#QjQA@p#E^muc{_V%a8V)NA>Zv~94#{S+YEgOHn;o;_7;?Y2-ZX~Au zHrqmPVO?kBV|Sb!kIE?MrNPi-lTRGWceW5-|E=Sd&U;1x|Ga@E8N#I7nb<`Q_MwSl zq{tza^$cH5wXVW(fK9shm5Nrr&>H5?Ce}&Qc&*47K~*nrbT~Kgd8T%}-YP`(z1!p< z4Tz8Jqdrw2$V@fj8@P)p9nj5Oq;x>u#HnzmauMiyF9WE&hBZ|k5Lfs;YOA#}h{-Z)ZUbxyx3>mUP6paf>M9^h-h0*xqGc8>%03a5}kz z?h|~x>e^(?YMrplEhFvHDFq z?`kO05)lApQ%}s)ASX?fEXKl_=+qMn1B_5ums?&|)KwTN0Q|DN4sc-EoQ9j)OSI+h z(SRfb&Fc?l;=#8CVLhGMjP4xB8$(`JRnXp?!Pb6PzAXLuoXp-s6v<{@Sw>BQsQ4AI z&FQ2)tXtw$2~@WBZKwcI#jPyUY~U-#Kn^UQt^$b(FneIprd=hKYPC`gl_%YE&8|ME zw~mo(n4}stp1D$WIeD`|Y4JO_ldm1r=Vi?G@V@=`vOpUJyJnJIt6%IM_`7CxS7=RW zddpLXV(FUVtc};0_amcU(UKPuIo_h{RwGaIl#a5K!H@jJ5+4AUZ77@;mW%TSzPmvO zg|)27hUT`cD{sq?9#2Zf-jf}9a}Ov_|Y-{%ZBPg#L*MD!q8l=7Vg{Zx^7UC3oZ4D3cZROViP9?osycni}9zH!Lg zSEh#SF|8M}?-G)hPU-x}{8os?q0-1js|x`p8D1Kq45F!)FagRRK_n$`ELcEtJvaab zSE(9WL9u4iIwP{#=aXpfS|hLkD+_GC*L4J@QL5uZyb8bNy)EU*4%?On`15KPSa?QZ zAF**^%oZDCl-fcO7P};6Xx@87fi*sh_y<5`&X-6&7$}SSCAKY{Mj*=Fq(PZ3V*7K6 zks88huj5~1K8II7THRsTt;hMu|6P<*OGnmDefS5^_ipU%H%a>FrJ9^KQj>dzyRsRS z_EzvlCXL%$Nc=lZuNbn}(-gDI&)0aLO~~%2+3OSy)YRgT+j6e@gDx%+K+Z0la`-l^ zp53dSiyNL^5U`Y8Tre5biK>{L;J0BafQxYaGB+A+6=i=SrTvMS#$Q^yhtQO&h|Si9 z47t$KPIf0x=292kLx)~*qt|SiXq9oBWFm6$y6FJ|Ox!QPYblFh{%1(rCOP@L1M><1 z{MC&lM)O5v7%SY3f0Al>JWwygSfzZJcBMRXrpK^I@s2x}GS$+{w?(Zq&;Sz~+G7GJ zDQ2@yow}OIG8&}eq}OAQy;k-;2tl=Hw`$%lbTIZbMB0mM^RNBQKgaf$95k5Y)0FtU zE(xZ#40ITL>#I;=@bXx_BPUj;C|b78JW#GuiY@nm)Z1PhnvR0*lXwUkBVY&DmlzA^ zZF_%JFK&;#HmSrnK9d!Exgd23RlG8d3eI09BFX@qT)`XJ6ef(0lF*k)t`S}-AJ8`w zb+)`00Hpx<>Q_M?I#yqBH1fwftvt_7S~p)~zJUFvwsh$Zp7DhObhzs(>DQ!pq{JO7`2>QcVc(ZEiOcg8teUkRz>;u zu3@Dv&73E6&#Z*JTJmh?l z%)Bl^M;Y%OQEDm&-yNdRh^lBYcB zUFqxeCpy{7Cj484*S(ZBaFok3y_yKgj-Mjn+^FEk0=&^C-H0KQtO}7@21;2@cQ zMV7M=SzQi|x_nt@s}D`uaIZ4N6`jX7r=u3Y8m)+qDdE@}cxUyC7C(c?NjEZ=xCdqm z6qSiyy%}4AiW>stH(Ig0|Ah}yu^9#H1xN%4#^xG9I;ZJjw zlS51BP^lWdUqrRN-}Cb(p#=oF!6A})JGhSf-e6&Mrv_%`XWgm^zmjxo6KNa1*pD+h z-*~8hwzk#2wdbT6bgR62;q}?%P|QDoE7;SUe0SEv0|WS@r9i9pR%^HB_r5W>bm?op9NRtSE2CRmD4=HAtL+%@CTvf~{gN zUE=qQ-!68C%H2&;T5ALj{Hk zMg;ziGVzV1ob-hNDAJCAYl5wFETdhMna$^b0~#wyiAWV-z6gQE*_;u==-OtYBjy}s zxhp4g`{2oQUl~@nQ}0V3Bx)j>M>+C}r3+HMY>!q0QMU3Q>Ysf!j5DG}W-He&zdsR~_9QWC<2_Cce>nna*En-hT?i9%<+_@zG$v z9^Z(ZUxmGHqv|(S`FJ|wT?Sdfq_w`B9h}_7b-sb_hA@>Gu`5SBkDYIq4Stz;%XeDx zp!k{bbDHCNZ`edbNX=1mV?4g_w^WV&Qc~@&r&~TsQaIGdJ*%L1f@KXkn#pGy>mB1& z-{oLi?4`G-v<}A(9dnuxR)zWC%g(_bifH^n_RFXKpjZs+y_^_}4=ys{G^`0+! zDTOvrFo@E+Khn8wycgG*rso0JNK+r1W-EFtFah`?7RlSuPL=l2`$ozS?~0a#AkstP zO<}7kjUfFG8T^JIzQ^RhN3PU`i6V^3Zp;A=lTCNnT51()Yho=xEBHK*vUiXq8CXYk zt@gGJ-bp&HAIOReR(>W=jCez!NLofOWIARlOqDMyrcQ0zFB6Y2tHMgb1YIhe}PZ53Y@5IcZX?)q4dxwtr2T2$^Q=7hd(sHrb5EN6tiedj#K*p7$Q-t^1&U zWsh?kwYTl~kbX04nNd7br&T>nzllETA3*Ex8Qe|SKAZYLWPHsBu3}p2lab!U{Giwv z_N9K65t+bG#7K%=%#uTb)mFO7$!&5^nf@R9M~Eq@dW6}&h0l%ov}zj@)-4U#zMQ+5 z@H8gFLm@VGt;GT>J@kJG=Kn(K!hgD_x`k!y4c!}RA2u`Fb-p8&-5@VRxz5eoCs!zr zsf_CEn_CX`rK{~}qSI5*W6ax<`lo+SHZ8l5?&8T8HQzL)X?%-j>s4*SUTI3(>U$4$ zf4S6ZiyNIQ$>wl|t>iyieIk4CS0|dY1qt|^Ll?$D=`lV0;9jI%t7!=HMkV_W-af|G zi67qzxA*nDzh9rN@tXfY`};}8Nakh4wLE@5wyjtv_vp&j_z%d~Tl-mqiy{1OQ2~xm z@C}6U#}S6XC!1-~aOfSf!tYDb%p`KXXwT1&d-7bVtDHxdY2T@comXn*)n{8P7y`eU zaty7D1hi%)d%V_U>vhwPoYDB&mSAy^dC#72Blzrzv?N7%-1*JA|5UVYaH87&A3!xI zK+Q&E*7P~fG@%A(TIfk{t3%XO!B@f}k4qS+GVdlMVu)ZMV-lgNf}di_ZUL1y?bo++ zo0U=XF37arjUZVZu<%+d=`$7C8Ch{xH@E=(@|*9mT{cjNP_l*&4`PTi;w06Uv4Mkk zF7E=%W4zuAZCgF(Q;&{pV1Wl^dyu5(1WP$U)uFRVlAWD+SO##=;zHu}J?9FKj;K&i z2SEcZwWX#hH}1vO!8Px2QizJT#Uop6#cW0 zCn6@`Y(~E7+g1BuylUx#)MHxSASfj*{o(62wgU^O-p*AH=wBbyz-yPcnZdRn3*n6d z2=7Dv0O-oHU)Be^AvEj8hfeRl?g)_v zO}$Y@cGDW`^MlX()&xINdCd1@AWtO3_*7FSM>%lo_wQ=A_bCZt9d-9!$|S8YdAZ#d z%3Oc3=E@bQA4FO#Dikl}Q>=S_n#dgnOqKJGnk@fA7H01&5BmhJ@DMo*$FD-_MyKPK zU3(-Ts)}YBmd-mRZ;))42i{X=@|ElrtuGyFD>z3~{?u3GsiZ=ODvCX3PFyBu@YV-m zufyqyA~e83pIcIkXK?BMW;8noWO5S1eiib;)fPpB&|vvD3I&-;QEbP(%}p8B4$C%* z?%@w*0@QuO{DA)e)V?WwM*0OGq(d$K=m#1abn)FA2|f5tGcsQ>Av=?#9(9=;?4(oI z(JRf>A){&VogD9eBdxD}N5e=E5aO;?=vR@yF_JtvTe{jqb8co7eIsytsG;cCQ8vo^ zIXk<~dFwIfRPAo*yQgnjH;d$d2CK!k5^Jj8D7IL)8w4chqzGDkO}n%hZgVdQ-eQGx z{mZpw{n&k#%mN2reAoXPoL9<)0B63 zK)vhgwD|KMfNm6<*3RX@7w$YtsO2wIwU=?k)9Y{LoTk2mm0IL!;XtYZI(mifLX1Q& z0%NEAUc!tYv3u+sm)Lvzj|}_CccWC6P0eb4Or7-QH+ZL-uxV-QsI(G(1)U0SMC~+KDtIsAB+ijr)e`^r#-2=uy(V@xGCasGKc*ubVmJB zB`676ahLTxOUbkU){iSBkLywNlh!*UQ!5!S6vz{ZoJulh0mhoao;B|!GK0Jxj#uzN zi8fJ~XLHn3c%!wp>D#KFU&Z3 zbDn4`O+q(odnR&ThVG9vaG&K;zji}=pJx?)M@TS^+&hb8-Z-)i`*Kq#io3zMb<&9C z;BkFfeeF_T|D<)7a7#?TXaZjh(JOEKj3#!A;stWuHSiERUgLc8BLgnTYoxUU^Hn=7 z%rv{JWc$_D;Tb%`cb1~@Y1(txKNF{}YoMTTA}KJ*FpxdWf_!Z0|BahJb_nu&<1v`P zm*mlmc^mO)a?#A}<7@k)ud?nz?+WuPR8jwH)-|O7^&CXylqf=y zrPk}`dr{b(jc8e}Xl;{-Pglf86`yR1{|zBI&e?AF^k^Towv>+pQ@Yt{2V;v+0_Yey&b zHWuRo*1GgEla8E&ICJuK&gDm0g5Wytg3j#xwNtJQl^8Kk1|{!1)`!`V?#;ifiZ5w5 ziVV3A4V{iE+35K!7%bIMPGQe#P>vDUGg6@PH^!!X%d(DPH{K~Zz?L^t(yLIK3Ly!j z$`PaFHkJXw1VhmnaVO*&;!2&-G>Ws4ct?JvmeWGbqi`U3Bc+g_r)~&))07;H*RZ5` zUqcm;TmW@zR6@DuzESF$d@oat=!|F%uqF|ogkHbm%wa`el0izpc!8d0fw@?3+q2{s z9KKlnW@0d-oTyMi^?OFw*SxsPQ}4{!4liy3V4@g`2z*(u(t zN&kBg{6_bkHZMOmsTGViCbgG2Y;j9ps3u=4x95;BYM6h>3y@gOj9P~J37m2WPM-}pz;;Qcx&R>iWQdj zdN_z-c2cz_RUycP1?vthBDSqFf*k84lJ@*mx6rf8MJmU*#mrCkV;0j994y$=P91Zpaz>&s<`S|3c_1_xITKlD6N9XqY*Ym=XgnjD;8Nc*aJYCU0( zKQM$v&&8<65c&JYeUjyUWt8_g*1#*ha-&KuW2|HkK@|d|OCcUV;icSOhu~5)*9fk+ z8YbMxY7nhClTiLRUGRda-Oo1B?9-xq+x_hIAEZO@4w%6Y+aw`s_|X1 zUHP0Hk;y;~Yp`v9Ake|^D^QEM`q{H@x4sT6SW&F=m~GM(GfflyYF9!GZi$3958>id z{ioap2SGx)u_on?<4vVzC%#T|@A`%wH~;zvaJ=2afIdug{8`fCNBs}OAa0wFU0o9Z^*OqNfRpu?axcpG7tYFlQ&Vu6k5u@w)pO(ewB#X zn}81dkdxdhiiWKOW6syD&#s1kJsJg7|1!psk65&hy)?d^~`MVS;byPgGFQ#E#NW$$)PI(mG4nmah|4I%r+6ColnWLw0y_A4Uhzb|LAG z2?p3aD_Tj%tK#V+2StmrJtV}P(fKxFIO$4?TOXC_XG75frpmxLB43axu58)B5{=w~ zV^f-8t9sWvArlD|%tjPtpuZcH{M|O(11P;$j903cwz^9$4X|85ikk{$RKOfiwN%0Q zr2rd89#)X%Qdy~R5k&TE1;xVXO+;T@3V_$it63%yX`2?23fe@ae);8zBZshRs$~20 zLugQe2iBAgApd)U=1u_?uz_)_86sGh-60y6quu1SaX9K+kRGR8RbmYtG9H<735W-7 z)Mn#BC2mxzmGLVTpG{eaB8lbeASr%ceN$;Gev3f*s$R&CpI~inoUlt$Vlp;)AK|-^ z_X$x2s`yZ4>t=;FCcA^Z)U&$rGKb75b$RWA*#QZR+?q%d2LNpihK?LWY?)r-fc31e zwo1H%^>0YmM%h<7WhKiYx6y8Gs>rhnN_W#OaRE(`{AB$ZMYO?$iBu@OYzso;lm5ioOSeaDRO>{lmC;`*Eep zDUI+9@>+jHa#76FkoVy}a#K6H_0WZ`nx1p1LnF;B=D^5!h}CuN+sA%m2dz%dSI`-x zpHoubzaasPTjnhh8Fkt$so8Yzn67!AFz!A0tR2r!XVI*Ut6`VYSstZJlKr?ulj!fvUPJ$nxg&x7zBB&bxe*HbBI3>sOk+s z|LC?EIOlzc}?4#|<$zHykq+xmAx1dnc{^MTztyJ9N`NwmlwX*QH&&RPSjT}f($0}a<^>A z&dkO|2V;l>}JGi!@ooC{1g_91=i|bz-N|8EtRPJ0L0n=x9}Z z6V3(0%>cQ8)98FC>vxf>U_pl?v>mJfan2r@{2s-ro&!?cqhCq7?=wlT!N#i3jRdOX z01K5p_!~e~n*ANIJu_gO^JEi|jsunzWFbPLI5|DD{mCW<-iCu$vEH%~qd+Puoq`Qa z4$d9mW~u+8gwZw#2ELc64MYo;Rhni~WAhpDcdr3E@aXVrab0knWd|N(Z2SN?A*~Cp zFsA`jQo-n?_{Yqth$_q$Ft9BKuu|BLwzt17(m_eK4x|QwAB5dEupfxVHrQKZGT9g1 zl$2KzbW6430ek>mt4axGVyQ@ZqLH3#)>s}PkJnF%Dv!tj+-XTJ2Lk#QEcNzDnmyH` z;S~xkF>Wi>s_#^50t-9{cRvuMs@G)PUt1g0E8jC}PoNQPlW;2MS!q_H{o=FsWLXj2 z5qs_D)T@X?g=h|zZSoaI`zGe-Bqh{W&+a_sE5%AQe>Jzu<^iIFTsoJOnILn!59do> zQ^QfuL8h}5ArPp*H1S7pXve`JwlXXsh-BsmP2;YfeDXKDX4Yb9J&$5AqT?^qF@7&J z&CBfK5JW8f?DF2`VOh#+%|@-mVH*R!&9vci;=lI8DmDWHlK3mCx7!Yu)9Mv;ik}@0}wR`>X2nbf}+JUXk{G$ z%Ah0+;yv(k=eJ_C0`4-{16{Ju_|2*7)e)&kOeuYZY6%T3`qp^n-FcMl1N#RAJ}i=l z!2J#u2}kJwSsy|cf=JMQ^aGv}H(h-N-%3=bMTJ(6M6DEc467TQlc@4j(PDR_ssxV> zSo6TDjZs@l?(t@)XqyIUv$lDxk>ojHbGjIS z1KBO}E|;BZWg{OlF%gq&DbDWce`nG(4DJfOa~EoILnKj+{47sxk_dn$;6cpfYOfc9 zQbg)PWBTk0(PF5lc$%*WNCdcGPE7>^2M~p~Dd9eHk^sP!Xq(cB9>V$|RdQ0|#&B1O z;$*COM+xJ}1Sb*N_YzI7`-vojqVtieS&PU$7HMRPd8s#f4#l*ny2odIJ8N& z{GeWEKt`#vTikp8?U25{q{6sYVE#$vK-Xqn(Yw}9*W=3)=(feE&Ss)$b!CE+S!n?_ zLgn}VG~7#mC<9n}+Ede7->nhcxw5huVQVi1v@sg^_qN|C6g&zE4{}KKZ~r!JUpFk& zx9R^|(7U_-E-~j^R=ATUVc7SA=xXOEpkjA0GN>ss=P!-hb`RLLksYP_9#=W+c3Lkv zdm7d(Iawxtpi_z=AM*h>Y!GlPc;9#cg)->PBfzcnJvn!i;Nu8r@2G})mJa< z#Pe^&@Y>K9F#?DACsOkvfR#CZ%c!VHa_Uih=So#+h0Du!iGoN3{atfvBciSW6~Rq| zEbA%mDX_&oMj`W<0nwZ5+*uL6NLgc3-OrAcPnM|6$oPnw7e14A99&; zj1Y%VPb#&^&_groTwmH03aZ}$J|+Ud5pPC@N>O#-h`=At<}}cX&t^oi;1aj%(h?+C zD0&b~sfM47*ZBUlBhv*f zRaP5QMnOUUyFaxJm+n0&H_VgzB)p`>?rmL;p&1y8pmo8+zEyP@DU$Gl?mZ86l^vwd zrz^fWV?8-+XB*bMZ-4r$tJ0wQZL}n=!F1Eeugmo}Ga<0I)nSQiq1gUo;fHiITs~n9 z5i;V{C9!IK%MXHA0x|KnS!7JvH}CqGP8ce+jkE5Xf21DJ0U=om^V^&m_pO@_rfk`n z1dn|%ZTZRVR~US46CY0jwyZ@)kc_ViEQc1Xf(fg@1XDcDf~Z%9N5q#f(4g~Gt1SHp z>6g4v84Vz*Eek1vPDwf>>&K)Cjjl(sM<%d^E3Tr_B@=Y(QeHU_Zccp#Wdd1IsZgnw zLRM4;7^QkrwZ!IBu43g$^73^BF8PIJI8A9u2Q*zEm`_rP<(A3qJF!-7x%9ZE)KL5` zg&9u*Lqrk+k=h%1l+5D~FRGT|Wt?0{u?J>XvcDaLGlO;a{` z@IW5NN>&J-olYeXoB`!ZM`_ctxAf=(n8BX>a11n^2yAhQxy0-MN8Lr!N=QLk%xQv5 zB#O~UAkbh+D6#y5)}4+7WB(chGwlaA|5ZeMCbe_{xt+H+)U;YOKwa)t`5Q)%}SVA%LykbLAq z)0Lf&np{0iocq9>SC5#-_?s+Ik2Zx`f0ux78?C{%w`b~gE3*@MIKECE*LpK)O-FZo zSN+UVub18Uy9*Q|PiJM%6^S}DKC}fR>25?apee!pf7!qM+b(GT=Ym)=#z0j+xb8h8 z)4um$a7BjqQ=#$9-Ib1j8p^!-mn0!fy&p03yWzG|h(1zqogwnRSm3Itpp2}6dJ58X zSue;=nd8tpaN)JLu3i=j8}2($mH%wt^r|p>%Y?KAgnEsvY`$`Nwrb)Vdyq-z8+y^! zQ2AyB8)_HL@8eohzo(4TTBg#U?%chykgabU$BT-x9M7PGVvr0PYBXkDq2F&Y!d zWR+-k%!3gyVQ_YP3BaPl*oHvqIC8 zim5^(p+XX^&+1X-nCfM)Z7c?fd7#882038U{o`dI!`W8TwNi0|sJ->Ng6i+Ls{6a- zx-Ij!Fm21ceM`pvD>k@2D}HB$ez4f{moX+Ga@prmCRS&CC7&#yLy*_&-L=k2#EmA0IC?%(V``MauB7i?}Qp>$5*qoE*y z9V@f``y8IK7}e;jdP|Nyy+Xm+T4}Jg^s}&--!moQ&t`V)uYbS&yZ-AjX!Af~-P(H1 z1{6{^dXGV-KWo}W{CR3?ZUjq%3Zc%=R9g~bFKj5cW}{Rej9(tnjS%1y=f&OFLd<|> z9k<3iw4`di>qM_V^py;L#EXf_Prt9uxnTKfky>uSa?1^q>&oh1nY>LnTx`uGD(Q#E*tR9-+B(cDRbUKjE)Qc(F(`$o(O5ipAin8%ry zWf`X8i>euXCiUcX_le9E)ba(B2lA7^;jg`vsvYb(H%*--=2mQPoq_SOfLF^{zyz+8 zz;i{)Z;D1(=_ynVTfcwNdYxP{z9WwI^kHtJe6uSfGM3hWM?ro|+ZzL3Iemp{2+JRj zQ(G9f>b?Ye)P@*O>!*z9K6^?@1?r>BJ&s(FAA^k0R`$*+loHBDjJytn(Xk)M#SI+C za`PjoP5MQ1#gr~Fpr5jPQiPGld#3d1&k-E<0jl_R7{IhF{|h$UhwQVkl0Hz>G6^%e z235GlPnMN_SalBrAJAO{=dTp~@seD&Y!nN6=GFX0g|KkZYU)Q-k)R`XU}6(GgbN*9 zwJ6H=H1?vtFN&AiS*mDFR<~+qPN)omdcso&?Cvp0wC+IQK&}=1tNA>YnhZyP96Pw8 zsJ7G8Oi!=z##^4+6!=r-SQ}ZtzkaQG<&Ph%b=xKXX!X^1u zz#o~>mahA)Uek@MV*cV$>?V=$Wac0Jb+QrRb4%6~hhNaNKi$8qZueiz3;aL(a}GVz za~DhCZcxslc@&}9DfE3dt-pQCDc|9K(v7F2r@2tMW>%ro=l=lg8|(PWo0)FgeeR4K zKkP*AQ|hHyS6jRm8 zAJ*JT;)k;GA^eDgeO& zd#2Tc%@g#xD}g{Ez-L}`JOtn}f=2mj%3U>mDZ+c7mHN!wADy5B`ug@4D-_U zBuRL3v$wRwt)!4y!HngUFwcZGM#QTy1Dv^WL@glYPGotNf!XWn6nv|RrzE=*Rdw5x zAqWk%@ruVAhb=sj4uAd;%qRa9{E3rf!pl_yz6$&xO|h*=xDEcqWn!F!rXAE5iQHra6+pG}AvFr^U0_aqjN@l^5wf>d*@DtB zNKmR8xe!2T)ohR%2{OUM1Ehit9f;otY91Z@Ne(%Iexi z?Z+=Tcwv(f8^|$`LQzOLM;E4nF0-9VziIU}o_ZC_*s9qFP$39AR?}S{&!`yt^a7|5 z?;zMRlZCAgY=}ijY9h-mmHiG8qKsrLkjmuxMB|IAL7lyVM-v@uKk9?uTzd88dOK5f)i$gv@;nM*BLj_P9r>3Gj3K2k32y3gZ3c88?KGPKMcxqo-Y^#AU!DRa}Y>HKM~ zsMOia=KZQVn6x`z4e08}-8AV#xlo}jbZGdty_4ibN@~f&L>Hbn4}L_Ui8kUk#VT_} zJ75A40;ACG{i-l5N{%VF>O54L+C8iChX{`6mQjkvNtcG_5@WaQF#dzv_RaNFzpdc` zV$3c%#HJyF_Lg}mL6M{gG|iSit`gHAe}a3OCg66=MF={0g@B0yHiJrZ_%@V@ybl+t zvfPK?Fn`euW0OO~QwyLejhzx&%Dd^Q3VsJK2gJuaAX25MTE0TTw8`u;wrX-U!}ChQMpA~|xi974Kty zmty|%dM`V@CG!#FD`D#5Zu1d&Xc!pA0pdsNGIvT)y561gSa%6T)>NT-0@2z(G-$fk zc3-%Jr?wI#rC})p2bZ9F4=KQWZSY4$<2$ceC(O;f=l}v>4#;jx02*NIfn63f0yzYDtGB$8Iz_eNS)OF0#EwwcarhSlU8ApmUmLkA*(|{;{I_3+Ej=M~v%FwpO zK=@$fTj513UWJSRXvH~jHST_G4^)U!k?OY2YVtVUf}Cr z;)(RPAG@c$jr?B9(pAv$dZBUM(L|WiiL?n=`|tEB`aiz~sHDz#%*TKlmCV*!)T|-7 zp>}qQu5!Dpi1s`Lz5ms2eBejCa9ztx7qR9& zrz76=88i^-%O&=yEy$We7p_a7v-S9_jg?vOXLAcKn9e$I<0XlmZlE+@WTp4v*@$}D ze8^8jcxN-d$5RhX5Xl!)NJ^DU5|$NQff~YWA_ADm=alFCURGRaR?N?sYK7uUpjtJqB<-j!=X1M|hgflz#vN?HYH=Zwzmk-Mry2`?E{M;~+a?*DKTU zuI0+e=-vIc%o@4_ytLWRp<91@WFpn8)`i5LY0W8ZjEDQ?GSABVkSQ74@o4eTUwpUq z8FxdCr{fy=NA@quNUMx2U*_rkSnC|KJ8&~_a3kbp?@BdKBIJuk&bDQ~?6tma?28Sn zzm!(>JDr2Tsciq#tLCrHitZh#{O|dR`xa1r04D@f`vJ3^&#kgQ1_leO@HsY zcjz4i5~cSVnn0rT-a82$Y&1zIQWOCNDFPB{p%Yq!1PDzLP-%)nXc9`ON)f5jyNV#c zlmD6Ld3W}nbDlHv?EiidGQ*Hzy~tYMd)?P{e=c(+Dm$d=gKU(dg;9g~J>*-@;+08Y zU=QTh&W8i$Qv zzV^{tK27zkbH+0s)(R=Om6Dw=qh#t?blJ1`qUlibQ z$7DG0+Q|-%x6kB6sKo)i)9??V=PSkj#agW!&Dh@;GUrUS;{m!eIO5Od{MGcvSd;+c zRG#lL^@|J<1&OpH&n6A#*VJz15rgkHTUUJuia^FqV+B-27TE48n~9njev{395bUru zIC;NTxl~pqxrY(i*j8?DyJuGJxZ>~(DY;>5(;WM%1&oQaw#ezRq7c8vRV~zOcqn61 zm3+Zkb~GtV#htPVm?svrYU2!U50QRbdK`i>KxQ^&Mnq)^#)}Vtg7<+e;FuO^kduce zuKDh;dJRmG&Cdc%!IvG%Kw2FvqJ$uWV;MrUf`LJ@g+gr zdahuh{0wYq9S*OQwXPXoX20X~QDEJ#%Ggi5aQ{ODb6ZF7l{bhVCOb2mGOBAb&<68m zV5y~I2)TJMJ{g6S`IgFV%q)Vt3k>i^Ii%oz$T`dsc*GLpym+b8Zx|MnE_-{IT3_Eb zj97kos~))x*xf=~MGQ}R>a;kC7Z%*L?#T*Cn ztVP$*>9SNPu(|E&)e%;k8;=Ssvl_3I<)y zyV-=v^IZqggc?)kUCXbO)Rwo{3+OpH6WauW^Jn>{Bk{KTkb zRD!))m>eIudSmd%;cC>wgS9u%GL|{l#=KG97n^#UAC_Gn6{`8DH}L8czm#@>dyzjV zSokPx+mpE#Mc3rPdT04_%_r5z&%N*dE{n6wv&CBAJR81bC5MUd09rp%Y?|fbkX7yKOBc;vz^99FL+?@ASTeV{r5>A9 zMVM_2Se!)D$e)dh%ZJ|iLy4253@gtAs$Ezot0=RHz`js=1M~Gb>n2DObOo;6ey2$w` zdRxCrMpC)gcG*4$ws;vFZ3;4w6sQ6=`EWxB!aUzF54zxoCxmx@3YoI@wnM=zg)9TF zC^TQ?XWiE{Ci$OzWL`XE(|_R_p7+&MtjgVIYlh739IU2mHIX@F>&VK0x4V&U8Bjd2 zb=gMtGx)ey6;_kpETx%|u65h-T`xVDr!T7DK~;!~BLUD~J>gKJR;f2iiNHq8V3i;M zW;UMkSRmZ3#LOB1Ij-c3+^|A2gk)F?B6>4X)Qi^2X=>v<6UwQ`xEkMTOg7ci3B`G& z38hCVcbi#Hym<1V2#u{5(ea&xLkh@8J*!+dqRhf6NT0C%{mgg>@plj3CIr=eQZw_A)(y}y-R z7gBn$_J%Sn%;xHRJc;I}1d=00m0T?s$*yNn3dxRa_TQ0azEm zQ@sBCw$p1@-DiADwj$FNTY3Kg{K%_3E3BxaDt;faR)57RU%?Ge>auIdPtjjfPl($3 z4PK3x#aG~cZl3=dRNgc(OQ0^+7&o;DP4|;QBkhm^6eK`Fio;r&o$OjAT#o~|kBHBU zrEIe;qEQtlqUO>vNrVT#Vfi)oGD=?`0vO;I2l4GITSOw$O+I3=zI`ATuRjxE!{S@y zTCL?E*H@iSm}zZDdvTN*>8OMqx@4NL>sH71jt?yJ-ZbWWU4ASO?Z@l1>!meh7?`5t zcp}|esWY>);-fAi6usG5psLTS`AU35J7W2Ca<*D`?3(OiT(LtZvsAOeL0{?e13$65 zG_7kDC}kcy2K_g6y#-$%kuROxAdswzsDSgWVktVm$R$Z*TIryOn#7MkkmkA(SmxoJ zHgyL4Q)j_f4&_Sf;M3gT-?%z`8y{#d3hCy!XQEr;6-BAn#<+>Vtb7Aj`XYAMVKLmMA*a_h$y%unA$E+#~@2mup+Hv*AH5?@*&gQs~7k3*0 z)(a5_lKrG8dOLv63w{RYi%TQ%;vj`szzNCZMD(nM7bluZDM-f$*2*al%IUKf7`*j; zF4ExIQY~jH#!xh5p{W5US!)76bh7T3IDo-;r`m)WU7vT}Fz0nSZFu|yym!vz_4=!2 z)>ZSaUkxbmp?U~cvzz86df5_>*wczuI`^n_1bs zL`=&j=wn3bj3I%mBAK^NPJtc8PHs$IsMJ%oJ99vVc$i{Sa*kk0%OK`-Ra>{L6#XjM zwFw7}PYOp3QQYjDS?av2 z*L)_F!R*Dr3tjsPB6LiCPi2h_^`ENH=L56Hdba;CC9He;)F=N-5eWHLP5E#6;QG(j z6{aKqW}Y>SYCPV*5?WR}`?ta>HzbSR*l2R>{yZ+@dJtc1BtbI6pL2tFRUz_lthC&F z^9%4;Yr_7R0CadgK=GXSS9etyLf^yvDxRLdcX?Wn6gGW5 zP`lfdNm%A9M^(M+WWz5}yNJE?Pc`mtOOrgY*Gtx?L551kJcpB4-byuy(SBk-AHQu( z7b{rv32+M%Yb&ReiS&sr#lpuNSkXpVbRMbWRF2gBLs4i{x_P`%i_OxB=SsgS<}>CL z-_Nn|yy#O=RMSoO?fX!eWv_bOK``)fALM;hy7^KmfDl_lZ4L1PF`m`GYnSO=SO=s-pJ()rH#wS=*5kI%oTNRTi;AF4oTD+^Sb^5xL+J}bJ8SY zc;S&#YwxG}g{EBx)@4kg99Ll;;qY;HXjar8N0;(UiOhi`uMFGZm77v*hokIJq=1J3wX3UNs%KLfvhyDGfHVOw2at6l_< z=1)5K{QzW^bWe#gcSLkb>_O8D<*JnGGJ9vOVJ2vZ^qD>7*^sAXDzi$=0|*%a`Gl~h zbXKL1*C9*DYg%L~+{8U{anv|Jo|UoS$gwhDV(6Q4&2DHV&aZOUlGAaPc4AzB%h=%A z&D#u7h#NB!r#2tD-td-MW@!Y=b=JM)m|+nFOZM_6StF-w84ro#{k4jSYZQUp&zth& ze|iT0TfI-&4LYQ=X(Z?gh1Gs7x?$oVzA7zN&Aj#4)u)x~yQvLZ=nb}Dr?kQ!W4WWp zge&!wqAxdUN?Kn;YrGBP!~Y6#*-8z#A7-D8w$QLs@qrwWj3^epEMt(QQN1k*pg=hW z0G@Q3;l?IIAf)Pb0XUU!_BYh0wl=lgrlY%_9D977k97LGE`A+1#JFm0H6Hnh42R5T zRFMlxmzfKFW%`DO!!q2+Z!?Ax*)>GY!E*jjWT}_ew6dKQMoo&Iv=gOzEw6w?g4v8a zWNMr#+1+l=vL~zCrHuz0i>(%@`1i zjsD_+L&te(C*mNjdo93G;y&K{hU>3td}Lf#P4!1}>UeW4J2tUtOo){Vg|)?rnuu`^ zs-d29e&liuVZ5EeZ(Nr1e)*)vrQ5UWX^>dF8>GIWl&dePpFrN!F8;f_Mo`(wh;u9o zt~AVm+Xr21`rg(6RyZBe24@wAT3;U?*qX{s_R{wO>ucIe$uOtr z-e-!h*UhM>|6={rag~>sVxYXK;pF>J)!S-LeL1*WX@c*q?}a-t_tY{tx{WVo`Sk71 zr&2cXUK*-3T zt0c&nKGu_-C=->|gmDyraf8x~dT~&GI~wb&CwQr%g>0(Ig5J_eZUNYBgXJt_M=>+T z_tPYTA1{UqJq&S+;eUO0vz}{b?#1gT%tKEKYuopoidNmBvkzkE?wMRilb)5Jr^KG> zRbYAW<*4Hk`G2RJp8wOeR1s$R>vddW2gFfFt;@2qv|gZ1P()%JGd&=1Frt8{72d$zkrV1CmbhVwV$thxz#dS#8$c=<$bMwCX}y+^DvY+GUKe^v z%NHl8s>eYekErpJDqviebM&}9Ssqc-RLrdBHQ#O-KKS8*$=n5Q(ofuF$?KA^vLGDhIgZ_2l64NHn<=sS{7q6sT0jaF#^BY+st7UKlfy!$aQ0`vRsDnpyl4bC0D4WmR za9AmB^6cq-kLqEKsRuowDyz;YeTCHAhZ4wzf0N}$ zG7n{TA3D`Y8$^G(x81mXHtSYgPpkP*<02Z_tReKRQ$G0sD*mIfv^~u&ZP01gMb+(u zy@h+hUG!`TeP6~D$&tJq6kjHaubA`S94$Xj%XS!kZ=bRI%kldQaB5)ntdU+QkPOWb z)qkw#I#PJA`4Q|xij(BuId*He8^Y_lO2QVAW4#Mk%JAkCS3m}_`!0zl@3@izh;rv2 zxFF;{I&g9~maburJBE(T=_b;wRDPCSa=$LH5(AQtnk=qLqSR{Vc8B{;nT=w8&)g4>W*oH@EV55NePk5KR_TAYqL%dMxEF()OLb}KQklS^1K!x2t zV;qxtgCRgo{O_)eN{F;&yi3$near5rR-!lkvtv1!cx;5oRKTER>m)HI70ccN?!OVS zGOc|tB$gp+G=2457gSoSVx~hs*s$J^_Ks^3Zw_}xl^cfX=wiCX{Fl@}0H5D~_z(rn zpM#YKDJf#sbjUc=(3^GPh53<*CnvR*kX;dJNFl{>s#P+Pqy;PW;~`#$uo}oVFCm_e z0<5oMV3c>7TX@xQdc;t$vVG$cT5OM`FM8T!I|99lxQb9l51y*kWL+P$HWv>mKB?J$ zI1opE$~Mzf>Bc|%W63_x>n7oDz?(d@K?N0Em=FuO1n))K+Sso&Bw!^ zD0S*+*ni79_5Z118U3Yiy*l&B+mG=({ezmOWdC|*J3E0Dhu1M?1-cft7UR?MG2KyJ z99cHTKF3QPr@g~{c%~a3nNag{C^P(5h-;CkQ!xic@N=Wr5>mEdNQIlac^tuLx674csBK=niGO}_8g?F2EzvWb>o(@`M_3HoITG| zrG*EJeg;1u88QuXnh7c%m|^9J@}mB0!hjdeouroyueT7#%bAroVefo8aA;6?Gtlb1 zzvG5C=iRW^Vfs*`)uOr=%Z->=DG%-Ty!_3yaTl`FJh zMe1KTWlkZUwJIZJtoVz1xPme*V}dr)Y>{JF*?_f}xj2iSPBXRPHh9xu+VN&&L zl^xyHc2%9B7a5;zEgi*or_za1#m{PFUDOLpeoE?go84d0?RIN-7DV7U+`{#ThRpAO zFQqe@a%ZrZ$*$K|WGwGjFU*{ZYeOa0Ik%xEB$&N!PkaFl4YJ%7(%I36#&eHOrKmxs zJy1^i;a)|cjW_53RH=&p4KdnG6L3V}I41daz`)!1`$3n`Zov|6UuBubl_Zn6Tge|;yyZ*cFxYhzop}{fs8cnsT1m5fXwzc(p>X{Tlk19ua zsTcfh2s|e7bUeh29Wk>M%T1{$UB;tEsk5!6637MC0NQaDShjt(9+f(X|2;d{$}!T$ z2be-1AP_aFnM_^RzYH~4Tp`Q&U;FOVT*MvyII~)!aec#KGzhu(SQP)abYi~$gXZ74 zKTgMYNaCd(r=It(z1i{)$i#Np!^zBU$0IWTr;nR|$G;z(4ltfixlRB3^jt0}Vz8CI z6YMb0tyE16zMgWnw!8*U@Mx)+*nFXwsF+k%3ckKbP9TOhwQ?zFTbpRWiU_2yP>0J7uU=j_d}N`w9B})eEsa>TW5?Bv>3#xosln zI?%IJMhgqM*_iP@S)Rom{_=1Ty9%inM*0hcInTNM0 zK3}l`^UJnh3eZdg_hpk@4*N%dH6;0&{)xTkZ&J~=z=ze#LCUuD*og>a_CpXo;&ch;^^v5AglLU7 zkwh+Kmx3MDK)h;rxZXg{ftBxC8pRR>2&XWNFu@zvP;nZf;%0`{s12PRF$l?U6M5rH zyXwJpcBe8kFF=*07_`(~9zc>)QpNheMP_a=T-g`T2c=iAJ?z)9WL?c~Jx6N8XT z7JSbe!^&qoKVc{fhH%j%uN9`?hZOJslseR3nV^u`3OU-p@Iy8Mbzm!c`wrUu$5g)4C~h#x1el3a@BYsD`| z+<3&sAT>zIJE+V{#`@*L?ZC_G)gzFgrObgaOt4yY+*(OnW?weyx^IOqUJx9cX=e@b zMK_rljGI?L`R_tlBapH{J8RLjJJtdYB?x7`-arzxf@oE1h^GNJM}n^8sggWq`z%r` zZ7r~}>W(De5z)s7Zeeas+;-3P9>Mv{1~6LKn%- ze2FH@B&ml`Q?2(dUFCAEO0OaJQ*uvgUCttF&$lK4UBpP+r;|PhpjUU=+1d|5n(sT-qF5P*5f!D_|0;>=)r8NS@&0^YvTPG~X3&E<2=ex>l()T?&oD97Y@bbSHc@_h|IWvJU^o=KFVh zS+^VYsiI9o=8NhW@L(Yald40f#fkcU7n?~h*H3_19U_%eA2}GmDNfJH+`eV53Kw^} z?+m~+zh*Wx&g_E(X>KZkOV8QE|-?@7RSBDB$B z&cSUsUdC=Bhq9hYoz zrx|cuFDf)Pl_3L_<0u$9j4$@YaRO9PZKH6lHYaw^*Y&!0wZXTG$=hos+x&7#B7LFQK|q1Dsdm*EBlN>+BI7%cB@bz0d!KwRfzyHm1UT<0*VcC zC<`EmnMJ{ri6$d@9E!?zO-ELD#jsf(aGq0%8nVu#iBQWuyXJE^r??eo#Fit*z{dJ& zqqJ0hy|Q@uzVmr%W;D-5?q=@F0$ZM1hcOlU4WXa*54!86D=7-7!%gaWtWTkg=nwz5 zYIy!Ho3xnE2bbU8`~LTGputLPV>;D^_=cXqUe7&#TnW=#mwQJe`x4WhLNzM6%1fj#3+!ebp|(*aiz{X+qsXWnLqY{AfCCe`@vpH^}3F z!vUR^u7Ml*-H7}%q>u0Ec}2!>Q`sjH;<0I+EbYXg8puhN=xUhTKp$lmqiW^71C!k| zFFrE!QNL8{?eNMLJwTA~NS(Y6=l8awFUW_2JAJy9p&^p}IJ(J@R9HNNMU4beTVlOl zG>`UOfM=a}2&VdKA5`T!50-`hwJalvGATCO4hH}iTXOf)o z$LV2@%d~Ix9(kQ3oz?}SWtHQP7BI`eMdh!VX_KrPslCQh<838mO3KLu4NnE6oqBvC z%uvx-nX|N+TW`yv%d;9lnXg%jMP4TXWCM;XAL4DA>pFkP7(`UwrO#}UdNlFOeXK#!z;PK+`zV1z;^j!O_y5_r;Z_`{w9Ic`K=D%2 z{F$a{2>X8a;TO|O$oK=<`)5BIL~|?9JZj(>In*BZSnJ$k;>;7ZbRiHirF7ZPYbaO= z3$Q3${qXhNzqtVyLXqyQ{>uu7|Jwp6!5B!z-&U@AeXvHw{AI~|`QJabG_nytUq5*< z%SErDenLsBJbw9HNQv)qd_esQOW1wJnC#Tn(U$Cb%XDuLX#!*yM#Y>I8QhbP#hhk>@d)_t7lIL`QIb;V_&ttayRRug@ zby?SN&q423YH7^ouj-XMd|~4hxTvbyz&W_3KinguC=UI5oZ8NuM+=~87Tl^GVpoyX z8e|h)b`An}yk7K7&dI>rbfLAz$HFpFLZ4iW|Z5gU3v7^m>wh;o_F zd}0}4-jg#+eGtI20v>8S4RtX+9S@E+0Njy`dDU0%96|#4Zux&OYQHfX#=F*mE@Jne z?9h9%WEi=%iaAt?1kESUEcWcCw||V0Y1^M_y%!y{5}@WY!dg`gJxNNgbNByh#Ftp> ztDY1xZyW5t^iIc0aE|ll^uBGcuFrc7$+@z&wl(>(w&k}=a`mcdn~&y!_D*{LhRGg) z|D82CT)D#Je^27@l|z~(S2Q^4$oBby4Bt%@r2iJT2kSt3O^d<|LI}G24}j~#v&Y6E z_qQ$Xa%Swtk>9z_h8}YHLx-)+1JW>9sH%gN3@2;-;k4;KB`}O=q`Ot*tgQaA zA)|{e?eRZ=cS!KLRP%d@M+zVAI4>X9oq7GT_A{P_t~X6c3OE#=A~j;v2r{8@hBDeR zGy>zX{Wx0rmYZgA0uO52jg1bD^mt$LGGJ2UE1LyXv^Sg4sh$O#l3iylT48Ut+3DbZ zZ}>t7(q(8|rTXAbw+j-|CsTd8>@l6JGhL9L0PzD%>9vpR#Z$03fw*(2dWmnc%%Pt$ zRet1ipdNir+=_G17YUMHgA7P18gTbTd0#f;QqO!VnEZ08V6i5vD%<*qx?xRRkdUR8 zrv(U&*=tdLF22KAve5Up!hU!tWruR0ytMCc|Af6)Xm&0u`L+=M@!ed(4@!`b%u1y2 zs;-E(zrLiS{mjq0c0U{=eE9FEL~zH+N$r~hU#m>pN^1yPzUSxnztm{3gpQ#&JFIU; zDdV1v$T~w+b&W*OFN7J@J3!Em)08bbhuxI7#yS+GT$;-;`&z>4&3=>}f4<}s1>h6D zmqKgUlBbCAmTU79Zw!t%&G#%JvDG676y2CM?3AuJV!z4s0kfz^dLDJkRffd?obDIWX!nT*P@(ev-IMQ9tbChWYTcV<0a|KbYa<_^o+dKZ&TlV_fpz< z0J`~J?jHa;X8rHiLzh3!57;s^xmyGIi~l&iM^~HsR1-tEK@l@5=$R=op%AA#k1v@b zH`HyLI;0!=mXz4tHW|Cx8zh9%HKr*1|IJ=>{>twZ7K-nI-t z#_wX)D9O;E(8=lFRJ!B%d$;8@J}|WirwFwp1{2)S?76w#-`eWUGN|8iSd|5uRfz*f zBdW>|FzDxRA%Phg_4fb)g! zIvUlr;YT!C>J0VDss`>!;l?)VTRU(mg9qyk{yORpJ@PL=JOs&NUhxzTX^Fow5>4W< zSnQl`A;IxWNEgQ(B|06B&@t;Qqfq`Fy^FaHPK&gLR@TvbdRLymFwxxSvTDR=WB^7|y|J&`My9N_5S9lbOBw=DO}K zs`uM_hs2@~nh@dn<9c`3=V=Q$+v0j$0!p`4t+SkHZ-7_H_F<;F-SBHF*c@@54rY$yIW{@LG4y-?wrW1}bE%HnSGWvGpc z0sgCjvlb&aT{(R+`}~Vb88n6Ftm0Ke)b}RCfx_TU1~>02DC4BIyklsXJzwtGgm0ll zAi2ilns{yCFJ#HgDTN4AJdIdXIys5~^-PB?D!x3a+oTj0I*_Q6j6aQTW5=LupHIva z&!$VGh}a0COGa+ZXNu4g3^p=H1#jrBz~c1S#L`+sfw6Y9k5mv60-ktnwd!I@&Rrvo z5|o=bj2s1|j=Ka^Y62xv9UmG{5&a+@NA_CLi2$GeMA7J0>FF(twAZ3M+R9F=&m(4Z z`Qpu7ZLZMJ)Z1|@eCjLSu|p7ob;gC%7FUb@IQl*df~|-As$!z{Xt;t^uXzP=&0D@a zJQnZ5uD0i(5I7W4=p%oqyZokZ=o8FhHHsM%0Ah^Ge5kf#MygP8Up#6%E$mZVh<$Rh zN#<4f?rMHEvvQE_7+5dR+|Wq-th7#Ddc$qZU7{ch!RPEzT$=C$T##V(CNjaSr>4m} zd-8qh%ng3-}6WhDaBEJERmL$5COu$7u@>CPRkoEiCkBI~b6w&4pf<-}KK!C`aEfr&XO z>%;4R{qOI^k;^A=hvkuxv(2f|epbf#3C>JFj4$sW`!F?^8NzAdRXHh8sz`p!`j>ve zfm%vOZMc0Jz)Y`D|NW3NdETLK)h))e0Cw&4A#<0LpFG;};q}7ijNc$+pN7@vz9DUd zGS1R(Y~Z2Hz3*PXvr%p2XWih89CnV*yI($@%_g0FMrU{|rFeIj6~)EtV_8&`^VG#R znrIGJCVZpo95XmeODBP_Aa#X4gJ|2Rm+FY?fx60FLW^YT?eUA1#S@UDgrd0DuYz># zMAMMq$}j!fR}o^;e$owUKM8{I<~$znwE5}FxTBY%gmP(H5liZeD|{sdG<_CXb(>8L`kICi6Dkc^G2V=&mR5Xk6v zQ|8M8h1BXF99_*xvksYW5f|)3A?V{WU#Ifn!xmn)eYb|a`{Bo@p1~DTKL6yMfB(ed zQfFAtw?o>te*nSLnd@ty*`zPUYV~6>z?14J+|AsHbTMPK+BZ#}O-FFO-w@@}m;2*O zG`}C(x%*cs_$YTLoJ&tQk4;Tqns>+Tt7Tp1brK{#hO^FukRCn!q$DhF4AC~wYy5CY zS(|C+7_v0r_jB{#g#q`!)0oC$z%0+7?I^62K4byjZc^T+1BAWTKJdN?+TU)X7e?j) z6Qln=VLpY!-sc@2npcMIt~5&jPJ8f;>#>hg%E`tJp67n)Gx=|{pYSc>qv!uP14)DsSaF12BHo!9eXrexOa%arKP%I7{vqg{dew|XeC_M z`L(3$bu!bD(1|51?WKZ{%uhhKz)6*6-l^e(7Y+iu0pUWQo)SG%?Rf%BBA#^U?R;&n z_&YcDa{p=Hs+~pc>a~THyWcRGgD!Euj74X#hWvZ;4U4`Aoml4av}-R(pFa#mch&J7mriuQ6Gr`)sTITt+2X?H!KShLl;9F{2t~7Rj=rFVDW1em+bn`;51>b~)%n zhAYEV)PoJ4bYfzRMXlOKfopN-vLd`D?@}>gfIxj8LItpyl*zzbG&(#N@}p{$v>}kp z1tJ7xzpG=5p4G-tEClM#sPM1UE8e~pcfMANFzdCdW3xEq=}0EX&%ub)7m-*WbB$;@ zoyW^tZm`_*EsX~f(K3AMx}f(YuJw8U0fuj&&G(Qezq5z68*jpe7aGoWhQ9Zl8!h9t z5hyloe}X^)VK5lce3{6aHE0=!8GZFWLXzSx8VqIJakU&AUw$muO5i=Y*R8}yT#CN3 z-PEr$A8=Lw0!rzw@$LDq2s1?FEqAxATzU15pC?EV^^5-fOY#Hv=vx+V-?!Wz9g0iV z);9g_R5E}h{*fs!6#e(V{rJW*9}1`GT)iAFZq%H;o?Y0ldp{hs;dW7fIiY+G$$ga? zDsi`UTW)29Ym3`oqwZH&GyAxzQ5l-UbJX@| zvsQ1qBsBRpaLj*1TV=^HXXS!I{zWwjTcEkZtCY0Jpnkl4f|b`1Ndr?mjQEYE!x0r_ zNMc4`mF+bqCi(5?!X+@w2e3dKgf%W1k^pdM53_$QK<0^~feXnyJV)D#KcH%Kzso~3 z3G50annWYNX9(Ciiv!tLX_&NS-1&%N--jeibX-i+ZKN^myQiW0yJqw!nSrSaFx1Z&}R6xI&|tUp*gbtZa^?y^GRw zPP=aAtz84Fi13sh;AD;lTjL$P?NqeFY{XG9gD{E>hn3Hc18>HK;~b^c2Ro$WTuupj zT6S`1`3GR^F@YNX2}V`Zyk9*2X`E6SiW^V@40L(0%${O$+1BFZN;RTU1SB>wU#duH zj%YCt?nsKu^$|c-1o!whOYFG*3XJuW6%bBa>&;b~>X_2~a5HYXs<$k|8PsK><8?EU zFD%l@p?mHAsjvD5FTXvJ7r|H{(5VVWdZQKicx0UI~0LZ~m-G zJ+a{UiEVF(0zas#^47qehXP~jrZ8somgSw@=qBSZ9mWH#7@7T!D>dE2#!M=Iu2Qzz zM(w{B9q51cT&?~x$yxZ`@uoR6qwaUN>FCav-QDE{4U_sT|Go~N1J6BplDj77Pyg^m z^si^XAVp?~YmUEPE|1qOTq{AYxEYz<`q{tfKj8DYc(T2=O#Q2gw8Gb>`PX1xPgUQF z5lCw#hF+EeHl%}QK)TqVZSX#eJ6n25+}1>`QbZlQY&_WJ-NzH?hwsa8BEG&_U%RT! z7WBh1LAcWk?iF>5bCVD5t9*m)!#rqcgYB&4T(0uXS<9ypvsRvgTBZ#bmW5@i z)R|nMv4)4Etqe`^x-I+>ctJm&SbH1l1T%CyqON&+)Ec0A4nDacsu`ZkyD?lHG{trwFEe*eVLiE@W;n0a>9%Jb+7%HpqvO=VtU?1HqL z+hQue^jcpfXz|@Px||(oafOnPN~VefJEq!kRT?eDaysFdY>YwRdV6t$wURfbwV0SW zJ6hzsEgPaPFP=J%m72mg=vl%TQ;U09_W=)85HAl=^b}qw)PQnN;!B~75Q42DmR|1; zQ;`s6VdU2@Z5begrkIGIhG*4(-JyNe4MlxiyAxxYbfmC?uf|c3-0}>K_Kv4CD0_W8 zsC7sNJBG0u1;~EFXpnzm8P)*~Kju(%QOprR#vrM>mUjE7~%1?!XCa&MzH5i|n#s?1w|SfRT+)mCJBDwj4^L<5>~`2cX{TtE`Z z^`_(+;;%lun{6koXJ8`4mIh;7wN#MIp5s}Bi8|6_O0`q3+QG@80C#|~4h1BBKWf!! z6o*BKP!3s!m!EZ<`cfpJtAYp1$g8IXXI#fyMUC4q5vUc97bB~7c=3+3<5YNS5j&)H zd9EGvoe(VDo&`Ii$NRX7mMHR$%0xE!p?yMITpA0J1(U-rYhXu98@6E?3l2@zR<-u| z2jClbO>cRjz+rjMG0CxSzPeaZ!!vX2^2hY%VL%l+p^HRIF-_2`@`8AUq1ppt72?`# zfa}Gh3=phmF#`mEbwE*4Y2n~e(f$g`&JwHFV)kaf98Zrhz}#pC&=Ex!$?H>@hkjHz zZ{JMTiGrn~aXi4J8^q6yer>Isf@4i2I%m~2)*N1AqcN7XQFXmif7s%070kcT2xX$~ zSwAa(YQIt{Q#3qP$$QqpJZRgdN^+aTADy^)$`xkG+%8|+ z2W2`QZgu?2cP?5rF2p*;%?>CA4HLfJ9SP~*VexXye>Hw@&#a3a*K0;2w`|_A(Z1wn zPqdMpbJFR?zbx(0Jjix>H4FM~*`F+|%k~c-+POO7(Na)cL!xQfwll+p>);H-k0|)N zy6|SWIQj;SI6Zt66%$5jmGk~rY*-TTe~MtfsZ-HETPim`U*txz#t@U9fR`UUdCuok zAm_#6@oM**3~oO57(DWRs`;HUq43&a$fMf(=BsbYItlY z-!SBg%RG6g4%H%tmgysffe;q?Iu-44j$Kxcp=qj@=RP0u?}v*jWd5BC{ZfkUvc{?~y*9@pR-fN-2CH)v^a}?#kcD+-% z81oDhPIdNNfG>xQ5NB_siMgNk_&2cY#*8>bGYHDhg;3R_5+HlV&r$qkqFYwdn1@mR zFaSvr(Tc}1*rNiKAYFziL;Xrj9yP~@8$tzTA`KD2pW3wL7Zt)LXEt86P9TyI)NQrg zM-;SCTEdduBWHWh-*-IjhIDcjKd40~5aDlDqb7g!cj0;mrt7U3K#YWOJiS*4N!t|I zjtJ2MsAI`-`fIVg7TOyh5o`b{RWim|8>y!vLo*eJ_7aq>#NQbK2KHkaIouE)nA~JY zk+mrLK4euK(jFR#KWtqpU>Vra7hy-}iKP`0;)J5v)Rd1aJ_E?7Pj+A|>iDE2xh?Gp|o9KVBA>yOFIgVck(mXYz0zlk@7VuN0J%A;01viL(AME;-*)Lv)31^CYe?H zv7o}Ol^HA$BS<$3+Uvj*vAGX;`mOZ&xl)fZDVJ~)snX!{R@?a=Z+Gw_x_AW zytL62@|d5Y!8rZI5ta0ve5vz!fOBg(e6$dhL5Pi1fggaRKnW!P=t6P|D; zr2Ibs#8a*qpSpm_=aJV+KPZSOMW&V@{qB{pJN85EUWq4Ie``LJp&5HDcVOr?$}4_F zYH98P$`JiqXmWc&;CJl{e?Rny(oB$h+H2vQGOt_m@)eClt95b~}GpO!$!W-se0m>Phm2`m7C?g75g}pd=TKC*Pz>1{$~T zJF(|3D=@gyY`Yb+_xvw)cj~V|=mwEPGGr6IJ^mO`*BrXV;aPwSU)HoAzQZ=i%m% zoLm|;*Ovb{5C6$eg$7!uy!%ji^raYm>Ll=Mw@*QIeIZ}jqw3uz1;Z=SKfc2U|FRmu z(eF%G_ZU9aqGC5!kogaQ_LfWo-5z)A2pOA4k-$GgXLk|TzL-k{dLZ+!v)^>(&>pBe zZ2`f%SEfAjIiwfD?6{atD)Qg$-pJc;-L!T_xKD#PwL*y3p-z_$FLKDespPwmuInLI z668`@+=`u9Zw!J#Dw`LwudK&D0wi}8kEbi!Qw?Cttq#m&0TA2~rL~>ItV|)O@KfOW z%hnJ83}utk%q6&i&>U~nV-?F57$;p97=`hl8?;%gt|j^x-$8vFD^iYgah%zxXeGM~ z$TVaT53g)^?7ZGrbq_a6FG&n;Y88amu7j$s$hl0}7W=l`xtE8vdXuXmL`iJKysAQ# zo9h#0a6DpVy)u``AtauD5<|9y8kP=T+Ki{;VZbZRZ|NKI1mf0k(hsit%0Ba>#2~+` z%(Mn5=aVxZO~i&+i^cN3#RPY4S*YQw3@829xN{LM060AcKLLSC^Kvt;3nj zyamWqBT%V_M`X;50F0^*cf6tJ0ngFBL)3PP<)Kq^C^C_;lcA#1V%ATa6cvC=SY`Q1 zx5zAd<|t8edj>#a)dHiyyBNlG6s_!1#T^(0NjrXtLTeDDjPPU=u#6OI-JGO)9~79p zN{~siz>BU-wrV(<3SgLxS&}|MBl}Vt*vYMXH&bFhdO=rjDwIWcD|I5QZ<-ah8?F91 zL_|~`nSwUNKRto}j9te*uvI zX2xr?h4Tp$Z7=;ugvtF6%H9L0sr3IBK1t{jKs+eDmjp#A2c>ENfe=VY0d)}-K`{YA zid0(yf_em`DH@6qQb0vl1qGE|ED=}@LV~D>sDL0KDgt)h`}q6+-#hocb7$`Rj^m8W zGP)bjd7jVr`)T7RoPOy3`mn!k>B9`5bbEqe<^m7>^4Ji$^XU5KJ?UOqB`IrkvM&0| z_XgEXQH(>0jvMKZ#oVj%A;5jX4kIPM!ZRGL`@P z_Za+lWu$%M^iR>j)!U!nCY>HimqHQZ$JOx%mLNAAdMA3DM~;nnt@!e|D*m)?THA)p zZ4Zwx{H6Ab@ym6U$+6v!yf)Up`MM-yM#b#*q164pE&F0d9L}4IE7{w2y{A0+shnAO z?|k>&o0UhVWXwHpPgl%KQc694t{;wVYj~%f*=zAjE&s*q$m*fiW!(`$N6TKgyXM&_ zuI}IW*CXvytgZZ%$%5z3`g^6FziYD@|_;af-7+mKUbcSXhMYi=gNgG5BO)N|( zQVv?+G*Ql26-n&EBVoiYrhR!w>2MFGl(UDfY-OyWt~?by#21&-p}s@8K1y;84YTVf zBMSSB=@5Q78_p1@@ONbw&xz^SUVm5@nhljYvZN3qH_U?N1j6kHg^m8g2G3=#9<@X#%nLCh=DNf`?CTvWdZE*qpsVW*d-U!*LlG0J1CMi1w<;z3f(l2^ z(Oop&45_L%V>rnH$v&$V?Q*q?W4kv%z+1B-d;cH`fdUK1aglwghrAT8!aGcL%5$5g z+McqgA+`~p3|I1@!s~sZOsB7fzCZfP{T#whMj19+jGd(o(wUCUS)WLcMu*JH@29_r zm^+{|m{2Iq%5N}_zp@Ele{*~IfbD)ap>%c0#zdp{QF9kwH;g+8nDR0{<+5+Czb)IE z*hmQymyu)C$GB{FB$^9iA01o^A!J_S5LgrEk3hewi8EbAc$a2dC(`0nz6z#7 zhsx4yIj01RNV?mYxgoV&)4LkD;o6;J)i& z>PO8tb3QI~XzFq{8}{SZ{N>+!bIVzF*|HN$QhWpJKuU)wJAgMMAM-Am?C{s#Q4nYSnBz?eydNdfd4me*N>z z+ru9NSU)WOTw}p{5OHK~xn#s)LC`;glM8p5Fth+LuxR)Z^XDPU+ucFOK8q6SKI*-^ z9CuCYZiUyQpykzF_sb}+V)~l59R2qEjvJae9ri&fzU(W1xY-PTzQsNMXu}2Lf1%8> z$Y(*5an8SvuZ+2roj_afl{|koC6YW|u_kYA1b;(#3is!WLWBBi*~~;cG{*FaH%Ihc zBjlTWF_sJKvQ>C8E4oFH&>;vU=B+@=cj=3l2x2}*A6r@1&3?v*m`{aLrRa)gM;(nc zh{YAC7^4MO4`}SEGUVLzmMNh(`N>L}im(`d>ZND7z}e`*)ByG{f#7{`$iv@u;U@wn zGna%jj3wTylyjHV3#A4knG#o$8-f)wNEl>r?DREQ-YRsusWL7KR;t+~zdWZ>=Y~W5 z3TA`Oozqi|%{bxEwxq^Mx@+L{^M(toJD*MWm43UZa{cJa!Ih6IU1PJ8G6IALG7TR! z@3q->rv}rgrlqfu^Yr4PZQ-IQ()?M+n&aE`RpBGs+qLzaqXhvrXSchqbq>*>#Eww# zI1`$IFrv(uj^ng{Q)Dyxxo}aczq#8>#o=}tv^H~fUx8bvq-r5qsMVD3w6WQ(eu{74 z(kaCCpVU@G*Fosob{Oo6jn8WIkl>SZ;7MWL1YLOLJCJ+=eLdW3s*rkScoVf!$x6no z%bXTTy4b0)j8PNEstVhdN?JI5OJs#+xz*tekb`TTY4|lpY=@figX1W~lC6S8>Jy=1 z`rr_A&HR0C`0>F?6I%B*Q)tY{$r8RNsRg|FE$y%g~Z-XX6I@&iqL*$X?oIxwFHHXd+GbP?h2j_;3IC z-gHC92XcmXe~ZbSsu0D=+z+W+4b@9R96%iuq;G^D5RHjIit=e!fIA)LW)SJf6A0GY16CuU_r#1Kg>uUvl zE>yjn3Clhn?#-?+*MRty)os+I8jCLyA!p<8<$s}&>ODnEX5VjSUHlQy@0xSprV9n$ zqh$*28vB~)*3k4vUR!G?43@0weQ(x&DEa5oLy!JKp56Wf`M$=bcXZdArFWO71w;pv zyTO*M^usk8Va&cG7@TzOmew$z&byveEb1GpbvC5D z+_>jhHcxEbXLCeoovpa+XXCu5Z~M=QgYx81AFV;cAm>zF+7e7E)tb2|fCE();;hi^;m-1mQ7^mm#rk>MwDkrc%FTE%l#}vGyKWnrTn#*AFLfazapYuT`9cT z3$4}v!3e$%@JVzo4gWqw>d6);cXOUAyG+nbfPtmWi{-!Kxd0M$bI!LKz&~>3d>>Ub z3UTs^5fKIkGeje%bCsmgG=Y-6kNB@wOx#1{t%^Xc_0$f1Q;; z#Nb0D@=MnB-U22VG>jU2R8Hkkmk%03$;fVT4ME_$h)<$%l2N~DAKwzH74p`Ym5eGO zGmK{PBO^nou(keV*pso^;^E?d%955=_DgR6DNB4F#En1p*zR?!M7L<^rt)2JV?U|! zaoyXFx1O(hxqhF`8R@1%;ifb9<_B9RD4eCKr6c3JPrMpHKK9IQq3i$(*LF^AuD|74 zWFT26h%r3HPgM{gkOG1XXc+%4L&L%QCd+>g9LZ+&jpeMBSjd#3SYgF|7;}B(?O3h5 z6Z<=O^XwQUHR8LEG0Cn7`>H|+3{e94)mp4xu3HFI#c#h>vCuT?YG=uJ-- zi~rtvNF1vD_4CEW+DvO{w%P6De7x!S<6ovT_YMs!GdSrDM6hvZ8n5~n+I&6Yi&(*| zd3wj{^1`0{i=iJX?j-~UKiyStc6DQsO6UXCOMs-~d|q@c=1um}6CXetI9~NG^eZ*Zvi>Nr+wh_PQ#`^;*MA|<&|Foy?0{Un^>OX7nTr-Pc6af?5#>L{udkYx zoVk7Mc1we+fwlkrgvj)>3B|?{CLCKeTYo`FB^IuKdvU?yJ-ohGX72PgAXBY(~cL|5i zvI&7K>ODNPG^pF(A|+GUO(Q0EvoQzqE1PzU%;~vytIMGnw$y#ysne(V72v*`DPR)L zJjVOHOeXzKBc~!_(SG$->%&4$I`rC;XPkyQ}^50LBl>Bee1o$ zckgDT?+^`)@*On0vY)Te@Ry2J%$FmGT56i8JAjD!%29?q1tMS_cYGf_P~w*xTg^j@xMUb(S;XmR zbTo@NzzD;}*dd25B5lgHt2ESgirmfw_(M^kNQ#&9^DXPsY=}liY`Lx>2($HM@gBw{ zrqbZM%LnCLWwSzy$j<`IVWAQ;Yn$Rz0<1Nn=4v1#Z-HadFv+O9xYHZ7V;|)83)foE z-f_lJ-#bV&FCVnsSunFfdFp~2yC@i2t~aN{q#lDekeU$HWjI3_--Dh+^yq;$$aOi? zl{{sHM+1icm<#L9(NfXN8#UspWcN5&=QQ<|b=i+q8wXSFcs<^7PImZNO+2@M`~6Gv zXOD!&RQz4$Y#LX#yWzq!=jbzQ(%bYMv{#?IVdM|5yNB_qEiHCL$_)uRVF^s|D3aBl z0M|U3Q?407iJdcRkDC+zEPYxuHtj#uu^$`GMv|FNbeD)Y`l8@m$WIm1U`k)yyrG z$71?#K5#O@dvBjNJ1mM?UH#$nJrkp_1tv2qook#2FOCniVdWnOK7ny%!fQvJm5Sa> zNX9bB>(osuxg#blzsd)fZ=BHeT$;4M_9)uctzXH?mjX-NsrY zG7$KQ?B9PT)uT{0>j?a|unA+@8{EX4nFt6JK}KBhDfRc1(8jf7nUj_^-y>a|=q|z3 zG-*c0@K^0XwM7>50+mkfxRi#q-xRAB?+nYk>@-1wXmcO6hKLPSFMB>FxWQ0vO_MO$ zht*%oU1H3_ujX?w2I9{WegfpB$+*tfM(P?Jm$=s!%>j7e!B0;{*)J8>e2dc` zG8+h&3p@0pS1OG@T04B+LKTg6*tP>ra@GA7R;{d5)PWfKbPB4HNh&Th18i|>s0k5< zwlRy!%Re|1FxA+V3EN`iuG*w%hLlDQ&Z{g(Lnon%tS4mjZWIUQB>W9&uATb=B}+?40&}A%YzT;mytwv*i)oid8^zc#tIO} zeB(-Bgt7zHkR#$yEo9L8^69hd@y&)8XaP%YKT_+ae#fMbe%)x>;t)Z-Q51i<-{yI1 zVBAaBu1m|8MkL?R3yQEs8r!AUegl#WWfbl}1M!;;}|A=D%isW#X}%}xzZ z>YvaZkpWLu3g`FxPn*jpNLcu2#7G{4@3E&TmdK$jA-ylz*q5Ta1STd$L~rIUlqB*hH6ig zay8|#<#IZ9tO6MzV7`j*4Dc%qAucn;UQ?bgSa8&Hh2Q$O&3ldp;iN8gxMCk^j0*0I zL><3WbN-mt-tCW(jt%dGOD<_C(tdn<-0&r1_+D<~GsE$zHA&>Tz2kY zE{m}}Df7h*Lg<_%B{DRmR&cEq%{)OC>RM^$Xr;2ok*?q3gw=+zw**dzob*{}eT`;M z4NRnGiBD;NK$S899!oH35vZuf(C%f+$11B7)=U925nI3_3u1UeyvznxV-EIESLU#F zUilz{z5P5Mf(al9R5gf;Pv0YA4T1-X=S497NPhuSz9>XB8FeDx>=~px6{4}qZu7zs zMCY!QT!-#5kl;w>_8xt-Xa1B9^xV=^QddI9gAIKx%3p*>C66~bn!Qc=Lr zK2GK{nH99rs70{M#%2Y@k95$sxEIxiq1yy0Q7BNUX=^9FFE)HM}No*>~L!Z?0pGIfHambH`awo^mZ4mKs?P1fWzYuho{Mm)EX$)yU1 zlbHUS%-gRTcSTpzG#p*uM(jQ6+LS`Ted%fYhwlX1_af?YMEn&Ks$q`r;>8@$12?Gkdx!cJzGak zYH%NmGE36%=1hf%RQWnxS;)zGk){(GZ#?6Nc}^6f9Aq$-rzIRN6!U2W$d=D}UN6M4 zr9eA2#>GsKZ`U}wi?3{(A*ny(gEL&nG*wZCkOB~=K=oqkkDvwfb!i0i9&EbkmSrx= zsAB01#xy#0>RHJAEP$Tv%2kXj@4)KtwsR35L-saz$7vso;rj)8HpoglyyJO~teu}< z^Im$X4?xkR&xOB=$A_-hzuUU;mH9m)(O(sP>!yE5q}0oI3h`w|jg^DYn|!N|%waKnaH2Pz96P~7J(#E}MH!3f7wVa1mr$L6FkK@{I7L#c zVjUhksVZZ}pip#`tD7-DCW-U12UqXS2I7&7Glvij?FeX>Wm!h^a=dXi9lMPQ5K9u? zVoNr}{42jgNh9$^c7{0}S1xx#&a&T=)Vl=96izHQYlj#c`yEtmW3non&<0|Hpb&4-UZF>L)*ZE$hn zhgJ7%kKMogc*D)8sL;WE@lSVNOqgE~)^hdKwj-eId%eXuGo6F;q6v9f23%RRk7%g< zd@ATVp9;lt;Sy?X1>CiWk2k18G!qGyRiKM5mLTiPMh2M*NEI%43ie0^e2CFsfkOzA85DybZ~+y;U-j#s)~mQ4QxObol6tI{ zI$U0|v&VYFHi%Fcn{CMH@~SDXOEQ2LADXs~X%4)$3ohzjVwtniUEntxL{+J1#%ju4 z2?A8bF--uZSP1kWHJHLi4%)>LpEroPx?BR>)SgsZ8BEe8>Pr?f>Kgzcx0s+}PY+a* z@Svi*v{>um{mmsWD}j`ItwYP*^j{|DWak<_P~VrG^}e*zx^7;taKm$>-s9EB*70&N zMf2$^J(K@fSwxb?wYLwq&_1TQHbt|jIT$L|m5@f18RDWT@yhwxiq%Hw#=pS%jMoy&y6tTMw zo^FaR)_fK)Pz=~mh?^|4@l;|i7Jg5l2albONsz+th?Dh?hT^fRl{Szm3rXy>FI1cA;$NX!hexCfyOwbYf5s)+HV}`!VD0U73vuewFRXJedkliYG^>1L{AH zR<6dGV3;=j;+!1g71kl*${IFikDRYe6hL&)?gTl7MnEFz>KrUv2Im)X32v@-;q@n9 z#jQ1O{C%2~wAaCDFa-2=suUNJhBmy&w}$ZWK1sy~T&qgaIcyA=^F)P!WEp{nIcJIY zcp#G_VZ6~ex>zf_2iM?48moZ8^ZS#SIvwCbQV-yy^W6kWGH{|tgAR_6UcSoj&}A*5 z@Rmr0YfNMhI#C@++Qz#ZbfYEwgN6*s9b_$9wDSIq*ALD9XRXRymkH;tCd=n{N&rudNF+%Cg;$0! zS&&c$$8r~!Rv@6u5FV5($byhaxf~;fH6vX79U!n~63QS?V>-8ZJel%|u6+ z0Aey}K@Xz0?pe%WDD#?OKyvvL(R;L*=LFFqnLsO}fd9fB86Pt3L3N5*0ir+2K zdoo$2c!T}(ypcaPbvPUAxzf{Qo&{rN>z8}D4D7xkYPl8!zupF+1H9|m>oG5@$%7ky zRv&w)KJq61Mm`DKaM2_Ainsoax^*tcRm7X|>khy6kMQ9!o&6&fSNV%XK?6L^Zhy?B z05{~(Xphs^F3C80c(jHXHqwZ*!C7*XI5Fwh>3kK zi3UzGXYy%_xV#Px>gMtv$4~A4>KCS>RXc7yulL}^O7(WV-WPlC=l)^#biLuhUy%zJ zBl4DKL7S?M5AL~~9NG44?Dfxc{Q&H3WbZe=ezP_0#OoN(fc^CB{b?f)oT+JwqoFKR z__er)YJt4VsobRMXy}77i7f~ul*j&lz>S7-gUQ)+oO=CtMP;K)n=)o9qyw3yz9)5n zs(HYomT&?iD1c=!brXrf^|%1_qls&%kZYc|8n&`sQk&yKR&p0#c&m6QLUhHvA_K&f zFv#{&=qxAp^*GT4N?weh-~pa`0Y5UjatQEo5s|EQrw6#Yr2xE{5HJ{b2`-2q+2fQ% zbo4X?<=h$>8SVndcje^*zKT-r3%I(CL(^sis5U|_#)^8W16GHXx2nwn@|+?E0$Adq z0l5Upi<->hpbUMH$|=B*Sj*fu>H%Xa1XR@R+gREBj5v~ozEGutuL?uu<&%`Cm8HM? zzim95%|(I`+MFRCRtsSQ!7km2T?bygMM#>Uh(iLktIL!9Enuh(M6Ve$3wkgGj4BO3 zA1^wlO1>z-iHfLkrWoYDvA82|u<@B)3BQiV*tvUi&T8CtqIlp)TAByYd?3WUl@Uf; z;swsAx;$)o4Uy;~!};dY^u@6!c!ytPb5+>TViDNH>@pCIz#@MHw7Uuwp%IzlHt3EQ zqld}tQ6s8LqRU_i=-kwVQpgjTch!+JVlO!gw+!<=PbA z6e325bXVMBk!#HJmA)3<`q;Op=J=FzVCuH-iCZ(D9LUQ2F?9#|e##+nq}pg_$)DS- z`gUJzXj(bUxl6J)Oilduihh)~~V!^_#zlHa4#!Vx!|-@aVCUI($54U_uv@IYLtL=ZhA?)S3oD< zCgQzNC#h3`)B+g+%oNv!%t<0@!(0w5C*Cy-VHy}|6f`t+iWl-pq!hgVL(7^#MDLa$ zdId6nFPVxnM(HC+dzr}T85%>Y8>(B|dVSxY0Ua-zBdEorCq~z6HzzH1UR#>+afI3Z zJa6}>U#9N(G~12e>qMIue~7ryS-W7a@r-|4ch=+JnDT2MF75T6joLSzd%=G6lFxP> z6kb?A_ikT2b24%re(veu0gcH*S2*RLx{~%E)V6=2BbIk+wk5ahTiWIMc;m=(>bv3nAqd*hL}h@4p>(;u$)1)62g9)L9y!INLaX{TdAI+9g@FXL4A|e zu!^Ov!qcGnyR>;%ox7!^*IfLR9m`9kD|&_1{~n3Pk_WTFs51{U6i1jq~Z36vjegdCO=@vvs3 z;>0OFgR~d|4ZeozLhM*EpQ0vEmMJONVyq!SsxMHgL!wV`HJRh^U@u7n{NAx;cHrtk z2};8#M%|wX%d-TkrsW4g7^?*Wy`2f-Qy8Q?0jl0EUMzr`1Kv`Ju12m9QZ=(S{x5oy z{Q2a@G;;$2c30KP?!FH#Q&gq0GTM;S!w86BzD~F^agZo%(0}jyCoTaZj+IupfeTk( zz-I;!TLri@XtcMWM}>fSDpmR|;{fLq>tNme8Kn%xkF^2GyN9eVaS0 zd}~YW*p=}f*Wmq(bwDK^HT!jaa#Qw9@E_NSl>tp9>sVsy^<7?_~yWnj9GU3 znp!HpbM@WmE4!`P=kD^d!x}<70wha*T+!8zp|URwfB*Du%}e+5&)u#3*R5#S%6Sx? z7#dJm@$WARD0J;#mdi)$S%mE`J_As@%?X6TEZ?h&^naSTF0N z?4LJ2ugpcdQmPCK6jY;HSfi~US3VH8wq{d zRXTYq%{&G@BHZ#m`l2>|$ZWnsL$#S|mRAP|mNL(MZfC=+cT5Wk0&f)3FkRwG!E;qt zh4j@ky<1L;zcoi5qg_4v&NufLZrz^`TUY2@w<_5f|LoX&vhkjo^@0zB?~i$qRFjsT z4cBl3U2q%i7Y68Fw7XQY>rBM6kpS76%+QBbqaf{J`%=mC$O<%#A>=IDTGj^b5SLYRMHeli2?HQKQ`qQ_Gu`2u^MWt`49&_ zn+(Jmr6&)KGWsj~E8GOC2Qc*$lPP#dx>I08${eDg(3GJS5gr=;ZlX9P$B&$K{ClTO zATNywf2^|}tS=JtXo4830jR|Qcmi)1LC|Sx5zk9BAk_%`asqGD42Z)Xz&jw| z`~8VRF}61dn0T=K1P6c{qAOrJE=Z)r=r4oSi!=$gWdJGo6*^t-q3Q!Rf;}B-h58aO ztzdu`%CN>9aZaC{jWK9AnCr(c09IJiedTip(pI7uHC})I7M{~y%*J%OjaB*bZpcq| zcoVdQ2?%tE0D=mS>(w{)ISz7fxHZf*${{?TrgaGs zwM32!B}^V&+|Ig_t=Pj_5Pd4J$4q7Z#nm?~j{NH8N0(3idi5`a+gy8S=&xO0sORx( zr1Wzub$2_Nbx5YR1o)=Jl~hJl=*%!L*k3n1MRn2w+E?)L)0LO^PLPQlD2h3)Qoy=I z!Vh>8pD7chE8z1=yEImXv*z_SQ&$g!{=Lg6{ktIiANwr&yWW!nHYMP#f%Xv#Uw!xM z@%U-jTrc|PhTg;FW5f@GW_Om={ZbD37dn`{W_);y{2IINg!iFoo54G#X4@yyU!Zam zt*!Dm3x7=~_6C>!=il7^b@%kI{eeGc13!VJx7pACLPh=4`gNsNpRT;yTBt%=9bdaY zr|LT9p)Adg_Ne)VX&(l4w+wAMoodZ?92v_pnvoQ$^u!$6^_k!76pZe@^mchxhD(0o zw!c5#I$#w^msZDR+cQ&ssBpXzFplM=r8F-(|vCkHDxo0t5M?N zs4e5cSF3}5A?+aypDBL`Q3L@M>piaibojQ!W*7d2In5kDDXX;6eeB#CK+B%8HErIr z_>@?d;j-0=dG}w){_Bi^DIX#Bw@wsbMm7fgjid|KCNRpFuD&!h?vdduyxuv zBj|RwJw3P^%nZ>q)r4(?3!?O;3v{@~iacZon8DQHQ-a|#1bRj56l-~;27`JdD9v^i zbK5?S576`ygx&0Jwmz6&a)VHanM^Ve>A&}foEn^HacrP~Wr6=<1Cbba_+&_RbPtpj~Gd`VOBa`}D?QMeDZ&nrV$kO~LX{IIVMN$0ZMSR>uBm2Z>kSL^mQtR&e zuJp)-n{Rx$a(9IKNab|u%4d7d+w1P1`UFx!$)BxVfsNXr^PF1TVOxnt)C3KNUIqVt zJ=+@gFf}&!{pX+97b+s~GL&^L5Tc28C>+re1z51pVKj!Pr6#ZQD{eL&o|Ef+%usPnn7_rM}X16{} zJpS5j@!s%H8&nDt?7Dya6Zm=xN*==8n5%J%`fETjAW01fGKd0P zv;BM>_$g4fEb5bpa3aXok?6!v-8T@|gP~#hKw_cOBGE|jW@GT?Q!=lY%qUmL9K;OR z7FC&+x4iS9<2KAto($mez+j_Gzyv<8Yr~<*mRxL{OP8dkynD{vv{tBR*q4)!j2M41-ox zAc%u=sx1w;C4Cx4 zj@fdJLSkssA#qS)p(B5tE8s@2l%Xf!XQ#6-v@W;*uo(lZ(LFRWUr}9#^1ZaPHH;qf zQ$W|obf{^$nkpnKT$cv)rjB-=-4%Snj;$c zkbpS%fVU1%Xae`##+6qt>x3B>9da#jW8|)D932SQ(Bm?(=Fol+PjxWYXJPmH=@U=> z*cW={f=qpCi}RONcT+NVZO)ljcUpY6YqD)>A~?AI!gO>?9`HmRQaYgH8dTbwtW7Bn zbE;Wg)prL37KZrcIUuY&3oR4k9;Q}#TLbr1Fjb|R1e7ggWgv{xZ~Cu*WAop1*8lSJ zf3_Us5%3LQgKYPM7$i3P)#ijx{Cm7{t?bzBP|f(7nmD`huRTiCQnz{8+K!t2%}b3_ zJX7h{j75#|(DD9I&EKk=ERe=YE z7UAcMMw?NH;E7hY>o0RZlh47HkQS8m%t=58)K*c*=Lv8~d9hLls(Tv*uL>b-@dvhX zvm4l3wABMlL?C%{=A`g9eN1f`(pIlOI2+9Ol+TdxqUj(s4MNf+hP$%UAc%pg*QtQ6 z&*)+%K!(3Hi11{B14I}r=+OcrQz#{5EGGs0qVMD<8(L?8)&lE)0 za9ZlTSn2{&++wdj37^-EceF~KWiF!H*_L_rSnk<9_|*HXd~{ow z-C>Mb97j?@1-N%k%nui+V_*dtEaxp$@+_+Y^$c(bG}ggjoaWO(gREon)LQ+ayI`L^ z`cjj%hoAh0U{DuN7+BMWetL0mzs-05s_S0wD-Mn2+MWGumX~QrW?aVa-fZ$nq;NIb z^JK#jtEl8jPgKWc@^1F=jr@*Y*Z6%qsOY!BTw{T0n|AV-b(fdFJHBnv$WZxmPlc~p z#FBJ&v~>qu2g^s3{5>;F2+%#EKOu_a69I)V&~zZWTm-j>r24}DUA_)^_Waim%s|mx zD(du}q#Ebh%k`9lgLV%i#386G#BV~>;En)1tZQ%EG0GX$G>nO{J*gS)y z>Tp^^Vr)#&;WnwxAbrmZAG_ZO@L$=!q!%Zf*8T%h58Z zR-wgvAHj7>bjN7DFLlZ8!jdE$U7rcTP35opPlBTrv~2w6v&{rs1I>_Bf1{SV41o1C z0KggIEf}isV(pRlV8BT{z&kQjJpec^bu@%o=8uILC<#1#7(@dgv#sEUk1g+kvVpE+ z5M&c53r*O+Q6WwHK9Hrzhl`8YMw;8kDjHRJBm@iAodUXrzlF%Ql&f3(oz5W``i>R% z>}w#Xv@32ffsjef#eY?s+nhjoI;f!c2DorFV>D=eA5(}2V|n=J^q{VW4N+lsjZO@l4u`E0pym41!aX_>x#2V?oHJeJ(1>h;C+)zVjn_-Q5#L7-r z(^ZTd0`AeT#b54!U-^M^&6&{L8s!|SHq`un;VSH3URSEm>}uhaBY zT-(DSM{lpc&$>()KB?Z?Mi?9_pCn)^+X{`^3fIik%?NQB+S#knG`KzSmj~nTfeN@~ ztQbGIkI*$Ii5WtpT*iuk^VO#{1;q3}U&Iubk zc_d76M66Xl3Asp3>bqpNS%R$tr4?-^Z=l+8)Q^-0*A~<9ki&{N%H}@J1c%@Jgc1W) zhSeZIE|%AFv21uD=yP>)3Jzcc8tHg8a|8v<1z~fOhR-d*0(1EynK`gWbQrQlLIL#K z;v~S4a#f1QDr}3#xHx%n62Y5|K;?%TRf_tc-7piVg5-t~6lil!nH4-z0A$cmqMK0i zNo;Hg6@`qr0twFQB!9nSe6D`r!*yqvC1ZlH!LR26-P;AK7gmmyUs$~>g1x!#?^P6t zdS1J~VppU6Xc6a$fgr|rq|tMNk}NNvThvw{yQe5lB)pw%fqY>%&}3m~5B0_4wpC;) zWUEEk?9tl-#BI=>{tEtP7fQDiNR{*kG$ds`qC>#6rv~@1NYD;M1%`wtkl~vkv(#N` zMG1v5Ltx6Za{W)d0!Dp1N#04K^$V>UV;he5T162XpVNu`0{=o>83X0l`>t66i{Wm z4{U@qAZk-f9db&bjI%BN(AU~7v-q0lXeed;6)9f=rSb!9$Y*Y~3rc*}nQ+#Z@#ynY zzKi`n@|8mFR7IDiyCHq4f3J`&)yo(`HJ}k!BQA~eJ?tt*>`~0td8F^fnk~sC7kC(^ zJF}r@Jcgrn-(cU)Wb)eWnPRRB%}bsjg}gHxFvE3>(1{SG_cUCZ#TYl4g!!7yc9vK9Mv%FGMSljQZ7R)P8gC`H++YJ^AGkX}C!T8To%>5?RTfLXKK z^=SZFn_9=w#FzfI-;~G!{&o;c>6FmA4EP!u=utcP>=PLJrEJUL&XR+GMC7M?UA=7W zgS#`dF0e1Wbwz5F&pDGUbA#BJRmO^crWK#E1907MtQEZxAHi4^Xa#+6}wfR zZ|UYb;5t&Nvy>V^_{x?8kzusAFQk0v=41P|h}!s9PxzwT06Y|IbN`xe+yguzcDMm;0rkHlM-T}UQ> zbG-rvmR~a{`MY=Qw#O%f_g~icq{~$c+}xG6t?&ADd3tAT+tjO$BL|x1PG9OK)jYwx zTi@)w?U>(@%l5bVb&DRgJOdH;@GC0(imb{6OB#S(RZdg_4jL<0qy_^&gMbN4!+bD$ zWGJpF+(g4*NQtfd{RZ=0s50;oi}@VjMJ3@~jSQ1P`Zp8+q~Wym`lGCci$276{zdrm z=dNE}O_df0`j5}gT~Krk5w>WtFK01V48#t*c8tZ3$3J_z{6fw8=uO>0NaZEQddrsb z5BBF;W5-V4NLHHDGb#>pwaqa-CH!S9-|<>%@D*&GWH5-qP|E_`C`rHmFmhIHillC# zKh9sfeXIOPIOStsUuM|WP(*Fd@>dg74b|+-sKFg@&&tygO1AFe>oFyv>y~}7g3qfY zSM@NU09G-_PY0x3k;6cxWBG_JPs8{9aE?Z^*3pM%q?zQv8%7DCFoaz4)FswbyfBFbe&^jignjs%E~D|D2Xh1ha&0R$0PeKUq~Nelc{fYHhmF!!^h1%cTvGN7Uce~CUK z84$(DR&oj0?IJHa!HbRsAb`mLzcoV+VDSPUA<)$W!D4E#3@IZ)w$;>IA^~K~XA$6C zF|pM31lQ_WQF(%FjU>)vPN0I)3D_#{Hy8+RLEdVzvZhH;goYxIC{X)^YVa$-3wban znS@`!SDO=D_Q1INjGM~OZt&S42%8G6F*o!Lcqgt~@z9tRK~h~t zz@`JQ^a=o}3Ox>FpG;ie!-ty365Q>Oy!$D&4b69IAo!tc=dGP|Y z6ZBS|4!#C4dxJmkgKiLHVqecYur*cjS5x?vE>BuA6-E&bNcl z9eQdUWBsUNP4>-0!yQLgImsPPzuvsPVR`k-b(Ce;r9aPHVTyW9;;y6#g_e_pIggw#Uz+-(E|EM7i`hG;fi&61 zJ4^7cqj_!Ej-jeQxE*my-u}c$h2B}xTsuj~+Az!futz}|7eVF|0hz)#L{GY-DnFZl zOymxL+j0)ERl>sH9noa4lYpR(i1gGD_laPAYq`$b;k?uv@XkB4L4_BkkiYZs6M5De zyC*yFIwjnrYJ;pIHy=^iX^;8yv^eY7{Mrnw^0E(58%>I4(jW{ zpdqXy?c!XL76{an*=7o1wMga&uzrKrVL4k^%R9>V1NXf7Sj`=jJzUO**l-h_3Q^@{ zjX-OK;ou3wxiZKcPX5itDZsK9C&TJ6iC^1c`!6+aCYr#TRg&?y)z>-@o2hMJx zH{my*7{jbfc#j&^#!8`$i@2+Qas13~k~>5Ut8@;_?&5~wE5 zFWxT+1c%C{D@S4q> zz?h-RUFqX$!HU1K@VPU4=W*9c-&Hu>dLf+5fR-T90s{k zx!FYf#^~JyfpWHhV}cq}K}_Vqn(7y-3_g&;3IUW(iQG#7gdSn;kki9GWI*;>&>A6Y zFm=u1XB!#FFp_m?TJeb=b6?XoF&qB=DPhuReB;TaT?I2nFRjnLb*FIXJ3RmR-{NKS zDy~f3u%^sq$?(cAzWOftA}8^@>7^qh8BS=M*1c0a zV^@p*;>Bs{JR|HyEcwFyyVJ#Zh2Qv>uy|px=U;Sv5hJ6e;nAlz+_$4L%~I*2uYyHc z7XALpZ_idq+i!%HZ;LuEv-#zxAAc*kv3#gZc;`*)r1n1=o(%m%9?bSy|3lm%&doC& zFFxsB4`!LZo#*IZ#Fu!{--El~bNu1J&Z$@S4-aWX->u%arl}K1GW(JKgUZ*38H=_` zTd$t>eEc?i+m*kH7N4Bc&1489FS;~ z>CuC=qbUdXdrN$5)vI=>*A7H>%v|h$Mpl=SuU$IpSAtTYiS&LYc)BUQZ1p4U&CRt! z>f_0`@p=8VX3(!jRzIi9eP<0MR!s^Oa>JDtS2CfQH#OW{h$s5Vr~Rm6>*+Teow+pM z>|%jY!(WIclHP5Eu z_M+vv(lebc3$`T?aJCfWXAdlH_Z<~?@G_&*Mem!*RS`lL!mv3T^sy*PDb67aD|wlq zMgVKnpbCNbQ@LC23VkNHLGNifNu_{iuo>MCz~;rGkd_@U6?mVV!GT?U zH-Sb);4Rb|Pyn_alv%~rs0`ow3Uij+?Q-{=9lEt>?ax?^*+*c#8u_WryU5^;?0o{B) zJN(Y1&OiRXk#ypZTV`mf~0e^mpwb4aqy0^W?Z+ocmA~9hY}vNF5t!myM?W_IQ+SHPBSA?=c~P9>5|t4 zF4Xt1zlzF(JdHm!C^gK6Gng!hl` z+rjH~e)~H%hOB>k$tHT2r99(JbIhVsJP7&QJG6PzC%ov*6HmO~5LNx$wDscrZ#g^r z4o=#f@yNoR&)Uqq=;AB-;}v^AG|;>F&52W+E=>FM$iCYJ$N%}{?B{ozCtR=mHU1{| zShL>d&uTG0Vp?(chgo(V-@B@JUrTk#-#%lW|A8z{bxrw~sj}Nx@7<%_CspSo7JRY& z_@~N7rZd*PMt!ORFGwzLU)W|`ex}$4=Sv&6=;|a?5AoJhA+3d7Yl_mJdu%k%_^QQ9 zXeA4gWC$9P;$}J^2^7D&06zs8%Jhgz{rR63pRHdEx(%h5w?wWW(f8lWw!i1beBSjU zTon1w3eO|Ue@f+NPTR^2xufu_2wgVv?jSR|Veo?7!?eoeu3a>JPBU?0wTkZGyJpmp zk-sizf-#IDm3u!cV^T$-QkNj@$KJ{jV?<)>lM=w(P@$zfhTOa~#&hvA2(!qEV|=+TPfe{^K-GkW8TW}7YhN%qI6Lz>F}cxCsn4B7JZB>s@;+6YQ% zKr*(fEjet$F(VG_cW~nw2+=z&ff~rTp5g=<1Y9tU)X7y~J;0?nm*#3>5)8D%SEI$+ zai?IW0VVeAE@)^{n-QExnNX;WWVA@gQPY}W3cS=Pq=x$pB$+{~n)0n`Yb&eo zEzueR5PTarF2&iY;2Ut4%>{EPKPysWldr23xfwy9wVzy_aTM#pZiI^W&aKlUbsD)s zXiM|tx;IvGFp9T(Bd8cytqs_FAs5Bl5q%NbG~+CxZ&5+xBrT?s`}-*-1>%5EMN6(W zT`&)+?a#5XxB4s6A~=!7q{$YAVVTDg*-i%c#>_)2RKU-R?`e%Ad`hKKG5dhnl`qG!9|I34F9) zH`gQo#HRXJQ&dNme~~BV^*)ri7qQI8YSZd)ojhpCZT#{2aI`T~cknNh&Hm>pPKDF2 zZ<*vYed*+C=OyN+jGV6J=g!8wYZ`iiGkq#an_U|wZh5wSa7q2r3phB&uxz{U=(878GHg zrhM3a^bc^q_!4`enV}?78!~f8`TZwUi_7NCzPdO*QSF@6H)Vfy+;6Y1%($Nu@r(QL zn+-3d&!jJA-#C<#^Xd8Z=`Phm&gvh7z5){?moaAGq}g>=6^FQBRprA}wd-X!XPCb#R|qWLrC z-u(~ee9ba?9xw0W+tU%hH-MX|e0B(aztupv93_Un?JwGoUsJR()$|DClEc|c-P5-v zv*k>v?1ZT1L5TZ&bu>f%O+_1ub11X;zkuu(2(sW`smX+l^NUvX< zy<67kJ{aHkeVz)L6Bw;4ztEc=9J?#XnmJeY@lk&t)7+caUtmSfTk}u$aPDa%!Zwd% zsIP`F7w?E!xPoSg{QVh%u%x+bYDK91S$e4p4GRV9pW{JQwqlmRFtgM`q30xB>8V~v zG#SyXUA=2Je!N$DBgYu}zWH*6%S*vba>K>A!4%L1sWX*bK;enZaFf%(2-XjdCc%6R zl;3owc8G^dDA8u?;@MJB7kf7eKXsxOv(=d@fFTji4ffSh_^iZ4BT9mge5Jld<0}Or zgX=WupM9z*NMb_CaLsRpx6E-QF{un^y30osGOS5&@CMe)ovofL2or<%5`*vWv^uR) z?3U~&_v<9wM47vp)Y<84)Bn&-QhywwrPgw(axXieLPSpq$GCW~qPeT-eG2baDcx&Y zP#B;@hLC88^>eqKnv)9s4l=fCC#bTqdO7T~)&IuyD{`C=-o(`LW7oO&g*(H16qa3moU%BP^^fmzk7|n%h{8AkV*z4Qu7*B{&wQ zf8KYz)U>3+tq*udHJRzjB}fGZK{u>K_^}3nFY~4*mbH)67!og7gBbAaRB$Gua?nD3 zsy`~B#JN4N?D=1`g~dh5A@;+f=Jo|#@nDj1?(SdL7ya2UN7>@#p;$Z&U`~HxY{&7q4Y5WRX%+>lVgY<@jlX# zle*+!wIaFV&8H_W3vISJopfmQ{b6l-!Na{?-B+{0Gbw(HM>Moiy|NQzd~vCenpFmg zt6)9xHV`1=n#-i9A6xcitZtmcr{jZM@1}MTOmdm^MYh&EGdV3O$w2$46eu5rr}8WPOlA*vJfSiQfJTuk`ZFumwi{L@~eWc&iWVHy* zcW(f_fNg&f8QFk6#sz3TRx`{oPU}YMJZ)zo8cKmB49TL*+I%&{yTYe02tW}+DJT?e zm})Z=0_BS2KB;iWuB%Yi1M?cVD_JVEj@Z$ZZ6JWHDXAa}uu>+~50W3BiM)h11^v|? z4KzYX+lO7!6_a@xbl=~#w+w-xSBaXxHhkS$r+dwd`{V-j>8;L_nqG&#)0&?jceg)V zE2JIh^nPhmXX|zNUE_84&DKDEhdabu&Lj_;i9@_*BW;FLiW83o+Qr{CBqe`Wb^BFh zy|UBKl{2mW{3%H}f$b7JIhe6veeA`y($ibguGlp=*I!%{{^Ym=NDZA5^{4mUn6h@` zi-xk}$98_csWn?LtD;LPhz|eAGMaMD3-ByWFqk`uhuwQyffJTek6tddAfi_!-SZfv z-v{nf4?j8|{y`QzgvNx6p40er<;>SflfzFm z%fe)T)!q|@h26Z-lYCZjLDzr4@lbO2&Tn|v4lk+LYez~4`42o(y6vSzW~o>SX+c=O zW^rE`B=LPzK}qRN%G5}%oh2v^(7o>-;kh#e&rH-qlN;9OK6S_TUK@)%*}DyvK|A{< zKaIJog?eOBu~fMW@NeO5Dr3H6V$<}FE4$m8P8C37=?Thy1JxC7Goa6;OyGaP zQvh=$t=X5zAy>^Qbe4D+f(T0Z99dk?ZYijo&CYr-kRNVp1F(x^4zkrW4~>CI04qUd zf|nM?*SHjoI!+4qBpYmx*&=a&Z7v3jDBC3nQ(BBn0_hZ(9z@F$Y>)Vy0Q8_*DQW~; zGL?}mGWS{S$k0OCOM&E17Q0)uvl5Cuw2w!5wlQW3Hv-Jb@yteNW&0{juZI`Zdgt=_ zJD`2lsS2sqE17OK#fg-%qZiJDne z%mc-!F=iniE#~#hL%7V<#VK}MGEO#k*k!2j`7TEl>{}L_$M1AEwbN-@pF0_q?whV} zlTeP1q%H+RT^*B#QC6+mp%pDY|HV-3RDYGy>ud{c-G``w=8&g#lY2+LUiGN&yCc8p z&Z~NgHWi-nI7Yo`m*u_Tx#p?1aQ(INi64rWMtwefVM5W~JM0hNM@dq}4~utNuJ63N zZe!7~h?ab4(|hp*OHQI{RrHRjO+ZH>&1k&zQon*|&n{dYNtDV0zmTHy7bs^P{y5zC z>(H@;*YjTgx=#M|)%CW#FWo+0{^6kRL)D|?Pg5&G3(^Kxy2Uiln!F8i@B>A0GnCBs zQ+uBEAD9}X+Ty3l{M``!=4{*iXaQt;QIgg-dl!m#j;6x-HFI=yd*dYZY`dm6m3ZNv zT&B0*zM=J*vvdk|qo;4}sfvWan#8OaSSFw7<&ZardSpgvhw8pCD^ArM(35gk9g_Uh zl&bIvsdbZ2%w9h1xln1JElVEd*+w_}U|!-Td1fsgiYXok+ZmwJEGta_o)ef&@a!Z| zIJQEeTrqKdpjI1;tjDA@)OO)PeS4N|V6S3RAd|X4U(=Lp-lVqQuQ4QDP;5d8WODjm zDZ^xwuW3B2>4oFgkE}S0T?2cE0(&i%_sKo$HEBi${zIgdl^)^QVUL;afZ#}Dp#mKY zzSESA(BeQSuOSc=7zl4{o>?cI+6=Eqw7HlISZ_4B4oF(7P+Ok{WI8D8MY#>YdHvqd z5}4meF`#**O9n2t3Rx3X(c+z!CP(NT6CcSb!<;^)88EGAGnm=H-=?8f13iuac*=n= zPVk;URvnoOgUMu|03SZH}@IQ2wJbNBYe4z#i8Ve;U+Re%?_D4;Po(BanCJL8YjbQ;HkQ(fvM$;Y+ zw7-tTuHFk&3jMYpt&^?%K0F;Hn}^r7Ljd-{!zq39gFkv5#LTX{>shRy6jRqtJ8*r{ zvs^W;anj6rhR)`w&t@O|bw{~uk7@3;i{Uc!^Aktb9waY^uDX6Nt;7N?kDa4UJ)iTj!IQmq{mI+HV!f_dg0vT*R+bwN59aI)5Hx zt((}F)KENFHs&;|ZgF3mOpv#{^-ARdb~dnV>sxYict7Pgy_qrN{N_R9l7XHRuSBNR zu>$7b-i^hS;Kymnb$R zbLV3ew+xoZ4NM=hGCVye6}r=_0M%!tbk`S^(IFy;O|13Cvth&|c4%W% zE18`Hq9pMs06B;t4TOoTL!b5EAzon`Tn1_ z4gin3Rg`68fXfN!QGaq3*wA50Wd>HmXf%=d&qn|Mewdn=1h?!at5-rK!5-9FPEt|{ zM^hx$0BGGE!qQVVmZy&0L2M`Tw2=RF2*RBxAp8wWx-4f%BZz;xug#T~LX@@wl!@|> zjy|d)Z35un;Ub?wegToExdSoRyH(Oct9lqa#st--w*+8?m3m|X%?iE3iKx*MwScH> z5E}^#HSpxkCRGDWYnzqFnqK3r8fspXZNj5ECp4!pjq9OaB?y4q9}`)m*MI}tY>38I z329H2rRi6TM^*omrXN;M2>B&uG&SPw$ge#omfkW%C~BP{8ZA9bENb*g=f|`7jhoP? z7mw6u>+hM4yL>BB{l3pD`H#yND4*qx6ng%7%lYojGYwZ)`Y1T}R@a_;yKIm0%kz@h z2Y;ph6x0^-nj}3YZ&-WSd2vVFPR|{m_evJxe~ROsW?Z@nWqd45wTtmPx|d+FoDbnSt_QbsKzc==Ep{y~Y;W zfNYHiy-HDQq5OiaG0X8nIHq^PVdn#9qK;G}Dfl!{O#}8MsRO155Z-2zkT>M4chNIl+ z>&9r@c3qkNta%Rtw`i~T0EBIs!AnL_fe-@VBB}yYqCTlp zKuA6?=UJuDGKT%?NRHJCBQ-Iqtsucs zYhp+jaG3nxl80+2$*ceyo1B)I5=2ThlthC&%v4r;7feJpYsL0WJe?Iu5cp|2iEDrz zX6+@K&8jvgHMxt2w+llJUP5UkJRv4=6ju36fWIRbS(9EEjdX2%pE^((PMf_o5_|5` z&HQjG^@>GnbI3J)#H>BIKazeT;f|elQ8Z{erj7r0@y2zr>0k8Ci_ah2hd8d3Uk~TC zeK-EO<%x@HqI$#c_lnA^rfw|0o0{=g)T93Bqs!j-uIjp}J`-J>UR0mip;~61Je?og zeFLO`EB6)N>Ish+dlFoEd2HG5!>o zEv*;0(ALKb2j;3>ZS$9?Oh8xe(9vKT?K!4O(F0~gCvot}l3PzvTD8zMX+WZy3l$e6 zV`@PTlLnq#Q0L@X;a$6C91J9&wTerS37KjewdGtQ+u^3xj?9P_(&8Po#KUjRCMqLe zN9OUN$L&*4k;RePumXyQF>d>XS3RhVpQ3?9Ick#sy#OFep_HUf$$ z*#z@&G$8_q)BpFqZPge(jDTk;jB{kMQs99ZaY>2E)>p6uf_kb{QuJ7Qtp*2@CabuZ zP3&u?I;L5gk>^w}2O9lFWfJQX;COYhor#~sP0<$jC+!*{4b*}3T0Bg};>DVl`c=^Q zFsW-jv@zz)nh7MHlL%6!`eHK(fNP@6X41;tTc>{k_>qtF!Mi18(<>ZfEX#{TZe%ie z35h-38b&FJ_BP4WpG)^-$~~jH4^J=5Cmh9GFmOXuPy?Z4pcQFJsyMbSJWr%d10~Ex zFGf{l?nv6ix@hxu%>KdyrynhCGcP>plwe5`-zPxa(HD9ENFVW@sMGvQ&4a@e|XNBCGHuqTggacz6%+?tj_=*qNGaP3?>qy$GpFqK50m}-4WtoO%iUtsNN{6b(bz<%Qm z_6!oSK>Ybrg~X zyDAVNBKhn9z&l~0YtJI%RB1QC{uVE3aMz)1fsd$CprQ8dmSBz6NSCs)X-XQKzO$>_ z#PzmCA*^@_to`bfPXzsYP~leSsOqaAs|NHfph=8^BbRv__H#TC(j-P4Q{KR!tLJg4 zB^nP6s~bXM^vKLK;s61GR3{tihZ`J#tY719ux$!6;$A}P8X}DkYS#e?AqSv?VXwi^ z_IjX*?HFkwiG+$%i$>T*i3VSfh6uo2we7${H1VK-*hTvJIUut*Gh z9x=#^P`{Ph=W>Gz?c1TxK_owel}v-;DuLF*lc}b83H5DFQJ#pYB>+oi0^aLhXySBY zPvf`ZsJM~1t*b52+=khTI4Xm@MG(pEZ&KB8QEURyHee{wwa{kJt34*UhR=M<1hW~L z0SLzS$9S%}i0wFRuKrEy|48+P-UUw61SdgyH3 z7t=Qw>+@B11q@E)Tg{7eFTad>7d^zEniSM}r2Sxg-r~T(vZa0VzW?1j`Za6OaN}T^ zdTIH&%wHCXGgkWE4d@9!@Mgt1oA=*cZ^>f4Z0H}?Twk5jY6;j?%TR2q|NCy_+S%DW z`-kUq1b~)8N;bMQfL?w?+t(~bTWgJ z#AetL6eE1-F#WY7Bi#Nlq@i4E#}Fj@|}FuA_|vYVPaAQ7<3^N^$zHsp?gUxH49nu(^MsR;1x_DwJkgv?}@n$4AoY_~RoQ9&jPZ8pvfniPrktT5S35SQKI&>ul zw6gLdw{mgZl!(CN*pU?zg}2@=xT-i-SYtvW37^sH zv@$Dvg-kfRUkRCQsMX*;%%@9pdMj=WCbZy>MVWF28-g>Ugg%(wSqbsQ`C1JKqA|t_ zp}qDdNb4954Y0fTKit61Sy`jGWsIt!j&Nd3`SlvxAwgprxe5%|K;0mIo-8?yJ)ze+ z)*zND4uHBwK;HVxltAE(LU$>TfT0M4=)giyG}slm@dKDd<}tPd#78_ZkL!?>8L7Di z?(RTI3`x;Y!)eVpB;i3=;D{hy9Cv8|p+Imm(Z;+jas4Tc1791Lq;hh%{%%fG`Q$1Rw}{?85dd*SS;bL zM!`0jWlDhMnSRj4)4DjC`&B1$RI%H-+aYlp)`zuftdBX6N$7K%D6P86lm;*{07q7v zS@RGq6@Z}Rg5k;jDCBJ(tEykCrp!!{h1;Yb(NK)Ojq8aT0+pzwfVK!6SL-Y{bt-p9 z;(F@pG?h*GDL@e|Fp$m4to;|w?f&FyQ|6z?F-q?3m_WhuLeXf zM;0#%IzSb2@`X%@VK%JHg)G)8`4nm?1CEdO`f4%)Ch$O95Ja?Z1uIn>4#(Vzw0K}0 zYn=Rv$MB;frduxR0(_yQD2bP^wgnvm14-2?;MO3tYDHg#BM3wzayY4Sb(MG@cAe#sa6c#8zV)^?mJW^pIRqPC9;9Vq%tyC13zbSJJghd2WT|XC#u^j} z{P|gO21Pr}HXt7@jpWa@jy`-VziSoYZs{q*QiNojb`g_IOa%!QVJam<*x=Oaa-cyLXL-1~Y zb**Vi50KZCRJy`0rmIX6X+%?-z#?93@R{O{ISZK2tg6E&R^mL<6KVaxfnBUx0ZT`c zL3+GGL~o=i*n6y~e;<4o1N>5$fgy0GroYD7n~8Y>lsTH>4U||0TG=hPw$Odm%|RKk z^<_n7X;vw;dT@ZiQd+u#$b#Iy`rXmyGOG}IFoD1Lu_*xxGAWUS;6wv5!wY?$B?zih zx=Nj;au8R?!zTtZjorl{hOSYoC=lE82vFfdv%(~zM2VjI9c&vAAR6R`u@b;OUnQ#X8&JKH`k`AWOi^vGrCRYp2UTTHqidew1ms;69ra!E2Ozu-|gY=L5hRVeE>454g z+NfpY>_si}l-o7oJ_}3J;+;YnkRqC|rjHLKT`1RjSd-XF$G|r>rnXHZVa1awgAO&9 z!-Y$e3G>T@fzKg2bpAlbgY}8VM2wCmRzj+Ze(OlmpHO z$hsPQ9Xu;)66*x18Yj3^&jKwHUMn&HdV%kRvgQCx@(O1ekUIbpN+jw(1butFRGnGs znZgC%-5Qd^p#xoJX#$rPE_Ff?X)2Cc9;u`DS1H;c&R`0mHs@;L!_li9&DKv^!;nJr zSwY90Z38VJ@T3V03oFc!vuNMw$x63bE0jJH3w$Vi95n(y5o$4pLQG4jNZ{%%OMUA?Gf8IP81kXfNvw{ zBPg&Izz|5s6F?w_48AK&TW!5hA}`UnwnO4qVJAbk;6Ut?PcOPq_UjK7p`8oxC)Hd0 z>$)A+_gJ;sx$!s-GfBPDEs={NE=XCbi25Qro&(q&4ImrvSE3F9?gbX`emTM=-f(l; z`ocgWL=g=h_0?+Vz0=4rs9bxAAs9TtcvuQd2F(7F2tv6-Z95bXX(|0M-jGPEp~AGw zVmr5M`V@iHkweKgeO0Ahx2?6PwYp_3+(lku9_%yH(o`#zkH+M=`3)_zwrgOZ_>Jjq zv?jiyf9!R&Vz|u9;BY6>%vD8lKrAN$OOnHr)Yt4Gl&|904&6d$TN+@Qlpe@E+1>zU zuEpFE(`nyuK}eS_{|mDIbfGY@K`DW+9K?#Pt;dcRqW8QG@|W|ZoF=^8B9M+YK4n9)r#2( zX#>$_8Xw;;9Zc#i?kVd=%4teUQkewOUQR$e8wx@i-$A})n1>klz><6k+nuHMkOupq zN#2_#062kd4~{?xG7MA@a3K8GB3gkBJ1v~i|01WEW9sc{;KMVbJnb?7_+WwYSQM71 z2WYrEPYq`hoYr+rLX*vQ#IkA=upf{(nRo>WVqx)8jTyX6Dtl`Pu=}-!a6H@tqfF=y zYD*q>7-qX!F|JMMlk?(Zck4ShE<5%6f&HCrg0Ro8lWe~+$jSVQ126>}bK}w`t2(MG zNC%2p(H5n^I;?$8CJEu&dN-aF>33-Cptp^HanxJdjXZ_<8X{MXOkg5xhf@$*87aX1 z)|whfdL*CN<~;z%2-cttm?K*SyL7=MR9~^{m9+`N#v9svVaSM&fJ;}->8Z`v#E8}} zkRY{Cf-TNhDFo2x`riOaV8f${{tY9AwJuGB~1QjAUU?Ey(>whPHle*Rv zp+@6sb0r6J+(c^z(U%D7D@Kn^#G|cJC>F2KvX8&M zkPBWAyrV`k1A4l8C0Myc#R39mA+TTK2{?4^;2khcY%taS7Tius4&kA6NSfj zTT81MN^6kqxYgFrcdj!&eVA-Pf9^#q4{TkyyXL`nE;8k=-qH_Gc{ZBWDQXjq4Xk3Y z_eu93?6t%2m*M@eLY$hh!?%QKh}aI$E(B*m{5v#estj==8+lmhiLRYjP_$Z~e z-UBG@p4Q2{2Wk~u@laqeNFpCmX2}2>U>1JliKZXHk;{ndfekBc4Lo Date: Sun, 1 Nov 2026 15:19:27 -0500 Subject: [PATCH 02/15] Refine AIOps page, member profiles, and publications --- assets/css/main.css | 7 +++ content/people/rev-liu.md | 2 +- content/people/yuanchen-gao.md | 15 ++++++ ...for-message-level-log-anomaly-detection.md | 7 ++- ...c-workflows-elevate-root-cause-analysis.md | 10 ---- ...-and-incident-response-in-microservices.md | 12 ++++- ...-for-autonomous-site-reliability-agents.md | 15 ++++++ ...ts-with-high-fidelity-failure-scenarios.md | 8 ++- ...ocalizing-runtime-faults-from-telemetry.md | 13 +++-- ...g-failures-in-distributed-gpu-workloads.md | 16 ++++++ .../distributed-machine-learning/_index.md | 49 +++++++----------- content/research/rca-aiops/_index.md | 49 +++++++++++++----- docs/aiops-sources.md | 43 +++++++++++++++ layouts/research/area.html | 30 +++++++++-- static/images/research/aiops-technical.svg | 30 +++++++++++ static/images/research/aiops-vision.svg | 30 +++++++++++ static/images/support/bell.svg | 3 ++ static/images/support/nserc.png | Bin 0 -> 4176 bytes 18 files changed, 270 insertions(+), 69 deletions(-) create mode 100644 content/people/yuanchen-gao.md delete mode 100644 content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md create mode 100644 content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md create mode 100644 content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md create mode 100644 docs/aiops-sources.md create mode 100644 static/images/research/aiops-technical.svg create mode 100644 static/images/research/aiops-vision.svg create mode 100644 static/images/support/bell.svg create mode 100644 static/images/support/nserc.png diff --git a/assets/css/main.css b/assets/css/main.css index 91dd9e4..fb97e03 100644 --- a/assets/css/main.css +++ b/assets/css/main.css @@ -1222,6 +1222,13 @@ button:disabled { cursor: default; opacity: 0.6; } font-size: 1.9rem; } +.research-support-logos { display: flex; flex-wrap: wrap; align-items: center; gap: 2rem; margin-block: 1rem; } +.research-support-logos img { display: block; object-fit: contain; } +.research-diagrams { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 20rem), 1fr)); gap: 2rem; margin-block: 2rem; } +.research-diagrams figure { margin: 0 auto; max-width: 25rem; } +.research-diagrams img { display: block; width: 100%; height: auto; } +.research-diagrams figcaption { color: var(--muted); font-size: 0.9rem; margin-top: 0.75rem; } + /* Long display equations scroll within their panel, including on phones. */ .math-block { max-width: 100%; diff --git a/content/people/rev-liu.md b/content/people/rev-liu.md index 6e41bf0..4a328b2 100644 --- a/content/people/rev-liu.md +++ b/content/people/rev-liu.md @@ -7,7 +7,7 @@ start_date = "2026-09" location = "Toronto, Canada" research = ["distributed-machine-learning", "rca-aiops"] email = "rev.liu@mail.utoronto.ca" -interests = ["VLM", "RL", "Artificial intelligence in IT operations"] +interests = ["VLM", "RL", "AIOps"] author_names = [] portrait = "/images/people/ruiwu-liu.jpg" linkedin = "https://www.linkedin.com/in/ruiwu-liu-b1534827b/" diff --git a/content/people/yuanchen-gao.md b/content/people/yuanchen-gao.md new file mode 100644 index 0000000..eb2efca --- /dev/null +++ b/content/people/yuanchen-gao.md @@ -0,0 +1,15 @@ ++++ +name = "Yuanchen Gao" +slug = "yuanchen-gao" +role = "PhD Student" +status = "current" +location = "HKUST, Hong Kong" +research = ["rca-aiops"] +interests = ["Root Cause Analysis", "Automated Program Repair", "AIOps"] +author_names = ["Yuanchen Gao"] +email = "ygaocv@connect.ust.hk" +# PhD role and current RCA participation confirmed by the maintainer, 2026-11-01. +# HKUST group: https://cse.hkust.edu.hk/~charlesz/ +# Affiliation and public email: https://arxiv.org/html/2608.17018v1 +# MSRG visit dates, portrait, and personal academic accounts are unverified. ++++ diff --git a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md index 1bb466b..27d7fb6 100644 --- a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md +++ b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md @@ -2,9 +2,12 @@ title = "FAME: Failure-Aware Mixture-of-Experts for Message-Level Log Anomaly Detection" year = 2026 authors = ["Huanchi Wang", "Zihang Huang", "Yifang Tian", "Kristina Dzeparoska", "Hans-Arno Jacobsen", "Alberto Leon-Garcia"] -venue = "arXiv" -publication_type = "ArXiv Preprint" +venue = "37th IEEE International Symposium on Software Reliability Engineering (ISSRE)" +publication_type = "Conference Paper" research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2605.22779" abstract = "Production systems generate millions of log lines daily, yet most anomaly detectors operate at the session or window-level, flagging groups of lines rather than identifying the specific message responsible. This coarse granularity forces operators to inspect many routine lines per alert. Message-level detection offers finer granularity, but remains challenging. A single event template may correspond to both normal and anomalous messages, failures arise from heterogeneous subsystems, and line-level labeling at scale is impractical. Although large language models (LLMs) can reason over log semantics, applying them to every line is too costly for continuous monitoring. We present FAME (Failure-Aware Mixture-of-Experts), a label-efficient message-level mixture-of-experts framework that uses an LLM only once offline. We annotate at most K labeled lines per template to derive binary normal/anomaly indicators and representative examples. The LLM proposes a partition of templates into failure domains, and a certification step validates the proposal before training. FAME trains a lightweight router and domain experts that run on-premise and output anomaly predictions and failure-domain labels. On BGL, FAME achieves F1 = 98.16 at K = 100 reducing annotation effort by 76x and detects 97.7% of anomalies from unseen EventIDs. On Thunderbird, FAME reaches F1 = 99.95 with perfect recall." +++ + +Presented at [ISSRE 2026](https://cyprusconferences.org/issre2026/program/). +The full-text link points to the arXiv version. diff --git a/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md b/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md deleted file mode 100644 index 301fa0a..0000000 --- a/content/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis.md +++ /dev/null @@ -1,10 +0,0 @@ -+++ -title = "GALA: Can Graph-Augmented Large Language Model Agentic Workflows Elevate Root Cause Analysis?" -year = 2025 -authors = ["Yifang Tian", "Yaming Liu", "Zichun Chong", "Zihang Huang", "Hans-Arno Jacobsen"] -venue = "arXiv" -publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning", "data-management", "rca-aiops"] -external_url = "https://arxiv.org/abs/2508.12472" -abstract = "Root cause analysis (RCA) in microservice systems is challenging, requiring on-call engineers to rapidly diagnose failures across heterogeneous telemetry such as metrics, logs, and traces. Traditional RCA methods often focus on single modalities or merely rank suspect services, falling short of providing actionable diagnostic insights with remediation guidance. This paper introduces GALA, a novel multi-modal framework that combines statistical causal inference with LLM-driven iterative reasoning for enhanced RCA. Evaluated on an open-source benchmark, GALA achieves substantial improvements over state-of-the-art methods of up to 42.22% accuracy. Our novel human-guided LLM evaluation score shows GALA generates significantly more causally sound and actionable diagnostic outputs than existing methods. Through comprehensive experiments and a case study, we show that GALA bridges the gap between automated failure diagnosis and practical incident resolution by providing both accurate root cause identification and human-interpretable remediation guidance." -+++ diff --git a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md index 45c5f22..75d7a02 100644 --- a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md +++ b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md @@ -1,10 +1,18 @@ +++ title = "GALA: Graph-Augmented LLM Agents for Root Cause Analysis and Incident Response in Microservices" +aliases = ["/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis/"] year = 2026 authors = ["Yifang Tian", "Yaming Liu", "Zichun Chong", "Zihang Huang", "Yiran Li", "Hans-Arno Jacobsen"] -venue = "arXiv" -publication_type = "ArXiv Preprint" +venue = "41st IEEE/ACM International Conference on Automated Software Engineering (ASE)" +publication_type = "Conference Paper" research = ["distributed-machine-learning", "data-management", "rca-aiops"] external_url = "https://arxiv.org/abs/2608.08968" abstract = "Microservice root cause analysis (RCA) requires correlating failures across heterogeneous telemetry within complex service dependency graphs. Existing methods often rely on a single telemetry modality; recent LLM-based approaches can suffer from unconstrained exploration and hallucination; and most systems stop at fault ranking without producing actionable incident response. We present GALA+, a graph-augmented LLM agentic framework centered on graph-guided investigation, which uses service dependencies to bound exploration and refine diagnosis through localized multi-modal evidence. For initial hypothesis generation, GALA+ combines complementary telemetry signals with STRIX, a novel trace- and graph-structure-aware scoring module. GALA+ then produces ranked diagnoses, incident summaries, and stratified action recommendations. We further introduce SURE-Score, a human-guided evaluation framework co-developed with industry SRE experts for assessing RCA-specific output quality beyond conventional text similarity metrics. On two microservice benchmarks, GALA+ consistently achieves the strongest overall results, surpassing the best LLM-based baseline by more than 25 percentage points in AC@1, while also receiving the highest ratings from both SURE-Score and independent human SRE evaluation." +++ + +[Published version](https://doi.org/10.1145/3832783.3834335). +The full-text link points to the arXiv version, which describes the GALA+ framework. + +This record consolidates the GALA work and its updated GALA+ framework. +The [earlier preprint](https://arxiv.org/abs/2508.12472) circulated under the title +*GALA: Can Graph-Augmented Large Language Model Agentic Workflows Elevate Root Cause Analysis?*. diff --git a/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md new file mode 100644 index 0000000..a00e09d --- /dev/null +++ b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md @@ -0,0 +1,15 @@ ++++ +title = "SRE-Marathon: A Continuous, Change-Driven Benchmark for Autonomous Site Reliability Agents" +year = 2026 +authors = ["Yifang Tian", "Yingjian Bai", "Yifeng He", "Zichun Chong", "Yuanchen Gao", "Yiran Li", "Hans-Arno Jacobsen"] +venue = "arXiv" +publication_type = "ArXiv Preprint" +research = ["distributed-machine-learning", "rca-aiops"] +external_url = "https://arxiv.org/abs/2609.33023" ++++ + +This preprint studies continuous SRE operation with overlapping incidents in a +live Kubernetes environment. Its evaluation follows the progression from +correlation and localization to repair using recorded system evidence. + +[Project website and leaderboard](https://bigbigbai.github.io/sre-marathon-leaderboard/). diff --git a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md index 66c447d..49dcd21 100644 --- a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md +++ b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md @@ -2,9 +2,13 @@ title = "SREGym: A Live Benchmark for AI SRE Agents with High-Fidelity Failure Scenarios" year = 2026 authors = ["Jackson Clark", "Yiming Su", "Saad Mohammad Rafid Pial", "Yifang Tian", "Lily Gniedziejko", "Hans-Arno Jacobsen", "Yinfang Chen", "Tianyin Xu"] -venue = "arXiv" -publication_type = "ArXiv Preprint" +venue = "NeurIPS Evaluations and Datasets Track (accepted; forthcoming)" +publication_type = "Conference Paper" research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2605.07161" abstract = "AI agents are increasingly used to diagnose and mitigate failures in production systems, known as agentic Site Reliability Engineering (SRE). Current SRE benchmarks are limited to oversimplistic SRE tasks and are unfortunately hard to extend due to bespoke designs. We present SREGym, a high-fidelity benchmark for SRE agents. SREGym exposes a live system environment built atop real-world cloud-native system stacks, where high-fidelity failure scenarios are simulated through fault injectors. SREGym models the complexity of production environments by simulating (1) a wide range of faults at different layers, (2) various ambient noises, and (3) diverse failure modes such as metastable failures and correlated failures. SREGym is architected as a modular, extensible framework that orchestrates fault and noise injectors across stacks. SREGym currently includes 90 realistic, challenging SRE problems. We use SREGym to evaluate frontier agents and show that their capabilities varies significantly in addressing different kinds of failures, with up to 40% differences in end-to-end results. SREGym is actively maintained as an open-source project and has been used by researchers and practitioners." +++ + +Accepted to NeurIPS 2026, as listed on the +[co-author's publication page](https://yinfangchen.github.io/publications/). +The full-text link points to the arXiv version; proceedings are forthcoming. diff --git a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md index dcc60d9..4c2f7d4 100644 --- a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md +++ b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md @@ -1,10 +1,15 @@ +++ -title = "When Agentic Executions Fail: Detecting and Localizing Runtime Faults from Telemetry" +title = "AgentChaosBench: Benchmarking External Fault Detection and Localization in Agentic Systems" +slug = "when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry" year = 2026 -authors = ["Chenkai Zhang", "Yiran Li", "Yifang Tian", "Michalis Bachras", "Hans-Arno Jacobsen"] -venue = "arXiv" -publication_type = "ArXiv Preprint" +authors = ["Chenkai Zhang", "Yiran Li", "Yifang Tian", "Michail Bachras", "Hans-Arno Jacobsen"] +venue = "International Workshop on Trustworthy and Responsible Autonomous Systems (TRUST)" +publication_type = "Workshop Paper" research = ["distributed-machine-learning", "rca-aiops"] external_url = "https://arxiv.org/abs/2608.14680" abstract = "Reliability in LLM-based agentic systems is a property of the whole execution (its tool calls, model calls, guardrails, and inter-agent messages), not of the final answer alone, yet evaluating only task outcomes reveals little about how or why a run fails. We present AGENTCHAOSBENCH, a benchmark for detecting and localizing runtime faults in agentic systems from their execution telemetry. We run five heterogeneous applications that coordinate agents over the Agent-to-Agent protocol and call tools through the Model Context Protocol, and inject ten types of operational fault (unavailable or slow tools, corrupted or oversized responses, and delayed, looped, or misrouted delegations and bypassed guardrails) at their tool, model, guardrail, and inter-agent boundaries, alongside a no-fault control. The resulting dataset contains 275 sanitized traces: 250 faulty executions spanning ten fault types and 25 no-fault controls. Each faulty trace is aligned with the no-fault execution of the same input; fault-type labels and, where applicable, location labels are held out from diagnosis. On structured single-trace inputs, a first set of zero-shot LLM baselines shows the task is far from solved: local detectors up to 14B parameters reach only 13.6-19.2% top-1 fault-type accuracy and the frontier DeepSeek-v4-pro only 24.8%, while jointly identifying the fault type and its location tops out at 22%; reference-dependent faults (above all a bypassed guardrail) stay near-unsolved from a single trace. An aligned reference improves selected relative faults but does not resolve guardrail bypass. The held-out labels and compact prediction format support reproducible comparison of LLM-based and non-LLM diagnosis methods." +++ + +[Published version](https://doi.org/10.1145/3843782.3844662). +The linked arXiv version uses the earlier title *When Agentic Executions Fail: +Detecting and Localizing Runtime Faults from Telemetry*. diff --git a/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md new file mode 100644 index 0000000..7919d71 --- /dev/null +++ b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md @@ -0,0 +1,16 @@ ++++ +title = "When Log Parsing Removes What Matters: Detecting Failures in Distributed GPU Workloads" +year = 2026 +authors = ["Boyuan Cui", "Yuyang Zeng", "Yifang Tian", "Hans-Arno Jacobsen"] +venue = "1st International Workshop on Reliable and trustworthy Automated Software Engineering (RASE), pp. 60–63" +publication_type = "Workshop Paper" +research = ["distributed-machine-learning", "rca-aiops"] +external_url = "https://arxiv.org/abs/2610.14009" ++++ + +[Published version](https://doi.org/10.1145/3820756.3844919). +The arXiv version was revised after publication and differs from the RASE 2026 paper. + +The paper introduces [GF-Bench](https://github.com/cuiboyuan/nccl_log_rca_bench) +for generating controlled faults in distributed GPU workloads and examines how +log parsing affects failure detection. diff --git a/content/research/distributed-machine-learning/_index.md b/content/research/distributed-machine-learning/_index.md index 2e34070..d1b8fbd 100644 --- a/content/research/distributed-machine-learning/_index.md +++ b/content/research/distributed-machine-learning/_index.md @@ -3,42 +3,29 @@ title = "Distributed Machine Learning" layout = "area" weight = 20 accent = "sage" -question = "How can distributed learning systems become more efficient and scalable?" -blurb = """At MSRG, we tackle core challenges in distributed deep learning systems. Our research focuses on optimizing training and inference across distributed infrastructure, developing novel approaches for **resource allocation** and **scheduling of training jobs**, and creating **system-level improvements** for **deep learning** and **graph learning** frameworks. We are advancing federated learning for resource-constrained and embedded devices, optimizing performance and communication efficiency in edge computing environments. Through our work, we aim to make distributed deep learning more efficient and accessible, enabling faster training and better resource utilization.""" -# Description aligned with https://msrg.org/research/distributed-machine-learning/ +question = "How can AI help diagnose failures and improve the reliability of distributed systems?" +blurb = """MSRG develops **AI for IT operations (AIOps)** to help engineers detect anomalies, identify root causes, and respond to incidents in distributed systems. We combine **metrics, logs, and traces**, service dependency graphs, and LLM agents to ground diagnosis and repair in operational evidence. Our research includes the reliability of agentic systems and realistic benchmarks for evaluating diagnosis and incident response. These efforts build on our broader work in distributed AI and machine learning, including scalable training, federated learning, and graph processing.""" +++ -MSRG studies performance optimization, privacy-preserving and federated -learning, and graph processing as connected challenges in distributed machine -learning. +Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research investigates how +learning-based methods can help operate complex distributed systems reliably. -## Distributed Deep Learning +## AIOps and incident response -Large training and inference workloads need substantial computing resources -and distributed infrastructure. We investigate how resource allocation, job -scheduling, and optimizations within learning systems can improve performance -and reduce costs. +We study anomaly detection, graph-guided root cause analysis, and +telemetry-grounded repair. By connecting operational signals to service +dependencies, we aim to produce diagnoses and response recommendations that +engineers can inspect and act on. -Our federated learning research also addresses embedded devices with limited -resources. We work across hardware-level optimizations and distributed learning -algorithms to improve efficiency in edge deployments. - -## Graph Processing & Learning - -Graph workloads combine irregular memory access, interdependent computations, -and large datasets. We develop partitioning and sampling methods alongside -system optimizations to make these workloads faster. - -Our distributed graph systems target graphs containing billions of vertices -and edges. We also optimize the training and inference stages of graph neural -networks. - -Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research connects these -interests with learning-based anomaly detection, LLM agents for failure -diagnosis, and the reliability of agent executions. +AI agents also introduce new operational failure modes. Our benchmarks study +fault detection and localization in agent executions, as well as diagnosis and +repair in live cloud-native environments. ## Current directions -- Resource allocation and scheduling for distributed training and inference -- Privacy-preserving and federated learning on embedded and edge devices -- Graph partitioning, sampling, and scalable graph neural network execution +- Anomaly detection from metrics, logs, and traces +- Graph-guided diagnosis and telemetry-grounded repair +- Reliable AI agents and realistic incident-response benchmarks + +We also continue research in distributed AI and machine learning, including +efficient training and inference, federated learning, and scalable graph processing. diff --git a/content/research/rca-aiops/_index.md b/content/research/rca-aiops/_index.md index d032169..252a65c 100644 --- a/content/research/rca-aiops/_index.md +++ b/content/research/rca-aiops/_index.md @@ -2,12 +2,18 @@ title = "Root Cause Analysis & AIOps" slug = "rca-aiops" layout = "area" +list_all_publications = true weight = 40 accent = "teal" question = "How can we turn operational telemetry into reliable diagnosis and actionable incident response?" summary = "Root cause analysis, anomaly detection, and telemetry-grounded repair for reliable microservices and AI agents." blurb = """MSRG develops methods for **root cause analysis** and **AI for IT operations (AIOps)** in distributed systems. We combine **metrics, logs, and traces** with service dependency graphs and LLM agents to investigate failures and produce actionable diagnoses. Our work connects anomaly detection, incident response, automated repair, and realistic evaluation, with a focus on microservices and the reliability of AI agent executions.""" # Research descriptions are based on the publication abstracts linked below. +acknowledgement = "We acknowledge the support of Bell Mobility and the Natural Sciences and Engineering Research Council of Canada (NSERC) under grant ALLRP614128-25." +supporters = [ + { name = "Bell Mobility", url = "https://www.bell.ca/Mobility", logo = "/images/support/bell.svg", width = 100, height = 60 }, + { name = "Natural Sciences and Engineering Research Council of Canada (NSERC)", url = "https://nserc-crsng.canada.ca/", logo = "/images/support/nserc.png", width = 145, height = 70 } +] +++ ## From telemetry to root causes @@ -22,19 +28,22 @@ an anomaly to understanding a failure and deciding what to do next. Our research brings together statistical analysis, graph structure, and LLM agents, with an emphasis on diagnoses and repair decisions grounded in operational evidence. -## Graph-guided diagnosis and incident response +
+ +{{< figure src="/images/research/aiops-vision.svg" alt="AIOps research vision: observe distributed systems, detect anomalies, diagnose causes, and support incident response, with realistic benchmark evaluation." caption="Research vision: connect operational evidence to diagnosis and response, and evaluate each step in realistic systems." >}} + +{{< figure src="/images/research/aiops-technical.svg" alt="GALA+ workflow: telemetry and service dependencies feed initial hypotheses, followed by graph-guided investigation using local evidence, producing diagnoses and incident response recommendations evaluated with SURE-Score." caption="Technical example: GALA+ uses service dependencies and local telemetry to guide LLM investigation; SURE-Score assesses the quality of its outputs." >}} + +
-[GALA](/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis/) -combines statistical causal inference with iterative LLM reasoning over metrics, -logs, and traces. It investigates both root cause identification and -human-readable remediation guidance. +## Graph-guided diagnosis and incident response -[GALA+](/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices/) -uses service dependencies to guide investigation and focus agents on local, -multimodal evidence. Its outputs include ranked diagnoses, incident summaries, -and action recommendations. The work also introduces SURE-Score, developed -with industry site reliability engineers to assess diagnostic quality beyond -text similarity. +[GALA](/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices/), +whose updated framework is called GALA+, uses service dependencies to guide +LLM agents through local metrics, logs, and traces. It produces ranked diagnoses, +incident summaries, and action recommendations. SURE-Score, developed with +industry site reliability engineers, assesses diagnostic quality beyond text +similarity. ## Message-level anomaly detection @@ -59,8 +68,8 @@ automated repair with the incident that motivated it. AI agents are both tools for incident response and systems whose own failures need diagnosis. -[When Agentic Executions Fail](/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry/) -introduces AGENTCHAOSBENCH to study fault detection and localization from agent +[AgentChaosBench](/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry/) +studies fault detection and localization from agent execution telemetry, including faults at tool, model, guardrail, and inter-agent boundaries. @@ -70,6 +79,11 @@ ambient noise. These complementary settings help us examine whether an agent can identify a fault, locate its origin, and support incident resolution under realistic operating conditions. +[GF-Bench](/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads/) +extends failure detection to distributed GPU workloads, while +[SRE-Marathon](/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents/) +studies continuous operation with overlapping incidents and code or configuration changes. + ## Current directions - Correlating metrics, logs, and traces across service dependencies @@ -80,3 +94,12 @@ realistic operating conditions. This research connects our work in [Data Management](/research/data-management/) and [Distributed Machine Learning](/research/distributed-machine-learning/). + +## Outcomes + +- **[AgentChaosBench](https://github.com/kevinzck8k/agentic-fault-diagnosis):** benchmark and diagnosis tools for faults in agentic executions. +- **[FAME](https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts):** implementation for message-level log anomaly detection. +- **[GALA evaluation materials](https://github.com/EvanTian233/GALA-for-RCA):** evaluation resources; the core implementation is not currently public. +- **[GF-Bench](https://github.com/cuiboyuan/nccl_log_rca_bench):** fault injection and log data for distributed GPU workloads. +- **[SREGym](https://github.com/SREGym/SREGym):** live environments and fault scenarios for evaluating SRE agents. +- **[SRE-Marathon](https://bigbigbai.github.io/sre-marathon-leaderboard/):** project website and leaderboard, with [website source on GitHub](https://github.com/bigbigbai/sre-marathon-leaderboard). diff --git a/docs/aiops-sources.md b/docs/aiops-sources.md new file mode 100644 index 0000000..2b74f54 --- /dev/null +++ b/docs/aiops-sources.md @@ -0,0 +1,43 @@ +# AIOps content sources + +Checked 2026-11-01. People and portraits come from `content/people/` through +the same roster and card partials as Quantum Computing Systems. Publications +come from `content/publications/`; the AIOps page displays all matching records. +The acknowledgement text and grant number were supplied by the project team. +The page retains the shared research layout, with People before Publications, +as requested by the maintainer. + +## Yuanchen Gao + +Added to the shared roster at the maintainer's request, with the PhD role +confirmed by the maintainer. [Charles Zhang's HKUST page](https://cse.hkust.edu.hk/~charlesz/) +lists him among current students. The [ORCA manuscript](https://arxiv.org/html/2608.17018v1) +confirms his HKUST affiliation, public university email, and an IVGS visit to +Toronto during the work. Visit dates are unknown and are not inferred. +His card uses the default initials avatar, as requested. No LinkedIn, ORCID, +or Google Scholar account was reliably matched; same-name materials researchers +were excluded. + +## Publication updates + +- GALA: [ASE programme](https://conf.researchr.org/details/ase-2026/ase-2026-research-track/5/GALA-Graph-Augmented-LLM-Agents-for-Root-Cause-Analysis-and-Incident-Response-in-Mic), [published version](https://doi.org/10.1145/3832783.3834335). At the maintainer's request, consolidate the earlier GALA preprint and the updated GALA+ work into this single record. Keep the earlier arXiv link in its notes and redirect the old website URL through an alias. +- AgentChaosBench: [TRUST programme](https://conf.researchr.org/details/ase-2026/trust-2026-papers/12/AgentChaosBench-Benchmarking-External-Fault-Detection-and-Localization-in-Agentic-Sy), [published version](https://doi.org/10.1145/3843782.3844662). Preserve the original page URL and identify the earlier arXiv title in the publication notes. +- FAME: [ISSRE programme](https://cyprusconferences.org/issre2026/program/). No unverified DOI or pagination added. +- SREGym: [co-author's publication list](https://yinfangchen.github.io/publications/). Accepted to NeurIPS 2026 Evaluations and Datasets; explicitly marked forthcoming. +- GF-Bench paper: [arXiv metadata](https://arxiv.org/abs/2610.14009), [published version](https://doi.org/10.1145/3820756.3844919). The revised arXiv version differs from the proceedings paper. +- SRE-Marathon: [arXiv](https://arxiv.org/abs/2609.33023); retained as a preprint. + +## Illustrations and sponsor marks + +The two `static/images/research/aiops-*.svg` diagrams are original schematic +illustrations, not reproductions of paper figures. The vision is a research +agenda; the technical workflow summarizes the [GALA+ abstract](https://arxiv.org/abs/2608.08968). +No experimental results are invented. Outcomes distinguish GALA evaluation +materials from its unreleased core code and SRE-Marathon website source from +benchmark implementation. + +- Bell: unmodified logo geometry and viewBox from the `icon-bell-logo` symbol + on [Bell's official site](https://www.bell.ca/), using Bell blue `#00549a`. +- NSERC: unmodified [official bilingual symbol](https://nserc-crsng.canada.ca/sites/default/files/styles/gcds_full_sm/public/migrated/img/logos/img-logo2-en.png?itok=1FYj3E2S). + +Sponsor marks identify project support and remain their owners' property. diff --git a/layouts/research/area.html b/layouts/research/area.html index f7d90be..36076ee 100644 --- a/layouts/research/area.html +++ b/layouts/research/area.html @@ -12,11 +12,22 @@

{{ .Title }}

{{ with .Params.question }}

{{ . }}

{{ end }}
{{ .Params.blurb | markdownify }}
+ {{ with .Params.acknowledgement }} +

{{ . }}

+
+ {{ range $.Params.supporters }} + {{ .name }} + {{ end }} +
+ {{ end }}

{{ len $people }} {{ if eq (len $people) 1 }}researcher{{ else }}researchers{{ end }} · {{ len $publications }} {{ if eq (len $publications) 1 }}publication{{ else }}publications{{ end }} · {{ len $datasets }} {{ if eq (len $datasets) 1 }}data set{{ else }}data sets{{ end }}

+ {{ if .Params.list_all_publications }} +

Outcomes · People · Publications

+ {{ end }}
@@ -28,26 +39,37 @@

{{ .Title }}

-
+
- + {{ if .Params.list_all_publications }}

People

{{ end }} {{ partial "people/group-cards.html" $people }}
-
+

Publications

{{ if $publications }} - {{ range first 6 (sort $publications "Params.year" "desc") }} + {{ $listed := first 6 (sort $publications "Params.year" "desc") }} + {{ if .Params.list_all_publications }}{{ $listed = sort $publications "Params.year" "desc" }}{{ end }} + {{ range $listed }} + {{ if $.Params.list_all_publications }} +
+ {{ .Title }} + {{ partial "publication-authors.html" (dict "authors" .Params.authors) }} + {{ .Params.venue }}, {{ .Params.year }} + {{ with .Params.external_url }}Read paper{{ end }} +
+ {{ else }} {{ .Title }} {{ .Params.venue }}, {{ .Params.year }} + {{ end }} {{ end }} {{ else }}

No publications are listed for this research area yet.

diff --git a/static/images/research/aiops-technical.svg b/static/images/research/aiops-technical.svg new file mode 100644 index 0000000..50bc2d3 --- /dev/null +++ b/static/images/research/aiops-technical.svg @@ -0,0 +1,30 @@ + +GALA+: technical workflowTelemetry and service dependencies seed hypotheses. Graph-guided LLM investigation gathers local evidence and refines the diagnosis. Outputs are assessed with SURE-Score. + + +GALA+: technical workflow + +Operational evidence +Metrics, logs, and traces +plus service dependencies + + +Initial hypotheses +Combine telemetry signals +with STRIX structural scoring + + +Graph-guided investigation +Gather local evidence +Refine hypotheses with LLM agents + + +Incident response outputs +Ranked causes and summaries +Stratified action recommendations + + +Quality assessment +SURE-Score and human review +Beyond text similarity + diff --git a/static/images/research/aiops-vision.svg b/static/images/research/aiops-vision.svg new file mode 100644 index 0000000..ef428b8 --- /dev/null +++ b/static/images/research/aiops-vision.svg @@ -0,0 +1,30 @@ + +AIOps research visionObserve distributed services, detect anomalies, diagnose causes, support response, and evaluate with realistic benchmarks. This is a research agenda, not a single deployed system. + + +AIOps research vision + +Observe +Metrics, logs, and traces +from distributed services + + +Detect +Identify anomalous signals +and unreliable agent behaviour + + +Diagnose +Connect symptoms to causes +using dependencies and evidence + + +Respond +Support incident decisions +and investigate automated repair + + +Evaluate +Realistic faults and benchmarks +Diagnostic and repair quality + diff --git a/static/images/support/bell.svg b/static/images/support/bell.svg new file mode 100644 index 0000000..ef4dc07 --- /dev/null +++ b/static/images/support/bell.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/images/support/nserc.png b/static/images/support/nserc.png new file mode 100644 index 0000000000000000000000000000000000000000..9a5277670a8e055ae55228d81a8abeabb2fd41ad GIT binary patch literal 4176 zcmV-W5U=lvP)n00009a7bBm000ie z000ie0hKEb8vp&kN?W4g|aMg0LwnoL@Y zYLki2`UB=je?YeXfMvBQq;r2GT1AR9=8)8ls{`UPMUG?9m3%}BfFu?R>|&Q7Ma6n% zJQnu@!0x_y@qU9aS~%k)$0}L? zX#-a;BmRK{z!E^}K7;$pP_`dlq5#>qoOq@Ey#bKtlb_-MKx_0%eiR%g`ye#{HDGm@ z(Y0-zhy}>s0dn7G-~wop^s$cm1F+g>w9?ck!DO6kHGsU9Vxxr{L(%vl*1%Y_zbrrB>965-1cPHZAO-N$W zK(?Lj4DpwUcZuyEaoq%QKDq{g*>)LN)yHO>7w7kQ?{36<>i~9*dmmjZ5lyH&@vn28 z<0=3zu0cS)Hx6k^jhNRGhKP4hk>GwJ(*8EMr8v>RezOIT-9;lPM%QXS6Jv29Xwf@oLPUz|H1F}=QOI_cUE&3R7fK`AUmPw_szDO` zSBIc7k(6<;D}^v~?F@^E_gYfW6B~;mLuCus0{9d^LefdC9BKHexOj@`O>^j5zNrd1`$|gR`M97gVbL7*j51qC;?b z_qqva1oqyvKW&_Dt#waU`blM%JpRx?BoLDcNA*5lE7=ATTcpOs@kvIni>!Fsf z><{1vIjg%nm=7Vf;y;&Dmo#agF|EF&&S*8uMl_7wQ4&8eTB(oz@w-3x*s1O?T7AT5 z(x z%r4|0qMfnvbTh?ae;@shc<(^|fANCRis@9| z25=N|oNn1;o&4A0en)ixIthz zD|17RCj?g<1-FGYKu-I7HyKVMBHlulH(Jx&OoGTmu@*xI#tO22 z|B|J-p6oMP*#HLHadmvq23GSnsUt>&(&)H)2%rHhA4A!cd=c>v{8w7}Xi4KKh^scR zoO*@IcG02=$Qh%RQ%H@@u$j5zdq@fYP!S3V2r_(dk6la~E_?z&&ji_GwDKMRPo~|I zwv`hJh>v;Y5BX6`${axIz$(+JRuWY4SZLi=B{|Q^8Gwd5^rQ$kA{id{Jn(y8S4~Tt z^_H2gEIHQaqfldWz2~qHsXI3Sp;E>Yru z_%q^aX<5y*=EyP&DK`|be@uu^dHG)>p#m?xX0+Or$3+bwkxYXj6cERwY8ghmM()+L zFs;CGMEnD1w7MgJH!iEWj8B?{qdNg-`_d@8wkbf~rCIk0@MFY3aMyBXouLu$wHU3I zCoGL%66EAE_LF@^D-VIe?i6;O7633>Jp^ESxJUie0lbTkWjrQ_JeD^Y-VCNU-;-?# ziuV!m?kS`9AE>}_w;V%ViIcjYO7 z?TL~h+5lGaKy{^ zhCl;YEy3y*5m82>p%v;qSRA;c%Bm~7j8Y>JG0iMOpzLX_q`PQb?90en1u(8SyWB`7 ze%kt5sbQ>W5Y65P03x<3x%&hK@J#R1B@caGK{Ed!+!qz9>$fs#-D*oMz2RG&5ivi9kciKxpLrHqi<#e=YH za!=e^>UWP1hI_KGrqhK`Bm~vMiEVWzNxfJFSW=6Z1YX@jBIH(_BbmA4ks;zcX})bB zK|rEoQK-tmw~Z9Rdb#d86Y=gT!EinA$5jSJOPtb}1KF&{=h;MN_JdE->g-N)Qj>({ zX;;ErCWdR}o*-q2WQU|1!ae13|U93BR+YOm%#!b@Cr)7x_!PSFo)!scxT(A{Aj}vNs zTbJaL#Bedi3SpTXfonfn#yOhan=dCaC(7ixEEDri8WClBw^bRJ%^Y%4+g|Wsbj-{q z1(XKj>nMFIYX$g(Wr)BL(|W_qN%iFFi>|tl)MIHM$<3 z0BpOW4(wR66;k+0Q1Yt&pR2|6)1;m!VsIiSx!fX>3#EDNlL2vqwwt?%4#99ePgaU5 zH8(3`=?#Dgo)qbkM!md`9P!KQgRe7M{RxI#HR3pmP*m=EA`G)1zh^{1X4IhQ6swiA zV%wx85HbV7tVp--m_hTVa4a(Wi&wcV8dBrz(<>MM38R(Q`5l?!j&2P$GuikG~a`nNq!gbN=vAI-NDUinnS0lMk zcL$_!lGwSHak8wqN)D{P#6qSglbDPa&-we^vQ(4Tc}nag6DtYLF}O^45)r9cs&}Qq zLUb{Mp}xD0CHB{L&Sp!p*Dqm>gR#x>b-NNH~7HdeL#W%6{c z?zgp;FLH!}ln#B&_{wBYlV@}_-L2-wq1%V7k&j;7NG)S?uNR$x20-?atQ5DK)L^Zt zMtKQIZ?oqA4YHB51aawvQTi}jn6()n^*ysL+Kg7zHD+;zK^s^Zooh{wf8oHAt=+Ux zUWJt`2j-$++ooyCRHL&zfyO%tQc(3hBfV3fPV`4|PM;b&srQY(w zK@}QrJ= zaV-_#Ga)O?E;<3lK@tT%(_IsikZdKDmok(Lv--b3B7V_UAukEL)RbFQk(w&pu)PRb z6)JA{H9R-=4d1^he3FmFHV>SHbVxa95(hM?INNJ6T0KU*%E)7+R{9;j2TJPta5W)A zY?@aF$Qh&67r^EIbqU;zR-Q_g`8u_*C5qVQE#e$gmk}1bMtqO1p_(=(n%-876O|#o z1sV5bBjom-&m>n`o2x-jZP)Z3APZ`Zo+a|xdMv924Ka$Kd%!{lZ@*!*I&)%D#B(Oq z&Ja$;n+SJ^j0>|U<2S1<&8Z@$eaJE-Z$j(~qGXQ8kfrJ9wCybpij~}vb%gv4hQPSy zL7?hz60;(r5h#FXKFrCWD`k92I9{1T?DT>eFI&K`~qY9I3f7WNwKlC)^tW>ku7akYlGu>oQrB{qert zSEltQW2krhQ;*VF Date: Sun, 1 Nov 2026 15:29:47 -0500 Subject: [PATCH 03/15] Clarify GALA evaluation materials description --- content/research/rca-aiops/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/research/rca-aiops/_index.md b/content/research/rca-aiops/_index.md index 252a65c..3f60359 100644 --- a/content/research/rca-aiops/_index.md +++ b/content/research/rca-aiops/_index.md @@ -99,7 +99,7 @@ This research connects our work in - **[AgentChaosBench](https://github.com/kevinzck8k/agentic-fault-diagnosis):** benchmark and diagnosis tools for faults in agentic executions. - **[FAME](https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts):** implementation for message-level log anomaly detection. -- **[GALA evaluation materials](https://github.com/EvanTian233/GALA-for-RCA):** evaluation resources; the core implementation is not currently public. +- **[GALA evaluation materials](https://github.com/EvanTian233/GALA-for-RCA):** human-annotated evaluation data, sample incident summaries and remediation recommendations, and third-party baselines for microservice root cause analysis. - **[GF-Bench](https://github.com/cuiboyuan/nccl_log_rca_bench):** fault injection and log data for distributed GPU workloads. - **[SREGym](https://github.com/SREGym/SREGym):** live environments and fault scenarios for evaluating SRE agents. - **[SRE-Marathon](https://bigbigbai.github.io/sre-marathon-leaderboard/):** project website and leaderboard, with [website source on GitHub](https://github.com/bigbigbai/sre-marathon-leaderboard). From e34cba4933f5f68b64303c0040628c372801227b Mon Sep 17 00:00:00 2001 From: ttrenty <154608953+ttrenty@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:58:25 -0500 Subject: [PATCH 04/15] Unify root cause analysis and AIOps research pages --- README.md | 18 +++ assets/css/main.css | 2 +- assets/js/site.js | 10 +- content/people/chenkai-zhang.md | 2 +- content/people/hans-arno-jacobsen.md | 2 +- content/people/mengli-duan.md | 2 +- content/people/michalis-bachras.md | 2 +- content/people/rev-liu.md | 2 +- content/people/yifang-tian.md | 2 +- content/people/yifeng-he.md | 2 +- content/people/yiran-li.md | 2 +- content/people/yuanchen-gao.md | 2 +- ...for-message-level-log-anomaly-detection.md | 2 +- ...-and-incident-response-in-microservices.md | 7 +- ...ogram-repair-for-microservice-incidents.md | 2 +- ...-for-autonomous-site-reliability-agents.md | 2 +- ...ts-with-high-fidelity-failure-scenarios.md | 2 +- ...ocalizing-runtime-faults-from-telemetry.md | 7 +- ...g-failures-in-distributed-gpu-workloads.md | 5 +- content/research/_index.md | 2 +- content/research/data-management/_index.md | 2 +- .../distributed-machine-learning/_index.md | 109 +++++++++++++++--- content/research/rca-aiops/_index.md | 105 ----------------- data/research_outcomes.toml | 58 ++++++++++ docs/aiops-sources.md | 7 +- hugo.yaml | 8 +- layouts/index.html | 2 +- layouts/partials/footer.html | 4 +- layouts/publications/archive.html | 2 +- layouts/research/area.html | 79 +++++++------ scripts/browser_check.cjs | 70 ++++++++++- tests/test_content_contracts.py | 23 ++++ 32 files changed, 352 insertions(+), 194 deletions(-) delete mode 100644 content/research/rca-aiops/_index.md create mode 100644 data/research_outcomes.toml diff --git a/README.md b/README.md index ddc3f47..c5c74bc 100644 --- a/README.md +++ b/README.md @@ -168,10 +168,28 @@ Link to an external repository for datasets larger than **20 MB**. | --- | --- | | Homepage text | `content/_index.md` | | Research themes | `content/research/` | +| Research software and benchmark outcomes | `data/research_outcomes.toml` | | Navigation and site settings | `hugo.yaml` | Use GitHub's file editor on a branch or fork, then submit a PR. +The **Root Cause Analysis & AIOps** area refocuses the existing Distributed +Machine Learning area. Its stable content identifier and URL remain +`distributed-machine-learning`; `/research/rca-aiops/` redirects there. Use +`distributed-machine-learning` in member, publication, dataset, and outcome +research tags. The old `rca-aiops` publication-archive query is supported through +the area's `legacy_research_ids` setting. It is not a separate research choice. +All three research areas appear in the header, homepage, and research overview. + +The combined page includes AIOps/RCA work and earlier distributed AI/ML work. +It reads people and publication references from the master collections, with +research outcomes before publications and the associated people. Add outcomes +only after verifying their public project or repository against a paper or an +official author page. Each `data/research_outcomes.toml` item has a stable `id`, +`title`, HTTPS `url`, short `summary`, `research` tags, and a master `publication` +slug. Record attribution in a comment and describe the actual release scope. +Alumni career outcomes remain separate member-profile data. + Edit source files; `public/` is generated. Everything in `content/` and `static/` may be published. Keep unfinished work on a branch until ready to merge. diff --git a/assets/css/main.css b/assets/css/main.css index fb97e03..60f2f28 100644 --- a/assets/css/main.css +++ b/assets/css/main.css @@ -1032,7 +1032,7 @@ a.chip { } .card-grid-wide { - grid-template-columns: repeat(2, minmax(0, 1fr)); + grid-template-columns: repeat(3, minmax(0, 1fr)); } .accent-copper { diff --git a/assets/js/site.js b/assets/js/site.js index e547dbd..d02a81e 100644 --- a/assets/js/site.js +++ b/assets/js/site.js @@ -73,6 +73,9 @@ document.addEventListener("DOMContentLoaded", () => { const emptyState = document.querySelector("[data-publication-empty]"); const controls = document.querySelector("[data-publication-controls]"); const researchButtons = [...document.querySelectorAll("[data-publication-research]")]; + const researchAliases = new Map(researchButtons.flatMap((button) => + (button.dataset.publicationResearchAliases || "").split(",").filter(Boolean) + .map((alias) => [alias, button.dataset.publicationResearch]))); const resultCount = document.querySelector("[data-publication-count]"); const clearButton = document.querySelector("[data-publication-clear]"); const normalize = (text) => text.normalize("NFD").replace(/\p{M}/gu, "").replace(/[‘’]/g, "'").toLowerCase(); @@ -197,8 +200,13 @@ document.addEventListener("DOMContentLoaded", () => { const url = new URL(location.href); const params = url.searchParams; searchInput.value = params.get("q") || params.get("author") || ""; - const area = params.get("research"); + const requestedArea = params.get("research"); + const area = researchAliases.get(requestedArea) || requestedArea; activeResearch = researchButtons.some((button) => button.dataset.publicationResearch === area) ? area : "all"; + if (requestedArea !== area) { + url.searchParams.set("research", area); + history.replaceState(null, "", url); + } const page = params.get("page") || url.pathname.match(/\/page\/(\d+)\//)?.[1] || "1"; currentPage = /^[1-9]\d*$/.test(page) && Number.isSafeInteger(Number(page)) ? Number(page) : 1; const filtered = searchInput.value || activeResearch !== "all"; diff --git a/content/people/chenkai-zhang.md b/content/people/chenkai-zhang.md index ddb1d92..17a2e48 100644 --- a/content/people/chenkai-zhang.md +++ b/content/people/chenkai-zhang.md @@ -9,6 +9,6 @@ email = "zhangchenkai.zhang@mail.utoronto.ca" interests = ["AI Agent Reliability", "Fault Diagnosis"] author_names = ["Chenkai Zhang"] portrait = "" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] linkedin = "https://www.linkedin.com/in/chenkaizhang1/" +++ diff --git a/content/people/hans-arno-jacobsen.md b/content/people/hans-arno-jacobsen.md index b1015bf..e7e6a45 100644 --- a/content/people/hans-arno-jacobsen.md +++ b/content/people/hans-arno-jacobsen.md @@ -5,7 +5,7 @@ role = "Professor" status = "current" start_date = "2000-01" location = "Toronto, Canada" -research = ["quantum-computing-systems", "distributed-machine-learning", "data-management", "rca-aiops"] +research = ["quantum-computing-systems", "distributed-machine-learning", "data-management"] email = "jacobsen@eecg.utoronto.ca" homepage = "https://www.eecg.toronto.edu/~jacobsen" orcid = "https://orcid.org/0000-0003-0813-0101" diff --git a/content/people/mengli-duan.md b/content/people/mengli-duan.md index 66a1de8..d6757d0 100644 --- a/content/people/mengli-duan.md +++ b/content/people/mengli-duan.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2026-09" location = "Toronto, Canada" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] email = "mengli.duan@mail.utoronto.ca" interests = ["Machine Learning Systems"] author_names = ["Mengli Duan", "Dawn (Mengli) Duan", "Dawn Duan"] diff --git a/content/people/michalis-bachras.md b/content/people/michalis-bachras.md index 25bac30..9acd5fd 100644 --- a/content/people/michalis-bachras.md +++ b/content/people/michalis-bachras.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2023-09" location = "Toronto, Canada" -research = ["data-management", "rca-aiops"] +research = ["data-management", "distributed-machine-learning"] interests = ["Data Management", "Distributed Systems"] portrait = "/images/people/michalis-bachras.jpg" linkedin = "https://www.linkedin.com/in/michalis-bachras/" diff --git a/content/people/rev-liu.md b/content/people/rev-liu.md index 4a328b2..a46d498 100644 --- a/content/people/rev-liu.md +++ b/content/people/rev-liu.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2026-09" location = "Toronto, Canada" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] email = "rev.liu@mail.utoronto.ca" interests = ["VLM", "RL", "AIOps"] author_names = [] diff --git a/content/people/yifang-tian.md b/content/people/yifang-tian.md index e7c96a1..ecbcab4 100644 --- a/content/people/yifang-tian.md +++ b/content/people/yifang-tian.md @@ -5,7 +5,7 @@ role = "PhD Student" status = "current" start_date = "2022-01" location = "Toronto, Canada" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] interests = ["Machine Learning Systems", "Distributed Systems"] portrait = "/images/people/yifang-tian.jpg" email = "yifang.tian@mail.utoronto.ca" diff --git a/content/people/yifeng-he.md b/content/people/yifeng-he.md index 3d6a69e..90e470c 100644 --- a/content/people/yifeng-he.md +++ b/content/people/yifeng-he.md @@ -5,7 +5,7 @@ role = "MSc Student" status = "current" start_date = "2025-09" location = "Toronto, Canada" -research = ["rca-aiops"] +research = ["distributed-machine-learning"] email = "yifeng.he@mail.utoronto.ca" interests = [] author_names = ["Yifeng He", "Yifeng (Jason) He"] diff --git a/content/people/yiran-li.md b/content/people/yiran-li.md index 912eaa0..50cb7af 100644 --- a/content/people/yiran-li.md +++ b/content/people/yiran-li.md @@ -5,7 +5,7 @@ role = "Post-Doc" status = "current" start_date = "2025-11" location = "Toronto, Canada" -research = ["quantum-computing-systems", "distributed-machine-learning", "data-management", "rca-aiops"] +research = ["quantum-computing-systems", "distributed-machine-learning", "data-management"] interests = ["Graph Analytics", "Graph Learning", "Quantum Data Management", "AI Agent Reliability"] email = "" homepage = "https://cyanidecentral.github.io/" diff --git a/content/people/yuanchen-gao.md b/content/people/yuanchen-gao.md index eb2efca..52aaa7a 100644 --- a/content/people/yuanchen-gao.md +++ b/content/people/yuanchen-gao.md @@ -4,7 +4,7 @@ slug = "yuanchen-gao" role = "PhD Student" status = "current" location = "HKUST, Hong Kong" -research = ["rca-aiops"] +research = ["distributed-machine-learning"] interests = ["Root Cause Analysis", "Automated Program Repair", "AIOps"] author_names = ["Yuanchen Gao"] email = "ygaocv@connect.ust.hk" diff --git a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md index 27d7fb6..eae4a3c 100644 --- a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md +++ b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Huanchi Wang", "Zihang Huang", "Yifang Tian", "Kristina Dzeparoska", "Hans-Arno Jacobsen", "Alberto Leon-Garcia"] venue = "37th IEEE International Symposium on Software Reliability Engineering (ISSRE)" publication_type = "Conference Paper" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2605.22779" abstract = "Production systems generate millions of log lines daily, yet most anomaly detectors operate at the session or window-level, flagging groups of lines rather than identifying the specific message responsible. This coarse granularity forces operators to inspect many routine lines per alert. Message-level detection offers finer granularity, but remains challenging. A single event template may correspond to both normal and anomalous messages, failures arise from heterogeneous subsystems, and line-level labeling at scale is impractical. Although large language models (LLMs) can reason over log semantics, applying them to every line is too costly for continuous monitoring. We present FAME (Failure-Aware Mixture-of-Experts), a label-efficient message-level mixture-of-experts framework that uses an LLM only once offline. We annotate at most K labeled lines per template to derive binary normal/anomaly indicators and representative examples. The LLM proposes a partition of templates into failure domains, and a certification step validates the proposal before training. FAME trains a lightweight router and domain experts that run on-premise and output anomaly predictions and failure-domain labels. On BGL, FAME achieves F1 = 98.16 at K = 100 reducing annotation effort by 76x and detects 97.7% of anomalies from unseen EventIDs. On Thunderbird, FAME reaches F1 = 99.95 with perfect recall." +++ diff --git a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md index 75d7a02..b4e6d87 100644 --- a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md +++ b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md @@ -3,10 +3,13 @@ title = "GALA: Graph-Augmented LLM Agents for Root Cause Analysis and Incident R aliases = ["/publications/gala-can-graph-augmented-large-language-model-agentic-workflows-elevate-root-cause-analysis/"] year = 2026 authors = ["Yifang Tian", "Yaming Liu", "Zichun Chong", "Zihang Huang", "Yiran Li", "Hans-Arno Jacobsen"] -venue = "41st IEEE/ACM International Conference on Automated Software Engineering (ASE)" +venue = "Proceedings of the 41st IEEE/ACM International Conference on Automated Software Engineering (ASE '26), pp. 55–67" publication_type = "Conference Paper" -research = ["distributed-machine-learning", "data-management", "rca-aiops"] +research = ["distributed-machine-learning", "data-management"] external_url = "https://arxiv.org/abs/2608.08968" +tags = ["GALA", "GALA: Can Graph-Augmented Large Language Model Agentic Workflows Elevate Root Cause Analysis?"] +# Keep the consolidated earlier manuscript title searchable. +# ACM publisher metadata: https://api.crossref.org/works/10.1145/3832783.3834335 abstract = "Microservice root cause analysis (RCA) requires correlating failures across heterogeneous telemetry within complex service dependency graphs. Existing methods often rely on a single telemetry modality; recent LLM-based approaches can suffer from unconstrained exploration and hallucination; and most systems stop at fault ranking without producing actionable incident response. We present GALA+, a graph-augmented LLM agentic framework centered on graph-guided investigation, which uses service dependencies to bound exploration and refine diagnosis through localized multi-modal evidence. For initial hypothesis generation, GALA+ combines complementary telemetry signals with STRIX, a novel trace- and graph-structure-aware scoring module. GALA+ then produces ranked diagnoses, incident summaries, and stratified action recommendations. We further introduce SURE-Score, a human-guided evaluation framework co-developed with industry SRE experts for assessing RCA-specific output quality beyond conventional text similarity metrics. On two microservice benchmarks, GALA+ consistently achieves the strongest overall results, surpassing the best LLM-based baseline by more than 25 percentage points in AC@1, while also receiving the highest ratings from both SURE-Score and independent human SRE evaluation." +++ diff --git a/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md b/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md index b87d2a2..89dfc65 100644 --- a/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md +++ b/content/publications/orca-observability-grounded-program-repair-for-microservice-incidents.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Yuanchen Gao", "Yifang Tian", "Yiran Li", "Charles Zhang", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["data-management", "rca-aiops"] +research = ["data-management", "distributed-machine-learning"] external_url = "https://arxiv.org/abs/2608.17018" abstract = "Microservice failures are often diagnosed from operational telemetry. However, automated program repair systems usually start from issue reports, localized code context, or failing tests. This mismatch leaves a gap between telemetry-based diagnosis and patch generation. We present ORCA, an observability-grounded APR pipeline for microservice incidents. ORCA first distills the differences in paired failure and reference telemetry into a fault signature, then uses the signature to identify candidate code and deployment-configuration locations. Repair graph agents and an Exploration agent generate unified-diff patch candidates from these locations. ORCA evaluates generated patches with a Telemetry-Grounded Patch Verifier that separates patch validity, syntactic and semantic correctness, test-oracle integrity, and telemetry replay. On a 575-case benchmark, ORCA outperforms all evaluated baselines in terms of cost-effectiveness. Results show that operational telemetry can be transformed from diagnostic evidence into actionable repair context: paired telemetry supports repair-oriented localization, while repair graph agents convert localized code and configuration evidence into constrained patch-generation context for the LLM. Telemetry-grounded verification then exposes repair outcomes that issue- or test-only evaluation would miss." +++ diff --git a/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md index a00e09d..cf6420d 100644 --- a/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md +++ b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Yifang Tian", "Yingjian Bai", "Yifeng He", "Zichun Chong", "Yuanchen Gao", "Yiran Li", "Hans-Arno Jacobsen"] venue = "arXiv" publication_type = "ArXiv Preprint" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2609.33023" +++ diff --git a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md index 49dcd21..114cbb3 100644 --- a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md +++ b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md @@ -4,7 +4,7 @@ year = 2026 authors = ["Jackson Clark", "Yiming Su", "Saad Mohammad Rafid Pial", "Yifang Tian", "Lily Gniedziejko", "Hans-Arno Jacobsen", "Yinfang Chen", "Tianyin Xu"] venue = "NeurIPS Evaluations and Datasets Track (accepted; forthcoming)" publication_type = "Conference Paper" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2605.07161" abstract = "AI agents are increasingly used to diagnose and mitigate failures in production systems, known as agentic Site Reliability Engineering (SRE). Current SRE benchmarks are limited to oversimplistic SRE tasks and are unfortunately hard to extend due to bespoke designs. We present SREGym, a high-fidelity benchmark for SRE agents. SREGym exposes a live system environment built atop real-world cloud-native system stacks, where high-fidelity failure scenarios are simulated through fault injectors. SREGym models the complexity of production environments by simulating (1) a wide range of faults at different layers, (2) various ambient noises, and (3) diverse failure modes such as metastable failures and correlated failures. SREGym is architected as a modular, extensible framework that orchestrates fault and noise injectors across stacks. SREGym currently includes 90 realistic, challenging SRE problems. We use SREGym to evaluate frontier agents and show that their capabilities varies significantly in addressing different kinds of failures, with up to 40% differences in end-to-end results. SREGym is actively maintained as an open-source project and has been used by researchers and practitioners." +++ diff --git a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md index 4c2f7d4..c46f316 100644 --- a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md +++ b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md @@ -3,10 +3,13 @@ title = "AgentChaosBench: Benchmarking External Fault Detection and Localization slug = "when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry" year = 2026 authors = ["Chenkai Zhang", "Yiran Li", "Yifang Tian", "Michail Bachras", "Hans-Arno Jacobsen"] -venue = "International Workshop on Trustworthy and Responsible Autonomous Systems (TRUST)" +venue = "Proceedings of the 1st International Workshop on Trustworthy and Responsible aUtonomous SysTems (TRUST '26), pp. 75–82" publication_type = "Workshop Paper" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2608.14680" +tags = ["AgentChaosBench", "When Agentic Executions Fail: Detecting and Localizing Runtime Faults from Telemetry"] +# Keep the earlier manuscript title searchable after updating the citation. +# ACM publisher metadata: https://api.crossref.org/works/10.1145/3843782.3844662 abstract = "Reliability in LLM-based agentic systems is a property of the whole execution (its tool calls, model calls, guardrails, and inter-agent messages), not of the final answer alone, yet evaluating only task outcomes reveals little about how or why a run fails. We present AGENTCHAOSBENCH, a benchmark for detecting and localizing runtime faults in agentic systems from their execution telemetry. We run five heterogeneous applications that coordinate agents over the Agent-to-Agent protocol and call tools through the Model Context Protocol, and inject ten types of operational fault (unavailable or slow tools, corrupted or oversized responses, and delayed, looped, or misrouted delegations and bypassed guardrails) at their tool, model, guardrail, and inter-agent boundaries, alongside a no-fault control. The resulting dataset contains 275 sanitized traces: 250 faulty executions spanning ten fault types and 25 no-fault controls. Each faulty trace is aligned with the no-fault execution of the same input; fault-type labels and, where applicable, location labels are held out from diagnosis. On structured single-trace inputs, a first set of zero-shot LLM baselines shows the task is far from solved: local detectors up to 14B parameters reach only 13.6-19.2% top-1 fault-type accuracy and the frontier DeepSeek-v4-pro only 24.8%, while jointly identifying the fault type and its location tops out at 22%; reference-dependent faults (above all a bypassed guardrail) stay near-unsolved from a single trace. An aligned reference improves selected relative faults but does not resolve guardrail bypass. The held-out labels and compact prediction format support reproducible comparison of LLM-based and non-LLM diagnosis methods." +++ diff --git a/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md index 7919d71..c40d2c0 100644 --- a/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md +++ b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md @@ -2,10 +2,11 @@ title = "When Log Parsing Removes What Matters: Detecting Failures in Distributed GPU Workloads" year = 2026 authors = ["Boyuan Cui", "Yuyang Zeng", "Yifang Tian", "Hans-Arno Jacobsen"] -venue = "1st International Workshop on Reliable and trustworthy Automated Software Engineering (RASE), pp. 60–63" +venue = "Proceedings of the 1st International Workshop on Reliable and trustworthy Automated Software Engineering (RASE '26), pp. 60–63" publication_type = "Workshop Paper" -research = ["distributed-machine-learning", "rca-aiops"] +research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2610.14009" +# ACM publisher metadata: https://api.crossref.org/works/10.1145/3820756.3844919 +++ [Published version](https://doi.org/10.1145/3820756.3844919). diff --git a/content/research/_index.md b/content/research/_index.md index b1a42e8..55a99bf 100644 --- a/content/research/_index.md +++ b/content/research/_index.md @@ -1,5 +1,5 @@ +++ title = "Research" layout = "overview" -summary = "Four active research themes connect MSRG's work in data management, distributed machine learning, quantum computing, and reliable systems." +summary = "Three research themes connect MSRG's work in data management, root cause analysis and AIOps, and quantum computing systems, including continuing work in distributed AI and machine learning." +++ diff --git a/content/research/data-management/_index.md b/content/research/data-management/_index.md index 88c67a9..895d803 100644 --- a/content/research/data-management/_index.md +++ b/content/research/data-management/_index.md @@ -20,7 +20,7 @@ can repeat computations and produce the same intermediate data. Our approach detects these overlaps online and coordinates execution so queries can share work and improve throughput. -Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research examines the +Our [Root Cause Analysis & AIOps](/research/distributed-machine-learning/) research examines the operational reliability of cloud-native systems, using metrics, logs, traces, and service dependencies to support fault diagnosis, incident response, and automated repair. diff --git a/content/research/distributed-machine-learning/_index.md b/content/research/distributed-machine-learning/_index.md index d1b8fbd..52837c9 100644 --- a/content/research/distributed-machine-learning/_index.md +++ b/content/research/distributed-machine-learning/_index.md @@ -1,31 +1,102 @@ +++ -title = "Distributed Machine Learning" +title = "Root Cause Analysis & AIOps" +slug = "distributed-machine-learning" +aliases = ["/research/rca-aiops/"] +legacy_research_ids = ["rca-aiops"] layout = "area" +list_all_publications = true weight = 20 -accent = "sage" -question = "How can AI help diagnose failures and improve the reliability of distributed systems?" -blurb = """MSRG develops **AI for IT operations (AIOps)** to help engineers detect anomalies, identify root causes, and respond to incidents in distributed systems. We combine **metrics, logs, and traces**, service dependency graphs, and LLM agents to ground diagnosis and repair in operational evidence. Our research includes the reliability of agentic systems and realistic benchmarks for evaluating diagnosis and incident response. These efforts build on our broader work in distributed AI and machine learning, including scalable training, federated learning, and graph processing.""" +accent = "teal" +question = "How can we turn operational telemetry into reliable diagnosis and actionable incident response?" +summary = "Root cause analysis, anomaly detection, and telemetry-grounded repair for reliable microservices and AI agents, alongside distributed AI and machine learning." +publication_blurb = "Publications in root cause analysis, AIOps, and our broader distributed AI and machine learning research." +team_blurb = "Current and visiting MSRG members associated with our AIOps and distributed AI/ML research are shown below." +blurb = """MSRG develops methods for **root cause analysis** and **AI for IT operations (AIOps)** in distributed systems. We combine **metrics, logs, and traces** with service dependency graphs and LLM agents to investigate failures and produce actionable diagnoses. Our work connects anomaly detection, incident response, automated repair, and realistic evaluation, with a focus on microservices and the reliability of AI agent executions.""" +# Research descriptions are based on the publication abstracts linked below. +acknowledgement = "We acknowledge the support of Bell Mobility and the Natural Sciences and Engineering Research Council of Canada (NSERC) under grant ALLRP614128-25." +supporters = [ + { name = "Bell Mobility", url = "https://www.bell.ca/Mobility", logo = "/images/support/bell.svg", width = 100, height = 60 }, + { name = "Natural Sciences and Engineering Research Council of Canada (NSERC)", url = "https://nserc-crsng.canada.ca/", logo = "/images/support/nserc.png", width = 145, height = 70 } +] +++ -Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research investigates how -learning-based methods can help operate complex distributed systems reliably. +## From telemetry to root causes -## AIOps and incident response +A failure in one service can surface as errors, slowdowns, or unusual resource +usage elsewhere. Root cause analysis (RCA) connects these symptoms to the +underlying fault. Doing so requires reasoning across heterogeneous telemetry +and the dependencies between services. -We study anomaly detection, graph-guided root cause analysis, and -telemetry-grounded repair. By connecting operational signals to service -dependencies, we aim to produce diagnoses and response recommendations that -engineers can inspect and act on. +We study how AI for IT operations (AIOps) can help engineers move from detecting +an anomaly to understanding a failure and deciding what to do next. Our research +brings together statistical analysis, graph structure, and LLM agents, with an +emphasis on diagnoses and repair decisions grounded in operational evidence. -AI agents also introduce new operational failure modes. Our benchmarks study -fault detection and localization in agent executions, as well as diagnosis and -repair in live cloud-native environments. +
+ +{{< figure src="/images/research/aiops-vision.svg" alt="AIOps research vision: observe distributed systems, detect anomalies, diagnose causes, and support incident response, with realistic benchmark evaluation." caption="Research vision: connect operational evidence to diagnosis and response, and evaluate each step in realistic systems." >}} + +{{< figure src="/images/research/aiops-technical.svg" alt="GALA+ workflow: telemetry and service dependencies feed initial hypotheses, followed by graph-guided investigation using local evidence, producing diagnoses and incident response recommendations evaluated with SURE-Score." caption="Technical example: GALA+ uses service dependencies and local telemetry to guide LLM investigation; SURE-Score assesses the quality of its outputs." >}} + +
+ +## Graph-guided diagnosis and incident response + +[GALA](/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices/), +whose updated framework is called GALA+, uses service dependencies to guide +LLM agents through local metrics, logs, and traces. It produces ranked diagnoses, +incident summaries, and action recommendations. SURE-Score, developed with +industry site reliability engineers, assesses diagnostic quality beyond text +similarity. + +## Message-level anomaly detection + +Effective investigation starts with finding useful signals in large volumes of +telemetry. +[FAME](/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection/) +studies anomaly detection at the level of individual log messages. It uses an +LLM during offline preparation to help organize failure domains, then trains +a lightweight router and specialized experts for online detection with limited +labeling effort. + +## Telemetry-grounded repair + +Diagnosis can also provide context for changing code and configuration. +[ORCA](/publications/orca-observability-grounded-program-repair-for-microservice-incidents/) +compares failure and reference telemetry to build a fault signature, locate +candidate repair sites, and guide patch generation. Its verification process +examines patch correctness, test integrity, and telemetry replay to connect +automated repair with the incident that motivated it. + +## Reliable agents and realistic evaluation + +AI agents are both tools for incident response and systems whose own failures +need diagnosis. +[AgentChaosBench](/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry/) +studies fault detection and localization from agent +execution telemetry, including faults at tool, model, guardrail, and inter-agent +boundaries. + +[SREGym](/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios/) +evaluates SRE agents in live cloud-native environments with injected faults and +ambient noise. These complementary settings help us examine whether an agent +can identify a fault, locate its origin, and support incident resolution under +realistic operating conditions. + +[GF-Bench](/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads/) +extends failure detection to distributed GPU workloads, while +[SRE-Marathon](/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents/) +studies continuous operation with overlapping incidents and code or configuration changes. ## Current directions -- Anomaly detection from metrics, logs, and traces -- Graph-guided diagnosis and telemetry-grounded repair -- Reliable AI agents and realistic incident-response benchmarks +- Correlating metrics, logs, and traces across service dependencies +- Graph-guided and evidence-grounded LLM investigation +- Connecting root cause diagnosis to remediation and automated repair +- Evaluating diagnostic quality, agent reliability, and operational usefulness -We also continue research in distributed AI and machine learning, including -efficient training and inference, federated learning, and scalable graph processing. +This research connects with our work in [Data Management](/research/data-management/). +We also continue research in **distributed AI and machine learning**, including +resource allocation and scheduling for training and inference, privacy-preserving +and federated learning on embedded and edge devices, and scalable graph processing +and graph neural networks. diff --git a/content/research/rca-aiops/_index.md b/content/research/rca-aiops/_index.md deleted file mode 100644 index 3f60359..0000000 --- a/content/research/rca-aiops/_index.md +++ /dev/null @@ -1,105 +0,0 @@ -+++ -title = "Root Cause Analysis & AIOps" -slug = "rca-aiops" -layout = "area" -list_all_publications = true -weight = 40 -accent = "teal" -question = "How can we turn operational telemetry into reliable diagnosis and actionable incident response?" -summary = "Root cause analysis, anomaly detection, and telemetry-grounded repair for reliable microservices and AI agents." -blurb = """MSRG develops methods for **root cause analysis** and **AI for IT operations (AIOps)** in distributed systems. We combine **metrics, logs, and traces** with service dependency graphs and LLM agents to investigate failures and produce actionable diagnoses. Our work connects anomaly detection, incident response, automated repair, and realistic evaluation, with a focus on microservices and the reliability of AI agent executions.""" -# Research descriptions are based on the publication abstracts linked below. -acknowledgement = "We acknowledge the support of Bell Mobility and the Natural Sciences and Engineering Research Council of Canada (NSERC) under grant ALLRP614128-25." -supporters = [ - { name = "Bell Mobility", url = "https://www.bell.ca/Mobility", logo = "/images/support/bell.svg", width = 100, height = 60 }, - { name = "Natural Sciences and Engineering Research Council of Canada (NSERC)", url = "https://nserc-crsng.canada.ca/", logo = "/images/support/nserc.png", width = 145, height = 70 } -] -+++ - -## From telemetry to root causes - -A failure in one service can surface as errors, slowdowns, or unusual resource -usage elsewhere. Root cause analysis (RCA) connects these symptoms to the -underlying fault. Doing so requires reasoning across heterogeneous telemetry -and the dependencies between services. - -We study how AI for IT operations (AIOps) can help engineers move from detecting -an anomaly to understanding a failure and deciding what to do next. Our research -brings together statistical analysis, graph structure, and LLM agents, with an -emphasis on diagnoses and repair decisions grounded in operational evidence. - -
- -{{< figure src="/images/research/aiops-vision.svg" alt="AIOps research vision: observe distributed systems, detect anomalies, diagnose causes, and support incident response, with realistic benchmark evaluation." caption="Research vision: connect operational evidence to diagnosis and response, and evaluate each step in realistic systems." >}} - -{{< figure src="/images/research/aiops-technical.svg" alt="GALA+ workflow: telemetry and service dependencies feed initial hypotheses, followed by graph-guided investigation using local evidence, producing diagnoses and incident response recommendations evaluated with SURE-Score." caption="Technical example: GALA+ uses service dependencies and local telemetry to guide LLM investigation; SURE-Score assesses the quality of its outputs." >}} - -
- -## Graph-guided diagnosis and incident response - -[GALA](/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices/), -whose updated framework is called GALA+, uses service dependencies to guide -LLM agents through local metrics, logs, and traces. It produces ranked diagnoses, -incident summaries, and action recommendations. SURE-Score, developed with -industry site reliability engineers, assesses diagnostic quality beyond text -similarity. - -## Message-level anomaly detection - -Effective investigation starts with finding useful signals in large volumes of -telemetry. -[FAME](/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection/) -studies anomaly detection at the level of individual log messages. It uses an -LLM during offline preparation to help organize failure domains, then trains -a lightweight router and specialized experts for online detection with limited -labeling effort. - -## Telemetry-grounded repair - -Diagnosis can also provide context for changing code and configuration. -[ORCA](/publications/orca-observability-grounded-program-repair-for-microservice-incidents/) -compares failure and reference telemetry to build a fault signature, locate -candidate repair sites, and guide patch generation. Its verification process -examines patch correctness, test integrity, and telemetry replay to connect -automated repair with the incident that motivated it. - -## Reliable agents and realistic evaluation - -AI agents are both tools for incident response and systems whose own failures -need diagnosis. -[AgentChaosBench](/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry/) -studies fault detection and localization from agent -execution telemetry, including faults at tool, model, guardrail, and inter-agent -boundaries. - -[SREGym](/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios/) -evaluates SRE agents in live cloud-native environments with injected faults and -ambient noise. These complementary settings help us examine whether an agent -can identify a fault, locate its origin, and support incident resolution under -realistic operating conditions. - -[GF-Bench](/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads/) -extends failure detection to distributed GPU workloads, while -[SRE-Marathon](/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents/) -studies continuous operation with overlapping incidents and code or configuration changes. - -## Current directions - -- Correlating metrics, logs, and traces across service dependencies -- Graph-guided and evidence-grounded LLM investigation -- Connecting root cause diagnosis to remediation and automated repair -- Evaluating diagnostic quality, agent reliability, and operational usefulness - -This research connects our work in -[Data Management](/research/data-management/) and -[Distributed Machine Learning](/research/distributed-machine-learning/). - -## Outcomes - -- **[AgentChaosBench](https://github.com/kevinzck8k/agentic-fault-diagnosis):** benchmark and diagnosis tools for faults in agentic executions. -- **[FAME](https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts):** implementation for message-level log anomaly detection. -- **[GALA evaluation materials](https://github.com/EvanTian233/GALA-for-RCA):** human-annotated evaluation data, sample incident summaries and remediation recommendations, and third-party baselines for microservice root cause analysis. -- **[GF-Bench](https://github.com/cuiboyuan/nccl_log_rca_bench):** fault injection and log data for distributed GPU workloads. -- **[SREGym](https://github.com/SREGym/SREGym):** live environments and fault scenarios for evaluating SRE agents. -- **[SRE-Marathon](https://bigbigbai.github.io/sre-marathon-leaderboard/):** project website and leaderboard, with [website source on GitHub](https://github.com/bigbigbai/sre-marathon-leaderboard). diff --git a/data/research_outcomes.toml b/data/research_outcomes.toml new file mode 100644 index 0000000..abd6b38 --- /dev/null +++ b/data/research_outcomes.toml @@ -0,0 +1,58 @@ +# Public artifacts are attributed by the linked papers or official author pages. +# Keep citation metadata in content/publications; these entries describe outputs. + +[[items]] +id = "sregym" +title = "SREGym" +url = "https://github.com/SREGym/SREGym" +summary = "A live cloud-native benchmark for evaluating how SRE agents diagnose and mitigate injected operational failures." +research = ["distributed-machine-learning"] +publication = "sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios" +# Attribution: https://arxiv.org/abs/2605.07161 and https://tianyin.github.io/pub.html + +[[items]] +id = "agentchaosbench" +title = "AgentChaosBench" +url = "https://github.com/kevinzck8k/agentic-fault-diagnosis" +summary = "Fault-detection and localization evaluation tools with 275 execution traces from five agentic applications." +research = ["distributed-machine-learning"] +publication = "when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry" +# Attribution: https://arxiv.org/abs/2608.14680 and the authors' project links. + +[[items]] +id = "fame" +title = "FAME" +url = "https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts" +summary = "Message-level log anomaly detection using offline failure-domain grouping and a lightweight router with specialized experts." +research = ["distributed-machine-learning"] +publication = "fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection" +# Attribution: repository URL in the paper, https://arxiv.org/abs/2605.22779 + +[[items]] +id = "gala-evaluation" +title = "GALA evaluation artifacts" +url = "https://github.com/EvanTian233/GALA-for-RCA" +summary = "Annotated incident reports, example outputs, a supplementary technical report, and third-party baseline frameworks. The core GALA implementation is not publicly released." +research = ["distributed-machine-learning"] +publication = "gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices" +# Attribution: https://arxiv.org/abs/2608.08968; repository README describes release scope. + +[[items]] +id = "gf-bench" +title = "GF-Bench artifacts" +url = "https://github.com/cuiboyuan/nccl_log_rca_bench" +summary = "Tools for generating multi-GPU failures, collecting NCCL logs, and evaluating anomaly detection, with sample data and scripts to reproduce the experiments." +research = ["distributed-machine-learning"] +publication = "when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads" +# Attribution: data availability statement, https://arxiv.org/abs/2610.14009 +# No top-level repository license was present when checked on 2026-10-22. + +[[items]] +id = "sre-marathon" +title = "SRE-Marathon project website and leaderboard" +url = "https://bigbigbai.github.io/sre-marathon-leaderboard/" +summary = "Project website and paper-reported leaderboard for continuous, change-driven SRE evaluation. Its GitHub repository contains the website source, not the benchmark implementation." +research = ["distributed-machine-learning"] +publication = "sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents" +# Attribution: https://arxiv.org/abs/2609.33023 +# Website source and scope: https://github.com/bigbigbai/sre-marathon-leaderboard/blob/main/README.md diff --git a/docs/aiops-sources.md b/docs/aiops-sources.md index 2b74f54..aef54de 100644 --- a/docs/aiops-sources.md +++ b/docs/aiops-sources.md @@ -4,8 +4,11 @@ Checked 2026-11-01. People and portraits come from `content/people/` through the same roster and card partials as Quantum Computing Systems. Publications come from `content/publications/`; the AIOps page displays all matching records. The acknowledgement text and grant number were supplied by the project team. -The page retains the shared research layout, with People before Publications, -as requested by the maintainer. +The consolidated page uses the shared research layout, with Outcomes followed +by Publications and then People, as requested in Arno's email. The former RCA +page redirects to the refocused Distributed Machine Learning area. Existing +RCA research tags were folded into its stable `distributed-machine-learning` +identifier without adding unrelated members or papers. ## Yuanchen Gao diff --git a/hugo.yaml b/hugo.yaml index 2ac0917..4637a9a 100644 --- a/hugo.yaml +++ b/hugo.yaml @@ -42,7 +42,7 @@ permalinks: data-sets: /data-sets/:slug/ params: - description: Systems research at the University of Toronto spanning data management, distributed machine learning, quantum computing systems, and root cause analysis and AIOps. + description: Systems research at the University of Toronto spanning data management, root cause analysis and AIOps, and quantum computing, with continuing work in distributed AI and machine learning. # Public contact: https://www.qscc.ca/open-positions/ contact_email: jacobsen@eecg.utoronto.ca contact_phone: "+1 416 946 7586" @@ -71,7 +71,7 @@ menu: parent: research pageRef: /research/quantum-computing-systems weight: 21 - - name: Distributed Machine Learning + - name: Root Cause Analysis & AIOps parent: research pageRef: /research/distributed-machine-learning weight: 22 @@ -79,10 +79,6 @@ menu: parent: research pageRef: /research/data-management weight: 23 - - name: Root Cause Analysis & AIOps - parent: research - pageRef: /research/rca-aiops - weight: 24 - identifier: people name: People pageRef: /people diff --git a/layouts/index.html b/layouts/index.html index 09ec66f..b7f343f 100644 --- a/layouts/index.html +++ b/layouts/index.html @@ -74,7 +74,7 @@

News

Research Themes

-

Four active tracks with room for overlap

+

Three active tracks with room for overlap

{{ range $researchSection.Pages.ByWeight }} diff --git a/layouts/partials/footer.html b/layouts/partials/footer.html index b7b5fef..fb2fada 100644 --- a/layouts/partials/footer.html +++ b/layouts/partials/footer.html @@ -6,8 +6,8 @@

- Research on data systems, distributed learning infrastructure, and - quantum computing systems at the University of Toronto. + Research on data systems, root cause analysis and AIOps, quantum computing, + and distributed learning infrastructure at the University of Toronto.

diff --git a/layouts/publications/archive.html b/layouts/publications/archive.html index 1a71193..41d9eb3 100644 --- a/layouts/publications/archive.html +++ b/layouts/publications/archive.html @@ -16,7 +16,7 @@

{{ .Title }}

{{ range (site.GetPage "/research").Pages.ByWeight }} - + {{ end }}
diff --git a/layouts/research/area.html b/layouts/research/area.html index 36076ee..1cd8b86 100644 --- a/layouts/research/area.html +++ b/layouts/research/area.html @@ -3,7 +3,10 @@ {{ $roster := partialCached "people/roster.html" site }} {{ $people := where $roster.current "Params.research" "intersect" (slice $slug) }} {{ $publications := where (site.GetPage "/publications").RegularPages "Params.research" "intersect" (slice $slug) }} + {{ $listedPublications := sort $publications "Params.year" "desc" }} + {{ if not .Params.list_all_publications }}{{ $listedPublications = first 6 $listedPublications }}{{ end }} {{ $datasets := where (site.GetPage "/data-sets").RegularPages "Params.research" "intersect" (slice $slug) }} + {{ $outcomes := where (hugo.Data.research_outcomes.items | default slice) "research" "intersect" (slice $slug) }}
@@ -21,12 +24,12 @@

{{ .Title }}

{{ end }}

- {{ len $people }} {{ if eq (len $people) 1 }}researcher{{ else }}researchers{{ end }} + {{ len $people }} {{ if eq (len $people) 1 }}researcher{{ else }}researchers{{ end }} in this area · {{ len $publications }} {{ if eq (len $publications) 1 }}publication{{ else }}publications{{ end }} · {{ len $datasets }} {{ if eq (len $datasets) 1 }}data set{{ else }}data sets{{ end }}

{{ if .Params.list_all_publications }} -

Outcomes · People · Publications

+

Outcomes · Publications · People

{{ end }}
@@ -39,37 +42,50 @@

{{ .Title }}

-
+ {{ if or $outcomes $datasets }} +
- {{ if .Params.list_all_publications }}

People

{{ end }} - {{ partial "people/group-cards.html" $people }} - +
+

Research outcomes

+
+
+ {{ range $outcomes }} + + {{ end }} + {{ range sort $datasets "Title" }} + + {{ end }} +
+ {{ end }}
-

Publications

+

{{ if lt (len $listedPublications) (len $publications) }}Recent publications{{ else }}Publications{{ end }}

+ {{ with .Params.publication_blurb }}

{{ . }}

{{ end }}
{{ if $publications }} - {{ $listed := first 6 (sort $publications "Params.year" "desc") }} - {{ if .Params.list_all_publications }}{{ $listed = sort $publications "Params.year" "desc" }}{{ end }} - {{ range $listed }} - {{ if $.Params.list_all_publications }} -
- {{ .Title }} - {{ partial "publication-authors.html" (dict "authors" .Params.authors) }} - {{ .Params.venue }}, {{ .Params.year }} - {{ with .Params.external_url }}Read paper{{ end }} + {{ range $listedPublications }} +
+

{{ partial "publication-authors.html" (dict "authors" .Params.authors) }}

+

{{ .Title }}

+

{{ .Params.venue }}, {{ .Params.year }}

+ {{ with .Params.external_url }} +

Read paper

+ {{ end }}
- {{ else }} - - {{ .Title }} - {{ .Params.venue }}, {{ .Params.year }} - - {{ end }} {{ end }} {{ else }}

No publications are listed for this research area yet.

@@ -79,22 +95,17 @@

Publications

- {{ if $datasets }} -
+
-

Data sets

-
-
- {{ range sort $datasets "Title" }} - - {{ end }} +

Researchers in this area

+

{{ with .Params.team_blurb }}{{ . }}{{ else }}Current and visiting MSRG members associated with {{ .Title }} are shown below.{{ end }} + View the full MSRG team.

+ + {{ partial "people/group-cards.html" $people }} +
- {{ end }} {{ end }} diff --git a/scripts/browser_check.cjs b/scripts/browser_check.cjs index 0c09051..264277b 100644 --- a/scripts/browser_check.cjs +++ b/scripts/browser_check.cjs @@ -173,11 +173,20 @@ let browser; await search.fill('Grier 2025'); await waitForSearch(); assert.equal(await visible.count(), 1); await search.fill('Woisetschlager'); await waitForSearch(); assert(await visible.count() > 0); await search.fill('Michael Dang’ana'); await waitForSearch(); assert(await visible.count() > 0); + for (const [formerTitle, slug] of [ + ['When Agentic Executions Fail: Detecting and Localizing Runtime Faults from Telemetry', 'when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry'], + ['GALA: Can Graph-Augmented Large Language Model Agentic Workflows Elevate Root Cause Analysis?', 'gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices'], + ]) { + await search.fill(formerTitle); await waitForSearch(); + assert.equal(await visible.count(), 1, 'The full former title must find its consolidated publication'); + assert.equal(await visible.locator('h2 a').getAttribute('href'), '/publications/' + slug + '/'); + } await search.fill('no-result-xyz'); await waitForSearch(); assert.equal(await visible.count(), 0); assert(await page.locator('[data-publication-empty]').isVisible()); await page.locator('[data-publication-clear]').click(); await waitForSearch(); assert.equal(await visible.count(), pageCount); const publicationIndex = JSON.parse(fs.readFileSync(path.join(root, 'publications/index.json'), 'utf8')); const researchButtons = page.getByRole('group', { name: 'Research area', exact: true }).getByRole('button'); + assert.equal(await researchButtons.count(), 4, 'One All button and exactly three research areas'); assert.equal(await page.locator('select[data-publication-research], [data-publication-tag]').count(), 0); for (const button of await researchButtons.all()) { await button.click(); @@ -213,6 +222,65 @@ let browser; await page.getByRole('link', { name: 'Browse all publications in this area' }).click(); await waitForSearch(); assert.equal(await visible.count(), quantumCount); + const aiopsRoute = '/research/distributed-machine-learning/'; + const aiopsPublications = publicationIndex.filter(record => record.research.includes('distributed-machine-learning')); + for (const route of ['/', '/research/']) { + await page.goto(base + route); + const themeLinks = page.locator('.card-grid-wide .card-readmore'); + assert.equal(await themeLinks.count(), 3, 'Exactly three research areas'); + const areaURLs = await themeLinks.evaluateAll(links => links.map(link => link.getAttribute('href')).sort()); + const menuURLs = await page.locator('#site-nav .nav-submenu a[href^="/research/"]').evaluateAll(links => links.map(link => link.getAttribute('href')).sort()); + assert.deepEqual(menuURLs, areaURLs, 'Every research area must be reachable from the header'); + assert.deepEqual(await page.locator('.site-footer .footer-list').first().locator('a').evaluateAll(links => links.map(link => link.getAttribute('href')).sort()), areaURLs); + } + for (const width of [1440, 390, 320]) { + await page.setViewportSize({ width, height: 900 }); + await page.goto(base + aiopsRoute); + assert.equal(await page.locator('h1').textContent(), 'Root Cause Analysis & AIOps'); + assert.deepEqual(await page.locator('.research-area > section .section-heading h2').allTextContents(), + ['Research outcomes', 'Publications', 'Researchers in this area']); + assert.deepEqual(await page.locator('#publications .list-card h3 a').evaluateAll(links => links.map(link => link.getAttribute('href'))), + aiopsPublications.map(record => record.url), 'The unified page must show all matching master references exactly once'); + assert.equal(await page.locator('#outcomes .card').count(), 6); + assert.equal(await page.locator('.research-support-logos img').count(), 2); + assert.equal(await page.locator('.research-diagrams img').count(), 2); + assert(await page.locator('.page-hero').textContent().then(text => text.includes('grant ALLRP614128-25'))); + assert(await page.getByRole('link', { name: 'View the full MSRG team', exact: true }).isVisible()); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth + 1), false); + if (width !== 320) { + await page.screenshot({ path: path.join(output, 'aiops-' + width + '.png') }); + for (const section of ['.research-diagrams', '#outcomes']) { + await page.locator(section).screenshot({ path: path.join(output, 'aiops-' + section.replace(/^[.#]/, '') + '-' + width + '.png'), + style: '.site-header { visibility: hidden !important; }' }); + } + } + } + // Hugo aliases can use the production origin. Local previews use a relative baseURL. + const alias = fs.readFileSync(path.join(root, 'research/rca-aiops/index.html'), 'utf8'); + assert.match(alias, /http-equiv=["']?refresh/i); + const aliasTarget = alias.match(/content=["']?[^>]*?url=([^"' >]+)/i)?.[1]; + assert.equal(new URL(aliasTarget, base).pathname, aiopsRoute, 'The old RCA page must redirect to the single area'); + if (new URL(aliasTarget, base).origin === new URL(base).origin) { + await page.goto(base + '/research/rca-aiops/'); + await page.waitForURL(base + aiopsRoute); + } + for (const research of ['rca-aiops', 'distributed-machine-learning']) { + await page.goto(base + '/publications/?research=' + research); await waitForSearch(); + assert.equal(new URL(page.url()).searchParams.get('research'), 'distributed-machine-learning'); + assert.deepEqual(await visible.locator('h2 a').evaluateAll(links => links.map(link => link.getAttribute('href'))), + aiopsPublications.map(record => record.url), 'Old and new archive links must select the same combined area'); + } + await page.goto(base + '/publications/?research=rca-aiops&q=GALA&page=1'); await waitForSearch(); + assert.equal(new URL(page.url()).searchParams.get('q'), 'GALA'); + assert.equal(new URL(page.url()).searchParams.get('page'), '1'); + const oldQueryMatches = await visible.locator('h2 a').allTextContents(); + assert(oldQueryMatches.length > 0); + await page.goto(base + '/publications/?research=quantum-computing-systems'); await waitForSearch(); + await page.goBack(); await waitForSearch(); + assert.deepEqual(await visible.locator('h2 a').allTextContents(), oldQueryMatches, 'Back must restore the canonicalized legacy query'); + await page.reload(); await waitForSearch(); + assert.deepEqual(await visible.locator('h2 a').allTextContents(), oldQueryMatches); + await page.setViewportSize({ width: 390, height: 667 }); await page.goto(base + '/people/'); assert.equal(await page.locator('.person-card h3 a, .alumni-list li > span:first-child a').count(), 0, 'Member names must be plain text'); // Snap at group starts; oversized groups must not create bottom-edge stops. @@ -315,7 +383,7 @@ let browser; await page.keyboard.type('msrg', { delay: 90 }); assert.equal(await page.locator('[data-msrg-rounds]').textContent(), '2', 'Hardware keyboard must complete a round'); - for (const route of ['/', '/people/', '/publications/', '/research/quantum-computing-systems/', '/data-sets/begen/', '/~thomas-trenty/', '/data-sets/']) { + for (const route of ['/', '/people/', '/publications/', '/research/quantum-computing-systems/', '/research/distributed-machine-learning/', '/data-sets/begen/', '/~thomas-trenty/', '/data-sets/']) { await page.goto(base + route); await page.addScriptTag({ path: require.resolve('axe-core/axe.min.js') }); const results = await page.evaluate(async () => axe.run(document, { runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'wcag21aa', 'best-practice'] } })); diff --git a/tests/test_content_contracts.py b/tests/test_content_contracts.py index 4839571..7f534b2 100644 --- a/tests/test_content_contracts.py +++ b/tests/test_content_contracts.py @@ -2,6 +2,7 @@ import re import sys +import tomllib import unittest from pathlib import Path from urllib.parse import urlsplit @@ -74,6 +75,8 @@ def test_research_and_related_content_references_exist(self) -> None: with self.subTest(path=path): for area in data.get("research", []): self.assertIn(area, areas) + self.assertEqual(len(data.get("research", [])), len(set(data.get("research", []))), + "A record must not repeat the consolidated research area") for dataset in data.get("related_datasets", []): self.assertIn(dataset, datasets) if data.get("related_publication"): @@ -90,6 +93,26 @@ def test_research_and_related_content_references_exist(self) -> None: self.assertIn(parsed.scheme, {"http", "https"}) self.assertTrue(parsed.netloc) + def test_research_outcomes_reference_master_records(self) -> None: + areas = {path.parent.name: load_toml_front_matter(path) + for path in (ROOT / "content/research").glob("*/_index.md")} + publications = {path.stem for path, _ in iter_publication_pages()} + legacy_ids = [alias for data in areas.values() for alias in data.get("legacy_research_ids", [])] + self.assertEqual(len(legacy_ids), len(set(legacy_ids))) + self.assertFalse(set(legacy_ids).intersection(areas), "A legacy filter must not be a separate research area") + outcomes = tomllib.loads((ROOT / "data/research_outcomes.toml").read_text())["items"] + self.assertEqual(len({item["id"] for item in outcomes}), len(outcomes)) + for item in outcomes: + with self.subTest(outcome=item["id"]): + self.assertTrue(item["title"].strip()) + self.assertTrue(item["summary"].strip()) + self.assertTrue(item["research"]) + self.assertTrue(set(item["research"]).issubset(areas)) + self.assertIn(item["publication"], publications) + parsed = urlsplit(item["url"]) + self.assertEqual(parsed.scheme, "https") + self.assertTrue(parsed.netloc) + if __name__ == "__main__": unittest.main() From c93ce8cf71c70c41fc9e04b0106eba4640390cf7 Mon Sep 17 00:00:00 2001 From: ttrenty <154608953+ttrenty@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:11:14 -0500 Subject: [PATCH 05/15] Preserve section bookmarks when redirecting the RCA page --- layouts/alias.html | 25 +++++++++++++++++++++++++ scripts/browser_check.cjs | 21 +++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 layouts/alias.html diff --git a/layouts/alias.html b/layouts/alias.html new file mode 100644 index 0000000..7f40f43 --- /dev/null +++ b/layouts/alias.html @@ -0,0 +1,25 @@ +{{- if and .Page .Page.Params.legacy_research_ids -}} + + + + + {{ .Page.Title }} + + + + +

Continue to {{ .Page.Title }}

+ +{{- else -}} + + + + {{ .Permalink }} + + + + + +{{- end -}} diff --git a/scripts/browser_check.cjs b/scripts/browser_check.cjs index 264277b..5d2e201 100644 --- a/scripts/browser_check.cjs +++ b/scripts/browser_check.cjs @@ -261,8 +261,25 @@ let browser; const aliasTarget = alias.match(/content=["']?[^>]*?url=([^"' >]+)/i)?.[1]; assert.equal(new URL(aliasTarget, base).pathname, aiopsRoute, 'The old RCA page must redirect to the single area'); if (new URL(aliasTarget, base).origin === new URL(base).origin) { - await page.goto(base + '/research/rca-aiops/'); - await page.waitForURL(base + aiopsRoute); + for (const hash of ['', '#outcomes', '#publications', '#people']) { + await page.goto(base + '/research/rca-aiops/' + hash); + await page.waitForURL(base + aiopsRoute + hash); + if (hash) { + await page.waitForFunction(hash => { + const section = document.querySelector(hash); + const top = section.getBoundingClientRect().top; + return scrollY > 0 && top >= 0 && top < innerHeight; + }, hash); + } + } + await page.goto(base + '/research/rca-aiops/?source=bookmark#people'); + await page.waitForURL(base + aiopsRoute + '?source=bookmark#people'); + const aliasNoJSContext = await createContext({ javaScriptEnabled: false }); + const aliasNoJSPage = await aliasNoJSContext.newPage(); + await aliasNoJSPage.goto(base + '/research/rca-aiops/'); + await aliasNoJSPage.waitForURL(base + aiopsRoute); + assert.equal(await aliasNoJSPage.locator('h1').textContent(), 'Root Cause Analysis & AIOps'); + await aliasNoJSContext.close(); } for (const research of ['rca-aiops', 'distributed-machine-learning']) { await page.goto(base + '/publications/?research=' + research); await waitForSearch(); From 50b7b711ee718b95651e9f2d43e904d80b413d8b Mon Sep 17 00:00:00 2001 From: ttrenty <154608953+ttrenty@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:34:18 -0500 Subject: [PATCH 06/15] Limit research previews and use the RCA page URL --- README.md | 12 ++++--- content/research/data-management/_index.md | 2 +- .../distributed-machine-learning/_index.md | 8 ++--- docs/aiops-sources.md | 11 ++++--- layouts/research/area.html | 5 ++- scripts/browser_check.cjs | 33 +++++++++++++------ 6 files changed, 43 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c5c74bc..2345917 100644 --- a/README.md +++ b/README.md @@ -174,17 +174,19 @@ Link to an external repository for datasets larger than **20 MB**. Use GitHub's file editor on a branch or fork, then submit a PR. The **Root Cause Analysis & AIOps** area refocuses the existing Distributed -Machine Learning area. Its stable content identifier and URL remain -`distributed-machine-learning`; `/research/rca-aiops/` redirects there. Use -`distributed-machine-learning` in member, publication, dataset, and outcome +Machine Learning area. Its canonical URL is `/research/rca-aiops/`; +`/research/distributed-machine-learning/` redirects there, preserving section +bookmarks. Its stable content identifier remains `distributed-machine-learning`. +Use `distributed-machine-learning` in member, publication, dataset, and outcome research tags. The old `rca-aiops` publication-archive query is supported through the area's `legacy_research_ids` setting. It is not a separate research choice. All three research areas appear in the header, homepage, and research overview. The combined page includes AIOps/RCA work and earlier distributed AI/ML work. It reads people and publication references from the master collections, with -research outcomes before publications and the associated people. Add outcomes -only after verifying their public project or repository against a paper or an +research outcomes before publications and the associated people. Every research +page shows up to six recent publications, with a link to its complete archive. +Add outcomes only after verifying their public project or repository against a paper or an official author page. Each `data/research_outcomes.toml` item has a stable `id`, `title`, HTTPS `url`, short `summary`, `research` tags, and a master `publication` slug. Record attribution in a comment and describe the actual release scope. diff --git a/content/research/data-management/_index.md b/content/research/data-management/_index.md index 895d803..88c67a9 100644 --- a/content/research/data-management/_index.md +++ b/content/research/data-management/_index.md @@ -20,7 +20,7 @@ can repeat computations and produce the same intermediate data. Our approach detects these overlaps online and coordinates execution so queries can share work and improve throughput. -Our [Root Cause Analysis & AIOps](/research/distributed-machine-learning/) research examines the +Our [Root Cause Analysis & AIOps](/research/rca-aiops/) research examines the operational reliability of cloud-native systems, using metrics, logs, traces, and service dependencies to support fault diagnosis, incident response, and automated repair. diff --git a/content/research/distributed-machine-learning/_index.md b/content/research/distributed-machine-learning/_index.md index 52837c9..c6814dc 100644 --- a/content/research/distributed-machine-learning/_index.md +++ b/content/research/distributed-machine-learning/_index.md @@ -1,15 +1,15 @@ +++ title = "Root Cause Analysis & AIOps" -slug = "distributed-machine-learning" -aliases = ["/research/rca-aiops/"] +slug = "rca-aiops" +aliases = ["/research/distributed-machine-learning/"] legacy_research_ids = ["rca-aiops"] layout = "area" -list_all_publications = true +show_section_links = true weight = 20 accent = "teal" question = "How can we turn operational telemetry into reliable diagnosis and actionable incident response?" summary = "Root cause analysis, anomaly detection, and telemetry-grounded repair for reliable microservices and AI agents, alongside distributed AI and machine learning." -publication_blurb = "Publications in root cause analysis, AIOps, and our broader distributed AI and machine learning research." +publication_blurb = "Recent publications in root cause analysis, AIOps, and our broader distributed AI and machine learning research." team_blurb = "Current and visiting MSRG members associated with our AIOps and distributed AI/ML research are shown below." blurb = """MSRG develops methods for **root cause analysis** and **AI for IT operations (AIOps)** in distributed systems. We combine **metrics, logs, and traces** with service dependency graphs and LLM agents to investigate failures and produce actionable diagnoses. Our work connects anomaly detection, incident response, automated repair, and realistic evaluation, with a focus on microservices and the reliability of AI agent executions.""" # Research descriptions are based on the publication abstracts linked below. diff --git a/docs/aiops-sources.md b/docs/aiops-sources.md index aef54de..8c74275 100644 --- a/docs/aiops-sources.md +++ b/docs/aiops-sources.md @@ -2,13 +2,14 @@ Checked 2026-11-01. People and portraits come from `content/people/` through the same roster and card partials as Quantum Computing Systems. Publications -come from `content/publications/`; the AIOps page displays all matching records. +come from `content/publications/`; the AIOps page displays up to six recent +records, matching the other research pages, and links to the complete archive. The acknowledgement text and grant number were supplied by the project team. The consolidated page uses the shared research layout, with Outcomes followed -by Publications and then People, as requested in Arno's email. The former RCA -page redirects to the refocused Distributed Machine Learning area. Existing -RCA research tags were folded into its stable `distributed-machine-learning` -identifier without adding unrelated members or papers. +by Publications and then People, as requested in Arno's email. The canonical +page is `/research/rca-aiops/`; the former Distributed Machine Learning URL +redirects there. Existing RCA research tags were folded into its stable +`distributed-machine-learning` identifier without adding unrelated members or papers. ## Yuanchen Gao diff --git a/layouts/research/area.html b/layouts/research/area.html index 1cd8b86..187dacc 100644 --- a/layouts/research/area.html +++ b/layouts/research/area.html @@ -3,8 +3,7 @@ {{ $roster := partialCached "people/roster.html" site }} {{ $people := where $roster.current "Params.research" "intersect" (slice $slug) }} {{ $publications := where (site.GetPage "/publications").RegularPages "Params.research" "intersect" (slice $slug) }} - {{ $listedPublications := sort $publications "Params.year" "desc" }} - {{ if not .Params.list_all_publications }}{{ $listedPublications = first 6 $listedPublications }}{{ end }} + {{ $listedPublications := first 6 (sort $publications "Params.year" "desc") }} {{ $datasets := where (site.GetPage "/data-sets").RegularPages "Params.research" "intersect" (slice $slug) }} {{ $outcomes := where (hugo.Data.research_outcomes.items | default slice) "research" "intersect" (slice $slug) }} @@ -28,7 +27,7 @@

{{ .Title }}

· {{ len $publications }} {{ if eq (len $publications) 1 }}publication{{ else }}publications{{ end }} · {{ len $datasets }} {{ if eq (len $datasets) 1 }}data set{{ else }}data sets{{ end }}

- {{ if .Params.list_all_publications }} + {{ if .Params.show_section_links }}

Outcomes · Publications · People

{{ end }} diff --git a/scripts/browser_check.cjs b/scripts/browser_check.cjs index 5d2e201..71dbd25 100644 --- a/scripts/browser_check.cjs +++ b/scripts/browser_check.cjs @@ -222,25 +222,38 @@ let browser; await page.getByRole('link', { name: 'Browse all publications in this area' }).click(); await waitForSearch(); assert.equal(await visible.count(), quantumCount); - const aiopsRoute = '/research/distributed-machine-learning/'; + const aiopsRoute = '/research/rca-aiops/'; + const previousAiopsRoute = '/research/distributed-machine-learning/'; const aiopsPublications = publicationIndex.filter(record => record.research.includes('distributed-machine-learning')); for (const route of ['/', '/research/']) { await page.goto(base + route); const themeLinks = page.locator('.card-grid-wide .card-readmore'); assert.equal(await themeLinks.count(), 3, 'Exactly three research areas'); const areaURLs = await themeLinks.evaluateAll(links => links.map(link => link.getAttribute('href')).sort()); + assert(areaURLs.includes(aiopsRoute) && !areaURLs.includes(previousAiopsRoute), 'Research links must use the new canonical RCA URL'); const menuURLs = await page.locator('#site-nav .nav-submenu a[href^="/research/"]').evaluateAll(links => links.map(link => link.getAttribute('href')).sort()); assert.deepEqual(menuURLs, areaURLs, 'Every research area must be reachable from the header'); assert.deepEqual(await page.locator('.site-footer .footer-list').first().locator('a').evaluateAll(links => links.map(link => link.getAttribute('href')).sort()), areaURLs); } + for (const [area, route] of [ + ['quantum-computing-systems', '/research/quantum-computing-systems/'], + ['distributed-machine-learning', aiopsRoute], + ['data-management', '/research/data-management/'], + ]) { + await page.goto(base + route); + const expected = publicationIndex.filter(record => record.research.includes(area)).slice(0, 6).map(record => record.url); + assert.deepEqual(await page.locator('#publications .list-card h3 a').evaluateAll(links => links.map(link => link.getAttribute('href'))), + expected, 'Every research area must use the same six-publication limit'); + } for (const width of [1440, 390, 320]) { await page.setViewportSize({ width, height: 900 }); await page.goto(base + aiopsRoute); assert.equal(await page.locator('h1').textContent(), 'Root Cause Analysis & AIOps'); + assert.equal(new URL(await page.locator('link[rel="canonical"]').getAttribute('href'), base).pathname, aiopsRoute); assert.deepEqual(await page.locator('.research-area > section .section-heading h2').allTextContents(), - ['Research outcomes', 'Publications', 'Researchers in this area']); + ['Research outcomes', 'Recent publications', 'Researchers in this area']); assert.deepEqual(await page.locator('#publications .list-card h3 a').evaluateAll(links => links.map(link => link.getAttribute('href'))), - aiopsPublications.map(record => record.url), 'The unified page must show all matching master references exactly once'); + aiopsPublications.slice(0, 6).map(record => record.url), 'The research page must show only its six most recent master references'); assert.equal(await page.locator('#outcomes .card').count(), 6); assert.equal(await page.locator('.research-support-logos img').count(), 2); assert.equal(await page.locator('.research-diagrams img').count(), 2); @@ -256,13 +269,13 @@ let browser; } } // Hugo aliases can use the production origin. Local previews use a relative baseURL. - const alias = fs.readFileSync(path.join(root, 'research/rca-aiops/index.html'), 'utf8'); + const alias = fs.readFileSync(path.join(root, 'research/distributed-machine-learning/index.html'), 'utf8'); assert.match(alias, /http-equiv=["']?refresh/i); const aliasTarget = alias.match(/content=["']?[^>]*?url=([^"' >]+)/i)?.[1]; - assert.equal(new URL(aliasTarget, base).pathname, aiopsRoute, 'The old RCA page must redirect to the single area'); + assert.equal(new URL(aliasTarget, base).pathname, aiopsRoute, 'The former Distributed Machine Learning URL must redirect to the RCA page'); if (new URL(aliasTarget, base).origin === new URL(base).origin) { for (const hash of ['', '#outcomes', '#publications', '#people']) { - await page.goto(base + '/research/rca-aiops/' + hash); + await page.goto(base + previousAiopsRoute + hash); await page.waitForURL(base + aiopsRoute + hash); if (hash) { await page.waitForFunction(hash => { @@ -272,11 +285,11 @@ let browser; }, hash); } } - await page.goto(base + '/research/rca-aiops/?source=bookmark#people'); + await page.goto(base + previousAiopsRoute + '?source=bookmark#people'); await page.waitForURL(base + aiopsRoute + '?source=bookmark#people'); const aliasNoJSContext = await createContext({ javaScriptEnabled: false }); const aliasNoJSPage = await aliasNoJSContext.newPage(); - await aliasNoJSPage.goto(base + '/research/rca-aiops/'); + await aliasNoJSPage.goto(base + previousAiopsRoute); await aliasNoJSPage.waitForURL(base + aiopsRoute); assert.equal(await aliasNoJSPage.locator('h1').textContent(), 'Root Cause Analysis & AIOps'); await aliasNoJSContext.close(); @@ -304,7 +317,7 @@ let browser; const snapContext = await createContext(); const snapPage = await snapContext.newPage(); await snapPage.bringToFront(); - const rosterRoutes = ['/people/', '/research/quantum-computing-systems/', '/research/distributed-machine-learning/', '/research/data-management/']; + const rosterRoutes = ['/people/', '/research/quantum-computing-systems/', '/research/rca-aiops/', '/research/data-management/']; for (const width of [1440, 390]) { await snapPage.setViewportSize({ width, height: 900 }); for (const route of rosterRoutes) { @@ -400,7 +413,7 @@ let browser; await page.keyboard.type('msrg', { delay: 90 }); assert.equal(await page.locator('[data-msrg-rounds]').textContent(), '2', 'Hardware keyboard must complete a round'); - for (const route of ['/', '/people/', '/publications/', '/research/quantum-computing-systems/', '/research/distributed-machine-learning/', '/data-sets/begen/', '/~thomas-trenty/', '/data-sets/']) { + for (const route of ['/', '/people/', '/publications/', '/research/quantum-computing-systems/', '/research/rca-aiops/', '/data-sets/begen/', '/~thomas-trenty/', '/data-sets/']) { await page.goto(base + route); await page.addScriptTag({ path: require.resolve('axe-core/axe.min.js') }); const results = await page.evaluate(async () => axe.run(document, { runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'wcag21aa', 'best-practice'] } })); From 0161ca4d559f74d5641467678d0b948fee9dc75c Mon Sep 17 00:00:00 2001 From: ttrenty <154608953+ttrenty@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:01:45 -0500 Subject: [PATCH 07/15] Add editable research outcomes to publications --- README.md | 30 +-- ...for-message-level-log-anomaly-detection.md | 5 + ...-and-incident-response-in-microservices.md | 5 + ...-for-autonomous-site-reliability-agents.md | 6 + ...ts-with-high-fidelity-failure-scenarios.md | 5 + ...ocalizing-runtime-faults-from-telemetry.md | 5 + ...g-failures-in-distributed-gpu-workloads.md | 6 + data/content_schema.json | 36 ++++ data/research_outcomes.toml | 58 ------ docs/aiops-sources.md | 7 + layouts/partials/research-outcome.html | 7 + layouts/publications/single.html | 10 + layouts/research/area.html | 18 +- scripts/content_model.py | 7 +- scripts/editor_check.cjs | 106 ++++++++++- scripts/profile_editor/editor.js | 13 +- tests/test_content_contracts.py | 16 +- tests/test_publication_outcomes_editor.py | 178 ++++++++++++++++++ tests/test_research_outcomes.py | 150 +++++++++++++++ 19 files changed, 556 insertions(+), 112 deletions(-) delete mode 100644 data/research_outcomes.toml create mode 100644 layouts/partials/research-outcome.html create mode 100644 tests/test_publication_outcomes_editor.py create mode 100644 tests/test_research_outcomes.py diff --git a/README.md b/README.md index 2345917..18d006a 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ in the terminal (usually **http://127.0.0.1:1313/**); **Ctrl+C** stops both. | Choose | What you can do | | --- | --- | | Member profiles | Update your details, research areas, links, awards, and portrait. | -| Publications | Add papers, abstracts, authors, tags, and related datasets. | +| Publications | Add papers, abstracts, authors, research outcomes, tags, and related datasets. | | Personal Pages | Build a page from a template or preview your custom website. | | Datasets and downloads | Add descriptions, related papers, download links, and files. | @@ -95,6 +95,13 @@ order shown on the paper. Tags and other item lists work the same way; you can also paste a list with line breaks. Add research areas and related datasets where relevant. +If the paper has public code, a benchmark, data, or a project website, choose +**Add research outcome**. Fill in its name, HTTPS link, and a short plain-text +description of what is available. You can add more than one outcome or leave +the section empty. Save and preview the publication to check the links and text. +Outcomes appear on the publication page and in its selected research areas; +changing those selections also updates where its outcomes appear. + The page identifier is assigned automatically from the original title and cannot be edited. If it is already in use, a number is added (for example, `paper-2`). Changing the title later keeps the original URL and adds a redirect from a URL @@ -168,30 +175,11 @@ Link to an external repository for datasets larger than **20 MB**. | --- | --- | | Homepage text | `content/_index.md` | | Research themes | `content/research/` | -| Research software and benchmark outcomes | `data/research_outcomes.toml` | +| Research software and benchmark outcomes | Publication editor, saved in `content/publications/` | | Navigation and site settings | `hugo.yaml` | Use GitHub's file editor on a branch or fork, then submit a PR. -The **Root Cause Analysis & AIOps** area refocuses the existing Distributed -Machine Learning area. Its canonical URL is `/research/rca-aiops/`; -`/research/distributed-machine-learning/` redirects there, preserving section -bookmarks. Its stable content identifier remains `distributed-machine-learning`. -Use `distributed-machine-learning` in member, publication, dataset, and outcome -research tags. The old `rca-aiops` publication-archive query is supported through -the area's `legacy_research_ids` setting. It is not a separate research choice. -All three research areas appear in the header, homepage, and research overview. - -The combined page includes AIOps/RCA work and earlier distributed AI/ML work. -It reads people and publication references from the master collections, with -research outcomes before publications and the associated people. Every research -page shows up to six recent publications, with a link to its complete archive. -Add outcomes only after verifying their public project or repository against a paper or an -official author page. Each `data/research_outcomes.toml` item has a stable `id`, -`title`, HTTPS `url`, short `summary`, `research` tags, and a master `publication` -slug. Record attribution in a comment and describe the actual release scope. -Alumni career outcomes remain separate member-profile data. - Edit source files; `public/` is generated. Everything in `content/` and `static/` may be published. Keep unfinished work on a branch until ready to merge. diff --git a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md index eae4a3c..33ec6b0 100644 --- a/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md +++ b/content/publications/fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection.md @@ -7,6 +7,11 @@ publication_type = "Conference Paper" research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2605.22779" abstract = "Production systems generate millions of log lines daily, yet most anomaly detectors operate at the session or window-level, flagging groups of lines rather than identifying the specific message responsible. This coarse granularity forces operators to inspect many routine lines per alert. Message-level detection offers finer granularity, but remains challenging. A single event template may correspond to both normal and anomalous messages, failures arise from heterogeneous subsystems, and line-level labeling at scale is impractical. Although large language models (LLMs) can reason over log semantics, applying them to every line is too costly for continuous monitoring. We present FAME (Failure-Aware Mixture-of-Experts), a label-efficient message-level mixture-of-experts framework that uses an LLM only once offline. We annotate at most K labeled lines per template to derive binary normal/anomaly indicators and representative examples. The LLM proposes a partition of templates into failure domains, and a certification step validates the proposal before training. FAME trains a lightweight router and domain experts that run on-premise and output anomaly predictions and failure-domain labels. On BGL, FAME achieves F1 = 98.16 at K = 100 reducing annotation effort by 76x and detects 97.7% of anomalies from unseen EventIDs. On Thunderbird, FAME reaches F1 = 99.95 with perfect recall." + +# Attribution: repository URL in the paper, https://arxiv.org/abs/2605.22779 +research_outcomes = [ + { title = "FAME", url = "https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts", summary = "Message-level log anomaly detection using offline failure-domain grouping and a lightweight router with specialized experts." }, +] +++ Presented at [ISSRE 2026](https://cyprusconferences.org/issre2026/program/). diff --git a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md index b4e6d87..acf977c 100644 --- a/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md +++ b/content/publications/gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices.md @@ -11,6 +11,11 @@ tags = ["GALA", "GALA: Can Graph-Augmented Large Language Model Agentic Workflow # Keep the consolidated earlier manuscript title searchable. # ACM publisher metadata: https://api.crossref.org/works/10.1145/3832783.3834335 abstract = "Microservice root cause analysis (RCA) requires correlating failures across heterogeneous telemetry within complex service dependency graphs. Existing methods often rely on a single telemetry modality; recent LLM-based approaches can suffer from unconstrained exploration and hallucination; and most systems stop at fault ranking without producing actionable incident response. We present GALA+, a graph-augmented LLM agentic framework centered on graph-guided investigation, which uses service dependencies to bound exploration and refine diagnosis through localized multi-modal evidence. For initial hypothesis generation, GALA+ combines complementary telemetry signals with STRIX, a novel trace- and graph-structure-aware scoring module. GALA+ then produces ranked diagnoses, incident summaries, and stratified action recommendations. We further introduce SURE-Score, a human-guided evaluation framework co-developed with industry SRE experts for assessing RCA-specific output quality beyond conventional text similarity metrics. On two microservice benchmarks, GALA+ consistently achieves the strongest overall results, surpassing the best LLM-based baseline by more than 25 percentage points in AC@1, while also receiving the highest ratings from both SURE-Score and independent human SRE evaluation." + +# Attribution: https://arxiv.org/abs/2608.08968; repository README describes release scope. +research_outcomes = [ + { title = "GALA evaluation artifacts", url = "https://github.com/EvanTian233/GALA-for-RCA", summary = "Annotated incident reports, example outputs, a supplementary technical report, and third-party baseline frameworks. The core GALA implementation is not publicly released." }, +] +++ [Published version](https://doi.org/10.1145/3832783.3834335). diff --git a/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md index cf6420d..6d9cdca 100644 --- a/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md +++ b/content/publications/sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents.md @@ -6,6 +6,12 @@ venue = "arXiv" publication_type = "ArXiv Preprint" research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2609.33023" + +# Attribution: https://arxiv.org/abs/2609.33023 +# Website source and scope: https://github.com/bigbigbai/sre-marathon-leaderboard/blob/main/README.md +research_outcomes = [ + { title = "SRE-Marathon project website and leaderboard", url = "https://bigbigbai.github.io/sre-marathon-leaderboard/", summary = "Project website and paper-reported leaderboard for continuous, change-driven SRE evaluation. Its GitHub repository contains the website source, not the benchmark implementation." }, +] +++ This preprint studies continuous SRE operation with overlapping incidents in a diff --git a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md index 114cbb3..4e2b74e 100644 --- a/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md +++ b/content/publications/sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios.md @@ -7,6 +7,11 @@ publication_type = "Conference Paper" research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2605.07161" abstract = "AI agents are increasingly used to diagnose and mitigate failures in production systems, known as agentic Site Reliability Engineering (SRE). Current SRE benchmarks are limited to oversimplistic SRE tasks and are unfortunately hard to extend due to bespoke designs. We present SREGym, a high-fidelity benchmark for SRE agents. SREGym exposes a live system environment built atop real-world cloud-native system stacks, where high-fidelity failure scenarios are simulated through fault injectors. SREGym models the complexity of production environments by simulating (1) a wide range of faults at different layers, (2) various ambient noises, and (3) diverse failure modes such as metastable failures and correlated failures. SREGym is architected as a modular, extensible framework that orchestrates fault and noise injectors across stacks. SREGym currently includes 90 realistic, challenging SRE problems. We use SREGym to evaluate frontier agents and show that their capabilities varies significantly in addressing different kinds of failures, with up to 40% differences in end-to-end results. SREGym is actively maintained as an open-source project and has been used by researchers and practitioners." + +# Attribution: https://arxiv.org/abs/2605.07161 and https://tianyin.github.io/pub.html +research_outcomes = [ + { title = "SREGym", url = "https://github.com/SREGym/SREGym", summary = "A live cloud-native benchmark for evaluating how SRE agents diagnose and mitigate injected operational failures." }, +] +++ Accepted to NeurIPS 2026, as listed on the diff --git a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md index c46f316..8dcb856 100644 --- a/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md +++ b/content/publications/when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry.md @@ -11,6 +11,11 @@ tags = ["AgentChaosBench", "When Agentic Executions Fail: Detecting and Localizi # Keep the earlier manuscript title searchable after updating the citation. # ACM publisher metadata: https://api.crossref.org/works/10.1145/3843782.3844662 abstract = "Reliability in LLM-based agentic systems is a property of the whole execution (its tool calls, model calls, guardrails, and inter-agent messages), not of the final answer alone, yet evaluating only task outcomes reveals little about how or why a run fails. We present AGENTCHAOSBENCH, a benchmark for detecting and localizing runtime faults in agentic systems from their execution telemetry. We run five heterogeneous applications that coordinate agents over the Agent-to-Agent protocol and call tools through the Model Context Protocol, and inject ten types of operational fault (unavailable or slow tools, corrupted or oversized responses, and delayed, looped, or misrouted delegations and bypassed guardrails) at their tool, model, guardrail, and inter-agent boundaries, alongside a no-fault control. The resulting dataset contains 275 sanitized traces: 250 faulty executions spanning ten fault types and 25 no-fault controls. Each faulty trace is aligned with the no-fault execution of the same input; fault-type labels and, where applicable, location labels are held out from diagnosis. On structured single-trace inputs, a first set of zero-shot LLM baselines shows the task is far from solved: local detectors up to 14B parameters reach only 13.6-19.2% top-1 fault-type accuracy and the frontier DeepSeek-v4-pro only 24.8%, while jointly identifying the fault type and its location tops out at 22%; reference-dependent faults (above all a bypassed guardrail) stay near-unsolved from a single trace. An aligned reference improves selected relative faults but does not resolve guardrail bypass. The held-out labels and compact prediction format support reproducible comparison of LLM-based and non-LLM diagnosis methods." + +# Attribution: https://arxiv.org/abs/2608.14680 and the authors' project links. +research_outcomes = [ + { title = "AgentChaosBench", url = "https://github.com/kevinzck8k/agentic-fault-diagnosis", summary = "Fault-detection and localization evaluation tools with 275 execution traces from five agentic applications." }, +] +++ [Published version](https://doi.org/10.1145/3843782.3844662). diff --git a/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md index c40d2c0..2b37d68 100644 --- a/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md +++ b/content/publications/when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads.md @@ -7,6 +7,12 @@ publication_type = "Workshop Paper" research = ["distributed-machine-learning"] external_url = "https://arxiv.org/abs/2610.14009" # ACM publisher metadata: https://api.crossref.org/works/10.1145/3820756.3844919 + +# Attribution: data availability statement, https://arxiv.org/abs/2610.14009 +# No top-level repository license was present when checked on 2026-10-22. +research_outcomes = [ + { title = "GF-Bench artifacts", url = "https://github.com/cuiboyuan/nccl_log_rca_bench", summary = "Tools for generating multi-GPU failures, collecting NCCL logs, and evaluating anomaly detection, with sample data and scripts to reproduce the experiments." }, +] +++ [Published version](https://doi.org/10.1145/3820756.3844919). diff --git a/data/content_schema.json b/data/content_schema.json index eaf7039..ed8b005 100644 --- a/data/content_schema.json +++ b/data/content_schema.json @@ -382,6 +382,42 @@ "default": [], "options_source": "research" }, + { + "key": "research_outcomes", + "label": "Research outcomes", + "add_label": "Add research outcome", + "type": "objects", + "default": [], + "help": "Optional public code, benchmarks, datasets, or project websites. Each outcome is linked automatically to this publication and its selected research areas.", + "fields": [ + { + "key": "title", + "label": "Outcome name", + "type": "string", + "default": "", + "required": true + }, + { + "key": "url", + "label": "Project or repository URL", + "type": "string", + "default": "", + "format": "url", + "https_only": true, + "required": true, + "help": "Use a public HTTPS link." + }, + { + "key": "summary", + "label": "Description", + "type": "string", + "default": "", + "multiline": true, + "required": true, + "help": "Briefly describe what is publicly available, using plain text." + } + ] + }, { "key": "abstract", "label": "Abstract", diff --git a/data/research_outcomes.toml b/data/research_outcomes.toml deleted file mode 100644 index abd6b38..0000000 --- a/data/research_outcomes.toml +++ /dev/null @@ -1,58 +0,0 @@ -# Public artifacts are attributed by the linked papers or official author pages. -# Keep citation metadata in content/publications; these entries describe outputs. - -[[items]] -id = "sregym" -title = "SREGym" -url = "https://github.com/SREGym/SREGym" -summary = "A live cloud-native benchmark for evaluating how SRE agents diagnose and mitigate injected operational failures." -research = ["distributed-machine-learning"] -publication = "sregym-a-live-benchmark-for-ai-sre-agents-with-high-fidelity-failure-scenarios" -# Attribution: https://arxiv.org/abs/2605.07161 and https://tianyin.github.io/pub.html - -[[items]] -id = "agentchaosbench" -title = "AgentChaosBench" -url = "https://github.com/kevinzck8k/agentic-fault-diagnosis" -summary = "Fault-detection and localization evaluation tools with 275 execution traces from five agentic applications." -research = ["distributed-machine-learning"] -publication = "when-agentic-executions-fail-detecting-and-localizing-runtime-faults-from-telemetry" -# Attribution: https://arxiv.org/abs/2608.14680 and the authors' project links. - -[[items]] -id = "fame" -title = "FAME" -url = "https://github.com/HuanchiW95/FAME-Failure-Aware-Mixture-of-Experts" -summary = "Message-level log anomaly detection using offline failure-domain grouping and a lightweight router with specialized experts." -research = ["distributed-machine-learning"] -publication = "fame-failure-aware-mixture-of-experts-for-message-level-log-anomaly-detection" -# Attribution: repository URL in the paper, https://arxiv.org/abs/2605.22779 - -[[items]] -id = "gala-evaluation" -title = "GALA evaluation artifacts" -url = "https://github.com/EvanTian233/GALA-for-RCA" -summary = "Annotated incident reports, example outputs, a supplementary technical report, and third-party baseline frameworks. The core GALA implementation is not publicly released." -research = ["distributed-machine-learning"] -publication = "gala-graph-augmented-llm-agents-for-root-cause-analysis-and-incident-response-in-microservices" -# Attribution: https://arxiv.org/abs/2608.08968; repository README describes release scope. - -[[items]] -id = "gf-bench" -title = "GF-Bench artifacts" -url = "https://github.com/cuiboyuan/nccl_log_rca_bench" -summary = "Tools for generating multi-GPU failures, collecting NCCL logs, and evaluating anomaly detection, with sample data and scripts to reproduce the experiments." -research = ["distributed-machine-learning"] -publication = "when-log-parsing-removes-what-matters-detecting-failures-in-distributed-gpu-workloads" -# Attribution: data availability statement, https://arxiv.org/abs/2610.14009 -# No top-level repository license was present when checked on 2026-10-22. - -[[items]] -id = "sre-marathon" -title = "SRE-Marathon project website and leaderboard" -url = "https://bigbigbai.github.io/sre-marathon-leaderboard/" -summary = "Project website and paper-reported leaderboard for continuous, change-driven SRE evaluation. Its GitHub repository contains the website source, not the benchmark implementation." -research = ["distributed-machine-learning"] -publication = "sre-marathon-a-continuous-change-driven-benchmark-for-autonomous-site-reliability-agents" -# Attribution: https://arxiv.org/abs/2609.33023 -# Website source and scope: https://github.com/bigbigbai/sre-marathon-leaderboard/blob/main/README.md diff --git a/docs/aiops-sources.md b/docs/aiops-sources.md index 8c74275..4a6a4f3 100644 --- a/docs/aiops-sources.md +++ b/docs/aiops-sources.md @@ -11,6 +11,13 @@ page is `/research/rca-aiops/`; the former Distributed Machine Learning URL redirects there. Existing RCA research tags were folded into its stable `distributed-machine-learning` identifier without adding unrelated members or papers. +Research outcomes are stored with their owning records in `content/publications/` +and can be maintained in the publication editor. They inherit the publication's +research areas; for example, GALA's evaluation artifacts also appear under Data +Management. Outcomes are collected from all matching publications, independently +of the six-reference preview. Source comments beside the outcome fields preserve +the attribution and release-scope evidence. + ## Yuanchen Gao Added to the shared roster at the maintainer's request, with the PhD role diff --git a/layouts/partials/research-outcome.html b/layouts/partials/research-outcome.html new file mode 100644 index 0000000..f2cbb34 --- /dev/null +++ b/layouts/partials/research-outcome.html @@ -0,0 +1,7 @@ + diff --git a/layouts/publications/single.html b/layouts/publications/single.html index c17b2a8..b4cd5fd 100644 --- a/layouts/publications/single.html +++ b/layouts/publications/single.html @@ -28,6 +28,16 @@

{{ .Title }}

{{ else }}

The abstract has not been added yet.

{{ end }} + {{ with .Params.research_outcomes }} +
+

Research outcomes

+
+ {{ range . }} + {{ partial "research-outcome.html" (dict "outcome" .) }} + {{ end }} +
+
+ {{ end }} {{ with .Content }}

Additional notes

{{ . }} diff --git a/layouts/research/area.html b/layouts/research/area.html index 187dacc..cd43c6a 100644 --- a/layouts/research/area.html +++ b/layouts/research/area.html @@ -5,7 +5,15 @@ {{ $publications := where (site.GetPage "/publications").RegularPages "Params.research" "intersect" (slice $slug) }} {{ $listedPublications := first 6 (sort $publications "Params.year" "desc") }} {{ $datasets := where (site.GetPage "/data-sets").RegularPages "Params.research" "intersect" (slice $slug) }} - {{ $outcomes := where (hugo.Data.research_outcomes.items | default slice) "research" "intersect" (slice $slug) }} + {{ $outcomes := slice }} + {{ range $publications.GroupByParam "year" "desc" }} + {{ range .Pages.ByTitle }} + {{ $publication := . }} + {{ range .Params.research_outcomes }} + {{ $outcomes = $outcomes | append (dict "outcome" . "publication" $publication) }} + {{ end }} + {{ end }} + {{ end }}
@@ -49,13 +57,7 @@

Research outcomes

{{ range $outcomes }} - + {{ partial "research-outcome.html" . }} {{ end }} {{ range sort $datasets "Title" }}
diff --git a/scripts/content_model.py b/scripts/content_model.py index 099ee4a..0a7014c 100644 --- a/scripts/content_model.py +++ b/scripts/content_model.py @@ -120,7 +120,12 @@ def validate_fields(fields: list[dict], data: dict, kind: str, root: Path = ROOT parsed = urlsplit(value) schemes = {"https"} if field.get("https_only") else {"http", "https"} host = parsed.hostname or "" - if parsed.scheme not in schemes or not host or any(c.isspace() for c in value): + if (parsed.scheme not in schemes or not host or + any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value) or "\\" in value): + raise ValueError + # urlsplit leaves invalid ports and escaped hostnames unchecked. + parsed.port + if parsed.username is not None or parsed.password is not None or "%" in host: raise ValueError if field.get("host") and not (host == field["host"] or host.endswith("." + field["host"])): raise ValueError diff --git a/scripts/editor_check.cjs b/scripts/editor_check.cjs index d3396fd..00bd89b 100644 --- a/scripts/editor_check.cjs +++ b/scripts/editor_check.cjs @@ -44,7 +44,9 @@ let browser; assert.equal(await page.locator('#slug').inputValue(), 'jane-doe'); await page.getByLabel('Upload a portrait').setInputFiles({ name: 'portrait.png', mimeType: 'image/png', buffer: Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=', 'base64') }); await page.waitForFunction(() => document.querySelector('#message').textContent.startsWith('Uploaded ')); - await page.getByRole('button', { name: 'Add awards and funding', exact: true }).click(); + await page.getByRole('button', { name: 'Add awards and funding', exact: true }).focus(); + await page.keyboard.press('Enter'); + assert.equal(await page.getByLabel('Award name').evaluate(input => input === document.activeElement), true); await page.getByLabel('Award name').fill('Example award'); await page.getByLabel('Award page *', { exact: true }).fill('https://example.org/award'); async function addItem(key, value) { @@ -71,10 +73,12 @@ let browser; assert.deepEqual(await items('interests'), ['Quantum Computing']); assert.equal(fs.existsSync(path.join(fixture, 'content/people/jane-doe.md')), false, 'Enter adds an item without submitting the form'); async function save() { - await page.locator('#save').click(); - await page.waitForFunction(() => document.querySelector('#message').textContent.startsWith('Saved ') || document.querySelector('#message').classList.contains('error')); - assert.match(await page.locator('#message').textContent(), /^Saved /); + await Promise.all([ + page.waitForResponse(response => response.url() === base + '/api/records' && response.request().method() === 'POST'), + page.locator('#save').click(), + ]); await page.waitForFunction(() => !document.querySelector('#save').disabled); + assert.match(await page.locator('#message').textContent(), /^Saved /); } async function choose(kind) { await page.locator('#kind').selectOption(kind); @@ -110,6 +114,7 @@ let browser; await page.getByRole('button', { name: 'Add highlights', exact: true }).click(); await page.getByLabel('Title *', { exact: true }).fill('Research highlight'); await page.getByRole('button', { name: 'Add primary link', exact: true }).click(); + assert.equal(await page.getByLabel('Label *', { exact: true }).evaluate(input => input === document.activeElement), true); await page.getByLabel('Label *', { exact: true }).fill('MSRG'); await page.getByLabel('Link *', { exact: true }).fill('https://msrg.github.io/'); await save(); @@ -174,6 +179,7 @@ let browser; await page.setViewportSize({ width: 1440, height: 900 }); console.log('Template fields, hidden-field preservation, and timeline reordering passed'); await page.getByRole('button', { name: 'Remove item', exact: true }).first().click(); + assert.equal(await page.getByRole('button', { name: 'Add primary link', exact: true }).evaluate(button => button === document.activeElement), true); await save(); await choose('publications'); @@ -222,7 +228,66 @@ let browser; const abstract = await page.locator('#abstract').inputValue(); assert.match(abstract, /\$\$\n\\sum_\{i=1\}\^\{n\} x_i\n\$\$/); await page.getByLabel('Additional notes', { exact: true }).fill('Companion notes. [Member](/people/#member-jane-doe)'); + const outcomeField = page.locator('#field-research_outcomes'); + const outcomeRows = outcomeField.locator('.award'); + const addOutcome = outcomeField.getByRole('button', { name: 'Add research outcome', exact: true }); + let expectedOutcomes = [ + { title: 'Public benchmark', url: 'https://example.org/benchmark', summary: 'Sample data and evaluation scripts.\nTwo public examples.' }, + { title: 'Project website', url: 'https://example.org/project', summary: 'Project overview and leaderboard.' }, + ]; + async function outcomeValues() { + return outcomeRows.evaluateAll(rows => rows.map(row => Object.fromEntries( + ['title', 'url', 'summary'].map(key => [key, row.querySelector(`[id$="-${key}"]`).value])))); + } + async function addOutcomeWithKeyboard(outcome) { + await addOutcome.focus(); await page.keyboard.press('Enter'); + const row = outcomeRows.last(); + const title = row.getByLabel('Outcome name *', { exact: true }); + const url = row.getByLabel('Project or repository URL *', { exact: true }); + const summary = row.getByLabel('Description *', { exact: true }); + assert.equal(await title.evaluate(input => input === document.activeElement), true, 'Add focuses the first new field'); + assert.equal(await title.evaluate(input => input.checkValidity()), false, 'An added outcome requires a name'); + await page.keyboard.insertText(outcome.title); + await page.keyboard.press('Tab'); + assert.equal(await url.evaluate(input => input === document.activeElement), true); + await page.keyboard.insertText(outcome.url); + await page.keyboard.press('Tab'); + assert.equal(await summary.evaluate(input => input === document.activeElement), true); + assert.equal(await summary.evaluate(input => input.tagName), 'TEXTAREA'); + await page.keyboard.insertText(outcome.summary); + return row; + } + async function removeOutcomeWithKeyboard(row) { + await row.getByRole('button', { name: 'Remove item', exact: true }).focus(); + await page.keyboard.press('Enter'); + assert.equal(await addOutcome.evaluate(button => button === document.activeElement), true, 'Remove returns focus to Add'); + } + assert.equal(await outcomeRows.count(), 0, 'Outcomes are optional and initially empty'); + assert.equal(await addOutcome.getAttribute('aria-describedby'), 'research_outcomes-help'); + assert.match(await page.locator('#research_outcomes-help').textContent(), /automatically.*publication.*research areas/); + await page.evaluate(() => { + window.outcomeFocusEvents = []; + document.addEventListener('focusin', event => { + if (event.target.closest('#field-research_outcomes')) window.outcomeFocusEvents.push(event.target.id || event.target.textContent); + }); + }); + await addOutcomeWithKeyboard(expectedOutcomes[0]); + await addOutcomeWithKeyboard({ title: 'Temporary outcome', url: 'https://example.org/temp', summary: 'Remove this before creating the paper.' }); + await removeOutcomeWithKeyboard(outcomeRows.last()); + assert.deepEqual(await outcomeValues(), expectedOutcomes.slice(0, 1)); + await addOutcomeWithKeyboard(expectedOutcomes[1]); + const outcomeUrl = outcomeRows.first().getByLabel('Project or repository URL *', { exact: true }); + await outcomeUrl.fill('http://example.org/benchmark'); + await page.locator('#save').click(); + await page.waitForFunction(() => document.querySelector('#message').classList.contains('error') && !document.querySelector('#save').disabled); + assert.match(await page.locator('#message').textContent(), /research_outcomes\[0\]\.url: use a valid HTTPS URL/); + assert.equal(fs.existsSync(path.join(fixture, 'content/publications/example-paper.md')), false); + await outcomeUrl.fill(expectedOutcomes[0].url); + await page.evaluate(() => { window.outcomeFocusEvents = []; }); await save(); + assert.deepEqual(await page.evaluate(() => window.outcomeFocusEvents), [], 'Rendering saved outcomes must not steal focus'); + assert.deepEqual(await outcomeValues(), expectedOutcomes); + assert.deepEqual((await (await page.request.get(base + '/api/records/publications/example-paper')).json()).data.research_outcomes, expectedOutcomes); assert.equal(await page.locator('#save').textContent(), 'Save changes'); assert.equal(await publicationType.inputValue(), 'Journal Article'); assert.deepEqual(await items('authors'), ['J. Doe', 'John Doe', 'Doe, Alex', 'External Author'], 'Saving includes unfinished input'); @@ -230,6 +295,7 @@ let browser; assert.equal(await page.locator('#record-slug').inputValue(), 'example-paper'); await page.locator('#profile').selectOption(''); await page.waitForFunction(() => !document.querySelector('#save').disabled && document.querySelector('#save').textContent === 'Create'); + assert.equal(await outcomeRows.count(), 0, 'A new publication starts with no outcomes'); assert.deepEqual(await items('authors'), [], 'New entry resets the list controls'); assert.deepEqual(await page.locator('#tags-suggestions option').evaluateAll(options => options.map(option => option.value)), ['quantum', 'systems'], 'Newly saved tags are immediately available for the next entry'); @@ -239,6 +305,7 @@ let browser; await page.locator('#external_url').fill('https://example.org/another-paper'); await save(); assert.equal(await page.locator('#record-slug').inputValue(), 'example-paper-2'); + assert.equal(await outcomeRows.count(), 0, 'Publications can be created without outcomes or research areas'); assert.equal(await page.locator('#profile').inputValue(), 'example-paper-2'); assert.match(await page.locator('#preview').getAttribute('href'), /\/publications\/example-paper-2\/$/); assert.equal(await page.locator('#record-slug').evaluate(input => input.readOnly), true); @@ -261,24 +328,51 @@ let browser; await choose('publications'); console.log('Publication form loaded'); assert.deepEqual(await page.locator('#tags-suggestions option').evaluateAll(options => options.map(option => option.value)), ['quantum', 'systems']); + await page.evaluate(() => { window.outcomeFocusEvents = []; }); await page.locator('#profile').selectOption('example-paper'); await page.waitForFunction(() => !document.querySelector('#save').disabled && document.querySelector('#title').value === 'Example Paper'); + assert.deepEqual(await page.evaluate(() => window.outcomeFocusEvents), [], 'Loading an existing publication must not focus its outcomes'); + assert.deepEqual(await outcomeValues(), expectedOutcomes); + await page.setViewportSize({ width: 390, height: 900 }); + const editedName = outcomeRows.first().getByLabel('Outcome name *', { exact: true }); + await editedName.focus(); await page.keyboard.press('ControlOrMeta+A'); await page.keyboard.insertText('Updated public benchmark'); + expectedOutcomes[0].title = 'Updated public benchmark'; + await removeOutcomeWithKeyboard(outcomeRows.last()); + expectedOutcomes = expectedOutcomes.slice(0, 1); + await save(); + assert.deepEqual(await outcomeValues(), expectedOutcomes); + const thirdOutcome = { title: 'New evaluation tools', url: 'https://example.org/evaluation', summary: 'Evaluation tools added to an existing publication.' }; + await addOutcomeWithKeyboard(thirdOutcome); + expectedOutcomes.push(thirdOutcome); + await save(); + await page.locator('#reload').focus(); + await page.evaluate(() => { window.outcomeFocusEvents = []; }); + await page.keyboard.press('Enter'); + await page.waitForFunction(() => !document.querySelector('#save').disabled && document.querySelector('#message').textContent.startsWith('Edit the form')); + assert.deepEqual(await page.evaluate(() => window.outcomeFocusEvents), [], 'Reload must not focus the populated outcome rows'); + assert.deepEqual(await outcomeValues(), expectedOutcomes); + await page.setViewportSize({ width: 1440, height: 900 }); + console.log('Optional publication outcomes: keyboard create/edit/remove/reload, HTTPS validation, and focus preservation passed'); assert.equal(await publicationType.inputValue(), 'Journal Article', 'Publication type survives reopening'); assert.equal(await page.locator('#abstract').inputValue(), abstract, 'LaTeX survives saving and reopening'); assert.deepEqual(await items('authors'), ['J. Doe', 'John Doe', 'Doe, Alex', 'External Author']); assert.equal(await page.locator('#save').textContent(), 'Save changes'); await page.getByLabel('Example Dataset', { exact: true }).check(); await save(); + assert.deepEqual(await outcomeValues(), expectedOutcomes, 'Unrelated changes preserve research outcomes'); const publication = path.join(fixture, 'content/publications/example-paper.md'); fs.appendFileSync(publication, '\nChanged outside the editor.\n'); + const externalPublication = fs.readFileSync(publication, 'utf8'); + await removeOutcomeWithKeyboard(outcomeRows.last()); await page.locator('#title').fill('Changed title'); await page.locator('#save').click(); await page.waitForFunction(() => document.querySelector('#message').classList.contains('error')); assert.match(await page.locator('#message').textContent(), /changed since you opened/); - assert.match(fs.readFileSync(publication, 'utf8'), /title = "Example Paper"/); + assert.equal(fs.readFileSync(publication, 'utf8'), externalPublication, 'A stale removal must not discard saved outcomes'); page.once('dialog', dialog => dialog.accept()); await page.getByRole('button', { name: 'Reload saved entry' }).click(); await page.waitForFunction(() => !document.querySelector('#save').disabled && !document.querySelector('#message').classList.contains('error')); + assert.deepEqual(await outcomeValues(), expectedOutcomes, 'Reload restores the saved outcomes after a conflict'); const originalPreview = await page.locator('#preview').getAttribute('href'); await page.locator('#title').fill('Revised Paper'); await save(); @@ -337,7 +431,7 @@ let browser; assert.equal(await page.locator('#preview').getAttribute('href'), null); assert.equal(await page.locator('#preview-start').isVisible(), true); assert.deepEqual(errors, []); - console.log(`${engine}: profile photo, awards, personal templates, publications, dataset uploads, conflicts, accessibility and desktop/mobile layouts passed.`); + console.log(`${engine}: profile photo, awards, personal templates, publications and outcomes, dataset uploads, conflicts, accessibility and desktop/mobile layouts passed.`); })().catch(error => { console.error(error); process.exitCode = 1; }).finally(async () => { if (browser) await browser.close(); if (server.exitCode === null) { const stopped = new Promise(resolve => server.once('exit', resolve)); server.kill(); await stopped; } diff --git a/scripts/profile_editor/editor.js b/scripts/profile_editor/editor.js index 68fa0dd..e8b1b01 100644 --- a/scripts/profile_editor/editor.js +++ b/scripts/profile_editor/editor.js @@ -149,7 +149,8 @@ function fieldControl(field, value, prefix = '') { if (field.type === 'objects' || field.type === 'object') { label.removeAttribute('for'); const rows = node('div'); - const add = node('button', 'Add ' + field.label.toLowerCase(), 'secondary'); add.type = 'button'; + const add = node('button', field.add_label || 'Add ' + field.label.toLowerCase(), 'secondary'); add.type = 'button'; + if (helpText) add.setAttribute('aria-describedby', help.id); function updateOrderControls() { Array.from(rows.children).forEach((row, index) => { if (!field.reorderable) return; @@ -182,12 +183,18 @@ function fieldControl(field, value, prefix = '') { } } const remove = node('button', 'Remove item', 'secondary'); remove.type = 'button'; - remove.addEventListener('click', () => { row.remove(); add.hidden = false; updateOrderControls(); markDirty(); }); + remove.addEventListener('click', () => { + row.remove(); add.hidden = false; updateOrderControls(); markDirty(); add.focus(); + }); actions.append(remove); row.append(actions); rows.append(row); updateOrderControls(); if (field.type === 'object') add.hidden = true; + return row; } (field.type === 'object' ? (value && Object.keys(value).length ? [value] : []) : value || []).forEach(addRow); - add.addEventListener('click', () => { addRow({}); markDirty(); }); + add.addEventListener('click', () => { + const row = addRow({}); markDirty(); + row.querySelector('input:not([type="hidden"]):enabled, select:enabled, textarea:enabled, button:enabled:not([hidden])')?.focus(); + }); wrapper.append(rows, add); read = () => field.type === 'object' ? (rows.firstElementChild?.readValue() || {}) : Array.from(rows.children, row => row.readValue()); } else if (field.type === 'strings' && field.choices) { diff --git a/tests/test_content_contracts.py b/tests/test_content_contracts.py index 7f534b2..a6458b9 100644 --- a/tests/test_content_contracts.py +++ b/tests/test_content_contracts.py @@ -2,7 +2,6 @@ import re import sys -import tomllib import unittest from pathlib import Path from urllib.parse import urlsplit @@ -93,25 +92,12 @@ def test_research_and_related_content_references_exist(self) -> None: self.assertIn(parsed.scheme, {"http", "https"}) self.assertTrue(parsed.netloc) - def test_research_outcomes_reference_master_records(self) -> None: + def test_legacy_research_filters_are_unique_and_not_separate_areas(self) -> None: areas = {path.parent.name: load_toml_front_matter(path) for path in (ROOT / "content/research").glob("*/_index.md")} - publications = {path.stem for path, _ in iter_publication_pages()} legacy_ids = [alias for data in areas.values() for alias in data.get("legacy_research_ids", [])] self.assertEqual(len(legacy_ids), len(set(legacy_ids))) self.assertFalse(set(legacy_ids).intersection(areas), "A legacy filter must not be a separate research area") - outcomes = tomllib.loads((ROOT / "data/research_outcomes.toml").read_text())["items"] - self.assertEqual(len({item["id"] for item in outcomes}), len(outcomes)) - for item in outcomes: - with self.subTest(outcome=item["id"]): - self.assertTrue(item["title"].strip()) - self.assertTrue(item["summary"].strip()) - self.assertTrue(item["research"]) - self.assertTrue(set(item["research"]).issubset(areas)) - self.assertIn(item["publication"], publications) - parsed = urlsplit(item["url"]) - self.assertEqual(parsed.scheme, "https") - self.assertTrue(parsed.netloc) if __name__ == "__main__": diff --git a/tests/test_publication_outcomes_editor.py b/tests/test_publication_outcomes_editor.py new file mode 100644 index 0000000..f8527ed --- /dev/null +++ b/tests/test_publication_outcomes_editor.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +from edit_profiles import Conflict, ProfileStore + + +class PublicationOutcomesEditorTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.store = ProfileStore(self.root) + + def paper(self, title='Example Paper'): + return {'title': title, 'authors': ['Example Author'], 'year': 2026, + 'venue': 'Example Conference', 'publication_type': 'Conference Paper', + 'external_url': 'https://example.org/paper'} + + def outcome(self, **changes): + return {'title': 'Public benchmark', 'url': 'https://example.org/benchmark', + 'summary': 'Sample data and evaluation scripts.\nIncludes "quoted" labels and Unicode: α.'} | changes + + def create(self, **changes): + return self.store.save({'kind': 'publications', 'data': self.paper() | changes}) + + def reopen(self): + return self.store.get('example-paper', 'publications', include_preview=False) + + def save(self, changes, revision=None): + return self.store.save({'kind': 'publications', 'slug': 'example-paper', + 'revision': revision or self.reopen()['revision'], 'data': changes}) + + def files(self): + return {path.relative_to(self.root): path.read_bytes() + for path in self.root.rglob('*') if path.is_file()} + + def legacy_paper(self): + self.create() + path = self.store.path('example-paper', 'publications') + source = path.read_text().replace('+++\n', '+++\n# Verified citation source.\n', 1) + source = source.replace('\n+++\n', '\ncustom_metadata = { owner = "Example", reviewed = true }\n+++\n') + source += '\n## Additional notes\nKeep this publication body and [source](https://example.org/source).\n' + path.write_text(source) + return path, source + + def test_schema_exposes_optional_repeatable_outcomes_without_requiring_research(self): + schema = self.store.schema('publications') + fields = {field['key']: field for field in schema['schema']['fields']} + outcomes = fields['research_outcomes'] + self.assertEqual(outcomes['type'], 'objects') + self.assertEqual(schema['defaults']['research_outcomes'], []) + self.assertEqual(outcomes['add_label'], 'Add research outcome') + self.assertFalse(outcomes.get('required', False)) + self.assertFalse(fields['research'].get('required', False)) + self.assertEqual({child['key'] for child in outcomes['fields']}, {'title', 'url', 'summary'}) + self.assertTrue(all(child['required'] for child in outcomes['fields'])) + self.assertIn('this publication', outcomes['help']) + self.assertIn('research areas', outcomes['help']) + + def test_create_reload_edit_and_remove_multiple_outcomes(self): + outcomes = [self.outcome(), self.outcome(title='Project website', url='https://example.org/project')] + self.create(research_outcomes=outcomes) + self.assertEqual(self.reopen()['data']['research_outcomes'], outcomes) + self.assertNotIn('research', self.reopen()['data']) + self.assertEqual(list(self.files()), [Path('content/publications/example-paper.md')]) + edited = [outcomes[0] | {'title': 'Updated benchmark', 'summary': 'Revised public release.'}, outcomes[1]] + self.save({'research_outcomes': edited}) + self.assertEqual(self.reopen()['data']['research_outcomes'], edited) + self.save({'research_outcomes': edited[1:]}) + self.assertEqual(self.reopen()['data']['research_outcomes'], edited[1:]) + self.save({'research_outcomes': []}) + self.assertEqual(self.reopen()['data']['research_outcomes'], []) + + def test_legacy_publication_can_save_without_adding_optional_fields(self): + path, source = self.legacy_paper() + self.save({'venue': 'Revised Conference'}) + self.assertEqual(path.read_text(), source.replace('venue = "Example Conference"', 'venue = "Revised Conference"')) + self.assertNotIn('research_outcomes', self.reopen()['data']) + self.assertNotIn('research', self.reopen()['data']) + + def test_adding_and_removing_outcomes_preserves_body_comments_and_unknown_metadata(self): + path, source = self.legacy_paper() + original = self.reopen() + self.save({'research_outcomes': [self.outcome()]}) + self.assertEqual(self.reopen()['data'], original['data'] | {'research_outcomes': [self.outcome()]}) + self.assertEqual(self.reopen()['body'], original['body']) + self.assertIn('# Verified citation source.\n', path.read_text()) + provenance = '# Public artifact provenance: https://example.org/source\n' + path.write_text(path.read_text().replace('research_outcomes = ', provenance + 'research_outcomes = ')) + revised = self.outcome(title='Revised public benchmark', summary='Updated release scope.') + self.save({'research_outcomes': [revised]}) + self.assertEqual(self.reopen()['data'], original['data'] | {'research_outcomes': [revised]}) + self.assertEqual(self.reopen()['body'], original['body']) + self.assertIn(provenance + 'research_outcomes = ', path.read_text()) + self.assertIn('# Verified citation source.\n', path.read_text()) + self.save({'research_outcomes': []}) + self.assertEqual(path.read_text(), source.replace('\n+++\n', '\n' + provenance + 'research_outcomes = []\n+++\n')) + + def test_unrelated_save_preserves_outcomes_and_their_source_comments_exactly(self): + path, _ = self.legacy_paper() + self.save({'research_outcomes': [self.outcome()]}) + source = path.read_text().replace('research_outcomes = [', '# Artifact provenance.\nresearch_outcomes = [\n ') + source = source.replace(' }]', ' }, # Keep this nested source note.\n]') + path.write_text(source) + outcomes = self.reopen()['data']['research_outcomes'] + self.save({'venue': 'Revised Conference'}) + expected = source.replace('venue = "Example Conference"', 'venue = "Revised Conference"') + self.assertEqual(path.read_text(), expected) + # The editor sends present fields even when their values have not changed. + self.save({'venue': 'Another Conference', 'research_outcomes': outcomes}) + self.assertEqual(path.read_text(), expected.replace('Revised Conference', 'Another Conference')) + + def test_stale_revision_cannot_remove_or_overwrite_newer_outcomes(self): + self.create(research_outcomes=[self.outcome()]) + stale = self.reopen() + newer = [self.outcome(title='New public release', url='https://example.org/new')] + self.save({'research_outcomes': newer}) + before = self.files() + for changes in ({'research_outcomes': []}, {'research_outcomes': stale['data']['research_outcomes']}, {'venue': 'Stale edit'}): + with self.subTest(changes=changes), self.assertRaises(Conflict): + self.save(changes, revision=stale['revision']) + self.assertEqual(self.files(), before) + self.assertEqual(self.reopen()['data']['research_outcomes'], newer) + + def assert_rejected_without_write(self, outcomes): + before = self.files() + with self.assertRaises(ValueError): + self.store.save({'kind': 'publications', 'data': self.paper('Invalid Paper') | {'research_outcomes': outcomes}}) + self.assertEqual(self.files(), before, 'An invalid new publication must not create a file') + with self.assertRaises(ValueError): + self.save({'research_outcomes': outcomes}) + self.assertEqual(self.files(), before, 'An invalid edit must not replace any existing file') + + def test_partial_and_invalid_nested_values_rejected_without_file_write(self): + self.create(research_outcomes=[self.outcome()]) + invalid = [None, {}, 'not a list', [None], ['not an object'], [1], [{}], + [{'title': 'Missing fields'}], + [{'title': 'Missing URL', 'summary': 'Description'}], + [self.outcome(title='')], [self.outcome(title=' ')], [self.outcome(title=1)], + [self.outcome(url=None)], [self.outcome(url=[])], + [self.outcome(summary='')], [self.outcome(summary=' \n ')], + [self.outcome(summary=False)], [self.outcome(summary={})], + [self.outcome(), {'title': 'Incomplete second outcome'}]] + for outcomes in invalid: + with self.subTest(outcomes=outcomes): + self.assert_rejected_without_write(outcomes) + + def test_unsafe_malformed_and_http_outcome_urls_rejected_without_file_write(self): + self.create(research_outcomes=[self.outcome()]) + urls = ['', 'http://example.org/project', 'javascript:alert(1)', 'data:text/html,test', + 'file:///tmp/project', '//example.org/project', '/project/', 'not a URL', + 'https://', 'https:///project', 'https://example.org/a b', + 'https://example.org/a\nb', 'https://example.org/\x00project', + 'https://example.org\\project', 'https://exa%6dple.org/project', + 'https://example.org:invalid/project', 'https://example.org:70000/project', + 'https://[broken/project', 'https://user:password@example.org/project'] + for url in urls: + with self.subTest(url=url): + self.assert_rejected_without_write([self.outcome(url=url)]) + + def test_valid_https_urls_and_plain_text_descriptions_round_trip(self): + outcomes = [self.outcome(url=url) for url in ( + 'https://github.com/example/benchmark', + 'https://example.org:8443/project?version=2&download=sample%20data#usage', + 'https://example.org/数据', 'https://[2001:db8::1]/project')] + outcomes[0]['summary'] = 'Literal , **asterisks**, and $x^2$.\nPlain text stays intact.' + self.create(research_outcomes=outcomes) + self.assertEqual(self.reopen()['data']['research_outcomes'], outcomes) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_research_outcomes.py b/tests/test_research_outcomes.py new file mode 100644 index 0000000..260db38 --- /dev/null +++ b/tests/test_research_outcomes.py @@ -0,0 +1,150 @@ +"""Exercise publication-owned outcomes through the editor store and real templates.""" +from __future__ import annotations + +from html import unescape +from html.parser import HTMLParser +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from edit_profiles import ProfileStore + +HUGO = os.environ.get("HUGO_BIN") or shutil.which("hugo") +AIOPS = "distributed-machine-learning" +QUANTUM = "quantum-computing-systems" + + +class Links(HTMLParser): + def __init__(self, html): + super().__init__() + self.hrefs = [] + self.feed(html) + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.hrefs.append(dict(attrs).get("href")) + + +@unittest.skipUnless(HUGO, "Hugo is required for outcome rendering tests") +class ResearchOutcomeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="msrg-outcome-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for directory in ("content", "layouts", "assets", "static", "data", "archetypes"): + shutil.copytree(ROOT / directory, self.root / directory) + shutil.copy2(ROOT / "hugo.yaml", self.root / "hugo.yaml") + self.store = ProfileStore(self.root) + + def outcome(self, name): + return {"title": name, "url": f"https://example.org/outcomes/{name}", + "summary": "Public evaluation scripts and sample data."} + + def paper(self, title, year, areas, outcomes=None): + data = {"title": title, "year": year, "authors": ["Example Author"], + "venue": "Example Conference", "publication_type": "Conference Paper", + "external_url": "https://example.org/paper", "research": areas} + if outcomes is not None: + data["research_outcomes"] = outcomes + result = self.store.save({"kind": "publications", "data": data}) + return result["slug"] + + def edit(self, slug, **changes): + record = self.store.get(slug, "publications", include_preview=False) + self.store.save({"kind": "publications", "slug": slug, + "revision": record["revision"], "data": changes}) + + def build(self): + result = subprocess.run( + [HUGO, "--source", str(self.root), "--destination", str(self.root / "public"), + "--cacheDir", str(self.root / "cache"), "--panicOnWarning"], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def page(self, route): + return (self.root / "public" / route.strip("/") / "index.html").read_text() + + def section(self, html, identifier): + match = re.search(rf']*\bid="{identifier}"[^>]*>(.*?)
', html, re.S) + self.assertIsNotNone(match, f"Missing section {identifier}") + return match[1] + + def assert_preview_limit(self, html): + section = self.section(html, "publications") + self.assertEqual(section.count('class="list-card"'), 6) + return Links(section).hrefs + + def test_all_papers_supply_ordered_outcomes_independently_of_recent_citations(self): + first = self.outcome("first") | { + "title": 'Benchmark & "examples"', + "summary": ' is plain text, not executable markup.', + } + second = self.outcome("second") + oldest = self.paper("Zebra old outcome paper", 2000, [AIOPS], [first, second]) + self.paper("Alpha old outcome paper", 2000, [AIOPS], [self.outcome("alphabetical")]) + self.paper("Newer outcome paper", 2001, [AIOPS], [self.outcome("newer")]) + unscoped = self.paper("Outcome without an area", 2000, [], [self.outcome("unscoped")]) + absent = self.paper("Paper without outcomes", 2000, [AIOPS]) + empty = self.paper("Paper with empty outcomes", 2000, [AIOPS], []) + self.build() + + research = self.page("research/rca-aiops") + outcomes = self.section(research, "outcomes") + links = Links(outcomes).hrefs + fixture_links = [url for url in links if url.startswith("https://example.org/outcomes/")] + self.assertEqual(fixture_links, [self.outcome(name)["url"] + for name in ("newer", "alphabetical", "first", "second")]) + owner_url = f"/publications/{oldest}/" + self.assertEqual(links.count(owner_url), 2, "Both outputs link to their owning publication") + self.assertNotIn(owner_url, self.assert_preview_limit(research), "The old paper must be outside the preview") + self.assertNotIn(first["url"], self.page("research/quantum-computing-systems")) + + detail = self.section(self.page(f"publications/{oldest}"), "research-outcomes") + for rendered in (outcomes, detail): + self.assertNotIn("