From ab0f08e9de4b4d94a204703cccb6a8d16c6d4dfe Mon Sep 17 00:00:00 2001 From: saket0187 Date: Wed, 3 Jun 2026 19:27:38 +0530 Subject: [PATCH 1/2] Add format options in t.rast.univar --- python/grass/temporal/univar_statistics.py | 45 ++++++-- temporal/t.rast.univar/t.rast.univar.md | 74 ++++++++++++ temporal/t.rast.univar/t.rast.univar.py | 36 +++++- .../testsuite/test_t_rast_univar.py | 108 ++++++++++++++++++ 4 files changed, 251 insertions(+), 12 deletions(-) diff --git a/python/grass/temporal/univar_statistics.py b/python/grass/temporal/univar_statistics.py index 54448d078f0..17ffa80d172 100755 --- a/python/grass/temporal/univar_statistics.py +++ b/python/grass/temporal/univar_statistics.py @@ -125,6 +125,7 @@ def print_gridded_dataset_univar_statistics( zones=None, percentile=None, nprocs: int = 1, + format: str = "plain", ) -> None: """Print univariate statistics for a space time raster or raster3d dataset. @@ -140,6 +141,7 @@ def print_gridded_dataset_univar_statistics( :param no_header: Suppress the printing of column names :param fs: Field separator :param nprocs: Number of cores to use for processing + :param format: Output format ("plain", "csv", or "json"). Default is "plain". :param rast_region: If set True ignore the current region settings and use the raster map regions for univar statistical calculation. :param region_relation: Process only maps with the given spatial relation @@ -192,7 +194,7 @@ def print_gridded_dataset_univar_statistics( if output is not None: out_file = open(output, "w") - if no_header is False: + if no_header is False or format == "json": cols = ( ["id", "semantic_label", "start", "end"] if type == "strds" @@ -225,12 +227,12 @@ def print_gridded_dataset_univar_statistics( for perc in percentile ] ) - string = fs.join(cols) - - if output is None: - print(string) - else: - out_file.write(string + "\n") + if no_header is False and format != "json": + string = fs.join(cols) + if output is None: + print(string) + else: + out_file.write(string + "\n") # Define flags flag = "g" @@ -265,10 +267,35 @@ def print_gridded_dataset_univar_statistics( compute_univar_stats, [(dict(row), univar_module, fs) for row in rows] ) + if format == "json": + # lazy import + import json + + json_out = [] + for block in filter(None, strings): + for line in block.splitlines(): + if line.strip(): + row = {} + for k, v in zip(cols, line.split(fs), strict=False): + if not v or v == "None": + row[k] = None + else: + try: + row[k] = ( + float(v) if "." in v or "e" in v.lower() else int(v) + ) + except ValueError: + row[k] = v + json_out.append(row) + + output_str = json.dumps(json_out, indent=4) + else: + output_str = "\n".join(filter(None, strings)) + if output is None: - print("\n".join(filter(None, strings))) + print(output_str) else: - out_file.write("\n".join(filter(None, strings))) + out_file.write(output_str + "\n") dbif.close() diff --git a/temporal/t.rast.univar/t.rast.univar.md b/temporal/t.rast.univar/t.rast.univar.md index ddfaa178198..031fc862a22 100644 --- a/temporal/t.rast.univar/t.rast.univar.md +++ b/temporal/t.rast.univar/t.rast.univar.md @@ -44,6 +44,80 @@ id|start|end|mean|min|max|mean_of_abs|stddev|variance|coeff_var|sum|null_cells|c 2012_12_tempmean@climate_2009_2012|2012-12-01 00:00:00|2013-01-01 00:00:00|8.71|1.76|11.98|8.71|1.72|2.95|19.74|4418403.77|503233|1010600|7.84|8.95|9.99|10.67 ``` +You can use format options as well to output the statistics in +JSON, CSV or Plain (default) formats. For example, to get JSON +output with extended statistics: + +```sh +t.rast.univar -e input=nc_lst_daily format=json +``` + +```json +[ + { + "id": "aqua_lst_day20020705@modis2002lst", + "semantic_label": null, + "start": "2002-01-02 00:00:00", + "end": "2002-01-03 00:00:00", + "mean": 36.6047213436794, + "min": 29.85, + "max": 41.11, + "mean_of_abs": 36.6047213436794, + "stddev": 2.74434797543588, + "variance": 7.53144581027899, + "coeff_var": 7.49725137822896, + "sum": 3726726.68, + "null_cells": 148515, + "cells": 101810, + "non_null_cells": 101810, + "first_quartile": 35.17, + "median": 36.23, + "third_quartile": 39.05 + }, + { + "id": "aqua_lst_day20020706@modis2002lst", + "semantic_label": null, + "start": "2002-01-03 00:00:00", + "end": "2002-01-04 00:00:00", + "mean": 28.5035320158103, + "min": 27.7500000000001, + "max": 29.5700000000001, + "mean_of_abs": 28.5035320158103, + "stddev": 0.570228624574918, + "variance": 0.325160684284603, + "coeff_var": 2.00055426204242, + "sum": 1081709.04, + "null_cells": 212375, + "cells": 37950, + "non_null_cells": 37950, + "first_quartile": 27.91, + "median": 28.41, + "third_quartile": 28.85 + }, + { + "id": "aqua_lst_day20020707@modis2002lst", + "semantic_label": null, + "start": "2002-01-04 00:00:00", + "end": "2002-01-05 00:00:00", + "mean": 34.6977710657791, + "min": 26.67, + "max": 43.27, + "mean_of_abs": 34.6977710657791, + "stddev": 3.42837919872729, + "variance": 11.7537839302659, + "coeff_var": 9.88069000809261, + "sum": 7771502.67000001, + "null_cells": 26348, + "cells": 223977, + "non_null_cells": 223977, + "first_quartile": 32.17, + "median": 35.07, + "third_quartile": 36.57 + }, + ... +] +``` + ## SEE ALSO *[t.create](t.create.md), [t.info](t.info.md) [t.create](r.univar.md),* diff --git a/temporal/t.rast.univar/t.rast.univar.py b/temporal/t.rast.univar/t.rast.univar.py index dbc2a31ab4d..59f628d5ec7 100755 --- a/temporal/t.rast.univar/t.rast.univar.py +++ b/temporal/t.rast.univar/t.rast.univar.py @@ -70,6 +70,13 @@ # %option G_OPT_F_SEP # % label: Field separator character between the output columns +# % answer: +# % guisection: Formatting +# %end + +# %option G_OPT_F_FORMAT +# % options: plain,json,csv +# % descriptions: plain;Plain text output;json;JSON (JavaScript Object Notation);csv;CSV (Comma Separated Values) # % guisection: Formatting # %end @@ -102,9 +109,6 @@ def main(): # Get the options and flags options, flags = gs.parser() - # lazy imports - import grass.temporal as tgis - # Define variables input = options["input"] zones = options["zones"] @@ -116,6 +120,31 @@ def main(): no_header = flags["u"] rast_region = bool(flags["r"]) separator = gs.separator(options["separator"]) + output_format = options.get("format", "plain") + + if output_format == "csv": + if not separator: + separator = "," + elif len(separator) > 1: + gs.fatal( + _("A standard CSV separator (delimiter) is only one character long") + ) + + elif output_format == "json": + if no_header: + gs.fatal(_("Column names are always included in JSON output")) + if separator: + gs.fatal(_("Separator option is not allowed with JSON format")) + + # We need this in univar_statistics.py for data split + separator = "|" + + else: + separator = "|" + + # lazy imports + import grass.temporal as tgis + percentile = None if options["percentile"]: try: @@ -150,6 +179,7 @@ def main(): rast_region=rast_region, region_relation=region_relation, nprocs=nprocs, + format=output_format, ) diff --git a/temporal/t.rast.univar/testsuite/test_t_rast_univar.py b/temporal/t.rast.univar/testsuite/test_t_rast_univar.py index c36fb4a9b9a..b77976411ba 100644 --- a/temporal/t.rast.univar/testsuite/test_t_rast_univar.py +++ b/temporal/t.rast.univar/testsuite/test_t_rast_univar.py @@ -8,6 +8,7 @@ @author Soeren Gebbert """ +import json from pathlib import Path from grass.gunittest.case import TestCase @@ -549,6 +550,113 @@ def test_with_spatial_filter_is_contained(self): res_line = res.split("|", 1)[1] self.assertLooksLike(ref_line, res_line) + def test_format_csv(self): + """Test output generation in CSV format""" + t_rast_univar = SimpleModule( + "t.rast.univar", + input="A", + where="start_time >= '2001-01-01'", + format="csv", + overwrite=True, + verbose=True, + ) + self.runModule("g.region", **self.default_region, res=1) + self.assertModule(t_rast_univar) + + univar_text = """id,semantic_label,start,end,mean,min,max,mean_of_abs,stddev,variance,coeff_var,sum,null_cells,cells,non_null_cells +a_1@testing,,2001-01-01 00:00:00,2001-04-01 00:00:00,100,100,100,100,0,0,0,960000,0,9600,9600 +a_2@testing,,2001-04-01 00:00:00,2001-07-01 00:00:00,200,200,200,200,0,0,0,1920000,0,9600,9600 +a_3@testing,,2001-07-01 00:00:00,2001-10-01 00:00:00,300,300,300,300,0,0,0,2880000,0,9600,9600 +a_4@testing,,2001-10-01 00:00:00,2002-01-01 00:00:00,400,400,400,400,0,0,0,3840000,0,9600,9600 +""" + for ref, res in zip( + univar_text.split("\n"), + t_rast_univar.outputs.stdout.split("\n"), + strict=False, + ): + if ref and res: + ref_line = ref.split(",", 1)[1] + res_line = res.split(",", 1)[1] + self.assertLooksLike(ref_line, res_line) + + def test_format_json(self): + """Test output generation in JSON format""" + t_rast_univar = SimpleModule( + "t.rast.univar", + input="A", + where="start_time >= '2001-04-01 00:00:00'", + format="json", + overwrite=True, + verbose=True, + ) + self.runModule("g.region", **self.default_region, res=1) + self.assertModule(t_rast_univar) + + output_json = json.loads(t_rast_univar.outputs.stdout) + + expected_json = [ + { + "id": "a_2", + "semantic_label": None, + "start": "2001-04-01 00:00:00", + "end": "2001-07-01 00:00:00", + "mean": 200, + "min": 200, + "max": 200, + "mean_of_abs": 200, + "stddev": 0, + "variance": 0, + "coeff_var": 0, + "sum": 1920000, + "null_cells": 0, + "cells": 9600, + "non_null_cells": 9600, + }, + { + "id": "a_3", + "semantic_label": None, + "start": "2001-07-01 00:00:00", + "end": "2001-10-01 00:00:00", + "mean": 300, + "min": 300, + "max": 300, + "mean_of_abs": 300, + "stddev": 0, + "variance": 0, + "coeff_var": 0, + "sum": 2880000, + "null_cells": 0, + "cells": 9600, + "non_null_cells": 9600, + }, + { + "id": "a_4", + "semantic_label": None, + "start": "2001-10-01 00:00:00", + "end": "2002-01-01 00:00:00", + "mean": 400, + "min": 400, + "max": 400, + "mean_of_abs": 400, + "stddev": 0, + "variance": 0, + "coeff_var": 0, + "sum": 3840000, + "null_cells": 0, + "cells": 9600, + "non_null_cells": 9600, + }, + ] + + self.assertEqual(len(output_json), len(expected_json)) + + for res_obj, exp_obj in zip(output_json, expected_json, strict=False): + for key in exp_obj: + if key == "id": + self.assertTrue(res_obj[key].startswith(exp_obj[key])) + else: + self.assertEqual(res_obj[key], exp_obj[key]) + if __name__ == "__main__": from grass.gunittest.main import test From a4f96e5456f94a5f35075e5b3ac4796bb47875f1 Mon Sep 17 00:00:00 2001 From: saket0187 Date: Mon, 8 Jun 2026 14:22:46 +0530 Subject: [PATCH 2/2] Fixed issues --- python/grass/temporal/univar_statistics.py | 71 +++++++---- temporal/t.rast.univar/t.rast.univar.md | 117 +++++++++++------- temporal/t.rast.univar/t.rast.univar.py | 5 +- .../testsuite/test_t_rast_univar.py | 14 +-- 4 files changed, 122 insertions(+), 85 deletions(-) diff --git a/python/grass/temporal/univar_statistics.py b/python/grass/temporal/univar_statistics.py index 17ffa80d172..730a55a26bc 100755 --- a/python/grass/temporal/univar_statistics.py +++ b/python/grass/temporal/univar_statistics.py @@ -23,6 +23,7 @@ from multiprocessing import Pool from subprocess import PIPE +import json import grass.script as gs from grass.pygrass.modules import Module @@ -35,7 +36,11 @@ def compute_univar_stats( - registered_map_info, stats_module, fs, rast_region: bool = False + registered_map_info, + stats_module, + fs, + rast_region: bool = False, + output_format: str | None = None, ): """Compute univariate statistics for a map of a space time raster or raster3d dataset @@ -47,6 +52,7 @@ def compute_univar_stats( :param rast_region: If set True ignore the current region settings and use the raster map regions for univar statistical calculation. Only available for strds. + :param output_format: Output format specification """ string = "" id = registered_map_info["id"] @@ -72,6 +78,34 @@ def compute_univar_stats( else _("Unable to get statistics for 3d raster map <%s>") % id ) return None + + if output_format == "json": + try: + stats = json.loads(univar_stats) + except ValueError: + gs.warning(_("Unable to parse JSON output for map <%s>") % id) + return None + + if isinstance(stats, dict): + stats = [stats] + + processed_stats = [] + for s in stats: + new_s = { + "id": id, + "semantic_label": semantic_label + if stats_module.name == "r.univar" and semantic_label + else None, + "start": str(start) if start else None, + "end": str(end) if end else None, + } + + new_s.update(s) + + processed_stats.append(new_s) + + return processed_stats + eol = "" for idx, stats_kv in enumerate(univar_stats.split(";")): @@ -194,7 +228,7 @@ def print_gridded_dataset_univar_statistics( if output is not None: out_file = open(output, "w") - if no_header is False or format == "json": + if no_header is False: cols = ( ["id", "semantic_label", "start", "end"] if type == "strds" @@ -235,7 +269,9 @@ def print_gridded_dataset_univar_statistics( out_file.write(string + "\n") # Define flags - flag = "g" + flag = "" + if format != "json": + flag = "g" if extended is True: flag += "e" if type == "strds" and rast_region is True and not zones: @@ -248,45 +284,28 @@ def print_gridded_dataset_univar_statistics( zones=zones, percentile=percentile, stdout_=PIPE, + format="json" if format == "json" else None, run_=False, ) nprocs = max(nprocs, 1) if nprocs == 1: strings = [ - compute_univar_stats( - row, - univar_module, - fs, - ) + compute_univar_stats(row, univar_module, fs, rast_region, format) for row in rows ] else: with Pool(min(nprocs, len(rows))) as pool: strings = pool.starmap( - compute_univar_stats, [(dict(row), univar_module, fs) for row in rows] + compute_univar_stats, + [(dict(row), univar_module, fs, rast_region, format) for row in rows], ) if format == "json": - # lazy import - import json - json_out = [] + for block in filter(None, strings): - for line in block.splitlines(): - if line.strip(): - row = {} - for k, v in zip(cols, line.split(fs), strict=False): - if not v or v == "None": - row[k] = None - else: - try: - row[k] = ( - float(v) if "." in v or "e" in v.lower() else int(v) - ) - except ValueError: - row[k] = v - json_out.append(row) + json_out.extend(block) output_str = json.dumps(json_out, indent=4) else: diff --git a/temporal/t.rast.univar/t.rast.univar.md b/temporal/t.rast.univar/t.rast.univar.md index 031fc862a22..46d0a0643bd 100644 --- a/temporal/t.rast.univar/t.rast.univar.md +++ b/temporal/t.rast.univar/t.rast.univar.md @@ -57,62 +57,83 @@ t.rast.univar -e input=nc_lst_daily format=json { "id": "aqua_lst_day20020705@modis2002lst", "semantic_label": null, - "start": "2002-01-02 00:00:00", - "end": "2002-01-03 00:00:00", - "mean": 36.6047213436794, - "min": 29.85, - "max": 41.11, - "mean_of_abs": 36.6047213436794, - "stddev": 2.74434797543588, - "variance": 7.53144581027899, - "coeff_var": 7.49725137822896, - "sum": 3726726.68, - "null_cells": 148515, - "cells": 101810, - "non_null_cells": 101810, - "first_quartile": 35.17, - "median": 36.23, - "third_quartile": 39.05 + "start": "2002-07-05 00:00:00", + "end": "2002-07-06 00:00:00", + "n": 448428, + "null_cells": 390977, + "cells": 839405, + "min": 25.730000000000018, + "max": 45.77000000000004, + "range": 20.04000000000002, + "mean": 36.24265068193782, + "mean_of_abs": 36.24265068193782, + "stddev": 2.6171016196159838, + "variance": 6.849220887396605, + "coeff_var": 7.221054670044495, + "sum": 16252219.360000012, + "first_quartile": 34.49000000000001, + "median": 36.07000000000005, + "third_quartile": 37.83000000000004, + "percentiles": [ + { + "percentile": 90, + "value": 40.01000000000005 + } + ] }, { "id": "aqua_lst_day20020706@modis2002lst", "semantic_label": null, - "start": "2002-01-03 00:00:00", - "end": "2002-01-04 00:00:00", - "mean": 28.5035320158103, - "min": 27.7500000000001, - "max": 29.5700000000001, - "mean_of_abs": 28.5035320158103, - "stddev": 0.570228624574918, - "variance": 0.325160684284603, - "coeff_var": 2.00055426204242, - "sum": 1081709.04, - "null_cells": 212375, - "cells": 37950, - "non_null_cells": 37950, - "first_quartile": 27.91, - "median": 28.41, - "third_quartile": 28.85 + "start": "2002-07-06 00:00:00", + "end": "2002-07-07 00:00:00", + "n": 246658, + "null_cells": 592747, + "cells": 839405, + "min": 25.630000000000052, + "max": 34.81, + "range": 9.17999999999995, + "mean": 28.219656852808374, + "mean_of_abs": 28.219656852808374, + "stddev": 1.6350578714604542, + "variance": 2.6734142430247916, + "coeff_var": 5.794038814819025, + "sum": 6960604.120000008, + "first_quartile": 27.110000000000014, + "median": 27.970000000000027, + "third_quartile": 28.970000000000027, + "percentiles": [ + { + "percentile": 90, + "value": 30.450000000000045 + } + ] }, { "id": "aqua_lst_day20020707@modis2002lst", "semantic_label": null, - "start": "2002-01-04 00:00:00", - "end": "2002-01-05 00:00:00", - "mean": 34.6977710657791, - "min": 26.67, - "max": 43.27, - "mean_of_abs": 34.6977710657791, - "stddev": 3.42837919872729, - "variance": 11.7537839302659, - "coeff_var": 9.88069000809261, - "sum": 7771502.67000001, - "null_cells": 26348, - "cells": 223977, - "non_null_cells": 223977, - "first_quartile": 32.17, - "median": 35.07, - "third_quartile": 36.57 + "start": "2002-07-07 00:00:00", + "end": "2002-07-08 00:00:00", + "n": 714454, + "null_cells": 124951, + "cells": 839405, + "min": 25.630000000000052, + "max": 44.670000000000016, + "range": 19.039999999999964, + "mean": 32.642155464172674, + "mean_of_abs": 32.642155464172674, + "stddev": 3.1874224286049917, + "variance": 10.159661738374142, + "coeff_var": 9.764742503305083, + "sum": 23321318.54000002, + "first_quartile": 30.350000000000023, + "median": 32.450000000000045, + "third_quartile": 34.77000000000004, + "percentiles": [ + { + "percentile": 90, + "value": 36.950000000000045 + } + ] }, ... ] diff --git a/temporal/t.rast.univar/t.rast.univar.py b/temporal/t.rast.univar/t.rast.univar.py index 59f628d5ec7..a01556ae233 100755 --- a/temporal/t.rast.univar/t.rast.univar.py +++ b/temporal/t.rast.univar/t.rast.univar.py @@ -136,10 +136,7 @@ def main(): if separator: gs.fatal(_("Separator option is not allowed with JSON format")) - # We need this in univar_statistics.py for data split - separator = "|" - - else: + elif not separator: separator = "|" # lazy imports diff --git a/temporal/t.rast.univar/testsuite/test_t_rast_univar.py b/temporal/t.rast.univar/testsuite/test_t_rast_univar.py index b77976411ba..d30994a0cfe 100644 --- a/temporal/t.rast.univar/testsuite/test_t_rast_univar.py +++ b/temporal/t.rast.univar/testsuite/test_t_rast_univar.py @@ -570,9 +570,9 @@ def test_format_csv(self): a_4@testing,,2001-10-01 00:00:00,2002-01-01 00:00:00,400,400,400,400,0,0,0,3840000,0,9600,9600 """ for ref, res in zip( - univar_text.split("\n"), - t_rast_univar.outputs.stdout.split("\n"), - strict=False, + univar_text.splitlines(), + t_rast_univar.outputs.stdout.splitlines(), + strict=True, ): if ref and res: ref_line = ref.split(",", 1)[1] @@ -610,7 +610,7 @@ def test_format_json(self): "sum": 1920000, "null_cells": 0, "cells": 9600, - "non_null_cells": 9600, + "n": 9600, }, { "id": "a_3", @@ -627,7 +627,7 @@ def test_format_json(self): "sum": 2880000, "null_cells": 0, "cells": 9600, - "non_null_cells": 9600, + "n": 9600, }, { "id": "a_4", @@ -644,13 +644,13 @@ def test_format_json(self): "sum": 3840000, "null_cells": 0, "cells": 9600, - "non_null_cells": 9600, + "n": 9600, }, ] self.assertEqual(len(output_json), len(expected_json)) - for res_obj, exp_obj in zip(output_json, expected_json, strict=False): + for res_obj, exp_obj in zip(output_json, expected_json, strict=True): for key in exp_obj: if key == "id": self.assertTrue(res_obj[key].startswith(exp_obj[key]))