Skip to content

Add temporal-numeric benchmark and performance evaluation #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
igonzf wants to merge 10 commits into
base: main
Choose a base branch
from
Open

Add temporal-numeric benchmark and performance evaluation #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
55b6582
add complex domain pddl data
igonzf Jul 23, 2026
66d9726
minor changes
igonzf Jul 23, 2026
4962f52
minor change
igonzf Jul 23, 2026
09b3021
rm pddl folder
igonzf Jul 23, 2026
46f26dd
fix dependencies
igonzf Jul 23, 2026
db80de9
add complex domain demo
igonzf Jul 24, 2026
d1702d5
fix dependencies
igonzf Jul 24, 2026
19f0346
restore generators
igonzf Jul 27, 2026
ba962f3
add rosidl-pycommon dependency
igonzf Jul 27, 2026
45a9083
rm rosidl-pycommon
igonzf Jul 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ For run this demo I used the simulated Leo Rover robot in Gazebo Ignition Fortre
| Demo 2 | UPF4ROS2 | 0.18429 | 0.1816 | 0.00868 | 0.1771 | 0.2035 | 10 |
| Demo 3 | UPF | 0.04703 | 0.0466 | 0.0013 | 0.0454 | 0.0498 | 10 |
| Demo 3 | UPF4ROS2 | 0.22318 | 0.2258 | 0.02218 | 0.1767 | 0.2592 | 10 |
| Demo 3 - Harvest | UPF | 0.18 | 0.04965 | 0.0048 | 0.0482 | 0.064 | 10 |
| Demo 3 - Harvest | UPF | 0.0513 | 0.04965 | 0.0048 | 0.0482 | 0.064 | 10 |
| Demo 3 - Harvest | UPF4ROS2 | 0.30883 | 0.31305 | 0.05294 | 0.1778 | 0.3678 | 10 |

## Acknowledgments
Expand Down
51 changes: 51 additions & 0 deletions data/demo_complex_integrated_upf.csv
View file
Open in desktop

Large diffs are not rendered by default.

51 changes: 51 additions & 0 deletions data/demo_complex_integrated_upf4ros2.csv
View file
Open in desktop

Large diffs are not rendered by default.

51 changes: 51 additions & 0 deletions data/demo_complex_pddlfile_upf.csv
View file
Open in desktop

Large diffs are not rendered by default.

51 changes: 51 additions & 0 deletions data/demo_complex_pddlfile_upf4ros2.csv
View file
Open in desktop

Large diffs are not rendered by default.

101 changes: 101 additions & 0 deletions data/overhead_breakdown_raw.csv
View file
Open in desktop

Large diffs are not rendered by default.

111 changes: 111 additions & 0 deletions data/overhead_breakdown_summary.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
demo,configuration,metric,mean,median,stdev,min,max,p95
demo1_pddl_file,direct_upf,total_latency_ms,74.343702,58.530951,30.266507,49.892931,119.468679,118.563279
demo1_pddl_file,direct_upf,server_planning_ms,74.314822,58.502147,30.266604,49.867684,119.436328,118.533449
demo1_pddl_file,direct_upf,server_response_preparation_ms,0.028880,0.028804,0.002737,0.025247,0.033713,0.033100
demo1_pddl_file,direct_upf,ros_and_queue_overhead_ms,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
demo1_pddl_file,direct_upf,plan_length,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,direct_upf,num_objects,2,2.000000,0.000000,2,2,2.000000
demo1_pddl_file,direct_upf,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,direct_upf,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,direct_upf,num_goals,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,direct_upf,domain_pddl_bytes,470,470.000000,0.000000,470,470,470.000000
demo1_pddl_file,direct_upf,problem_pddl_bytes,268,268.000000,0.000000,268,268,268.000000
demo1_pddl_file,upf4ros2,total_latency_ms,62.478503,62.435538,1.953442,59.355084,65.166639,64.878202
demo1_pddl_file,upf4ros2,server_planning_ms,60.743779,60.454833,1.811879,58.416687,63.335109,63.255427
demo1_pddl_file,upf4ros2,server_response_preparation_ms,0.164634,0.158547,0.020299,0.139241,0.198993,0.197437
demo1_pddl_file,upf4ros2,ros_and_queue_overhead_ms,1.570090,1.701387,0.407962,0.742861,1.999124,1.950528
demo1_pddl_file,upf4ros2,plan_length,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,upf4ros2,num_objects,2,2.000000,0.000000,2,2,2.000000
demo1_pddl_file,upf4ros2,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,upf4ros2,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,upf4ros2,num_goals,1,1.000000,0.000000,1,1,1.000000
demo1_pddl_file,upf4ros2,domain_pddl_bytes,470,470.000000,0.000000,470,470,470.000000
demo1_pddl_file,upf4ros2,problem_pddl_bytes,268,268.000000,0.000000,268,268,268.000000
demo1_ros2_node,direct_upf,total_latency_ms,1.393708,1.278314,0.336403,1.060245,2.050310,1.964528
demo1_ros2_node,direct_upf,server_planning_ms,1.360200,1.243860,0.335251,1.028235,2.022598,1.928782
demo1_ros2_node,direct_upf,server_response_preparation_ms,0.033508,0.031630,0.005869,0.027712,0.045566,0.044118
demo1_ros2_node,direct_upf,ros_and_queue_overhead_ms,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
demo1_ros2_node,direct_upf,plan_length,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,direct_upf,num_objects,2,2.000000,0.000000,2,2,2.000000
demo1_ros2_node,direct_upf,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,direct_upf,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,direct_upf,num_goals,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,direct_upf,domain_pddl_bytes,312,312.000000,0.000000,312,312,312.000000
demo1_ros2_node,direct_upf,problem_pddl_bytes,209,209.000000,0.000000,209,209,209.000000
demo1_ros2_node,upf4ros2,total_latency_ms,4.984597,4.461436,0.960380,4.062149,6.659513,6.596607
demo1_ros2_node,upf4ros2,server_planning_ms,1.838025,1.701863,0.379500,1.583777,2.812708,2.510146
demo1_ros2_node,upf4ros2,server_response_preparation_ms,0.124399,0.116984,0.024459,0.096430,0.185017,0.166383
demo1_ros2_node,upf4ros2,ros_and_queue_overhead_ms,3.022174,2.683171,0.684069,2.335895,4.391174,4.164503
demo1_ros2_node,upf4ros2,plan_length,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,upf4ros2,num_objects,2,2.000000,0.000000,2,2,2.000000
demo1_ros2_node,upf4ros2,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,upf4ros2,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,upf4ros2,num_goals,1,1.000000,0.000000,1,1,1.000000
demo1_ros2_node,upf4ros2,domain_pddl_bytes,312,312.000000,0.000000,312,312,312.000000
demo1_ros2_node,upf4ros2,problem_pddl_bytes,209,209.000000,0.000000,209,209,209.000000
demo2,direct_upf,total_latency_ms,8.892544,1.792037,22.794483,0.999612,73.742852,41.920060
demo2,direct_upf,server_planning_ms,8.855397,1.748155,22.795593,0.972892,73.709159,41.888310
demo2,direct_upf,server_response_preparation_ms,0.037147,0.033823,0.010083,0.025147,0.053050,0.052148
demo2,direct_upf,ros_and_queue_overhead_ms,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
demo2,direct_upf,plan_length,1,1.000000,0.000000,1,1,1.000000
demo2,direct_upf,num_objects,2,2.000000,0.000000,2,2,2.000000
demo2,direct_upf,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo2,direct_upf,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo2,direct_upf,num_goals,1,1.000000,0.000000,1,1,1.000000
demo2,direct_upf,domain_pddl_bytes,312,312.000000,0.000000,312,312,312.000000
demo2,direct_upf,problem_pddl_bytes,209,209.000000,0.000000,209,209,209.000000
demo2,upf4ros2,total_latency_ms,4.661713,4.569724,1.459277,2.887929,7.178225,6.853053
demo2,upf4ros2,server_planning_ms,1.908865,1.817354,0.548383,1.311577,3.283670,2.804679
demo2,upf4ros2,server_response_preparation_ms,0.119443,0.118368,0.028695,0.062797,0.175008,0.158493
demo2,upf4ros2,ros_and_queue_overhead_ms,2.633405,2.476368,1.173841,1.513555,5.312640,4.355894
demo2,upf4ros2,plan_length,1,1.000000,0.000000,1,1,1.000000
demo2,upf4ros2,num_objects,2,2.000000,0.000000,2,2,2.000000
demo2,upf4ros2,num_fluents,1,1.000000,0.000000,1,1,1.000000
demo2,upf4ros2,num_domain_actions,1,1.000000,0.000000,1,1,1.000000
demo2,upf4ros2,num_goals,1,1.000000,0.000000,1,1,1.000000
demo2,upf4ros2,domain_pddl_bytes,312,312.000000,0.000000,312,312,312.000000
demo2,upf4ros2,problem_pddl_bytes,209,209.000000,0.000000,209,209,209.000000
demo3,direct_upf,total_latency_ms,3.251885,2.736641,1.052083,2.240046,4.994094,4.817697
demo3,direct_upf,server_planning_ms,3.170578,2.674450,1.023490,2.183550,4.865072,4.691934
demo3,direct_upf,server_response_preparation_ms,0.081306,0.063189,0.028850,0.056496,0.129022,0.125763
demo3,direct_upf,ros_and_queue_overhead_ms,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
demo3,direct_upf,plan_length,6,6.000000,0.000000,6,6,6.000000
demo3,direct_upf,num_objects,6,6.000000,0.000000,6,6,6.000000
demo3,direct_upf,num_fluents,2,2.000000,0.000000,2,2,2.000000
demo3,direct_upf,num_domain_actions,2,2.000000,0.000000,2,2,2.000000
demo3,direct_upf,num_goals,3,3.000000,0.000000,3,3,3.000000
demo3,direct_upf,domain_pddl_bytes,479,479.000000,0.000000,479,479,479.000000
demo3,direct_upf,problem_pddl_bytes,296,296.000000,0.000000,296,296,296.000000
demo3,upf4ros2,total_latency_ms,9.508368,7.684634,5.332190,6.741537,24.437402,17.711876
demo3,upf4ros2,server_planning_ms,3.643518,3.446035,0.474181,3.272049,4.857368,4.405266
demo3,upf4ros2,server_response_preparation_ms,0.203812,0.193637,0.040151,0.162264,0.289643,0.265825
demo3,upf4ros2,ros_and_queue_overhead_ms,5.661038,3.957477,5.217640,3.167693,20.378480,13.663161
demo3,upf4ros2,plan_length,6,6.000000,0.000000,6,6,6.000000
demo3,upf4ros2,num_objects,6,6.000000,0.000000,6,6,6.000000
demo3,upf4ros2,num_fluents,2,2.000000,0.000000,2,2,2.000000
demo3,upf4ros2,num_domain_actions,2,2.000000,0.000000,2,2,2.000000
demo3,upf4ros2,num_goals,3,3.000000,0.000000,3,3,3.000000
demo3,upf4ros2,domain_pddl_bytes,479,479.000000,0.000000,479,479,479.000000
demo3,upf4ros2,problem_pddl_bytes,296,296.000000,0.000000,296,296,296.000000
demo3_harvest,direct_upf,total_latency_ms,6.785032,6.280664,1.167131,5.951267,9.464347,8.967715
demo3_harvest,direct_upf,server_planning_ms,6.723338,6.221458,1.167098,5.890754,9.403994,8.906334
demo3_harvest,direct_upf,server_response_preparation_ms,0.061694,0.061400,0.002516,0.056987,0.065753,0.065456
demo3_harvest,direct_upf,ros_and_queue_overhead_ms,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
demo3_harvest,direct_upf,plan_length,4,4.000000,0.000000,4,4,4.000000
demo3_harvest,direct_upf,num_objects,7,7.000000,0.000000,7,7,7.000000
demo3_harvest,direct_upf,num_fluents,6,6.000000,0.000000,6,6,6.000000
demo3_harvest,direct_upf,num_domain_actions,3,3.000000,0.000000,3,3,3.000000
demo3_harvest,direct_upf,num_goals,2,2.000000,0.000000,2,2,2.000000
demo3_harvest,direct_upf,domain_pddl_bytes,903,903.000000,0.000000,903,903,903.000000
demo3_harvest,direct_upf,problem_pddl_bytes,597,597.000000,0.000000,597,597,597.000000
demo3_harvest,upf4ros2,total_latency_ms,19.605915,18.674889,3.599084,16.763027,28.757614,25.872038
demo3_harvest,upf4ros2,server_planning_ms,10.340151,9.683382,2.109900,8.826442,15.977937,13.708332
demo3_harvest,upf4ros2,server_response_preparation_ms,0.247226,0.216681,0.073864,0.189135,0.435586,0.373856
demo3_harvest,upf4ros2,ros_and_queue_overhead_ms,9.018538,8.395325,1.622045,7.601078,12.344091,11.834375
demo3_harvest,upf4ros2,plan_length,4,4.000000,0.000000,4,4,4.000000
demo3_harvest,upf4ros2,num_objects,7,7.000000,0.000000,7,7,7.000000
demo3_harvest,upf4ros2,num_fluents,6,6.000000,0.000000,6,6,6.000000
demo3_harvest,upf4ros2,num_domain_actions,3,3.000000,0.000000,3,3,3.000000
demo3_harvest,upf4ros2,num_goals,2,2.000000,0.000000,2,2,2.000000
demo3_harvest,upf4ros2,domain_pddl_bytes,903,903.000000,0.000000,903,903,903.000000
demo3_harvest,upf4ros2,problem_pddl_bytes,597,597.000000,0.000000,597,597,597.000000
18 changes: 9 additions & 9 deletions upf4ros2/upf4ros2/ros2_interface_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@

from unified_planning import Environment
from unified_planning import model
from unified_planning.exceptions import UPException, UPValueError
from unified_planning import shortcuts
from unified_planning.engines.results import PlanGenerationResultStatus
from unified_planning.exceptions import UPException, UPValueError
from unified_planning.model import (
DurativeAction,
Effect,
Expand All @@ -45,9 +46,8 @@
from unified_planning.model.effect import EffectKind
from unified_planning.model.operators import OperatorKind
import unified_planning.plans
from upf4ros2.converter import Converter, handles
from unified_planning.plans.plan import PlanKind
from unified_planning.engines.results import PlanGenerationResultStatus
from upf4ros2.converter import Converter, handles
# from upf4ros2.ros2_utils import print_expr

from upf_msgs import msg as msgs
Expand Down Expand Up @@ -367,7 +367,7 @@ def _convert_decomposition(

else:
raise ValueError(
f"Unknown subtask type: {type(subtask.task)}")
f'Unknown subtask type: {type(subtask.task)}')

return decomposition

Expand All @@ -383,18 +383,18 @@ def _convert_schedule(

for item in expr.expressions:
var_name = item.type
container, kind = var_name.split(".")
container, kind = var_name.split('.')
activity = problem.get_activity(container)

if ".start" in var_name or ".end" in var_name:
if kind == "start":
if '.start' in var_name or '.end' in var_name:
if kind == 'start':
var = model.Timepoint(
model.TimepointKind.START, container)
elif kind == "end":
elif kind == 'end':
var = model.Timepoint(
model.TimepointKind.END, container)
else:
raise ValueError(f"Invalid timepoint kind: {kind}")
raise ValueError(f'Invalid timepoint kind: {kind}')
else:
var = activity.get_parameter(kind)
if var is None:
Expand Down
10 changes: 5 additions & 5 deletions upf4ros2/upf4ros2/ros2_interface_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

from unified_planning import model
from unified_planning.engines import PlanGenerationResult
from unified_planning.environment import get_environment
from unified_planning.exceptions import UPException
import unified_planning.model.htn
from unified_planning.model.operators import (
Expand All @@ -33,7 +34,6 @@
)
from unified_planning.model.timing import TimepointKind
from unified_planning.model.types import domain_item, domain_size
from unified_planning.environment import get_environment
import unified_planning.model.walkers as walkers
from upf4ros2.converter import Converter, handles
# from upf4ros2.ros2_utils import print_expr
Expand Down Expand Up @@ -691,11 +691,11 @@ def _convert_schedule(

if isinstance(var, model.Timepoint):
if var.kind == TimepointKind.START:
var = f"{var.container}.start"
var = f'{var.container}.start'
elif var.kind == TimepointKind.END:
var = f"{var.container}.end"
var = f'{var.container}.end'
else:
raise ValueError(f"Invalid timepoint in assignment: {var}")
raise ValueError(f'Invalid timepoint in assignment: {var}')
else:
assert isinstance(var, model.Parameter)
var = var.name
Expand Down Expand Up @@ -892,7 +892,7 @@ def _convert_hierarchical_plan(
elif isinstance(plan._flat_plan, unified_planning.plans.TimeTriggeredPlan):
ret = self._convert_time_triggered_plan(plan._flat_plan)
else:
raise UPException(f"Unknown plan: {type(plan._flat_plan)}")
raise UPException(f'Unknown plan: {type(plan._flat_plan)}')

ret.kind = 6
return ret
Expand Down
Loading