From f080d4c3f38648e965296320ad17590586dc194f Mon Sep 17 00:00:00 2001 From: Tanmayee Sharvani Date: Fri, 31 Jul 2026 12:26:43 +0530 Subject: [PATCH 1/2] Add explicit dGPU target device support --- README.md | 26 +++++++++++++ benchmark-scripts/Makefile | 2 +- benchmark-scripts/benchmark.py | 4 +- benchmark-scripts/benchmark_order_accuracy.py | 6 +-- benchmark-scripts/benchmark_test.py | 24 ++++++++++++ benchmark-scripts/device_validation.py | 34 +++++++++++++++++ benchmark-scripts/device_validation_test.py | 37 +++++++++++++++++++ 7 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 benchmark-scripts/device_validation.py create mode 100644 benchmark-scripts/device_validation_test.py diff --git a/README.md b/README.md index fe2daad4..68a28c61 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,32 @@ - [Open source code for setup, install, and execution of software, with complete developer documentation](https://intel-retail.github.io/documentation/performance-tools/benchmark.html) - [Developer focused website to enable developers to engage and build our partner community](https://www.intel.com/content/www/us/en/developer/articles/reference-implementation/automated-self-checkout.html) +## Benchmark Target Device + +Benchmark scripts support explicit target-device selection via `--target_device` or `TARGET_DEVICE`. + +Accepted values: + +- `CPU` +- `GPU` +- `GPU.` (for example: `GPU.0`, `GPU.1`, `GPU.2`) +- `NPU` + +Examples: + +```bash +# Existing workflow (still supported) +make benchmark TARGET_DEVICE=GPU + +# Explicit GPU device selection +make benchmark TARGET_DEVICE=GPU.1 + +# Direct script usage +python benchmark-scripts/benchmark.py \ + --compose_file ./docker/docker-compose.yaml \ + --target_device GPU.1 +``` + ## Disclaimer GStreamer is an open source framework licensed under LGPL. See https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language=c. You are solely responsible for determining if your use of Gstreamer requires any additional licenses. Intel is not responsible for obtaining any such licenses, nor liable for any licensing fees due, in connection with your use of Gstreamer diff --git a/benchmark-scripts/Makefile b/benchmark-scripts/Makefile index bfca2548..3725c68a 100644 --- a/benchmark-scripts/Makefile +++ b/benchmark-scripts/Makefile @@ -14,7 +14,7 @@ plot: init-packages python3 usage_graph_plot.py --dir $(ROOT_DIRECTORY)/ python-test: - python -m coverage run -m unittest benchmark_test.py stream_density_test.py + python -m coverage run -m unittest benchmark_test.py stream_density_test.py device_validation_test.py python-integration: python -m coverage run -m unittest benchmark_integration.py diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 2ede4a72..7fe6d063 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -13,6 +13,7 @@ import csv import json import stream_density +from device_validation import validate_target_device def parse_args(print=False): @@ -77,7 +78,8 @@ def parse_args(print=False): 'starting metric data collection') # TODO: change target_device to an env variable in docker compose parser.add_argument('--target_device', default='CPU', - help='desired running platform [cpu|core|xeon|dgpu.x]') + type=validate_target_device, + help='desired running platform [CPU|GPU|GPU.|NPU]') parser.add_argument('--compose_file', default=None, action='append', help='path to docker compose files. ' + 'can be used multiple times') diff --git a/benchmark-scripts/benchmark_order_accuracy.py b/benchmark-scripts/benchmark_order_accuracy.py index e8d5e2e3..9e8457ad 100644 --- a/benchmark-scripts/benchmark_order_accuracy.py +++ b/benchmark-scripts/benchmark_order_accuracy.py @@ -25,6 +25,7 @@ import traceback from pathlib import Path from typing import List, Dict, Optional +from device_validation import validate_target_device # Import from performance-tools benchmark scripts sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -688,10 +689,9 @@ def parse_args(): parser.add_argument( '--target_device', - type=str, + type=validate_target_device, default='GPU', - choices=['CPU', 'GPU', 'NPU'], - help='Target inference device' + help='Target inference device [CPU|GPU|GPU.|NPU]' ) parser.add_argument( diff --git a/benchmark-scripts/benchmark_test.py b/benchmark-scripts/benchmark_test.py index 581bca91..cc698728 100644 --- a/benchmark-scripts/benchmark_test.py +++ b/benchmark-scripts/benchmark_test.py @@ -9,6 +9,8 @@ import unittest import benchmark import os +import argparse +from device_validation import validate_target_device class Testing(unittest.TestCase): @@ -55,6 +57,28 @@ def test_docker_compose_containers_fail(self): mock_popen.communicate.assert_called_once_with() mock_returncode.assert_called() + def test_validate_target_device_valid_values(self): + test_cases = { + 'CPU': 'CPU', + 'GPU': 'GPU', + 'GPU.0': 'GPU.0', + 'GPU.1': 'GPU.1', + 'GPU.2': 'GPU.2', + 'NPU': 'NPU', + } + + for user_value, expected in test_cases.items(): + with self.subTest(user_value=user_value): + self.assertEqual(validate_target_device(user_value), expected) + + def test_validate_target_device_invalid_values(self): + invalid_values = ['GPU.', 'GPU.A', 'GPU.-1', 'GPU.abc'] + + for user_value in invalid_values: + with self.subTest(user_value=user_value): + with self.assertRaises(argparse.ArgumentTypeError): + validate_target_device(user_value) + if __name__ == '__main__': unittest.main() diff --git a/benchmark-scripts/device_validation.py b/benchmark-scripts/device_validation.py new file mode 100644 index 00000000..70a2c9f4 --- /dev/null +++ b/benchmark-scripts/device_validation.py @@ -0,0 +1,34 @@ +""" +Utilities for validating benchmark target device arguments. +""" + +import argparse +import re + + +_GPU_INDEX_PATTERN = re.compile(r"^GPU\.(\d+)$", re.IGNORECASE) + + +def validate_target_device(value: str) -> str: + """Validate and normalize target device values. + + Accepted values: + - CPU + - GPU + - GPU. where index is a non-negative integer + - NPU + """ + normalized = value.strip() + upper_value = normalized.upper() + + if upper_value in {"CPU", "GPU", "NPU"}: + return upper_value + + gpu_match = _GPU_INDEX_PATTERN.fullmatch(normalized) + if gpu_match: + return f"GPU.{gpu_match.group(1)}" + + raise argparse.ArgumentTypeError( + "invalid target device '%s'. Expected one of: CPU, GPU, NPU, GPU." + % value + ) diff --git a/benchmark-scripts/device_validation_test.py b/benchmark-scripts/device_validation_test.py new file mode 100644 index 00000000..c1dcc613 --- /dev/null +++ b/benchmark-scripts/device_validation_test.py @@ -0,0 +1,37 @@ +""" +Unit tests for benchmark target device validation. +""" + +import argparse +import unittest + +from device_validation import validate_target_device + + +class TestDeviceValidation(unittest.TestCase): + + def test_valid_target_devices(self): + test_cases = { + 'CPU': 'CPU', + 'GPU': 'GPU', + 'GPU.0': 'GPU.0', + 'GPU.1': 'GPU.1', + 'GPU.2': 'GPU.2', + 'NPU': 'NPU', + } + + for user_value, expected in test_cases.items(): + with self.subTest(user_value=user_value): + self.assertEqual(validate_target_device(user_value), expected) + + def test_invalid_target_devices(self): + invalid_values = ['GPU.', 'GPU.A', 'GPU.-1', 'GPU.abc'] + + for user_value in invalid_values: + with self.subTest(user_value=user_value): + with self.assertRaises(argparse.ArgumentTypeError): + validate_target_device(user_value) + + +if __name__ == '__main__': + unittest.main() From 5e7b192f6d16c0ac0007993d94f2813fc74bca42 Mon Sep 17 00:00:00 2001 From: Tanmayee Sharvani Date: Wed, 5 Aug 2026 12:49:06 +0530 Subject: [PATCH 2/2] Address Copilot review comments --- benchmark-scripts/benchmark.py | 8 ++++-- benchmark-scripts/benchmark_order_accuracy.py | 9 +++++-- benchmark-scripts/benchmark_test.py | 25 ------------------- benchmark-scripts/device_validation.py | 15 +++++++++++ benchmark-scripts/device_validation_test.py | 21 +++++++++++++++- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 7fe6d063..52568cbc 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -13,7 +13,7 @@ import csv import json import stream_density -from device_validation import validate_target_device +from device_validation import validate_target_device, resolve_target_device_default def parse_args(print=False): @@ -77,7 +77,11 @@ def parse_args(print=False): help='initial time in seconds before ' + 'starting metric data collection') # TODO: change target_device to an env variable in docker compose - parser.add_argument('--target_device', default='CPU', + try: + default_target_device = resolve_target_device_default('CPU') + except argparse.ArgumentTypeError as exc: + parser.error(str(exc)) + parser.add_argument('--target_device', default=default_target_device, type=validate_target_device, help='desired running platform [CPU|GPU|GPU.|NPU]') parser.add_argument('--compose_file', default=None, action='append', diff --git a/benchmark-scripts/benchmark_order_accuracy.py b/benchmark-scripts/benchmark_order_accuracy.py index 9e8457ad..b6ab8043 100644 --- a/benchmark-scripts/benchmark_order_accuracy.py +++ b/benchmark-scripts/benchmark_order_accuracy.py @@ -25,7 +25,7 @@ import traceback from pathlib import Path from typing import List, Dict, Optional -from device_validation import validate_target_device +from device_validation import validate_target_device, resolve_target_device_default # Import from performance-tools benchmark scripts sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -687,10 +687,15 @@ def parse_args(): help='Directory for results output' ) + try: + default_target_device = resolve_target_device_default('GPU') + except argparse.ArgumentTypeError as exc: + parser.error(str(exc)) + parser.add_argument( '--target_device', type=validate_target_device, - default='GPU', + default=default_target_device, help='Target inference device [CPU|GPU|GPU.|NPU]' ) diff --git a/benchmark-scripts/benchmark_test.py b/benchmark-scripts/benchmark_test.py index cc698728..b05f63dc 100644 --- a/benchmark-scripts/benchmark_test.py +++ b/benchmark-scripts/benchmark_test.py @@ -9,8 +9,6 @@ import unittest import benchmark import os -import argparse -from device_validation import validate_target_device class Testing(unittest.TestCase): @@ -57,28 +55,5 @@ def test_docker_compose_containers_fail(self): mock_popen.communicate.assert_called_once_with() mock_returncode.assert_called() - def test_validate_target_device_valid_values(self): - test_cases = { - 'CPU': 'CPU', - 'GPU': 'GPU', - 'GPU.0': 'GPU.0', - 'GPU.1': 'GPU.1', - 'GPU.2': 'GPU.2', - 'NPU': 'NPU', - } - - for user_value, expected in test_cases.items(): - with self.subTest(user_value=user_value): - self.assertEqual(validate_target_device(user_value), expected) - - def test_validate_target_device_invalid_values(self): - invalid_values = ['GPU.', 'GPU.A', 'GPU.-1', 'GPU.abc'] - - for user_value in invalid_values: - with self.subTest(user_value=user_value): - with self.assertRaises(argparse.ArgumentTypeError): - validate_target_device(user_value) - - if __name__ == '__main__': unittest.main() diff --git a/benchmark-scripts/device_validation.py b/benchmark-scripts/device_validation.py index 70a2c9f4..2bb6edba 100644 --- a/benchmark-scripts/device_validation.py +++ b/benchmark-scripts/device_validation.py @@ -3,6 +3,7 @@ """ import argparse +import os import re @@ -32,3 +33,17 @@ def validate_target_device(value: str) -> str: "invalid target device '%s'. Expected one of: CPU, GPU, NPU, GPU." % value ) + + +def resolve_target_device_default(default_value: str, + env_var_name: str = "TARGET_DEVICE") -> str: + """Resolve default target device using env var and validate/normalize it. + + Precedence: + 1) explicit CLI value (handled by argparse separately) + 2) environment variable + 3) hard-coded default + """ + env_value = os.getenv(env_var_name) + candidate = env_value if env_value and env_value.strip() else default_value + return validate_target_device(candidate) diff --git a/benchmark-scripts/device_validation_test.py b/benchmark-scripts/device_validation_test.py index c1dcc613..d2f8319b 100644 --- a/benchmark-scripts/device_validation_test.py +++ b/benchmark-scripts/device_validation_test.py @@ -3,9 +3,11 @@ """ import argparse +import os import unittest +from unittest import mock -from device_validation import validate_target_device +from device_validation import validate_target_device, resolve_target_device_default class TestDeviceValidation(unittest.TestCase): @@ -32,6 +34,23 @@ def test_invalid_target_devices(self): with self.assertRaises(argparse.ArgumentTypeError): validate_target_device(user_value) + def test_env_target_device_valid(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'gpu.2'}, clear=False): + self.assertEqual(resolve_target_device_default('CPU'), 'GPU.2') + + def test_env_target_device_invalid(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'GPU.-1'}, clear=False): + with self.assertRaises(argparse.ArgumentTypeError): + resolve_target_device_default('CPU') + + def test_env_target_device_fallback_when_missing(self): + with mock.patch.dict(os.environ, {}, clear=True): + self.assertEqual(resolve_target_device_default('CPU'), 'CPU') + + def test_env_target_device_fallback_when_empty(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': ' '}, clear=False): + self.assertEqual(resolve_target_device_default('GPU'), 'GPU') + if __name__ == '__main__': unittest.main()