From fee7ab04b62ac172e796d25e02efbd5f3b852b20 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Thu, 22 Jan 2026 08:49:27 +0000 Subject: [PATCH 01/18] chore: update devcontainer config --- .devcontainer/dev.compose.yaml | 6 ----- .devcontainer/devcontainer.json | 33 ++++++++++++++++------- .devcontainer/setup.sh | 48 ++++++++++++++++++++------------- compose.yaml | 9 +++++++ pyproject.toml | 7 ++++- weko_group_cache_db/config.py | 2 +- weko_group_cache_db/groups.py | 2 +- weko_group_cache_db/logger.py | 2 +- 8 files changed, 71 insertions(+), 38 deletions(-) delete mode 100644 .devcontainer/dev.compose.yaml diff --git a/.devcontainer/dev.compose.yaml b/.devcontainer/dev.compose.yaml deleted file mode 100644 index 7478122..0000000 --- a/.devcontainer/dev.compose.yaml +++ /dev/null @@ -1,6 +0,0 @@ -services: - dev: - image: mcr.microsoft.com/devcontainers/base:debian - volumes: - - .:/workspace - command: sleep infinity diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e03ff8f..bfdfa1e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,30 +1,41 @@ { "name": "weko-group-cache-db Dev Container", - "dockerComposeFile": [ - "../compose.yaml", - "dev.compose.yaml" - ], - "runServices": ["dev", "redis"], - "service": "dev", + "image": "mcr.microsoft.com/devcontainers/base:bookworm", "remoteUser": "vscode", - "workspaceFolder": "/workspace", + + "initializeCommand": "docker network create weko-group-cache-db_default || true", + "runArgs": [ + "--network", + "weko-group-cache-db_default" + ], "features": { "ghcr.io/devcontainers/features/python:1": { "version": "3.14" + }, + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { + "version": "latest", + "dockerDashComposeVersion": "latest", + "enableNonRootDocker": true, + "moby": false } }, "remoteEnv": { "VIRTUAL_ENV": "/home/vscode/.venv", "UV_PROJECT_ENVIRONMENT": "/home/vscode/.venv" }, - "postCreateCommand": "chmod +x .devcontainer/setup.sh && .devcontainer/setup.sh vscode", + "mounts": [ + "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" + ], + + "onCreateCommand": "/bin/sh .devcontainer/setup.sh install_deps", + "postAttachCommand": "/bin/sh .devcontainer/setup.sh compose_up", "customizations": { "vscode": { "settings": { - "python.pythonPath": "/code/.venv/bin/python", "python.languageServer": "Pylance", + "python.defaultInterpreterPath": "~/.venv/bin/python", "[python]": { "editor.rulers": [88], "editor.formatOnSave": true, @@ -33,7 +44,9 @@ "source.organizeImports": "explicit" }, "editor.defaultFormatter": "charliermarsh.ruff" - } + }, + "python.useEnvironmentsExtension": false, + "python.terminal.activateEnvironment": false }, "extensions": [ "visualstudioexptteam.vscodeintellicode", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 048ca33..0e369ac 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,26 +1,38 @@ #!/bin/bash -set -e -REMOTE_USER=$1 -BASHRC_FILE="/home/${REMOTE_USER}/.bashrc" +install_deps() { + # Install Python dependencies + pip install --upgrade pip && pip install uv + uv sync && uv pip install -e . -if [ -z "$REMOTE_USER" ]; then - echo "Error: User name argument is missing." - exit 1 -fi + if ! grep -q "venv_activate_reload" ~/.bashrc 2>/dev/null; then + { + echo "" + echo "# venv_activate_reload" + echo "if [ -f ~/.venv/bin/activate ]; then" + echo " source ~/.venv/bin/activate" + echo "fi" + } >> ~/.bashrc + fi +} + + +compose_up() { + docker compose up -d --force-recreate redis +} -if ! grep -q "venv_activate_reload" "$BASHRC_FILE" 2>/dev/null; then - { - echo "" - echo "# venv_activate_reload" - echo "if [ -f /home/${REMOTE_USER}/.venv/bin/activate ]; then" - echo " source /home/${REMOTE_USER}/.venv/bin/activate" - echo "fi" - } >> "$BASHRC_FILE" +FUNCTION_NAME=$1 - sudo chown ${REMOTE_USER}:${REMOTE_USER} "$BASHRC_FILE" +if [ -z "$FUNCTION_NAME" ]; then + echo "Error: Function name argument is missing." + exit 1 fi +shift -pip install --upgrade pip -pip install uv && uv sync && uv pip install -e . +if type -t "$FUNCTION_NAME" | grep -q 'function'; then + "$FUNCTION_NAME" "$@" +else + echo "Error: Function '$FUNCTION_NAME' not found." + exit 1 +fi diff --git a/compose.yaml b/compose.yaml index 7b7faa0..60d86bf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,8 @@ services: volumes: - .:/code - /code/.venv + networks: + - default redis: restart: "always" @@ -19,3 +21,10 @@ services: resources: limits: memory: 100m + networks: + - default + +networks: + default: + external: true + name: weko-group-cache-db_default diff --git a/pyproject.toml b/pyproject.toml index b1581dc..0844993 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,12 @@ addopts = [ [tool.coverage.run] source = ["weko_group_cache_db"] -omit = ["*/__init__.py", "*/tests/*"] + +[tool.coverage.report] +omit = ["__init__.py", "tests/*"] +exclude_also = [ + "if t.TYPE_CHECKING:", +] [tool.ruff.lint] # refer to https://docs.astral.sh/ruff/settings/#lint diff --git a/weko_group_cache_db/config.py b/weko_group_cache_db/config.py index af5c5a6..1198942 100644 --- a/weko_group_cache_db/config.py +++ b/weko_group_cache_db/config.py @@ -21,7 +21,7 @@ from werkzeug.local import LocalProxy if t.TYPE_CHECKING: - from pydantic.fields import FieldInfo # pragma: no cover + from pydantic.fields import FieldInfo class Settings(BaseSettings): diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index bfd9ec5..1e2d355 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -24,7 +24,7 @@ from .redis import connection if t.TYPE_CHECKING: - from redis import Redis # pragma: no cover + from redis import Redis def fetch_all(**kwargs: t.Unpack[InstitutionSource]): diff --git a/weko_group_cache_db/logger.py b/weko_group_cache_db/logger.py index 3501124..2eba9c8 100644 --- a/weko_group_cache_db/logger.py +++ b/weko_group_cache_db/logger.py @@ -17,7 +17,7 @@ from .config import config if t.TYPE_CHECKING: - from datetime import datetime # pragma: no cover + from datetime import datetime def _log_time_format(datetime: datetime) -> Text: From cf24d25e5e1fa9af15bbe8a89d400c53256347d5 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Thu, 22 Jan 2026 08:51:11 +0000 Subject: [PATCH 02/18] downgrade: require redis upto 6.4.0 - for backward compatibility --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0844993..3870261 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "backoff>=2.2.1", "inflect>=7.5.0", "pydantic-settings>=2.12.0", - "redis>=7.0.1", + "redis>=6.4.0", "requests>=2.32.5", "rich-click>=1.9.4", "werkzeug>=3.1.3", diff --git a/uv.lock b/uv.lock index 4fd256c..61059fd 100644 --- a/uv.lock +++ b/uv.lock @@ -542,7 +542,7 @@ requires-dist = [ { name = "backoff", specifier = ">=2.2.1" }, { name = "inflect", specifier = ">=7.5.0" }, { name = "pydantic-settings", specifier = ">=2.12.0" }, - { name = "redis", specifier = ">=7.0.1" }, + { name = "redis", specifier = ">=6.4.0" }, { name = "requests", specifier = ">=2.32.5" }, { name = "rich-click", specifier = ">=1.9.4" }, { name = "werkzeug", specifier = ">=3.1.3" }, From 4b41ff4d972487a6812499f43d10d956b6f72721 Mon Sep 17 00:00:00 2001 From: ivis-kuroda <174298009+ivis-kuroda@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:00:52 +0000 Subject: [PATCH 03/18] docs: update coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a70014d..e2bd2a9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # weko-group-cache-db / グループ情報キャッシュ機能 -Coverage
Coverage Report
FileStmtsMissCover
weko_group_cache_db
   cli.py520100%
   config.py840100%
   exc.py90100%
   groups.py860100%
   loader.py1110100%
   logger.py250100%
   redis.py260100%
TOTAL3930100%
+Coverage
Coverage Report
FileStmtsMissCover
weko_group_cache_db
   cli.py520100%
   config.py830100%
   exc.py90100%
   groups.py850100%
   loader.py1110100%
   logger.py240100%
   redis.py260100%
TOTAL3900100%
## 概要 From 81c275ad5aaba07d8bfd140d5b2176fbd24621b2 Mon Sep 17 00:00:00 2001 From: ryoya-hayase Date: Tue, 27 Jan 2026 00:54:19 +0000 Subject: [PATCH 04/18] add signals for groups manager --- pyproject.toml | 1 + uv.lock | 11 ++++++++++ weko_group_cache_db/config.py | 16 ++++++++++++-- weko_group_cache_db/groups.py | 40 ++++++++++++++++++++++++++++++++++ weko_group_cache_db/signals.py | 13 +++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 weko_group_cache_db/signals.py diff --git a/pyproject.toml b/pyproject.toml index 3870261..41b41b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" requires-python = ">=3.14" dependencies = [ "backoff>=2.2.1", + "blinker>=1.9.0", "inflect>=7.5.0", "pydantic-settings>=2.12.0", "redis>=6.4.0", diff --git a/uv.lock b/uv.lock index 61059fd..8acd225 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "certifi" version = "2025.10.5" @@ -520,6 +529,7 @@ version = "1.0.0rc4" source = { virtual = "." } dependencies = [ { name = "backoff" }, + { name = "blinker" }, { name = "inflect" }, { name = "pydantic-settings" }, { name = "redis" }, @@ -540,6 +550,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "backoff", specifier = ">=2.2.1" }, + { name = "blinker", specifier = ">=1.9.0" }, { name = "inflect", specifier = ">=7.5.0" }, { name = "pydantic-settings", specifier = ">=2.12.0" }, { name = "redis", specifier = ">=6.4.0" }, diff --git a/weko_group_cache_db/config.py b/weko_group_cache_db/config.py index 1198942..0e7f4d9 100644 --- a/weko_group_cache_db/config.py +++ b/weko_group_cache_db/config.py @@ -9,6 +9,7 @@ from contextvars import ContextVar from pathlib import Path +from typing import overload import rich_click as click @@ -193,9 +194,20 @@ def __repr__(self) -> str: _current_config: ContextVar[Settings] = ContextVar("current_config") -def setup_config(toml_path: str) -> None: +@overload +def setup_config(config: dict[str, t.Any]) -> None: ... + + +@overload +def setup_config(config: str) -> None: ... + + +def setup_config(config: dict[str, t.Any] | str) -> None: """Initialize the global config instance.""" - _current_config.set(Settings(toml_path=toml_path)) # pyright: ignore[reportCallIssue] + if isinstance(config, dict): + _current_config.set(Settings(**config)) # pyright: ignore[reportCallIssue] + else: + _current_config.set(Settings(toml_path=config)) # pyright: ignore[reportCallIssue] config = t.cast(Settings, LocalProxy(_current_config, unbound_message=_no_config_msg)) diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 1e2d355..fe9fc63 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -22,8 +22,10 @@ from .loader import Institution, InstitutionSource, load_institutions from .logger import console, logger from .redis import connection +from .signals import update_count_signal, update_result_signal if t.TYPE_CHECKING: + from backoff.types import Details from redis import Redis @@ -77,6 +79,12 @@ def fetch_all(**kwargs: t.Unpack[InstitutionSource]): exceptions.append(UpdateError(institution.fqdn, origin=ex)) finally: progress.update(task, advance=1) + count_data = { + "total": len(institutions), + "done": index + 1, + "current": institution.fqdn, + } + update_count_signal.send(fetch_all, **count_data) if index != total - 1: time.sleep(config.REQUEST_INTERVAL) @@ -137,6 +145,33 @@ def fetch_and_cache(): """ + def _update_retry_count(details: Details) -> None: + """Update the retry count based on backoff details. + + Arguments: + details (Details): Details from backoff. + + """ + nonlocal retry_count + retry_count = details.get("tries", 0) + + def _set_update_result_signal(fqdn: str, code: str = "") -> None: + """Send update result signal. + + Arguments: + fqdn (str): FQDN of the institution. + code (str): Result code. Default is empty string. + + """ + nonlocal retry_count + signal_data = { + "task_name": f"{fqdn}_{retry_count!s}", + "status": "success" if not code else "failed", + "code": code, + "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + } + update_result_signal.send(fetch_and_cache, **signal_data) # pyright: ignore[reportArgumentType] + @backoff.on_exception( lambda: backoff.expo( base=config.REQUEST_RETRY_BASE, @@ -146,25 +181,30 @@ def fetch_and_cache(): (requests.RequestException, redis.RedisError), max_tries=config.REQUEST_RETRIES + 1, jitter=backoff.full_jitter, + on_backoff=_update_retry_count, ) def _retrieve_fetch_and_cache(institution: Institution, store: Redis) -> int: try: group_ids = fetch_map_groups(institution) set_groups_to_redis(institution.fqdn, group_ids, store=store) + _set_update_result_signal(institution.fqdn) return len(group_ids) except requests.RequestException: logger.warning( "Failed to fetch groups from mAP API for institution: %s", institution.fqdn, ) + _set_update_result_signal(institution.fqdn, code="timeout") raise except redis.RedisError: logger.warning( "Failed to cache groups to Redis for institution: %s", institution.fqdn, ) + _set_update_result_signal(institution.fqdn, code="redis_error") raise + retry_count = 0 return _retrieve_fetch_and_cache diff --git a/weko_group_cache_db/signals.py b/weko_group_cache_db/signals.py new file mode 100644 index 0000000..fe77be4 --- /dev/null +++ b/weko_group_cache_db/signals.py @@ -0,0 +1,13 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Signal definitions for weko-group-cache-db.""" + +from blinker import signal + +# Define a custom signal for updating count +update_count_signal = signal("update_count") + +# Define a custom signal for updating result +update_result_signal = signal("update_result") From 721b235994a57132e191d362d68accf7d0638137 Mon Sep 17 00:00:00 2001 From: ivis-kuroda <174298009+ivis-kuroda@users.noreply.github.com> Date: Mon, 9 Mar 2026 05:21:43 +0000 Subject: [PATCH 05/18] docs: update coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a70014d..a1873d2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # weko-group-cache-db / グループ情報キャッシュ機能 -Coverage
Coverage Report
FileStmtsMissCover
weko_group_cache_db
   cli.py520100%
   config.py840100%
   exc.py90100%
   groups.py860100%
   loader.py1110100%
   logger.py250100%
   redis.py260100%
TOTAL3930100%
+Coverage
Coverage Report
FileStmtsMissBranchBrPartCover
weko_group_cache_db
   cli.py520120100%
   config.py83060100%
   exc.py9000100%
   groups.py850140100%
   loader.py1110220100%
   logger.py24020100%
   redis.py26020100%
TOTAL3900580100%
## 概要 From ec056ef6e587a11db5ed25382a0e1886248e8f29 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Mon, 9 Mar 2026 07:47:13 +0000 Subject: [PATCH 06/18] update: refine configurations --- README.md | 2 +- config.toml | 6 ++-- tests/conftest.py | 2 +- tests/test_config.py | 26 +++++----------- tests/test_redis.py | 8 ++--- weko_group_cache_db/config.py | 57 ++++++++--------------------------- weko_group_cache_db/redis.py | 2 +- 7 files changed, 30 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index e2bd2a9..df03866 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ $ wgcd one --help | REQUEST_RETRY_BASE | 数値 | 4 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの基準時間（秒） | | REQUEST_RETRY_FACTOR | 数値 | 5 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの係数（秒） | | REQUEST_RETRY_MAX | 数値 | 90 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの最大時間（秒） | -| REDIS_TYPE | 文字列 | redis | 使用するRedisの種類
"redis", "sentinel"のうちから指定 | +| REDIS_TYPE | 文字列 | redis | 使用するRedisの種類
"RedisCache", "RedisSentinelCache"のうちから指定 | | REDIS_HOST | 文字列 | localhost | Redisのホスト名 | | REDIS_PORT | 数値 | 6379 | Redisのポート番号 | | REDIS_DB_INDEX | 数値 | 4 | グループ情報を格納するRedisのDBインデックス | diff --git a/config.toml b/config.toml index 089793b..dc83e77 100644 --- a/config.toml +++ b/config.toml @@ -18,7 +18,7 @@ # cache_ttl = 86400 # === Map groups API endpoint. === -map_groups_api_endpoint = "https://sample.gakunin.jp/api/groups/" +map_groups_api_endpoint = "https://sptest.cg.gakunin.jp/api/groups/" # === Request timeout (in seconds) when connecting to mAP API. === # request_timeout = 20 @@ -42,8 +42,8 @@ map_groups_api_endpoint = "https://sample.gakunin.jp/api/groups/" # === Maximum time (in seconds) for exponential backoff during request retries. === # request_retry_max = 90 -# === Redis type to use. `redis` or `sentinel` is allowed. === -# redis_type = "redis" +# === Redis type to use. `RedisCache` or `RedisSentinelCache` is allowed. === +# redis_type = "RedisCache" # === Redis service host name. === # redis_host = "localhost" diff --git a/tests/conftest.py b/tests/conftest.py index 5c65837..84fcd06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,7 +51,7 @@ def row_config(): "REQUEST_RETRY_BASE": 2, "REQUEST_RETRY_FACTOR": 15, "REQUEST_RETRY_MAX": 60, - "REDIS_TYPE": "sentinel", + "REDIS_TYPE": "RedisSentinelCache", "REDIS_HOST": "redis", "REDIS_PORT": 26379, "REDIS_DB_INDEX": 2, diff --git a/tests/test_config.py b/tests/test_config.py index b347f1a..7504e46 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,10 +3,8 @@ # import typing as t -import pytest import toml -from pydantic_core import ValidationError from werkzeug.local import LocalProxy from weko_group_cache_db.config import Settings, _current_config, config, setup_config @@ -37,16 +35,6 @@ def test_settings_default_values(): assert settings.REDIS_URL == "redis://localhost:6379/4" -def test_settings_missing_required_field(): - with pytest.raises(ValidationError) as excinfo: - Settings() # pyright: ignore[reportCallIssue] - - errors = excinfo.value.errors() - assert len(errors) == 1 - assert errors[0]["loc"] == ("MAP_GROUPS_API_ENDPOINT",) - assert errors[0]["msg"] == "Field required" - - def test_setup_config(tmp_path, row_config): config_path = tmp_path / "config.toml" row_config = {key: value for key, value in row_config.items() if key == "MAP_GROUPS_API_ENDPOINT"} @@ -90,11 +78,11 @@ def test_setup_config_overrides_lower_case(tmp_path, row_config): assert config_dict == row_config -def test_setup_config_invalid_path(): - with pytest.raises(ValidationError) as excinfo: - setup_config("non_existent_config.toml") +def test_setup_config_dict(row_config): + setup_config(row_config) + assert t.cast(LocalProxy, config)._get_current_object() is _current_config.get() + - errors = excinfo.value.errors() - assert len(errors) == 1 - assert errors[0]["loc"] == ("MAP_GROUPS_API_ENDPOINT",) - assert errors[0]["msg"] == "Field required" +def test_setup_config_instance(row_config): + setup_config(Settings(**row_config)) + assert t.cast(LocalProxy, config)._get_current_object() is _current_config.get() diff --git a/tests/test_redis.py b/tests/test_redis.py index 55299a8..8b3c03b 100644 --- a/tests/test_redis.py +++ b/tests/test_redis.py @@ -16,7 +16,7 @@ # def connection() -> Redis: def test_connection_redis(set_test_config, log_capture: pytest.LogCaptureFixture): - set_test_config(REDIS_TYPE="redis") + set_test_config(REDIS_TYPE="RedisCache") with patch("weko_group_cache_db.redis._redis_connection") as mock_redis_connection: mock_redis_connection.return_value = MagicMock(spec=redis.Redis) store = connection() @@ -28,7 +28,7 @@ def test_connection_redis(set_test_config, log_capture: pytest.LogCaptureFixture def test_connection_sentinel(set_test_config, log_capture: pytest.LogCaptureFixture): - set_test_config(REDIS_TYPE="sentinel") + set_test_config(REDIS_TYPE="RedisSentinelCache") with patch("weko_group_cache_db.redis._sentinel_connection") as mock_sentinel_connection: mock_sentinel_connection.return_value = MagicMock(spec=redis.Redis) store = connection() @@ -40,7 +40,7 @@ def test_connection_sentinel(set_test_config, log_capture: pytest.LogCaptureFixt def test_connection_catch_value_error(set_test_config, log_capture: pytest.LogCaptureFixture): - set_test_config(REDIS_TYPE="redis") + set_test_config(REDIS_TYPE="RedisCache") with patch("weko_group_cache_db.redis._redis_connection") as mock_redis_connection: mock_redis_connection.side_effect = ValueError("Test ValueError") with pytest.raises(ValueError, match="Test ValueError"): @@ -50,7 +50,7 @@ def test_connection_catch_value_error(set_test_config, log_capture: pytest.LogCa def test_connection_catch_connection_error(set_test_config, log_capture: pytest.LogCaptureFixture): - set_test_config(REDIS_TYPE="redis") + set_test_config(REDIS_TYPE="RedisCache") with patch("weko_group_cache_db.redis._redis_connection") as mock_redis_connection: mock_store = MagicMock(spec=redis.Redis) mock_store.ping.side_effect = RedisConnectionError("Test ConnectionError") diff --git a/weko_group_cache_db/config.py b/weko_group_cache_db/config.py index 0e7f4d9..b4feba8 100644 --- a/weko_group_cache_db/config.py +++ b/weko_group_cache_db/config.py @@ -4,26 +4,21 @@ """Settings module for weko-group-cache-db.""" -import tomllib import typing as t from contextvars import ContextVar from pathlib import Path from typing import overload -import rich_click as click - from pydantic import BaseModel, computed_field from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, + TomlConfigSettingsSource, ) from werkzeug.local import LocalProxy -if t.TYPE_CHECKING: - from pydantic.fields import FieldInfo - class Settings(BaseSettings): """Settings for application with validation.""" @@ -49,7 +44,7 @@ class Settings(BaseSettings): If it specified less than 0, it will be considered as no expiration. """ - MAP_GROUPS_API_ENDPOINT: str + MAP_GROUPS_API_ENDPOINT: str = "https://sample.gakunin.jp/api/groups/" """Map groups API endpoint.""" REQUEST_TIMEOUT: t.Annotated[int, "seconds"] = 20 @@ -70,8 +65,8 @@ class Settings(BaseSettings): REQUEST_RETRY_MAX: t.Annotated[int | float, "seconds"] = 90 """Maximum time for exponential backoff during request retries.""" - REDIS_TYPE: t.Literal["redis", "sentinel"] = "redis" - """Redis type to use. `redis` or `sentinel` is allowed.""" + REDIS_TYPE: t.Literal["RedisCache", "RedisSentinelCache"] = "RedisCache" + """Redis type to use. `RedisCache` or `RedisSentinelCache` is allowed.""" REDIS_HOST: str = "localhost" """Redis service host name.""" @@ -107,7 +102,10 @@ def REDIS_SENTINELS(self) -> list[tuple[str, str]]: # noqa: N802 model_config = SettingsConfigDict( extra="forbid", frozen=True, + alias_generator=lambda s: s.lower(), validate_default=True, + validate_by_name=True, + validate_by_alias=True, ) @classmethod @@ -159,55 +157,26 @@ class Sentinel(BaseModel): """Sentinel port number.""" -class TomlConfigSettingsSource(PydanticBaseSettingsSource): - """TOML configuration settings source.""" - - def __init__(self, settings_cls: type[BaseSettings], toml_path: Path) -> None: - """Initialize TOML config settings source.""" - super().__init__(settings_cls) - self.toml_path = toml_path - - def get_field_value( # noqa: D102 - self, field: FieldInfo, field_name: str - ) -> tuple[t.Any, str, bool]: ... - - def __call__(self) -> dict[str, t.Any]: - """Load settings from TOML file.""" # noqa: DOC201 - if not self.toml_path.exists(): - click.echo("[WARN] Settings file not found. Default values will be used.") - click.echo(f"[INFO] Looking for settings file at: {self.toml_path}") - return {} - - with self.toml_path.open("rb") as f: - data = tomllib.load(f) - - return {k.upper(): v for k, v in data.items()} - - def __repr__(self) -> str: - """Representation.""" # noqa: DOC201 - return ( - f"TomlConfigSettingsSource(toml_path={self.toml_path})" # pragma: no cover - ) - - _no_config_msg = "Config has not been initialized." _current_config: ContextVar[Settings] = ContextVar("current_config") @overload def setup_config(config: dict[str, t.Any]) -> None: ... - - @overload def setup_config(config: str) -> None: ... +@overload +def setup_config(config: Settings) -> None: ... -def setup_config(config: dict[str, t.Any] | str) -> None: +def setup_config(config: dict[str, t.Any] | str | Settings) -> None: """Initialize the global config instance.""" if isinstance(config, dict): _current_config.set(Settings(**config)) # pyright: ignore[reportCallIssue] - else: + elif isinstance(config, str): _current_config.set(Settings(toml_path=config)) # pyright: ignore[reportCallIssue] + else: + _current_config.set(config) config = t.cast(Settings, LocalProxy(_current_config, unbound_message=_no_config_msg)) diff --git a/weko_group_cache_db/redis.py b/weko_group_cache_db/redis.py index 3ad30b3..bd206b6 100644 --- a/weko_group_cache_db/redis.py +++ b/weko_group_cache_db/redis.py @@ -23,7 +23,7 @@ def connection() -> Redis: """ try: - if config.REDIS_TYPE == "redis": + if config.REDIS_TYPE == "RedisCache": store = _redis_connection() store.ping() logger.info("Successfully connected to Redis.") From 28e95094be923f429f18e6eaea1251e1e171ad1f Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 10:39:51 +0000 Subject: [PATCH 07/18] refactor: refine signals and support fqdn list --- weko_group_cache_db/groups.py | 79 +++++++++++++++++++++++----------- weko_group_cache_db/loader.py | 28 +++++++++++- weko_group_cache_db/signals.py | 48 +++++++++++++++++++-- 3 files changed, 124 insertions(+), 31 deletions(-) diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index fe9fc63..1d25a6d 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -22,7 +22,7 @@ from .loader import Institution, InstitutionSource, load_institutions from .logger import console, logger from .redis import connection -from .signals import update_count_signal, update_result_signal +from .signals import ExecutedData, ProgressData, executed_signal, progress_signal if t.TYPE_CHECKING: from backoff.types import Details @@ -79,12 +79,7 @@ def fetch_all(**kwargs: t.Unpack[InstitutionSource]): exceptions.append(UpdateError(institution.fqdn, origin=ex)) finally: progress.update(task, advance=1) - count_data = { - "total": len(institutions), - "done": index + 1, - "current": institution.fqdn, - } - update_count_signal.send(fetch_all, **count_data) + _send_progress_signal(institutions, index) if index != total - 1: time.sleep(config.REQUEST_INTERVAL) @@ -145,32 +140,36 @@ def fetch_and_cache(): """ - def _update_retry_count(details: Details) -> None: + def _count(details: Details) -> None: """Update the retry count based on backoff details. Arguments: details (Details): Details from backoff. """ - nonlocal retry_count - retry_count = details.get("tries", 0) + nonlocal retries + retries = details.get("tries", 0) - def _set_update_result_signal(fqdn: str, code: str = "") -> None: + def _executed_signal( + fqdn: str, status: str, error: Exception | None = None + ) -> None: """Send update result signal. Arguments: fqdn (str): FQDN of the institution. - code (str): Result code. Default is empty string. + status (str): Status of the fetch and cache operation. + error (Exception): Error object. Default is None. """ - nonlocal retry_count - signal_data = { - "task_name": f"{fqdn}_{retry_count!s}", - "status": "success" if not code else "failed", - "code": code, - "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + nonlocal retries + data = { + "fqdn": fqdn, + "retries": retries, + "status": status, + "error": error, + "updated": datetime.now(UTC), } - update_result_signal.send(fetch_and_cache, **signal_data) # pyright: ignore[reportArgumentType] + executed_signal.send(fetch_and_cache, **data) @backoff.on_exception( lambda: backoff.expo( @@ -181,30 +180,31 @@ def _set_update_result_signal(fqdn: str, code: str = "") -> None: (requests.RequestException, redis.RedisError), max_tries=config.REQUEST_RETRIES + 1, jitter=backoff.full_jitter, - on_backoff=_update_retry_count, + on_backoff=_count, ) def _retrieve_fetch_and_cache(institution: Institution, store: Redis) -> int: + nonlocal retries try: group_ids = fetch_map_groups(institution) set_groups_to_redis(institution.fqdn, group_ids, store=store) - _set_update_result_signal(institution.fqdn) + _send_executed_signal(institution, "success", retries=retries) return len(group_ids) - except requests.RequestException: + except requests.RequestException as ex: logger.warning( "Failed to fetch groups from mAP API for institution: %s", institution.fqdn, ) - _set_update_result_signal(institution.fqdn, code="timeout") + _send_executed_signal(institution, "failed", retries=retries, error=ex) raise - except redis.RedisError: + except redis.RedisError as ex: logger.warning( "Failed to cache groups to Redis for institution: %s", institution.fqdn, ) - _set_update_result_signal(institution.fqdn, code="redis_error") + _send_executed_signal(institution, "failed", retries=retries, error=ex) raise - retry_count = 0 + retries = 0 return _retrieve_fetch_and_cache @@ -258,3 +258,30 @@ def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None store.persist(redis_key) if config.CACHE_TTL >= 0: store.expire(redis_key, config.CACHE_TTL) + + +def _send_progress_signal(institutions: list[Institution], index: int) -> None: + total = len(institutions) + done = index + 1 + current = institutions[index].fqdn if index < total else "N/A" + data = ProgressData(total=total, done=done, current=current) + + progress_signal.send(fetch_all, data=data) + + +def _send_executed_signal( + institution: Institution, + status: t.Literal["success", "failed"], + retries: int, + error: Exception | None = None, +) -> None: + data = ExecutedData( + fqdn=institution.fqdn, + status=status, + retries=retries, + error_type=type(error).__name__ if error else None, + error_message=str(error) if error else None, + updated_at=datetime.now(UTC), + ) + + executed_signal.send(fetch_all, data=data) diff --git a/weko_group_cache_db/loader.py b/weko_group_cache_db/loader.py index d66220d..4c8fd19 100644 --- a/weko_group_cache_db/loader.py +++ b/weko_group_cache_db/loader.py @@ -34,6 +34,9 @@ class InstitutionSource(t.TypedDict): fqdn_list_file: t.NotRequired[str | Path] """Path to the file containing FQDN list.""" + fqdn_list: t.NotRequired[list[str]] + """List of FQDNs.""" + def load_institutions(**kwargs: t.Unpack[InstitutionSource]) -> list[Institution]: """Load institution information from the configured source. @@ -52,6 +55,10 @@ def load_institutions(**kwargs: t.Unpack[InstitutionSource]) -> list[Institution return load_institutions_from_directory( kwargs["directory_path"], kwargs["fqdn_list_file"] ) + if "directory_path" in kwargs and "fqdn_list" in kwargs: + return load_institutions_from_list( + kwargs["directory_path"], kwargs["fqdn_list"] + ) logger.error("Invalid institution source configuration.") return [] @@ -157,8 +164,6 @@ def load_institutions_from_directory( list[Institution]: List of Institution objects. """ - if isinstance(directory_path, str): - directory_path = Path(directory_path) if isinstance(fqdn_list_file, str): fqdn_list_file = Path(fqdn_list_file) @@ -168,6 +173,25 @@ def load_institutions_from_directory( for line in f if (L := line.strip()) and not L.startswith("#") ] + return load_institutions_from_list(directory_path, fqdn_list) + + +def load_institutions_from_list( + directory_path: str | Path, fqdn_list: list[str] +) -> list[Institution]: + """Load institution information from the provided FQDN list. + + Arguments: + directory_path (str | Path): Path to the directory containing TOML files. + fqdn_list (list[str]): List of FQDNs. + + Returns: + list[Institution]: List of Institution objects. + + """ + if isinstance(directory_path, str): + directory_path = Path(directory_path) + p = inflect.engine() institutions: list[Institution] = [] with Progress( diff --git a/weko_group_cache_db/signals.py b/weko_group_cache_db/signals.py index fe77be4..378276b 100644 --- a/weko_group_cache_db/signals.py +++ b/weko_group_cache_db/signals.py @@ -4,10 +4,52 @@ """Signal definitions for weko-group-cache-db.""" -from blinker import signal +import typing as t + +from datetime import datetime # noqa: TC003 + +from blinker import Namespace +from pydantic import BaseModel + +namespace = Namespace() # Define a custom signal for updating count -update_count_signal = signal("update_count") +progress_signal = namespace.signal("progress") # Define a custom signal for updating result -update_result_signal = signal("update_result") +executed_signal = namespace.signal("executed") + + +class ProgressData(BaseModel): + """Data model for progress signal.""" + + total: int + """Total number of institutions to process.""" + + done: int + """Number of institutions processed so far.""" + + current: str + """FQDN of the current institution being processed.""" + + +class ExecutedData(BaseModel): + """Data model for executed signal.""" + + fqdn: str + """FQDN of the institution.""" + + status: t.Literal["success", "failed"] + """Status of the fetch and cache operation.""" + + retries: int = 0 + """Number of retries attempted.""" + + error_type: str | None = None + """Type of error, if any.""" + + error_message: str | None = None + """Error message, if any.""" + + updated_at: datetime + """Timestamp of the last update.""" From bc764705663446c99e5b6a760851b06bb22ec8a4 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 14:54:16 +0000 Subject: [PATCH 08/18] update: add cache key function --- weko_group_cache_db/groups.py | 38 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 1d25a6d..76173df 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -150,27 +150,6 @@ def _count(details: Details) -> None: nonlocal retries retries = details.get("tries", 0) - def _executed_signal( - fqdn: str, status: str, error: Exception | None = None - ) -> None: - """Send update result signal. - - Arguments: - fqdn (str): FQDN of the institution. - status (str): Status of the fetch and cache operation. - error (Exception): Error object. Default is None. - - """ - nonlocal retries - data = { - "fqdn": fqdn, - "retries": retries, - "status": status, - "error": error, - "updated": datetime.now(UTC), - } - executed_signal.send(fetch_and_cache, **data) - @backoff.on_exception( lambda: backoff.expo( base=config.REQUEST_RETRY_BASE, @@ -245,8 +224,7 @@ def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None Redis store object. If None, a new connection will be established. """ - transformed_fqdn = fqdn.replace(".", "_").replace("-", "_") - redis_key = transformed_fqdn + config.CACHE_KEY_SUFFIX + redis_key = cache_key(fqdn) updated_at = datetime.now(UTC).isoformat(timespec="seconds") if store is None: @@ -260,6 +238,20 @@ def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None store.expire(redis_key, config.CACHE_TTL) +def cache_key(fqdn: str) -> str: + """Generate the Redis key for the given FQDN. + + Arguments: + fqdn (str): FQDN of the institution. + + Returns: + str: Redis key for the institution's groups. + + """ + transformed_fqdn = fqdn.replace(".", "_").replace("-", "_") + return transformed_fqdn + config.CACHE_KEY_SUFFIX + + def _send_progress_signal(institutions: list[Institution], index: int) -> None: total = len(institutions) done = index + 1 From e3b0b94f1266b8f5980742aa8bfb4eea718dd710 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 15:04:34 +0000 Subject: [PATCH 09/18] fix: updated value for signal --- weko_group_cache_db/groups.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 76173df..52c345a 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -165,8 +165,10 @@ def _retrieve_fetch_and_cache(institution: Institution, store: Redis) -> int: nonlocal retries try: group_ids = fetch_map_groups(institution) - set_groups_to_redis(institution.fqdn, group_ids, store=store) - _send_executed_signal(institution, "success", retries=retries) + updated_at = set_groups_to_redis(institution.fqdn, group_ids, store=store) + _send_executed_signal( + institution, "success", retries=retries, updated_at=updated_at + ) return len(group_ids) except requests.RequestException as ex: logger.warning( @@ -214,7 +216,9 @@ def fetch_map_groups(institution: Institution) -> list[str]: return group_ids -def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None = None): +def set_groups_to_redis( + fqdn: str, group_ids: list[str], *, store: Redis | None = None +) -> datetime: """Set groups to redis. Arguments: @@ -223,9 +227,13 @@ def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None store(Redis | None): Redis store object. If None, a new connection will be established. + Returns: + datetime: The timestamp when the groups were cached. + """ redis_key = cache_key(fqdn) - updated_at = datetime.now(UTC).isoformat(timespec="seconds") + now = datetime.now(UTC) + updated_at = now.isoformat(timespec="seconds") if store is None: store = connection() @@ -237,6 +245,8 @@ def set_groups_to_redis(fqdn: str, group_ids: list[str], *, store: Redis | None if config.CACHE_TTL >= 0: store.expire(redis_key, config.CACHE_TTL) + return now + def cache_key(fqdn: str) -> str: """Generate the Redis key for the given FQDN. @@ -264,7 +274,8 @@ def _send_progress_signal(institutions: list[Institution], index: int) -> None: def _send_executed_signal( institution: Institution, status: t.Literal["success", "failed"], - retries: int, + updated_at: datetime | None = None, + retries: int = 0, error: Exception | None = None, ) -> None: data = ExecutedData( @@ -273,7 +284,7 @@ def _send_executed_signal( retries=retries, error_type=type(error).__name__ if error else None, error_message=str(error) if error else None, - updated_at=datetime.now(UTC), + updated_at=updated_at or datetime.now(UTC), ) executed_signal.send(fetch_all, data=data) From 885fd44e16d86a2790365af41c767af50571cec2 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 15:11:25 +0000 Subject: [PATCH 10/18] chore: update to 1.0.0rc5 --- pyproject.toml | 2 +- uv.lock | 2 +- weko_group_cache_db/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41b41b0..e345ba8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "weko-group-cache-db" -version = "1.0.0rc4" +version = "1.0.0rc5" description = "CLI tool for caching mAP group information on JAIRO Cloud." readme = "README.md" requires-python = ">=3.14" diff --git a/uv.lock b/uv.lock index 8acd225..3467025 100644 --- a/uv.lock +++ b/uv.lock @@ -525,7 +525,7 @@ wheels = [ [[package]] name = "weko-group-cache-db" -version = "1.0.0rc4" +version = "1.0.0rc5" source = { virtual = "." } dependencies = [ { name = "backoff" }, diff --git a/weko_group_cache_db/__init__.py b/weko_group_cache_db/__init__.py index 70e895d..6449845 100644 --- a/weko_group_cache_db/__init__.py +++ b/weko_group_cache_db/__init__.py @@ -4,4 +4,4 @@ """weko-group-cache-db is a caching solution for group information in Redis.""" -__version__ = "1.0.0rc4" +__version__ = "1.0.0rc5" From 7e8bd737b024c6c7f7c1453d633d62ef263e18e4 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 16:06:01 +0000 Subject: [PATCH 11/18] feat: add progress status to signal args --- weko_group_cache_db/groups.py | 24 +++++++++++++++++++----- weko_group_cache_db/signals.py | 3 +++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 52c345a..4ee9671 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -63,6 +63,8 @@ def fetch_all(**kwargs: t.Unpack[InstitutionSource]): ) for index, institution in enumerate(institutions): + _send_progress_signal(institutions, index) + try: group_count = fetch_and_cache()(institution, store) logger.info( @@ -79,11 +81,12 @@ def fetch_all(**kwargs: t.Unpack[InstitutionSource]): exceptions.append(UpdateError(institution.fqdn, origin=ex)) finally: progress.update(task, advance=1) - _send_progress_signal(institutions, index) if index != total - 1: time.sleep(config.REQUEST_INTERVAL) + _send_progress_signal(institutions, total) + if exceptions: error_message = "Failed to update information from %d institution(s)." logger.error(error_message, len(exceptions)) @@ -264,11 +267,19 @@ def cache_key(fqdn: str) -> str: def _send_progress_signal(institutions: list[Institution], index: int) -> None: total = len(institutions) - done = index + 1 + status = "started" + if index > 0: + status = "in_progress" + elif index == total: + status = "completed" current = institutions[index].fqdn if index < total else "N/A" - data = ProgressData(total=total, done=done, current=current) - progress_signal.send(fetch_all, data=data) + data = ProgressData(status=status, total=total, done=index, current=current) + + try: + progress_signal.send(fetch_all, data=data) + except Exception: # noqa: BLE001 + traceback.print_exc() def _send_executed_signal( @@ -287,4 +298,7 @@ def _send_executed_signal( updated_at=updated_at or datetime.now(UTC), ) - executed_signal.send(fetch_all, data=data) + try: + executed_signal.send(fetch_all, data=data) + except Exception: # noqa: BLE001 + traceback.print_exc() diff --git a/weko_group_cache_db/signals.py b/weko_group_cache_db/signals.py index 378276b..ec68751 100644 --- a/weko_group_cache_db/signals.py +++ b/weko_group_cache_db/signals.py @@ -23,6 +23,9 @@ class ProgressData(BaseModel): """Data model for progress signal.""" + status: t.Literal["started", "in_progress", "completed"] + """Status of the fetch and cache operation.""" + total: int """Total number of institutions to process.""" From 16716717c4a919d79345acc95b76eedd4bf46dc8 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 16:12:48 +0000 Subject: [PATCH 12/18] chore: update to 1.0.0rc6 --- pyproject.toml | 2 +- uv.lock | 2 +- weko_group_cache_db/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e345ba8..3ff8678 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "weko-group-cache-db" -version = "1.0.0rc5" +version = "1.0.0rc6" description = "CLI tool for caching mAP group information on JAIRO Cloud." readme = "README.md" requires-python = ">=3.14" diff --git a/uv.lock b/uv.lock index 3467025..b6fea35 100644 --- a/uv.lock +++ b/uv.lock @@ -525,7 +525,7 @@ wheels = [ [[package]] name = "weko-group-cache-db" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { virtual = "." } dependencies = [ { name = "backoff" }, diff --git a/weko_group_cache_db/__init__.py b/weko_group_cache_db/__init__.py index 6449845..b537b8f 100644 --- a/weko_group_cache_db/__init__.py +++ b/weko_group_cache_db/__init__.py @@ -4,4 +4,4 @@ """weko-group-cache-db is a caching solution for group information in Redis.""" -__version__ = "1.0.0rc5" +__version__ = "1.0.0rc6" From a8d12f62b486e8805a7b9dab059d8be4f9ce1a10 Mon Sep 17 00:00:00 2001 From: ivis-kuroda <174298009+ivis-kuroda@users.noreply.github.com> Date: Wed, 11 Mar 2026 05:45:12 +0000 Subject: [PATCH 13/18] docs: update coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 46b7c02..9a51fed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # weko-group-cache-db / グループ情報キャッシュ機能 -Coverage
Coverage Report
FileStmtsMissBranchBrPartCover
weko_group_cache_db
   cli.py520120100%
   config.py83060100%
   exc.py9000100%
   groups.py850140100%
   loader.py1110220100%
   logger.py24020100%
   redis.py26020100%
TOTAL3900580100%
+Coverage
Coverage Report
FileStmtsMissBranchBrPartCoverMissing
weko_group_cache_db
   cli.py520120100% 
   config.py74080100% 
   exc.py9000100% 
   groups.py11751811 96%
   loader.py11612411 99%
   logger.py24020100% 
   redis.py26020100% 
   signals.py22000100% 
TOTAL440666298% 
## 概要 From 7046dbc23658290f1d0aee2d4ae59e0e3f56a892 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Mon, 26 Oct 2026 02:30:58 +0000 Subject: [PATCH 14/18] fix: send completed status in progress signal --- tests/test_groups.py | 99 +++++++++++++++++++++++++++++++++++ weko_group_cache_db/groups.py | 9 ++-- 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/tests/test_groups.py b/tests/test_groups.py index 859028f..9d4df82 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -12,6 +12,7 @@ from weko_group_cache_db.exc import UpdateError from weko_group_cache_db.groups import fetch_all, fetch_and_cache, fetch_map_groups, fetch_one, set_groups_to_redis from weko_group_cache_db.loader import Institution +from weko_group_cache_db.signals import executed_signal, progress_signal def test_fetch_all_success_toml(institutions_data, set_test_config, log_capture): @@ -21,11 +22,13 @@ def test_fetch_all_success_toml(institutions_data, set_test_config, log_capture) set_test_config() mock_store = MagicMock() + mock_receiver = MagicMock() with ( patch("weko_group_cache_db.groups.connection", return_value=mock_store), patch("weko_group_cache_db.groups.load_institutions") as mock_load, patch("weko_group_cache_db.groups.fetch_and_cache") as mock_fetch_and_cache, patch("weko_group_cache_db.groups.time") as mock_time, + progress_signal.connected_to(mock_receiver), ): mock_load.return_value = institutions mock_func = MagicMock(side_effect=[2, 3]) @@ -40,6 +43,13 @@ def test_fetch_all_success_toml(institutions_data, set_test_config, log_capture) assert log_capture.records[0].getMessage() == f"Successfully cached 2 groups for {data[0]['fqdn']}." assert log_capture.records[1].getMessage() == f"Successfully cached 3 groups for {data[1]['fqdn']}." mock_time.sleep.assert_called_once() + assert mock_receiver.call_count == num_institutions + 1 + assert all(call.args == (fetch_all,) for call in mock_receiver.call_args_list) + sent = [call.kwargs["data"] for call in mock_receiver.call_args_list] + assert [d.status for d in sent] == ["started", "in_progress", "completed"] + assert [d.done for d in sent] == [0, 1, 2] + assert [d.current for d in sent] == [data[0]["fqdn"], data[1]["fqdn"], "N/A"] + assert all(d.total == num_institutions for d in sent) def test_fetch_all_success_directory(institutions_data, set_test_config, log_capture): @@ -87,11 +97,13 @@ def test_fetch_all_request_error(institutions_data, set_test_config, log_capture test_config = set_test_config() mock_store = MagicMock() + mock_receiver = MagicMock() with ( patch("weko_group_cache_db.groups.connection", return_value=mock_store), patch("weko_group_cache_db.groups.load_institutions", return_value=institutions), patch("weko_group_cache_db.groups.fetch_and_cache") as mock_fetch_and_cache, patch("weko_group_cache_db.groups.time"), + progress_signal.connected_to(mock_receiver), ): num_groups = 2 mock_func = MagicMock(side_effect=[num_groups, requests.RequestException("Request failed")]) @@ -111,6 +123,10 @@ def test_fetch_all_request_error(institutions_data, set_test_config, log_capture assert isinstance(error_instance, UpdateError) assert isinstance(error_instance.origin, requests.RequestException) assert str(error_instance) == f"FQDN: {data[1]['fqdn']}, Request failed" + sent = [call.kwargs["data"] for call in mock_receiver.call_args_list] + assert [d.status for d in sent] == ["started", "in_progress", "completed"] + assert [d.done for d in sent] == [0, 1, 2] + assert [d.current for d in sent] == [data[0]["fqdn"], data[1]["fqdn"], "N/A"] def test_fetch_all_redis_error(institutions_data, set_test_config, log_capture): @@ -254,13 +270,17 @@ def test_fetch_and_cache_success(institutions_data, set_test_config): institution = Institution(**data[0]) set_test_config() groups = ["jc_group1", "jc_group2"] + updated_at = datetime(2025, 1, 1, tzinfo=UTC) mock_store = MagicMock() + mock_receiver = MagicMock() with ( patch("weko_group_cache_db.groups.fetch_map_groups") as mock_fetch, patch("weko_group_cache_db.groups.set_groups_to_redis") as mock_set, + executed_signal.connected_to(mock_receiver), ): mock_fetch.return_value = groups + mock_set.return_value = updated_at func = fetch_and_cache() count = func(institution, mock_store) @@ -268,6 +288,15 @@ def test_fetch_and_cache_success(institutions_data, set_test_config): assert count == len(groups) mock_fetch.assert_called_once_with(institution) mock_set.assert_called_once_with(institution.fqdn, groups, store=mock_store) + mock_receiver.assert_called_once() + assert mock_receiver.call_args.args == (fetch_all,) + sent = mock_receiver.call_args.kwargs["data"] + assert sent.fqdn == institution.fqdn + assert sent.status == "success" + assert sent.retries == 0 + assert sent.error_type is None + assert sent.error_message is None + assert sent.updated_at == updated_at def test_fetch_and_cache_all_request_exception(institutions_data, set_test_config, log_capture): @@ -276,9 +305,11 @@ def test_fetch_and_cache_all_request_exception(institutions_data, set_test_confi test_config = set_test_config(REQUEST_RETRY_FACTOR=0, REQUEST_RETRIES=2) mock_store = MagicMock() + mock_receiver = MagicMock() with ( patch("weko_group_cache_db.groups.fetch_map_groups") as mock_fetch, patch("weko_group_cache_db.groups.set_groups_to_redis") as mock_set, + executed_signal.connected_to(mock_receiver), ): mock_fetch.side_effect = requests.RequestException("Request failed") @@ -292,6 +323,13 @@ def test_fetch_and_cache_all_request_exception(institutions_data, set_test_confi record.getMessage() == f"Failed to fetch groups from mAP API for institution: {institution.fqdn}" for record in log_capture.records ) + assert mock_receiver.call_count == test_config.REQUEST_RETRIES + 1 + sent = [call.kwargs["data"] for call in mock_receiver.call_args_list] + assert all(d.fqdn == institution.fqdn for d in sent) + assert all(d.status == "failed" for d in sent) + assert [d.retries for d in sent] == [0, 1, 2] + assert all(d.error_type == "RequestException" for d in sent) + assert all(d.error_message == "Request failed" for d in sent) def test_fetch_and_cache_all_redis_error(institutions_data, set_test_config, log_capture): @@ -327,9 +365,11 @@ def test_fetch_and_cache_success_after_retries(institutions_data, set_test_confi groups = ["jc_group1", "jc_group2"] mock_store = MagicMock() + mock_receiver = MagicMock() with ( patch("weko_group_cache_db.groups.fetch_map_groups") as mock_fetch, patch("weko_group_cache_db.groups.set_groups_to_redis") as mock_set, + executed_signal.connected_to(mock_receiver), ): retries_fetch = 2 mock_fetch.side_effect = [requests.RequestException("Request failed"), groups, groups] @@ -346,6 +386,11 @@ def test_fetch_and_cache_success_after_retries(institutions_data, set_test_confi record.getMessage() == f"Failed to cache groups to Redis for institution: {institution.fqdn}" for record in log_capture.records ) + sent = [call.kwargs["data"] for call in mock_receiver.call_args_list] + assert [d.status for d in sent] == ["failed", "failed", "success"] + assert [d.retries for d in sent] == [0, 1, 2] + assert [d.error_type for d in sent] == ["RequestException", "RedisError", None] + assert [d.error_message for d in sent] == ["Request failed", "Cache error", None] def test_fetch_map_groups(institutions_data, set_test_config): @@ -438,3 +483,57 @@ def test_set_groups_to_redis_no_expire(institutions_data, set_test_config): mock_store.hset.assert_called_once_with(redis_key, mapping={"updated_at": timestamp, "groups": ",".join(groups)}) mock_store.persist.assert_called_once_with(redis_key) mock_store.expire.assert_not_called() + + +def test_fetch_all_progress_signal_receiver_error(institutions_data, set_test_config, log_capture): + num_institutions = 2 + data = institutions_data(num_institutions) + institutions = [Institution(**data[0]), Institution(**data[1])] + set_test_config() + + mock_receiver = MagicMock(side_effect=Exception("Receiver error")) + with ( + patch("weko_group_cache_db.groups.connection"), + patch("weko_group_cache_db.groups.load_institutions", return_value=institutions), + patch("weko_group_cache_db.groups.fetch_and_cache") as mock_fetch_and_cache, + patch("weko_group_cache_db.groups.time"), + patch("weko_group_cache_db.groups.traceback") as mock_traceback, + progress_signal.connected_to(mock_receiver), + ): + mock_func = MagicMock(side_effect=[2, 3]) + mock_fetch_and_cache.return_value = mock_func + + fetch_all(toml_path="institutions.toml") + + assert mock_func.call_count == num_institutions + assert mock_receiver.call_count == num_institutions + 1 + assert mock_traceback.print_exc.call_count == num_institutions + 1 + assert log_capture.records[0].getMessage() == f"Successfully cached 2 groups for {data[0]['fqdn']}." + assert log_capture.records[1].getMessage() == f"Successfully cached 3 groups for {data[1]['fqdn']}." + + +def test_fetch_and_cache_executed_signal_receiver_error(institutions_data, set_test_config): + data = institutions_data(1) + institution = Institution(**data[0]) + set_test_config() + groups = ["jc_group1", "jc_group2"] + + mock_store = MagicMock() + mock_receiver = MagicMock(side_effect=Exception("Receiver error")) + with ( + patch("weko_group_cache_db.groups.fetch_map_groups") as mock_fetch, + patch("weko_group_cache_db.groups.set_groups_to_redis") as mock_set, + patch("weko_group_cache_db.groups.traceback") as mock_traceback, + executed_signal.connected_to(mock_receiver), + ): + mock_fetch.return_value = groups + mock_set.return_value = datetime(2025, 1, 1, tzinfo=UTC) + + func = fetch_and_cache() + count = func(institution, mock_store) + + assert count == len(groups) + mock_fetch.assert_called_once_with(institution) + mock_set.assert_called_once_with(institution.fqdn, groups, store=mock_store) + mock_receiver.assert_called_once() + mock_traceback.print_exc.assert_called_once() diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 4ee9671..6e3a2bd 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -267,11 +267,12 @@ def cache_key(fqdn: str) -> str: def _send_progress_signal(institutions: list[Institution], index: int) -> None: total = len(institutions) - status = "started" - if index > 0: - status = "in_progress" - elif index == total: + if index >= total: status = "completed" + elif index > 0: + status = "in_progress" + else: + status = "started" current = institutions[index].fqdn if index < total else "N/A" data = ProgressData(status=status, total=total, done=index, current=current) From 059ce4e58d75088524bc2bdb55507625ed30d9af Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Mon, 26 Oct 2026 02:33:29 +0000 Subject: [PATCH 15/18] doc: add description of signals --- README.md | 56 +++++++++++++++++++++++++++++------ weko_group_cache_db/loader.py | 2 +- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9a51fed..b4c46ba 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ $ wgcd run --help 1つの機関に接続しているグループ情報を取得し、キャッシュDBへ登録する。 マウントされたディレクトリからFQDNリストファイルとSSL証明書・SSL鍵ファイルを使用して実行するディレクトリベースの実行方法と、機関情報を記述したTOMLファイルを使用して実行するファイルベースの実行方法がある。 -### ヘルプ表示 +#### ヘルプ表示 ``` $ wgcd one --help @@ -208,7 +208,6 @@ $ wgcd one --help ### カスタム設定値 [config.toml](./config.toml)に設定値を記載することで、実行時に設定値を反映して実行することが可能。 -また、環境変数、`.env`ファイル、Kubernetesのシークレットマウントによっても設定値を反映することが可能。 `MAP_GROUPS_API_ENDPOINT`以外はデフォルトの設定値があるため、それ以外は必要に応じて上書きしたい設定値のみを記載すればよい。 `config.toml`に記述する場合、大文字小文字は区別されない. @@ -227,7 +226,7 @@ $ wgcd one --help | REQUEST_RETRY_BASE | 数値 | 4 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの基準時間（秒） | | REQUEST_RETRY_FACTOR | 数値 | 5 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの係数（秒） | | REQUEST_RETRY_MAX | 数値 | 90 | グループ情報取得・キャッシュDBへの登録処理のリトライ時の指数バックオフの最大時間（秒） | -| REDIS_TYPE | 文字列 | redis | 使用するRedisの種類
"RedisCache", "RedisSentinelCache"のうちから指定 | +| REDIS_TYPE | 文字列 | RedisCache | 使用するRedisの種類
"RedisCache", "RedisSentinelCache"のうちから指定 | | REDIS_HOST | 文字列 | localhost | Redisのホスト名 | | REDIS_PORT | 数値 | 6379 | Redisのポート番号 | | REDIS_DB_INDEX | 数値 | 4 | グループ情報を格納するRedisのDBインデックス | @@ -239,15 +238,15 @@ $ wgcd one --help ※ Redis Sentinel の設定値をTOMLファイルに記載する場合の例 ``` [[sentinels]] -host = "sendinel-1" +host = "sentinel-1" port = 26379 [[sentinels]] -host = "sendinel-2" +host = "sentinel-2" port = 26378 [[sentinels]] -host = "sendinel-3" +host = "sentinel-3" port = 26377 ``` @@ -255,9 +254,48 @@ port = 26377 1. TOMLファイル 2. 環境変数 -3. `.env`ファイル -4. Kubernetesのシークレットマウント -5. デフォルト値（`MAP_GROUPS_API_ENDPOINT`以外） +3. デフォルト値（`MAP_GROUPS_API_ENDPOINT`以外） + + +## シグナル + +[blinker](https://blinker.readthedocs.io/) のシグナルにより、処理の進捗と各機関の処理結果を外部から受け取ることができる。 + +```python +from weko_group_cache_db.signals import executed_signal, progress_signal + +@progress_signal.connect +def on_progress(sender, data): + print(f"{data.status}: {data.done}/{data.total}") +``` + +| シグナル | 送信タイミング | 対象コマンド | データ型 | +| ----------------- | --------------------------------------------------------- | -------------- | -------------- | +| `progress_signal` | 各機関の処理開始前、および全機関の処理完了後 | `run` | `ProgressData` | +| `executed_signal` | 各機関の処理の試行ごと（リトライを含む）の成功時・失敗時 | `run`, `one` | `ExecutedData` | + +### progress_signal + +| 項目 | 型 | 説明 | +| --------- | ------ | -------------------------------------------------------------------- | +| `status` | 文字列 | `started`：最初の機関の処理開始前
`in_progress`：2件目以降の機関の処理開始前
`completed`：全機関の処理完了後（失敗した機関があっても送信される） | +| `total` | 数値 | 処理対象の機関数 | +| `done` | 数値 | 処理済みの機関数 | +| `current` | 文字列 | これから処理する機関のFQDN（`completed` の場合は `N/A`） | + +### executed_signal + +| 項目 | 型 | 説明 | +| --------------- | -------- | -------------------------------------------------------------------- | +| `fqdn` | 文字列 | 機関のFQDN | +| `status` | 文字列 | `success`：グループ情報のRedisへの登録に成功
`failed`：グループ情報の取得またはRedisへの登録に失敗 | +| `retries` | 数値 | その試行までのリトライ回数（初回の試行は 0） | +| `error_type` | 文字列 | 失敗時の例外クラス名（成功時は `None`） | +| `error_message` | 文字列 | 失敗時のエラーメッセージ（成功時は `None`） | +| `updated_at` | 日時 | 成功時はRedisへの登録日時、失敗時はシグナルの送信日時（UTC） | + +※ `failed` はリトライのたびに送信される。 + ## 開発環境構築 diff --git a/weko_group_cache_db/loader.py b/weko_group_cache_db/loader.py index 4c8fd19..b342d54 100644 --- a/weko_group_cache_db/loader.py +++ b/weko_group_cache_db/loader.py @@ -264,7 +264,7 @@ def check_existence_file(institution: Institution) -> bool: class Institution(BaseModel): """Institution model for loading from TOML file.""" - name: str | None + name: str | None = None """Name of the institution. Optional.""" fqdn: str From 8fc3c82af367f26426f407839b361f2cc696946a Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Mon, 26 Oct 2026 02:45:59 +0000 Subject: [PATCH 16/18] chore: specify build backend --- Dockerfile | 3 +-- pyproject.toml | 10 ++++++++++ uv.lock | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce47ac2..db62bb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,6 @@ FROM base AS production ENV VIRTUAL_ENV="/code/.venv" ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN uv sync --no-dev --frozen && \ - uv pip install . +RUN uv sync --no-dev --frozen CMD ["sh", "-c", "COLUMNS=120 exec wgcd run -d /var/mnt -l /var/repositories.txt"] diff --git a/pyproject.toml b/pyproject.toml index 3ff8678..a3b32c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ +#:schema https://json.schemastore.org/pyproject.json + +[build-system] +requires = ["uv_build>=0.13.5,<0.14"] +build-backend = "uv_build" + [project] name = "weko-group-cache-db" version = "1.0.0rc6" @@ -30,6 +36,10 @@ dev = [ "toml>=0.10.2", ] +[tool.uv.build-backend] +module-name = "weko_group_cache_db" +module-root = "" + [tool.taskipy.tasks] # refer to https://github.com/taskipy/taskipy?tab=readme-ov-file#usage test = "pytest" diff --git a/uv.lock b/uv.lock index b6fea35..cfb9afb 100644 --- a/uv.lock +++ b/uv.lock @@ -526,7 +526,7 @@ wheels = [ [[package]] name = "weko-group-cache-db" version = "1.0.0rc6" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "backoff" }, { name = "blinker" }, From c79859d50c32c7cfaaf5d2018ddcd86f411c70fb Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Mon, 26 Oct 2026 02:51:54 +0000 Subject: [PATCH 17/18] style: update lint suppression comments --- README.md | 1 + pyproject.toml | 12 ++++++------ tests/ruff.toml | 14 +++++++------- weko_group_cache_db/config.py | 4 ++-- weko_group_cache_db/exc.py | 2 +- weko_group_cache_db/groups.py | 4 ++-- weko_group_cache_db/redis.py | 2 +- weko_group_cache_db/signals.py | 2 +- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b4c46ba..a454f95 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,7 @@ port = 26377 ```python from weko_group_cache_db.signals import executed_signal, progress_signal + @progress_signal.connect def on_progress(sender, data): print(f"{data.status}: {data.done}/{data.total}") diff --git a/pyproject.toml b/pyproject.toml index a3b32c1..821b9d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,12 +67,12 @@ exclude_also = [ [tool.ruff.lint] # refer to https://docs.astral.sh/ruff/settings/#lint ignore = [ - "D203", - "D213", - "TC006", - "COM812", - "ANN201", - "TRY400", + "incorrect-blank-line-before-class", + "multi-line-summary-second-line", + "runtime-cast-value", + "missing-trailing-comma", + "missing-return-type-undocumented-public-function", + "error-instead-of-exception", ] preview = true select = ["ALL"] diff --git a/tests/ruff.toml b/tests/ruff.toml index bdb096a..81d25b4 100644 --- a/tests/ruff.toml +++ b/tests/ruff.toml @@ -4,14 +4,14 @@ target-version = "py314" [lint] ignore = [ "D", - "S101", + "assert", "ANN", - "COM812", - "TC006", - "DTZ005", - "PLC2701", - "SLF001", - "SIM300", + "missing-trailing-comma", + "runtime-cast-value", + "call-datetime-now-without-tzinfo", + "import-private-name", + "private-member-access", + "yoda-conditions", ] preview = true select = ["ALL"] diff --git a/weko_group_cache_db/config.py b/weko_group_cache_db/config.py index b4feba8..101b957 100644 --- a/weko_group_cache_db/config.py +++ b/weko_group_cache_db/config.py @@ -79,7 +79,7 @@ class Settings(BaseSettings): @computed_field @property - def REDIS_URL(self) -> str: # noqa: N802 + def REDIS_URL(self) -> str: # ruff: ignore[invalid-function-name] """Redis URL for caching group information.""" return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_DB_INDEX}" @@ -91,7 +91,7 @@ def REDIS_URL(self) -> str: # noqa: N802 @computed_field @property - def REDIS_SENTINELS(self) -> list[tuple[str, str]]: # noqa: N802 + def REDIS_SENTINELS(self) -> list[tuple[str, str]]: # ruff: ignore[invalid-function-name] """A list of Redis sentinel host names and their ports.""" return ( [(sentinel.host, str(sentinel.port)) for sentinel in self.SENTINELS] diff --git a/weko_group_cache_db/exc.py b/weko_group_cache_db/exc.py index 96d097c..08583fa 100644 --- a/weko_group_cache_db/exc.py +++ b/weko_group_cache_db/exc.py @@ -24,7 +24,7 @@ def __init__(self, fqdn: str, origin: Exception | None = None) -> None: self.fqdn = fqdn self.origin = origin - def __str__(self) -> str: # noqa: D105 + def __str__(self) -> str: # ruff: ignore[undocumented-magic-method] return f"FQDN: {self.fqdn}, {self.origin}" diff --git a/weko_group_cache_db/groups.py b/weko_group_cache_db/groups.py index 6e3a2bd..b2ad2c9 100644 --- a/weko_group_cache_db/groups.py +++ b/weko_group_cache_db/groups.py @@ -279,7 +279,7 @@ def _send_progress_signal(institutions: list[Institution], index: int) -> None: try: progress_signal.send(fetch_all, data=data) - except Exception: # noqa: BLE001 + except Exception: # ruff: ignore[blind-except] traceback.print_exc() @@ -301,5 +301,5 @@ def _send_executed_signal( try: executed_signal.send(fetch_all, data=data) - except Exception: # noqa: BLE001 + except Exception: # ruff: ignore[blind-except] traceback.print_exc() diff --git a/weko_group_cache_db/redis.py b/weko_group_cache_db/redis.py index bd206b6..6e77173 100644 --- a/weko_group_cache_db/redis.py +++ b/weko_group_cache_db/redis.py @@ -22,7 +22,7 @@ def connection() -> Redis: ConnectionError: If failed to connect to Redis. """ - try: + try: # ruff: ignore[too-many-statements-in-try-clause] if config.REDIS_TYPE == "RedisCache": store = _redis_connection() store.ping() diff --git a/weko_group_cache_db/signals.py b/weko_group_cache_db/signals.py index ec68751..5797c40 100644 --- a/weko_group_cache_db/signals.py +++ b/weko_group_cache_db/signals.py @@ -6,7 +6,7 @@ import typing as t -from datetime import datetime # noqa: TC003 +from datetime import datetime # ruff: ignore[typing-only-standard-library-import] from blinker import Namespace from pydantic import BaseModel From 47200219e3405e3f17775fe79d1d943d403b04ca Mon Sep 17 00:00:00 2001 From: ivis-kuroda <174298009+ivis-kuroda@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:54:15 +0000 Subject: [PATCH 18/18] docs: update coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a454f95..c144930 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # weko-group-cache-db / グループ情報キャッシュ機能 -Coverage
Coverage Report
FileStmtsMissBranchBrPartCoverMissing
weko_group_cache_db
   cli.py520120100% 
   config.py74080100% 
   exc.py9000100% 
   groups.py11751811 96%
   loader.py11612411 99%
   logger.py24020100% 
   redis.py26020100% 
   signals.py22000100% 
TOTAL440666298% 
+

Coverage

Coverage Report
FileStmtsMissBranchBrPartCoverMissing
weko_group_cache_db
   cli.py520120100% 
   config.py74080100% 
   exc.py9000100% 
   groups.py1170180100% 
   loader.py11712411 99%
   logger.py24020100% 
   redis.py26020100% 
   signals.py22000100% 
TOTAL441166199% 
## 概要