diff --git a/src/fosslight_android/_binary_db_controller.py b/src/fosslight_android/_binary_db_controller.py index a39e1f2..adc6584 100644 --- a/src/fosslight_android/_binary_db_controller.py +++ b/src/fosslight_android/_binary_db_controller.py @@ -10,6 +10,8 @@ import urllib.request from typing import Dict, List, Optional, Tuple +from tqdm import tqdm + from ._common import CONST_TLSH_NULL from fosslight_util.constant import LOGGER_NAME @@ -21,7 +23,7 @@ _CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "1000")) MatchKey = Tuple[str, str] -# (response_or_None, unreachable) — unreachable stops remaining chunks +# (response_or_None, stop_remaining) — stop on unreachable, HTTP 404, or HTTP 401 PostMatchResult = Tuple[Optional[dict], bool] @@ -102,20 +104,21 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): if not items_payload: return bin_info_list - endpoint = f"{base_url.rstrip('/')}{_BINARY_MATCH_PATH}" - logger.info(f"Querying KB binary match: {endpoint}") - results_by_id = {} kb_reachable_logged = False try: - for chunk_start in range(0, len(items_payload), _CHUNK_SIZE): + for chunk_start in tqdm( + range(0, len(items_payload), _CHUNK_SIZE), + desc="Binary DB match", + unit="chunk", + ): chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] - response, unreachable = _post_binary_match(base_url, token, chunk) - if unreachable: - # Host not reachable — do not attempt remaining chunks + response, stop_remaining = _post_binary_match(base_url, token, chunk) + if stop_remaining: + # Unreachable, auth denied (401), or /binary/match missing (404) break if not kb_reachable_logged: - logger.info(f"KB({base_url}) reachable") + logger.debug(f"KB({base_url}) reachable") kb_reachable_logged = True if response is None: logger.warning( @@ -143,7 +146,11 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResult: - """POST one chunk. Returns (body, unreachable). unreachable stops further chunks.""" + """POST one chunk. Returns (body, stop_remaining). + + stop_remaining is True for host unreachable, HTTP 401 (auth denied), + or HTTP 404 (endpoint missing). + """ data = json.dumps({"items": items}).encode("utf-8") request = urllib.request.Request( f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}", @@ -165,7 +172,17 @@ def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResu body = ex.read().decode() except Exception: pass - # Host responded → reachable; caller may continue with next chunks + if ex.code in (401, 404): + reason = ( + "auth denied (HTTP 401)" + if ex.code == 401 + else "endpoint not found (HTTP 404)" + ) + logger.warning( + f"KB({kb_url}) binary match {reason}; skipping remaining chunks." + ) + return None, True + # Other HTTP errors → reachable; caller may continue with next chunks logger.debug( f"KB({kb_url}) reachable but binary match HTTP {ex.code}: {body or ex.reason}" )