From b3aed61e193fe973c32616fee65111bb8fb1815b Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 31 Jul 2026 09:59:12 +0900 Subject: [PATCH 1/3] feat(android): show binary match progress and abort on HTTP 404 Add a tqdm progress bar for match chunks, and stop remaining chunks when /binary/match is missing (404) instead of retrying every chunk. --- .../_binary_db_controller.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/fosslight_android/_binary_db_controller.py b/src/fosslight_android/_binary_db_controller.py index a39e1f2..9e99fd8 100644 --- a/src/fosslight_android/_binary_db_controller.py +++ b/src/fosslight_android/_binary_db_controller.py @@ -10,6 +10,8 @@ import urllib.request from typing import Dict, List, Optional, Tuple +from tqdm import tqdm + from ._common import CONST_TLSH_NULL from fosslight_util.constant import LOGGER_NAME @@ -21,7 +23,7 @@ _CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "1000")) MatchKey = Tuple[str, str] -# (response_or_None, unreachable) — unreachable stops remaining chunks +# (response_or_None, stop_remaining) — stop on unreachable or HTTP 404 PostMatchResult = Tuple[Optional[dict], bool] @@ -103,19 +105,23 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): return bin_info_list endpoint = f"{base_url.rstrip('/')}{_BINARY_MATCH_PATH}" - logger.info(f"Querying KB binary match: {endpoint}") + logger.info(f"Querying KB binary match") results_by_id = {} kb_reachable_logged = False try: - for chunk_start in range(0, len(items_payload), _CHUNK_SIZE): + for chunk_start in tqdm( + range(0, len(items_payload), _CHUNK_SIZE), + desc="Binary DB match", + unit="chunk", + ): chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] - response, unreachable = _post_binary_match(base_url, token, chunk) - if unreachable: - # Host not reachable — do not attempt remaining chunks + response, stop_remaining = _post_binary_match(base_url, token, chunk) + if stop_remaining: + # Unreachable or /binary/match missing (404) — do not retry chunks break if not kb_reachable_logged: - logger.info(f"KB({base_url}) reachable") + logger.debug(f"KB({base_url}) reachable") kb_reachable_logged = True if response is None: logger.warning( @@ -143,7 +149,10 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResult: - """POST one chunk. Returns (body, unreachable). unreachable stops further chunks.""" + """POST one chunk. Returns (body, stop_remaining). + + stop_remaining is True for host unreachable or HTTP 404 (endpoint missing). + """ data = json.dumps({"items": items}).encode("utf-8") request = urllib.request.Request( f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}", @@ -165,7 +174,13 @@ def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResu body = ex.read().decode() except Exception: pass - # Host responded → reachable; caller may continue with next chunks + if ex.code == 404: + logger.warning( + f"KB({kb_url}) binary match endpoint not found (HTTP 404); " + "skipping remaining chunks." + ) + return None, True + # Other HTTP errors → reachable; caller may continue with next chunks logger.debug( f"KB({kb_url}) reachable but binary match HTTP {ex.code}: {body or ex.reason}" ) From 44977edae3a9b752981fa14f469fd9ef42acf772 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 31 Jul 2026 10:04:47 +0900 Subject: [PATCH 2/3] fix(android): silence flake8 F841/F541 in binary DB controller Use the KB match endpoint in the info log instead of an unused local and a placeholder-less f-string. --- src/fosslight_android/_binary_db_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fosslight_android/_binary_db_controller.py b/src/fosslight_android/_binary_db_controller.py index 9e99fd8..7752aaa 100644 --- a/src/fosslight_android/_binary_db_controller.py +++ b/src/fosslight_android/_binary_db_controller.py @@ -104,8 +104,7 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): if not items_payload: return bin_info_list - endpoint = f"{base_url.rstrip('/')}{_BINARY_MATCH_PATH}" - logger.info(f"Querying KB binary match") + logger.info(f"Querying KB binary match: {base_url.rstrip('/')}{_BINARY_MATCH_PATH}") results_by_id = {} kb_reachable_logged = False From 68064bb162cfaa52e6ecd99ebb64804d07067e16 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 31 Jul 2026 10:07:48 +0900 Subject: [PATCH 3/3] fix(android): abort binary match chunks on HTTP 401 Stop remaining /binary/match chunks when the token is denied, same as endpoint-missing 404, instead of retrying every chunk. --- .../_binary_db_controller.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/fosslight_android/_binary_db_controller.py b/src/fosslight_android/_binary_db_controller.py index 7752aaa..adc6584 100644 --- a/src/fosslight_android/_binary_db_controller.py +++ b/src/fosslight_android/_binary_db_controller.py @@ -23,7 +23,7 @@ _CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "1000")) MatchKey = Tuple[str, str] -# (response_or_None, stop_remaining) — stop on unreachable or HTTP 404 +# (response_or_None, stop_remaining) — stop on unreachable, HTTP 404, or HTTP 401 PostMatchResult = Tuple[Optional[dict], bool] @@ -104,8 +104,6 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): if not items_payload: return bin_info_list - logger.info(f"Querying KB binary match: {base_url.rstrip('/')}{_BINARY_MATCH_PATH}") - results_by_id = {} kb_reachable_logged = False try: @@ -117,7 +115,7 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] response, stop_remaining = _post_binary_match(base_url, token, chunk) if stop_remaining: - # Unreachable or /binary/match missing (404) — do not retry chunks + # Unreachable, auth denied (401), or /binary/match missing (404) break if not kb_reachable_logged: logger.debug(f"KB({base_url}) reachable") @@ -150,7 +148,8 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResult: """POST one chunk. Returns (body, stop_remaining). - stop_remaining is True for host unreachable or HTTP 404 (endpoint missing). + stop_remaining is True for host unreachable, HTTP 401 (auth denied), + or HTTP 404 (endpoint missing). """ data = json.dumps({"items": items}).encode("utf-8") request = urllib.request.Request( @@ -173,10 +172,14 @@ def _post_binary_match(kb_url: str, kb_token: str, items: list) -> PostMatchResu body = ex.read().decode() except Exception: pass - if ex.code == 404: + if ex.code in (401, 404): + reason = ( + "auth denied (HTTP 401)" + if ex.code == 401 + else "endpoint not found (HTTP 404)" + ) logger.warning( - f"KB({kb_url}) binary match endpoint not found (HTTP 404); " - "skipping remaining chunks." + f"KB({kb_url}) binary match {reason}; skipping remaining chunks." ) return None, True # Other HTTP errors → reachable; caller may continue with next chunks