Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions openhack/deterministic_recon.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ def _detect_features_fast(
find_cmd.append("-o")
find_cmd.extend(["-name", f"*{ext}"])
find_cmd.append(")")
# Keep filenames literal across find, xargs, and grep.
find_cmd.append("-print0")
for d in _GREP_EXCLUDE_DIRS:
clean = d.rstrip("*").rstrip(".")
find_cmd[2:2] = ["-not", "-path", f"*/{clean}/*"]
Expand All @@ -158,10 +160,10 @@ def _detect_features_fast(
logger.warning(f"Find failed: {e}")
return {}

if not file_list.strip():
if not file_list:
return {}

file_count = file_list.count("\n")
file_count = file_list.count("\0")

if file_count > 5000:
# Large repo: fixed-string grep in parallel threads
Expand All @@ -179,7 +181,7 @@ def _detect_features_fast(

def _check_feature(name_and_keywords):
fname, keywords = name_and_keywords
cmd = ["xargs", "grep", "-Fl", "--max-count=1",
cmd = ["xargs", "-0", "grep", "-FlZ", "--max-count=1",
"--binary-files=without-match"]
for kw in keywords:
cmd.extend(["-e", kw])
Expand All @@ -188,7 +190,7 @@ def _check_feature(name_and_keywords):
cmd, input=file_list, capture_output=True,
text=True, timeout=15,
)
matches = [l for l in proc.stdout.strip().split("\n") if l.strip()]
matches = [fp for fp in proc.stdout.split("\0") if fp]
if matches:
readable = fname.replace("_", " ").title()
return fname, [f"{readable} ({len(matches)} files)"]
Expand All @@ -210,16 +212,15 @@ def _check_feature(name_and_keywords):
result: dict[str, list[str]] = {}
for feature_name, patterns in feature_indicators.items():
combined = "|".join(p for p, _ in patterns)
cmd_parts = ["xargs", "grep", "-El", "--max-count=1",
cmd_parts = ["xargs", "-0", "grep", "-ElZ", "--max-count=1",
"--binary-files=without-match", combined]
try:
proc = subprocess.run(
cmd_parts, input=file_list, capture_output=True,
text=True, timeout=30,
)
files = []
for line in proc.stdout.strip().split("\n"):
fp = line.strip()
for fp in proc.stdout.split("\0"):
if not fp:
continue
try:
Expand Down
37 changes: 34 additions & 3 deletions tests/test_deterministic_recon.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
import json
from pathlib import Path
import os

import pytest
from tests.conftest import write_file, write_json

from openhack.tools.registry import ToolRegistry
from openhack.deterministic_recon import run_deterministic_recon
from openhack.deterministic_recon import (
_FEATURE_INDICATORS,
_detect_features_fast,
run_deterministic_recon,
)
from openhack.tools.filesystem import FileSystemTools


class TestFeatureDetection:
Expand Down Expand Up @@ -53,3 +59,28 @@ def test_returns_expected_keys(self, tmp_path):
assert "type" in result
assert result["type"] == "recon_complete"
assert "attack_surface" in result


class TestFeatureDetectionPaths:
@pytest.mark.parametrize("file_count", [2, 5001], ids=["regex", "fixed-string"])
def test_quoted_paths_preserve_all_features(self, tmp_path, file_count):
write_file(tmp_path, "src/owner's uploads/don't_break.js", "multer();\n")
write_file(tmp_path, "auth.js", "bcrypt();\n")
for index in range(file_count - 2):
write_file(tmp_path, f"src/empty_{index}.js", "")

features = _detect_features_fast(FileSystemTools(tmp_path), _FEATURE_INDICATORS)

assert features == {
"file_uploads": ["File Uploads (1 files)"],
"auth_system": ["Auth System (1 files)"],
}

@pytest.mark.skipif(os.name == "nt", reason="Requires POSIX filename characters")
@pytest.mark.parametrize("filename", ['double"quote.js', "back\\slash.js", "new\nline.js", "tab\tname.js"])
def test_special_filename_is_counted_once(self, tmp_path, filename):
write_file(tmp_path, filename, "multer();\n")

features = _detect_features_fast(FileSystemTools(tmp_path), _FEATURE_INDICATORS)

assert features == {"file_uploads": ["File Uploads (1 files)"]}