diff --git a/.github/workflows/smoke-test-ffi.yml b/.github/workflows/smoke-test-ffi.yml index a6705062b..7d2dff2ee 100644 --- a/.github/workflows/smoke-test-ffi.yml +++ b/.github/workflows/smoke-test-ffi.yml @@ -48,7 +48,36 @@ jobs: run: iex "& { $(iwr 'https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.ps1' -UseBasicParsing) } -ci" - name: Install dependencies - run: make install && cp -r .cache/binaries/* aikido_zen/libs + run: make install - name: Run tests run: poetry run pytest aikido_zen/vulnerabilities/sql_injection/init_test.py + + smoke-test-musl: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-22.04-arm] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download binaries + run: make binaries && cp -r .cache/binaries/* aikido_zen/libs + + # pip runs inside the container, so safe-chain is set up there and not on the runner. + # build-base compiles pytricia, which has no prebuilt wheel. + - name: Run tests on python:${{ matrix.python-version }}-alpine + run: | + docker run --rm -v "$(pwd):/app" -w /app python:${{ matrix.python-version }}-alpine sh -eo pipefail -c " + apk add --no-cache build-base curl + curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci + export PATH=\"\$HOME/.safe-chain/shims:\$HOME/.safe-chain/bin:\$PATH\" + pip install --root-user-action=ignore . pytest + python -m pytest aikido_zen/helpers/zen_internals_test.py + " diff --git a/Makefile b/Makefile index 0dfd05eeb..eed0f48be 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,6 @@ BUILD_DIR := dist/ .PHONY: build build: check_binaries @echo "Building using poetry, creates $(BUILD_DIR) folder." - @echo "Copying binaries from $(CACHE_DIR) folder : " - cp -r .cache/binaries/* aikido_zen/libs poetry build .PHONY: clean clean: @@ -46,20 +44,27 @@ benchmark: # Binaries cache : -BASE_URL = https://github.com/AikidoSec/zen-internals/releases/download/v0.1.60 +BASE_URL = https://github.com/AikidoSec/zen-internals/releases/download/v0.1.74 FILES = \ libzen_internals_aarch64-apple-darwin.dylib \ libzen_internals_aarch64-apple-darwin.dylib.sha256sum \ + libzen_internals_aarch64-pc-windows-msvc.dll \ + libzen_internals_aarch64-pc-windows-msvc.dll.sha256sum \ libzen_internals_aarch64-unknown-linux-gnu.so \ libzen_internals_aarch64-unknown-linux-gnu.so.sha256sum \ + libzen_internals_aarch64-unknown-linux-musl.so \ + libzen_internals_aarch64-unknown-linux-musl.so.sha256sum \ libzen_internals_x86_64-apple-darwin.dylib \ libzen_internals_x86_64-apple-darwin.dylib.sha256sum \ libzen_internals_x86_64-pc-windows-gnu.dll \ libzen_internals_x86_64-pc-windows-gnu.dll.sha256sum \ libzen_internals_x86_64-unknown-linux-gnu.so \ - libzen_internals_x86_64-unknown-linux-gnu.so.sha256sum + libzen_internals_x86_64-unknown-linux-gnu.so.sha256sum \ + libzen_internals_x86_64-unknown-linux-musl.so \ + libzen_internals_x86_64-unknown-linux-musl.so.sha256sum binaries: binaries_make_dir $(addprefix .cache/binaries/, $(FILES)) + echo "$(BASE_URL)" > $(CACHE_DIR)/.version binaries_make_dir: rm -rf .cache/binaries mkdir -p .cache/binaries/ @@ -67,14 +72,16 @@ binaries_make_dir: @echo "Downloading $*..." curl -L -o $@ $(BASE_URL)/$* +# Binaries from another release may not match the Python bindings, so download them again. .PHONY: check_binaries check_binaries: - @if [ -d "$(CACHE_DIR)" ]; then \ - echo "Directory $(CACHE_DIR) exists."; \ + @if [ "$$(cat $(CACHE_DIR)/.version 2>/dev/null)" = "$(BASE_URL)" ]; then \ + echo "Directory $(CACHE_DIR) has the binaries from $(BASE_URL)."; \ else \ - echo "Directory $(CACHE_DIR) is empty. Running 'make binaries'..."; \ + echo "Directory $(CACHE_DIR) is missing or outdated. Running 'make binaries'..."; \ $(MAKE) binaries; \ fi + cp -r $(CACHE_DIR)/* aikido_zen/libs # Replace version number automatically on publish : diff --git a/aikido_zen/helpers/get_lib_path.py b/aikido_zen/helpers/get_lib_path.py deleted file mode 100644 index 72536409c..000000000 --- a/aikido_zen/helpers/get_lib_path.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Exports get_binary_path""" - -import platform -import os - - -def get_binary_path(): - """Returns an absolute path for Rust binary file""" - current_dir = os.path.dirname(os.path.abspath(__file__)) - lib_path = os.path.join(current_dir, "../libs", get_file_name()) - return lib_path - - -def get_file_name(): - """Gives you the file name for the binary based on platform info""" - os_name = platform.system().lower() - machine = platform.machine().lower() - file_name = "libzen_internals_" - - # On macOS, platform.machine() returns "arm64" for Apple Silicon - # On Linux, platform.machine() returns "aarch64" for ARM64 - if "arm64" in machine or "aarch64" in machine: - file_name += "aarch64-" - # On macOS, platform.machine() returns "x86_64" for Intel - # On Linux, platform.machine() returns "x86_64" for AMD64 - elif "x86_64" in machine or "amd64" in machine: - file_name += "x86_64-" # x86_64 or AMD64 - - if os_name == "windows": - file_name += "pc-windows-gnu.dll" # Windows - elif os_name == "darwin": - file_name += "apple-darwin.dylib" # macOS - elif os_name in ["linux", "linux2"]: - file_name += "unknown-linux-gnu.so" # Linux - - return file_name diff --git a/aikido_zen/helpers/zen_internals.py b/aikido_zen/helpers/zen_internals.py index a706a6c51..b2027b44d 100644 --- a/aikido_zen/helpers/zen_internals.py +++ b/aikido_zen/helpers/zen_internals.py @@ -6,30 +6,74 @@ import ctypes import os import platform +import sysconfig -from aikido_zen.helpers.get_lib_path import get_binary_path from aikido_zen.helpers.logging import logger +_TARGET = f"{platform.system().lower()}-{platform.machine().lower()}" + + +def _binary_path(): + os_name = platform.system().lower() + machine = platform.machine().lower() + is_arm = "arm64" in machine or "aarch64" in machine + file_name = "libzen_internals_" + + # On macOS, platform.machine() returns "arm64" for Apple Silicon + # On Linux, platform.machine() returns "aarch64" for ARM64 + if is_arm: + file_name += "aarch64-" + # On macOS, platform.machine() returns "x86_64" for Intel + # On Linux, platform.machine() returns "x86_64" for AMD64 + elif "x86_64" in machine or "amd64" in machine: + file_name += "x86_64-" + + if os_name == "windows": + file_name += "pc-windows-msvc.dll" if is_arm else "pc-windows-gnu.dll" + elif os_name == "darwin": + file_name += "apple-darwin.dylib" + elif os_name in ["linux", "linux2"]: + # The library must use the same libc as this Python, which records it at build + # time (for example "x86_64-pc-linux-musl"). platform.libc_ver() and files under + # /lib describe the system, which can have both glibc and musl installed. + host = sysconfig.get_config_var("HOST_GNU_TYPE") or "" + libc = "musl" if "musl" in host else "gnu" + file_name += f"unknown-linux-{libc}.so" + + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../libs", file_name + ) + + def _load(): - binary_path = get_binary_path() - target = f"{platform.system().lower()}-{platform.machine().lower()}" + binary_path = _binary_path() if not os.path.exists(binary_path): logger.warning( - "Zen will NOT block SQL injection attacks. Cannot find the zen-internals library for %s. " + "Cannot find the zen-internals library for %s. " "Request support: https://github.com/AikidoSec/firewall-python/issues", - target, + _TARGET, ) return None try: return ctypes.CDLL(binary_path) except OSError as e: - logger.warning( - "Zen will NOT block SQL injection attacks. Failed to load the zen-internals library for %s: %s", - target, - e, - ) - return None + error = e + # A musl Python built without HOST_GNU_TYPE picks the glibc build, which musl + # cannot load. + musl_path = os.path.join( + os.path.dirname(binary_path), + os.path.basename(binary_path).replace("-gnu.so", "-musl.so"), + ) + if musl_path != binary_path and os.path.exists(musl_path): + try: + return ctypes.CDLL(musl_path) + except OSError: + pass + logger.warning( + "Failed to load the zen-internals library for %s: %s", _TARGET, error + ) + return None def _bind(internals_lib, name, argtypes, restype): @@ -39,7 +83,9 @@ def _bind(internals_lib, name, argtypes, restype): try: function = getattr(internals_lib, name) except AttributeError: - logger.debug("The zen-internals library has no function %s", name) + logger.warning( + "The zen-internals library for %s has no %s function.", _TARGET, name + ) return None function.argtypes = argtypes function.restype = restype @@ -60,6 +106,8 @@ def _bind(internals_lib, name, argtypes, restype): ], ctypes.c_int, ) +if _detect_sql_injection is None: + logger.warning("Zen will NOT block SQL injection attacks.") def detect_sql_injection(query: bytes, user_input: bytes, dialect: int) -> int: diff --git a/aikido_zen/helpers/zen_internals_test.py b/aikido_zen/helpers/zen_internals_test.py index 483e532b2..cdc59ac5f 100644 --- a/aikido_zen/helpers/zen_internals_test.py +++ b/aikido_zen/helpers/zen_internals_test.py @@ -1,4 +1,10 @@ import ctypes +import os +import shutil +import subprocess +import sys + +import pytest from aikido_zen.helpers import zen_internals @@ -37,30 +43,46 @@ def test_detect_sql_injection_returns_error_when_function_is_not_loaded(monkeypa def test_load_returns_none_when_library_file_is_missing(monkeypatch, caplog): monkeypatch.setattr( - "aikido_zen.helpers.zen_internals.get_binary_path", + "aikido_zen.helpers.zen_internals._binary_path", lambda: "/nonexistent/libzen_internals.so", ) assert zen_internals._load() is None - assert "Zen will NOT block SQL injection attacks" in caplog.text assert "Cannot find the zen-internals library" in caplog.text + assert "Request support" in caplog.text def test_load_returns_none_when_library_fails_to_load(monkeypatch, caplog): monkeypatch.setattr( - "aikido_zen.helpers.zen_internals.get_binary_path", lambda: __file__ + "aikido_zen.helpers.zen_internals._binary_path", lambda: __file__ ) assert zen_internals._load() is None - assert "Zen will NOT block SQL injection attacks" in caplog.text assert "Failed to load the zen-internals library" in caplog.text +def test_load_falls_back_to_the_musl_build_when_the_glibc_build_fails( + monkeypatch, tmp_path +): + libs = tmp_path / "app-gnu.so" / "libs" + libs.mkdir(parents=True) + gnu_path = libs / "libzen_internals_x86_64-unknown-linux-gnu.so" + gnu_path.write_text("not a library") + shutil.copy( + zen_internals._binary_path(), libs / gnu_path.name.replace("-gnu", "-musl") + ) + monkeypatch.setattr( + "aikido_zen.helpers.zen_internals._binary_path", lambda: str(gnu_path) + ) + + assert zen_internals._load() is not None + + def test_bind_returns_none_when_library_is_not_loaded(): assert zen_internals._bind(None, "detect_sql_injection", [], ctypes.c_int) is None -def test_bind_returns_none_when_function_is_missing(): +def test_bind_returns_none_when_function_is_missing(caplog): class LibraryWithoutFunctions: pass @@ -70,3 +92,109 @@ class LibraryWithoutFunctions: ) is None ) + assert "has no detect_sql_injection function" in caplog.text + + +def _warnings_at_import(cdll_patch): + script = ( + "import ctypes\n" f"{cdll_patch}\n" "import aikido_zen.helpers.zen_internals\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=True + ) + return result.stdout + result.stderr + + +def test_warns_once_per_disabled_feature_when_library_fails_to_load(): + output = _warnings_at_import( + "def fail(*args, **kwargs):\n" + " raise OSError('cannot open shared object file')\n" + "ctypes.CDLL = fail" + ) + + assert "Failed to load the zen-internals library" in output + assert "cannot open shared object file" in output + assert "Zen will NOT block SQL injection attacks." in output + + +def test_warns_about_the_missing_function_when_library_is_loaded(): + output = _warnings_at_import( + "real_cdll = ctypes.CDLL\n" + "class WithoutSql:\n" + " def __init__(self, lib):\n" + " self._lib = lib\n" + " def __getattr__(self, name):\n" + " if name == 'detect_sql_injection':\n" + " raise AttributeError(name)\n" + " return getattr(self._lib, name)\n" + "ctypes.CDLL = lambda path, *args, **kwargs: WithoutSql(real_cdll(path))" + ) + + assert "has no detect_sql_injection function" in output + assert "Zen will NOT block SQL injection attacks." in output + assert "Failed to load the zen-internals library" not in output + + +@pytest.mark.parametrize( + "system, machine, host, file_name", + [ + ("Darwin", "arm64", "", "libzen_internals_aarch64-apple-darwin.dylib"), + ("Darwin", "x86_64", "", "libzen_internals_x86_64-apple-darwin.dylib"), + ( + "Linux", + "aarch64", + "aarch64-unknown-linux-gnu", + "libzen_internals_aarch64-unknown-linux-gnu.so", + ), + ( + "Linux", + "x86_64", + "x86_64-pc-linux-gnu", + "libzen_internals_x86_64-unknown-linux-gnu.so", + ), + ( + "Linux", + "aarch64", + "aarch64-alpine-linux-musl", + "libzen_internals_aarch64-unknown-linux-musl.so", + ), + ( + "Linux", + "x86_64", + "x86_64-pc-linux-musl", + "libzen_internals_x86_64-unknown-linux-musl.so", + ), + ("Linux", "x86_64", None, "libzen_internals_x86_64-unknown-linux-gnu.so"), + ("Windows", "AMD64", None, "libzen_internals_x86_64-pc-windows-gnu.dll"), + ("Windows", "ARM64", None, "libzen_internals_aarch64-pc-windows-msvc.dll"), + ], +) +def test_binary_path_picks_the_build_for_the_platform( + monkeypatch, system, machine, host, file_name +): + monkeypatch.setattr("platform.system", lambda: system) + monkeypatch.setattr("platform.machine", lambda: machine) + monkeypatch.setattr("sysconfig.get_config_var", lambda name: host) + + path = zen_internals._binary_path() + + assert os.path.basename(path) == file_name + assert os.path.dirname(os.path.normpath(path)).endswith( + os.path.join("aikido_zen", "libs") + ) + + +def test_binary_path_ignores_what_the_system_reports_about_libc(monkeypatch): + monkeypatch.setattr("platform.system", lambda: "Linux") + monkeypatch.setattr("platform.machine", lambda: "x86_64") + monkeypatch.setattr("sysconfig.get_config_var", lambda name: "x86_64-pc-linux-gnu") + monkeypatch.setattr("platform.libc_ver", lambda: ("", "")) + monkeypatch.setattr("glob.glob", lambda pattern: ["/lib/ld-musl-x86_64.so.1"]) + + assert os.path.basename(zen_internals._binary_path()) == ( + "libzen_internals_x86_64-unknown-linux-gnu.so" + ) + + +def test_binary_path_exists_for_this_machine(): + assert os.path.exists(zen_internals._binary_path()) diff --git a/aikido_zen/vulnerabilities/sql_injection/init_test.py b/aikido_zen/vulnerabilities/sql_injection/init_test.py index d017bc442..e13ec4b3a 100644 --- a/aikido_zen/vulnerabilities/sql_injection/init_test.py +++ b/aikido_zen/vulnerabilities/sql_injection/init_test.py @@ -1,9 +1,11 @@ import os import pytest +from aikido_zen.helpers import zen_internals from aikido_zen.vulnerabilities.sql_injection import ( detect_sql_injection, should_return_early, ) +from .map_dialect_to_rust_int import map_dialect_to_rust_int """Removed tests : -> `I'm writting you` : Invalid SQL @@ -86,6 +88,17 @@ def is_not_sql_injection(sql, input, dialect="all"): ), f"Expected no SQL injection for SQL: {sql} and input: {input} in {current} dialect" +def is_tokenize_error(sql, input, dialect): + result = zen_internals.detect_sql_injection( + sql.lower().encode(), + input.lower().strip().encode(), + map_dialect_to_rust_int(dialect), + ) + assert ( + result == 3 + ), f"Expected tokenize error for SQL: {sql} and input: {input} in {dialect} dialect" + + def test_should_return_early(): # Test cases where the function should return True @@ -218,7 +231,7 @@ def test_check_string_safely_escaped(): ) # Positive example of same query : is_sql_injection( - "SELECT * FROM comments WHERE comment = 'I'm writting you--'", + "SELECT * FROM comments WHERE comment = 'I'm writting you-- '", "I'm writting you--", ) is_sql_injection( @@ -415,11 +428,29 @@ def test_function_calls_as_sql_injections(): def test_trimmed_user_input_bypass(): is_sql_injection( - "INSERT INTO pets (name, owner) VALUES ('x', 'dummy'), ('injected', 'hacker'); --', 'owner')", + "INSERT INTO pets (name, owner) VALUES ('x', 'dummy'), ('injected', 'hacker'); -- ', 'owner')", "x', 'dummy'), ('injected', 'hacker'); -- ", ) +def test_mysql_does_not_detect_dash_dash_without_space(): + for sql, input in [ + ( + "SELECT * FROM comments WHERE comment = 'I'm writting you--'", + "I'm writting you--", + ), + ( + "INSERT INTO pets (name, owner) VALUES ('x', 'dummy'), ('injected', 'hacker'); --', 'owner')", + "x', 'dummy'), ('injected', 'hacker'); -- ", + ), + ]: + is_tokenize_error(sql, input, "mysql") + is_not_sql_injection(sql, input, "mysql") + is_sql_injection(sql, input, "generic") + is_sql_injection(sql, input, "postgres") + is_sql_injection(sql, input, "sqlite") + + def test_returns_false_when_library_cannot_be_loaded(monkeypatch): monkeypatch.setattr("aikido_zen.helpers.zen_internals._detect_sql_injection", None)