disable os acces

.ci/compose/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export FLEDGE_ROOT=/usr/local/fledge
+export FLEDGE_DATA="${FLEDGE_DATA:-/usr/local/fledge/data}"
+
+# Fledge logs to syslog
+rm -f /var/run/rsyslogd.pid
+touch /var/log/syslog
+# Ubuntu's rsyslog drops privileges to the syslog user, Debian's stays root
+chown syslog:adm /var/log/syslog 2>/dev/null || chmod 666 /var/log/syslog
+rsyslogd
+
+"${FLEDGE_ROOT}/bin/fledge" start
+
+# Keep the container in the foreground and show the Fledge logs
+exec tail -F /var/log/syslog

.ci/deb/Dockerfile

@@ -0,0 +1,107 @@
+ARG BASE_IMAGE=ubuntu:22.04
+
+FROM ${BASE_IMAGE} AS deps
+ENV DEBIAN_FRONTEND=noninteractive \
+    TZ=UTC \
+    LANG=C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        sudo ca-certificates curl wget git unzip dos2unix \
+        build-essential cmake g++ make autoconf automake libtool pkg-config \
+        uuid-dev libssl-dev libz-dev libbz2-dev libreadline-dev liblzma-dev libffi-dev \
+        libboost-dev libboost-system-dev libboost-thread-dev \
+        libpq-dev libcurl4-openssl-dev \
+        python3 python3-dev python3-pip python3-setuptools python3-numpy python3-distutils-extra \
+        sqlite3 \
+        krb5-user \
+        avahi-daemon \
+        dpkg-dev
+
+FROM deps AS builder
+
+WORKDIR /build
+COPY . /build/plugin
+
+RUN find /build/plugin -type f -print0 \
+        | xargs -0 -r dos2unix -q 2>/dev/null || true
+
+# Fledge core, built from source. It is not part of this repository (there is no
+# submodule). If a checkout is already in ./fledge (as in CI) it is used, otherwise
+# it is cloned here. Override with --build-arg FLEDGE_REF=<branch|tag>
+ARG FLEDGE_REPO=https://github.com/fledge-iot/fledge.git
+ARG FLEDGE_REF=develop
+RUN if [ ! -f /build/plugin/fledge/requirements.sh ]; then \
+        rm -rf /build/plugin/fledge \
+        && git clone --quiet --depth 1 --branch "${FLEDGE_REF}" "${FLEDGE_REPO}" /build/plugin/fledge; \
+    fi \
+    && chmod +x /build/plugin/fledge/requirements.sh
+
+WORKDIR /build/plugin/fledge
+RUN bash ./requirements.sh
+RUN make -j"$(nproc)"
+
+WORKDIR /build/plugin
+RUN if [ -f requirements.sh ]; then bash ./requirements.sh; fi
+
+ENV FLEDGE_INSTALL=/usr/local/fledge
+ENV LD_LIBRARY_PATH=/usr/local/fledge/lib
+
+WORKDIR /build/plugin
+RUN mkdir -p build && cd build \
+    && cmake -DFLEDGE_SRC=/build/plugin/fledge -DFLEDGE_INSTALL="${FLEDGE_INSTALL}" .. \
+    && make -j"$(nproc)" \
+    && make install
+
+
+# Run the plugin unit tests (gtest) against the Fledge build above:
+#   docker build -f .ci/deb/Dockerfile --target test .
+FROM builder AS test
+
+RUN apt-get update && apt-get install -y --no-install-recommends libgtest-dev googletest \
+    && rm -rf /var/lib/apt/lists/*
+# Some distros ship the gtest sources only
+RUN if ! ls /usr/lib/*/libgtest.* /usr/lib/libgtest.* >/dev/null 2>&1; then \
+        cmake -S /usr/src/googletest -B /tmp/gtest-build \
+        && cmake --build /tmp/gtest-build -j"$(nproc)" \
+        && cp /tmp/gtest-build/lib/*.a /usr/lib/; \
+    fi
+
+ENV FLEDGE_DATA=/tmp
+ENV LD_LIBRARY_PATH=/build/plugin/fledge/cmake_build/C/lib:/usr/local/fledge/lib
+
+WORKDIR /build/plugin
+# Current gtest releases need C++14, the test CMakeLists pins C++11
+RUN sed -i 's/-std=c++11/-std=c++14/' tests/CMakeLists.txt \
+    && mkdir -p tests-build && cd tests-build \
+    && cmake -DFLEDGE_SRC=/build/plugin/fledge ../tests \
+    && make -j"$(nproc)"
+RUN cd tests-build && ./RunTests
+
+# Runnable Fledge (built from source) with this plugin installed, see
+# docker-compose.yml in the repository root.
+FROM builder AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends rsyslog curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && sed -i '/imklog/d' /etc/rsyslog.conf
+
+WORKDIR /build/plugin/fledge
+RUN make install
+
+# South sinusoid plugin, pinned to a tag and verified by commit
+ARG SINUSOID_TAG=v3.1.0
+ARG SINUSOID_COMMIT=4ff6eab5f21671fbcfd244e716572e699d0974da
+RUN git clone --quiet --depth 1 --branch "${SINUSOID_TAG}" \
+        https://github.com/fledge-iot/fledge-south-sinusoid.git /tmp/sinusoid \
+    && test "$(git -C /tmp/sinusoid rev-parse HEAD)" = "${SINUSOID_COMMIT}" \
+    && cp -R /tmp/sinusoid/python/fledge/plugins/south/sinusoid \
+        /usr/local/fledge/python/fledge/plugins/south/ \
+    && rm -rf /tmp/sinusoid
+
+# Taken from the builder stage, where line endings have been normalised
+RUN cp /build/plugin/.ci/compose/entrypoint.sh /usr/local/bin/entrypoint.sh \
+    && chmod +x /usr/local/bin/entrypoint.sh
+
+ENV FLEDGE_ROOT=/usr/local/fledge
+EXPOSE 8081 1995
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

.ci/deb/test.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build Fledge and the plugin in Docker and run the unit tests.
+# Usage: .ci/deb/test.sh [base image] [platform]
+set -euo pipefail
+
+BASE_IMAGE="${1:-ubuntu:24.04}"
+PLATFORM="${2:-linux/amd64}"
+
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+cd "$REPO_ROOT"
+
+docker build \
+    --file .ci/deb/Dockerfile \
+    --target test \
+    --platform "$PLATFORM" \
+    --build-arg BASE_IMAGE="$BASE_IMAGE" \
+    --progress=plain \
+    .

.dockerignore

@@ -0,0 +1,6 @@
+# Keep host build output out of the image
+fledge/cmake_build
+build
+build-out
+tests-build
+__pycache__

CMakeLists.txt

@@ -42,12 +42,12 @@ set(NEEDED_FLEDGE_LIBS common-lib plugins-common-lib filters-common-lib services
 # Find source files
 file(GLOB SOURCES *.cpp)
 
-# Find python3.x dev/lib package
+# Find python3.x dev/lib package, 3.8 or later
 find_package(PkgConfig REQUIRED)
 if(${CMAKE_VERSION} VERSION_LESS "3.12.0") 
-    pkg_check_modules(PYTHON REQUIRED python3)
+    pkg_check_modules(PYTHON REQUIRED python3>=3.8)
 else()
-    find_package(Python COMPONENTS Interpreter Development)
+    find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development)
 endif()
 
 # Find Fledge includes and libs, by including FindFledge.cmak file
@@ -107,7 +107,7 @@ add_library(${PROJECT_NAME} SHARED ${SOURCES} version.h)
 target_link_libraries(${PROJECT_NAME} ${NEEDED_FLEDGE_LIBS})
 
 # Add additional libraries
-# Add Python 3.5 library
+# Add Python library
 if(${CMAKE_VERSION} VERSION_LESS "3.12.0") 
     target_link_libraries(${PROJECT_NAME} ${PYTHON_LIBRARIES})
 else()

README.rst

@@ -2,11 +2,11 @@
 Fledge "Python35" C++ Filter plugin
 =========================================
 
-A readings data transformation plugin using Python 3.5
-embedded interpreter.
+A readings data transformation plugin using an embedded Python 3
+interpreter. The name is historic, Python 3.8 or later is required.
 
 NOTE:
-Python3.5 development package is required.
+Python 3.8 or later and its development package are required.
 
 Build
 -----
@@ -67,3 +67,46 @@ Examples:
   $ cmake -DFLEDGE_INSTALL=/home/source/develop/Fledge
 
   $ cmake -DFLEDGE_INSTALL=/usr/local/fledge
+
+Script restrictions
+-------------------
+
+Filter scripts (south and north, including those authored through the
+Configurator) run in a restricted environment (PT-HLN-HDP-26A-002, CWE-78).
+The script is checked when the filter starts and on every reconfigure, before
+any of its code runs:
+
+- an AST check rejects imports of anything outside the allow-list, relative
+  and star imports, dunder attribute access (``__class__``, ``__subclasses__``
+  ...), references to ``__import__``/``__builtins__``, and to ``eval``,
+  ``exec``, ``compile``, ``open``, ``getattr`` and similar builtins;
+- the script is then executed with restricted builtins and an ``__import__``
+  that enforces the same allow-list at run time.
+
+A rejected script is logged with the reason and line, the filter stays failed
+and drops readings (fail closed).
+
+Default allow-list: ``json math cmath time datetime re copy collections
+itertools functools string decimal fractions statistics numbers random bisect
+heapq struct base64 binascii hashlib hmac enum typing textwrap unicodedata``.
+Never importable: ``os subprocess socket sys ctypes importlib shutil io
+threading multiprocessing pickle urllib http logging.config`` and others, see
+``include/script_sandbox.h``. Scripts that used ``logging``, ``os`` or ``sys``
+(for example the old ``examples/scale35.py``) must be changed.
+
+**Decision: the allow-list is configurable per deployment, by the operator
+only.** The environment variable ``FLEDGE_PYTHON35_EXTRA_IMPORTS``
+(comma separated exact module names) adds modules to the default list. It is
+read from the Fledge service environment, not from the filter configuration
+category, because that category is editable by the same users who write the
+script. The deny-list can not be overridden. Adding modules widens the attack
+surface, review each one for ways to reach the OS (``logging`` can write
+files, for instance).
+
+This is defence in depth, not a complete sandbox: in-process Python
+restrictions are known to be bypassable by a determined author. Keep running
+Fledge with least privilege and restrict who can edit filter scripts. Checks
+at save time in the Portal/Configurator are a courtesy only; the enforcement
+is here.
+
+Tests: the ``PYTHON35.Sandbox*`` cases in ``tests/test_python35.cpp``.

docker-compose.yml

@@ -0,0 +1,50 @@
+# Fledge built from source (cloned during the image build, see FLEDGE_REF in the Dockerfile) with this plugin installed.
+#
+#   docker compose up --build    start Fledge (REST API on :8081)
+#   docker compose down -v       stop and remove the data
+#   docker compose --profile test run --build --rm test
+#                                build and run the unit tests (see the test service)
+#
+# The first build compiles all of Fledge and takes a long time.
+# The south sinusoid plugin is installed, add it with:
+#   curl -X POST localhost:8081/fledge/service -d '{"name":"sin","type":"south","plugin":"sinusoid","enabled":true}'
+# The example scripts are mounted read-only at /examples.
+services:
+  fledge:
+    image: fledge-python35:dev
+    build:
+      context: .
+      dockerfile: .ci/deb/Dockerfile
+      target: runtime
+      args:
+        BASE_IMAGE: ${BASE_IMAGE:-ubuntu:24.04}
+    ports:
+      - "8081:8081"
+      - "1995:1995"
+    volumes:
+      - fledge-data:/usr/local/fledge/data
+      - ./examples:/examples:ro
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:8081/fledge/ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 20s
+
+  # Unit tests. Building the image already runs them (the Dockerfile test stage
+  # fails the build if a test fails); running the service runs them again, e.g.
+  #   docker compose --profile test run --rm test --gtest_filter='*Sandbox*'
+  test:
+    image: fledge-python35:test
+    profiles: ["test"]
+    build:
+      context: .
+      dockerfile: .ci/deb/Dockerfile
+      target: test
+      args:
+        BASE_IMAGE: ${BASE_IMAGE:-ubuntu:24.04}
+    working_dir: /build/plugin/tests-build
+    entrypoint: ["./RunTests"]
+
+volumes:
+  fledge-data:

examples/scale35.py

@@ -10,22 +10,18 @@
 __license__ = "Apache 2.0"
 __version__ = "${VERSION}"
 
-import sys
 import json
-import logging
-from logging.handlers import SysLogHandler
-import os
-
-# Log to syslog
-handler = SysLogHandler(address='/dev/log')
-formatter = logging.Formatter(fmt='Fledge[%(process)d] %(levelname)s: %(name)s: %(message)s')
-handler.setFormatter(formatter)
-# When embedded Python is in use sys.argv is not defined,
-logger = logging.getLogger(os.path.basename(__file__))
-
-# Set mininum log severity
-logger.setLevel(logging.DEBUG)
-logger.addHandler(handler)
+
+# Filter scripts run in a restricted environment: os, sys, subprocess, socket,
+# logging and similar modules can not be imported, so diagnostics go to stdout.
+
+
+class _Logger(object):
+    def debug(self, msg):
+        print("DEBUG: " + msg)
+
+
+logger = _Logger()
 
 """
 Filter configuration set by set_filter_config(config)

include/python35.h

@@ -40,6 +40,7 @@ class Python35Filter : public FledgeFilter
 		{
 			m_pModule = NULL;
 			m_pFunc = NULL;
+			m_sandbox = NULL;
 			m_init = false;
 			m_encode_names = true;
 			m_logger = Logger::getLogger();
@@ -64,6 +65,7 @@ class Python35Filter : public FledgeFilter
 		void	lock() { m_configMutex.lock(); };
 		void	unlock() { m_configMutex.unlock(); };
 		void	logErrorMessage();
+		PyObject*	loadScript(const std::string& scriptName);
 		// Filtering methods for Reading objects
 		PyObject*
 			createReadingsList(const std::vector<Reading *>& readings);
@@ -75,6 +77,8 @@ class Python35Filter : public FledgeFilter
 		PyObject*	m_pModule;
 		// Python 3.5 callable method handle
 		PyObject*	m_pFunc;
+		// Restricted script loader function
+		PyObject*	m_sandbox;
 		// Python 3.5  script name
 		std::string	m_pythonScript;
 		// Python interpreter has been started by this plugin