fix(KDP): repair core defects, enable the linter and move to the Keras 3 API

.github/templates/Python/mkdocs/action.yml

@@ -11,7 +11,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "${{ inputs.PYTHON_VERSION }}"
         cache: "pip"

.github/templates/Python/poetry_build/action.yml

@@ -29,7 +29,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "${{ inputs.PYTHON_VERSION }}"
 

.github/templates/Python/unittests/action.yml

@@ -14,9 +14,9 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "${{ inputs.PYTHON_VERSION }}"
     - name: Install dependencies

.github/workflows/MANUAL_DOCS_PUBLISH.yml

@@ -21,13 +21,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
           cache: 'pip'

.github/workflows/MANUAL_PYPI_PUBLISH.yml

@@ -16,13 +16,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
           cache: 'pip'

.github/workflows/PRECOMMITS.yml

@@ -21,30 +21,44 @@ jobs:
   pre-commit-checks:
     runs-on: ["ubuntu-latest"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: Check Pull Request Title
-        uses: Slashgear/action-check-pr-title@main
+
+      # Kept in step with the local commit-msg hook in .pre-commit-config.yaml.
+      # Checking it here rather than through a third-party action keeps the two
+      # definitions in one repository and removes an unpinned dependency.
+      - name: Check pull request title
+        if: github.event_name == 'pull_request'
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+        run: |
+          PATTERN='^(break|build|ci|docs|feat|fix|perf|refactor|style|test|ops|hotfix|release|maint|init|enh|revert)\([^)]+\)!?: .+'
+          if ! printf '%s' "$PR_TITLE" | grep -Eq "$PATTERN"; then
+            echo "::error::Pull request title does not follow the conventional-commit format."
+            echo "Got:      $PR_TITLE"
+            echo "Expected: type(scope): description"
+            echo "Types:    break, build, ci, docs, feat, fix, perf, refactor, style,"
+            echo "          test, ops, hotfix, release, maint, init, enh, revert"
+            exit 1
+          fi
+          echo "Pull request title is valid."
+
+      - name: Cache pre-commit environments
+        uses: actions/cache@v4
         with:
-          regexp: '(break|build|ci|docs|feat|fix|perf|refactor|style|test|ops|hotfix)\([a-z,A-Z,0-9,\-,\_,\/,:]+\)(:)\s{1}([\w\s]+)' # Regex the title should match.
-      - name: Getting changed files list
-        id: files
-        uses: jitterbit/get-changed-files@master
-      - name: Checking chaged files
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      # `--all-files` rather than a changed-files list: the repository is clean
+      # under every hook, so this is both simpler and stricter, and it drops the
+      # unpinned third-party action the changed-files plumbing needed.
+      - name: Pre-commit checks
         shell: bash
-        run: |
-          echo "Changed files"
-          echo ${{ steps.files.outputs.all }}
-          echo "Github Client version"
-          echo $(gh --version)
-      - name: Pre-Commit Checks
         run: |
           python -m pip install --upgrade pip
-          pip install pre-commit==3.3.3
-          echo "Running pre-commit scans:"
-          # adding log display in case of pre-commit errors
-          pre-commit run -v --files ${{ steps.files.outputs.all }}
-        shell: bash
+          pip install "pre-commit>=3.6,<4"
+          pre-commit run --all-files --show-diff-on-failure

.github/workflows/PR_PREVIEW.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ["ubuntu-latest"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: "pip"

.github/workflows/RELEASE.yml

@@ -32,7 +32,7 @@ jobs:
       release_created: ${{ steps.semantic-release.outputs.new_release_published }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -68,11 +68,13 @@ jobs:
 
   PYPI_PUBLISH:
     needs: [SEMANTIC_RELEASE]
-    if: ${{ needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE }}
+    # `!cancelled()` is required so this still runs when SKIP_RELEASE skips
+    # SEMANTIC_RELEASE: a skipped dependency otherwise skips this job too.
+    if: ${{ !cancelled() && (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE) }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           fetch-depth: 1
@@ -100,7 +102,7 @@ jobs:
           fi
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
           cache: 'pip'
@@ -143,11 +145,15 @@ jobs:
 
   UPDATE_DOCS:
     needs: [SEMANTIC_RELEASE, PYPI_PUBLISH]
-    if: ${{ (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' && !failure('PYPI_PUBLISH')) || inputs.SKIP_RELEASE }}
+    # A job's result is read from `needs.<job>.result`; `failure()` takes no
+    # arguments, so `failure('PYPI_PUBLISH')` never tested what it looked like.
+    if: >-
+      ${{ !cancelled() && needs.PYPI_PUBLISH.result != 'failure'
+      && (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE) }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           fetch-depth: 1
@@ -175,7 +181,7 @@ jobs:
           fi
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
           cache: 'pip'

.github/workflows/UTESTS.yml

@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - name: Install dependencies
@@ -82,7 +82,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -133,6 +133,33 @@ jobs:
             htmlcov/
           retention-days: 7
 
+  # The package targets the Keras 3 API directly rather than the tf.keras shim,
+  # so it must keep working with Keras releases newer than the one TensorFlow
+  # happens to pin. This job catches a Keras 3 change before users do.
+  keras-latest:
+    runs-on: ubuntu-latest
+    needs: [smoke-test]
+    if: always() && (needs.smoke-test.result == 'success' || github.event_name != 'pull_request')
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies with the newest Keras 3
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry install --no-interaction
+          poetry run pip install --upgrade "keras>=3,<4"
+          poetry run python -c "import keras; print('keras', keras.__version__)"
+
+      - name: Run tests
+        run: poetry run pytest -c pytest-fast.ini --timeout=180
+        timeout-minutes: 15
+
   # Coverage job (only on main Python version)
   coverage:
     runs-on: ubuntu-latest
@@ -141,7 +168,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -178,7 +205,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 

Makefile

@@ -3,7 +3,12 @@
 # ------------------------------------
 
 HAS_POETRY := $(shell command -v poetry 2> /dev/null)
-POETRY_VERSION := $(shell poetry version $(shell git describe --tags --abbrev=0))
+
+# Sets the package version from the latest git tag. This MUST stay a lazily
+# expanded recipe command: as a `:=` shell assignment it ran `poetry version
+# <tag>` while make was still parsing the file, so every single make target --
+# `make help` included -- silently rewrote the version in pyproject.toml.
+SET_VERSION_FROM_TAG = poetry version $$(git describe --tags --abbrev=0)
 
 # ------------------------------------
 # Test
@@ -101,10 +106,10 @@ unittests: test
 .PHONY: clean_tests
 ## Remove pytest cache and test artifacts
 clean_tests:
-	find . -type d -name .pytest_cache -exec rm -r {} +
-	find . -type d -name __pycache__ -exec rm -r {} +
-	find . -type f -name '*junit_report.xml' -exec rm {} +
-	find . -type f -name '*.pyc' -exec rm {} +
+	find . -type d -name .pytest_cache -prune -exec rm -rf {} + 2>/dev/null || true
+	find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
+	find . -type f -name '*junit_report.xml' -delete
+	find . -type f -name '*.pyc' -delete
 	rm -rf htmlcov/
 	rm -f .coverage
 	rm -f coverage.xml
@@ -125,7 +130,7 @@ coverage:
 build_pkg:
 	@echo "Start to build pkg"
 ifdef HAS_POETRY
-	@$(POETRY_VERSION)
+	@$(SET_VERSION_FROM_TAG)
 	poetry build
 else
 	@echo "To build the package, you need to have poetry first"
@@ -139,8 +144,9 @@ build: clean_built build_pkg
 .PHONY: clean_built
 ## Remove cache, built package, and docs directories after build or installation
 clean_built:
-	find . -type d -name dist -exec rm -r {} +
-	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+	rm -rf dist
+	find . -type f -name '*.py[co]' -delete
+	find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
 
 # ------------------------------------
 # Build doc
@@ -172,7 +178,7 @@ docs_deploy: generate_all_diagrams
 	@echo "Starting to build docs"
 	@echo "more info: https://squidfunk.github.io/mkdocs-material/setup/setting-up-versioning/"
 ifdef HAS_POETRY
-	@$(POETRY_VERSION)
+	@$(SET_VERSION_FROM_TAG)
 	poetry version -s | xargs -I {} sh -c 'echo Deploying version {} && mike deploy --push --update-aliases {} latest'
 else
 	@echo "To build the docs, you need to have poetry first"
@@ -223,10 +229,8 @@ clean_old_diagrams: identify_unused_diagrams
 
 .PHONY: clean
 ## Remove cache, built package, and docs directories after build or installation
-clean: clean_old_diagrams
-	find . -type d -name dist -exec rm -r {} +
-	find . -type f -name '*.rst' ! -name 'index.rst' -delete
-	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+clean: clean_tests clean_built clean_old_diagrams
+	@echo "Workspace cleaned"
 
 # ------------------------------------
 # Default