diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml
index fa5e8e0..673437c 100644
--- a/.github/workflows/plugin_test.yaml
+++ b/.github/workflows/plugin_test.yaml
@@ -38,6 +38,20 @@ jobs:
           update-environment: false
       - name: Check out code
         uses: actions/checkout@v5
+      - name: Determine Please version
+        if: ${{ inputs.platform != 'freebsd_amd64' }}
+        id: please_version
+        run: echo "version=$(./pleasew --version | awk '{print $NF}')" >> "$GITHUB_OUTPUT"
+      - name: Cache Please downloads
+        # The FreeBSD tests run inside a nested VM, so the host's cache directory isn't visible to them.
+        if: ${{ inputs.platform != 'freebsd_amd64' }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/please
+          key: please-${{ inputs.platform }}-${{ steps.please_version.outputs.version }}-${{ hashFiles('plugins/BUILD', 'third_party/**/BUILD') }}
+          restore-keys: |
+            please-${{ inputs.platform }}-${{ steps.please_version.outputs.version }}-
+            please-${{ inputs.platform }}-
       - name: Configure plugin to use please_pex tool built from source
         if: inputs.please_pex_from_repo
         # PexTool can't be set to //tools/please_pex for e2e tests because it can't be built inside the e2e test
@@ -58,18 +72,18 @@ jobs:
             _external_interpreter="python${{ inputs.python }}"
             _in_repo_interpreter="//third_party/cc/cpython:cpython_${{ inputs.python }}|python"
             echo "*** Running tests using external Python interpreter: $_external_interpreter"
-            ./pleasew -o "plugin.python.defaultinterpreter:$_external_interpreter" test --rerun --keep_going --log_file plz-out/log/test-external_interpreter.log ${{ inputs.test_targets }}
+            ./pleasew -o "plugin.python.defaultinterpreter:$_external_interpreter" test --profile ci --keep_going --log_file plz-out/log/test-external_interpreter.log ${{ inputs.test_targets }}
             echo "*** Running tests using in-repo Python interpreter: $_in_repo_interpreter"
-            ./pleasew -o "plugin.python.defaultinterpreter:$_in_repo_interpreter" test --rerun --keep_going --log_file plz-out/log/test-in_repo_interpreter.log ${{ inputs.test_targets }}
+            ./pleasew -o "plugin.python.defaultinterpreter:$_in_repo_interpreter" test --profile ci --keep_going --log_file plz-out/log/test-in_repo_interpreter.log ${{ inputs.test_targets }}
       - name: Run tests (on host)
         if: ${{ inputs.platform != 'freebsd_amd64' }}
         run: |
           _external_interpreter="${{ steps.python.outputs.python-path }}"
           _in_repo_interpreter="//third_party/cc/cpython:cpython_${{ inputs.python }}|python"
           echo "*** Running tests using external Python interpreter: $_external_interpreter"
-          ./pleasew -o "plugin.python.defaultinterpreter:$_external_interpreter" test --rerun --keep_going --log_file plz-out/log/test-external_interpreter.log ${{ inputs.test_targets }}
+          ./pleasew -o "plugin.python.defaultinterpreter:$_external_interpreter" test --profile ci --keep_going --log_file plz-out/log/test-external_interpreter.log ${{ inputs.test_targets }}
           echo "*** Running tests using in-repo Python interpreter: $_in_repo_interpreter"
-          ./pleasew -o "plugin.python.defaultinterpreter:$_in_repo_interpreter" test --rerun --keep_going --log_file plz-out/log/test-in_repo_interpreter.log ${{ inputs.test_targets }}
+          ./pleasew -o "plugin.python.defaultinterpreter:$_in_repo_interpreter" test --profile ci --keep_going --log_file plz-out/log/test-in_repo_interpreter.log ${{ inputs.test_targets }}
       - name: Archive logs
         uses: actions/upload-artifact@v7
         with:
diff --git a/.plzconfig.ci b/.plzconfig.ci
new file mode 100644
index 0000000..0646402
--- /dev/null
+++ b/.plzconfig.ci
@@ -0,0 +1,3 @@
+[cache]
+dircompress = true
+dirclean = false