Run the SQL Server integration tests on Testcontainers, with a CI job

.fallout/build.schema.json

@@ -33,6 +33,7 @@
         "Docs",
         "DocsReference",
         "DocsSdk",
+        "IntegrationTest",
         "Pack",
         "PackBinaries",
         "Publish",

.github/workflows/ci.yml

@@ -37,3 +37,23 @@ jobs:
       - name: Run './build.cmd ci'
         run: ./build.sh ci
 
+  # The SQL Server backed integration tests. Each test assembly starts its own SQL Server container with Testcontainers,
+  # which needs only the Docker daemon that ubuntu-latest provides.
+  integration-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/setup-dotnet@v6
+        with:
+          dotnet-version: 10.0
+      - uses: actions/checkout@v7
+      - name: Run './build.sh Compile IntegrationTest'
+        run: ./build.sh Compile IntegrationTest
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: integration-test-results
+          path: artifacts/TestResults
+          if-no-files-found: ignore
+

AGENTS.md

@@ -36,7 +36,7 @@ dotnet test --project test/Spring/Spring.Core.Tests/Spring.Core.Tests.csproj -f
 
 Fallout's `Test` target excludes `*.Integration.Tests` projects (and `Spring.Web.Conversation.NHibernate5.Tests`) by default; they need external infrastructure and are gated behind flags: `--test-full`, `--test-integration-data`, `--test-integration-nms`, `--test-integration-ms-mq`. Several test projects are Windows-only (net472: Web, Mvc5, Messaging/MSMQ).
 
-- SQL Server-backed tests (`Spring.Data.Integration.Tests`, `Spring.Data.NHibernate5.Integration.Tests`, `Spring.Scheduling.Quartz3.Integration.Tests`): `docker-compose.yml` brings up SQL Server; setup scripts in `build-support/*.sql` and the test projects' `Data/` folders. See README.md "Running tests" for the full recipe.
+- SQL Server-backed tests (`Spring.Data.Integration.Tests`, `Spring.Data.NHibernate5.Integration.Tests`, `Spring.Data.NHibernate5.NestedTxSuspension.Integration.Tests`, `Spring.Scheduling.Quartz3.Integration.Tests`) provision their own database with Testcontainers: a running Docker daemon is all they need. `./build.cmd IntegrationTest` runs them (CI: the `integration-linux` job); `--test-integration-data` adds them to `Test`. Each assembly's `[SetUpFixture]` calls `test/Shared/SqlServerTestEnvironment.cs` (a linked source file), which starts a pinned SQL Server container, runs `build-support/create-integration-test-databases-and-users.sql` and the project's `*_database.sql` scripts, and publishes `SPRING_TEST_SQLSERVER_*` environment variables that the XML contexts read as `${...}` placeholders. Setting `SPRING_TEST_SQLSERVER` to an admin connection string uses that server instead of a container. Tests that need MSDTC are `[Explicit]`. See README.md "Running tests".
 
 ## Architecture
 

Directory.Packages.props

@@ -40,6 +40,7 @@
     <PackageVersion Include="System.Runtime.Serialization.Formatters" Version="10.0.14" />
     <PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.14" />
     <PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.14" />
+    <PackageVersion Include="Testcontainers.MsSql" Version="4.16.0" />
     <PackageVersion Include="log4net" Version="3.5.0" />
   </ItemGroup>
 </Project>

README.md

@@ -158,17 +158,50 @@ suffice.
 
 ### Running tests
 
-- To run tests that require a database connection, you need:
-    - SqlServer running in (local)
-      - Check in `SQL Server Configuration Manager` that `Named Pipes` and `TCP/IP` protocols are enabled
-    - From `build-support` execute 
-	    - `create-integration-test-databases-and-users.sql` 
-    - From `\test\Spring\Spring.Data.Integration.Tests\Data` execute 
-	    - `Spring.Data.Integration.Tests_Spring_database.sql`
-	    - `Spring.Data.Integration.Tests_Credits_database.sql`
-	    - `Spring.Data.Integration.Tests_Debits_database.sql`
-	    - `Spring.Data.Integration.Tests_CreditsAndDebits_database.sql`
-    - Create a database named `quartz` and execute `tables_sqlServer_Below2016.sql` from [Quartz.NET](https://github.com/quartznet/quartznet/blob/main/database/tables/tables_sqlServer_Below2016.sql)
+- `./build.cmd Test` (`./build.sh Test`) runs the unit tests. They need nothing but the .NET SDK.
+- The SQL Server backed integration tests (`Spring.Data.Integration.Tests`, `Spring.Data.NHibernate5.Integration.Tests`,
+  `Spring.Data.NHibernate5.NestedTxSuspension.Integration.Tests` and `Spring.Scheduling.Quartz3.Integration.Tests`) provision their
+  own database with [Testcontainers for .NET](https://dotnet.testcontainers.org/). All you need is a running Docker daemon
+  (Linux containers):
+
+  ```
+  ./build.cmd IntegrationTest
+  ```
+
+  Each test assembly starts a pinned SQL Server 2022 container, creates the `springqa` login and the databases from
+  `build-support/create-integration-test-databases-and-users.sql` and the project's own `*_database.sql` script (the `quartz`
+  database gets the Quartz.NET tables), runs, and removes the container. There is no database to install, no hosts entry and no
+  script to run by hand. CI runs the same target on Linux (the `integration-linux` job).
+- To run against a SQL Server you already have, for example on a Windows machine without a Linux container engine, set
+  `SPRING_TEST_SQLSERVER` to an administrator's connection string. No container is started and the same scripts run against that
+  server (they drop and recreate the test databases and the `springqa` login, so do not point it at a server you care about).
+  The server must allow SQL Server (mixed-mode) authentication, because the tests log in as `springqa`. In PowerShell:
+
+  ```
+  $env:SPRING_TEST_SQLSERVER = "Server=localhost;Integrated Security=true;TrustServerCertificate=true"
+  ./build.cmd IntegrationTest
+  ```
+
+  `./build.cmd IntegrationTest` runs the net10.0 and net472 builds of a project one after the other when the variable is set.
+  When you call `dotnet test` yourself, pass `--max-parallel-test-modules 1`, because Microsoft.Testing.Platform otherwise runs
+  both builds at once and they would drop and recreate the same databases under each other:
+
+  ```
+  dotnet test --project test/Spring/Spring.Data.Integration.Tests/Spring.Data.Integration.Tests.csproj --max-parallel-test-modules 1
+  ```
+- The tests learn where the server is from these environment variables, which the test assembly sets once the scripts have run
+  (the XML contexts use them as `${...}` placeholders):
+
+  | Variable | Value |
+  |---|---|
+  | `SPRING_TEST_SQLSERVER_HOST` | `host,port` of the server |
+  | `SPRING_TEST_SQLSERVER_USER`, `SPRING_TEST_SQLSERVER_PASSWORD` | the `springqa` login (`springqa` / `springqa`) |
+  | `SPRING_TEST_SQLSERVER_CS_<DATABASE>` | a complete connection string, as `springqa`, for each database the scripts created, with the name upper-cased: `..._CS_SPRING`, `..._CS_CREDITS`, `..._CS_DEBITS`, `..._CS_CREDITSANDDEBITS`, `..._CS_QUARTZ` |
+
+- Tests that need a distributed transaction (MSDTC) are `[Explicit]`: a SQL Server container cannot take part in one. Run them
+  by name against a Windows server with MSDTC enabled if you work on that code.
+- `./build.cmd Test --test-integration-data` also runs the four SQL Server projects as part of the `Test` target, and
+  `--test-full` runs every integration project, including the ones that need ActiveMQ, MSMQ or a SQLite file.
 
 
 ## Acknowledgements

build-support/build/Build.Test.cs

@@ -1,8 +1,10 @@
+using System.Collections.Generic;
 using System.Linq;
 using Fallout.Common;
 using Fallout.Common.IO;
 using Fallout.Common.Tooling;
 using Fallout.Common.Tools.DotNet;
+using Fallout.Solutions;
 using static Fallout.Common.Tools.DotNet.DotNetTasks;
 
 public partial class Build
@@ -14,6 +16,24 @@ public partial class Build
 
     AbsolutePath TestResultsDirectory => ArtifactsDirectory / "TestResults";
 
+    // The integration test projects that provision their own SQL Server with Testcontainers (test/Shared/SqlServerTestEnvironment.cs).
+    // They need a running Docker daemon, or a server named by the SPRING_TEST_SQLSERVER environment variable, and nothing else.
+    static readonly string[] SqlServerIntegrationTestProjects =
+    [
+        "Spring.Data.Integration.Tests",
+        "Spring.Data.NHibernate5.Integration.Tests",
+        "Spring.Data.NHibernate5.NestedTxSuspension.Integration.Tests",
+        "Spring.Scheduling.Quartz3.Integration.Tests",
+    ];
+
+    // Projects that target net472 only (no net10.0 build to run on Linux).
+    static readonly string[] WindowsOnlyIntegrationTestProjects =
+    [
+        "Spring.Data.NHibernate5.NestedTxSuspension.Integration.Tests",
+    ];
+
+    static bool IsSqlServerIntegrationTestProject(Project project) => SqlServerIntegrationTestProjects.Contains(project.Name);
+
     Target Test => _ => _
         .DependsOn(Restore)
         .After(Compile)
@@ -23,45 +43,77 @@ public partial class Build
                 // The test projects only: Spring.Testing.NUnit and Spring.Testing.Microsoft are shipped libraries that
                 // merely contain ".Test", and a Microsoft.Testing.Platform run of a project without tests is an error.
                 .Where(x => x.Name.EndsWith(".Tests"))
-                .Where(x => !x.Name.Contains(".Integration") || TestFull)
+                // Integration tests need infrastructure, so they run only when asked for: --test-full for all of them, or one
+                // flag for the projects behind it. (The four SQL Server projects also have the IntegrationTest target.)
+                .Where(x => !x.Name.Contains(".Integration")
+                            || TestFull
+                            || (TestIntegrationData && IsSqlServerIntegrationTestProject(x))
+                            || (TestIntegrationNms && x.Name == "Spring.Messaging.Nms.Integration.Tests"))
                 .Where(x => !x.Name.Contains("Spring.Web.Conversation.NHibernate5.Tests") || TestFull)
-                .Where(x => !x.Name.Contains("Spring.Data.Integration.Tests") || TestIntegrationData)
                 .Where(x => !x.Name.Contains("Spring.Messaging.Tests") || TestIntegrationMsMq || TestFull)
-                .Where(x => !x.Name.Contains("Spring.Nms.Integration.Tests") || TestIntegrationNms || TestFull)
                 .Where(x => !x.Name.Contains("Spring.Testing.Microsoft.Test") || EnvironmentInfo.IsWin)
                 .Where(x => !x.Name.Contains("Spring.Web.Tests") || EnvironmentInfo.IsWin)
-                .Where(x => !x.Name.Contains("Spring.Web.Mvc5.Tests") || EnvironmentInfo.IsWin);
+                .Where(x => !x.Name.Contains("Spring.Web.Mvc5.Tests") || EnvironmentInfo.IsWin)
+                .Where(x => !WindowsOnlyIntegrationTestProjects.Contains(x.Name) || EnvironmentInfo.IsWin);
+
+            RunTestProjects(testTargets);
+        });
+
+    // Runs the four SQL Server backed integration test projects. Each starts its own SQL Server container, so a Docker
+    // daemon must be running (or SPRING_TEST_SQLSERVER must name a server). Unlike CI-only targets this one is meant to be
+    // run locally: ./build.cmd IntegrationTest
+    Target IntegrationTest => _ => _
+        .DependsOn(Restore)
+        .After(Compile)
+        .Executes(() =>
+        {
+            var testTargets = GetActiveProjects()
+                .Where(IsSqlServerIntegrationTestProject)
+                .Where(x => !WindowsOnlyIntegrationTestProjects.Contains(x.Name) || EnvironmentInfo.IsWin);
+
+            RunTestProjects(testTargets);
+        });
+
+    void RunTestProjects(IEnumerable<Project> testTargets)
+    {
+        var failedProjects = new List<string>();
+
+        // Microsoft.Testing.Platform runs the modules of a multi-targeted project (net10.0 and net472) at the same time. Against
+        // a server named by SPRING_TEST_SQLSERVER they would drop and recreate the same databases under each other, so they run
+        // one after the other then. With a container per module (the default) there is nothing to share.
+        var maxParallelModules = string.IsNullOrWhiteSpace(System.Environment.GetEnvironmentVariable("SPRING_TEST_SQLSERVER"))
+            ? System.Environment.ProcessorCount
+            : 1;
 
-            var failedProjects = new System.Collections.Generic.List<string>();
-            foreach (var project in testTargets)
+        foreach (var project in testTargets)
+        {
+            // The repo runs tests on Microsoft.Testing.Platform (global.json), where the project goes in --project
+            // rather than as a positional argument, which DotNetTest's settings cannot express. Each run also
+            // writes a TRX file under artifacts/TestResults/<project>, so a red CI log has something to open.
+            // The interpolated string has to go to DotNet() as is: its handler quotes each {hole}, a pre-built string
+            // would be passed as a single argument.
+            var results = TestResultsDirectory / project.Name;
+            try
             {
-                // The repo runs tests on Microsoft.Testing.Platform (global.json), where the project goes in --project
-                // rather than as a positional argument, which DotNetTest's settings cannot express. Each run also
-                // writes a TRX file under artifacts/TestResults/<project>, so a red CI log has something to open.
-                // The interpolated string has to go to DotNet() as is: its handler quotes each {hole}, a pre-built string
-                // would be passed as a single argument.
-                var results = TestResultsDirectory / project.Name;
-                try
+                if (EnvironmentInfo.IsWin)
                 {
-                    if (EnvironmentInfo.IsWin)
-                    {
-                        DotNet($"test --project {project.Path} --configuration {Configuration} --no-restore --no-progress --report-trx --results-directory {results}");
-                    }
-                    else
-                    {
-                        DotNet($"test --project {project.Path} --configuration {Configuration} --no-restore --no-progress --report-trx --results-directory {results} --framework net10.0");
-                    }
+                    DotNet($"test --project {project.Path} --configuration {Configuration} --no-restore --no-progress --report-trx --results-directory {results} --max-parallel-test-modules {maxParallelModules}");
                 }
-                catch (ProcessException)
+                else
                 {
-                    // Keep going so one red project does not hide the state of the others; the target still fails below.
-                    failedProjects.Add(project.Name);
+                    DotNet($"test --project {project.Path} --configuration {Configuration} --no-restore --no-progress --report-trx --results-directory {results} --framework net10.0");
                 }
             }
-
-            if (failedProjects.Count > 0)
+            catch (ProcessException)
             {
-                throw new System.Exception("Test projects failed: " + string.Join(", ", failedProjects));
+                // Keep going so one red project does not hide the state of the others; the target still fails below.
+                failedProjects.Add(project.Name);
             }
-        });
+        }
+
+        if (failedProjects.Count > 0)
+        {
+            throw new System.Exception("Test projects failed: " + string.Join(", ", failedProjects));
+        }
+    }
 }