[#72] Add find-refs test coverage and fix its database/null-ref bugs

.gitignore

@@ -32,12 +32,13 @@
 
 .claude
 
-UnityFileSystemTestData/AssetBundles/
-UnityFileSystemTestData/**/*.csproj
-UnityFileSystemTestData/**/*.sln
-UnityFileSystemTestData/ProjectSettings/
-UnityFileSystemTestData/UserSettings/
-UnityFileSystemTestData/Packages/
+UnityProjects/**/*.csproj
+UnityProjects/**/*.sln
+UnityProjects/**/AssetBundles/
+UnityProjects/**/ProjectSettings/
+UnityProjects/**/UserSettings/
+UnityProjects/**/Packages/
+UnityProjects/**/Builds/
 *.db
 *.csv
 

AGENTS.md

@@ -174,4 +174,6 @@ To use a specific Unity version's library:
 
 * TestCommon/Data contains small reference files extracted from Unity builds (player and AssetBundles). These are used by the automated tests and also useful for manual testing.
 
-* UnityFileSystemTestData is a Unity project that generates test data for the test suites.
+* UnityProjects contains two Unity projects that generate test data for the test suites:
+  * `Baseline` - tracks stable, broadly-used Unity versions and is upgraded only when necessary. Most UnityDataTools users inspect output from these versions. Its `TypeIdRegistryGenerator` regenerates `UnityFileSystem/TypeIdRegistry.cs`.
+  * `LeadingEdge` - tracks the newest Unity version (currently the 6.6 beta) and is updated proactively so newer build features (Content Directory builds, serialized dictionaries, etc.) can be tested.

CLAUDE.md

@@ -1 +1 @@
-AGENTS.md
+@AGENTS.md

Documentation/command-find-refs.md

@@ -1,9 +1,9 @@
 # find-refs Command
 
-> ⚠️ **Experimental:** This command may not work as expected in all cases.
-
 The `find-refs` command traces reference chains leading to specific objects. Use it to understand why an asset was included (and potentially duplicated) in a build.
 
+It walks *up* the reference graph from the target object and **stops at the first asset it reaches**. The reported chains therefore end at the immediate containing asset, not at the ultimate root that transitively depends on the target. For example, if `RootAsset` references `LeafAsset` which references a texture, searching for the texture reports the chain ending at `LeafAsset`; to see that `RootAsset` pulls it in, search for `LeafAsset` instead.
+
 ## Quick Reference
 
 ```
@@ -16,10 +16,13 @@ UnityDataTool find-refs <database> [options]
 | `-i, --object-id <id>` | ID of object to analyze (from `id` column) | — |
 | `-n, --object-name <name>` | Name of objects to analyze | — |
 | `-t, --object-type <type>` | Type filter when using `-n` | — |
-| `-o, --output-file <file>` | Output filename | — |
+| `-o, --output-file <file>` | Output filename | `references.txt` |
+| `--stdout` | Write the reference chains to stdout instead of a file | `false` |
 | `-a, --find-all` | Find all chains instead of stopping at first | `false` |
 
 > **Note:** Either `--object-id` or `--object-name` must be provided.
+>
+> `--stdout` and `-o/--output-file` are mutually exclusive.
 
 ## Prerequisites
 
@@ -29,14 +32,19 @@ This command requires a database created by the [`analyze`](command-analyze.md)
 
 ## Examples
 
-Find references to an object by name and type:
+Find references to an object by name and type, printing directly to the console:
+```bash
+UnityDataTool find-refs my_database.db -n "MyTexture" -t "Texture2D" --stdout
+```
+
+Write the reference chains to a file instead:
 ```bash
 UnityDataTool find-refs my_database.db -n "MyTexture" -t "Texture2D" -o refs.txt
 ```
 
 Find references to a specific object by ID:
 ```bash
-UnityDataTool find-refs my_database.db -i 12345 -o refs.txt
+UnityDataTool find-refs my_database.db -i 12345 --stdout
 ```
 
 Find all duplicate references (useful for finding why an asset is duplicated):

README.md

@@ -25,7 +25,7 @@ The repository contains the following items:
 * UnityFileSystem: source code and binaries of a .NET class library exposing the functionalities or the
   UnityFileSystemApi native library.
 * UnityFileSystem.Tests: test suite for the UnityFileSystem library.
-* UnityFileSystemTestData: the Unity project used to generate the test data.
+* UnityProjects: Unity projects used to generate some of the test data.
 * TestCommon: a helper library used by the test projects.
 
 ## Downloads

ReferenceFinder/ReferenceFinderTool.cs

@@ -24,21 +24,14 @@ public class ReferenceFinderTool
     List<ReferenceTreeNode> m_Roots = new List<ReferenceTreeNode>();
     HashSet<(long, string)> m_ProcessedObjects = new HashSet<(long, string)>();
 
-    StreamWriter m_Writer;
+    TextWriter m_Writer;
 
-    public int FindReferences(string objectName, string objectType, string databasePath, string outputFile, bool findAll)
+    public int FindReferences(string objectName, string objectType, string databasePath, string outputFile, bool findAll, bool toStdout = false)
     {
         var objectIds = new List<long>();
-        SqliteConnection db;
-
-        try
-        {
-            db = new SqliteConnection($"Data Source={databasePath};Version=3;Foreign Keys=False;");
-            db.Open();
-        }
-        catch (Exception e)
+        using var db = OpenDatabase(databasePath);
+        if (db == null)
         {
-            Console.WriteLine($"Error opening database: {e.Message}");
             return 1;
         }
 
@@ -81,33 +74,50 @@ public int FindReferences(string objectName, string objectType, string databaseP
             return 1;
         }
 
-        return FindReferences(db, outputFile, objectIds, findAll);
+        return FindReferences(db, outputFile, objectIds, findAll, toStdout);
     }
 
-    public int FindReferences(long objectId, string databasePath, string outputFile, bool findAll)
+    public int FindReferences(long objectId, string databasePath, string outputFile, bool findAll, bool toStdout = false)
     {
         var objectIds = new List<long>();
-        SqliteConnection db;
+        using var db = OpenDatabase(databasePath);
+        if (db == null)
+        {
+            return 1;
+        }
 
+        objectIds.Add(objectId);
+
+        return FindReferences(db, outputFile, objectIds, findAll, toStdout);
+    }
+
+    // Opens the analyze database for reading. Uses SqliteConnectionStringBuilder (matching SQLiteWriter) rather than a
+    // hand-written connection string, which used a legacy System.Data.SQLite keyword that Microsoft.Data.Sqlite rejects.
+    static SqliteConnection OpenDatabase(string databasePath)
+    {
         try
         {
-            db = new SqliteConnection($"Data Source={databasePath};Version=3;Foreign Keys=False;");
+            var connectionString = new SqliteConnectionStringBuilder
+            {
+                DataSource = databasePath,
+                Mode = SqliteOpenMode.ReadOnly,
+                Pooling = false,
+                ForeignKeys = false,
+            }.ConnectionString;
+            var db = new SqliteConnection(connectionString);
             db.Open();
+            return db;
         }
         catch (Exception e)
         {
             Console.WriteLine($"Error opening database: {e.Message}");
-            return 1;
+            return null;
         }
-
-        objectIds.Add(objectId);
-
-        return FindReferences(db, outputFile, objectIds, findAll);
     }
 
-    int FindReferences(SqliteConnection db, string outputFile, IList<long> objectIds, bool findAll)
+    int FindReferences(SqliteConnection db, string outputFile, IList<long> objectIds, bool findAll, bool toStdout)
     {
-        m_Writer = new StreamWriter(outputFile);
+        m_Writer = toStdout ? Console.Out : new StreamWriter(outputFile);
 
         m_GetRefsCommand = db.CreateCommand();
         m_GetRefsCommand.CommandText = @"SELECT object, property_path, EXISTS (SELECT * FROM assets a WHERE a.object = r.object) FROM refs r WHERE referenced_object = @id";
@@ -181,7 +191,11 @@ FROM object_view o
             }
         }
 
-        m_Writer.Close();
+        // Don't close Console.Out when writing to stdout; just flush it.
+        if (toStdout)
+            m_Writer.Flush();
+        else
+            m_Writer.Close();
 
         return 0;
     }
@@ -196,10 +210,13 @@ void OutputReferenceNode(ReferenceTreeNode node, string propertyPath, int indent
         {
             reader.Read();
 
+            // game_object and script come from correlated subqueries that yield NULL when there is no matching row
+            // (e.g. a ScriptableObject is a MonoBehaviour whose m_GameObject PPtr is 0, or a MonoBehaviour with no
+            // m_Script reference), so both must be null-checked.
             var objectType = reader.GetString(0);
             var objectName = reader.GetString(1);
-            var gameObject = reader.GetString(2);
-            var script = reader.GetString(3);
+            var gameObject = reader.IsDBNull(2) ? "" : reader.GetString(2);
+            var script = reader.IsDBNull(3) ? "" : reader.GetString(3);
 
             if (propertyPath != "")
             {

TestCommon/Data/AssetBundles/AGENTS.md

@@ -0,0 +1,5 @@
+# AssetBundle test data
+
+AssetBundle build output used by the UnityDataTools test suites, kept in a subfolder per Unity version (`2019.4.0f1` through `2023.1.0a16`). Each holds the `assetbundle` and `scenes` bundles built from the project's assets; some versions also include additional fixtures used by specific tests.
+
+Generated by the `UnityProjects/Baseline` project - run **Tools > Generate AssetBundles** there (built for StandaloneOSX and copied here under the Unity version). See that project's `AGENTS.md` for the assets that go into the bundles.

TestCommon/Data/AssetBundles/CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

TestCommon/Data/LeadingEdgeBuilds/AGENTS.md

@@ -0,0 +1,12 @@
+# LeadingEdge reference builds
+
+The checked-in build output of the `UnityProjects/LeadingEdge` project, for use in automated and ad hoc tests. See that project's `AGENTS.md` for the test scenarios and how the assets are set up.
+
+The LeadingEdge build scripts regenerate this folder directly, so to update it, rebuild in that project and check in the results.
+
+## Layout
+
+* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle.
+* `ContentDirectory/` - the Content Directory build: content (`.cf`) files, `.resource` files and the build manifest.
+* `BuildReport-AssetBundles/LastBuild.buildreport` - the AssetBundle build report.
+* `BuildReport-ContentDirectory/` - the Content Directory build report folder, including `ContentLayout.json`.

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/6.manifest

@@ -0,0 +1,21 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 4272264984
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 256fc0686cc70288ca0c8f6d88bc4cfc
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 9a2ca7bdbd1871f7131daf57de908e0c
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: e4d5b2148b8da35f0919e0024f8e9095
+HashAppended: 0
+ClassTypes:
+- Class: 83
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers: []
+Assets:
+- Assets/Audio/6.mp3
+Dependencies: []

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/AssetBundles.manifest

@@ -0,0 +1,30 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 3235539447
+HashAppended: 0
+AssetBundleManifest:
+  AssetBundleInfos:
+    Info_0:
+      Name: assetbundleroot
+      Dependencies:
+        Dependency_0: directaudioclipreference
+        Dependency_1: singleaudioclipdirectreference
+        Dependency_2: serializationdemo
+    Info_1:
+      Name: directaudioclipreference
+      Dependencies:
+        Dependency_0: 6
+        Dependency_1: a
+    Info_2:
+      Name: singleaudioclipdirectreference
+      Dependencies:
+        Dependency_0: a
+    Info_3:
+      Name: serializationdemo
+      Dependencies: {}
+    Info_4:
+      Name: 6
+      Dependencies: {}
+    Info_5:
+      Name: a
+      Dependencies: {}

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/a.manifest

@@ -0,0 +1,21 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 3543150657
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 27584a7667a63eb913bc1cc20ff4c416
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 9a2ca7bdbd1871f7131daf57de908e0c
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: 8e2f9ab0bd28d271a63e0ad1c392e317
+HashAppended: 0
+ClassTypes:
+- Class: 83
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers: []
+Assets:
+- Assets/Audio/a.mp3
+Dependencies: []

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/assetbundleroot.manifest

@@ -0,0 +1,34 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 1569140202
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: d7f919acab03f8314047c9954b2f35d3
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: e4c92e7ce30487f41e62ec475c399971
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: 1dbce57dbf052f5659ad2cc7eb1ce772
+HashAppended: 0
+ClassTypes:
+- Class: 114
+  Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3}
+- Class: 114
+  Script: {fileID: 11500000, guid: 8623c5efbb626994da80931050f0aba0, type: 3}
+- Class: 114
+  Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3}
+- Class: 115
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers:
+- AssemblyName: Assembly-CSharp
+  ClassName: SerializationDemo/SerializedData
+- AssemblyName: UnityEngine.CoreModule
+  ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2
+Assets:
+- Assets/ScriptableObjects/AssetBundleRoot.asset
+Dependencies:
+- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundles/directaudioclipreference
+- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundles/serializationdemo
+- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundles/singleaudioclipdirectreference

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/directaudioclipreference.manifest

@@ -0,0 +1,29 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 2898859186
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 81b1dfeb6b75ecfa0e6d2ac8a2ace1a1
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 4c88c857f41d6968a1090f557f31c5ed
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: 550b678d70753dc54c9731a6aac37c3b
+HashAppended: 0
+ClassTypes:
+- Class: 83
+  Script: {instanceID: 0}
+- Class: 114
+  Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3}
+- Class: 115
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers:
+- AssemblyName: UnityEngine.CoreModule
+  ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2
+Assets:
+- Assets/ScriptableObjects/DirectAudioClipReference.asset
+Dependencies:
+- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundles/6
+- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundles/a

TestCommon/Data/LeadingEdgeBuilds/AssetBundles/serializationdemo.manifest

@@ -0,0 +1,25 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.6.0b3
+CRC: 1465243436
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 126f385e23d41aca13d7233b6051d3a1
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: d0726b8e1199eb6125e2245ce15343f9
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: 6b339242357a8a3b47aa14706dd55d35
+HashAppended: 0
+ClassTypes:
+- Class: 114
+  Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3}
+- Class: 115
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers:
+- AssemblyName: Assembly-CSharp
+  ClassName: SerializationDemo/SerializedData
+Assets:
+- Assets/ScriptableObjects/SerializationDemo.asset
+Dependencies: []