apache · lenamonj · Sep 6, 2026
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
@@ -45,6 +45,7 @@ The <action> type attribute can be add,update,fix,remove.
   <body>
     <release version="1.23.0" date="YYYY-MM-DD" description="This is a feature and maintenance release. Java 8 or later is required.">
       <!-- FIX -->
+      <action type="fix" dev="ggregory" due-to="Jeff Lenamon">GitIdentifiers orders tree entries by unsigned UTF-8 bytes, as Git does, instead of UTF-16 code units.</action>
       <action type="fix" dev="ggregory" due-to="Yu Bao, Gary Gregory">Optimize PhoneticEngine.encode(String, LanguageSet) for speed.</action>
       <action type="fix" dev="ggregory" due-to="Yu Bao, Gary Gregory">RFC1522Codec.decodeText(String) now throws a DecoderException instead of a StringIndexOutOfBoundsException when a separator is missing.</action>
       <action type="fix" dev="ggregory" due-to="Yu Bao, Gary Gregory">Optimize Base58.convertFromBase58(byte[], Context) for speed and temporary object allocation.</action>

diff --git a/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java b/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java
@@ -57,8 +57,9 @@ public class GitIdentifiers {
      *   <li>the raw object id of the referenced blob or sub-tree.</li>
      * </ul>
      *
-     * <p>Entries are ordered by {@link #compareTo} using Git's tree-sort rule: directory names are compared as if they ended with {@code '/'}, so that {@code foo/}
-     * sorts after {@code foobar}.</p>
+     * <p>Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if
+     * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} is what keeps
+     * the order Git's for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order.</p>
      *
      * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
      * @see <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID Directory Identifier</a>
@@ -76,11 +77,11 @@ static class DirectoryEntry implements Comparable<DirectoryEntry> {
         private final byte[] rawObjectId;
 
         /**
-         * The key used for ordering entries within a tree object.
+         * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares.
          *
-         * <p>>Git appends {@code '/'} to directory names before comparing.</p>
+         * <p>Git appends {@code '/'} to directory names before comparing.</p>
          */
-        private final String sortKey;
+        private final byte[] sortKey;
 
         /**
          * The Git object type, which determines the Unix file-mode prefix.
@@ -100,13 +101,22 @@ static class DirectoryEntry implements Comparable<DirectoryEntry> {
             }
             this.name = name;
             this.type = Objects.requireNonNull(type, "type");
-            this.sortKey = type == FileMode.DIRECTORY ? name + "/" : name;
+            this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8);
             this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId");
         }
 
         @Override
         public int compareTo(final DirectoryEntry o) {
-            return sortKey.compareTo(o.sortKey);
+            final byte[] a = sortKey;
+            final byte[] b = o.sortKey;
+            final int shared = Math.min(a.length, b.length);
+            for (int i = 0; i < shared; i++) {
+                final int diff = (a[i] & 0xff) - (b[i] & 0xff);
+                if (diff != 0) {
+                    return diff;
+                }
+            }
+            return a.length - b.length;
         }
 
         @Override

diff --git a/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java b/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java
@@ -24,6 +24,7 @@
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -221,6 +222,36 @@ void testDirectoryEntryEqualityBasedOnNameOnly() {
         assertFalse(regular.equals("foo"));
     }
 
+    /**
+     * Tree entry names are ordered by their UTF-8 bytes, which is not the order {@link String#compareTo(String)} gives for names outside the Basic Multilingual
+     * Plane: U+FF21 encodes to {@code EF BC A1} and U+1F600 to {@code F0 9F 98 80}, so Git sorts U+FF21 first, while the UTF-16 code units place the surrogate
+     * pair of U+1F600 first.
+     *
+     * <p>The expected identifier is the one {@code git write-tree} produces for a tree holding the same two entries.</p>
+     */
+    @Test
+    void testTreeIdSortsSupplementaryPlaneNamesLikeGit(@TempDir final Path tempDir) throws Exception {
+        final String fullWidthA = "\uFF21";
+        final String grinningFace = "\uD83D\uDE00";
+        final byte[] content = "x".getBytes(StandardCharsets.UTF_8);
+        final String expected = "9f9c1fc3580195f51d3e71b384ef1d57740e2151";
+        final MessageDigest md = DigestUtils.getSha1Digest();
+
+        // Entries are added in the wrong order on purpose, so only the sort decides the result.
+        final GitIdentifiers.TreeIdBuilder builder = GitIdentifiers.treeIdBuilder(md);
+        builder.addFile(GitIdentifiers.FileMode.REGULAR, grinningFace, content);
+        builder.addFile(GitIdentifiers.FileMode.REGULAR, fullWidthA, content);
+        assertEquals(expected, Hex.encodeHexString(builder.get()));
+
+        try {
+            Files.write(tempDir.resolve(fullWidthA), content);
+            Files.write(tempDir.resolve(grinningFace), content);
+        } catch (final IOException e) {
+            Assumptions.abort("Filesystem cannot hold the test entry names: " + e);
+        }
+        assertEquals(expected, Hex.encodeHexString(GitIdentifiers.treeId(md, tempDir)));
+    }
+
     /**
      * Entries should be sorted by Git sort rule.
      *