diff --git a/src/changes/changes.xml b/src/changes/changes.xml index a2e3824deb..805725fa4f 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -45,6 +45,7 @@ The type attribute can be add,update,fix,remove. + GitIdentifiers orders tree entries by unsigned UTF-8 bytes, as Git does, instead of UTF-16 code units. Optimize PhoneticEngine.encode(String, LanguageSet) for speed. RFC1522Codec.decodeText(String) now throws a DecoderException instead of a StringIndexOutOfBoundsException when a separator is missing. Optimize Base58.convertFromBase58(byte[], Context) for speed and temporary object allocation. diff --git a/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java b/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java index 0f7a3015e3..959eb35334 100644 --- a/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java +++ b/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java @@ -57,8 +57,9 @@ public class GitIdentifiers { *
  • the raw object id of the referenced blob or sub-tree.
    • * * - *

    Entries are ordered by {@link #compareTo} using Git's tree-sort rule: directory names are compared as if they ended with {@code '/'}, so that {@code foo/} - * sorts after {@code foobar}.

    + *

    Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if + * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} is what keeps + * the order Git's for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order.

    * * @see Git Internals – Git Objects * @see SWHID Directory Identifier @@ -76,11 +77,11 @@ static class DirectoryEntry implements Comparable { private final byte[] rawObjectId; /** - * The key used for ordering entries within a tree object. + * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares. * - *

    >Git appends {@code '/'} to directory names before comparing.

    + *

    Git appends {@code '/'} to directory names before comparing.

    */ - private final String sortKey; + private final byte[] sortKey; /** * The Git object type, which determines the Unix file-mode prefix. @@ -100,13 +101,22 @@ static class DirectoryEntry implements Comparable { } this.name = name; this.type = Objects.requireNonNull(type, "type"); - this.sortKey = type == FileMode.DIRECTORY ? name + "/" : name; + this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8); this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId"); } @Override public int compareTo(final DirectoryEntry o) { - return sortKey.compareTo(o.sortKey); + final byte[] a = sortKey; + final byte[] b = o.sortKey; + final int shared = Math.min(a.length, b.length); + for (int i = 0; i < shared; i++) { + final int diff = (a[i] & 0xff) - (b[i] & 0xff); + if (diff != 0) { + return diff; + } + } + return a.length - b.length; } @Override diff --git a/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java b/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java index e455173d82..257a8873d3 100644 --- a/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java +++ b/src/test/java/org/apache/commons/codec/digest/GitIdentifiersTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -221,6 +222,36 @@ void testDirectoryEntryEqualityBasedOnNameOnly() { assertFalse(regular.equals("foo")); } + /** + * Tree entry names are ordered by their UTF-8 bytes, which is not the order {@link String#compareTo(String)} gives for names outside the Basic Multilingual + * Plane: U+FF21 encodes to {@code EF BC A1} and U+1F600 to {@code F0 9F 98 80}, so Git sorts U+FF21 first, while the UTF-16 code units place the surrogate + * pair of U+1F600 first. + * + *

    The expected identifier is the one {@code git write-tree} produces for a tree holding the same two entries.

    + */ + @Test + void testTreeIdSortsSupplementaryPlaneNamesLikeGit(@TempDir final Path tempDir) throws Exception { + final String fullWidthA = "\uFF21"; + final String grinningFace = "\uD83D\uDE00"; + final byte[] content = "x".getBytes(StandardCharsets.UTF_8); + final String expected = "9f9c1fc3580195f51d3e71b384ef1d57740e2151"; + final MessageDigest md = DigestUtils.getSha1Digest(); + + // Entries are added in the wrong order on purpose, so only the sort decides the result. + final GitIdentifiers.TreeIdBuilder builder = GitIdentifiers.treeIdBuilder(md); + builder.addFile(GitIdentifiers.FileMode.REGULAR, grinningFace, content); + builder.addFile(GitIdentifiers.FileMode.REGULAR, fullWidthA, content); + assertEquals(expected, Hex.encodeHexString(builder.get())); + + try { + Files.write(tempDir.resolve(fullWidthA), content); + Files.write(tempDir.resolve(grinningFace), content); + } catch (final IOException e) { + Assumptions.abort("Filesystem cannot hold the test entry names: " + e); + } + assertEquals(expected, Hex.encodeHexString(GitIdentifiers.treeId(md, tempDir))); + } + /** * Entries should be sorted by Git sort rule. *