From 6fbdb0ec14b201fabbc0efd22ecfb57b4d9368ad Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:13 +0200 Subject: [PATCH 01/14] feat(iwork): unpack the snappy framing of an apple `.iwa` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The framing an iWork package uses is Apple's own — a four-byte header per block, `0x00` and a little-endian 24-bit compressed length — so stock Snappy stream decoding does not apply and only the block decoder does. That is a varint length plus literal and copy tags, which is less code than a dependency would be and keeps it out of the wasm, android and apple builds. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 2 + src/odr/internal/iwork/iwork_snappy.cpp | 130 ++++++++++++++++++ src/odr/internal/iwork/iwork_snappy.hpp | 18 +++ test/CMakeLists.txt | 2 + test/src/internal/iwork/iwork_snappy_test.cpp | 122 ++++++++++++++++ 5 files changed, 274 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_snappy.cpp create mode 100644 src/odr/internal/iwork/iwork_snappy.hpp create mode 100644 test/src/internal/iwork/iwork_snappy_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 04feb219..f58ca415 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,8 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_snappy.cpp" + "src/odr/internal/json/json_file.cpp" "src/odr/internal/json/json_util.cpp" diff --git a/src/odr/internal/iwork/iwork_snappy.cpp b/src/odr/internal/iwork/iwork_snappy.cpp new file mode 100644 index 00000000..63a8f931 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include + +namespace odr::internal { + +namespace { + +/// Reads @p size little-endian bytes as an unsigned integer. +std::uint32_t read_little_endian(const std::string_view in, + const std::size_t position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: snappy block ends mid-tag"); + } + + std::uint32_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + return result; +} + +/// Reads the block's uncompressed length and advances @p position past it. +std::uint32_t read_uncompressed_length(const std::string_view in, + std::size_t &position) { + std::uint32_t result = 0; + for (std::uint32_t shift = 0; shift <= 28; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error( + "iwork: snappy length varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: snappy length varint does not terminate"); +} + +} // namespace + +std::string iwork::snappy_decompress_block(const std::string_view compressed) { + std::size_t position = 0; + const std::uint32_t uncompressed_length = + read_uncompressed_length(compressed, position); + + std::string result; + result.reserve(uncompressed_length); + + while (position < compressed.size()) { + const auto tag = static_cast(compressed[position++]); + + if ((tag & 0x03) == 0) { + // literal: the length is in the tag, or in the bytes following it + std::size_t length = tag >> 2; + if (length >= 60) { + const std::size_t length_size = length - 59; + length = read_little_endian(compressed, position, length_size); + position += length_size; + } + ++length; + + if (position + length > compressed.size()) { + throw std::runtime_error("iwork: snappy literal runs past the block"); + } + result.append(compressed, position, length); + position += length; + continue; + } + + // copy: a length and a back reference into what has been written already + std::size_t length = 0; + std::size_t offset = 0; + if ((tag & 0x03) == 1) { + length = 4 + ((tag >> 2) & 0x07); + offset = (static_cast(tag >> 5) << 8) | + read_little_endian(compressed, position, 1); + position += 1; + } else { + const std::size_t offset_size = (tag & 0x03) == 2 ? 2 : 4; + length = (tag >> 2) + 1; + offset = read_little_endian(compressed, position, offset_size); + position += offset_size; + } + + if (offset == 0 || offset > result.size()) { + throw std::runtime_error("iwork: snappy copy points outside the block"); + } + // the copy may overlap what it writes, so it runs byte by byte + for (std::size_t i = 0, from = result.size() - offset; i < length; ++i) { + result.push_back(result[from + i]); + } + } + + if (result.size() != uncompressed_length) { + throw std::runtime_error("iwork: snappy block does not fill its length"); + } + return result; +} + +std::string iwork::iwa_decompress(const std::string_view framed) { + std::string result; + + std::size_t position = 0; + while (position < framed.size()) { + if (position + 4 > framed.size()) { + throw std::runtime_error("iwork: iwa block header is cut off"); + } + if (framed[position] != '\0') { + throw std::runtime_error("iwork: iwa block header is not zero"); + } + const std::uint32_t length = read_little_endian(framed, position + 1, 3); + position += 4; + + if (position + length > framed.size()) { + throw std::runtime_error("iwork: iwa block runs past the file"); + } + result += snappy_decompress_block(framed.substr(position, length)); + position += length; + } + + return result; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_snappy.hpp b/src/odr/internal/iwork/iwork_snappy.hpp new file mode 100644 index 00000000..158d97a5 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace odr::internal::iwork { + +/// Decompresses one Snappy block — a varint uncompressed length followed by +/// literal and copy tags. The stream framing Snappy ships with (the `sNaPpY` +/// identifier, per-chunk CRC-32C) is not involved, see @ref iwa_decompress. +std::string snappy_decompress_block(std::string_view compressed); + +/// Undoes the framing of an `.iwa`: `0x00`, a little-endian 24-bit compressed +/// length, then that many bytes of a Snappy block, repeated to the end. +/// Verified on `empty.pages Index/Document.iwa +0`. +std::string iwa_decompress(std::string_view framed); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b516952..0cb5c128 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,8 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_snappy_test.cpp" + "src/internal/odf/odf_table_test.cpp" "src/internal/oldms/doc_test.cpp" diff --git a/test/src/internal/iwork/iwork_snappy_test.cpp b/test/src/internal/iwork/iwork_snappy_test.cpp new file mode 100644 index 00000000..1ceaf495 --- /dev/null +++ b/test/src/internal/iwork/iwork_snappy_test.cpp @@ -0,0 +1,122 @@ +#include + +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +/// A Snappy block: the uncompressed length as a varint, then @p body. +std::string block(const std::size_t uncompressed_length, + const std::string &body) { + std::string result; + for (std::size_t rest = uncompressed_length;;) { + const auto byte = static_cast(rest & 0x7f); + rest >>= 7; + result.push_back(rest == 0 ? byte : static_cast(byte | 0x80)); + if (rest == 0) { + break; + } + } + return result + body; +} + +/// A literal tag for @p text, in the form that carries the length inline. +std::string literal(const std::string &text) { + return std::string(1, static_cast((text.size() - 1) << 2)) + text; +} + +/// One `.iwa` block header plus @p body. +std::string framed(const std::string &body) { + const std::size_t length = body.size(); + const std::string header{'\0', static_cast(length & 0xff), + static_cast((length >> 8) & 0xff), + static_cast((length >> 16) & 0xff)}; + return header + body; +} + +} // namespace + +TEST(SnappyDecompressBlock, literal) { + EXPECT_EQ(snappy_decompress_block(block(5, literal("hello"))), "hello"); +} + +TEST(SnappyDecompressBlock, empty) { + EXPECT_EQ(snappy_decompress_block(block(0, "")), ""); +} + +// A literal of 61 bytes or more names its length in the bytes after the tag. +TEST(SnappyDecompressBlock, long_literal) { + const std::string text(300, 'x'); + const std::string body = std::string{'\xf4', '\x2b', '\x01'} + text; + EXPECT_EQ(snappy_decompress_block(block(text.size(), body)), text); +} + +// Copy tag 1: a three-bit length and a ten-bit offset. +TEST(SnappyDecompressBlock, copy_with_one_byte_offset) { + const std::string body = literal("abc") + std::string{'\x09', '\x03'}; + EXPECT_EQ(snappy_decompress_block(block(9, body)), "abcabcabc"); +} + +// Copy tag 2: a six-bit length and a two-byte offset. +TEST(SnappyDecompressBlock, copy_with_two_byte_offset) { + const std::string body = + literal("abcd") + std::string{'\x0e', '\x04', '\x00'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abcdabcd"); +} + +// The run a copy reads may be the one it is writing. +TEST(SnappyDecompressBlock, overlapping_copy) { + const std::string body = literal("ab") + std::string{'\x09', '\x02'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abababab"); +} + +TEST(SnappyDecompressBlock, length_does_not_match) { + EXPECT_ANY_THROW(std::ignore = + snappy_decompress_block(block(6, literal("hello")))); +} + +TEST(SnappyDecompressBlock, literal_runs_past_the_block) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block( + block(5, std::string{'\x10'} + "hel"))); +} + +TEST(SnappyDecompressBlock, copy_points_outside_the_block) { + const std::string body = literal("abc") + std::string{'\x09', '\x09'}; + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(block(9, body))); +} + +TEST(SnappyDecompressBlock, length_varint_does_not_terminate) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block("\x80\x80\x80")); +} + +TEST(IwaDecompress, one_block) { + EXPECT_EQ(iwa_decompress(framed(block(5, literal("hello")))), "hello"); +} + +// A file is as many blocks as it takes; they concatenate. +TEST(IwaDecompress, two_blocks) { + const std::string data = + framed(block(5, literal("hello"))) + framed(block(6, literal(" world"))); + EXPECT_EQ(iwa_decompress(data), "hello world"); +} + +TEST(IwaDecompress, empty_file) { EXPECT_EQ(iwa_decompress(""), ""); } + +TEST(IwaDecompress, header_is_not_zero) { + std::string data = framed(block(5, literal("hello"))); + data[0] = '\x01'; + EXPECT_ANY_THROW(std::ignore = iwa_decompress(data)); +} + +TEST(IwaDecompress, truncated_mid_block) { + const std::string data = framed(block(5, literal("hello"))); + EXPECT_ANY_THROW(std::ignore = + iwa_decompress(data.substr(0, data.size() - 2))); +} + +TEST(IwaDecompress, truncated_header) { + EXPECT_ANY_THROW(std::ignore = iwa_decompress(std::string{'\0', '\x07'})); +} From afa207928e7a18ce4a16e796ed8e5586c232580e Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:23 +0200 Subject: [PATCH 02/14] feat(iwork): read the protobuf wire format An iWork archive is protobuf, but Apple has never published the `.proto` schemas, so there is nothing for a code generator to generate and linking conan `protobuf` would drag it into every downstream build to replace this. Only the wire format is needed: varints, the three fixed and length-delimited forms, and unknown fields carried along rather than dropped. A group means the parse went wrong, so it throws. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 1 + src/odr/internal/iwork/iwork_protobuf.cpp | 129 +++++++++++++++++ src/odr/internal/iwork/iwork_protobuf.hpp | 61 ++++++++ test/CMakeLists.txt | 1 + .../internal/iwork/iwork_protobuf_test.cpp | 134 ++++++++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_protobuf.cpp create mode 100644 src/odr/internal/iwork/iwork_protobuf.hpp create mode 100644 test/src/internal/iwork/iwork_protobuf_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f58ca415..6fa2cd08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,7 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" "src/odr/internal/json/json_file.cpp" diff --git a/src/odr/internal/iwork/iwork_protobuf.cpp b/src/odr/internal/iwork/iwork_protobuf.cpp new file mode 100644 index 00000000..df203b33 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.cpp @@ -0,0 +1,129 @@ +#include + +#include + +namespace odr::internal { + +namespace { + +std::uint64_t read_fixed(const std::string_view in, std::size_t &position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: protobuf fixed field is cut off"); + } + + std::uint64_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + position += size; + return result; +} + +} // namespace + +std::uint64_t iwork::read_varint(const std::string_view in, + std::size_t &position) { + std::uint64_t result = 0; + for (std::uint32_t shift = 0; shift <= 63; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error("iwork: protobuf varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: protobuf varint does not terminate"); +} + +iwork::Message::Message(const std::string_view bytes) { + std::size_t position = 0; + + while (position < bytes.size()) { + const std::uint64_t key = read_varint(bytes, position); + const auto wire_type = static_cast(key & 0x07); + const auto number = static_cast(key >> 3); + if (number == 0) { + throw std::runtime_error("iwork: protobuf field number zero"); + } + + Field field; + field.number = number; + field.type = wire_type; + + switch (wire_type) { + case WireType::varint: + field.number_value = read_varint(bytes, position); + break; + case WireType::fixed64: + field.number_value = read_fixed(bytes, position, 8); + break; + case WireType::fixed32: + field.number_value = read_fixed(bytes, position, 4); + break; + case WireType::length_delimited: { + const std::uint64_t length = read_varint(bytes, position); + if (length > bytes.size() - position) { + throw std::runtime_error("iwork: protobuf field runs past the message"); + } + field.bytes = bytes.substr(position, length); + position += length; + } break; + case WireType::start_group: + case WireType::end_group: + throw std::runtime_error("iwork: protobuf group field"); + } + + m_fields.push_back(field); + } +} + +const std::vector &iwork::Message::fields() const noexcept { + return m_fields; +} + +std::optional +iwork::Message::field(const std::uint32_t number) const { + std::optional result; + for (const Field &field : m_fields) { + if (field.number == number) { + result = field; + } + } + return result; +} + +std::vector +iwork::Message::repeated_field(const std::uint32_t number) const { + std::vector result; + for (const Field &field : m_fields) { + if (field.number == number) { + result.push_back(field); + } + } + return result; +} + +std::optional +iwork::Message::number_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type == WireType::length_delimited) { + return {}; + } + return field->number_value; +} + +std::optional +iwork::Message::bytes_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type != WireType::length_delimited) { + return {}; + } + return field->bytes; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_protobuf.hpp b/src/odr/internal/iwork/iwork_protobuf.hpp new file mode 100644 index 00000000..85b75a82 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include + +namespace odr::internal::iwork { + +/// The protobuf wire types. Groups (3 and 4) are deprecated and never appear +/// in an iWork archive, so reading one is a parse error rather than a field to +/// skip. +enum class WireType : std::uint8_t { + varint = 0, + fixed64 = 1, + length_delimited = 2, + start_group = 3, + end_group = 4, + fixed32 = 5, +}; + +/// One field of a protobuf message. @ref number_value carries a varint or a +/// fixed-width field, @ref bytes a length-delimited one — a nested message, a +/// string or a packed repeated field. +struct Field final { + std::uint32_t number{}; + WireType type{WireType::varint}; + std::uint64_t number_value{}; + std::string_view bytes; +}; + +/// A protobuf message read by field number: there are no schemas to generate +/// accessors from, so the archives are read against hand-written ones. +/// +/// Nested messages stay as views into the buffer the message was read from, +/// which has to outlive it. +class Message final { +public: + explicit Message(std::string_view bytes); + + [[nodiscard]] const std::vector &fields() const noexcept; + + /// The last field numbered @p number, which is what protobuf makes of a + /// non-repeated field appearing more than once. + [[nodiscard]] std::optional field(std::uint32_t number) const; + [[nodiscard]] std::vector repeated_field(std::uint32_t number) const; + + [[nodiscard]] std::optional + number_field(std::uint32_t number) const; + [[nodiscard]] std::optional + bytes_field(std::uint32_t number) const; + +private: + std::vector m_fields; +}; + +/// Reads a varint at @p position and advances it past the field. Throws when +/// the varint does not terminate within ten bytes. +std::uint64_t read_varint(std::string_view in, std::size_t &position); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0cb5c128..0161fc4b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" "src/internal/odf/odf_table_test.cpp" diff --git a/test/src/internal/iwork/iwork_protobuf_test.cpp b/test/src/internal/iwork/iwork_protobuf_test.cpp new file mode 100644 index 00000000..6cc44acc --- /dev/null +++ b/test/src/internal/iwork/iwork_protobuf_test.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string key(const std::uint32_t number, const WireType type) { + return varint((number << 3) | static_cast(type)); +} + +std::string length_delimited(const std::uint32_t number, + const std::string &bytes) { + return key(number, WireType::length_delimited) + varint(bytes.size()) + bytes; +} + +void parse(const std::string &data) { + const Message message(data); + (void)message; +} + +} // namespace + +TEST(ProtobufMessage, varint_field) { + const std::string data = key(1, WireType::varint) + varint(10000); + const Message message(data); + EXPECT_EQ(message.number_field(1), 10000); +} + +// The wire format's largest varint is ten bytes. +TEST(ProtobufMessage, largest_varint) { + const std::string data = key(1, WireType::varint) + + varint(std::numeric_limits::max()); + const Message message(data); + EXPECT_EQ(message.number_field(1), std::numeric_limits::max()); +} + +TEST(ProtobufMessage, varint_does_not_terminate) { + EXPECT_ANY_THROW(parse(key(1, WireType::varint) + std::string(11, '\xff'))); +} + +TEST(ProtobufMessage, fixed_fields) { + const std::string data = + key(1, WireType::fixed32) + std::string{'\x04', '\x03', '\x02', '\x01'} + + key(2, WireType::fixed64) + std::string{'\x08', '\x07', '\x06', '\x05', + '\x04', '\x03', '\x02', '\x01'}; + const Message message(data); + EXPECT_EQ(message.number_field(1), 0x01020304); + EXPECT_EQ(message.number_field(2), 0x0102030405060708); +} + +TEST(ProtobufMessage, bytes_field) { + const std::string data = length_delimited(3, "Table of Contents"); + const Message message(data); + EXPECT_EQ(message.bytes_field(3), "Table of Contents"); +} + +TEST(ProtobufMessage, nested_message) { + const std::string data = + length_delimited(2, key(1, WireType::varint) + varint(1732588)); + const Message message(data); + const Message nested(message.bytes_field(2).value()); + EXPECT_EQ(nested.number_field(1), 1732588); +} + +TEST(ProtobufMessage, repeated_field) { + const std::string data = length_delimited(3, "a") + length_delimited(4, "b") + + length_delimited(3, "c"); + const Message message(data); + const std::vector repeated = message.repeated_field(3); + ASSERT_EQ(repeated.size(), 2); + EXPECT_EQ(repeated[0].bytes, "a"); + EXPECT_EQ(repeated[1].bytes, "c"); +} + +// A field we have no accessor for is read like any other and left alone. +TEST(ProtobufMessage, unknown_field_is_kept) { + const std::string data = + key(999, WireType::varint) + varint(1) + length_delimited(3, "text"); + const Message message(data); + EXPECT_EQ(message.fields().size(), 2); + EXPECT_EQ(message.bytes_field(3), "text"); + EXPECT_FALSE(message.bytes_field(999).has_value()); + EXPECT_FALSE(message.number_field(3).has_value()); +} + +TEST(ProtobufMessage, absent_field) { + const std::string data = length_delimited(3, "text"); + const Message message(data); + EXPECT_FALSE(message.field(4).has_value()); + EXPECT_TRUE(message.repeated_field(4).empty()); +} + +TEST(ProtobufMessage, field_runs_past_the_message) { + EXPECT_ANY_THROW( + parse(key(3, WireType::length_delimited) + varint(10) + "short")); +} + +// Groups are deprecated and no iWork archive carries one, so reading one means +// the parse went wrong rather than that a field needs skipping. +TEST(ProtobufMessage, group_field) { + EXPECT_ANY_THROW(parse(key(1, WireType::start_group))); +} + +TEST(ProtobufMessage, field_number_zero) { + EXPECT_ANY_THROW(parse(key(0, WireType::varint) + varint(1))); +} + +TEST(ReadVarint, advances_past_the_field) { + const std::string data = varint(300) + varint(1); + std::size_t position = 0; + EXPECT_EQ(read_varint(data, position), 300); + EXPECT_EQ(position, 2); + EXPECT_EQ(read_varint(data, position), 1); + EXPECT_EQ(position, 3); +} From e42219fdcccc969d098fdf071640016e71c4a437 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:33 +0200 Subject: [PATCH 03/14] feat(iwork): index the objects an iwork package holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An `.iwa` is not a tree but a sequence of archived objects, each a `TSP.ArchiveInfo` naming an identifier and the type of the messages that follow. Objects reference each other by identifier across components, so the package reads its component list from `Index/Metadata.iwa` first — the file names carry identifier suffixes often enough that globbing for them finds nothing — and decompresses a component when something in it is asked for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 1 + src/odr/internal/iwork/iwork_archive.cpp | 180 ++++++++++++++++++ src/odr/internal/iwork/iwork_archive.hpp | 97 ++++++++++ test/CMakeLists.txt | 1 + .../src/internal/iwork/iwork_archive_test.cpp | 120 ++++++++++++ 5 files changed, 399 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_archive.cpp create mode 100644 src/odr/internal/iwork/iwork_archive.hpp create mode 100644 test/src/internal/iwork/iwork_archive_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fa2cd08..fed40439 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,7 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_archive.cpp" "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp new file mode 100644 index 00000000..487ddcd1 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -0,0 +1,180 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// Field numbers of `TSP.PackageMetadata` and the `ComponentInfo` it repeats, +/// read off `empty.pages Index/Metadata.iwa` (object 2, type 11006). +constexpr std::uint32_t package_metadata_components = 3; +constexpr std::uint32_t component_info_identifier = 1; +constexpr std::uint32_t component_info_preferred_locator = 2; +constexpr std::uint32_t component_info_locator = 3; + +/// Field numbers of `TSP.ArchiveInfo` and the `MessageInfo` it repeats. +constexpr std::uint32_t archive_info_identifier = 1; +constexpr std::uint32_t archive_info_messages = 2; +constexpr std::uint32_t message_info_type = 1; +constexpr std::uint32_t message_info_length = 3; + +AbsPath component_path(const std::string &locator) { + return AbsPath("/Index").join(RelPath(locator + ".iwa")); +} + +} // namespace + +std::string iwork::read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path) { + const std::shared_ptr file = filesystem.open(path); + if (!file) { + throw std::runtime_error("iwork: missing " + path.string()); + } + const std::unique_ptr stream = file->stream(); + return iwa_decompress(util::stream::read(*stream)); +} + +std::vector iwork::read_objects(const std::string_view data) { + std::vector result; + + std::size_t position = 0; + while (position < data.size()) { + const std::uint64_t info_length = read_varint(data, position); + if (info_length > data.size() - position) { + throw std::runtime_error("iwork: archive info runs past the component"); + } + const Message info(data.substr(position, info_length)); + position += info_length; + + Object object; + object.identifier = info.number_field(archive_info_identifier).value_or(0); + + // the payload holds every message the info names, back to back; only the + // first is modelled, the length of the rest is what skips them + std::size_t payload_length = 0; + std::size_t first_length = 0; + bool first = true; + for (const Field &message : info.repeated_field(archive_info_messages)) { + if (message.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed message info"); + } + const Message message_info(message.bytes); + const std::uint64_t length = + message_info.number_field(message_info_length).value_or(0); + if (first) { + object.type = static_cast( + message_info.number_field(message_info_type).value_or(0)); + first_length = length; + first = false; + } + payload_length += length; + } + + if (payload_length > data.size() - position) { + throw std::runtime_error("iwork: object payload runs past the component"); + } + object.payload = data.substr(position, first_length); + position += payload_length; + + result.push_back(object); + } + + return result; +} + +iwork::Component::Component(std::string locator, std::string data) + : m_locator{std::move(locator)}, + m_data{std::make_unique(std::move(data))}, + m_objects{read_objects(*m_data)} {} + +const std::string &iwork::Component::locator() const noexcept { + return m_locator; +} + +const std::vector &iwork::Component::objects() const noexcept { + return m_objects; +} + +iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) + : m_filesystem{&filesystem} { + const std::string data = read_iwa(filesystem, AbsPath("/Index/Metadata.iwa")); + const std::vector objects = read_objects(data); + if (objects.empty()) { + throw std::runtime_error("iwork: empty package metadata"); + } + + const Message metadata(objects.front().payload); + for (const Field &component : + metadata.repeated_field(package_metadata_components)) { + if (component.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed component info"); + } + const Message info(component.bytes); + + ComponentInfo result; + result.identifier = + info.number_field(component_info_identifier).value_or(0); + result.name = std::string(info.bytes_field(component_info_preferred_locator) + .value_or(std::string_view())); + result.locator = std::string( + info.bytes_field(component_info_locator).value_or(result.name)); + if (result.name.empty()) { + throw std::runtime_error("iwork: component without a name"); + } + m_component_infos.push_back(std::move(result)); + } +} + +const iwork::Component &iwork::Package::component(const std::string &name) { + const auto it = + std::ranges::find(m_component_infos, name, &ComponentInfo::name); + if (it == std::ranges::end(m_component_infos)) { + throw std::runtime_error("iwork: no component named " + name); + } + return load_(*it); +} + +const iwork::Object &iwork::Package::object(const std::uint64_t identifier) { + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + + for (const ComponentInfo &info : m_component_infos) { + load_(info); + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + } + + throw std::runtime_error("iwork: no object " + std::to_string(identifier)); +} + +const iwork::Component &iwork::Package::load_(const ComponentInfo &info) { + // by locator, not by name: a name is shared across components, so keying on + // it would hand back the wrong file and leave the other never loaded + if (const auto it = + std::ranges::find(m_components, info.locator, &Component::locator); + it != std::ranges::end(m_components)) { + return *it; + } + + const Component &component = m_components.emplace_back( + info.locator, read_iwa(*m_filesystem, component_path(info.locator))); + for (const Object &object : component.objects()) { + m_objects.emplace(object.identifier, &object); + } + return component; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp new file mode 100644 index 00000000..7a286e01 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal { +class AbsPath; +} // namespace odr::internal + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// One archived object. `TSP.ArchiveInfo` names its identifier (field 1) and, +/// per payload message, a `MessageInfo` (field 2) carrying the message type +/// and its length. An object usually holds one message; where it holds more, +/// only the first is modelled. +/// +/// Verified on `empty.pages Index/Document.iwa +0`: `08 01` (identifier 1), +/// `12 52` (an 82-byte `MessageInfo`), `08 90 4e` (type 10000), `18 e0 0c` +/// (payload length 1632). +struct Object final { + std::uint64_t identifier{}; + std::uint32_t type{}; + std::string_view payload; +}; + +/// The objects of one `.iwa`, over the bytes it decompressed to. Object +/// payloads are views into those bytes. +class Component final { +public: + Component(std::string locator, std::string data); + + /// The file the component was loaded from, without `/Index/` and `.iwa`. + /// Unlike its name, this is unique across the package. + [[nodiscard]] const std::string &locator() const noexcept; + [[nodiscard]] const std::vector &objects() const noexcept; + +private: + std::string m_locator; + std::unique_ptr m_data; + std::vector m_objects; +}; + +/// An iWork package: the component list from `Index/Metadata.iwa`, and the +/// components loaded from it so far. +/// +/// Objects reference each other by identifier across components, so the list +/// is read first and a component is decompressed when something in it is +/// asked for. +class Package final { +public: + explicit Package(const abstract::ReadableFilesystem &filesystem); + + /// The first component named @p name in the package's component list — a + /// name is not unique, `Tables/DataList` names dozens. Throws when the + /// package holds none. + const Component &component(const std::string &name); + + /// The object @p identifier names, loading components until it is found. + /// Throws when no component holds it. + const Object &object(std::uint64_t identifier); + +private: + /// One entry of `TSP.PackageMetadata`'s component list: the identifier of + /// the component's root object, the name it is known by, and the file it + /// lives in — which carries an identifier suffix often enough that the file + /// name is not a way to find it. + struct ComponentInfo final { + std::uint64_t identifier{}; + std::string name; + std::string locator; + }; + + const abstract::ReadableFilesystem *m_filesystem{nullptr}; + std::vector m_component_infos; + std::deque m_components; + std::unordered_map m_objects; + + const Component &load_(const ComponentInfo &info); +}; + +/// Reads @p path off @p filesystem and undoes its `.iwa` framing. +std::string read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path); + +/// Splits a decompressed `.iwa` into its objects, over @p data. +std::vector read_objects(std::string_view data); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0161fc4b..96063143 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_archive_test.cpp" "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp new file mode 100644 index 00000000..192e02cb --- /dev/null +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -0,0 +1,120 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string number_field(const std::uint32_t number, + const std::uint64_t value) { + return varint(number << 3) + varint(value); +} + +std::string message_field(const std::uint32_t number, + const std::string &bytes) { + return varint((number << 3) | 2) + varint(bytes.size()) + bytes; +} + +/// `TSP.ArchiveInfo`: an identifier and one `MessageInfo` per payload message. +std::string archive_info( + const std::uint64_t identifier, + const std::vector> &messages) { + std::string result = number_field(1, identifier); + for (const auto &[type, length] : messages) { + result += message_field(2, number_field(1, type) + number_field(3, length)); + } + return result; +} + +std::string +object(const std::uint64_t identifier, + const std::vector> &messages, + const std::string &payload) { + const std::string info = archive_info(identifier, messages); + return varint(info.size()) + info + payload; +} + +} // namespace + +TEST(ReadObjects, one_object) { + const std::string data = object(1, {{10000, 5}}, "hello"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].identifier, 1); + EXPECT_EQ(objects[0].type, 10000); + EXPECT_EQ(objects[0].payload, "hello"); +} + +TEST(ReadObjects, objects_follow_one_another) { + const std::string data = + object(1, {{10000, 5}}, "hello") + object(1732514, {{2001, 5}}, "world"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[1].identifier, 1732514); + EXPECT_EQ(objects[1].type, 2001); + EXPECT_EQ(objects[1].payload, "world"); +} + +// An object may hold more than one message; only the first is modelled, and +// the length of the rest is what keeps the reader in step. +TEST(ReadObjects, later_messages_are_skipped) { + const std::string data = object(1732594, {{6247, 3}, {6247, 3}}, "onetwo") + + object(2, {{222, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[0].type, 6247); + EXPECT_EQ(objects[0].payload, "one"); + EXPECT_EQ(objects[1].identifier, 2); +} + +// There is no spec and no schema registry, so a type we have not mapped is an +// app version we have not seen — the reader keeps it and moves on. +TEST(ReadObjects, unknown_type_is_kept) { + const std::string data = object(7, {{123456, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].type, 123456); +} + +TEST(ReadObjects, empty_payload) { + const std::string data = object(1732550, {{3047, 0}}, ""); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_TRUE(objects[0].payload.empty()); +} + +TEST(ReadObjects, empty_component) { EXPECT_TRUE(read_objects("").empty()); } + +TEST(ReadObjects, archive_info_runs_past_the_component) { + const std::string data = object(1, {{10000, 5}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data.substr(0, 4))); +} + +TEST(ReadObjects, payload_runs_past_the_component) { + const std::string data = object(1, {{10000, 500}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data)); +} From e571f7f5b047aaaff8359fef728e9ce1897f4234 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:55:48 +0200 Subject: [PATCH 04/14] feat(iwork): name the three apple iwork file types `.pages`, `.numbers` and `.key` are zips, so today they are reported as `[zip]` and open as an archive rather than a document. Naming them gives a caller the extensions and MIME types to route one and hand a file picker, which is what has to be decided before the file is held. Classification only for now: which app wrote a package is read off its root archive, and only `.pages` has a fixture to pin that against, so the rows declare no capabilities and nothing detects or decodes one yet. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- apple/include/OdrCoreObjC/ODRFile.h | 4 ++ apple/src/ODRFile.mm | 4 ++ jni/java/app/opendocument/core/FileType.java | 3 +- python/src/bind_file.cpp | 5 ++- src/odr/file.hpp | 8 ++++ src/odr/internal/file_type_table.cpp | 42 ++++++++++++++++++++ test/src/odr_test.cpp | 2 +- 7 files changed, 65 insertions(+), 3 deletions(-) diff --git a/apple/include/OdrCoreObjC/ODRFile.h b/apple/include/OdrCoreObjC/ODRFile.h index d1b07078..ac19d087 100644 --- a/apple/include/OdrCoreObjC/ODRFile.h +++ b/apple/include/OdrCoreObjC/ODRFile.h @@ -83,6 +83,10 @@ typedef NS_ENUM(NSInteger, ODRFileType) { ODRFileTypeEnhancedMetafile, ODRFileTypeXml, + + ODRFileTypeIworkPages, + ODRFileTypeIworkNumbers, + ODRFileTypeIworkKeynote, } NS_SWIFT_NAME(FileType); typedef NS_ENUM(NSInteger, ODRFileCategory) { diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index 472f05f8..d616b1ab 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -92,6 +92,10 @@ ODR_SAME_ENUM(ODRFileTypeEnhancedMetafile, odr::FileType::enhanced_metafile); ODR_SAME_ENUM(ODRFileTypeXml, odr::FileType::xml); +ODR_SAME_ENUM(ODRFileTypeIworkPages, odr::FileType::iwork_pages); +ODR_SAME_ENUM(ODRFileTypeIworkNumbers, odr::FileType::iwork_numbers); +ODR_SAME_ENUM(ODRFileTypeIworkKeynote, odr::FileType::iwork_keynote); + ODR_SAME_ENUM(ODRFileCategoryUnknown, odr::FileCategory::unknown); ODR_SAME_ENUM(ODRFileCategoryText, odr::FileCategory::text); ODR_SAME_ENUM(ODRFileCategoryImage, odr::FileCategory::image); diff --git a/jni/java/app/opendocument/core/FileType.java b/jni/java/app/opendocument/core/FileType.java index 80d13963..74b1f5e1 100644 --- a/jni/java/app/opendocument/core/FileType.java +++ b/jni/java/app/opendocument/core/FileType.java @@ -16,7 +16,8 @@ public enum FileType { OGG_AUDIO, WAVEFORM_AUDIO, FREE_LOSSLESS_AUDIO_CODEC, MPEG4_VIDEO, QUICKTIME_VIDEO, THIRD_GENERATION_PARTNERSHIP_VIDEO, MATROSKA_VIDEO, AUDIO_VIDEO_INTERLEAVE, SCALABLE_VECTOR_GRAPHICS, WINDOWS_ICON, JPEG_XL, - JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML; + JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML, + IWORK_PAGES, IWORK_NUMBERS, IWORK_KEYNOTE; static FileType fromNative(int code) { return code < 0 ? null : values()[code]; diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index ac48ef54..0d1163d8 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -83,7 +83,10 @@ void odr_python::bind_file(py::module_ &m) { .value("photoshop_document", odr::FileType::photoshop_document) .value("windows_metafile", odr::FileType::windows_metafile) .value("enhanced_metafile", odr::FileType::enhanced_metafile) - .value("xml", odr::FileType::xml); + .value("xml", odr::FileType::xml) + .value("iwork_pages", odr::FileType::iwork_pages) + .value("iwork_numbers", odr::FileType::iwork_numbers) + .value("iwork_keynote", odr::FileType::iwork_keynote); py::enum_(m, "FileCategory") .value("unknown", odr::FileCategory::unknown) diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 45aea937..930be535 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -149,6 +149,14 @@ enum class FileType { // `[text_file, xml, scalable_vector_graphics]`. // https://en.wikipedia.org/wiki/XML xml, + + // https://en.wikipedia.org/wiki/IWork + iwork_pages, + // Classification only - `.numbers` and `.key` sit in the same package the + // pages engine reads, but which app wrote one is read off its root archive + // and no fixture pins those two, so nothing detects or decodes them yet. + iwork_numbers, + iwork_keynote, }; /// @brief Collection of file categories. diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 61c6ecdd..c5235642 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -250,6 +250,24 @@ constexpr std::array avi_mimetypes{"video/x-msvideo"sv, "video/avi"sv, // `decrypt` on an OOXML document type means a password-protected package, // detected as `office_open_xml_encrypted` and decrypting into the type named // here. ODF files decrypt in place and keep their type. +constexpr std::array pages_extensions{"pages"sv}; +constexpr std::array pages_mimetypes{ + "application/vnd.apple.pages"sv, + "application/x-iwork-pages-sffpages"sv, +}; + +constexpr std::array numbers_extensions{"numbers"sv}; +constexpr std::array numbers_mimetypes{ + "application/vnd.apple.numbers"sv, + "application/x-iwork-numbers-sffnumbers"sv, +}; + +constexpr std::array keynote_extensions{"key"sv}; +constexpr std::array keynote_mimetypes{ + "application/vnd.apple.keynote"sv, + "application/x-iwork-keynote-sffkey"sv, +}; + constexpr std::array table{ Row{FileType::unknown, "unknown"sv, @@ -755,6 +773,30 @@ constexpr std::array table{ .open = true, .translate_html = true, .color_scheme = true}}, + + // Classified so a caller can name the three and hand their MIME types to a + // file picker; no engine reads one yet. + Row{FileType::iwork_pages, + "pages"sv, + pages_extensions, + pages_mimetypes, + FileCategory::document, + DocumentType::text, + {}}, + Row{FileType::iwork_numbers, + "numbers"sv, + numbers_extensions, + numbers_mimetypes, + FileCategory::document, + DocumentType::spreadsheet, + {}}, + Row{FileType::iwork_keynote, + "key"sv, + keynote_extensions, + keynote_mimetypes, + FileCategory::document, + DocumentType::presentation, + {}}, }; /// Finds the row whose list, selected by @p list, contains @p needle. diff --git a/test/src/odr_test.cpp b/test/src/odr_test.cpp index 11bd2d6e..1f7a5fd2 100644 --- a/test/src/odr_test.cpp +++ b/test/src/odr_test.cpp @@ -27,7 +27,7 @@ namespace { std::vector every_file_type() { std::vector result; for (auto i = static_cast(FileType::unknown); - i <= static_cast(FileType::xml); ++i) { + i <= static_cast(FileType::iwork_keynote); ++i) { result.push_back(static_cast(i)); } return result; From 1ee90b6d5c64f8093e165fe559b076d98b0e75d9 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:02:06 +0200 Subject: [PATCH 05/14] feat(pages): open a `.pages` document and read its body text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.pages` package now opens as a text document rather than as the zip it is made of, and its body comes out as paragraphs. Which app wrote the package is read off the type of the root archive in `Index/Document.iwa` — the extension is not consulted, since a caller may not have one. Paragraph boundaries come from the storage's paragraph run table rather than from splitting the text on `\n`, and `U+2028` inside a paragraph becomes a line break. The anchor a drawable leaves in the text is dropped: styles, page geometry, drawables, images and tables are all still to come, so this is the text and nothing else. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 4 + src/odr/exceptions.cpp | 2 + src/odr/exceptions.hpp | 5 + src/odr/internal/file_type_table.cpp | 9 +- src/odr/internal/iwork/iwork_document.cpp | 185 +++++++++++++++ src/odr/internal/iwork/iwork_document.hpp | 27 +++ .../internal/iwork/iwork_element_registry.cpp | 96 ++++++++ .../internal/iwork/iwork_element_registry.hpp | 52 +++++ src/odr/internal/iwork/iwork_file.cpp | 94 ++++++++ src/odr/internal/iwork/iwork_file.hpp | 41 ++++ src/odr/internal/iwork/iwork_parser.cpp | 220 ++++++++++++++++++ src/odr/internal/iwork/iwork_parser.hpp | 18 ++ src/odr/internal/iwork/iwork_types.hpp | 49 ++++ src/odr/internal/open_strategy.cpp | 34 +++ test/CMakeLists.txt | 1 + test/src/internal/iwork/pages_test.cpp | 127 ++++++++++ 16 files changed, 961 insertions(+), 3 deletions(-) create mode 100644 src/odr/internal/iwork/iwork_document.cpp create mode 100644 src/odr/internal/iwork/iwork_document.hpp create mode 100644 src/odr/internal/iwork/iwork_element_registry.cpp create mode 100644 src/odr/internal/iwork/iwork_element_registry.hpp create mode 100644 src/odr/internal/iwork/iwork_file.cpp create mode 100644 src/odr/internal/iwork/iwork_file.hpp create mode 100644 src/odr/internal/iwork/iwork_parser.cpp create mode 100644 src/odr/internal/iwork/iwork_parser.hpp create mode 100644 src/odr/internal/iwork/iwork_types.hpp create mode 100644 test/src/internal/iwork/pages_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fed40439..ab090b17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,10 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/xml_file.cpp" "src/odr/internal/iwork/iwork_archive.cpp" + "src/odr/internal/iwork/iwork_document.cpp" + "src/odr/internal/iwork/iwork_element_registry.cpp" + "src/odr/internal/iwork/iwork_file.cpp" + "src/odr/internal/iwork/iwork_parser.cpp" "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index aaa8d361..c94619a8 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -68,6 +68,8 @@ NoFontFile::NoFontFile() : Exception("not a font file") {} NoLegacyMicrosoftFile::NoLegacyMicrosoftFile() : Exception("not a legacy microsoft office file") {} +NoIworkFile::NoIworkFile() : Exception("not an iwork file") {} + NoXmlFile::NoXmlFile() : Exception("not an xml file") {} NoSvgFile::NoSvgFile() : Exception("not an svg file") {} diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index c9d55fc1..597bc863 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -142,6 +142,11 @@ struct NoLegacyMicrosoftFile final : Exception { NoLegacyMicrosoftFile(); }; +/// @brief No iWork file exception +struct NoIworkFile final : Exception { + NoIworkFile(); +}; + /// @brief No XML file exception struct NoXmlFile final : Exception { NoXmlFile(); diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index c5235642..5333423e 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -774,15 +774,18 @@ constexpr std::array table{ .translate_html = true, .color_scheme = true}}, - // Classified so a caller can name the three and hand their MIME types to a - // file picker; no engine reads one yet. Row{FileType::iwork_pages, "pages"sv, pages_extensions, pages_mimetypes, FileCategory::document, DocumentType::text, - {}}, + {.detect_by_content = true, + .open = true, + .translate_html = true, + .color_scheme = true}}, + // Classified so a caller can name these two and hand their MIME types to a + // file picker; no engine reads either yet. Row{FileType::iwork_numbers, "numbers"sv, numbers_extensions, diff --git a/src/odr/internal/iwork/iwork_document.cpp b/src/odr/internal/iwork/iwork_document.cpp new file mode 100644 index 00000000..5d6848f7 --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.cpp @@ -0,0 +1,185 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace odr::internal::iwork { + +namespace { +std::unique_ptr +create_element_adapter(ElementRegistry ®istry); +} + +Document::Document(std::shared_ptr files) + : internal::Document(FileType::iwork_pages, DocumentType::text, + std::move(files)) { + m_root_element = parse_pages_tree(m_element_registry, *m_files); + + m_element_adapter = create_element_adapter(m_element_registry); +} + +const ElementRegistry &Document::element_registry() const { + return m_element_registry; +} + +bool Document::is_editable() const noexcept { return false; } + +bool Document::is_savable(const bool encrypted) const noexcept { + (void)encrypted; + return false; +} + +void Document::save(const Path &path) const { + (void)path; + throw UnsupportedOperation(); +} + +void Document::save(const Path &path, const char *password) const { + (void)path; + (void)password; + throw UnsupportedOperation(); +} + +namespace { + +class ElementAdapter final : public abstract::ElementAdapter, + public abstract::TextRootAdapter, + public abstract::LineBreakAdapter, + public abstract::ParagraphAdapter, + public abstract::TextAdapter { +public: + explicit ElementAdapter(ElementRegistry ®istry) : m_registry(®istry) {} + + [[nodiscard]] ElementType + element_type(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).type; + } + + [[nodiscard]] ElementIdentifier + element_parent(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).parent_id; + } + [[nodiscard]] ElementIdentifier + element_first_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).first_child_id; + } + [[nodiscard]] ElementIdentifier + element_last_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).last_child_id; + } + [[nodiscard]] ElementIdentifier + element_previous_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).previous_sibling_id; + } + [[nodiscard]] ElementIdentifier + element_next_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).next_sibling_id; + } + + [[nodiscard]] bool + element_is_unique(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_self_locatable(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_editable(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] DocumentPath + element_document_path(const ElementIdentifier element_id) const override { + return util::document::extract_path(*this, element_id, null_element_id); + } + [[nodiscard]] ElementIdentifier + element_navigate_path(const ElementIdentifier element_id, + const DocumentPath &path) const override { + return util::document::navigate_path(*this, element_id, path); + } + + [[nodiscard]] const TextRootAdapter * + text_root_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::root ? this : nullptr; + } + [[nodiscard]] const LineBreakAdapter * + line_break_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::line_break ? this : nullptr; + } + [[nodiscard]] const ParagraphAdapter * + paragraph_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::paragraph ? this : nullptr; + } + [[nodiscard]] const TextAdapter * + text_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::text ? this : nullptr; + } + + // The page geometry sits in the document archive and the styles in + // `Index/DocumentStylesheet.iwa`; neither is read yet. + [[nodiscard]] PageLayout + text_root_page_layout(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] ElementIdentifier text_root_first_master_page( + const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TextStyle + line_break_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] ParagraphStyle + paragraph_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] TextStyle + paragraph_text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] std::string + text_content(const ElementIdentifier element_id) const override { + return m_registry->text_element_at(element_id).text; + } + void text_set_content(const ElementIdentifier element_id, + const std::string &text) const override { + (void)element_id; + (void)text; + throw UnsupportedOperation(); + } + [[nodiscard]] TextStyle + text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + +private: + ElementRegistry *m_registry{nullptr}; +}; + +std::unique_ptr +create_element_adapter(ElementRegistry ®istry) { + return std::make_unique(registry); +} + +} // namespace + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_document.hpp b/src/odr/internal/iwork/iwork_document.hpp new file mode 100644 index 00000000..d73c8b0d --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include + +namespace odr::internal::iwork { + +/// A `.pages` package, read as a text document. +class Document final : public internal::Document { +public: + explicit Document(std::shared_ptr files); + + [[nodiscard]] const ElementRegistry &element_registry() const; + + [[nodiscard]] bool is_editable() const noexcept override; + [[nodiscard]] bool is_savable(bool encrypted) const noexcept override; + + void save(const Path &path) const override; + void save(const Path &path, const char *password) const override; + +private: + ElementRegistry m_element_registry; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.cpp b/src/odr/internal/iwork/iwork_element_registry.cpp new file mode 100644 index 00000000..2ce6952f --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.cpp @@ -0,0 +1,96 @@ +#include + +#include + +namespace odr::internal::iwork { + +void ElementRegistry::clear() noexcept { + m_elements.clear(); + m_texts.clear(); +} + +[[nodiscard]] std::size_t ElementRegistry::size() const noexcept { + return m_elements.size(); +} + +std::tuple +ElementRegistry::create_element(const ElementType type) { + Element &element = m_elements.emplace_back(); + ElementIdentifier element_id = m_elements.size(); + element.type = type; + return {element_id, element}; +} + +std::tuple +ElementRegistry::create_text_element() { + const auto &[element_id, element] = create_element(ElementType::text); + auto [it, success] = m_texts.emplace(element_id, Text{}); + return {element_id, element, it->second}; +} + +ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) { + check_element_id(id); + return m_elements.at(id - 1); +} + +ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) { + check_text_id(id); + return m_texts.at(id); +} + +const ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) const { + check_element_id(id); + return m_elements.at(id - 1); +} + +const ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) const { + check_text_id(id); + return m_texts.at(id); +} + +void ElementRegistry::append_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id) { + check_element_id(parent_id); + check_element_id(child_id); + if (element_at(child_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_child: child already has a parent"); + } + + const ElementIdentifier previous_sibling_id = + element_at(parent_id).last_child_id; + + element_at(child_id).parent_id = parent_id; + element_at(child_id).previous_sibling_id = previous_sibling_id; + + if (element_at(parent_id).first_child_id == null_element_id) { + element_at(parent_id).first_child_id = child_id; + } else { + element_at(previous_sibling_id).next_sibling_id = child_id; + } + element_at(parent_id).last_child_id = child_id; +} + +void ElementRegistry::check_element_id(const ElementIdentifier id) const { + if (id == null_element_id) { + throw std::out_of_range("ElementRegistry::check_id: null identifier"); + } + if (id - 1 >= m_elements.size()) { + throw std::out_of_range( + "ElementRegistry::check_id: identifier out of range"); + } +} + +void ElementRegistry::check_text_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_texts.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.hpp b/src/odr/internal/iwork/iwork_element_registry.hpp new file mode 100644 index 00000000..a7a7bbaa --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace odr::internal::iwork { + +class ElementRegistry final { +public: + struct Element final { + ElementIdentifier parent_id{null_element_id}; + ElementIdentifier first_child_id{null_element_id}; + ElementIdentifier last_child_id{null_element_id}; + ElementIdentifier previous_sibling_id{null_element_id}; + ElementIdentifier next_sibling_id{null_element_id}; + ElementType type{ElementType::none}; + }; + + struct Text final { + std::string text; + }; + + void clear() noexcept; + + [[nodiscard]] std::size_t size() const noexcept; + + std::tuple create_element(ElementType type); + std::tuple create_text_element(); + + [[nodiscard]] Element &element_at(ElementIdentifier id); + [[nodiscard]] Text &text_element_at(ElementIdentifier id); + + [[nodiscard]] const Element &element_at(ElementIdentifier id) const; + [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; + + void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + +private: + std::vector m_elements; + std::unordered_map m_texts; + + void check_element_id(ElementIdentifier id) const; + void check_text_id(ElementIdentifier id) const; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_file.cpp b/src/odr/internal/iwork/iwork_file.cpp new file mode 100644 index 00000000..22bcdfab --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// The type of the root archive says which app wrote the package. Only +/// `.pages` is pinned — a `.numbers` or `.key` fixture would be needed to read +/// theirs off, and the extension is not an answer. +FileType file_type_by_archive_type(const std::uint32_t type) { + switch (type) { + case iwork::archive_type::pages_document: + return FileType::iwork_pages; + default: + return FileType::unknown; + } +} + +/// Reads the root archive of the package's `Document` component. The component +/// list in `Index/Metadata.iwa` is not consulted: this runs on every zip a +/// caller opens, and the `Document` component is the one whose file name never +/// carries an identifier suffix. +FileType parse_file_type(const abstract::ReadableFilesystem &filesystem) { + const std::string data = + iwork::read_iwa(filesystem, AbsPath("/Index/Document.iwa")); + const std::vector objects = iwork::read_objects(data); + if (objects.empty()) { + throw NoIworkFile(); + } + + const FileType file_type = file_type_by_archive_type(objects.front().type); + if (file_type == FileType::unknown) { + throw NoIworkFile(); + } + return file_type; +} + +} // namespace + +iwork::IworkFile::IworkFile( + std::shared_ptr filesystem) + : m_filesystem{std::move(filesystem)} { + if (!m_filesystem->is_file(AbsPath("/Index/Document.iwa"))) { + throw NoIworkFile(); + } + + m_file_meta.type = parse_file_type(*m_filesystem); + m_file_meta.mimetype = mimetype_by_file_type(m_file_meta.type); + m_file_meta.document_type = document_type_by_file_type(m_file_meta.type); +} + +std::shared_ptr iwork::IworkFile::file() const noexcept { + return {}; +} + +FileType iwork::IworkFile::file_type() const noexcept { + return m_file_meta.type; +} + +std::string_view iwork::IworkFile::mimetype() const noexcept { + return m_file_meta.mimetype; +} + +FileMeta iwork::IworkFile::file_meta() const noexcept { return m_file_meta; } + +DocumentType iwork::IworkFile::document_type() const { + return m_file_meta.document_type; +} + +bool iwork::IworkFile::is_decodable() const noexcept { return true; } + +std::shared_ptr iwork::IworkFile::document() const { + switch (file_type()) { + case FileType::iwork_pages: + return std::make_shared(m_filesystem); + default: + throw UnsupportedFileType(file_type()); + } +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_file.hpp b/src/odr/internal/iwork/iwork_file.hpp new file mode 100644 index 00000000..ce86d519 --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace odr::internal::abstract { +class Document; +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// An iWork package (`.pages`, `.numbers`, `.key`). Which app wrote it is read +/// off the root archive of `Index/Document.iwa`, not off the file name, which +/// a caller may have lost. +class IworkFile final : public abstract::DocumentFile { +public: + explicit IworkFile(std::shared_ptr filesystem); + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] DocumentType document_type() const override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + [[nodiscard]] std::shared_ptr document() const override; + +private: + std::shared_ptr m_filesystem; + FileMeta m_file_meta; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp new file mode 100644 index 00000000..20056440 --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -0,0 +1,220 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// `U+2028 LINE SEPARATOR` — a line break inside a paragraph. +constexpr std::string_view line_separator = "\xe2\x80\xa8"; +/// `U+FFFC OBJECT REPLACEMENT CHARACTER` — where a drawable is anchored in the +/// text. Nothing reads drawables yet, so the anchor is dropped rather than +/// rendered as a glyph. +constexpr std::string_view object_replacement = "\xef\xbf\xbc"; + +/// The byte length of the UTF-8 sequence @p lead starts. +std::size_t utf8_length(const std::uint8_t lead) { + if (lead < 0x80) { + return 1; + } + if ((lead & 0xe0) == 0xc0) { + return 2; + } + if ((lead & 0xf0) == 0xe0) { + return 3; + } + if ((lead & 0xf8) == 0xf0) { + return 4; + } + throw std::runtime_error("iwork: text is not utf-8"); +} + +/// Translates the ascending UTF-16 code unit @p indices a storage's run tables +/// count in into byte offsets into @p text. +std::vector +utf16_offsets(const std::string_view text, + const std::vector &indices) { + std::vector result; + result.reserve(indices.size()); + + std::size_t offset = 0; + std::uint64_t unit = 0; + auto next = indices.begin(); + + for (;;) { + while (next != indices.end() && *next == unit) { + result.push_back(offset); + ++next; + } + if (next == indices.end()) { + return result; + } + if (offset >= text.size()) { + throw std::runtime_error("iwork: run table points past the text"); + } + + const std::size_t length = + utf8_length(static_cast(text[offset])); + if (offset + length > text.size()) { + throw std::runtime_error("iwork: text ends mid-character"); + } + offset += length; + // everything outside the basic multilingual plane is a surrogate pair + unit += length == 4 ? 2 : 1; + } +} + +/// The character index each paragraph of @p storage starts at. Paragraph +/// boundaries are the run table's rather than every `\n` in the text — the two +/// agree today, but the table is what says so. +std::vector paragraph_starts(const iwork::Message &storage) { + std::vector result; + + const std::optional table = + storage.bytes_field(iwork::text_storage::paragraph_styles); + if (!table.has_value()) { + return {0}; + } + + for (const iwork::Field &entry : + iwork::Message(*table).repeated_field(iwork::attribute_table::entries)) { + if (entry.type != iwork::WireType::length_delimited) { + throw std::runtime_error("iwork: malformed paragraph style table"); + } + const iwork::Message run(entry.bytes); + result.push_back( + run.number_field(iwork::attribute_table_entry::character_index) + .value_or(0)); + } + + // an empty document carries an empty table; either way the body starts at + // its first character + if (result.empty() || result.front() != 0) { + result.insert(result.begin(), 0); + } + return result; +} + +/// @p text without the drawable anchors it holds. +std::string without_anchors(const std::string_view text) { + std::string result; + result.reserve(text.size()); + + for (std::size_t position = 0; position < text.size();) { + const std::size_t anchor = text.find(object_replacement, position); + if (anchor == std::string_view::npos) { + result += text.substr(position); + break; + } + result += text.substr(position, anchor - position); + position = anchor + object_replacement.size(); + } + return result; +} + +/// Fills @p paragraph_id with the text of one paragraph, breaking it at the +/// line separators it holds. +void parse_paragraph(iwork::ElementRegistry ®istry, + const ElementIdentifier paragraph_id, + std::string_view content) { + const auto append_text = [&](const std::string_view part) { + std::string text = without_anchors(part); + if (text.empty()) { + return; + } + auto [text_id, element, payload] = registry.create_text_element(); + payload.text = std::move(text); + registry.append_child(paragraph_id, text_id); + }; + + for (std::size_t position = content.find(line_separator); + position != std::string_view::npos; + position = content.find(line_separator)) { + append_text(content.substr(0, position)); + + auto [break_id, element] = registry.create_element(ElementType::line_break); + registry.append_child(paragraph_id, break_id); + + content.remove_prefix(position + line_separator.size()); + } + append_text(content); +} + +} // namespace + +ElementIdentifier +iwork::parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files) { + Package package(files); + + const std::vector &objects = package.component("Document").objects(); + if (objects.empty() || objects.front().type != archive_type::pages_document) { + throw std::runtime_error("iwork: no pages document archive"); + } + + const Message document(objects.front().payload); + const std::optional body = + document.bytes_field(document_archive::body_storage); + if (!body.has_value()) { + throw std::runtime_error("iwork: document archive holds no body"); + } + const std::optional body_identifier = + Message(*body).number_field(reference::identifier); + if (!body_identifier.has_value()) { + throw std::runtime_error("iwork: body reference names no object"); + } + + const Object &body_object = package.object(*body_identifier); + if (body_object.type != archive_type::text_storage) { + throw std::runtime_error("iwork: body is not a text storage"); + } + const Message storage(body_object.payload); + + // the text arrives as a small number of large strings; the run tables index + // it as one + std::string text; + for (const Field &part : storage.repeated_field(text_storage::text)) { + if (part.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed text storage"); + } + text += part.bytes; + } + + const std::vector starts = + utf16_offsets(text, paragraph_starts(storage)); + + auto [root_id, root] = registry.create_element(ElementType::root); + + const std::string_view body_text(text); + for (std::size_t i = 0; i < starts.size(); ++i) { + const std::size_t begin = starts[i]; + const std::size_t end = i + 1 < starts.size() ? starts[i + 1] : text.size(); + + std::string_view content = body_text.substr(begin, end - begin); + // the paragraph mark belongs to the paragraph it ends, and the last + // paragraph of a body does not carry one + if (content.ends_with('\n')) { + content.remove_suffix(1); + } + + auto [paragraph_id, paragraph] = + registry.create_element(ElementType::paragraph); + registry.append_child(root_id, paragraph_id); + parse_paragraph(registry, paragraph_id, content); + } + + return root_id; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_parser.hpp b/src/odr/internal/iwork/iwork_parser.hpp new file mode 100644 index 00000000..82cbd72c --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { +class ElementRegistry; + +/// Parses the body of a `.pages` package into root → paragraph → text +/// elements. +/// \return the root element id. +ElementIdentifier parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_types.hpp b/src/odr/internal/iwork/iwork_types.hpp new file mode 100644 index 00000000..7cf0f962 --- /dev/null +++ b/src/odr/internal/iwork/iwork_types.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace odr::internal::iwork { + +/// The archive types and field numbers the engine reads. +/// +/// There is no spec and Apple has never published the `.proto` schemas, so +/// each of these is cited to the fixture it was read off rather than to a +/// section number, and holds for the iWork version that wrote it — see +/// `Metadata/BuildVersionHistory.plist`. A type id that is not here is one we +/// have not mapped, which the reader skips rather than throws on. +namespace archive_type { +/// `TP.DocumentArchive`, the root of a `.pages` package. +/// `empty.pages Index/Document.iwa` object 1 (iWork 13.2). +constexpr std::uint32_t pages_document = 10000; +/// `TSWP.StorageArchive`, a run of text with its run tables. +/// `empty.pages Index/Document.iwa` object 1732514 (iWork 13.2). +constexpr std::uint32_t text_storage = 2001; +} // namespace archive_type + +namespace document_archive { +/// The body text storage, as a `TSP.Reference`. +constexpr std::uint32_t body_storage = 4; +} // namespace document_archive + +namespace text_storage { +/// The text, in a small number of large strings. +constexpr std::uint32_t text = 3; +/// The paragraph style run table: one entry per paragraph, holding the +/// character index the paragraph starts at and, where it has one, its style. +constexpr std::uint32_t paragraph_styles = 5; +} // namespace text_storage + +/// A run table parallel to the text, as `TSWP.ObjectAttributeTable`. +namespace attribute_table { +constexpr std::uint32_t entries = 1; +} // namespace attribute_table + +namespace reference { +constexpr std::uint32_t identifier = 1; +} // namespace reference + +namespace attribute_table_entry { +constexpr std::uint32_t character_index = 1; +} // namespace attribute_table_entry + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 68a4e260..4a53a4ad 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,18 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoOpenDocumentFile(); } + if (as == FileType::iwork_pages) { + ODR_VERBOSE(logger, "open as iwork"); + try { + auto zip_file = std::make_unique(file); + auto filesystem = zip_file->archive()->as_filesystem(); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + throw NoIworkFile(); + } + if (as == FileType::office_open_xml_document || as == FileType::office_open_xml_presentation || as == FileType::office_open_xml_workbook || @@ -258,6 +271,13 @@ open_strategy::list_file_types(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + result.push_back(iwork::IworkFile(filesystem).file_type()); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } catch (...) { ODR_VERBOSE(logger, "failed to open as zip"); } @@ -367,6 +387,13 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "failed to open as ooxml"); } + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + return zip_file; } if (file_type == FileType::compound_file_binary_format) { @@ -545,6 +572,13 @@ open_strategy::open_document_file(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } else if (file_type == FileType::compound_file_binary_format) { ODR_VERBOSE(logger, "open as cbf"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 96063143..315e50d3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -62,6 +62,7 @@ add_executable(odr_test "src/internal/iwork/iwork_archive_test.cpp" "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" + "src/internal/iwork/pages_test.cpp" "src/internal/odf/odf_table_test.cpp" diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp new file mode 100644 index 00000000..820dfc95 --- /dev/null +++ b/test/src/internal/iwork/pages_test.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +using namespace odr; +using odr::test::TestData; + +namespace { + +/// The paragraphs of a text root, a line break reading as a newline. +std::vector paragraphs(const Element root) { + std::vector result; + + for (const Element paragraph : root.children()) { + EXPECT_EQ(paragraph.type(), ElementType::paragraph); + + std::string text; + for (const Element child : paragraph.children()) { + if (child.type() == ElementType::line_break) { + text += '\n'; + } else { + text += child.as_text().content(); + } + } + result.push_back(std::move(text)); + } + + return result; +} + +} // namespace + +TEST(Iwork, pages_is_detected_by_content) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + const std::string path = + TestData::test_file_path("odr-public/pages/style-various-1.pages"); + + EXPECT_THAT(list_file_types(path, logger), + testing::Contains(FileType::iwork_pages)); + + const DecodedFile file(path, logger); + EXPECT_EQ(file.file_type(), FileType::iwork_pages); + EXPECT_EQ(file.file_category(), FileCategory::document); + EXPECT_EQ(file.as_document_file().document_type(), DocumentType::text); +} + +// A document with nothing in it must come back with an empty body rather than +// throw: `empty.pages` carries a body storage that holds no text at all. +TEST(Iwork, pages_empty) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/empty.pages"), logger); + EXPECT_EQ(document_file.file_type(), FileType::iwork_pages); + + const Document document = document_file.document(); + EXPECT_EQ(document.document_type(), DocumentType::text); + EXPECT_FALSE(document.is_editable()); + EXPECT_FALSE(document.is_savable(false)); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{""})); +} + +TEST(Iwork, pages_body_text) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/style-various-1.pages"), + logger); + + const Document document = document_file.document(); + const std::vector text = paragraphs(document.root_element()); + + // one element per paragraph of the body, including the empty ones that + // separate its sections + ASSERT_EQ(text.size(), 54); + EXPECT_EQ(text[0], "Table of Contents"); + // the anchor of a drawable is dropped: nothing reads drawables yet + EXPECT_EQ(text[1], ""); + EXPECT_EQ(text[4], "Headline"); + EXPECT_EQ(text[5], "Nested Headline"); + EXPECT_EQ(text[7], "Text"); + EXPECT_EQ(text[9], "Hyperlink google"); + EXPECT_EQ(text[11], "Default"); + EXPECT_EQ(text[12], "Bold"); + EXPECT_EQ(text.back(), "image"); +} + +// Components share names — `style-various-1.pages` holds two dozen called +// `Tables/DataList` — so the package has to load them by locator. Keying on +// the name hands back the wrong file and leaves the rest never loaded, which +// shows up as an object nothing can resolve. +TEST(Iwork, package_resolves_across_components) { + using odr::internal::iwork::Package; + + const auto file = + std::make_shared(odr::internal::AbsPath( + TestData::test_file_path("odr-public/pages/style-various-1.pages"))); + const auto filesystem = + odr::internal::zip::ZipFile(file).archive()->as_filesystem(); + + Package package(*filesystem); + + EXPECT_EQ(package.component("Document").objects().front().identifier, 1); + // the stylesheet, which is a component of its own + EXPECT_EQ(package.object(1732588).identifier, 1732588); + // the root of a `Tables/DataList` that is not the first one of that name + EXPECT_EQ(package.object(1732940).identifier, 1732940); +} From 32284fd61c8db0b5ca68d51717b9eaa93ba86899 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:03:59 +0200 Subject: [PATCH 06/14] docs(iwork): record what the first two stages decided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PLAN.md` was written before any of it existed; this is the module's `AGENTS.md` alongside it — why a fixture is the citation here rather than a spec section, why snappy and protobuf are in-tree, the `Message` lifetime the whole engine rests on, and where the run tables sit. `PLAN.md` marks the two landed stages and the three places the plan and the code disagreed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- AGENTS.md | 10 +++ CHANGELOG.md | 6 ++ src/odr/internal/iwork/AGENTS.md | 109 +++++++++++++++++++++++++++++++ src/odr/internal/iwork/PLAN.md | 65 +++++++++++------- 4 files changed, 167 insertions(+), 23 deletions(-) create mode 100644 src/odr/internal/iwork/AGENTS.md diff --git a/AGENTS.md b/AGENTS.md index d7b27100..0eafe31f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/odf/` | OpenDocument (odt/ods/odp/odg); see [`odf/AGENTS.md`](src/odr/internal/odf/AGENTS.md). | | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | +| `src/odr/internal/iwork/` | Apple iWork (`.pages` today); see [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) + [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md). | | `src/odr/internal/pdf/` | PDF (own parser). | | `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | @@ -218,6 +219,15 @@ Dispatch `release.yml` against main, publish the draft that appears — 4. Register the factory (e.g. `oldms_file.cpp::document()` switches on `file_type()`), add sources to `CMakeLists.txt`, add a GoogleTest. +## Apple iWork (`iwork`) + +`.pages` opens as a text document and renders its body text; `.numbers` and +`.key` are named but not decoded. There is no spec — the module cites fixtures +instead, keeps its own Snappy and protobuf readers, and fails soft on archive +types it has not mapped. Read [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) +before touching it, and [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md) for +what comes next. + ## Legacy Microsoft binary formats (`oldms`) CFB container handling exists; each format is a small module under `oldms/` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc85b98..678452ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- Apple iWork: a `.pages` file opens as a text document and renders its body + text, instead of coming back as the zip it is made of. Styles, page geometry, + images and tables are not read yet. `.numbers` and `.key` are named — + `FileType::iwork_numbers`, `FileType::iwork_keynote`, their extensions and + MIME types — but there is no decoder behind either. + ## v6.10.1 - 2026-08-21 - A linked image in a docx or xlsx (`embed_images = false`) is named relative diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md new file mode 100644 index 00000000..4223d55f --- /dev/null +++ b/src/odr/internal/iwork/AGENTS.md @@ -0,0 +1,109 @@ +# AGENTS.md — `internal/iwork` + +Read the root [`AGENTS.md`](../../../../AGENTS.md) first, then +[`PLAN.md`](PLAN.md), which is where this module is going and in what order. +This file is what the landed stages decided, and why. + +Landed: **stage 1** (detection and the container) and **stage 2** (Pages body +text). A `.pages` opens as a text document and renders its paragraphs. +Everything else in `PLAN.md` is still ahead. + +## There is no spec, so a fixture is the citation + +Apple has never published the `.proto` schemas and nothing is vendored under +`offline/documentation/`. Where `oldms/` writes `[MS-XLS] §2.4.1`, this module +writes `empty.pages Index/Document.iwa +0` — the byte layout verified against a +file in the repo is the only claim treated as fact. + +Everything the engine reads by number lives in `iwork_types.hpp`, each constant +cited to the fixture it was read off. Read `numbers-parser`, `keynote-parser`, +`obriensp/iWorkFileFormat` and `libetonyek` for facts; **copy code from none of +them**. + +**Fail soft on a type id we have not mapped, fail fast on broken framing.** The +root `AGENTS.md` says to throw where the spec dictates what to expect. Here +there is no spec, and an unknown type id means Apple shipped a version we have +not seen — a reader that throws on one cannot open next year's files. What does +throw: framing that overruns the file, a Snappy block that does not fill its +declared length, a varint that does not terminate, an identifier the package +does not hold, and text that is not UTF-8. + +## No new dependencies + +Two pieces would normally be a conan line each, and both would be wrong. + +- **Snappy** — the `.iwa` framing is Apple's own (`0x00`, a little-endian + 24-bit compressed length, repeated to EOF), not Snappy's stream framing, so + only the *block* decoder applies. `iwork_snappy.cpp` is that, in about a + hundred lines. +- **Protobuf** — only the wire format is needed, and with no schemas a code + generator has nothing to generate. Linking conan `protobuf` would drag it + into the wasm, android and apple builds to replace `iwork_protobuf.cpp`. + +Both stay inside `iwork/` until something else wants them; a wire reader with +one user has not earned a package. + +## `Message` views the buffer it was read from + +`iwork::Message` parses one level eagerly and leaves nested messages, strings +and packed fields as `std::string_view`s into the bytes it was handed. So the +buffer has to outlive it — `Component` owns its decompressed data behind a +`unique_ptr` for exactly that reason, and a `Message(some_temporary())` is a +dangling read rather than a compile error. + +## An `.iwa` is an object graph, not a tree + +A component file is a flat sequence of `(varint length, TSP.ArchiveInfo, +payload)`, and objects reference each other by identifier — across components. +So `Package` reads the component list from `Index/Metadata.iwa` first and +decompresses a component when something in it is asked for; `object(id)` loads +further components until the identifier turns up. Walking files in directory +order and hoping a tree falls out is the mistake to avoid. + +**Component names are not file names.** `Index/Metadata.iwa` maps a component's +name to its locator, and the locator carries an identifier suffix often enough +that globbing for `CalculationEngine.iwa` finds it in one fixture and not in +the other. + +The one place that skips the component list is detection: `IworkFile` reads +`/Index/Document.iwa` directly, because it runs on every zip a caller opens and +`Document` is the component whose file name never carries a suffix. + +## Which app wrote the package comes off the root archive + +`TP.DocumentArchive` is type 10000, verified on both `.pages` fixtures. The +extension is not consulted — a caller may have lost it — and neither is +`Metadata/Properties.plist`, which names an app version but not the app. + +That is also why only `.pages` is detected. `iwork_numbers` and `iwork_keynote` +have `file_type_table.cpp` rows so a caller can name them and hand a file +picker their MIME types, but no capabilities: reading their root archive types +off a guess is exactly what this module does not do, and neither has a fixture +in the test data yet. + +## Paragraphs come from the run table + +A `TSWP.StorageArchive` holds its text as a few large strings plus run tables +parallel to it — index/value pairs for paragraph styles, character styles and +attachments. Paragraph boundaries are the **paragraph style table's**, not +every `\n` in the text. The two agree on both fixtures, but the table is what +says so, and `U+2028` is a line break *inside* a paragraph rather than a +paragraph boundary. + +Run tables count in **UTF-16 code units** while the text is UTF-8, so +`iwork_parser.cpp` translates the indices in one pass over the text. An index +that lands mid-character is an error, not a rounding. + +`U+FFFC` is where a drawable is anchored. Nothing reads drawables yet, so the +anchor is dropped rather than rendered as a glyph — see stage 4. + +`empty.pages` is the regression that matters at this level: a body storage that +carries no text at all must produce an empty body, not an exception. + +## Not read yet + +`Index/DocumentStylesheet.iwa` (so `text_root_page_layout` is empty and every +style is the default), drawables and images, `Index/Tables/`, and everything +`PLAN.md` lists as deferred. `password_encrypted()` is not answered either: an +encrypted package is one whose `Index/Document.iwa` does not decompress, which +falls back to reporting the file as a zip. diff --git a/src/odr/internal/iwork/PLAN.md b/src/odr/internal/iwork/PLAN.md index f57831fc..a8bbf830 100644 --- a/src/odr/internal/iwork/PLAN.md +++ b/src/odr/internal/iwork/PLAN.md @@ -1,28 +1,24 @@ # iWork plan -Where an iwork module would go, and in what order. Written before stage 1; keep -it honest as stages land. +Where an iwork module goes, and in what order. Written before stage 1; kept +honest as stages land. **Stages 1 and 2 have landed** — see +[`AGENTS.md`](AGENTS.md) for what they decided. Stage 3 is next. ## Today -Nothing decodes, and unlike rtf there is not even a `FileType` yet. A `.pages` -is a zip, so `magic.cpp:95` reports `FileType::zip`, `list_file_types` probes -odf then ooxml (`open_strategy.cpp:213-238`), both fail, and the caller gets -`[zip]`. `odr::open` hands back a `zip::ZipFile` — an archive, not a document. +A `.pages` opens as a text document and renders its body text. `.numbers` and +`.key` have `FileType` entries and `file_type_table.cpp` rows so a caller can +name them, but no capabilities and no engine behind them: which app wrote a +package is read off its root archive type, and neither has a fixture to pin +that against. -Two fixtures are already committed: +Two fixtures are committed: `test/data/input/odr-public/pages/{empty.pages,style-various-1.pages}`, both written by iWork 13.2 (`Metadata/BuildVersionHistory.plist`). Neither is listed -in `index.csv` and neither has reference output, so nothing exercises them. -`style-various-1.pages` carries `Index/Tables/` and nine files under `Data/`, -which is most of the surface below. - -New `FileType` entries append at the end of the enum — `file.hpp:98` says so — -and **the bindings do need updating** here, unlike rtf: `python/src/bind_file.cpp`, -`jni/java/app/opendocument/core/FileType.java`, -`apple/include/OdrCoreObjC/ODRFile.h` + `apple/src/ODRFile.mm`. Wasm does not: -it derives its enums from `odr::all_file_types()` at runtime -(`wasm/src/wasm_core.cpp:35`, `:69`). +in `index.csv` — they do not need to be, `TestData` picks up anything the file +type table knows an extension for — and they gained reference output when stage +2 turned `translate_html` on. `style-various-1.pages` carries `Index/Tables/` +and nine files under `Data/`, which is most of the surface below. ## Spec @@ -145,11 +141,26 @@ silently. --- -## Stage 1 — detection and the container +## Stage 1 — detection and the container *(landed)* Nothing renders yet. The point is that the bytes come apart correctly and the type is reported, which is also the whole of what a file picker needs. +Landed as planned, with three deviations: + +- **Only `iwork_pages` detects and opens.** `iwork_numbers` and `iwork_keynote` + are classification-only rows, because the root archive type of a `.numbers` + or a `.key` cannot be read off a fixture that does not exist, and this module + does not guess. +- **`password_encrypted()` is not answered.** `Index/Metadata.iwph` was going + to report it, but nothing here has ever seen an encrypted package; an + encrypted one is one whose `Index/Document.iwa` does not decompress, and it + falls back to being reported as a zip. +- **Detection does not read `Index/Metadata.iwa`.** It reads + `/Index/Document.iwa` straight, since it runs on every zip a caller opens and + `Document` is the one component whose file name never carries a suffix. The + component list is read when the document is. + - `iwork_snappy.{hpp,cpp}` — Apple framing plus block decompression, over the `std::istream *` / `std::streambuf *` shape `pdf::ObjectParser` uses (`pdf_object_parser.hpp`). @@ -183,7 +194,7 @@ type ID that must be skipped rather than thrown on, and framing truncated mid-block. Only the type-reporting test needs the fixtures — the data repos are fetched and optional, so everything that can be inline is. -## Stage 2 — Pages text +## Stage 2 — Pages text *(landed)* - walk from the document archive to the body's text storage (`TSWP.StorageArchive` in the reverse-engineering literature; confirm the type @@ -199,6 +210,13 @@ fetched and optional, so everything that can be inline is. produce an empty body and not an exception. - table row: `iwork_pages` gains `.translate_html = true`. +Landed as planned. What the fixtures settled: the body storage is field 4 of +`TP.DocumentArchive` (type 10000) and is a `TSWP.StorageArchive` (type 2001); +its paragraph style table is field 5, a `TSWP.ObjectAttributeTable` whose +field 1 repeats the entries — so the run tables are one level deeper than +"repeated entries on the storage". Run-table indices count UTF-16 code units +against UTF-8 text, which the parser translates in one pass. + ## Stage 3 — Pages styles - `Index/DocumentStylesheet.iwa`. Style archives are sparse property sets with a @@ -286,10 +304,11 @@ without a Numbers fixture existing. ## Test data -`empty.pages` and `style-various-1.pages` are already in -`test/data/input/odr-public/pages/` but absent from `index.csv` and from -reference output. Add them to the index in stage 1, and regenerate reference -output when stage 2 flips `translate_html` on. +`empty.pages` and `style-various-1.pages` are in +`test/data/input/odr-public/pages/`. They need no `index.csv` row — +`TestData::test_files` picks up any file whose extension the file type table +knows — and reference output was regenerated when stage 2 flipped +`translate_html` on. Stages 5 and 7 each need a fixture that does not exist yet — one `.key` and one `.numbers` in the public repo. Everything at container level stays inline, per From 98c418d4feb774c475db0ab121e7914501ee0cb4 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:06:19 +0200 Subject: [PATCH 07/14] test(data): pin the reference output the two pages fixtures render Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- test/data.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/data.cmake b/test/data.cmake index ffdcf2fc..26286106 100644 --- a/test/data.cmake +++ b/test/data.cmake @@ -17,7 +17,7 @@ odr_test_data( odr_test_data( PATH "reference-output/odr-public" URL "https://github.com/opendocument-app/OpenDocument.test.output.git" - REVISION "d9cb5666399ab5873152953106dce53723fcbced") + REVISION "f8994d207ce3869e05e4ba126cd9d3de1fc73bc7") odr_test_data( PATH "reference-output/odr-private" From ddcf8e8e34ae666315f34b8194c5b5b60572e40e Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:17:47 +0200 Subject: [PATCH 08/14] docs(readme): name pages, numbers and key in the format lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch decodes `.pages` and classifies `.numbers` and `.key`, but README still listed `pages` as unsupported and named neither of the other two — the one place a caller looks to decide which MIME types to advertise. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3837e147..4ee06f28 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ C++ library to visualize files, especially documents, in HTML. - [csv](https://github.com/opendocument-app/OpenDocument.core/issues/107) - [doc](https://github.com/opendocument-app/OpenDocument.core/issues/104), [ppt](https://github.com/opendocument-app/OpenDocument.core/issues/106), [xls](https://github.com/opendocument-app/OpenDocument.core/issues/105) - [pdf](https://github.com/opendocument-app/OpenDocument.core/issues/108) +- pages (Apple Pages — body text only; styles, page geometry, images and + tables are not read yet) - txt - json - [zip](https://github.com/opendocument-app/OpenDocument.core/issues/109) @@ -33,6 +35,8 @@ decoder, and opening one throws: - md (Markdown) - xlsb (Excel binary workbook — an OOXML package whose workbook parts are binary rather than spreadsheetml) +- numbers (Apple Numbers) +- key (Apple Keynote) ## Asking what is supported @@ -49,7 +53,6 @@ supported for any format. ## Unsupported files -- pages - xml - yaml From 4d1a27dbda915500b7cfd309a3af3ed6f9cce579 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:17:54 +0200 Subject: [PATCH 09/14] refactor(iwork): drop what nothing reads and reuse what exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ComponentInfo::identifier` was filled from every component info and never read: `component` finds by name, `load_` keys on the locator, and `object` goes through the object map. It goes back in when something reads it. `without_anchors` was `util::string::replace_all` written out, and `paragraph_starts` said "the body starts at its first character" twice — once as an early return for a missing table, once as the normalisation that already covers it. The three iWork alias arrays had landed between `table`'s doc comment and `table`, so the comment read as if it described them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/file_type_table.cpp | 12 +++--- src/odr/internal/iwork/iwork_archive.cpp | 3 -- src/odr/internal/iwork/iwork_archive.hpp | 8 ++-- src/odr/internal/iwork/iwork_parser.cpp | 51 ++++++++---------------- 4 files changed, 26 insertions(+), 48 deletions(-) diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 5333423e..a336f541 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -244,12 +244,6 @@ constexpr std::array avi_extensions{"avi"sv}; constexpr std::array avi_mimetypes{"video/x-msvideo"sv, "video/avi"sv, "video/msvideo"sv}; -// The single source of truth behind every public format lookup; `odr_test` -// asserts one row per `FileType` and capabilities that match the engines. -// -// `decrypt` on an OOXML document type means a password-protected package, -// detected as `office_open_xml_encrypted` and decrypting into the type named -// here. ODF files decrypt in place and keep their type. constexpr std::array pages_extensions{"pages"sv}; constexpr std::array pages_mimetypes{ "application/vnd.apple.pages"sv, @@ -268,6 +262,12 @@ constexpr std::array keynote_mimetypes{ "application/x-iwork-keynote-sffkey"sv, }; +// The single source of truth behind every public format lookup; `odr_test` +// asserts one row per `FileType` and capabilities that match the engines. +// +// `decrypt` on an OOXML document type means a password-protected package, +// detected as `office_open_xml_encrypted` and decrypting into the type named +// here. ODF files decrypt in place and keep their type. constexpr std::array table{ Row{FileType::unknown, "unknown"sv, diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp index 487ddcd1..b9b94b05 100644 --- a/src/odr/internal/iwork/iwork_archive.cpp +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -19,7 +19,6 @@ namespace { /// Field numbers of `TSP.PackageMetadata` and the `ComponentInfo` it repeats, /// read off `empty.pages Index/Metadata.iwa` (object 2, type 11006). constexpr std::uint32_t package_metadata_components = 3; -constexpr std::uint32_t component_info_identifier = 1; constexpr std::uint32_t component_info_preferred_locator = 2; constexpr std::uint32_t component_info_locator = 3; @@ -123,8 +122,6 @@ iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) const Message info(component.bytes); ComponentInfo result; - result.identifier = - info.number_field(component_info_identifier).value_or(0); result.name = std::string(info.bytes_field(component_info_preferred_locator) .value_or(std::string_view())); result.locator = std::string( diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp index 7a286e01..b8363edc 100644 --- a/src/odr/internal/iwork/iwork_archive.hpp +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -69,12 +69,10 @@ class Package final { const Object &object(std::uint64_t identifier); private: - /// One entry of `TSP.PackageMetadata`'s component list: the identifier of - /// the component's root object, the name it is known by, and the file it - /// lives in — which carries an identifier suffix often enough that the file - /// name is not a way to find it. + /// One entry of `TSP.PackageMetadata`'s component list: the name a component + /// is known by, and the file it lives in — which carries an identifier + /// suffix often enough that the file name is not a way to find it. struct ComponentInfo final { - std::uint64_t identifier{}; std::string name; std::string locator; }; diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp index 20056440..0a0942e0 100644 --- a/src/odr/internal/iwork/iwork_parser.cpp +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -81,55 +82,37 @@ utf16_offsets(const std::string_view text, std::vector paragraph_starts(const iwork::Message &storage) { std::vector result; - const std::optional table = - storage.bytes_field(iwork::text_storage::paragraph_styles); - if (!table.has_value()) { - return {0}; - } - - for (const iwork::Field &entry : - iwork::Message(*table).repeated_field(iwork::attribute_table::entries)) { - if (entry.type != iwork::WireType::length_delimited) { - throw std::runtime_error("iwork: malformed paragraph style table"); + if (const std::optional table = + storage.bytes_field(iwork::text_storage::paragraph_styles); + table.has_value()) { + for (const iwork::Field &entry : iwork::Message(*table).repeated_field( + iwork::attribute_table::entries)) { + if (entry.type != iwork::WireType::length_delimited) { + throw std::runtime_error("iwork: malformed paragraph style table"); + } + const iwork::Message run(entry.bytes); + result.push_back( + run.number_field(iwork::attribute_table_entry::character_index) + .value_or(0)); } - const iwork::Message run(entry.bytes); - result.push_back( - run.number_field(iwork::attribute_table_entry::character_index) - .value_or(0)); } - // an empty document carries an empty table; either way the body starts at - // its first character + // no table at all, or an empty one as an empty document carries: either way + // the body starts at its first character if (result.empty() || result.front() != 0) { result.insert(result.begin(), 0); } return result; } -/// @p text without the drawable anchors it holds. -std::string without_anchors(const std::string_view text) { - std::string result; - result.reserve(text.size()); - - for (std::size_t position = 0; position < text.size();) { - const std::size_t anchor = text.find(object_replacement, position); - if (anchor == std::string_view::npos) { - result += text.substr(position); - break; - } - result += text.substr(position, anchor - position); - position = anchor + object_replacement.size(); - } - return result; -} - /// Fills @p paragraph_id with the text of one paragraph, breaking it at the /// line separators it holds. void parse_paragraph(iwork::ElementRegistry ®istry, const ElementIdentifier paragraph_id, std::string_view content) { const auto append_text = [&](const std::string_view part) { - std::string text = without_anchors(part); + std::string text(part); + util::string::replace_all(text, std::string(object_replacement), ""); if (text.empty()) { return; } From ceb791f2dacebfd0bf0f05323199962800dd716c Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:18:01 +0200 Subject: [PATCH 10/14] fix(iwork): bound a snappy block by what it can hold, not what it claims `reserve` was sized from the block header's 32-bit uncompressed length before a single tag was read, so a zip carrying `Index/Document.iwa` with the ten bytes `00 06 00 00 | FF FF FF FF 0F | 00` reserved ~4 GiB and only then threw. Every zip a caller opens reaches this, decoding no iWork content at all. The reservation is now capped at what the compressed bytes could expand to, and a tag is checked against what the block has left to give before anything is appended, so nothing is written or allocated against a length the file merely claims. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/iwork/AGENTS.md | 5 +++++ src/odr/internal/iwork/iwork_snappy.cpp | 21 ++++++++++++++++++- test/src/internal/iwork/iwork_snappy_test.cpp | 10 +++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md index 4223d55f..977c06d8 100644 --- a/src/odr/internal/iwork/AGENTS.md +++ b/src/odr/internal/iwork/AGENTS.md @@ -28,6 +28,11 @@ throw: framing that overruns the file, a Snappy block that does not fill its declared length, a varint that does not terminate, an identifier the package does not hold, and text that is not UTF-8. +A declared length is the file's word rather than a fact, so nothing is +allocated or written against one before it is known to fit: +`snappy_decompress_block` caps its reservation at what the compressed bytes +could expand to and checks every tag against what the block has left to give. + ## No new dependencies Two pieces would normally be a conan line each, and both would be wrong. diff --git a/src/odr/internal/iwork/iwork_snappy.cpp b/src/odr/internal/iwork/iwork_snappy.cpp index 63a8f931..027c6b49 100644 --- a/src/odr/internal/iwork/iwork_snappy.cpp +++ b/src/odr/internal/iwork/iwork_snappy.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -42,6 +43,11 @@ std::uint32_t read_uncompressed_length(const std::string_view in, throw std::runtime_error("iwork: snappy length varint does not terminate"); } +/// The most a block can emit per compressed byte it holds: the tag that +/// writes the most for its size is a two-byte-offset copy, three bytes for at +/// most 64. Generous headroom over that, so no real block trips it. +constexpr std::size_t max_expansion = 64; + } // namespace std::string iwork::snappy_decompress_block(const std::string_view compressed) { @@ -50,7 +56,18 @@ std::string iwork::snappy_decompress_block(const std::string_view compressed) { read_uncompressed_length(compressed, position); std::string result; - result.reserve(uncompressed_length); + // the declared length is the file's word, not a fact, so the allocation is + // capped by what the block could hold rather than by what it claims + result.reserve(std::min(uncompressed_length, + max_expansion * compressed.size())); + + // nothing is appended before it is known to fit, so a block never + // materialises more than it declared + const auto check_fits = [&](const std::size_t length) { + if (length > uncompressed_length - result.size()) { + throw std::runtime_error("iwork: snappy block writes past its length"); + } + }; while (position < compressed.size()) { const auto tag = static_cast(compressed[position++]); @@ -68,6 +85,7 @@ std::string iwork::snappy_decompress_block(const std::string_view compressed) { if (position + length > compressed.size()) { throw std::runtime_error("iwork: snappy literal runs past the block"); } + check_fits(length); result.append(compressed, position, length); position += length; continue; @@ -91,6 +109,7 @@ std::string iwork::snappy_decompress_block(const std::string_view compressed) { if (offset == 0 || offset > result.size()) { throw std::runtime_error("iwork: snappy copy points outside the block"); } + check_fits(length); // the copy may overlap what it writes, so it runs byte by byte for (std::size_t i = 0, from = result.size() - offset; i < length; ++i) { result.push_back(result[from + i]); diff --git a/test/src/internal/iwork/iwork_snappy_test.cpp b/test/src/internal/iwork/iwork_snappy_test.cpp index 1ceaf495..6a69e31f 100644 --- a/test/src/internal/iwork/iwork_snappy_test.cpp +++ b/test/src/internal/iwork/iwork_snappy_test.cpp @@ -1,6 +1,8 @@ #include +#include #include +#include #include @@ -88,6 +90,14 @@ TEST(SnappyDecompressBlock, copy_points_outside_the_block) { EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(block(9, body))); } +// The declared length is what the file claims, not what it holds: a block +// claiming 4 GiB and carrying one literal byte is rejected off its tags rather +// than after allocating for the claim. +TEST(SnappyDecompressBlock, declared_length_is_not_trusted) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(std::string{ + '\xff', '\xff', '\xff', '\xff', '\x0f', '\x00'})); +} + TEST(SnappyDecompressBlock, length_varint_does_not_terminate) { EXPECT_ANY_THROW(std::ignore = snappy_decompress_block("\x80\x80\x80")); } From 7f3fc670226d2c1aab329f674fe7468d9f75b35a Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:18:11 +0200 Subject: [PATCH 11/14] test(iwork): build packages inline instead of only through fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both `.pages` fixtures are ASCII with no line break, so the UTF-16 index translation, its three throws and the `U+2028` branch were reachable but never run — `unit += length == 4 ? 2 : 1` could be dropped with the suite green. Nothing reached `Package` or `read_iwa`'s error paths either, and nothing pinned the `.numbers`/`.key` fallback both `PLAN.md` and `CHANGELOG.md` promise. `iwork_test_util.hpp` assembles the layers a package is made of — protobuf fields, `TSP.ArchiveInfo`, a literal-only Snappy block, the component list — so a storage of any shape is stated inline over a `VirtualFilesystem`. It is a test-only assembler and must never grow into a writer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/iwork/AGENTS.md | 7 + .../src/internal/iwork/iwork_archive_test.cpp | 146 +++++++++---- test/src/internal/iwork/iwork_test_util.hpp | 197 ++++++++++++++++++ test/src/internal/iwork/pages_test.cpp | 87 ++++++++ 4 files changed, 391 insertions(+), 46 deletions(-) create mode 100644 test/src/internal/iwork/iwork_test_util.hpp diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md index 977c06d8..8076cb1c 100644 --- a/src/odr/internal/iwork/AGENTS.md +++ b/src/odr/internal/iwork/AGENTS.md @@ -20,6 +20,13 @@ cited to the fixture it was read off. Read `numbers-parser`, `keynote-parser`, `obriensp/iWorkFileFormat` and `libetonyek` for facts; **copy code from none of them**. +A fixture is the citation for what the format *is*; it is not the only way to +state a test input. Shapes no fixture holds — a surrogate pair the run table +counts across, a component the list names but the package does not carry — are +built inline by `test/src/internal/iwork/iwork_test_util.hpp`, which assembles +the protobuf, archive and Snappy layers a package is made of. It is a test-only +assembler and must never grow into a writer. + **Fail soft on a type id we have not mapped, fail fast on broken framing.** The root `AGENTS.md` says to throw where the spec dictates what to expect. Here there is no spec, and an unknown type id means Apple shipped a version we have diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp index 192e02cb..ac6ceabd 100644 --- a/test/src/internal/iwork/iwork_archive_test.cpp +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -1,59 +1,22 @@ #include -#include +#include +#include + +#include + #include #include -#include #include #include using namespace odr::internal::iwork; +namespace builder = odr::test::iwork; -namespace { - -std::string varint(std::uint64_t value) { - std::string result; - for (;;) { - const auto byte = static_cast(value & 0x7f); - value >>= 7; - result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); - if (value == 0) { - return result; - } - } -} - -std::string number_field(const std::uint32_t number, - const std::uint64_t value) { - return varint(number << 3) + varint(value); -} - -std::string message_field(const std::uint32_t number, - const std::string &bytes) { - return varint((number << 3) | 2) + varint(bytes.size()) + bytes; -} - -/// `TSP.ArchiveInfo`: an identifier and one `MessageInfo` per payload message. -std::string archive_info( - const std::uint64_t identifier, - const std::vector> &messages) { - std::string result = number_field(1, identifier); - for (const auto &[type, length] : messages) { - result += message_field(2, number_field(1, type) + number_field(3, length)); - } - return result; -} - -std::string -object(const std::uint64_t identifier, - const std::vector> &messages, - const std::string &payload) { - const std::string info = archive_info(identifier, messages); - return varint(info.size()) + info + payload; -} - -} // namespace +using builder::message_field; +using builder::number_field; +using builder::object; TEST(ReadObjects, one_object) { const std::string data = object(1, {{10000, 5}}, "hello"); @@ -118,3 +81,94 @@ TEST(ReadObjects, payload_runs_past_the_component) { const std::string data = object(1, {{10000, 500}}, "hello"); EXPECT_ANY_THROW(std::ignore = read_objects(data)); } + +TEST(ReadIwa, undoes_the_framing) { + const auto files = + builder::filesystem({{"/Index/Document.iwa", builder::iwa("hello")}}); + + EXPECT_EQ(read_iwa(*files, odr::internal::AbsPath("/Index/Document.iwa")), + "hello"); +} + +TEST(ReadIwa, missing_file) { + const auto files = builder::filesystem({}); + + EXPECT_ANY_THROW(std::ignore = read_iwa( + *files, odr::internal::AbsPath("/Index/Document.iwa"))); +} + +TEST(IworkPackage, loads_a_component_by_name) { + const auto files = + builder::package({{"Document", object(1, {{10000, 5}}, "hello")}}); + + Package package(*files); + const std::vector &objects = package.component("Document").objects(); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].payload, "hello"); + EXPECT_EQ(package.object(1).type, 10000); +} + +TEST(IworkPackage, missing_metadata) { + const auto files = builder::filesystem({}); + + EXPECT_ANY_THROW(Package{*files}); +} + +TEST(IworkPackage, empty_metadata) { + const auto files = + builder::filesystem({{"/Index/Metadata.iwa", builder::iwa("")}}); + + EXPECT_ANY_THROW(Package{*files}); +} + +// The component list is a repeated message; a varint where one belongs is a +// package that cannot be read rather than an entry to skip. +TEST(IworkPackage, malformed_component_info) { + const std::string list = + number_field(builder::package_metadata_components, 1); + const auto files = builder::filesystem( + {{"/Index/Metadata.iwa", + builder::iwa(object(2, {{builder::package_metadata_type, list.size()}}, + list))}}); + + EXPECT_ANY_THROW(Package{*files}); +} + +TEST(IworkPackage, component_without_a_name) { + const std::string list = + message_field(builder::package_metadata_components, + message_field(builder::component_info_locator, "Document")); + const auto files = builder::filesystem( + {{"/Index/Metadata.iwa", + builder::iwa(object(2, {{builder::package_metadata_type, list.size()}}, + list))}}); + + EXPECT_ANY_THROW(Package{*files}); +} + +TEST(IworkPackage, no_component_of_that_name) { + const auto files = + builder::package({{"Document", object(1, {{10000, 5}}, "hello")}}); + + Package package(*files); + EXPECT_ANY_THROW(package.component("Stylesheet")); +} + +TEST(IworkPackage, no_object_of_that_identifier) { + const auto files = + builder::package({{"Document", object(1, {{10000, 5}}, "hello")}}); + + Package package(*files); + EXPECT_ANY_THROW(package.object(1732514)); +} + +// A component the list names but the package does not hold is broken framing, +// not a component to pass over while looking for an object elsewhere. +TEST(IworkPackage, component_file_is_missing) { + const auto files = builder::filesystem( + {{"/Index/Metadata.iwa", + builder::iwa(builder::package_metadata({"Document"}))}}); + + Package package(*files); + EXPECT_ANY_THROW(package.component("Document")); +} diff --git a/test/src/internal/iwork/iwork_test_util.hpp b/test/src/internal/iwork/iwork_test_util.hpp new file mode 100644 index 00000000..a83c0dab --- /dev/null +++ b/test/src/internal/iwork/iwork_test_util.hpp @@ -0,0 +1,197 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/// Test-only assembler for the layers an iWork package is made of: protobuf +/// fields, `TSP.ArchiveInfo` framing, a Snappy block, and the component list +/// that names the files. The engine has no writer and is not getting one — +/// this exists so a parser test can state its input inline rather than needing +/// a fixture for every shape, and must never grow into a writer API. +/// +/// Field numbers are the ones `iwork_archive.cpp` reads, cited there to +/// `empty.pages Index/Metadata.iwa`. +namespace odr::test::iwork { + +/// `TSP.PackageMetadata`, the object the component list lives in. +constexpr std::uint32_t package_metadata_type = 11006; +constexpr std::uint32_t package_metadata_components = 3; +constexpr std::uint32_t component_info_preferred_locator = 2; +constexpr std::uint32_t component_info_locator = 3; + +inline std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +inline std::string number_field(const std::uint32_t number, + const std::uint64_t value) { + return varint(number << 3) + varint(value); +} + +inline std::string message_field(const std::uint32_t number, + const std::string &bytes) { + return varint((number << 3) | 2) + varint(bytes.size()) + bytes; +} + +/// `TSP.ArchiveInfo`: an identifier and one `MessageInfo` per payload message. +inline std::string archive_info( + const std::uint64_t identifier, + const std::vector> &messages) { + std::string result = number_field(1, identifier); + for (const auto &[type, length] : messages) { + result += message_field(2, number_field(1, type) + number_field(3, length)); + } + return result; +} + +/// One archived object: its info, length-prefixed, then @p payload. +inline std::string +object(const std::uint64_t identifier, + const std::vector> &messages, + const std::string &payload) { + const std::string info = archive_info(identifier, messages); + return varint(info.size()) + info + payload; +} + +/// One Snappy block holding @p data as a single literal — the compressor the +/// engine never needs, in the shape `snappy_decompress_block` reads. +inline std::string snappy_block(const std::string &data) { + std::string result = varint(data.size()); + if (data.empty()) { + return result; + } + + // a literal tag carries its length inline up to 60 bytes, in the bytes + // after it beyond that + const std::size_t length = data.size() - 1; + if (length < 60) { + result.push_back(static_cast(length << 2)); + } else { + std::string bytes; + for (std::size_t rest = length; rest != 0; rest >>= 8) { + bytes.push_back(static_cast(rest & 0xff)); + } + result.push_back(static_cast((59 + bytes.size()) << 2)); + result += bytes; + } + return result + data; +} + +/// @p data as one `.iwa` file: a single block behind Apple's framing. +inline std::string iwa(const std::string &data) { + const std::string block = snappy_block(data); + const std::size_t length = block.size(); + const std::string header{'\0', static_cast(length & 0xff), + static_cast((length >> 8) & 0xff), + static_cast((length >> 16) & 0xff)}; + return header + block; +} + +/// A filesystem holding @p files as they are given. +inline std::shared_ptr +filesystem(const std::vector> &files) { + auto result = std::make_shared(); + for (const auto &[path, data] : files) { + result->copy(std::make_shared(data), + internal::AbsPath(path)); + } + return result; +} + +/// The component list `Index/Metadata.iwa` carries, naming @p locators. A +/// component's name is its locator here — the fixtures are where the two +/// differ. +inline std::string package_metadata(const std::vector &locators) { + std::string list; + for (const std::string &locator : locators) { + list += + message_field(package_metadata_components, + message_field(component_info_preferred_locator, locator) + + message_field(component_info_locator, locator)); + } + return object(2, {{package_metadata_type, list.size()}}, list); +} + +/// A package: one `/Index/.iwa` per component of @p components, plus +/// the `Index/Metadata.iwa` naming them. +inline std::shared_ptr +package(const std::vector> &components) { + std::vector locators; + std::vector> files; + for (const auto &[locator, data] : components) { + locators.push_back(locator); + files.emplace_back("/Index/" + locator + ".iwa", iwa(data)); + } + + files.emplace_back("/Index/Metadata.iwa", iwa(package_metadata(locators))); + return filesystem(files); +} + +/// A `TP.DocumentArchive` whose body is the object @p body_identifier. +inline std::string document_archive(const std::uint64_t body_identifier) { + return message_field( + ::odr::internal::iwork::document_archive::body_storage, + number_field(::odr::internal::iwork::reference::identifier, + body_identifier)); +} + +/// A `TSWP.StorageArchive` holding @p text, with a paragraph style run table +/// over the UTF-16 code unit indices @p paragraphs. `std::nullopt` writes no +/// table at all, which is not the same as an empty one. +inline std::string +text_storage(const std::string &text, + const std::optional> ¶graphs) { + namespace types = ::odr::internal::iwork; + + std::string result = message_field(types::text_storage::text, text); + if (paragraphs.has_value()) { + std::string table; + for (const std::uint64_t index : *paragraphs) { + table += message_field( + types::attribute_table::entries, + number_field(types::attribute_table_entry::character_index, index)); + } + result += message_field(types::text_storage::paragraph_styles, table); + } + return result; +} + +/// The object a synthetic package's body storage is filed under. +constexpr std::uint64_t body_identifier = 5; + +/// The one-component package a `.pages` is: a root archive of @p root_type +/// whose body storage is @p storage. +inline std::shared_ptr +pages_package(const std::string &storage, + const std::uint32_t root_type = + ::odr::internal::iwork::archive_type::pages_document) { + const std::string root = document_archive(body_identifier); + const std::string document = + object(1, {{root_type, root.size()}}, root) + + object(body_identifier, + {{::odr::internal::iwork::archive_type::text_storage, + storage.size()}}, + storage); + return package({{"Document", document}}); +} + +} // namespace odr::test::iwork diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp index 820dfc95..42337c84 100644 --- a/test/src/internal/iwork/pages_test.cpp +++ b/test/src/internal/iwork/pages_test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -8,19 +9,25 @@ #include #include #include +#include +#include #include +#include #include #include #include +#include #include +#include #include #include using namespace odr; using odr::test::TestData; +namespace builder = odr::test::iwork; namespace { @@ -45,6 +52,15 @@ std::vector paragraphs(const Element root) { return result; } +/// The document a synthetic one-component package decodes to, so a shape the +/// fixtures do not hold can be stated inline. +Document pages_document( + const std::string &text, + const std::optional> ¶graph_indices) { + return Document(std::make_shared( + builder::pages_package(builder::text_storage(text, paragraph_indices)))); +} + } // namespace TEST(Iwork, pages_is_detected_by_content) { @@ -125,3 +141,74 @@ TEST(Iwork, package_resolves_across_components) { // the root of a `Tables/DataList` that is not the first one of that name EXPECT_EQ(package.object(1732940).identifier, 1732940); } + +// Run tables count in UTF-16 code units while the text is UTF-8. A character +// outside the basic multilingual plane is two units but four bytes, so a +// paragraph starting after one lands mid-text if the two are conflated. +TEST(Iwork, pages_paragraph_starts_after_a_surrogate_pair) { + const Document document = + pages_document("a\xf0\x9f\x98\x80\nbcd\n", {{0, 4}}); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{"a\xf0\x9f\x98\x80", "bcd"})); +} + +TEST(Iwork, pages_run_table_points_past_the_text) { + EXPECT_ANY_THROW(std::ignore = pages_document("abc", {{0, 9}})); +} + +TEST(Iwork, pages_text_ends_mid_character) { + EXPECT_ANY_THROW(std::ignore = pages_document("a\xe2\x80", {{0, 3}})); +} + +TEST(Iwork, pages_text_is_not_utf8) { + EXPECT_ANY_THROW(std::ignore = pages_document("\x80x", {{0, 1}})); +} + +// `U+2028` breaks a line inside a paragraph rather than starting a new one — +// the run table is what says where a paragraph begins. +TEST(Iwork, pages_line_separator_breaks_a_line_inside_a_paragraph) { + const Document document = pages_document("one\xe2\x80\xa8two\n", {{0}}); + + const Element root = document.root_element(); + ASSERT_EQ(paragraphs(root), (std::vector{"one\ntwo"})); + + std::vector types; + for (const Element child : (*root.children().begin()).children()) { + types.push_back(child.type()); + } + EXPECT_EQ(types, (std::vector{ElementType::text, + ElementType::line_break, + ElementType::text})); +} + +TEST(Iwork, pages_without_a_paragraph_style_table) { + const Document document = pages_document("only\n", std::nullopt); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{"only"})); +} + +TEST(Iwork, pages_with_an_empty_paragraph_style_table) { + const Document document = + pages_document("only\n", std::vector{}); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{"only"})); +} + +// Which app wrote a package comes off its root archive type, and only +// `.pages` is mapped: a `.numbers` or a `.key` falls back to being reported as +// the zip it is rather than guessed at from its extension. +TEST(Iwork, unmapped_root_archive_is_not_an_iwork_file) { + const auto files = + builder::pages_package(builder::text_storage("", std::nullopt), 10001); + + EXPECT_THROW(internal::iwork::IworkFile{files}, NoIworkFile); +} + +TEST(Iwork, package_without_a_document_component_is_not_an_iwork_file) { + const auto files = builder::filesystem({}); + + EXPECT_THROW(internal::iwork::IworkFile{files}, NoIworkFile); +} From f4dad4725cb9f25f00d132c8bbdc216b64fcff99 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:28:21 +0200 Subject: [PATCH 12/14] refactor(iwork): read bytes and utf-16 indices through the utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `iwork_snappy.cpp` and `iwork_protobuf.cpp` each carried the same variable-width little-endian accumulation loop; `util::byte::from_little_endian` now takes a size for a field narrower than the type it is read into. `iwork_parser.cpp` decoded UTF-8 by hand to count UTF-16 code units, which utfcpp — already a dependency behind `util::string` — does properly. `util::string::utf16_offsets` is that walk, with its own tests. Also cuts the doc comments in `iwork_archive.hpp` and the ones the last three commits added back to a line or two. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/iwork/AGENTS.md | 24 ++++---- src/odr/internal/iwork/iwork_archive.hpp | 44 +++++--------- src/odr/internal/iwork/iwork_parser.cpp | 58 +------------------ src/odr/internal/iwork/iwork_protobuf.cpp | 11 ++-- src/odr/internal/iwork/iwork_snappy.cpp | 22 +++---- src/odr/internal/util/byte_util.hpp | 14 +++++ src/odr/internal/util/string_util.cpp | 29 ++++++++++ src/odr/internal/util/string_util.hpp | 8 +++ .../src/internal/iwork/iwork_archive_test.cpp | 6 +- test/src/internal/iwork/iwork_snappy_test.cpp | 5 +- test/src/internal/iwork/iwork_test_util.hpp | 39 +++++-------- test/src/internal/iwork/pages_test.cpp | 16 ++--- test/src/internal/util/string_util_test.cpp | 23 ++++++++ 13 files changed, 140 insertions(+), 159 deletions(-) diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md index 8076cb1c..0378dca6 100644 --- a/src/odr/internal/iwork/AGENTS.md +++ b/src/odr/internal/iwork/AGENTS.md @@ -20,12 +20,10 @@ cited to the fixture it was read off. Read `numbers-parser`, `keynote-parser`, `obriensp/iWorkFileFormat` and `libetonyek` for facts; **copy code from none of them**. -A fixture is the citation for what the format *is*; it is not the only way to -state a test input. Shapes no fixture holds — a surrogate pair the run table -counts across, a component the list names but the package does not carry — are -built inline by `test/src/internal/iwork/iwork_test_util.hpp`, which assembles -the protobuf, archive and Snappy layers a package is made of. It is a test-only -assembler and must never grow into a writer. +A fixture is the citation for what the format *is*, not the only way to state +a test input. Shapes no fixture holds are built inline by +`test/src/internal/iwork/iwork_test_util.hpp`, which assembles the protobuf, +archive and Snappy layers. It is test-only and must never grow into a writer. **Fail soft on a type id we have not mapped, fail fast on broken framing.** The root `AGENTS.md` says to throw where the spec dictates what to expect. Here @@ -35,10 +33,10 @@ throw: framing that overruns the file, a Snappy block that does not fill its declared length, a varint that does not terminate, an identifier the package does not hold, and text that is not UTF-8. -A declared length is the file's word rather than a fact, so nothing is -allocated or written against one before it is known to fit: -`snappy_decompress_block` caps its reservation at what the compressed bytes -could expand to and checks every tag against what the block has left to give. +A declared length is the file's word, so nothing is allocated or written +against one before it is known to fit: `snappy_decompress_block` caps its +reservation at what the compressed bytes could expand to and checks every tag +against what the block has left. ## No new dependencies @@ -102,9 +100,9 @@ every `\n` in the text. The two agree on both fixtures, but the table is what says so, and `U+2028` is a line break *inside* a paragraph rather than a paragraph boundary. -Run tables count in **UTF-16 code units** while the text is UTF-8, so -`iwork_parser.cpp` translates the indices in one pass over the text. An index -that lands mid-character is an error, not a rounding. +Run tables count in **UTF-16 code units** while the text is UTF-8; +`util::string::utf16_offsets` translates the indices in one pass. An index that +lands mid-character is an error, not a rounding. `U+FFFC` is where a drawable is anchored. Nothing reads drawables yet, so the anchor is dropped rather than rendered as a glyph — see stage 4. diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp index b8363edc..a12c4775 100644 --- a/src/odr/internal/iwork/iwork_archive.hpp +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -18,28 +18,22 @@ class ReadableFilesystem; namespace odr::internal::iwork { -/// One archived object. `TSP.ArchiveInfo` names its identifier (field 1) and, -/// per payload message, a `MessageInfo` (field 2) carrying the message type -/// and its length. An object usually holds one message; where it holds more, -/// only the first is modelled. -/// -/// Verified on `empty.pages Index/Document.iwa +0`: `08 01` (identifier 1), -/// `12 52` (an 82-byte `MessageInfo`), `08 90 4e` (type 10000), `18 e0 0c` -/// (payload length 1632). +/// One object of an `.iwa`, per `TSP.ArchiveInfo`. Where it holds more than +/// one message, only the first is modelled. struct Object final { std::uint64_t identifier{}; std::uint32_t type{}; std::string_view payload; }; -/// The objects of one `.iwa`, over the bytes it decompressed to. Object -/// payloads are views into those bytes. +/// The objects of one `.iwa`, over the bytes it decompressed to; payloads are +/// views into those bytes. class Component final { public: Component(std::string locator, std::string data); - /// The file the component was loaded from, without `/Index/` and `.iwa`. - /// Unlike its name, this is unique across the package. + /// The file it was loaded from, without `/Index/` and `.iwa`. Unlike its + /// name, this is unique across the package. [[nodiscard]] const std::string &locator() const noexcept; [[nodiscard]] const std::vector &objects() const noexcept; @@ -49,29 +43,23 @@ class Component final { std::vector m_objects; }; -/// An iWork package: the component list from `Index/Metadata.iwa`, and the -/// components loaded from it so far. -/// -/// Objects reference each other by identifier across components, so the list -/// is read first and a component is decompressed when something in it is -/// asked for. +/// An iWork package. Objects reference each other across components, so the +/// list from `Index/Metadata.iwa` is read first and a component decompressed +/// when something in it is asked for. class Package final { public: explicit Package(const abstract::ReadableFilesystem &filesystem); - /// The first component named @p name in the package's component list — a - /// name is not unique, `Tables/DataList` names dozens. Throws when the - /// package holds none. + /// The first component named @p name — a name is not unique. Throws when + /// the package holds none. const Component &component(const std::string &name); - /// The object @p identifier names, loading components until it is found. - /// Throws when no component holds it. + /// The object @p identifier names, loading components until it turns up. const Object &object(std::uint64_t identifier); private: - /// One entry of `TSP.PackageMetadata`'s component list: the name a component - /// is known by, and the file it lives in — which carries an identifier - /// suffix often enough that the file name is not a way to find it. + /// One entry of the component list: the name a component is known by, and + /// the file it lives in. struct ComponentInfo final { std::string name; std::string locator; @@ -85,11 +73,11 @@ class Package final { const Component &load_(const ComponentInfo &info); }; -/// Reads @p path off @p filesystem and undoes its `.iwa` framing. +/// Reads @p path and undoes its `.iwa` framing. std::string read_iwa(const abstract::ReadableFilesystem &filesystem, const AbsPath &path); -/// Splits a decompressed `.iwa` into its objects, over @p data. +/// Splits a decompressed `.iwa` into its objects, viewing @p data. std::vector read_objects(std::string_view data); } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp index 0a0942e0..10f8ac5b 100644 --- a/src/odr/internal/iwork/iwork_parser.cpp +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -24,58 +24,6 @@ constexpr std::string_view line_separator = "\xe2\x80\xa8"; /// rendered as a glyph. constexpr std::string_view object_replacement = "\xef\xbf\xbc"; -/// The byte length of the UTF-8 sequence @p lead starts. -std::size_t utf8_length(const std::uint8_t lead) { - if (lead < 0x80) { - return 1; - } - if ((lead & 0xe0) == 0xc0) { - return 2; - } - if ((lead & 0xf0) == 0xe0) { - return 3; - } - if ((lead & 0xf8) == 0xf0) { - return 4; - } - throw std::runtime_error("iwork: text is not utf-8"); -} - -/// Translates the ascending UTF-16 code unit @p indices a storage's run tables -/// count in into byte offsets into @p text. -std::vector -utf16_offsets(const std::string_view text, - const std::vector &indices) { - std::vector result; - result.reserve(indices.size()); - - std::size_t offset = 0; - std::uint64_t unit = 0; - auto next = indices.begin(); - - for (;;) { - while (next != indices.end() && *next == unit) { - result.push_back(offset); - ++next; - } - if (next == indices.end()) { - return result; - } - if (offset >= text.size()) { - throw std::runtime_error("iwork: run table points past the text"); - } - - const std::size_t length = - utf8_length(static_cast(text[offset])); - if (offset + length > text.size()) { - throw std::runtime_error("iwork: text ends mid-character"); - } - offset += length; - // everything outside the basic multilingual plane is a surrogate pair - unit += length == 4 ? 2 : 1; - } -} - /// The character index each paragraph of @p storage starts at. Paragraph /// boundaries are the run table's rather than every `\n` in the text — the two /// agree today, but the table is what says so. @@ -97,8 +45,8 @@ std::vector paragraph_starts(const iwork::Message &storage) { } } - // no table at all, or an empty one as an empty document carries: either way - // the body starts at its first character + // no table, or an empty one: either way the body starts at its first + // character if (result.empty() || result.front() != 0) { result.insert(result.begin(), 0); } @@ -175,7 +123,7 @@ iwork::parse_pages_tree(ElementRegistry ®istry, } const std::vector starts = - utf16_offsets(text, paragraph_starts(storage)); + util::string::utf16_offsets(text, paragraph_starts(storage)); auto [root_id, root] = registry.create_element(ElementType::root); diff --git a/src/odr/internal/iwork/iwork_protobuf.cpp b/src/odr/internal/iwork/iwork_protobuf.cpp index df203b33..9f069bf1 100644 --- a/src/odr/internal/iwork/iwork_protobuf.cpp +++ b/src/odr/internal/iwork/iwork_protobuf.cpp @@ -1,5 +1,7 @@ #include +#include + #include namespace odr::internal { @@ -11,13 +13,8 @@ std::uint64_t read_fixed(const std::string_view in, std::size_t &position, if (position + size > in.size()) { throw std::runtime_error("iwork: protobuf fixed field is cut off"); } - - std::uint64_t result = 0; - for (std::size_t i = 0; i < size; ++i) { - result |= - static_cast(static_cast(in[position + i])) - << (8 * i); - } + const std::uint64_t result = + util::byte::from_little_endian(in.substr(position), size); position += size; return result; } diff --git a/src/odr/internal/iwork/iwork_snappy.cpp b/src/odr/internal/iwork/iwork_snappy.cpp index 027c6b49..74eb71ba 100644 --- a/src/odr/internal/iwork/iwork_snappy.cpp +++ b/src/odr/internal/iwork/iwork_snappy.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -15,14 +17,8 @@ std::uint32_t read_little_endian(const std::string_view in, if (position + size > in.size()) { throw std::runtime_error("iwork: snappy block ends mid-tag"); } - - std::uint32_t result = 0; - for (std::size_t i = 0; i < size; ++i) { - result |= - static_cast(static_cast(in[position + i])) - << (8 * i); - } - return result; + return util::byte::from_little_endian(in.substr(position), + size); } /// Reads the block's uncompressed length and advances @p position past it. @@ -43,9 +39,7 @@ std::uint32_t read_uncompressed_length(const std::string_view in, throw std::runtime_error("iwork: snappy length varint does not terminate"); } -/// The most a block can emit per compressed byte it holds: the tag that -/// writes the most for its size is a two-byte-offset copy, three bytes for at -/// most 64. Generous headroom over that, so no real block trips it. +/// Headroom over the real ceiling of a three-byte copy tag for 64 bytes. constexpr std::size_t max_expansion = 64; } // namespace @@ -55,14 +49,12 @@ std::string iwork::snappy_decompress_block(const std::string_view compressed) { const std::uint32_t uncompressed_length = read_uncompressed_length(compressed, position); + // the declared length is the file's word, so the allocation is capped by + // what the block could hold rather than by what it claims std::string result; - // the declared length is the file's word, not a fact, so the allocation is - // capped by what the block could hold rather than by what it claims result.reserve(std::min(uncompressed_length, max_expansion * compressed.size())); - // nothing is appended before it is known to fit, so a block never - // materialises more than it declared const auto check_fits = [&](const std::size_t length) { if (length > uncompressed_length - result.size()) { throw std::runtime_error("iwork: snappy block writes past its length"); diff --git a/src/odr/internal/util/byte_util.hpp b/src/odr/internal/util/byte_util.hpp index 2d3b4ebf..98cb1787 100644 --- a/src/odr/internal/util/byte_util.hpp +++ b/src/odr/internal/util/byte_util.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -57,6 +58,19 @@ O from_little_endian(const I &in) { return out; } +/// Reads @p size bytes rather than `sizeof(O)`, for a field narrower than the +/// type it is read into — a 24-bit length, a one-byte offset. +template +O from_little_endian(const I &in, const std::size_t size) { + assert(size <= sizeof(O) && "output type too small for input size"); + assert(std::ranges::size(in) >= size && "input range too small"); + O out{}; + for (std::size_t i = 0; i < size; ++i) { + out |= (static_cast(in[i]) & 0xff) << (i * 8); + } + return out; +} + template void from_big_endian(const I &in, O &out) { assert(std::ranges::size(in) >= sizeof(out) && diff --git a/src/odr/internal/util/string_util.cpp b/src/odr/internal/util/string_util.cpp index 8d1109cf..9d4bf72b 100644 --- a/src/odr/internal/util/string_util.cpp +++ b/src/odr/internal/util/string_util.cpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace odr::internal::util { @@ -175,6 +176,34 @@ std::size_t string::utf8_length(const std::string &string) { return utf8::unchecked::distance(string.begin(), string.end()); } +std::vector +string::utf16_offsets(const std::string_view string, + const std::vector &indices) { + std::vector result; + result.reserve(indices.size()); + + const char *const begin = string.data(); + const char *const end = begin + string.size(); + const char *position = begin; + std::uint64_t unit = 0; + auto next = indices.begin(); + + for (;;) { + while (next != indices.end() && *next == unit) { + result.push_back(static_cast(position - begin)); + ++next; + } + if (next == indices.end()) { + return result; + } + if (position == end) { + throw std::runtime_error("utf-16 index past the end of the string"); + } + // everything outside the basic multilingual plane is a surrogate pair + unit += utf8::next(position, end) >= 0x10000 ? 2 : 1; + } +} + std::string string::u16string_to_string(const std::u16string &string) { static constexpr char32_t replacement = 0xfffd; diff --git a/src/odr/internal/util/string_util.hpp b/src/odr/internal/util/string_util.hpp index b9889dc8..a4907c5c 100644 --- a/src/odr/internal/util/string_util.hpp +++ b/src/odr/internal/util/string_util.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -66,6 +67,13 @@ std::string to_string(double d, int precision); std::size_t utf8_length(const std::string &string); +/// The byte offsets into @p string of the ascending UTF-16 code unit +/// @p indices — the indexing a format that counts UTF-16 uses over UTF-8 text. +/// Throws when @p string is not UTF-8 or an index is past its end. +std::vector +utf16_offsets(std::string_view string, + const std::vector &indices); + /// A surrogate completing no pair becomes U+FFFD rather than throwing. std::string u16string_to_string(const std::u16string &string); std::u16string string_to_u16string(std::string_view string); diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp index ac6ceabd..3efe1d42 100644 --- a/test/src/internal/iwork/iwork_archive_test.cpp +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -121,8 +121,7 @@ TEST(IworkPackage, empty_metadata) { EXPECT_ANY_THROW(Package{*files}); } -// The component list is a repeated message; a varint where one belongs is a -// package that cannot be read rather than an entry to skip. +// A varint where a component info belongs is a package we cannot read. TEST(IworkPackage, malformed_component_info) { const std::string list = number_field(builder::package_metadata_components, 1); @@ -162,8 +161,7 @@ TEST(IworkPackage, no_object_of_that_identifier) { EXPECT_ANY_THROW(package.object(1732514)); } -// A component the list names but the package does not hold is broken framing, -// not a component to pass over while looking for an object elsewhere. +// A component the list names but the package does not hold is broken framing. TEST(IworkPackage, component_file_is_missing) { const auto files = builder::filesystem( {{"/Index/Metadata.iwa", diff --git a/test/src/internal/iwork/iwork_snappy_test.cpp b/test/src/internal/iwork/iwork_snappy_test.cpp index 6a69e31f..62699e09 100644 --- a/test/src/internal/iwork/iwork_snappy_test.cpp +++ b/test/src/internal/iwork/iwork_snappy_test.cpp @@ -90,9 +90,8 @@ TEST(SnappyDecompressBlock, copy_points_outside_the_block) { EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(block(9, body))); } -// The declared length is what the file claims, not what it holds: a block -// claiming 4 GiB and carrying one literal byte is rejected off its tags rather -// than after allocating for the claim. +// A block claiming 4 GiB and carrying one literal byte is rejected off its +// tags, not after allocating for the claim. TEST(SnappyDecompressBlock, declared_length_is_not_trusted) { EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(std::string{ '\xff', '\xff', '\xff', '\xff', '\x0f', '\x00'})); diff --git a/test/src/internal/iwork/iwork_test_util.hpp b/test/src/internal/iwork/iwork_test_util.hpp index a83c0dab..877ffb10 100644 --- a/test/src/internal/iwork/iwork_test_util.hpp +++ b/test/src/internal/iwork/iwork_test_util.hpp @@ -14,17 +14,12 @@ #include #include -/// Test-only assembler for the layers an iWork package is made of: protobuf -/// fields, `TSP.ArchiveInfo` framing, a Snappy block, and the component list -/// that names the files. The engine has no writer and is not getting one — -/// this exists so a parser test can state its input inline rather than needing -/// a fixture for every shape, and must never grow into a writer API. -/// -/// Field numbers are the ones `iwork_archive.cpp` reads, cited there to -/// `empty.pages Index/Metadata.iwa`. +/// Test-only assembler for the layers an iWork package is made of, so a test +/// can state its input inline instead of needing a fixture. Field numbers are +/// the ones `iwork_archive.cpp` reads. Must never grow into a writer API. namespace odr::test::iwork { -/// `TSP.PackageMetadata`, the object the component list lives in. +/// The object the component list lives in. constexpr std::uint32_t package_metadata_type = 11006; constexpr std::uint32_t package_metadata_components = 3; constexpr std::uint32_t component_info_preferred_locator = 2; @@ -72,16 +67,14 @@ object(const std::uint64_t identifier, return varint(info.size()) + info + payload; } -/// One Snappy block holding @p data as a single literal — the compressor the -/// engine never needs, in the shape `snappy_decompress_block` reads. +/// One Snappy block holding @p data as a single literal. inline std::string snappy_block(const std::string &data) { std::string result = varint(data.size()); if (data.empty()) { return result; } - // a literal tag carries its length inline up to 60 bytes, in the bytes - // after it beyond that + // a literal tag carries its length inline up to 60 bytes const std::size_t length = data.size() - 1; if (length < 60) { result.push_back(static_cast(length << 2)); @@ -117,9 +110,8 @@ filesystem(const std::vector> &files) { return result; } -/// The component list `Index/Metadata.iwa` carries, naming @p locators. A -/// component's name is its locator here — the fixtures are where the two -/// differ. +/// The component list `Index/Metadata.iwa` carries. A component's name is its +/// locator here; the fixtures are where the two differ. inline std::string package_metadata(const std::vector &locators) { std::string list; for (const std::string &locator : locators) { @@ -131,8 +123,7 @@ inline std::string package_metadata(const std::vector &locators) { return object(2, {{package_metadata_type, list.size()}}, list); } -/// A package: one `/Index/.iwa` per component of @p components, plus -/// the `Index/Metadata.iwa` naming them. +/// One `/Index/.iwa` per component, plus the metadata naming them. inline std::shared_ptr package(const std::vector> &components) { std::vector locators; @@ -154,9 +145,9 @@ inline std::string document_archive(const std::uint64_t body_identifier) { body_identifier)); } -/// A `TSWP.StorageArchive` holding @p text, with a paragraph style run table -/// over the UTF-16 code unit indices @p paragraphs. `std::nullopt` writes no -/// table at all, which is not the same as an empty one. +/// A `TSWP.StorageArchive`: @p text and a paragraph style run table over the +/// UTF-16 code unit indices @p paragraphs. `std::nullopt` writes no table, +/// which is not an empty one. inline std::string text_storage(const std::string &text, const std::optional> ¶graphs) { @@ -175,11 +166,11 @@ text_storage(const std::string &text, return result; } -/// The object a synthetic package's body storage is filed under. +/// The identifier a synthetic body storage is filed under. constexpr std::uint64_t body_identifier = 5; -/// The one-component package a `.pages` is: a root archive of @p root_type -/// whose body storage is @p storage. +/// A one-component package: a root archive of @p root_type whose body is +/// @p storage. inline std::shared_ptr pages_package(const std::string &storage, const std::uint32_t root_type = diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp index 42337c84..ac93d2b5 100644 --- a/test/src/internal/iwork/pages_test.cpp +++ b/test/src/internal/iwork/pages_test.cpp @@ -52,8 +52,7 @@ std::vector paragraphs(const Element root) { return result; } -/// The document a synthetic one-component package decodes to, so a shape the -/// fixtures do not hold can be stated inline. +/// The document a synthetic package decodes to. Document pages_document( const std::string &text, const std::optional> ¶graph_indices) { @@ -142,9 +141,8 @@ TEST(Iwork, package_resolves_across_components) { EXPECT_EQ(package.object(1732940).identifier, 1732940); } -// Run tables count in UTF-16 code units while the text is UTF-8. A character -// outside the basic multilingual plane is two units but four bytes, so a -// paragraph starting after one lands mid-text if the two are conflated. +// Run tables count UTF-16 code units over UTF-8 text: a character outside the +// basic multilingual plane is two units but four bytes. TEST(Iwork, pages_paragraph_starts_after_a_surrogate_pair) { const Document document = pages_document("a\xf0\x9f\x98\x80\nbcd\n", {{0, 4}}); @@ -165,8 +163,7 @@ TEST(Iwork, pages_text_is_not_utf8) { EXPECT_ANY_THROW(std::ignore = pages_document("\x80x", {{0, 1}})); } -// `U+2028` breaks a line inside a paragraph rather than starting a new one — -// the run table is what says where a paragraph begins. +// `U+2028` breaks a line inside a paragraph rather than starting a new one. TEST(Iwork, pages_line_separator_breaks_a_line_inside_a_paragraph) { const Document document = pages_document("one\xe2\x80\xa8two\n", {{0}}); @@ -197,9 +194,8 @@ TEST(Iwork, pages_with_an_empty_paragraph_style_table) { (std::vector{"only"})); } -// Which app wrote a package comes off its root archive type, and only -// `.pages` is mapped: a `.numbers` or a `.key` falls back to being reported as -// the zip it is rather than guessed at from its extension. +// Only `.pages` is mapped, so a `.numbers` or `.key` falls back to the zip it +// is rather than being guessed at from its extension. TEST(Iwork, unmapped_root_archive_is_not_an_iwork_file) { const auto files = builder::pages_package(builder::text_storage("", std::nullopt), 10001); diff --git a/test/src/internal/util/string_util_test.cpp b/test/src/internal/util/string_util_test.cpp index 3ac2ea26..f7ae4386 100644 --- a/test/src/internal/util/string_util_test.cpp +++ b/test/src/internal/util/string_util_test.cpp @@ -2,6 +2,10 @@ #include +#include +#include +#include + using namespace odr::internal::util::string; TEST(string_util, split) { @@ -173,6 +177,25 @@ TEST(string_util, find_ignore_case) { EXPECT_EQ(find_ignore_case("abc", "a", 99), std::string_view::npos); } +TEST(string_util, utf16_offsets) { + // a character outside the basic multilingual plane is two units, four bytes + EXPECT_EQ(utf16_offsets("a\xf0\x9f\x98\x80z", {0, 1, 3, 4}), + (std::vector{0, 1, 5, 6})); + // and one inside it is one unit, up to three bytes + EXPECT_EQ(utf16_offsets("\xe2\x80\xa8" + "b", + {1}), + (std::vector{3})); + EXPECT_EQ(utf16_offsets("abc", {}), (std::vector{})); + EXPECT_EQ(utf16_offsets("", {0}), (std::vector{0})); + + EXPECT_ANY_THROW(std::ignore = utf16_offsets("abc", {4})); + EXPECT_ANY_THROW(std::ignore = utf16_offsets("\x80", {1})); + EXPECT_ANY_THROW(std::ignore = utf16_offsets("a\xe2\x80", {2})); + // an index landing inside a character is never reached + EXPECT_ANY_THROW(std::ignore = utf16_offsets("\xf0\x9f\x98\x80", {1})); +} + TEST(string_util, u16string_to_string) { EXPECT_EQ(u16string_to_string(u"abc"), "abc"); EXPECT_EQ(u16string_to_string(u"\U0001f600"), "\xf0\x9f\x98\x80"); From 79587bb85d77dba05e80eaf23ab0c7082f46f125 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:38:46 +0200 Subject: [PATCH 13/14] style(iwork): define members in their namespace, not by qualified name `iwork::Component::locator`, `iwork::IworkFile::document`, `iwork::Message::field` and the rest are now defined inside `namespace odr::internal::iwork` like every sibling engine; only the free functions keep the qualified form the root `AGENTS.md` asks for. That drops the `iwork::` prefix off the types they use, and off the test helpers via a namespace alias. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/iwork/AGENTS.md | 2 +- src/odr/internal/iwork/iwork_archive.cpp | 138 +++++++++--------- src/odr/internal/iwork/iwork_file.cpp | 32 ++-- src/odr/internal/iwork/iwork_parser.cpp | 23 +-- src/odr/internal/iwork/iwork_protobuf.cpp | 54 +++---- .../src/internal/iwork/iwork_archive_test.cpp | 8 +- test/src/internal/iwork/iwork_test_util.hpp | 21 ++- test/src/internal/iwork/pages_test.cpp | 18 +-- 8 files changed, 145 insertions(+), 151 deletions(-) diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md index 0378dca6..6b0b9b68 100644 --- a/src/odr/internal/iwork/AGENTS.md +++ b/src/odr/internal/iwork/AGENTS.md @@ -55,7 +55,7 @@ one user has not earned a package. ## `Message` views the buffer it was read from -`iwork::Message` parses one level eagerly and leaves nested messages, strings +`Message` parses one level eagerly and leaves nested messages, strings and packed fields as `std::string_view`s into the bytes it was handed. So the buffer has to outlive it — `Component` owns its decompressed data behind a `unique_ptr` for exactly that reason, and a `Message(some_temporary())` is a diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp index b9b94b05..f53a888d 100644 --- a/src/odr/internal/iwork/iwork_archive.cpp +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -12,7 +12,7 @@ #include #include -namespace odr::internal { +namespace odr::internal::iwork { namespace { @@ -34,78 +34,18 @@ AbsPath component_path(const std::string &locator) { } // namespace -std::string iwork::read_iwa(const abstract::ReadableFilesystem &filesystem, - const AbsPath &path) { - const std::shared_ptr file = filesystem.open(path); - if (!file) { - throw std::runtime_error("iwork: missing " + path.string()); - } - const std::unique_ptr stream = file->stream(); - return iwa_decompress(util::stream::read(*stream)); -} - -std::vector iwork::read_objects(const std::string_view data) { - std::vector result; - - std::size_t position = 0; - while (position < data.size()) { - const std::uint64_t info_length = read_varint(data, position); - if (info_length > data.size() - position) { - throw std::runtime_error("iwork: archive info runs past the component"); - } - const Message info(data.substr(position, info_length)); - position += info_length; - - Object object; - object.identifier = info.number_field(archive_info_identifier).value_or(0); - - // the payload holds every message the info names, back to back; only the - // first is modelled, the length of the rest is what skips them - std::size_t payload_length = 0; - std::size_t first_length = 0; - bool first = true; - for (const Field &message : info.repeated_field(archive_info_messages)) { - if (message.type != WireType::length_delimited) { - throw std::runtime_error("iwork: malformed message info"); - } - const Message message_info(message.bytes); - const std::uint64_t length = - message_info.number_field(message_info_length).value_or(0); - if (first) { - object.type = static_cast( - message_info.number_field(message_info_type).value_or(0)); - first_length = length; - first = false; - } - payload_length += length; - } - - if (payload_length > data.size() - position) { - throw std::runtime_error("iwork: object payload runs past the component"); - } - object.payload = data.substr(position, first_length); - position += payload_length; - - result.push_back(object); - } - - return result; -} - -iwork::Component::Component(std::string locator, std::string data) +Component::Component(std::string locator, std::string data) : m_locator{std::move(locator)}, m_data{std::make_unique(std::move(data))}, m_objects{read_objects(*m_data)} {} -const std::string &iwork::Component::locator() const noexcept { - return m_locator; -} +const std::string &Component::locator() const noexcept { return m_locator; } -const std::vector &iwork::Component::objects() const noexcept { +const std::vector &Component::objects() const noexcept { return m_objects; } -iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) +Package::Package(const abstract::ReadableFilesystem &filesystem) : m_filesystem{&filesystem} { const std::string data = read_iwa(filesystem, AbsPath("/Index/Metadata.iwa")); const std::vector objects = read_objects(data); @@ -133,7 +73,7 @@ iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) } } -const iwork::Component &iwork::Package::component(const std::string &name) { +const Component &Package::component(const std::string &name) { const auto it = std::ranges::find(m_component_infos, name, &ComponentInfo::name); if (it == std::ranges::end(m_component_infos)) { @@ -142,7 +82,7 @@ const iwork::Component &iwork::Package::component(const std::string &name) { return load_(*it); } -const iwork::Object &iwork::Package::object(const std::uint64_t identifier) { +const Object &Package::object(const std::uint64_t identifier) { if (const auto it = m_objects.find(identifier); it != m_objects.end()) { return *it->second; } @@ -157,7 +97,7 @@ const iwork::Object &iwork::Package::object(const std::uint64_t identifier) { throw std::runtime_error("iwork: no object " + std::to_string(identifier)); } -const iwork::Component &iwork::Package::load_(const ComponentInfo &info) { +const Component &Package::load_(const ComponentInfo &info) { // by locator, not by name: a name is shared across components, so keying on // it would hand back the wrong file and leave the other never loaded if (const auto it = @@ -174,4 +114,66 @@ const iwork::Component &iwork::Package::load_(const ComponentInfo &info) { return component; } +} // namespace odr::internal::iwork + +namespace odr::internal { + +std::string iwork::read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path) { + const std::shared_ptr file = filesystem.open(path); + if (!file) { + throw std::runtime_error("iwork: missing " + path.string()); + } + const std::unique_ptr stream = file->stream(); + return iwa_decompress(util::stream::read(*stream)); +} + +std::vector iwork::read_objects(const std::string_view data) { + std::vector result; + + std::size_t position = 0; + while (position < data.size()) { + const std::uint64_t info_length = read_varint(data, position); + if (info_length > data.size() - position) { + throw std::runtime_error("iwork: archive info runs past the component"); + } + const Message info(data.substr(position, info_length)); + position += info_length; + + Object object; + object.identifier = info.number_field(archive_info_identifier).value_or(0); + + // the payload holds every message the info names, back to back; only the + // first is modelled, the length of the rest is what skips them + std::size_t payload_length = 0; + std::size_t first_length = 0; + bool first = true; + for (const Field &message : info.repeated_field(archive_info_messages)) { + if (message.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed message info"); + } + const Message message_info(message.bytes); + const std::uint64_t length = + message_info.number_field(message_info_length).value_or(0); + if (first) { + object.type = static_cast( + message_info.number_field(message_info_type).value_or(0)); + first_length = length; + first = false; + } + payload_length += length; + } + + if (payload_length > data.size() - position) { + throw std::runtime_error("iwork: object payload runs past the component"); + } + object.payload = data.substr(position, first_length); + position += payload_length; + + result.push_back(object); + } + + return result; +} + } // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_file.cpp b/src/odr/internal/iwork/iwork_file.cpp index 22bcdfab..af1c2c7b 100644 --- a/src/odr/internal/iwork/iwork_file.cpp +++ b/src/odr/internal/iwork/iwork_file.cpp @@ -13,7 +13,7 @@ #include #include -namespace odr::internal { +namespace odr::internal::iwork { namespace { @@ -22,7 +22,7 @@ namespace { /// theirs off, and the extension is not an answer. FileType file_type_by_archive_type(const std::uint32_t type) { switch (type) { - case iwork::archive_type::pages_document: + case archive_type::pages_document: return FileType::iwork_pages; default: return FileType::unknown; @@ -34,9 +34,8 @@ FileType file_type_by_archive_type(const std::uint32_t type) { /// caller opens, and the `Document` component is the one whose file name never /// carries an identifier suffix. FileType parse_file_type(const abstract::ReadableFilesystem &filesystem) { - const std::string data = - iwork::read_iwa(filesystem, AbsPath("/Index/Document.iwa")); - const std::vector objects = iwork::read_objects(data); + const std::string data = read_iwa(filesystem, AbsPath("/Index/Document.iwa")); + const std::vector objects = read_objects(data); if (objects.empty()) { throw NoIworkFile(); } @@ -50,8 +49,7 @@ FileType parse_file_type(const abstract::ReadableFilesystem &filesystem) { } // namespace -iwork::IworkFile::IworkFile( - std::shared_ptr filesystem) +IworkFile::IworkFile(std::shared_ptr filesystem) : m_filesystem{std::move(filesystem)} { if (!m_filesystem->is_file(AbsPath("/Index/Document.iwa"))) { throw NoIworkFile(); @@ -62,27 +60,23 @@ iwork::IworkFile::IworkFile( m_file_meta.document_type = document_type_by_file_type(m_file_meta.type); } -std::shared_ptr iwork::IworkFile::file() const noexcept { - return {}; -} +std::shared_ptr IworkFile::file() const noexcept { return {}; } -FileType iwork::IworkFile::file_type() const noexcept { - return m_file_meta.type; -} +FileType IworkFile::file_type() const noexcept { return m_file_meta.type; } -std::string_view iwork::IworkFile::mimetype() const noexcept { +std::string_view IworkFile::mimetype() const noexcept { return m_file_meta.mimetype; } -FileMeta iwork::IworkFile::file_meta() const noexcept { return m_file_meta; } +FileMeta IworkFile::file_meta() const noexcept { return m_file_meta; } -DocumentType iwork::IworkFile::document_type() const { +DocumentType IworkFile::document_type() const { return m_file_meta.document_type; } -bool iwork::IworkFile::is_decodable() const noexcept { return true; } +bool IworkFile::is_decodable() const noexcept { return true; } -std::shared_ptr iwork::IworkFile::document() const { +std::shared_ptr IworkFile::document() const { switch (file_type()) { case FileType::iwork_pages: return std::make_shared(m_filesystem); @@ -91,4 +85,4 @@ std::shared_ptr iwork::IworkFile::document() const { } } -} // namespace odr::internal +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp index 10f8ac5b..ea4ce844 100644 --- a/src/odr/internal/iwork/iwork_parser.cpp +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -13,7 +13,7 @@ #include #include -namespace odr::internal { +namespace odr::internal::iwork { namespace { @@ -27,21 +27,20 @@ constexpr std::string_view object_replacement = "\xef\xbf\xbc"; /// The character index each paragraph of @p storage starts at. Paragraph /// boundaries are the run table's rather than every `\n` in the text — the two /// agree today, but the table is what says so. -std::vector paragraph_starts(const iwork::Message &storage) { +std::vector paragraph_starts(const Message &storage) { std::vector result; if (const std::optional table = - storage.bytes_field(iwork::text_storage::paragraph_styles); + storage.bytes_field(text_storage::paragraph_styles); table.has_value()) { - for (const iwork::Field &entry : iwork::Message(*table).repeated_field( - iwork::attribute_table::entries)) { - if (entry.type != iwork::WireType::length_delimited) { + for (const Field &entry : + Message(*table).repeated_field(attribute_table::entries)) { + if (entry.type != WireType::length_delimited) { throw std::runtime_error("iwork: malformed paragraph style table"); } - const iwork::Message run(entry.bytes); + const Message run(entry.bytes); result.push_back( - run.number_field(iwork::attribute_table_entry::character_index) - .value_or(0)); + run.number_field(attribute_table_entry::character_index).value_or(0)); } } @@ -55,7 +54,7 @@ std::vector paragraph_starts(const iwork::Message &storage) { /// Fills @p paragraph_id with the text of one paragraph, breaking it at the /// line separators it holds. -void parse_paragraph(iwork::ElementRegistry ®istry, +void parse_paragraph(ElementRegistry ®istry, const ElementIdentifier paragraph_id, std::string_view content) { const auto append_text = [&](const std::string_view part) { @@ -84,6 +83,10 @@ void parse_paragraph(iwork::ElementRegistry ®istry, } // namespace +} // namespace odr::internal::iwork + +namespace odr::internal { + ElementIdentifier iwork::parse_pages_tree(ElementRegistry ®istry, const abstract::ReadableFilesystem &files) { diff --git a/src/odr/internal/iwork/iwork_protobuf.cpp b/src/odr/internal/iwork/iwork_protobuf.cpp index 9f069bf1..bddb1103 100644 --- a/src/odr/internal/iwork/iwork_protobuf.cpp +++ b/src/odr/internal/iwork/iwork_protobuf.cpp @@ -4,7 +4,7 @@ #include -namespace odr::internal { +namespace odr::internal::iwork { namespace { @@ -21,23 +21,7 @@ std::uint64_t read_fixed(const std::string_view in, std::size_t &position, } // namespace -std::uint64_t iwork::read_varint(const std::string_view in, - std::size_t &position) { - std::uint64_t result = 0; - for (std::uint32_t shift = 0; shift <= 63; shift += 7) { - if (position >= in.size()) { - throw std::runtime_error("iwork: protobuf varint does not terminate"); - } - const auto byte = static_cast(in[position++]); - result |= static_cast(byte & 0x7f) << shift; - if ((byte & 0x80) == 0) { - return result; - } - } - throw std::runtime_error("iwork: protobuf varint does not terminate"); -} - -iwork::Message::Message(const std::string_view bytes) { +Message::Message(const std::string_view bytes) { std::size_t position = 0; while (position < bytes.size()) { @@ -79,12 +63,9 @@ iwork::Message::Message(const std::string_view bytes) { } } -const std::vector &iwork::Message::fields() const noexcept { - return m_fields; -} +const std::vector &Message::fields() const noexcept { return m_fields; } -std::optional -iwork::Message::field(const std::uint32_t number) const { +std::optional Message::field(const std::uint32_t number) const { std::optional result; for (const Field &field : m_fields) { if (field.number == number) { @@ -94,8 +75,7 @@ iwork::Message::field(const std::uint32_t number) const { return result; } -std::vector -iwork::Message::repeated_field(const std::uint32_t number) const { +std::vector Message::repeated_field(const std::uint32_t number) const { std::vector result; for (const Field &field : m_fields) { if (field.number == number) { @@ -106,7 +86,7 @@ iwork::Message::repeated_field(const std::uint32_t number) const { } std::optional -iwork::Message::number_field(const std::uint32_t number) const { +Message::number_field(const std::uint32_t number) const { const std::optional field = this->field(number); if (!field.has_value() || field->type == WireType::length_delimited) { return {}; @@ -115,7 +95,7 @@ iwork::Message::number_field(const std::uint32_t number) const { } std::optional -iwork::Message::bytes_field(const std::uint32_t number) const { +Message::bytes_field(const std::uint32_t number) const { const std::optional field = this->field(number); if (!field.has_value() || field->type != WireType::length_delimited) { return {}; @@ -123,4 +103,24 @@ iwork::Message::bytes_field(const std::uint32_t number) const { return field->bytes; } +} // namespace odr::internal::iwork + +namespace odr::internal { + +std::uint64_t iwork::read_varint(const std::string_view in, + std::size_t &position) { + std::uint64_t result = 0; + for (std::uint32_t shift = 0; shift <= 63; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error("iwork: protobuf varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: protobuf varint does not terminate"); +} + } // namespace odr::internal diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp index 3efe1d42..1777ea23 100644 --- a/test/src/internal/iwork/iwork_archive_test.cpp +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -12,6 +12,7 @@ #include using namespace odr::internal::iwork; +using odr::internal::AbsPath; namespace builder = odr::test::iwork; using builder::message_field; @@ -86,15 +87,14 @@ TEST(ReadIwa, undoes_the_framing) { const auto files = builder::filesystem({{"/Index/Document.iwa", builder::iwa("hello")}}); - EXPECT_EQ(read_iwa(*files, odr::internal::AbsPath("/Index/Document.iwa")), - "hello"); + EXPECT_EQ(read_iwa(*files, AbsPath("/Index/Document.iwa")), "hello"); } TEST(ReadIwa, missing_file) { const auto files = builder::filesystem({}); - EXPECT_ANY_THROW(std::ignore = read_iwa( - *files, odr::internal::AbsPath("/Index/Document.iwa"))); + EXPECT_ANY_THROW(std::ignore = + read_iwa(*files, AbsPath("/Index/Document.iwa"))); } TEST(IworkPackage, loads_a_component_by_name) { diff --git a/test/src/internal/iwork/iwork_test_util.hpp b/test/src/internal/iwork/iwork_test_util.hpp index 877ffb10..c5d80a9d 100644 --- a/test/src/internal/iwork/iwork_test_util.hpp +++ b/test/src/internal/iwork/iwork_test_util.hpp @@ -19,6 +19,9 @@ /// the ones `iwork_archive.cpp` reads. Must never grow into a writer API. namespace odr::test::iwork { +/// The engine's field numbers and archive types, which this namespace shadows. +namespace types = internal::iwork; + /// The object the component list lives in. constexpr std::uint32_t package_metadata_type = 11006; constexpr std::uint32_t package_metadata_components = 3; @@ -140,9 +143,8 @@ package(const std::vector> &components) { /// A `TP.DocumentArchive` whose body is the object @p body_identifier. inline std::string document_archive(const std::uint64_t body_identifier) { return message_field( - ::odr::internal::iwork::document_archive::body_storage, - number_field(::odr::internal::iwork::reference::identifier, - body_identifier)); + types::document_archive::body_storage, + number_field(types::reference::identifier, body_identifier)); } /// A `TSWP.StorageArchive`: @p text and a paragraph style run table over the @@ -151,8 +153,6 @@ inline std::string document_archive(const std::uint64_t body_identifier) { inline std::string text_storage(const std::string &text, const std::optional> ¶graphs) { - namespace types = ::odr::internal::iwork; - std::string result = message_field(types::text_storage::text, text); if (paragraphs.has_value()) { std::string table; @@ -171,17 +171,14 @@ constexpr std::uint64_t body_identifier = 5; /// A one-component package: a root archive of @p root_type whose body is /// @p storage. -inline std::shared_ptr -pages_package(const std::string &storage, - const std::uint32_t root_type = - ::odr::internal::iwork::archive_type::pages_document) { +inline std::shared_ptr pages_package( + const std::string &storage, + const std::uint32_t root_type = types::archive_type::pages_document) { const std::string root = document_archive(body_identifier); const std::string document = object(1, {{root_type, root.size()}}, root) + object(body_identifier, - {{::odr::internal::iwork::archive_type::text_storage, - storage.size()}}, - storage); + {{types::archive_type::text_storage, storage.size()}}, storage); return package({{"Document", document}}); } diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp index ac93d2b5..d10c64fe 100644 --- a/test/src/internal/iwork/pages_test.cpp +++ b/test/src/internal/iwork/pages_test.cpp @@ -28,6 +28,7 @@ using namespace odr; using odr::test::TestData; namespace builder = odr::test::iwork; +namespace iwork = odr::internal::iwork; namespace { @@ -56,7 +57,7 @@ std::vector paragraphs(const Element root) { Document pages_document( const std::string &text, const std::optional> ¶graph_indices) { - return Document(std::make_shared( + return Document(std::make_shared( builder::pages_package(builder::text_storage(text, paragraph_indices)))); } @@ -124,15 +125,12 @@ TEST(Iwork, pages_body_text) { // the name hands back the wrong file and leaves the rest never loaded, which // shows up as an object nothing can resolve. TEST(Iwork, package_resolves_across_components) { - using odr::internal::iwork::Package; - - const auto file = - std::make_shared(odr::internal::AbsPath( - TestData::test_file_path("odr-public/pages/style-various-1.pages"))); + const auto file = std::make_shared(internal::AbsPath( + TestData::test_file_path("odr-public/pages/style-various-1.pages"))); const auto filesystem = - odr::internal::zip::ZipFile(file).archive()->as_filesystem(); + internal::zip::ZipFile(file).archive()->as_filesystem(); - Package package(*filesystem); + iwork::Package package(*filesystem); EXPECT_EQ(package.component("Document").objects().front().identifier, 1); // the stylesheet, which is a component of its own @@ -200,11 +198,11 @@ TEST(Iwork, unmapped_root_archive_is_not_an_iwork_file) { const auto files = builder::pages_package(builder::text_storage("", std::nullopt), 10001); - EXPECT_THROW(internal::iwork::IworkFile{files}, NoIworkFile); + EXPECT_THROW(iwork::IworkFile{files}, NoIworkFile); } TEST(Iwork, package_without_a_document_component_is_not_an_iwork_file) { const auto files = builder::filesystem({}); - EXPECT_THROW(internal::iwork::IworkFile{files}, NoIworkFile); + EXPECT_THROW(iwork::IworkFile{files}, NoIworkFile); } From 787e311e312805f263b2bec130a4fedfd9638b26 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 24 Aug 2026 20:41:54 +0200 Subject: [PATCH 14/14] refactor(util): take the utf-16 indices as a span `utf16_offsets` only reads them, so an array or a subrange is as good as the vector the one caller happens to hold. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CfnhZeFgZh84WMmeKK5cp2 --- src/odr/internal/util/string_util.cpp | 2 +- src/odr/internal/util/string_util.hpp | 6 ++-- test/src/internal/util/string_util_test.cpp | 33 ++++++++++++++------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/odr/internal/util/string_util.cpp b/src/odr/internal/util/string_util.cpp index 9d4bf72b..85c52faa 100644 --- a/src/odr/internal/util/string_util.cpp +++ b/src/odr/internal/util/string_util.cpp @@ -178,7 +178,7 @@ std::size_t string::utf8_length(const std::string &string) { std::vector string::utf16_offsets(const std::string_view string, - const std::vector &indices) { + const std::span indices) { std::vector result; result.reserve(indices.size()); diff --git a/src/odr/internal/util/string_util.hpp b/src/odr/internal/util/string_util.hpp index a4907c5c..25de9077 100644 --- a/src/odr/internal/util/string_util.hpp +++ b/src/odr/internal/util/string_util.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -70,9 +71,8 @@ std::size_t utf8_length(const std::string &string); /// The byte offsets into @p string of the ascending UTF-16 code unit /// @p indices — the indexing a format that counts UTF-16 uses over UTF-8 text. /// Throws when @p string is not UTF-8 or an index is past its end. -std::vector -utf16_offsets(std::string_view string, - const std::vector &indices); +std::vector utf16_offsets(std::string_view string, + std::span indices); /// A surrogate completing no pair becomes U+FFFD rather than throwing. std::string u16string_to_string(const std::u16string &string); diff --git a/test/src/internal/util/string_util_test.cpp b/test/src/internal/util/string_util_test.cpp index f7ae4386..383b9a9f 100644 --- a/test/src/internal/util/string_util_test.cpp +++ b/test/src/internal/util/string_util_test.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include @@ -177,23 +179,34 @@ TEST(string_util, find_ignore_case) { EXPECT_EQ(find_ignore_case("abc", "a", 99), std::string_view::npos); } +namespace { + +/// `utf16_offsets` over a braced index list, which its `std::span` does not +/// take. +std::vector offsets(const std::string_view string, + const std::vector &indices) { + return utf16_offsets(string, indices); +} + +} // namespace + TEST(string_util, utf16_offsets) { // a character outside the basic multilingual plane is two units, four bytes - EXPECT_EQ(utf16_offsets("a\xf0\x9f\x98\x80z", {0, 1, 3, 4}), + EXPECT_EQ(offsets("a\xf0\x9f\x98\x80z", {0, 1, 3, 4}), (std::vector{0, 1, 5, 6})); // and one inside it is one unit, up to three bytes - EXPECT_EQ(utf16_offsets("\xe2\x80\xa8" - "b", - {1}), + EXPECT_EQ(offsets("\xe2\x80\xa8" + "b", + {1}), (std::vector{3})); - EXPECT_EQ(utf16_offsets("abc", {}), (std::vector{})); - EXPECT_EQ(utf16_offsets("", {0}), (std::vector{0})); + EXPECT_EQ(offsets("abc", {}), (std::vector{})); + EXPECT_EQ(offsets("", {0}), (std::vector{0})); - EXPECT_ANY_THROW(std::ignore = utf16_offsets("abc", {4})); - EXPECT_ANY_THROW(std::ignore = utf16_offsets("\x80", {1})); - EXPECT_ANY_THROW(std::ignore = utf16_offsets("a\xe2\x80", {2})); + EXPECT_ANY_THROW(std::ignore = offsets("abc", {4})); + EXPECT_ANY_THROW(std::ignore = offsets("\x80", {1})); + EXPECT_ANY_THROW(std::ignore = offsets("a\xe2\x80", {2})); // an index landing inside a character is never reached - EXPECT_ANY_THROW(std::ignore = utf16_offsets("\xf0\x9f\x98\x80", {1})); + EXPECT_ANY_THROW(std::ignore = offsets("\xf0\x9f\x98\x80", {1})); } TEST(string_util, u16string_to_string) {