Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 43 additions & 26 deletions core/src/main/java/org/apache/iceberg/ContentFileParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ public class ContentFileParser {
private static final String EQUALITY_IDS = "equality-ids";
private static final String SORT_ORDER_ID = "sort-order-id";
private static final String FIRST_ROW_ID = "first-row-id";
private static final String DATA_SEQUENCE_NUMBER = "data-sequence-number";
private static final String FILE_SEQUENCE_NUMBER = "file-sequence-number";
private static final String REFERENCED_DATA_FILE = "referenced-data-file";
private static final String CONTENT_OFFSET = "content-offset";
private static final String CONTENT_SIZE = "content-size-in-bytes";
Expand Down Expand Up @@ -123,6 +125,11 @@ public static void toJson(ContentFile<?> contentFile, PartitionSpec spec, JsonGe

JsonUtil.writeLongFieldIfPresent(FIRST_ROW_ID, contentFile.firstRowId(), generator);

JsonUtil.writeLongFieldIfPresent(
DATA_SEQUENCE_NUMBER, contentFile.dataSequenceNumber(), generator);
JsonUtil.writeLongFieldIfPresent(
FILE_SEQUENCE_NUMBER, contentFile.fileSequenceNumber(), generator);

if (contentFile instanceof DeleteFile) {
DeleteFile deleteFile = (DeleteFile) contentFile;

Expand Down Expand Up @@ -170,38 +177,48 @@ public static ContentFile<?> fromJson(JsonNode jsonNode, Map<Integer, PartitionS
int[] equalityFieldIds = JsonUtil.getIntArrayOrNull(EQUALITY_IDS, jsonNode);
Integer sortOrderId = JsonUtil.getIntOrNull(SORT_ORDER_ID, jsonNode);
Long firstRowId = JsonUtil.getLongOrNull(FIRST_ROW_ID, jsonNode);
Long dataSequenceNumber = JsonUtil.getLongOrNull(DATA_SEQUENCE_NUMBER, jsonNode);
Long fileSequenceNumber = JsonUtil.getLongOrNull(FILE_SEQUENCE_NUMBER, jsonNode);
String referencedDataFile = JsonUtil.getStringOrNull(REFERENCED_DATA_FILE, jsonNode);
Long contentOffset = JsonUtil.getLongOrNull(CONTENT_OFFSET, jsonNode);
Long contentSizeInBytes = JsonUtil.getLongOrNull(CONTENT_SIZE, jsonNode);

if (fileContent == FileContent.DATA) {
return new GenericDataFile(
specId,
filePath,
fileFormat,
partitionData,
fileSizeInBytes,
metrics,
keyMetadata,
splitOffsets,
sortOrderId,
firstRowId);
GenericDataFile dataFile =
new GenericDataFile(
specId,
filePath,
fileFormat,
partitionData,
fileSizeInBytes,
metrics,
keyMetadata,
splitOffsets,
sortOrderId,
firstRowId);
dataFile.setDataSequenceNumber(dataSequenceNumber);
dataFile.setFileSequenceNumber(fileSequenceNumber);
return dataFile;
} else {
return new GenericDeleteFile(
specId,
fileContent,
filePath,
fileFormat,
partitionData,
fileSizeInBytes,
metrics,
equalityFieldIds,
sortOrderId,
splitOffsets,
keyMetadata,
referencedDataFile,
contentOffset,
contentSizeInBytes);
GenericDeleteFile deleteFile =
new GenericDeleteFile(
specId,
fileContent,
filePath,
fileFormat,
partitionData,
fileSizeInBytes,
metrics,
equalityFieldIds,
sortOrderId,
splitOffsets,
keyMetadata,
referencedDataFile,
contentOffset,
contentSizeInBytes);
deleteFile.setDataSequenceNumber(dataSequenceNumber);
deleteFile.setFileSequenceNumber(fileSequenceNumber);
return deleteFile;
}
}

Expand Down
8 changes: 8 additions & 0 deletions open-api/rest-catalog-open-api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5222,6 +5222,14 @@ components:
description: "List of splittable offsets"
sort-order-id:
type: integer
data-sequence-number:
type: integer
format: int64
description: The data sequence number of the file
file-sequence-number:
type: integer
format: int64
description: The file sequence number of the file

DataFile:
allOf:
Expand Down
Loading