From 6560609e04255a9c0859c7772496d1f3c729fa37 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Wed, 18 Oct 2023 15:56:19 -0400 Subject: [PATCH] fix(manifests): Make `added_snapshot_id` mandatory in V1 --- internal/avro_schemas.go | 2 +- manifest.go | 21 +++++++-------------- manifest_test.go | 3 +-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/internal/avro_schemas.go b/internal/avro_schemas.go index 4628787..893b7b2 100644 --- a/internal/avro_schemas.go +++ b/internal/avro_schemas.go @@ -40,7 +40,7 @@ func init() { {"name": "partition_spec_id", "type": "int", "doc": "Spec ID used to write", "field-id": 502}, { "name": "added_snapshot_id", - "type": ["null", "long"], + "type": "long", "doc": "Snapshot ID that added the manifest", "field-id": 503 }, diff --git a/manifest.go b/manifest.go index 628b721..1d1113e 100644 --- a/manifest.go +++ b/manifest.go @@ -52,21 +52,17 @@ type ManifestV1Builder struct { // NewManifestV1Builder is passed all of the required fields and then allows // all of the optional fields to be set by calling the corresponding methods // before calling [ManifestV1Builder.Build] to construct the object. -func NewManifestV1Builder(path string, length int64, partitionSpecID int32) *ManifestV1Builder { +func NewManifestV1Builder(path string, length int64, partitionSpecID int32, addedSnapshotID int64) *ManifestV1Builder { return &ManifestV1Builder{ m: &manifestFileV1{ - Path: path, - Len: length, - SpecID: partitionSpecID, + Path: path, + Len: length, + SpecID: partitionSpecID, + AddedSnapshotID: addedSnapshotID, }, } } -func (b *ManifestV1Builder) AddedSnapshotID(id int64) *ManifestV1Builder { - b.m.AddedSnapshotID = &id - return b -} - func (b *ManifestV1Builder) AddedFiles(cnt int32) *ManifestV1Builder { b.m.AddedFilesCount = &cnt return b @@ -119,7 +115,7 @@ type manifestFileV1 struct { Path string `avro:"manifest_path"` Len int64 `avro:"manifest_length"` SpecID int32 `avro:"partition_spec_id"` - AddedSnapshotID *int64 `avro:"added_snapshot_id"` + AddedSnapshotID int64 `avro:"added_snapshot_id"` AddedFilesCount *int32 `avro:"added_data_files_count"` ExistingFilesCount *int32 `avro:"existing_data_files_count"` DeletedFilesCount *int32 `avro:"deleted_data_files_count"` @@ -138,10 +134,7 @@ func (m *manifestFileV1) ManifestContent() ManifestContent { return ManifestContentData } func (m *manifestFileV1) SnapshotID() int64 { - if m.AddedSnapshotID == nil { - return 0 - } - return *m.AddedSnapshotID + return m.AddedSnapshotID } func (m *manifestFileV1) AddedDataFiles() int32 { diff --git a/manifest_test.go b/manifest_test.go index 74d063e..486c6a8 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -33,8 +33,7 @@ var ( addedRows int64 = 237993 manifestFileRecordsV1 = []ManifestFile{ NewManifestV1Builder("/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", - 7989, 0). - AddedSnapshotID(snapshotID). + 7989, 0, snapshotID). AddedFiles(3). ExistingFiles(0). DeletedFiles(0).