From 7baec1f35ac0a238491a17f6e5161174efe4517a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Wed, 16 Oct 2024 16:32:00 -0400 Subject: [PATCH 01/14] IO Implementation using Go CDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Alleyne --- io/blob.go | 333 ++++++++++++++++++++++++++++++++ io/gcs_cdk.go | 63 +++++++ io/io.go | 514 ++++++++++++++++++++++++++------------------------ io/local.go | 70 +++---- io/s3.go | 256 ++++++++++++++----------- io/s3_cdk.go | 45 +++++ 6 files changed, 884 insertions(+), 397 deletions(-) create mode 100644 io/blob.go create mode 100644 io/gcs_cdk.go create mode 100644 io/s3_cdk.go diff --git a/io/blob.go b/io/blob.go new file mode 100644 index 0000000..474ba25 --- /dev/null +++ b/io/blob.go @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "io" + "io/fs" + "net/url" + "path/filepath" + "strings" + "time" + + "gocloud.dev/blob" + "gocloud.dev/blob/memblob" +) + +// iofsFileInfo describes a single file in an io/fs.FS. +// It implements fs.FileInfo and fs.DirEntry. +// Copied from `gocloud.dev/blob.iofsDir` because it is private +type iofsFileInfo struct { + lo *blob.ListObject + name string +} + +func (f *iofsFileInfo) Name() string { return f.name } +func (f *iofsFileInfo) Size() int64 { return f.lo.Size } +func (f *iofsFileInfo) Mode() fs.FileMode { return fs.ModeIrregular } +func (f *iofsFileInfo) ModTime() time.Time { return f.lo.ModTime } +func (f *iofsFileInfo) IsDir() bool { return false } +func (f *iofsFileInfo) Sys() interface{} { return f.lo } +func (f *iofsFileInfo) Info() (fs.FileInfo, error) { return f, nil } +func (f *iofsFileInfo) Type() fs.FileMode { return fs.ModeIrregular } + +// iofsDir describes a single directory in an `iceberg-go/io.FS`. +// It implements `io/fs.FileInfo`, `io/fs.DirEntry`, and `io/fs.File`. +// Copied from `gocloud.dev/blob.iofsDir`, but modified to use `iceberg-go/io.File` instead of `io/fs.File` +type iofsDir struct { + b *BlobFileIO + key string + name string + // If opened is true, we've read entries via openOnce(). + opened bool + entries []fs.DirEntry + offset int +} + +func newDir(b *BlobFileIO, key, name string) *iofsDir { + return &iofsDir{b: b, key: key, name: name} +} + +func (d *iofsDir) Name() string { return d.name } +func (d *iofsDir) Size() int64 { return 0 } +func (d *iofsDir) Mode() fs.FileMode { return fs.ModeDir } +func (d *iofsDir) Type() fs.FileMode { return fs.ModeDir } +func (d *iofsDir) ModTime() time.Time { return time.Time{} } +func (d *iofsDir) IsDir() bool { return true } +func (d *iofsDir) Sys() interface{} { return d } +func (d *iofsDir) Info() (fs.FileInfo, error) { return d, nil } +func (d *iofsDir) Stat() (fs.FileInfo, error) { return d, nil } +func (d *iofsDir) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) ReadAt(p []byte, off int64) (int, error) { + return 0, &fs.PathError{Op: "readAt", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) Seek(offset int64, whence int) (int64, error) { + return 0, &fs.PathError{Op: "seek", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) Close() error { return nil } +func (d *iofsDir) ReadDir(count int) ([]fs.DirEntry, error) { + if err := d.openOnce(); err != nil { + return nil, err + } + n := len(d.entries) - d.offset + if n == 0 && count > 0 { + return nil, io.EOF + } + if count > 0 && n > count { + n = count + } + list := make([]fs.DirEntry, n) + for i := range list { + list[i] = d.entries[d.offset+i] + } + d.offset += n + return list, nil +} + +func (d *iofsDir) openOnce() error { + if d.opened { + return nil + } + d.opened = true + + // blob expects directories to end in the delimiter, except at the top level. + prefix := d.key + if prefix != "" { + prefix += "/" + } + listOpts := blob.ListOptions{ + Prefix: prefix, + Delimiter: "/", + } + ctx := d.b.ctx + + // Fetch all the directory entries. + // Conceivably we could only fetch a few here, and fetch the rest lazily + // on demand, but that would add significant complexity. + iter := d.b.List(&listOpts) + for { + item, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return err + } + name := filepath.Base(item.Key) + if item.IsDir { + d.entries = append(d.entries, newDir(d.b, item.Key, name)) + } else { + d.entries = append(d.entries, &iofsFileInfo{item, name}) + } + } + // There is no such thing as an empty directory in Bucket, so if + // we didn't find anything, it doesn't exist. + if len(d.entries) == 0 { + return fs.ErrNotExist + } + return nil +} + +// blobOpenFile describes a single open blob as a File. +// It implements the iceberg-go/io.File interface. +// It is based on gocloud.dev/blob.iofsOpenFile which: +// - Doesn't support the `io.ReaderAt` interface +// - Is not externally accessible, so copied here +type blobOpenFile struct { + *blob.Reader + name string +} + +func (f *blobOpenFile) ReadAt(p []byte, off int64) (int, error) { + finalOff, err := f.Reader.Seek(off, io.SeekStart) + if err != nil { + return -1, err + } else if finalOff != off { + return -1, io.ErrUnexpectedEOF + } + + return f.Read(p) +} + +// Functions to implement the `Stat()` function in the `io/fs.File` interface + +func (f *blobOpenFile) Name() string { return f.name } +func (f *blobOpenFile) Mode() fs.FileMode { return fs.ModeIrregular } +func (f *blobOpenFile) Sys() interface{} { return f.Reader } +func (f *blobOpenFile) IsDir() bool { return false } +func (f *blobOpenFile) Stat() (fs.FileInfo, error) { return f, nil } + +// BlobFileIO represents a file system backed by a bucket in object store. It implements the `iceberg-go/io.FileIO` interface. +type BlobFileIO struct { + *blob.Bucket + ctx context.Context + opts *blob.ReaderOptions + prefix string +} + +func (io *BlobFileIO) preprocess(n string) string { + _, after, found := strings.Cut(n, "://") + if found { + n = after + } + + out := strings.TrimPrefix(n, io.prefix) + if out == "/" { + out = "." + } else { + out = strings.TrimPrefix(out, "/") + } + + return out +} + +// Open a Blob from a Bucket using the BlobFileIO. Note this +// function is copied from blob.Bucket.Open, but extended to +// return a iceberg-go/io.File instance instead of io/fs.File +func (io *BlobFileIO) Open(path string) (File, error) { + if _, err := url.Parse(path); err != nil { + return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + + var isDir bool + var key, name string // name is the last part of the path + if path == "." { + // Root is always a directory, but blob doesn't want the "." in the key. + isDir = true + key, name = "", "." + } else { + exists, _ := io.Bucket.Exists(io.ctx, path) + isDir = !exists + key, name = path, filepath.Base(path) + } + + // If it's a directory, list the directory contents. We can't do this lazily + // because we need to error out here if it doesn't exist. + if isDir { + dir := newDir(io, key, name) + err := dir.openOnce() + if err != nil { + if err == fs.ErrNotExist && path == "." { + // The root directory must exist. + return dir, nil + } + return nil, &fs.PathError{Op: "open", Path: path, Err: err} + } + return dir, nil + } + + // It's a file; open it and return a wrapper. + r, err := io.Bucket.NewReader(io.ctx, path, io.opts) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: path, Err: err} + } + + return &blobOpenFile{Reader: r, name: name}, nil +} + +// Remove a Blob from a Bucket using the BlobFileIO +func (io *BlobFileIO) Remove(path string) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "remove", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + + return io.Bucket.Delete(io.ctx, path) +} + +// NewWriter returns a Writer that writes to the blob stored at path. +// A nil WriterOptions is treated the same as the zero value. +// +// If overwrite is disabled and a blob with this path already exists, +// an error will be returned. +// +// The returned Writer will store ctx for later use in Write and/or Close. +// To abort a write, cancel ctx; otherwise, it must remain open until +// Close is called. +// +// The caller must call Close on the returned Writer, even if the write is +// aborted. +func (io *BlobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOptions) (w *BlobWriteFile, err error) { + if !fs.ValidPath(path) { + return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + if !overwrite { + if exists, err := io.Bucket.Exists(io.ctx, path); exists { + if err != nil { + return nil, &fs.PathError{Op: "new writer", Path: path, Err: err} + } + return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} + } + } + bw, err := io.Bucket.NewWriter(io.ctx, path, opts) + if err != nil { + return nil, err + } + return &BlobWriteFile{ + Writer: bw, + name: path, + opts: opts}, + nil +} + +func urlToBucketPath(parsed *url.URL) (string, string) { + return parsed.Host, parsed.Path +} + +// Create a new BlobFileIO instance +func CreateBlobFileIO(parsed *url.URL, props map[string]string) (*BlobFileIO, error) { + ctx := context.Background() + + var bucket *blob.Bucket + var err error + switch parsed.Scheme { + case "mem": + // memblob doesn't use the URL host or path + bucket = memblob.OpenBucket(nil) + case "s3", "s3a", "s3n": + bucket, err = createS3Bucket(ctx, parsed, props) + case "gs": + bucket, err = createGCSBucket(ctx, parsed, props) + } + + if err != nil { + return nil, err + } + + if parsed.Path != "" && parsed.Path != "/" { + bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) + } + + return &BlobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path}, nil +} + +type BlobWriteFile struct { + *blob.Writer + name string + opts *blob.WriterOptions +} + +func (f *BlobWriteFile) Name() string { return f.name } +func (f *BlobWriteFile) Sys() interface{} { return f.Writer } +func (f *BlobWriteFile) Close() error { return f.Writer.Close() } +func (f *BlobWriteFile) Write(p []byte) (int, error) { return f.Writer.Write(p) } diff --git a/io/gcs_cdk.go b/io/gcs_cdk.go new file mode 100644 index 0000000..232306f --- /dev/null +++ b/io/gcs_cdk.go @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "net/url" + + "gocloud.dev/blob" + "gocloud.dev/blob/gcsblob" + "gocloud.dev/gcp" + "google.golang.org/api/option" +) + +// Constants for GCS configuration options +const ( + GCSEndpoint = "gcs.endpoint" + GCSKeyPath = "gcs.keypath" + GCSJSONKey = "gcs.jsonkey" +) + +func ParseGCSConfig(props map[string]string) *gcsblob.Options { + var o []option.ClientOption + if url := props[GCSEndpoint]; url != "" { + o = append(o, option.WithEndpoint(url)) + } + if key := props[GCSJSONKey]; key != "" { + o = append(o, option.WithCredentialsJSON([]byte(key))) + } + if path := props[GCSKeyPath]; path != "" { + o = append(o, option.WithCredentialsFile(path)) + } + return &gcsblob.Options{ + ClientOptions: o, + } +} + +// Construct a S3 bucket from a URL +func createGCSBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + gcscfg := ParseGCSConfig(props) + client := gcp.NewAnonymousHTTPClient(gcp.DefaultTransport()) + bucket, err := gcsblob.OpenBucket(ctx, client, parsed.Host, gcscfg) + if err != nil { + return nil, err + } + + return bucket, nil +} diff --git a/io/io.go b/io/io.go index abe5971..6aeee25 100644 --- a/io/io.go +++ b/io/io.go @@ -1,248 +1,266 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "errors" - "fmt" - "io" - "io/fs" - "net/url" - "strings" -) - -// IO is an interface to a hierarchical file system. -// -// The IO interface is the minimum implementation required for a file -// system to utilize an iceberg table. A file system may implement -// additional interfaces, such as ReadFileIO, to provide additional or -// optimized functionality. -type IO interface { - // Open opens the named file. - // - // When Open returns an error, it should be of type *PathError - // with the Op field set to "open", the Path field set to name, - // and the Err field describing the problem. - // - // Open should reject attempts to open names that do not satisfy - // fs.ValidPath(name), returning a *PathError with Err set to - // ErrInvalid or ErrNotExist. - Open(name string) (File, error) - - // Remove removes the named file or (empty) directory. - // - // If there is an error, it will be of type *PathError. - Remove(name string) error -} - -// ReadFileIO is the interface implemented by a file system that -// provides an optimized implementation of ReadFile. -type ReadFileIO interface { - IO - - // ReadFile reads the named file and returns its contents. - // A successful call returns a nil error, not io.EOF. - // (Because ReadFile reads the whole file, the expected EOF - // from the final Read is not treated as an error to be reported.) - // - // The caller is permitted to modify the returned byte slice. - // This method should return a copy of the underlying data. - ReadFile(name string) ([]byte, error) -} - -// A File provides access to a single file. The File interface is the -// minimum implementation required for Iceberg to interact with a file. -// Directory files should also implement -type File interface { - fs.File - io.ReadSeekCloser - io.ReaderAt -} - -// A ReadDirFile is a directory file whose entries can be read with the -// ReadDir method. Every directory file should implement this interface. -// (It is permissible for any file to implement this interface, but -// if so ReadDir should return an error for non-directories.) -type ReadDirFile interface { - File - - // ReadDir read the contents of the directory and returns a slice - // of up to n DirEntry values in directory order. Subsequent calls - // on the same file will yield further DirEntry values. - // - // If n > 0, ReadDir returns at most n DirEntry structures. In this - // case, if ReadDir returns an empty slice, it will return a non-nil - // error explaining why. - // - // At the end of a directory, the error is io.EOF. (ReadDir must return - // io.EOF itself, not an error wrapping io.EOF.) - // - // If n <= 0, ReadDir returns all the DirEntry values from the directory - // in a single slice. In this case, if ReadDir succeeds (reads all the way - // to the end of the directory), it returns the slice and a nil error. - // If it encounters an error before the end of the directory, ReadDir - // returns the DirEntry list read until that point and a non-nil error. - ReadDir(n int) ([]fs.DirEntry, error) -} - -// FS wraps an io/fs.FS as an IO interface. -func FS(fsys fs.FS) IO { - if _, ok := fsys.(fs.ReadFileFS); ok { - return readFileFS{ioFS{fsys, nil}} - } - return ioFS{fsys, nil} -} - -// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then -// it is called to preprocess any filenames before they are passed to -// the underlying fsys. -func FSPreProcName(fsys fs.FS, fn func(string) string) IO { - if _, ok := fsys.(fs.ReadFileFS); ok { - return readFileFS{ioFS{fsys, fn}} - } - return ioFS{fsys, fn} -} - -type readFileFS struct { - ioFS -} - -func (r readFileFS) ReadFile(name string) ([]byte, error) { - if r.preProcessName != nil { - name = r.preProcessName(name) - } - - rfs, ok := r.fsys.(fs.ReadFileFS) - if !ok { - return nil, errMissingReadFile - } - return rfs.ReadFile(name) -} - -type ioFS struct { - fsys fs.FS - - preProcessName func(string) string -} - -func (f ioFS) Open(name string) (File, error) { - if f.preProcessName != nil { - name = f.preProcessName(name) - } - - if name == "/" { - name = "." - } else { - name = strings.TrimPrefix(name, "/") - } - file, err := f.fsys.Open(name) - if err != nil { - return nil, err - } - - return ioFile{file}, nil -} - -func (f ioFS) Remove(name string) error { - r, ok := f.fsys.(interface{ Remove(name string) error }) - if !ok { - return errMissingRemove - } - return r.Remove(name) -} - -var ( - errMissingReadDir = errors.New("fs.File directory missing ReadDir method") - errMissingSeek = errors.New("fs.File missing Seek method") - errMissingReadAt = errors.New("fs.File missing ReadAt") - errMissingRemove = errors.New("fs.FS missing Remove method") - errMissingReadFile = errors.New("fs.FS missing ReadFile method") -) - -type ioFile struct { - file fs.File -} - -func (f ioFile) Close() error { return f.file.Close() } -func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } -func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } -func (f ioFile) Seek(offset int64, whence int) (int64, error) { - s, ok := f.file.(io.Seeker) - if !ok { - return 0, errMissingSeek - } - return s.Seek(offset, whence) -} - -func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) { - r, ok := f.file.(io.ReaderAt) - if !ok { - return 0, errMissingReadAt - } - return r.ReadAt(p, off) -} - -func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) { - d, ok := f.file.(fs.ReadDirFile) - if !ok { - return nil, errMissingReadDir - } - - return d.ReadDir(count) -} - -func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { - parsed, err := url.Parse(path) - if err != nil { - return nil, err - } - - switch parsed.Scheme { - case "s3", "s3a", "s3n": - return createS3FileIO(parsed, props) - case "file", "": - return LocalFS{}, nil - default: - return nil, fmt.Errorf("IO for file '%s' not implemented", path) - } -} - -// LoadFS takes a map of properties and an optional URI location -// and attempts to infer an IO object from it. -// -// A schema of "file://" or an empty string will result in a LocalFS -// implementation. Otherwise this will return an error if the schema -// does not yet have an implementation here. -// -// Currently only LocalFS and S3 are implemented. -func LoadFS(props map[string]string, location string) (IO, error) { - if location == "" { - location = props["warehouse"] - } - - iofs, err := inferFileIOFromSchema(location, props) - if err != nil { - return nil, err - } - - if iofs == nil { - iofs = LocalFS{} - } - - return iofs, nil -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "strings" +) + +// IO is an interface to a hierarchical file system. +// +// The IO interface is the minimum implementation required for a file +// system to utilize an iceberg table. A file system may implement +// additional interfaces, such as ReadFileIO, to provide additional or +// optimized functionality. +type IO interface { + // Open opens the named file. + // + // When Open returns an error, it should be of type *PathError + // with the Op field set to "open", the Path field set to name, + // and the Err field describing the problem. + // + // Open should reject attempts to open names that do not satisfy + // fs.ValidPath(name), returning a *PathError with Err set to + // ErrInvalid or ErrNotExist. + Open(name string) (File, error) + + // Remove removes the named file or (empty) directory. + // + // If there is an error, it will be of type *PathError. + Remove(name string) error +} + +// ReadFileIO is the interface implemented by a file system that +// provides an optimized implementation of ReadFile. +type ReadFileIO interface { + IO + + // ReadFile reads the named file and returns its contents. + // A successful call returns a nil error, not io.EOF. + // (Because ReadFile reads the whole file, the expected EOF + // from the final Read is not treated as an error to be reported.) + // + // The caller is permitted to modify the returned byte slice. + // This method should return a copy of the underlying data. + ReadFile(name string) ([]byte, error) +} + +// WriteFileIO is the interface implemented by a file system that +// provides an optimized implementation of WriteFile +type WriteFileIO interface { + IO + + // WriteFile writes p to the named file. + // An error will be returned if the file already exists. + Write(name string, p []byte) error + Close() error +} + +// A File provides access to a single file. The File interface is the +// minimum implementation required for Iceberg to interact with a file. +// Directory files should also implement +type File interface { + fs.File + io.ReadSeekCloser + io.ReaderAt +} + +// A ReadDirFile is a directory file whose entries can be read with the +// ReadDir method. Every directory file should implement this interface. +// (It is permissible for any file to implement this interface, but +// if so ReadDir should return an error for non-directories.) +type ReadDirFile interface { + File + + // ReadDir read the contents of the directory and returns a slice + // of up to n DirEntry values in directory order. Subsequent calls + // on the same file will yield further DirEntry values. + // + // If n > 0, ReadDir returns at most n DirEntry structures. In this + // case, if ReadDir returns an empty slice, it will return a non-nil + // error explaining why. + // + // At the end of a directory, the error is io.EOF. (ReadDir must return + // io.EOF itself, not an error wrapping io.EOF.) + // + // If n <= 0, ReadDir returns all the DirEntry values from the directory + // in a single slice. In this case, if ReadDir succeeds (reads all the way + // to the end of the directory), it returns the slice and a nil error. + // If it encounters an error before the end of the directory, ReadDir + // returns the DirEntry list read until that point and a non-nil error. + ReadDir(n int) ([]fs.DirEntry, error) +} + +// FS wraps an io/fs.FS as an IO interface. +func FS(fsys fs.FS) IO { + if _, ok := fsys.(fs.ReadFileFS); ok { + return readFileFS{ioFS{fsys, nil}} + } + return ioFS{fsys, nil} +} + +// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then +// it is called to preprocess any filenames before they are passed to +// the underlying fsys. +func FSPreProcName(fsys fs.FS, fn func(string) string) IO { + if _, ok := fsys.(fs.ReadFileFS); ok { + return readFileFS{ioFS{fsys, fn}} + } + return ioFS{fsys, fn} +} + +type readFileFS struct { + ioFS +} + +func (r readFileFS) ReadFile(name string) ([]byte, error) { + if r.preProcessName != nil { + name = r.preProcessName(name) + } + + rfs, ok := r.fsys.(fs.ReadFileFS) + if !ok { + return nil, errMissingReadFile + } + return rfs.ReadFile(name) +} + +type ioFS struct { + fsys fs.FS + + preProcessName func(string) string +} + +func (f ioFS) Open(name string) (File, error) { + if f.preProcessName != nil { + name = f.preProcessName(name) + } + + if name == "/" { + name = "." + } else { + name = strings.TrimPrefix(name, "/") + } + file, err := f.fsys.Open(name) + if err != nil { + return nil, err + } + + return ioFile{file}, nil +} + +func (f ioFS) Remove(name string) error { + r, ok := f.fsys.(interface{ Remove(name string) error }) + if !ok { + return errMissingRemove + } + return r.Remove(name) +} + +var ( + errMissingReadDir = errors.New("fs.File directory missing ReadDir method") + errMissingSeek = errors.New("fs.File missing Seek method") + errMissingReadAt = errors.New("fs.File missing ReadAt") + errMissingRemove = errors.New("fs.FS missing Remove method") + errMissingReadFile = errors.New("fs.FS missing ReadFile method") +) + +type ioFile struct { + file fs.File +} + +func (f ioFile) Close() error { return f.file.Close() } +func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } +func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } +func (f ioFile) Seek(offset int64, whence int) (int64, error) { + s, ok := f.file.(io.Seeker) + if !ok { + return 0, errMissingSeek + } + return s.Seek(offset, whence) +} + +func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) { + r, ok := f.file.(io.ReaderAt) + if !ok { + return 0, errMissingReadAt + } + return r.ReadAt(p, off) +} + +func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, errMissingReadDir + } + + return d.ReadDir(count) +} + +func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { + parsed, err := url.Parse(path) + if err != nil { + return nil, err + } + + switch parsed.Scheme { + case "s3", "s3a", "s3n": + if props["s3.use-cdk"] == "true" { + return CreateBlobFileIO(parsed, props) + } + return createS3FileIO(parsed, props) + case "gs": + return CreateBlobFileIO(parsed, props) + case "mem": + return CreateBlobFileIO(parsed, props) + case "file", "": + return LocalFS{}, nil + default: + return nil, fmt.Errorf("IO for file '%s' not implemented", path) + } +} + +// LoadFS takes a map of properties and an optional URI location +// and attempts to infer an IO object from it. +// +// A schema of "file://" or an empty string will result in a LocalFS +// implementation. Otherwise this will return an error if the schema +// does not yet have an implementation here. +// +// Currently local, S3, GCS, and In-Memory FSs are implemented. +func LoadFS(props map[string]string, location string) (IO, error) { + if location == "" { + location = props["warehouse"] + } + + iofs, err := inferFileIOFromSchema(location, props) + if err != nil { + return nil, err + } + + if iofs == nil { + iofs = LocalFS{} + } + + return iofs, nil +} diff --git a/io/local.go b/io/local.go index 560d9be..cb80e2a 100644 --- a/io/local.go +++ b/io/local.go @@ -1,35 +1,35 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "os" - "strings" -) - -// LocalFS is an implementation of IO that implements interaction with -// the local file system. -type LocalFS struct{} - -func (LocalFS) Open(name string) (File, error) { - return os.Open(strings.TrimPrefix(name, "file://")) -} - -func (LocalFS) Remove(name string) error { - return os.Remove(name) -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "os" + "strings" +) + +// LocalFS is an implementation of IO that implements interaction with +// the local file system. +type LocalFS struct{} + +func (LocalFS) Open(name string) (File, error) { + return os.Open(strings.TrimPrefix(name, "file://")) +} + +func (LocalFS) Remove(name string) error { + return os.Remove(name) +} diff --git a/io/s3.go b/io/s3.go index 7396130..31fb074 100644 --- a/io/s3.go +++ b/io/s3.go @@ -1,114 +1,142 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "fmt" - "net/http" - "net/url" - "os" - "strings" - - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/smithy-go/auth/bearer" - "github.com/wolfeidau/s3iofs" -) - -// Constants for S3 configuration options -const ( - S3Region = "s3.region" - S3SessionToken = "s3.session-token" - S3SecretAccessKey = "s3.secret-access-key" - S3AccessKeyID = "s3.access-key-id" - S3EndpointURL = "s3.endpoint" - S3ProxyURI = "s3.proxy-uri" -) - -func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) { - cfgOpts := []func(*config.LoadOptions) error{} - opts := []func(*s3.Options){} - - endpoint, ok := props[S3EndpointURL] - if !ok { - endpoint = os.Getenv("AWS_S3_ENDPOINT") - } - - if endpoint != "" { - opts = append(opts, func(o *s3.Options) { - o.BaseEndpoint = &endpoint - }) - } - - if tok, ok := props["token"]; ok { - cfgOpts = append(cfgOpts, config.WithBearerAuthTokenProvider( - &bearer.StaticTokenProvider{Token: bearer.Token{Value: tok}})) - } - - if region, ok := props[S3Region]; ok { - opts = append(opts, func(o *s3.Options) { - o.Region = region - }) - } else if region, ok := props["client.region"]; ok { - opts = append(opts, func(o *s3.Options) { - o.Region = region - }) - } - - accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey] - token := props[S3SessionToken] - if accessKey != "" || secretAccessKey != "" || token != "" { - opts = append(opts, func(o *s3.Options) { - o.Credentials = credentials.NewStaticCredentialsProvider( - props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken]) - }) - } - - if proxy, ok := props[S3ProxyURI]; ok { - proxyURL, err := url.Parse(proxy) - if err != nil { - return nil, fmt.Errorf("invalid s3 proxy url '%s'", proxy) - } - - opts = append(opts, func(o *s3.Options) { - o.HTTPClient = awshttp.NewBuildableClient().WithTransportOptions( - func(t *http.Transport) { t.Proxy = http.ProxyURL(proxyURL) }) - }) - } - - awscfg, err := config.LoadDefaultConfig(context.Background(), cfgOpts...) - if err != nil { - return nil, err - } - - s3Client := s3.NewFromConfig(awscfg, opts...) - preprocess := func(n string) string { - _, after, found := strings.Cut(n, "://") - if found { - n = after - } - - return strings.TrimPrefix(n, parsed.Host) - } - - s3fs := s3iofs.NewWithClient(parsed.Host, s3Client) - return FSPreProcName(s3fs, preprocess), nil -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "slices" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/auth/bearer" + "github.com/wolfeidau/s3iofs" +) + +// Constants for S3 configuration options +const ( + S3Region = "s3.region" + S3SessionToken = "s3.session-token" + S3SecretAccessKey = "s3.secret-access-key" + S3AccessKeyID = "s3.access-key-id" + S3EndpointURL = "s3.endpoint" + S3ProxyURI = "s3.proxy-uri" + S3ConnectTimeout = "s3.connect-timeout" + S3SignerUri = "s3.signer.uri" +) + +var unsupportedS3Props = []string{ + S3ConnectTimeout, + S3SignerUri, +} + +func ParseAWSConfig(props map[string]string) (*aws.Config, error) { + // If any unsupported properties are set, return an error. + for k := range props { + if slices.Contains(unsupportedS3Props, k) { + return nil, fmt.Errorf("unsupported S3 property %q", k) + } + } + + opts := []func(*config.LoadOptions) error{} + endpoint, ok := props[S3EndpointURL] + if !ok { + endpoint = os.Getenv("AWS_S3_ENDPOINT") + } + + if endpoint != "" { + opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if service != s3.ServiceID { + // fallback to default resolution for the service + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + } + + return aws.Endpoint{ + URL: endpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + }))) + } + + if tok, ok := props["token"]; ok { + opts = append(opts, config.WithBearerAuthTokenProvider( + &bearer.StaticTokenProvider{Token: bearer.Token{Value: tok}})) + } + + if region, ok := props[S3Region]; ok { + opts = append(opts, config.WithRegion(region)) + } else if region, ok := props["client.region"]; ok { + opts = append(opts, config.WithRegion(region)) + } + + accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey] + token := props[S3SessionToken] + if accessKey != "" || secretAccessKey != "" || token != "" { + opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken]))) + } + + if proxy, ok := props[S3ProxyURI]; ok { + proxyURL, err := url.Parse(proxy) + if err != nil { + return nil, fmt.Errorf("invalid s3 proxy url '%s'", proxy) + } + + opts = append(opts, config.WithHTTPClient(awshttp.NewBuildableClient().WithTransportOptions( + func(t *http.Transport) { + t.Proxy = http.ProxyURL(proxyURL) + }, + ))) + } + + awscfg := new(aws.Config) + var err error + *awscfg, err = config.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, err + } + + return awscfg, nil +} + +func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) { + awscfg, err := ParseAWSConfig(props) + if err != nil { + return nil, err + } + + preprocess := func(n string) string { + _, after, found := strings.Cut(n, "://") + if found { + n = after + } + + return strings.TrimPrefix(n, parsed.Host) + } + + s3fs := s3iofs.New(parsed.Host, *awscfg) + return FSPreProcName(s3fs, preprocess), nil +} diff --git a/io/s3_cdk.go b/io/s3_cdk.go new file mode 100644 index 0000000..b712bb7 --- /dev/null +++ b/io/s3_cdk.go @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "net/url" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "gocloud.dev/blob" + "gocloud.dev/blob/s3blob" +) + +// Construct a S3 bucket from a URL +func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + awscfg, err := ParseAWSConfig(props) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(*awscfg) + + // Create a *blob.Bucket. + bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) + if err != nil { + return nil, err + } + + return bucket, nil +} From 9e89fa935f1bdf67b3b0c6121db5c08e8c6fc983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Wed, 16 Oct 2024 16:44:07 -0400 Subject: [PATCH 02/14] fix comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Alleyne --- io/io.go | 1 - 1 file changed, 1 deletion(-) diff --git a/io/io.go b/io/io.go index 6aeee25..54a5e6d 100644 --- a/io/io.go +++ b/io/io.go @@ -71,7 +71,6 @@ type WriteFileIO interface { IO // WriteFile writes p to the named file. - // An error will be returned if the file already exists. Write(name string, p []byte) error Close() error } From 69c1e2ae62bb1ad97673196cbb793023bb4ae7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Thu, 17 Oct 2024 16:53:07 -0400 Subject: [PATCH 03/14] refactor CreateBlobFileIO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Alleyne --- go.mod | 37 +++++++- go.sum | 186 ++++++++++++++++++++++++++++++++++++-- io/blob.go | 28 +----- io/{gcs_cdk.go => gcs.go} | 0 io/io.go | 24 ++++- io/s3.go | 20 ++-- io/s3_cdk.go | 45 --------- 7 files changed, 242 insertions(+), 98 deletions(-) rename io/{gcs_cdk.go => gcs.go} (100%) delete mode 100644 io/s3_cdk.go diff --git a/go.mod b/go.mod index 76f3594..178f9de 100644 --- a/go.mod +++ b/go.mod @@ -36,8 +36,9 @@ require ( github.com/stretchr/testify v1.10.0 github.com/substrait-io/substrait-go v1.2.0 github.com/twmb/murmur3 v1.1.8 - github.com/wolfeidau/s3iofs v1.5.2 - golang.org/x/sync v0.10.0 + gocloud.dev v0.40.0 + golang.org/x/sync v0.8.0 + google.golang.org/api v0.201.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,12 +46,20 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.8 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/storage v1.43.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/alecthomas/participle/v2 v2.1.0 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/apache/thrift v0.21.0 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect @@ -66,11 +75,20 @@ require ( github.com/creasty/defaults v1.8.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/goccy/go-yaml v1.11.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/wire v0.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gookit/color v1.5.4 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -91,15 +109,26 @@ require ( github.com/substrait-io/substrait v0.57.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/zeebo/xxh3 v1.0.2 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.26.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect + google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/go.sum b/go.sum index 4a35697..ac5b567 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,22 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= +cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= +cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= @@ -30,6 +46,8 @@ github.com/apache/arrow-go/v18 v18.0.1-0.20241029153821-f0c5d9939d3f/go.mod h1:k github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= @@ -40,6 +58,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgb github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 h1:zeN9UtUlA6FTx0vFSayxSX32HDw73Yb6Hh2izDSFxXY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10/go.mod h1:3HKuexPDcwLWPaqpW2UR/9n8N/u/3CKcGAzSs8p8u8g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= @@ -68,6 +88,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLb github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= @@ -77,8 +100,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= @@ -89,15 +123,53 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54= github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= +github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -106,6 +178,10 @@ github.com/hamba/avro/v2 v2.27.0 h1:IAM4lQ0VzUIKBuo4qlAiLKfqALSrFC+zi1iseTtbBKU= github.com/hamba/avro/v2 v2.27.0/go.mod h1:jN209lopfllfrz7IGoZErlDz+AyUJ3vrBePQFZwYf5I= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= @@ -151,6 +227,7 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE= @@ -168,12 +245,17 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/substrait-io/substrait v0.57.1 h1:GW8nnYfSowMseHR8Os82/X6lNtQGIK7p4p+lr6r+auw= @@ -182,8 +264,6 @@ github.com/substrait-io/substrait-go v1.2.0 h1:3ZNRkc8FYD7ifCagKEOZQtUcgMceMQfwo github.com/substrait-io/substrait-go v1.2.0/go.mod h1:IPsy24rdjp/buXR+T8ENl6QCnSCS6h+uM8P+GaZez7c= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/wolfeidau/s3iofs v1.5.2 h1:2dmzSxdrSY29GsILVheJqBbURVQX3KglggSBtVWCYj4= -github.com/wolfeidau/s3iofs v1.5.2/go.mod h1:fPAKzdWmZ1Z2L9vnqL6d1eb7pVsUgkUstxQUkG1HIHA= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= @@ -194,28 +274,72 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +gocloud.dev v0.40.0 h1:f8LgP+4WDqOG/RXoUcyLpeIAGOcAbZrZbDQCUee10ng= +gocloud.dev v0.40.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -227,6 +351,9 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -234,6 +361,9 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -241,23 +371,59 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= +google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 h1:nFS3IivktIU5Mk6KQa+v6RKkHUpdQpphqGNLxqNnbEk= +google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:tEzYTYZxbmVNOu0OAFH9HzdJtLn6h4Aj89zzlBCdHms= +google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f h1:jTm13A2itBi3La6yTGqn8bVSrc3ZZ1r8ENHlIXBfnRA= +google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f/go.mod h1:CLGoBuH1VHxAUXVPP8FfPwPEVJB6lz3URE5mY2SuayE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -266,7 +432,11 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/io/blob.go b/io/blob.go index 474ba25..1b15ec6 100644 --- a/io/blob.go +++ b/io/blob.go @@ -27,7 +27,6 @@ import ( "time" "gocloud.dev/blob" - "gocloud.dev/blob/memblob" ) // iofsFileInfo describes a single file in an io/fs.FS. @@ -294,31 +293,10 @@ func urlToBucketPath(parsed *url.URL) (string, string) { return parsed.Host, parsed.Path } -// Create a new BlobFileIO instance -func CreateBlobFileIO(parsed *url.URL, props map[string]string) (*BlobFileIO, error) { +// CreateBlobFileIO creates a new BlobFileIO instance +func CreateBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *BlobFileIO { ctx := context.Background() - - var bucket *blob.Bucket - var err error - switch parsed.Scheme { - case "mem": - // memblob doesn't use the URL host or path - bucket = memblob.OpenBucket(nil) - case "s3", "s3a", "s3n": - bucket, err = createS3Bucket(ctx, parsed, props) - case "gs": - bucket, err = createGCSBucket(ctx, parsed, props) - } - - if err != nil { - return nil, err - } - - if parsed.Path != "" && parsed.Path != "/" { - bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) - } - - return &BlobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path}, nil + return &BlobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} } type BlobWriteFile struct { diff --git a/io/gcs_cdk.go b/io/gcs.go similarity index 100% rename from io/gcs_cdk.go rename to io/gcs.go diff --git a/io/io.go b/io/io.go index 54a5e6d..799c878 100644 --- a/io/io.go +++ b/io/io.go @@ -18,12 +18,16 @@ package io import ( + "context" "errors" "fmt" "io" "io/fs" "net/url" "strings" + + "gocloud.dev/blob" + "gocloud.dev/blob/memblob" ) // IO is an interface to a hierarchical file system. @@ -221,22 +225,32 @@ func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { if err != nil { return nil, err } + var bucket *blob.Bucket + ctx := context.Background() switch parsed.Scheme { case "s3", "s3a", "s3n": - if props["s3.use-cdk"] == "true" { - return CreateBlobFileIO(parsed, props) + bucket, err = createS3Bucket(ctx, parsed, props) + if err != nil { + return nil, err } - return createS3FileIO(parsed, props) case "gs": - return CreateBlobFileIO(parsed, props) + bucket, err = createGCSBucket(ctx, parsed, props) + if err != nil { + return nil, err + } case "mem": - return CreateBlobFileIO(parsed, props) + // memblob doesn't use the URL host or path + bucket = memblob.OpenBucket(nil) case "file", "": return LocalFS{}, nil default: return nil, fmt.Errorf("IO for file '%s' not implemented", path) } + if parsed.Path != "" && parsed.Path != "/" { + bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) + } + return CreateBlobFileIO(parsed, bucket), nil } // LoadFS takes a map of properties and an optional URI location diff --git a/io/s3.go b/io/s3.go index 31fb074..22bb784 100644 --- a/io/s3.go +++ b/io/s3.go @@ -24,7 +24,6 @@ import ( "net/url" "os" "slices" - "strings" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" @@ -32,7 +31,8 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go/auth/bearer" - "github.com/wolfeidau/s3iofs" + "gocloud.dev/blob" + "gocloud.dev/blob/s3blob" ) // Constants for S3 configuration options @@ -122,21 +122,19 @@ func ParseAWSConfig(props map[string]string) (*aws.Config, error) { return awscfg, nil } -func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) { +func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { awscfg, err := ParseAWSConfig(props) if err != nil { return nil, err } - preprocess := func(n string) string { - _, after, found := strings.Cut(n, "://") - if found { - n = after - } + client := s3.NewFromConfig(*awscfg) - return strings.TrimPrefix(n, parsed.Host) + // Create a *blob.Bucket. + bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) + if err != nil { + return nil, err } - s3fs := s3iofs.New(parsed.Host, *awscfg) - return FSPreProcName(s3fs, preprocess), nil + return bucket, nil } diff --git a/io/s3_cdk.go b/io/s3_cdk.go deleted file mode 100644 index b712bb7..0000000 --- a/io/s3_cdk.go +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "net/url" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "gocloud.dev/blob" - "gocloud.dev/blob/s3blob" -) - -// Construct a S3 bucket from a URL -func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { - awscfg, err := ParseAWSConfig(props) - if err != nil { - return nil, err - } - - client := s3.NewFromConfig(*awscfg) - - // Create a *blob.Bucket. - bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) - if err != nil { - return nil, err - } - - return bucket, nil -} From b89431d41564559f460e00036dc20e60fa3ab256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Thu, 14 Nov 2024 17:32:14 -0500 Subject: [PATCH 04/14] line endings to LF --- io/blob.go | 622 ++++++++++++++++++++++++++-------------------------- io/gcs.go | 127 +++++------ io/io.go | 558 +++++++++++++++++++++++----------------------- io/local.go | 70 +++--- io/s3.go | 281 ++++++++++++------------ 5 files changed, 830 insertions(+), 828 deletions(-) diff --git a/io/blob.go b/io/blob.go index 1b15ec6..8fa4b15 100644 --- a/io/blob.go +++ b/io/blob.go @@ -1,311 +1,311 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "io" - "io/fs" - "net/url" - "path/filepath" - "strings" - "time" - - "gocloud.dev/blob" -) - -// iofsFileInfo describes a single file in an io/fs.FS. -// It implements fs.FileInfo and fs.DirEntry. -// Copied from `gocloud.dev/blob.iofsDir` because it is private -type iofsFileInfo struct { - lo *blob.ListObject - name string -} - -func (f *iofsFileInfo) Name() string { return f.name } -func (f *iofsFileInfo) Size() int64 { return f.lo.Size } -func (f *iofsFileInfo) Mode() fs.FileMode { return fs.ModeIrregular } -func (f *iofsFileInfo) ModTime() time.Time { return f.lo.ModTime } -func (f *iofsFileInfo) IsDir() bool { return false } -func (f *iofsFileInfo) Sys() interface{} { return f.lo } -func (f *iofsFileInfo) Info() (fs.FileInfo, error) { return f, nil } -func (f *iofsFileInfo) Type() fs.FileMode { return fs.ModeIrregular } - -// iofsDir describes a single directory in an `iceberg-go/io.FS`. -// It implements `io/fs.FileInfo`, `io/fs.DirEntry`, and `io/fs.File`. -// Copied from `gocloud.dev/blob.iofsDir`, but modified to use `iceberg-go/io.File` instead of `io/fs.File` -type iofsDir struct { - b *BlobFileIO - key string - name string - // If opened is true, we've read entries via openOnce(). - opened bool - entries []fs.DirEntry - offset int -} - -func newDir(b *BlobFileIO, key, name string) *iofsDir { - return &iofsDir{b: b, key: key, name: name} -} - -func (d *iofsDir) Name() string { return d.name } -func (d *iofsDir) Size() int64 { return 0 } -func (d *iofsDir) Mode() fs.FileMode { return fs.ModeDir } -func (d *iofsDir) Type() fs.FileMode { return fs.ModeDir } -func (d *iofsDir) ModTime() time.Time { return time.Time{} } -func (d *iofsDir) IsDir() bool { return true } -func (d *iofsDir) Sys() interface{} { return d } -func (d *iofsDir) Info() (fs.FileInfo, error) { return d, nil } -func (d *iofsDir) Stat() (fs.FileInfo, error) { return d, nil } -func (d *iofsDir) Read([]byte) (int, error) { - return 0, &fs.PathError{Op: "read", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) ReadAt(p []byte, off int64) (int, error) { - return 0, &fs.PathError{Op: "readAt", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) Seek(offset int64, whence int) (int64, error) { - return 0, &fs.PathError{Op: "seek", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) Close() error { return nil } -func (d *iofsDir) ReadDir(count int) ([]fs.DirEntry, error) { - if err := d.openOnce(); err != nil { - return nil, err - } - n := len(d.entries) - d.offset - if n == 0 && count > 0 { - return nil, io.EOF - } - if count > 0 && n > count { - n = count - } - list := make([]fs.DirEntry, n) - for i := range list { - list[i] = d.entries[d.offset+i] - } - d.offset += n - return list, nil -} - -func (d *iofsDir) openOnce() error { - if d.opened { - return nil - } - d.opened = true - - // blob expects directories to end in the delimiter, except at the top level. - prefix := d.key - if prefix != "" { - prefix += "/" - } - listOpts := blob.ListOptions{ - Prefix: prefix, - Delimiter: "/", - } - ctx := d.b.ctx - - // Fetch all the directory entries. - // Conceivably we could only fetch a few here, and fetch the rest lazily - // on demand, but that would add significant complexity. - iter := d.b.List(&listOpts) - for { - item, err := iter.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return err - } - name := filepath.Base(item.Key) - if item.IsDir { - d.entries = append(d.entries, newDir(d.b, item.Key, name)) - } else { - d.entries = append(d.entries, &iofsFileInfo{item, name}) - } - } - // There is no such thing as an empty directory in Bucket, so if - // we didn't find anything, it doesn't exist. - if len(d.entries) == 0 { - return fs.ErrNotExist - } - return nil -} - -// blobOpenFile describes a single open blob as a File. -// It implements the iceberg-go/io.File interface. -// It is based on gocloud.dev/blob.iofsOpenFile which: -// - Doesn't support the `io.ReaderAt` interface -// - Is not externally accessible, so copied here -type blobOpenFile struct { - *blob.Reader - name string -} - -func (f *blobOpenFile) ReadAt(p []byte, off int64) (int, error) { - finalOff, err := f.Reader.Seek(off, io.SeekStart) - if err != nil { - return -1, err - } else if finalOff != off { - return -1, io.ErrUnexpectedEOF - } - - return f.Read(p) -} - -// Functions to implement the `Stat()` function in the `io/fs.File` interface - -func (f *blobOpenFile) Name() string { return f.name } -func (f *blobOpenFile) Mode() fs.FileMode { return fs.ModeIrregular } -func (f *blobOpenFile) Sys() interface{} { return f.Reader } -func (f *blobOpenFile) IsDir() bool { return false } -func (f *blobOpenFile) Stat() (fs.FileInfo, error) { return f, nil } - -// BlobFileIO represents a file system backed by a bucket in object store. It implements the `iceberg-go/io.FileIO` interface. -type BlobFileIO struct { - *blob.Bucket - ctx context.Context - opts *blob.ReaderOptions - prefix string -} - -func (io *BlobFileIO) preprocess(n string) string { - _, after, found := strings.Cut(n, "://") - if found { - n = after - } - - out := strings.TrimPrefix(n, io.prefix) - if out == "/" { - out = "." - } else { - out = strings.TrimPrefix(out, "/") - } - - return out -} - -// Open a Blob from a Bucket using the BlobFileIO. Note this -// function is copied from blob.Bucket.Open, but extended to -// return a iceberg-go/io.File instance instead of io/fs.File -func (io *BlobFileIO) Open(path string) (File, error) { - if _, err := url.Parse(path); err != nil { - return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid} - } - path = io.preprocess(path) - - var isDir bool - var key, name string // name is the last part of the path - if path == "." { - // Root is always a directory, but blob doesn't want the "." in the key. - isDir = true - key, name = "", "." - } else { - exists, _ := io.Bucket.Exists(io.ctx, path) - isDir = !exists - key, name = path, filepath.Base(path) - } - - // If it's a directory, list the directory contents. We can't do this lazily - // because we need to error out here if it doesn't exist. - if isDir { - dir := newDir(io, key, name) - err := dir.openOnce() - if err != nil { - if err == fs.ErrNotExist && path == "." { - // The root directory must exist. - return dir, nil - } - return nil, &fs.PathError{Op: "open", Path: path, Err: err} - } - return dir, nil - } - - // It's a file; open it and return a wrapper. - r, err := io.Bucket.NewReader(io.ctx, path, io.opts) - if err != nil { - return nil, &fs.PathError{Op: "open", Path: path, Err: err} - } - - return &blobOpenFile{Reader: r, name: name}, nil -} - -// Remove a Blob from a Bucket using the BlobFileIO -func (io *BlobFileIO) Remove(path string) error { - if !fs.ValidPath(path) { - return &fs.PathError{Op: "remove", Path: path, Err: fs.ErrInvalid} - } - path = io.preprocess(path) - - return io.Bucket.Delete(io.ctx, path) -} - -// NewWriter returns a Writer that writes to the blob stored at path. -// A nil WriterOptions is treated the same as the zero value. -// -// If overwrite is disabled and a blob with this path already exists, -// an error will be returned. -// -// The returned Writer will store ctx for later use in Write and/or Close. -// To abort a write, cancel ctx; otherwise, it must remain open until -// Close is called. -// -// The caller must call Close on the returned Writer, even if the write is -// aborted. -func (io *BlobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOptions) (w *BlobWriteFile, err error) { - if !fs.ValidPath(path) { - return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} - } - path = io.preprocess(path) - if !overwrite { - if exists, err := io.Bucket.Exists(io.ctx, path); exists { - if err != nil { - return nil, &fs.PathError{Op: "new writer", Path: path, Err: err} - } - return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} - } - } - bw, err := io.Bucket.NewWriter(io.ctx, path, opts) - if err != nil { - return nil, err - } - return &BlobWriteFile{ - Writer: bw, - name: path, - opts: opts}, - nil -} - -func urlToBucketPath(parsed *url.URL) (string, string) { - return parsed.Host, parsed.Path -} - -// CreateBlobFileIO creates a new BlobFileIO instance -func CreateBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *BlobFileIO { - ctx := context.Background() - return &BlobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} -} - -type BlobWriteFile struct { - *blob.Writer - name string - opts *blob.WriterOptions -} - -func (f *BlobWriteFile) Name() string { return f.name } -func (f *BlobWriteFile) Sys() interface{} { return f.Writer } -func (f *BlobWriteFile) Close() error { return f.Writer.Close() } -func (f *BlobWriteFile) Write(p []byte) (int, error) { return f.Writer.Write(p) } +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "io" + "io/fs" + "net/url" + "path/filepath" + "strings" + "time" + + "gocloud.dev/blob" +) + +// iofsFileInfo describes a single file in an io/fs.FS. +// It implements fs.FileInfo and fs.DirEntry. +// Copied from `gocloud.dev/blob.iofsDir` because it is private +type iofsFileInfo struct { + lo *blob.ListObject + name string +} + +func (f *iofsFileInfo) Name() string { return f.name } +func (f *iofsFileInfo) Size() int64 { return f.lo.Size } +func (f *iofsFileInfo) Mode() fs.FileMode { return fs.ModeIrregular } +func (f *iofsFileInfo) ModTime() time.Time { return f.lo.ModTime } +func (f *iofsFileInfo) IsDir() bool { return false } +func (f *iofsFileInfo) Sys() interface{} { return f.lo } +func (f *iofsFileInfo) Info() (fs.FileInfo, error) { return f, nil } +func (f *iofsFileInfo) Type() fs.FileMode { return fs.ModeIrregular } + +// iofsDir describes a single directory in an `iceberg-go/io.FS`. +// It implements `io/fs.FileInfo`, `io/fs.DirEntry`, and `io/fs.File`. +// Copied from `gocloud.dev/blob.iofsDir`, but modified to use `iceberg-go/io.File` instead of `io/fs.File` +type iofsDir struct { + b *blobFileIO + key string + name string + // If opened is true, we've read entries via openOnce(). + opened bool + entries []fs.DirEntry + offset int +} + +func newDir(b *blobFileIO, key, name string) *iofsDir { + return &iofsDir{b: b, key: key, name: name} +} + +func (d *iofsDir) Name() string { return d.name } +func (d *iofsDir) Size() int64 { return 0 } +func (d *iofsDir) Mode() fs.FileMode { return fs.ModeDir } +func (d *iofsDir) Type() fs.FileMode { return fs.ModeDir } +func (d *iofsDir) ModTime() time.Time { return time.Time{} } +func (d *iofsDir) IsDir() bool { return true } +func (d *iofsDir) Sys() interface{} { return d } +func (d *iofsDir) Info() (fs.FileInfo, error) { return d, nil } +func (d *iofsDir) Stat() (fs.FileInfo, error) { return d, nil } +func (d *iofsDir) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) ReadAt(p []byte, off int64) (int, error) { + return 0, &fs.PathError{Op: "readAt", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) Seek(offset int64, whence int) (int64, error) { + return 0, &fs.PathError{Op: "seek", Path: d.key, Err: fs.ErrInvalid} +} +func (d *iofsDir) Close() error { return nil } +func (d *iofsDir) ReadDir(count int) ([]fs.DirEntry, error) { + if err := d.openOnce(); err != nil { + return nil, err + } + n := len(d.entries) - d.offset + if n == 0 && count > 0 { + return nil, io.EOF + } + if count > 0 && n > count { + n = count + } + list := make([]fs.DirEntry, n) + for i := range list { + list[i] = d.entries[d.offset+i] + } + d.offset += n + return list, nil +} + +func (d *iofsDir) openOnce() error { + if d.opened { + return nil + } + d.opened = true + + // blob expects directories to end in the delimiter, except at the top level. + prefix := d.key + if prefix != "" { + prefix += "/" + } + listOpts := blob.ListOptions{ + Prefix: prefix, + Delimiter: "/", + } + ctx := d.b.ctx + + // Fetch all the directory entries. + // Conceivably we could only fetch a few here, and fetch the rest lazily + // on demand, but that would add significant complexity. + iter := d.b.List(&listOpts) + for { + item, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return err + } + name := filepath.Base(item.Key) + if item.IsDir { + d.entries = append(d.entries, newDir(d.b, item.Key, name)) + } else { + d.entries = append(d.entries, &iofsFileInfo{item, name}) + } + } + // There is no such thing as an empty directory in Bucket, so if + // we didn't find anything, it doesn't exist. + if len(d.entries) == 0 { + return fs.ErrNotExist + } + return nil +} + +// blobOpenFile describes a single open blob as a File. +// It implements the iceberg-go/io.File interface. +// It is based on gocloud.dev/blob.iofsOpenFile which: +// - Doesn't support the `io.ReaderAt` interface +// - Is not externally accessible, so copied here +type blobOpenFile struct { + *blob.Reader + name string +} + +func (f *blobOpenFile) ReadAt(p []byte, off int64) (int, error) { + finalOff, err := f.Reader.Seek(off, io.SeekStart) + if err != nil { + return -1, err + } else if finalOff != off { + return -1, io.ErrUnexpectedEOF + } + + return f.Read(p) +} + +// Functions to implement the `Stat()` function in the `io/fs.File` interface + +func (f *blobOpenFile) Name() string { return f.name } +func (f *blobOpenFile) Mode() fs.FileMode { return fs.ModeIrregular } +func (f *blobOpenFile) Sys() interface{} { return f.Reader } +func (f *blobOpenFile) IsDir() bool { return false } +func (f *blobOpenFile) Stat() (fs.FileInfo, error) { return f, nil } + +// blobFileIO represents a file system backed by a bucket in object store. It implements the `iceberg-go/io.FileIO` interface. +type blobFileIO struct { + *blob.Bucket + ctx context.Context + opts *blob.ReaderOptions + prefix string +} + +func (io *blobFileIO) preprocess(n string) string { + _, after, found := strings.Cut(n, "://") + if found { + n = after + } + + out := strings.TrimPrefix(n, io.prefix) + if out == "/" { + out = "." + } else { + out = strings.TrimPrefix(out, "/") + } + + return out +} + +// Open a Blob from a Bucket using the blobFileIO. Note this +// function is copied from blob.Bucket.Open, but extended to +// return a iceberg-go/io.File instance instead of io/fs.File +func (io *blobFileIO) Open(path string) (File, error) { + if _, err := url.Parse(path); err != nil { + return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + + var isDir bool + var key, name string // name is the last part of the path + if path == "." { + // Root is always a directory, but blob doesn't want the "." in the key. + isDir = true + key, name = "", "." + } else { + exists, _ := io.Bucket.Exists(io.ctx, path) + isDir = !exists + key, name = path, filepath.Base(path) + } + + // If it's a directory, list the directory contents. We can't do this lazily + // because we need to error out here if it doesn't exist. + if isDir { + dir := newDir(io, key, name) + err := dir.openOnce() + if err != nil { + if err == fs.ErrNotExist && path == "." { + // The root directory must exist. + return dir, nil + } + return nil, &fs.PathError{Op: "open", Path: path, Err: err} + } + return dir, nil + } + + // It's a file; open it and return a wrapper. + r, err := io.Bucket.NewReader(io.ctx, path, io.opts) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: path, Err: err} + } + + return &blobOpenFile{Reader: r, name: name}, nil +} + +// Remove a Blob from a Bucket using the blobFileIO +func (io *blobFileIO) Remove(path string) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "remove", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + + return io.Bucket.Delete(io.ctx, path) +} + +// NewWriter returns a Writer that writes to the blob stored at path. +// A nil WriterOptions is treated the same as the zero value. +// +// If overwrite is disabled and a blob with this path already exists, +// an error will be returned. +// +// The returned Writer will store ctx for later use in Write and/or Close. +// To abort a write, cancel ctx; otherwise, it must remain open until +// Close is called. +// +// The caller must call Close on the returned Writer, even if the write is +// aborted. +func (io *blobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOptions) (w *BlobWriteFile, err error) { + if !fs.ValidPath(path) { + return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} + } + path = io.preprocess(path) + if !overwrite { + if exists, err := io.Bucket.Exists(io.ctx, path); exists { + if err != nil { + return nil, &fs.PathError{Op: "new writer", Path: path, Err: err} + } + return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} + } + } + bw, err := io.Bucket.NewWriter(io.ctx, path, opts) + if err != nil { + return nil, err + } + return &BlobWriteFile{ + Writer: bw, + name: path, + opts: opts}, + nil +} + +func urlToBucketPath(parsed *url.URL) (string, string) { + return parsed.Host, parsed.Path +} + +// createblobFileIO creates a new blobFileIO instance +func createBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { + ctx := context.Background() + return &blobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} +} + +type BlobWriteFile struct { + *blob.Writer + name string + opts *blob.WriterOptions +} + +func (f *BlobWriteFile) Name() string { return f.name } +func (f *BlobWriteFile) Sys() interface{} { return f.Writer } +func (f *BlobWriteFile) Close() error { return f.Writer.Close() } +func (f *BlobWriteFile) Write(p []byte) (int, error) { return f.Writer.Write(p) } diff --git a/io/gcs.go b/io/gcs.go index 232306f..d7da24b 100644 --- a/io/gcs.go +++ b/io/gcs.go @@ -1,63 +1,64 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "net/url" - - "gocloud.dev/blob" - "gocloud.dev/blob/gcsblob" - "gocloud.dev/gcp" - "google.golang.org/api/option" -) - -// Constants for GCS configuration options -const ( - GCSEndpoint = "gcs.endpoint" - GCSKeyPath = "gcs.keypath" - GCSJSONKey = "gcs.jsonkey" -) - -func ParseGCSConfig(props map[string]string) *gcsblob.Options { - var o []option.ClientOption - if url := props[GCSEndpoint]; url != "" { - o = append(o, option.WithEndpoint(url)) - } - if key := props[GCSJSONKey]; key != "" { - o = append(o, option.WithCredentialsJSON([]byte(key))) - } - if path := props[GCSKeyPath]; path != "" { - o = append(o, option.WithCredentialsFile(path)) - } - return &gcsblob.Options{ - ClientOptions: o, - } -} - -// Construct a S3 bucket from a URL -func createGCSBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { - gcscfg := ParseGCSConfig(props) - client := gcp.NewAnonymousHTTPClient(gcp.DefaultTransport()) - bucket, err := gcsblob.OpenBucket(ctx, client, parsed.Host, gcscfg) - if err != nil { - return nil, err - } - - return bucket, nil -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "net/url" + + "gocloud.dev/blob" + "gocloud.dev/blob/gcsblob" + "gocloud.dev/gcp" + "google.golang.org/api/option" +) + +// Constants for GCS configuration options +const ( + GCSEndpoint = "gcs.endpoint" + GCSKeyPath = "gcs.keypath" + GCSJSONKey = "gcs.jsonkey" +) + +// ParseGCSConfig parses GCS properties and returns a configuration. +func ParseGCSConfig(props map[string]string) *gcsblob.Options { + var o []option.ClientOption + if url := props[GCSEndpoint]; url != "" { + o = append(o, option.WithEndpoint(url)) + } + if key := props[GCSJSONKey]; key != "" { + o = append(o, option.WithCredentialsJSON([]byte(key))) + } + if path := props[GCSKeyPath]; path != "" { + o = append(o, option.WithCredentialsFile(path)) + } + return &gcsblob.Options{ + ClientOptions: o, + } +} + +// Construct a GCS bucket from a URL +func createGCSBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + gcscfg := ParseGCSConfig(props) + client := gcp.NewAnonymousHTTPClient(gcp.DefaultTransport()) + bucket, err := gcsblob.OpenBucket(ctx, client, parsed.Host, gcscfg) + if err != nil { + return nil, err + } + + return bucket, nil +} diff --git a/io/io.go b/io/io.go index 799c878..1f38bbe 100644 --- a/io/io.go +++ b/io/io.go @@ -1,279 +1,279 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "net/url" - "strings" - - "gocloud.dev/blob" - "gocloud.dev/blob/memblob" -) - -// IO is an interface to a hierarchical file system. -// -// The IO interface is the minimum implementation required for a file -// system to utilize an iceberg table. A file system may implement -// additional interfaces, such as ReadFileIO, to provide additional or -// optimized functionality. -type IO interface { - // Open opens the named file. - // - // When Open returns an error, it should be of type *PathError - // with the Op field set to "open", the Path field set to name, - // and the Err field describing the problem. - // - // Open should reject attempts to open names that do not satisfy - // fs.ValidPath(name), returning a *PathError with Err set to - // ErrInvalid or ErrNotExist. - Open(name string) (File, error) - - // Remove removes the named file or (empty) directory. - // - // If there is an error, it will be of type *PathError. - Remove(name string) error -} - -// ReadFileIO is the interface implemented by a file system that -// provides an optimized implementation of ReadFile. -type ReadFileIO interface { - IO - - // ReadFile reads the named file and returns its contents. - // A successful call returns a nil error, not io.EOF. - // (Because ReadFile reads the whole file, the expected EOF - // from the final Read is not treated as an error to be reported.) - // - // The caller is permitted to modify the returned byte slice. - // This method should return a copy of the underlying data. - ReadFile(name string) ([]byte, error) -} - -// WriteFileIO is the interface implemented by a file system that -// provides an optimized implementation of WriteFile -type WriteFileIO interface { - IO - - // WriteFile writes p to the named file. - Write(name string, p []byte) error - Close() error -} - -// A File provides access to a single file. The File interface is the -// minimum implementation required for Iceberg to interact with a file. -// Directory files should also implement -type File interface { - fs.File - io.ReadSeekCloser - io.ReaderAt -} - -// A ReadDirFile is a directory file whose entries can be read with the -// ReadDir method. Every directory file should implement this interface. -// (It is permissible for any file to implement this interface, but -// if so ReadDir should return an error for non-directories.) -type ReadDirFile interface { - File - - // ReadDir read the contents of the directory and returns a slice - // of up to n DirEntry values in directory order. Subsequent calls - // on the same file will yield further DirEntry values. - // - // If n > 0, ReadDir returns at most n DirEntry structures. In this - // case, if ReadDir returns an empty slice, it will return a non-nil - // error explaining why. - // - // At the end of a directory, the error is io.EOF. (ReadDir must return - // io.EOF itself, not an error wrapping io.EOF.) - // - // If n <= 0, ReadDir returns all the DirEntry values from the directory - // in a single slice. In this case, if ReadDir succeeds (reads all the way - // to the end of the directory), it returns the slice and a nil error. - // If it encounters an error before the end of the directory, ReadDir - // returns the DirEntry list read until that point and a non-nil error. - ReadDir(n int) ([]fs.DirEntry, error) -} - -// FS wraps an io/fs.FS as an IO interface. -func FS(fsys fs.FS) IO { - if _, ok := fsys.(fs.ReadFileFS); ok { - return readFileFS{ioFS{fsys, nil}} - } - return ioFS{fsys, nil} -} - -// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then -// it is called to preprocess any filenames before they are passed to -// the underlying fsys. -func FSPreProcName(fsys fs.FS, fn func(string) string) IO { - if _, ok := fsys.(fs.ReadFileFS); ok { - return readFileFS{ioFS{fsys, fn}} - } - return ioFS{fsys, fn} -} - -type readFileFS struct { - ioFS -} - -func (r readFileFS) ReadFile(name string) ([]byte, error) { - if r.preProcessName != nil { - name = r.preProcessName(name) - } - - rfs, ok := r.fsys.(fs.ReadFileFS) - if !ok { - return nil, errMissingReadFile - } - return rfs.ReadFile(name) -} - -type ioFS struct { - fsys fs.FS - - preProcessName func(string) string -} - -func (f ioFS) Open(name string) (File, error) { - if f.preProcessName != nil { - name = f.preProcessName(name) - } - - if name == "/" { - name = "." - } else { - name = strings.TrimPrefix(name, "/") - } - file, err := f.fsys.Open(name) - if err != nil { - return nil, err - } - - return ioFile{file}, nil -} - -func (f ioFS) Remove(name string) error { - r, ok := f.fsys.(interface{ Remove(name string) error }) - if !ok { - return errMissingRemove - } - return r.Remove(name) -} - -var ( - errMissingReadDir = errors.New("fs.File directory missing ReadDir method") - errMissingSeek = errors.New("fs.File missing Seek method") - errMissingReadAt = errors.New("fs.File missing ReadAt") - errMissingRemove = errors.New("fs.FS missing Remove method") - errMissingReadFile = errors.New("fs.FS missing ReadFile method") -) - -type ioFile struct { - file fs.File -} - -func (f ioFile) Close() error { return f.file.Close() } -func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } -func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } -func (f ioFile) Seek(offset int64, whence int) (int64, error) { - s, ok := f.file.(io.Seeker) - if !ok { - return 0, errMissingSeek - } - return s.Seek(offset, whence) -} - -func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) { - r, ok := f.file.(io.ReaderAt) - if !ok { - return 0, errMissingReadAt - } - return r.ReadAt(p, off) -} - -func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) { - d, ok := f.file.(fs.ReadDirFile) - if !ok { - return nil, errMissingReadDir - } - - return d.ReadDir(count) -} - -func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { - parsed, err := url.Parse(path) - if err != nil { - return nil, err - } - var bucket *blob.Bucket - ctx := context.Background() - - switch parsed.Scheme { - case "s3", "s3a", "s3n": - bucket, err = createS3Bucket(ctx, parsed, props) - if err != nil { - return nil, err - } - case "gs": - bucket, err = createGCSBucket(ctx, parsed, props) - if err != nil { - return nil, err - } - case "mem": - // memblob doesn't use the URL host or path - bucket = memblob.OpenBucket(nil) - case "file", "": - return LocalFS{}, nil - default: - return nil, fmt.Errorf("IO for file '%s' not implemented", path) - } - if parsed.Path != "" && parsed.Path != "/" { - bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) - } - return CreateBlobFileIO(parsed, bucket), nil -} - -// LoadFS takes a map of properties and an optional URI location -// and attempts to infer an IO object from it. -// -// A schema of "file://" or an empty string will result in a LocalFS -// implementation. Otherwise this will return an error if the schema -// does not yet have an implementation here. -// -// Currently local, S3, GCS, and In-Memory FSs are implemented. -func LoadFS(props map[string]string, location string) (IO, error) { - if location == "" { - location = props["warehouse"] - } - - iofs, err := inferFileIOFromSchema(location, props) - if err != nil { - return nil, err - } - - if iofs == nil { - iofs = LocalFS{} - } - - return iofs, nil -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "strings" + + "gocloud.dev/blob" + "gocloud.dev/blob/memblob" +) + +// IO is an interface to a hierarchical file system. +// +// The IO interface is the minimum implementation required for a file +// system to utilize an iceberg table. A file system may implement +// additional interfaces, such as ReadFileIO, to provide additional or +// optimized functionality. +type IO interface { + // Open opens the named file. + // + // When Open returns an error, it should be of type *PathError + // with the Op field set to "open", the Path field set to name, + // and the Err field describing the problem. + // + // Open should reject attempts to open names that do not satisfy + // fs.ValidPath(name), returning a *PathError with Err set to + // ErrInvalid or ErrNotExist. + Open(name string) (File, error) + + // Remove removes the named file or (empty) directory. + // + // If there is an error, it will be of type *PathError. + Remove(name string) error +} + +// ReadFileIO is the interface implemented by a file system that +// provides an optimized implementation of ReadFile. +type ReadFileIO interface { + IO + + // ReadFile reads the named file and returns its contents. + // A successful call returns a nil error, not io.EOF. + // (Because ReadFile reads the whole file, the expected EOF + // from the final Read is not treated as an error to be reported.) + // + // The caller is permitted to modify the returned byte slice. + // This method should return a copy of the underlying data. + ReadFile(name string) ([]byte, error) +} + +// WriteFileIO is the interface implemented by a file system that +// provides an optimized implementation of WriteFile +type WriteFileIO interface { + IO + + // WriteFile writes p to the named file. + Write(name string, p []byte) error + Close() error +} + +// A File provides access to a single file. The File interface is the +// minimum implementation required for Iceberg to interact with a file. +// Directory files should also implement +type File interface { + fs.File + io.ReadSeekCloser + io.ReaderAt +} + +// A ReadDirFile is a directory file whose entries can be read with the +// ReadDir method. Every directory file should implement this interface. +// (It is permissible for any file to implement this interface, but +// if so ReadDir should return an error for non-directories.) +type ReadDirFile interface { + File + + // ReadDir read the contents of the directory and returns a slice + // of up to n DirEntry values in directory order. Subsequent calls + // on the same file will yield further DirEntry values. + // + // If n > 0, ReadDir returns at most n DirEntry structures. In this + // case, if ReadDir returns an empty slice, it will return a non-nil + // error explaining why. + // + // At the end of a directory, the error is io.EOF. (ReadDir must return + // io.EOF itself, not an error wrapping io.EOF.) + // + // If n <= 0, ReadDir returns all the DirEntry values from the directory + // in a single slice. In this case, if ReadDir succeeds (reads all the way + // to the end of the directory), it returns the slice and a nil error. + // If it encounters an error before the end of the directory, ReadDir + // returns the DirEntry list read until that point and a non-nil error. + ReadDir(n int) ([]fs.DirEntry, error) +} + +// FS wraps an io/fs.FS as an IO interface. +func FS(fsys fs.FS) IO { + if _, ok := fsys.(fs.ReadFileFS); ok { + return readFileFS{ioFS{fsys, nil}} + } + return ioFS{fsys, nil} +} + +// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then +// it is called to preprocess any filenames before they are passed to +// the underlying fsys. +func FSPreProcName(fsys fs.FS, fn func(string) string) IO { + if _, ok := fsys.(fs.ReadFileFS); ok { + return readFileFS{ioFS{fsys, fn}} + } + return ioFS{fsys, fn} +} + +type readFileFS struct { + ioFS +} + +func (r readFileFS) ReadFile(name string) ([]byte, error) { + if r.preProcessName != nil { + name = r.preProcessName(name) + } + + rfs, ok := r.fsys.(fs.ReadFileFS) + if !ok { + return nil, errMissingReadFile + } + return rfs.ReadFile(name) +} + +type ioFS struct { + fsys fs.FS + + preProcessName func(string) string +} + +func (f ioFS) Open(name string) (File, error) { + if f.preProcessName != nil { + name = f.preProcessName(name) + } + + if name == "/" { + name = "." + } else { + name = strings.TrimPrefix(name, "/") + } + file, err := f.fsys.Open(name) + if err != nil { + return nil, err + } + + return ioFile{file}, nil +} + +func (f ioFS) Remove(name string) error { + r, ok := f.fsys.(interface{ Remove(name string) error }) + if !ok { + return errMissingRemove + } + return r.Remove(name) +} + +var ( + errMissingReadDir = errors.New("fs.File directory missing ReadDir method") + errMissingSeek = errors.New("fs.File missing Seek method") + errMissingReadAt = errors.New("fs.File missing ReadAt") + errMissingRemove = errors.New("fs.FS missing Remove method") + errMissingReadFile = errors.New("fs.FS missing ReadFile method") +) + +type ioFile struct { + file fs.File +} + +func (f ioFile) Close() error { return f.file.Close() } +func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } +func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } +func (f ioFile) Seek(offset int64, whence int) (int64, error) { + s, ok := f.file.(io.Seeker) + if !ok { + return 0, errMissingSeek + } + return s.Seek(offset, whence) +} + +func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) { + r, ok := f.file.(io.ReaderAt) + if !ok { + return 0, errMissingReadAt + } + return r.ReadAt(p, off) +} + +func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, errMissingReadDir + } + + return d.ReadDir(count) +} + +func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { + parsed, err := url.Parse(path) + if err != nil { + return nil, err + } + var bucket *blob.Bucket + ctx := context.Background() + + switch parsed.Scheme { + case "s3", "s3a", "s3n": + bucket, err = createS3Bucket(ctx, parsed, props) + if err != nil { + return nil, err + } + case "gs": + bucket, err = createGCSBucket(ctx, parsed, props) + if err != nil { + return nil, err + } + case "mem": + // memblob doesn't use the URL host or path + bucket = memblob.OpenBucket(nil) + case "file", "": + return LocalFS{}, nil + default: + return nil, fmt.Errorf("IO for file '%s' not implemented", path) + } + if parsed.Path != "" && parsed.Path != "/" { + bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) + } + return createBlobFileIO(parsed, bucket), nil +} + +// LoadFS takes a map of properties and an optional URI location +// and attempts to infer an IO object from it. +// +// A schema of "file://" or an empty string will result in a LocalFS +// implementation. Otherwise this will return an error if the schema +// does not yet have an implementation here. +// +// Currently local, S3, GCS, and In-Memory FSs are implemented. +func LoadFS(props map[string]string, location string) (IO, error) { + if location == "" { + location = props["warehouse"] + } + + iofs, err := inferFileIOFromSchema(location, props) + if err != nil { + return nil, err + } + + if iofs == nil { + iofs = LocalFS{} + } + + return iofs, nil +} diff --git a/io/local.go b/io/local.go index cb80e2a..560d9be 100644 --- a/io/local.go +++ b/io/local.go @@ -1,35 +1,35 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "os" - "strings" -) - -// LocalFS is an implementation of IO that implements interaction with -// the local file system. -type LocalFS struct{} - -func (LocalFS) Open(name string) (File, error) { - return os.Open(strings.TrimPrefix(name, "file://")) -} - -func (LocalFS) Remove(name string) error { - return os.Remove(name) -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "os" + "strings" +) + +// LocalFS is an implementation of IO that implements interaction with +// the local file system. +type LocalFS struct{} + +func (LocalFS) Open(name string) (File, error) { + return os.Open(strings.TrimPrefix(name, "file://")) +} + +func (LocalFS) Remove(name string) error { + return os.Remove(name) +} diff --git a/io/s3.go b/io/s3.go index 22bb784..15bf759 100644 --- a/io/s3.go +++ b/io/s3.go @@ -1,140 +1,141 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package io - -import ( - "context" - "fmt" - "net/http" - "net/url" - "os" - "slices" - - "github.com/aws/aws-sdk-go-v2/aws" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/smithy-go/auth/bearer" - "gocloud.dev/blob" - "gocloud.dev/blob/s3blob" -) - -// Constants for S3 configuration options -const ( - S3Region = "s3.region" - S3SessionToken = "s3.session-token" - S3SecretAccessKey = "s3.secret-access-key" - S3AccessKeyID = "s3.access-key-id" - S3EndpointURL = "s3.endpoint" - S3ProxyURI = "s3.proxy-uri" - S3ConnectTimeout = "s3.connect-timeout" - S3SignerUri = "s3.signer.uri" -) - -var unsupportedS3Props = []string{ - S3ConnectTimeout, - S3SignerUri, -} - -func ParseAWSConfig(props map[string]string) (*aws.Config, error) { - // If any unsupported properties are set, return an error. - for k := range props { - if slices.Contains(unsupportedS3Props, k) { - return nil, fmt.Errorf("unsupported S3 property %q", k) - } - } - - opts := []func(*config.LoadOptions) error{} - endpoint, ok := props[S3EndpointURL] - if !ok { - endpoint = os.Getenv("AWS_S3_ENDPOINT") - } - - if endpoint != "" { - opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - if service != s3.ServiceID { - // fallback to default resolution for the service - return aws.Endpoint{}, &aws.EndpointNotFoundError{} - } - - return aws.Endpoint{ - URL: endpoint, - SigningRegion: region, - HostnameImmutable: true, - }, nil - }))) - } - - if tok, ok := props["token"]; ok { - opts = append(opts, config.WithBearerAuthTokenProvider( - &bearer.StaticTokenProvider{Token: bearer.Token{Value: tok}})) - } - - if region, ok := props[S3Region]; ok { - opts = append(opts, config.WithRegion(region)) - } else if region, ok := props["client.region"]; ok { - opts = append(opts, config.WithRegion(region)) - } - - accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey] - token := props[S3SessionToken] - if accessKey != "" || secretAccessKey != "" || token != "" { - opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken]))) - } - - if proxy, ok := props[S3ProxyURI]; ok { - proxyURL, err := url.Parse(proxy) - if err != nil { - return nil, fmt.Errorf("invalid s3 proxy url '%s'", proxy) - } - - opts = append(opts, config.WithHTTPClient(awshttp.NewBuildableClient().WithTransportOptions( - func(t *http.Transport) { - t.Proxy = http.ProxyURL(proxyURL) - }, - ))) - } - - awscfg := new(aws.Config) - var err error - *awscfg, err = config.LoadDefaultConfig(context.Background(), opts...) - if err != nil { - return nil, err - } - - return awscfg, nil -} - -func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { - awscfg, err := ParseAWSConfig(props) - if err != nil { - return nil, err - } - - client := s3.NewFromConfig(*awscfg) - - // Create a *blob.Bucket. - bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) - if err != nil { - return nil, err - } - - return bucket, nil -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "slices" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/auth/bearer" + "gocloud.dev/blob" + "gocloud.dev/blob/s3blob" +) + +// Constants for S3 configuration options +const ( + S3Region = "s3.region" + S3SessionToken = "s3.session-token" + S3SecretAccessKey = "s3.secret-access-key" + S3AccessKeyID = "s3.access-key-id" + S3EndpointURL = "s3.endpoint" + S3ProxyURI = "s3.proxy-uri" + S3ConnectTimeout = "s3.connect-timeout" + S3SignerUri = "s3.signer.uri" +) + +var unsupportedS3Props = []string{ + S3ConnectTimeout, + S3SignerUri, +} + +// ParseAWSConfig parses S3 properties and returns a configuration. +func ParseAWSConfig(props map[string]string) (*aws.Config, error) { + // If any unsupported properties are set, return an error. + for k := range props { + if slices.Contains(unsupportedS3Props, k) { + return nil, fmt.Errorf("unsupported S3 property %q", k) + } + } + + opts := []func(*config.LoadOptions) error{} + endpoint, ok := props[S3EndpointURL] + if !ok { + endpoint = os.Getenv("AWS_S3_ENDPOINT") + } + + if endpoint != "" { + opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if service != s3.ServiceID { + // fallback to default resolution for the service + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + } + + return aws.Endpoint{ + URL: endpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + }))) + } + + if tok, ok := props["token"]; ok { + opts = append(opts, config.WithBearerAuthTokenProvider( + &bearer.StaticTokenProvider{Token: bearer.Token{Value: tok}})) + } + + if region, ok := props[S3Region]; ok { + opts = append(opts, config.WithRegion(region)) + } else if region, ok := props["client.region"]; ok { + opts = append(opts, config.WithRegion(region)) + } + + accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey] + token := props[S3SessionToken] + if accessKey != "" || secretAccessKey != "" || token != "" { + opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken]))) + } + + if proxy, ok := props[S3ProxyURI]; ok { + proxyURL, err := url.Parse(proxy) + if err != nil { + return nil, fmt.Errorf("invalid s3 proxy url '%s'", proxy) + } + + opts = append(opts, config.WithHTTPClient(awshttp.NewBuildableClient().WithTransportOptions( + func(t *http.Transport) { + t.Proxy = http.ProxyURL(proxyURL) + }, + ))) + } + + awscfg := new(aws.Config) + var err error + *awscfg, err = config.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, err + } + + return awscfg, nil +} + +func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + awscfg, err := ParseAWSConfig(props) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(*awscfg) + + // Create a *blob.Bucket. + bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) + if err != nil { + return nil, err + } + + return bucket, nil +} From 77e216ab72c76726da45f02d31ab5b6a63b81c59 Mon Sep 17 00:00:00 2001 From: Daniel Wilson Date: Sun, 1 Dec 2024 16:04:56 -0700 Subject: [PATCH 05/14] remove deprecated endpoint resolver --- io/blob.go | 4 ++-- io/io.go | 2 +- io/s3.go | 47 +++++++++++++++++++++++++++-------------------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/io/blob.go b/io/blob.go index 8fa4b15..bc58218 100644 --- a/io/blob.go +++ b/io/blob.go @@ -293,8 +293,8 @@ func urlToBucketPath(parsed *url.URL) (string, string) { return parsed.Host, parsed.Path } -// createblobFileIO creates a new blobFileIO instance -func createBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { +// CreateblobFileIO creates a new blobFileIO instance +func CreateBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { ctx := context.Background() return &blobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} } diff --git a/io/io.go b/io/io.go index 1f38bbe..96c08ea 100644 --- a/io/io.go +++ b/io/io.go @@ -250,7 +250,7 @@ func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { if parsed.Path != "" && parsed.Path != "/" { bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) } - return createBlobFileIO(parsed, bucket), nil + return CreateBlobFileIO(parsed, bucket), nil } // LoadFS takes a map of properties and an optional URI location diff --git a/io/s3.go b/io/s3.go index 15bf759..4679a0a 100644 --- a/io/s3.go +++ b/io/s3.go @@ -31,6 +31,7 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go/auth/bearer" + smithyendpoints "github.com/aws/smithy-go/endpoints" "gocloud.dev/blob" "gocloud.dev/blob/s3blob" ) @@ -52,6 +53,24 @@ var unsupportedS3Props = []string{ S3SignerUri, } +type s3endpointResolver struct { + endpoint string +} + +func (r *s3endpointResolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { + if r.endpoint == "" { + return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) + } + + u, err := url.Parse(r.endpoint) + if err != nil { + return smithyendpoints.Endpoint{}, fmt.Errorf("invalid s3 endpoint url '%s'", r.endpoint) + } + return smithyendpoints.Endpoint{ + URI: *u, + }, nil +} + // ParseAWSConfig parses S3 properties and returns a configuration. func ParseAWSConfig(props map[string]string) (*aws.Config, error) { // If any unsupported properties are set, return an error. @@ -62,25 +81,6 @@ func ParseAWSConfig(props map[string]string) (*aws.Config, error) { } opts := []func(*config.LoadOptions) error{} - endpoint, ok := props[S3EndpointURL] - if !ok { - endpoint = os.Getenv("AWS_S3_ENDPOINT") - } - - if endpoint != "" { - opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - if service != s3.ServiceID { - // fallback to default resolution for the service - return aws.Endpoint{}, &aws.EndpointNotFoundError{} - } - - return aws.Endpoint{ - URL: endpoint, - SigningRegion: region, - HostnameImmutable: true, - }, nil - }))) - } if tok, ok := props["token"]; ok { opts = append(opts, config.WithBearerAuthTokenProvider( @@ -129,7 +129,14 @@ func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]strin return nil, err } - client := s3.NewFromConfig(*awscfg) + endpoint, ok := props[S3EndpointURL] + if !ok { + endpoint = os.Getenv("AWS_S3_ENDPOINT") + } + + client := s3.NewFromConfig(*awscfg, func(o *s3.Options) { + o.EndpointResolverV2 = &s3endpointResolver{endpoint: endpoint} + }) // Create a *blob.Bucket. bucket, err := s3blob.OpenBucketV2(ctx, client, parsed.Host, nil) From 015f893cbe2fa2aa31c000ec6707e990b882beb1 Mon Sep 17 00:00:00 2001 From: Daniel Wilson Date: Wed, 11 Dec 2024 13:46:00 -0700 Subject: [PATCH 06/14] unexport createBlobFileIO --- io/blob.go | 4 ++-- io/io.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/io/blob.go b/io/blob.go index bc58218..8fa4b15 100644 --- a/io/blob.go +++ b/io/blob.go @@ -293,8 +293,8 @@ func urlToBucketPath(parsed *url.URL) (string, string) { return parsed.Host, parsed.Path } -// CreateblobFileIO creates a new blobFileIO instance -func CreateBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { +// createblobFileIO creates a new blobFileIO instance +func createBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { ctx := context.Background() return &blobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} } diff --git a/io/io.go b/io/io.go index 96c08ea..1f38bbe 100644 --- a/io/io.go +++ b/io/io.go @@ -250,7 +250,7 @@ func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { if parsed.Path != "" && parsed.Path != "/" { bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) } - return CreateBlobFileIO(parsed, bucket), nil + return createBlobFileIO(parsed, bucket), nil } // LoadFS takes a map of properties and an optional URI location From 251985cd33c12355748866fc2410d8837399e9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Wed, 11 Dec 2024 16:13:12 -0500 Subject: [PATCH 07/14] remove unused urlToBucketPath --- io/blob.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/io/blob.go b/io/blob.go index 8fa4b15..33043ef 100644 --- a/io/blob.go +++ b/io/blob.go @@ -289,10 +289,6 @@ func (io *blobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOp nil } -func urlToBucketPath(parsed *url.URL) (string, string) { - return parsed.Host, parsed.Path -} - // createblobFileIO creates a new blobFileIO instance func createBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { ctx := context.Background() From 93011667ef6b6f8ee47eaad5e207ee20db3e7659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Alleyne?= Date: Thu, 12 Dec 2024 10:36:19 -0500 Subject: [PATCH 08/14] reinit go.mod go.sum --- go.mod | 80 +++++++++++++--------------------- go.sum | 134 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 97 insertions(+), 117 deletions(-) diff --git a/go.mod b/go.mod index 178f9de..220ee40 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,3 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - module github.com/apache/iceberg-go go 1.23 @@ -24,21 +7,21 @@ toolchain go1.23.2 require ( github.com/apache/arrow-go/v18 v18.0.1-0.20241029153821-f0c5d9939d3f github.com/aws/aws-sdk-go-v2 v1.32.6 - github.com/aws/aws-sdk-go-v2/config v1.28.5 - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 - github.com/aws/aws-sdk-go-v2/service/glue v1.102.0 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 + github.com/aws/aws-sdk-go-v2/service/glue v1.103.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/smithy-go v1.22.1 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 github.com/google/uuid v1.6.0 github.com/hamba/avro/v2 v2.27.0 - github.com/pterm/pterm v0.12.79 + github.com/pterm/pterm v0.12.80 github.com/stretchr/testify v1.10.0 - github.com/substrait-io/substrait-go v1.2.0 + github.com/substrait-io/substrait-go v1.1.0 github.com/twmb/murmur3 v1.1.8 gocloud.dev v0.40.0 - golang.org/x/sync v0.8.0 - google.golang.org/api v0.201.0 + golang.org/x/sync v0.10.0 + google.golang.org/api v0.211.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -46,11 +29,11 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.9.8 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.12.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/iam v1.1.13 // indirect cloud.google.com/go/storage v1.43.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/alecthomas/participle/v2 v2.1.0 // indirect @@ -58,7 +41,7 @@ require ( github.com/apache/thrift v0.21.0 // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect @@ -68,9 +51,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/containerd/console v1.0.3 // indirect github.com/creasty/defaults v1.8.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -86,7 +69,7 @@ require ( github.com/google/s2a-go v0.1.8 // indirect github.com/google/wire v0.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/googleapis/gax-go/v2 v2.14.0 // indirect github.com/gookit/color v1.5.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -96,7 +79,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -106,7 +89,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/substrait-io/substrait v0.57.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -115,20 +97,20 @@ require ( go.opentelemetry.io/otel v1.29.0 // indirect go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/term v0.25.0 // indirect - golang.org/x/text v0.19.0 // indirect - golang.org/x/time v0.7.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.8.0 // indirect + golang.org/x/tools v0.28.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect - google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect ) diff --git a/go.sum b/go.sum index ac5b567..c1a5723 100644 --- a/go.sum +++ b/go.sum @@ -7,18 +7,18 @@ atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtE atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= -cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= -cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= -cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4= +cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4= +cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= +cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= -cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= -cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= -cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= -cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= +cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= +cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE= +cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -52,12 +52,12 @@ github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 h1:zeN9UtUlA6FTx0vFSayxSX32HDw73Yb6Hh2izDSFxXY= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10/go.mod h1:3HKuexPDcwLWPaqpW2UR/9n8N/u/3CKcGAzSs8p8u8g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= @@ -68,8 +68,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvK github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= -github.com/aws/aws-sdk-go-v2/service/glue v1.102.0 h1:D6OOWCPCSpjzwfya9hOgDQk3BNvgN1N8ie8bzszq3VU= -github.com/aws/aws-sdk-go-v2/service/glue v1.102.0/go.mod h1:TNh83y7HCK7s/ImCZkiJF/a5/25XZwkvGHtmvDM4y7I= +github.com/aws/aws-sdk-go-v2/service/glue v1.103.0 h1:aWkaWcO6AWKOZgnm45es3lL+jZ9tece2kfQ2Na4xoQs= +github.com/aws/aws-sdk-go-v2/service/glue v1.103.0/go.mod h1:ajiRue7mZ0vQjVHQkQG2KBaPHW8lL5GtvmjRTHWDaqk= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= @@ -80,12 +80,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlm github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -168,8 +168,8 @@ github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= +github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -210,8 +210,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= @@ -235,8 +235,8 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= -github.com/pterm/pterm v0.12.79 h1:lH3yrYMhdpeqX9y5Ep1u7DejyHy7NSQg9qrBjF9dFT4= -github.com/pterm/pterm v0.12.79/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo= +github.com/pterm/pterm v0.12.80 h1:mM55B+GnKUnLMUSqhdINe4s6tOuVQIetQ3my8JGyAIg= +github.com/pterm/pterm v0.12.80/go.mod h1:c6DeF9bSnOSeFPZlfs4ZRAFcf5SCoTwvwQ5xaKGQlHo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -258,10 +258,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/substrait-io/substrait v0.57.1 h1:GW8nnYfSowMseHR8Os82/X6lNtQGIK7p4p+lr6r+auw= -github.com/substrait-io/substrait v0.57.1/go.mod h1:q9s+tjo+gK0lsA+SqYB0lhojNuxvdPdfYlGUP0hjbrA= -github.com/substrait-io/substrait-go v1.2.0 h1:3ZNRkc8FYD7ifCagKEOZQtUcgMceMQfwo2N1NGaK4Q4= -github.com/substrait-io/substrait-go v1.2.0/go.mod h1:IPsy24rdjp/buXR+T8ENl6QCnSCS6h+uM8P+GaZez7c= +github.com/substrait-io/substrait-go v1.1.0 h1:wUXoXV/ESMXgaOWu/05kvMI/UBUyhtaTRfkT5p1b5Ck= +github.com/substrait-io/substrait-go v1.1.0/go.mod h1:LHzL5E0VL620yw4kBQCP+sQPmxhepPTQMDJQRbOe/T4= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -284,8 +282,8 @@ go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= -go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= gocloud.dev v0.40.0 h1:f8LgP+4WDqOG/RXoUcyLpeIAGOcAbZrZbDQCUee10ng= @@ -295,11 +293,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU= +golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -307,8 +305,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -322,11 +320,11 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -334,8 +332,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -354,8 +352,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -364,8 +362,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -373,10 +371,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -387,27 +385,27 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= -google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= -google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= +google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg= +google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 h1:nFS3IivktIU5Mk6KQa+v6RKkHUpdQpphqGNLxqNnbEk= -google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:tEzYTYZxbmVNOu0OAFH9HzdJtLn6h4Aj89zzlBCdHms= -google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f h1:jTm13A2itBi3La6yTGqn8bVSrc3ZZ1r8ENHlIXBfnRA= -google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f/go.mod h1:CLGoBuH1VHxAUXVPP8FfPwPEVJB6lz3URE5mY2SuayE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 h1:CT2Thj5AuPV9phrYMtzX11k+XkzMGfRAet42PmoTATM= +google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1:pgr/4QbFyktUv9CtQ/Fq4gzEE6/Xs7iCXbktaGzLHbQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 h1:IfdSdTcLFy4lqUQrQJLkLt1PB+AsqVz6lwkWPzWEz10= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -424,8 +422,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From b3bb87634ff2e53fb4da4550e5518f92274cb7c4 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 12 Dec 2024 13:02:52 -0500 Subject: [PATCH 09/14] run go mod tidy to update deps --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 220ee40..e74ca6e 100644 --- a/go.mod +++ b/go.mod @@ -98,15 +98,15 @@ require ( go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect golang.org/x/crypto v0.30.0 // indirect - golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/mod v0.22.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect - golang.org/x/tools v0.28.0 // indirect + golang.org/x/tools v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect diff --git a/go.sum b/go.sum index c1a5723..f9e3679 100644 --- a/go.sum +++ b/go.sum @@ -296,8 +296,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -305,8 +305,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -385,8 +385,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= From f38e790646bc2c4f92ba9a7e038155b1ee63500d Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Fri, 13 Dec 2024 15:00:57 -0500 Subject: [PATCH 10/14] clean-up and fixing buckets --- go.mod | 21 ++--- go.sum | 50 ++++++----- io/blob.go | 255 +++++++++++------------------------------------------ io/io.go | 12 +-- io/s3.go | 24 +---- 5 files changed, 100 insertions(+), 262 deletions(-) diff --git a/go.mod b/go.mod index e74ca6e..a76a714 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hamba/avro/v2 v2.27.0 github.com/pterm/pterm v0.12.80 github.com/stretchr/testify v1.10.0 - github.com/substrait-io/substrait-go v1.1.0 + github.com/substrait-io/substrait-go v1.2.0 github.com/twmb/murmur3 v1.1.8 gocloud.dev v0.40.0 golang.org/x/sync v0.10.0 @@ -29,11 +29,11 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect - cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go v0.116.0 // indirect cloud.google.com/go/auth v0.12.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.1.13 // indirect + cloud.google.com/go/iam v1.2.2 // indirect cloud.google.com/go/storage v1.43.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/alecthomas/participle/v2 v2.1.0 // indirect @@ -42,7 +42,7 @@ require ( github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect @@ -75,10 +75,10 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect @@ -89,6 +89,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/substrait-io/substrait v0.57.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -99,16 +100,16 @@ require ( go.opentelemetry.io/otel/trace v1.29.0 // indirect golang.org/x/crypto v0.30.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/mod v0.21.0 // indirect + golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect - golang.org/x/tools v0.26.0 // indirect - golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect - google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect + golang.org/x/tools v0.28.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect google.golang.org/grpc v1.67.1 // indirect diff --git a/go.sum b/go.sum index f9e3679..f25391b 100644 --- a/go.sum +++ b/go.sum @@ -7,18 +7,18 @@ atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtE atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4= cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4= cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= -cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= -cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= -cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE= -cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -58,8 +58,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 h1:zeN9UtUlA6FTx0vFSayxSX32HDw73Yb6Hh2izDSFxXY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10/go.mod h1:3HKuexPDcwLWPaqpW2UR/9n8N/u/3CKcGAzSs8p8u8g= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.11 h1:FEDZD/Axt5tKSkPAs967KZ++MkvYdBqr0a+cetRbjLM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.11/go.mod h1:dvlsbA32KfvCzqwTiX7maABgFek2RyUuYEJ3kyn/PmQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= @@ -191,8 +191,8 @@ github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -207,8 +207,8 @@ github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -258,8 +258,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/substrait-io/substrait-go v1.1.0 h1:wUXoXV/ESMXgaOWu/05kvMI/UBUyhtaTRfkT5p1b5Ck= -github.com/substrait-io/substrait-go v1.1.0/go.mod h1:LHzL5E0VL620yw4kBQCP+sQPmxhepPTQMDJQRbOe/T4= +github.com/substrait-io/substrait v0.57.1 h1:GW8nnYfSowMseHR8Os82/X6lNtQGIK7p4p+lr6r+auw= +github.com/substrait-io/substrait v0.57.1/go.mod h1:q9s+tjo+gK0lsA+SqYB0lhojNuxvdPdfYlGUP0hjbrA= +github.com/substrait-io/substrait-go v1.2.0 h1:3ZNRkc8FYD7ifCagKEOZQtUcgMceMQfwo2N1NGaK4Q4= +github.com/substrait-io/substrait-go v1.2.0/go.mod h1:IPsy24rdjp/buXR+T8ENl6QCnSCS6h+uM8P+GaZez7c= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -282,8 +284,8 @@ go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= gocloud.dev v0.40.0 h1:f8LgP+4WDqOG/RXoUcyLpeIAGOcAbZrZbDQCUee10ng= @@ -305,8 +307,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -385,12 +387,12 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg= @@ -400,8 +402,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 h1:CT2Thj5AuPV9phrYMtzX11k+XkzMGfRAet42PmoTATM= -google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= +google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f h1:zDoHYmMzMacIdjNe+P2XiTmPsLawi/pCbSPfxt6lTfw= +google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f/go.mod h1:Q5m6g8b5KaFFzsQFIGdJkSJDGeJiybVenoYFMMa3ohI= google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1:pgr/4QbFyktUv9CtQ/Fq4gzEE6/Xs7iCXbktaGzLHbQ= google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 h1:IfdSdTcLFy4lqUQrQJLkLt1PB+AsqVz6lwkWPzWEz10= diff --git a/io/blob.go b/io/blob.go index 33043ef..54c7e79 100644 --- a/io/blob.go +++ b/io/blob.go @@ -19,132 +19,15 @@ package io import ( "context" + "errors" "io" "io/fs" - "net/url" "path/filepath" "strings" - "time" "gocloud.dev/blob" ) -// iofsFileInfo describes a single file in an io/fs.FS. -// It implements fs.FileInfo and fs.DirEntry. -// Copied from `gocloud.dev/blob.iofsDir` because it is private -type iofsFileInfo struct { - lo *blob.ListObject - name string -} - -func (f *iofsFileInfo) Name() string { return f.name } -func (f *iofsFileInfo) Size() int64 { return f.lo.Size } -func (f *iofsFileInfo) Mode() fs.FileMode { return fs.ModeIrregular } -func (f *iofsFileInfo) ModTime() time.Time { return f.lo.ModTime } -func (f *iofsFileInfo) IsDir() bool { return false } -func (f *iofsFileInfo) Sys() interface{} { return f.lo } -func (f *iofsFileInfo) Info() (fs.FileInfo, error) { return f, nil } -func (f *iofsFileInfo) Type() fs.FileMode { return fs.ModeIrregular } - -// iofsDir describes a single directory in an `iceberg-go/io.FS`. -// It implements `io/fs.FileInfo`, `io/fs.DirEntry`, and `io/fs.File`. -// Copied from `gocloud.dev/blob.iofsDir`, but modified to use `iceberg-go/io.File` instead of `io/fs.File` -type iofsDir struct { - b *blobFileIO - key string - name string - // If opened is true, we've read entries via openOnce(). - opened bool - entries []fs.DirEntry - offset int -} - -func newDir(b *blobFileIO, key, name string) *iofsDir { - return &iofsDir{b: b, key: key, name: name} -} - -func (d *iofsDir) Name() string { return d.name } -func (d *iofsDir) Size() int64 { return 0 } -func (d *iofsDir) Mode() fs.FileMode { return fs.ModeDir } -func (d *iofsDir) Type() fs.FileMode { return fs.ModeDir } -func (d *iofsDir) ModTime() time.Time { return time.Time{} } -func (d *iofsDir) IsDir() bool { return true } -func (d *iofsDir) Sys() interface{} { return d } -func (d *iofsDir) Info() (fs.FileInfo, error) { return d, nil } -func (d *iofsDir) Stat() (fs.FileInfo, error) { return d, nil } -func (d *iofsDir) Read([]byte) (int, error) { - return 0, &fs.PathError{Op: "read", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) ReadAt(p []byte, off int64) (int, error) { - return 0, &fs.PathError{Op: "readAt", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) Seek(offset int64, whence int) (int64, error) { - return 0, &fs.PathError{Op: "seek", Path: d.key, Err: fs.ErrInvalid} -} -func (d *iofsDir) Close() error { return nil } -func (d *iofsDir) ReadDir(count int) ([]fs.DirEntry, error) { - if err := d.openOnce(); err != nil { - return nil, err - } - n := len(d.entries) - d.offset - if n == 0 && count > 0 { - return nil, io.EOF - } - if count > 0 && n > count { - n = count - } - list := make([]fs.DirEntry, n) - for i := range list { - list[i] = d.entries[d.offset+i] - } - d.offset += n - return list, nil -} - -func (d *iofsDir) openOnce() error { - if d.opened { - return nil - } - d.opened = true - - // blob expects directories to end in the delimiter, except at the top level. - prefix := d.key - if prefix != "" { - prefix += "/" - } - listOpts := blob.ListOptions{ - Prefix: prefix, - Delimiter: "/", - } - ctx := d.b.ctx - - // Fetch all the directory entries. - // Conceivably we could only fetch a few here, and fetch the rest lazily - // on demand, but that would add significant complexity. - iter := d.b.List(&listOpts) - for { - item, err := iter.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return err - } - name := filepath.Base(item.Key) - if item.IsDir { - d.entries = append(d.entries, newDir(d.b, item.Key, name)) - } else { - d.entries = append(d.entries, &iofsFileInfo{item, name}) - } - } - // There is no such thing as an empty directory in Bucket, so if - // we didn't find anything, it doesn't exist. - if len(d.entries) == 0 { - return fs.ErrNotExist - } - return nil -} - // blobOpenFile describes a single open blob as a File. // It implements the iceberg-go/io.File interface. // It is based on gocloud.dev/blob.iofsOpenFile which: @@ -152,105 +35,75 @@ func (d *iofsDir) openOnce() error { // - Is not externally accessible, so copied here type blobOpenFile struct { *blob.Reader - name string + + name, key string + b *blobFileIO } func (f *blobOpenFile) ReadAt(p []byte, off int64) (int, error) { - finalOff, err := f.Reader.Seek(off, io.SeekStart) + rdr, err := f.b.Bucket.NewRangeReader(context.Background(), f.key, off, int64(len(p)), nil) if err != nil { - return -1, err - } else if finalOff != off { - return -1, io.ErrUnexpectedEOF + return 0, err } - return f.Read(p) + // ensure the buffer is read, or EOF is reached for this read of this "chunk" + // given we are using offsets to read this block, it is constrained by size of 'p' + size, err := io.ReadFull(rdr, p) + if err != nil { + if errors.Is(err, io.EOF) { + return size, err + } + // check we are at the end of the underlying file + if off+int64(size) > f.Size() { + return size, err + } + } + + return size, rdr.Close() } // Functions to implement the `Stat()` function in the `io/fs.File` interface func (f *blobOpenFile) Name() string { return f.name } func (f *blobOpenFile) Mode() fs.FileMode { return fs.ModeIrregular } -func (f *blobOpenFile) Sys() interface{} { return f.Reader } +func (f *blobOpenFile) Sys() interface{} { return f.b } func (f *blobOpenFile) IsDir() bool { return false } func (f *blobOpenFile) Stat() (fs.FileInfo, error) { return f, nil } -// blobFileIO represents a file system backed by a bucket in object store. It implements the `iceberg-go/io.FileIO` interface. type blobFileIO struct { *blob.Bucket - ctx context.Context - opts *blob.ReaderOptions - prefix string + + bucketName string } -func (io *blobFileIO) preprocess(n string) string { - _, after, found := strings.Cut(n, "://") +func (bfs *blobFileIO) preprocess(key string) string { + _, after, found := strings.Cut(key, "://") if found { - n = after - } - - out := strings.TrimPrefix(n, io.prefix) - if out == "/" { - out = "." - } else { - out = strings.TrimPrefix(out, "/") + key = after } - return out + return strings.TrimPrefix(key, bfs.bucketName) } -// Open a Blob from a Bucket using the blobFileIO. Note this -// function is copied from blob.Bucket.Open, but extended to -// return a iceberg-go/io.File instance instead of io/fs.File -func (io *blobFileIO) Open(path string) (File, error) { - if _, err := url.Parse(path); err != nil { +func (bfs *blobFileIO) Open(path string) (fs.File, error) { + if !fs.ValidPath(path) { return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid} } - path = io.preprocess(path) - var isDir bool - var key, name string // name is the last part of the path - if path == "." { - // Root is always a directory, but blob doesn't want the "." in the key. - isDir = true - key, name = "", "." - } else { - exists, _ := io.Bucket.Exists(io.ctx, path) - isDir = !exists + var ( key, name = path, filepath.Base(path) - } + ) - // If it's a directory, list the directory contents. We can't do this lazily - // because we need to error out here if it doesn't exist. - if isDir { - dir := newDir(io, key, name) - err := dir.openOnce() - if err != nil { - if err == fs.ErrNotExist && path == "." { - // The root directory must exist. - return dir, nil - } - return nil, &fs.PathError{Op: "open", Path: path, Err: err} - } - return dir, nil - } - - // It's a file; open it and return a wrapper. - r, err := io.Bucket.NewReader(io.ctx, path, io.opts) + r, err := bfs.NewReader(context.Background(), key, nil) if err != nil { - return nil, &fs.PathError{Op: "open", Path: path, Err: err} + return nil, err } - return &blobOpenFile{Reader: r, name: name}, nil + return &blobOpenFile{Reader: r, name: name, key: key, b: bfs}, nil } -// Remove a Blob from a Bucket using the blobFileIO -func (io *blobFileIO) Remove(path string) error { - if !fs.ValidPath(path) { - return &fs.PathError{Op: "remove", Path: path, Err: fs.ErrInvalid} - } - path = io.preprocess(path) - - return io.Bucket.Delete(io.ctx, path) +func (bfs *blobFileIO) Remove(name string) error { + return bfs.Bucket.Delete(context.Background(), name) } // NewWriter returns a Writer that writes to the blob stored at path. @@ -259,49 +112,45 @@ func (io *blobFileIO) Remove(path string) error { // If overwrite is disabled and a blob with this path already exists, // an error will be returned. // -// The returned Writer will store ctx for later use in Write and/or Close. -// To abort a write, cancel ctx; otherwise, it must remain open until -// Close is called. -// // The caller must call Close on the returned Writer, even if the write is // aborted. -func (io *blobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOptions) (w *BlobWriteFile, err error) { +func (io *blobFileIO) NewWriter(path string, overwrite bool, opts *blob.WriterOptions) (w *blobWriteFile, err error) { if !fs.ValidPath(path) { return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} } path = io.preprocess(path) + + ctx := context.Background() if !overwrite { - if exists, err := io.Bucket.Exists(io.ctx, path); exists { + if exists, err := io.Bucket.Exists(ctx, path); exists { if err != nil { return nil, &fs.PathError{Op: "new writer", Path: path, Err: err} } return nil, &fs.PathError{Op: "new writer", Path: path, Err: fs.ErrInvalid} } } - bw, err := io.Bucket.NewWriter(io.ctx, path, opts) + bw, err := io.Bucket.NewWriter(ctx, path, opts) if err != nil { return nil, err } - return &BlobWriteFile{ + return &blobWriteFile{ Writer: bw, - name: path, - opts: opts}, + name: path}, nil } -// createblobFileIO creates a new blobFileIO instance -func createBlobFileIO(parsed *url.URL, bucket *blob.Bucket) *blobFileIO { - ctx := context.Background() - return &blobFileIO{Bucket: bucket, ctx: ctx, opts: &blob.ReaderOptions{}, prefix: parsed.Host + parsed.Path} +func createBlobFS(bucket *blob.Bucket, bucketName string) IO { + iofs := &blobFileIO{Bucket: bucket, bucketName: bucketName} + return FSPreProcName(iofs, iofs.preprocess) } -type BlobWriteFile struct { +type blobWriteFile struct { *blob.Writer name string - opts *blob.WriterOptions + b *blobFileIO } -func (f *BlobWriteFile) Name() string { return f.name } -func (f *BlobWriteFile) Sys() interface{} { return f.Writer } -func (f *BlobWriteFile) Close() error { return f.Writer.Close() } -func (f *BlobWriteFile) Write(p []byte) (int, error) { return f.Writer.Write(p) } +func (f *blobWriteFile) Name() string { return f.name } +func (f *blobWriteFile) Sys() interface{} { return f.b } +func (f *blobWriteFile) Close() error { return f.Writer.Close() } +func (f *blobWriteFile) Write(p []byte) (int, error) { return f.Writer.Write(p) } diff --git a/io/io.go b/io/io.go index 1f38bbe..a019a47 100644 --- a/io/io.go +++ b/io/io.go @@ -76,7 +76,6 @@ type WriteFileIO interface { // WriteFile writes p to the named file. Write(name string, p []byte) error - Close() error } // A File provides access to a single file. The File interface is the @@ -88,6 +87,12 @@ type File interface { io.ReaderAt } +// A FileWriter represents an open writable file. +type FileWriter interface { + io.WriteCloser + io.ReaderFrom +} + // A ReadDirFile is a directory file whose entries can be read with the // ReadDir method. Every directory file should implement this interface. // (It is permissible for any file to implement this interface, but @@ -247,10 +252,7 @@ func inferFileIOFromSchema(path string, props map[string]string) (IO, error) { default: return nil, fmt.Errorf("IO for file '%s' not implemented", path) } - if parsed.Path != "" && parsed.Path != "/" { - bucket = blob.PrefixedBucket(bucket, strings.TrimPrefix(parsed.Path, "/")) - } - return createBlobFileIO(parsed, bucket), nil + return createBlobFS(bucket, parsed.Host), nil } // LoadFS takes a map of properties and an optional URI location diff --git a/io/s3.go b/io/s3.go index 4679a0a..47ad028 100644 --- a/io/s3.go +++ b/io/s3.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go/auth/bearer" - smithyendpoints "github.com/aws/smithy-go/endpoints" "gocloud.dev/blob" "gocloud.dev/blob/s3blob" ) @@ -53,24 +52,6 @@ var unsupportedS3Props = []string{ S3SignerUri, } -type s3endpointResolver struct { - endpoint string -} - -func (r *s3endpointResolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { - if r.endpoint == "" { - return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) - } - - u, err := url.Parse(r.endpoint) - if err != nil { - return smithyendpoints.Endpoint{}, fmt.Errorf("invalid s3 endpoint url '%s'", r.endpoint) - } - return smithyendpoints.Endpoint{ - URI: *u, - }, nil -} - // ParseAWSConfig parses S3 properties and returns a configuration. func ParseAWSConfig(props map[string]string) (*aws.Config, error) { // If any unsupported properties are set, return an error. @@ -135,7 +116,10 @@ func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]strin } client := s3.NewFromConfig(*awscfg, func(o *s3.Options) { - o.EndpointResolverV2 = &s3endpointResolver{endpoint: endpoint} + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + } + o.UsePathStyle = true }) // Create a *blob.Bucket. From bdac9d498a7dfb46ecdb73c7ac37e30bade4591a Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Fri, 13 Dec 2024 15:03:24 -0500 Subject: [PATCH 11/14] somehow lost the license in go.mod --- go.mod | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/go.mod b/go.mod index a76a714..602eebe 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + module github.com/apache/iceberg-go go 1.23 From 39c7baa9f0cfc7f3065abd3bd2921730e5d54e21 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Fri, 13 Dec 2024 15:19:36 -0500 Subject: [PATCH 12/14] fix spacing due to upgraded dependency --- cmd/iceberg/output_test.go | 39 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/cmd/iceberg/output_test.go b/cmd/iceberg/output_test.go index 5907660..01dcebf 100644 --- a/cmd/iceberg/output_test.go +++ b/cmd/iceberg/output_test.go @@ -63,13 +63,13 @@ var testArgs = []struct { "snapshot-log": [ ], "metadata-log": [ ], "refs": { } -}`, -`Table format version | 2 -Metadata location | +}`, + `Table format version | 2 +Metadata location | Table UUID | 9c12d441-03fe-4693-9a96-a0705ddf69c1 -Last updated | 1602638573590 -Sort Order | 0: [] -Partition Spec | [] +Last updated | 1602638573590 +Sort Order | 0: [] +Partition Spec | [] Current Schema, id=0 └──1: x: required long @@ -79,7 +79,7 @@ Current Snapshot | Snapshots Properties -key | value +key | value ---------------------------------- read.split.target.size | 134217728 @@ -145,18 +145,18 @@ read.split.target.size | 134217728 ], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}} -}`, -`Table format version | 2 -Metadata location | +}`, + `Table format version | 2 +Metadata location | Table UUID | 9c12d441-03fe-4693-9a96-a0705ddf69c1 -Last updated | 1602638573590 -Sort Order | 3: [ - | 2 asc nulls-first - | bucket[4](3) desc nulls-last - | ] -Partition Spec | [ - | 1000: x: identity(1) - | ] +Last updated | 1602638573590 +Sort Order | 3: [ + | 2 asc nulls-first + | bucket[4](3) desc nulls-last + | ] +Partition Spec | [ + | 1000: x: identity(1) + | ] Current Schema, id=1 ├──1: x: required long @@ -170,14 +170,13 @@ Snapshots └──Snapshot 3055729675574597004, schema 1: s3://a/b/2.avro Properties -key | value +key | value ---------------------------------- read.split.target.size | 134217728 `}, } - func TestDescribeTable(t *testing.T) { var buf bytes.Buffer pterm.SetDefaultOutput(&buf) From 0b374790e21e1aa185f379b483ce53ccb04ee9bd Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Fri, 13 Dec 2024 15:25:14 -0500 Subject: [PATCH 13/14] missed a close for the parquet reader --- table/arrow_scanner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/table/arrow_scanner.go b/table/arrow_scanner.go index 97147ce..7155ab5 100644 --- a/table/arrow_scanner.go +++ b/table/arrow_scanner.go @@ -390,6 +390,7 @@ func (as *arrowScan) recordsFromTask(ctx context.Context, task internal.Enumerat if err != nil { return } + defer rdr.Close() pipeline := make([]recProcessFn, 0, 2) if len(positionalDeletes) > 0 { From 6861e09f80173a567f8454fc55ee427c3402af5f Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Fri, 13 Dec 2024 16:18:37 -0500 Subject: [PATCH 14/14] add config option to force s3 virtual addressing --- io/s3.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/io/s3.go b/io/s3.go index 47ad028..430ac2f 100644 --- a/io/s3.go +++ b/io/s3.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "slices" + "strconv" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" @@ -37,14 +38,15 @@ import ( // Constants for S3 configuration options const ( - S3Region = "s3.region" - S3SessionToken = "s3.session-token" - S3SecretAccessKey = "s3.secret-access-key" - S3AccessKeyID = "s3.access-key-id" - S3EndpointURL = "s3.endpoint" - S3ProxyURI = "s3.proxy-uri" - S3ConnectTimeout = "s3.connect-timeout" - S3SignerUri = "s3.signer.uri" + S3Region = "s3.region" + S3SessionToken = "s3.session-token" + S3SecretAccessKey = "s3.secret-access-key" + S3AccessKeyID = "s3.access-key-id" + S3EndpointURL = "s3.endpoint" + S3ProxyURI = "s3.proxy-uri" + S3ConnectTimeout = "s3.connect-timeout" + S3SignerUri = "s3.signer.uri" + S3ForceVirtualAddressing = "s3.force-virtual-addressing" ) var unsupportedS3Props = []string{ @@ -115,11 +117,18 @@ func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]strin endpoint = os.Getenv("AWS_S3_ENDPOINT") } + usePathStyle := true + if forceVirtual, ok := props[S3ForceVirtualAddressing]; ok { + if cfgForceVirtual, err := strconv.ParseBool(forceVirtual); err == nil { + usePathStyle = !cfgForceVirtual + } + } + client := s3.NewFromConfig(*awscfg, func(o *s3.Options) { if endpoint != "" { o.BaseEndpoint = aws.String(endpoint) } - o.UsePathStyle = true + o.UsePathStyle = usePathStyle }) // Create a *blob.Bucket.