Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Runtime: Azure storage connector #3036

Merged
merged 35 commits into from
Sep 26, 2023
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
cbaf361
azure storage changes
rakeshsharma14317 Sep 4, 2023
b9639bc
testing azure
rakeshsharma14317 Sep 5, 2023
5e47c0e
interim commit
k-anshul Sep 5, 2023
2e3650d
changes with SharedKeyCredential
rakeshsharma14317 Sep 7, 2023
d2c05c9
nit changes
rakeshsharma14317 Sep 7, 2023
6733396
nit
rakeshsharma14317 Sep 7, 2023
1a6fe58
Merge remote-tracking branch 'origin/main' into 2992-runtime-azure-st…
rakeshsharma14317 Sep 7, 2023
29b6a44
linter fix
rakeshsharma14317 Sep 7, 2023
e75d21e
review changes 1
rakeshsharma14317 Sep 8, 2023
b4d64b0
review changes
rakeshsharma14317 Sep 11, 2023
5a98e5e
added access for public objects
rakeshsharma14317 Sep 11, 2023
0d1d099
ci fix
rakeshsharma14317 Sep 11, 2023
b7c6228
Merge remote-tracking branch 'origin/main' into 2992-runtime-azure-st…
rakeshsharma14317 Sep 11, 2023
b9883a3
minor changes
rakeshsharma14317 Sep 13, 2023
53ad4ff
changes
rakeshsharma14317 Sep 14, 2023
f9c37d9
cleanup
rakeshsharma14317 Sep 14, 2023
1249f21
more cleanup and added az login auth
rakeshsharma14317 Sep 14, 2023
41c15d6
merge with main branch
rakeshsharma14317 Sep 14, 2023
b0d93e1
reverted package-lock.json
rakeshsharma14317 Sep 14, 2023
96324dd
nit
rakeshsharma14317 Sep 15, 2023
f8a82a1
nit
rakeshsharma14317 Sep 15, 2023
eab2b25
linter fix
rakeshsharma14317 Sep 18, 2023
c00f42a
more changes
rakeshsharma14317 Sep 18, 2023
705c7a8
review changes
rakeshsharma14317 Sep 19, 2023
a27577c
added public bucket access
rakeshsharma14317 Sep 20, 2023
00b99a4
merge with main branch
rakeshsharma14317 Sep 20, 2023
c6b3543
added public blob access
rakeshsharma14317 Sep 21, 2023
7951f4c
review changes
rakeshsharma14317 Sep 22, 2023
fe46fc2
review changes
rakeshsharma14317 Sep 25, 2023
558503d
merge with main branch
rakeshsharma14317 Sep 25, 2023
4c85059
Return err in all cases
begelundmuller Sep 25, 2023
5ac5cdd
git pull and added log.info
rakeshsharma14317 Sep 25, 2023
4af6c48
default error fallback issue
rakeshsharma14317 Sep 26, 2023
040a8ae
conflicts resolved
rakeshsharma14317 Sep 26, 2023
523afec
Fix CI
begelundmuller Sep 26, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
review changes 1
rakeshsharma14317 committed Sep 8, 2023
commit e75d21e3b7f79df0a1a26a2299990c43ad66b2a2
39 changes: 16 additions & 23 deletions runtime/drivers/azure/azure.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"os"
"strings"

"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
@@ -13,7 +12,6 @@ import (
"github.com/rilldata/rill/runtime/drivers"
rillblob "github.com/rilldata/rill/runtime/drivers/blob"
"github.com/rilldata/rill/runtime/pkg/activity"
"github.com/rilldata/rill/runtime/pkg/fileutil"
"github.com/rilldata/rill/runtime/pkg/globutil"
"go.uber.org/zap"
"gocloud.dev/blob/azureblob"
@@ -33,7 +31,7 @@ var spec = drivers.Spec{
Key: "path",
DisplayName: "Blob URI",
Description: "Path to file on the disk.",
Placeholder: "az://container-name/path/to/file.csv",
Placeholder: "azblob://container-name/path/to/file.csv",
rakeshsharma14317 marked this conversation as resolved.
Show resolved Hide resolved
Type: drivers.StringPropertyType,
Required: true,
Hint: "Glob patterns are supported",
@@ -49,31 +47,27 @@ var spec = drivers.Spec{
},
ConfigProperties: []drivers.PropertySchema{
rakeshsharma14317 marked this conversation as resolved.
Show resolved Hide resolved
{
Key: "azure.storage.account",
Hint: "Enter path of file to load from.",
ValidateFunc: func(any interface{}) error {
val := any.(string)
if val == "" {
// user can chhose to leave empty for public sources
return nil
}

path, err := fileutil.ExpandHome(strings.TrimSpace(val))
if err != nil {
return err
}

_, err = os.Stat(path)
return err
},
Key: "azure_storage_account",
Secret: true,
},
{
Key: "azure_storage_key",
Secret: true,
},
{
Key: "azure_storage_sas_token",
Secret: true,
},
},
}

type driver struct{}

type configProperties struct {
Account string `mapstructure:"azure.storage.account"`
Account string `mapstructure:"azure_storage_account"`
Key string `mapstructure:"azure_storage_key"`
SASToken string `mapstructure:"azure_storage_sas_token"`
AllowHostAccess bool `mapstructure:"allow_host_access"`
}

func (d driver) Open(config map[string]any, shared bool, client activity.Client, logger *zap.Logger) (drivers.Handle, error) {
@@ -102,6 +96,7 @@ func (d driver) Spec() drivers.Spec {
}

func (d driver) HasAnonymousSourceAccess(ctx context.Context, src drivers.Source, logger *zap.Logger) (bool, error) {
rakeshsharma14317 marked this conversation as resolved.
Show resolved Hide resolved
// TODO: implement
return false, nil
}

@@ -240,8 +235,6 @@ func parseSourceProperties(props map[string]any) (*sourceProperties, error) {
return nil, err
}
if !doublestar.ValidatePattern(conf.Path) {
// ideally this should be validated at much earlier stage
// keeping it here to have gcs specific validations
return nil, fmt.Errorf("glob pattern %s is invalid", conf.Path)
rakeshsharma14317 marked this conversation as resolved.
Show resolved Hide resolved
}
url, err := globutil.ParseBucketURL(conf.Path)
2 changes: 1 addition & 1 deletion web-common/src/features/sources/modal/yupSchemas.ts
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ export function getYupSchema(connector: V1ConnectorSpec) {
return yup.object().shape({
path: yup
.string()
// .matches(/^https?:\/\//, 'Path must start with "http(s)://"')
.matches(/^azblob:\/\//, "Must be an Azure URI (e.g. azblob://container/path)")
.required("Path is required"),
rakeshsharma14317 marked this conversation as resolved.
Show resolved Hide resolved
sourceName: yup
.string()