diff --git a/core/orchestrator_core.go b/core/orchestrator_core.go index 3e0fc6327..faab85a9b 100644 --- a/core/orchestrator_core.go +++ b/core/orchestrator_core.go @@ -1083,17 +1083,20 @@ func (o *TridentOrchestrator) validateAndCreateBackendFromConfig( // If Credentials are set, fetch them and set them in the configJSON matching field names if len(commonConfig.Credentials) != 0 { - secretName, _, err := commonConfig.GetCredentials() - if err != nil { - return nil, err + secretName, secretType, secretErr := commonConfig.GetCredentials() + if secretErr != nil { + return nil, secretErr } else if secretName == "" { return nil, fmt.Errorf("credentials `name` field cannot be empty") } - if backendSecret, err = o.storeClient.GetBackendSecret(ctx, secretName); err != nil { - return nil, err - } else if backendSecret == nil { - return nil, fmt.Errorf("backend credentials not found") + // Handle known secret store types here, but driver-specific ones may be handled in the drivers. + if secretType == string(drivers.CredentialStoreK8sSecret) { + if backendSecret, err = o.storeClient.GetBackendSecret(ctx, secretName); err != nil { + return nil, err + } else if backendSecret == nil { + return nil, fmt.Errorf("backend credentials not found") + } } } diff --git a/go.mod b/go.mod index 2761b4640..6b5bac228 100755 --- a/go.mod +++ b/go.mod @@ -8,6 +8,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.7.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.1.0 github.com/RoaringBitmap/roaring v1.5.0 + github.com/aws/aws-sdk-go-v2 v1.21.1 + github.com/aws/aws-sdk-go-v2/config v1.18.44 + github.com/aws/aws-sdk-go-v2/credentials v1.13.42 + github.com/aws/aws-sdk-go-v2/service/fsx v1.33.0 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4 github.com/cenkalti/backoff/v4 v4.2.1 github.com/container-storage-interface/spec v1.8.0 github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651 @@ -49,7 +54,7 @@ require ( golang.org/x/sys v0.13.0 // github.com/golang/sys golang.org/x/text v0.13.0 // github.com/golang/text golang.org/x/time v0.3.0 // github.com/golang/time - google.golang.org/grpc v1.58.2 // github.com/grpc/grpc-go + google.golang.org/grpc v1.58.3 // github.com/grpc/grpc-go k8s.io/api v0.28.2 // github.com/kubernetes/api k8s.io/apiextensions-apiserver v0.28.2 // github.com/kubernetes/apiextensions-apiserver k8s.io/apimachinery v0.28.2 // github.com/kubernetes/apimachinery @@ -72,6 +77,15 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 77c4306ed..2e1832e34 100755 --- a/go.sum +++ b/go.sum @@ -84,6 +84,34 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.21.1 h1:wjHYshtPpYOZm+/mu3NhVgRRc0baM6LJZOmxPZ5Cwzs= +github.com/aws/aws-sdk-go-v2 v1.21.1/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/config v1.18.44 h1:U10NQ3OxiY0dGGozmVIENIDnCT0W432PWxk2VO8wGnY= +github.com/aws/aws-sdk-go-v2/config v1.18.44/go.mod h1:pHxnQBldd0heEdJmolLBk78D1Bf69YnKLY3LOpFImlU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42 h1:KMkjpZqcMOwtRHChVlHdNxTUUAC6NC/b58mRZDIdcRg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42/go.mod h1:7ltKclhvEB8305sBhrpls24HGxORl6qgnQqSJ314Uw8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 h1:3j5lrl9kVQrJ1BU4O0z7MQ8sa+UXdiLuo4j0V+odNI8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12/go.mod h1:JbFpcHDBdsex1zpIKuVRorZSQiZEyc3MykNCcjgz174= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 h1:817VqVe6wvwE46xXy6YF5RywvjOX6U2zRQQ6IbQFK0s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42/go.mod h1:oDfgXoBBmj+kXnqxDDnIDnC56QBosglKp8ftRCTxR+0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 h1:7ZApaXzWbo8slc+W5TynuUlB4z66g44h7uqa3/d/BsY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36/go.mod h1:rwr4WnmFi3RJO0M4dxbJtgi9BPLMpVBMX1nUte5ha9U= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 h1:quOJOqlbSfeJTboXLjYXM1M9T52LBXqLoTPlmsKLpBo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44/go.mod h1:LNy+P1+1LiRcCsVYr/4zG5n8zWFL0xsvZkOybjbftm8= +github.com/aws/aws-sdk-go-v2/service/fsx v1.33.0 h1:EXZbHn1FRk+Y2g0KNCeXOeOIus7S2BScyH5vbDMvo0k= +github.com/aws/aws-sdk-go-v2/service/fsx v1.33.0/go.mod h1:XNZWr4vExL2LXPyccQX7uAcGF9so8k+RA9xChAhU0/g= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 h1:YXlm7LxwNlauqb2OrinWlcvtsflTzP8GaMvYfQBhoT4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36/go.mod h1:ou9ffqJ9hKOVZmjlC6kQ6oROAyG1M4yBKzR+9BKbDwk= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4 h1:LUtjmUxYPkiFkiVyvLmHVcuthVPnEKd0hEprTOVRTS0= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4/go.mod h1:Bph0xA97xjEciochtR3JKrgGHt1psILMtFgu3KAbiBE= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 h1:ZN3bxw9OYC5D6umLw6f57rNJfGfhg1DIAAcKpzyUTOE= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1/go.mod h1:PieckvBoT5HtyB9AsJRrYZFY2Z+EyfVM/9zG6gbV8DQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 h1:fSCCJuT5i6ht8TqGdZc5Q5K9pz/atrf7qH4iK5C9XzU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2/go.mod h1:5eNtr+vNc5vVd92q7SJ+U/HszsIdhZBEyi9dkMRKsp8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7HROOpeSMu1xDA/I8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= @@ -267,6 +295,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -310,6 +339,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -818,8 +849,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go b/mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go new file mode 100644 index 000000000..285638165 --- /dev/null +++ b/mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go @@ -0,0 +1,325 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/netapp/trident/storage_drivers/ontap/awsapi (interfaces: AWSAPI) + +// Package mock_api is a generated GoMock package. +package mock_api + +import ( + context "context" + reflect "reflect" + time "time" + + gomock "github.com/golang/mock/gomock" + storage "github.com/netapp/trident/storage" + awsapi "github.com/netapp/trident/storage_drivers/ontap/awsapi" +) + +// MockAWSAPI is a mock of AWSAPI interface. +type MockAWSAPI struct { + ctrl *gomock.Controller + recorder *MockAWSAPIMockRecorder +} + +// MockAWSAPIMockRecorder is the mock recorder for MockAWSAPI. +type MockAWSAPIMockRecorder struct { + mock *MockAWSAPI +} + +// NewMockAWSAPI creates a new mock instance. +func NewMockAWSAPI(ctrl *gomock.Controller) *MockAWSAPI { + mock := &MockAWSAPI{ctrl: ctrl} + mock.recorder = &MockAWSAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAWSAPI) EXPECT() *MockAWSAPIMockRecorder { + return m.recorder +} + +// CreateVolume mocks base method. +func (m *MockAWSAPI) CreateVolume(arg0 context.Context, arg1 *awsapi.VolumeCreateRequest) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVolume", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateVolume indicates an expected call of CreateVolume. +func (mr *MockAWSAPIMockRecorder) CreateVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVolume", reflect.TypeOf((*MockAWSAPI)(nil).CreateVolume), arg0, arg1) +} + +// DeleteVolume mocks base method. +func (m *MockAWSAPI) DeleteVolume(arg0 context.Context, arg1 *awsapi.Volume) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteVolume indicates an expected call of DeleteVolume. +func (mr *MockAWSAPIMockRecorder) DeleteVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVolume", reflect.TypeOf((*MockAWSAPI)(nil).DeleteVolume), arg0, arg1) +} + +// GetFilesystemByID mocks base method. +func (m *MockAWSAPI) GetFilesystemByID(arg0 context.Context, arg1 string) (*awsapi.Filesystem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFilesystemByID", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Filesystem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFilesystemByID indicates an expected call of GetFilesystemByID. +func (mr *MockAWSAPIMockRecorder) GetFilesystemByID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFilesystemByID", reflect.TypeOf((*MockAWSAPI)(nil).GetFilesystemByID), arg0, arg1) +} + +// GetFilesystems mocks base method. +func (m *MockAWSAPI) GetFilesystems(arg0 context.Context) (*[]*awsapi.Filesystem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFilesystems", arg0) + ret0, _ := ret[0].(*[]*awsapi.Filesystem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFilesystems indicates an expected call of GetFilesystems. +func (mr *MockAWSAPIMockRecorder) GetFilesystems(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFilesystems", reflect.TypeOf((*MockAWSAPI)(nil).GetFilesystems), arg0) +} + +// GetSVMByID mocks base method. +func (m *MockAWSAPI) GetSVMByID(arg0 context.Context, arg1 string) (*awsapi.SVM, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSVMByID", arg0, arg1) + ret0, _ := ret[0].(*awsapi.SVM) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSVMByID indicates an expected call of GetSVMByID. +func (mr *MockAWSAPIMockRecorder) GetSVMByID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSVMByID", reflect.TypeOf((*MockAWSAPI)(nil).GetSVMByID), arg0, arg1) +} + +// GetSVMs mocks base method. +func (m *MockAWSAPI) GetSVMs(arg0 context.Context) (*[]*awsapi.SVM, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSVMs", arg0) + ret0, _ := ret[0].(*[]*awsapi.SVM) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSVMs indicates an expected call of GetSVMs. +func (mr *MockAWSAPIMockRecorder) GetSVMs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSVMs", reflect.TypeOf((*MockAWSAPI)(nil).GetSVMs), arg0) +} + +// GetSecret mocks base method. +func (m *MockAWSAPI) GetSecret(arg0 context.Context, arg1 string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSecret", arg0, arg1) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSecret indicates an expected call of GetSecret. +func (mr *MockAWSAPIMockRecorder) GetSecret(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecret", reflect.TypeOf((*MockAWSAPI)(nil).GetSecret), arg0, arg1) +} + +// GetVolume mocks base method. +func (m *MockAWSAPI) GetVolume(arg0 context.Context, arg1 *storage.VolumeConfig) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolume", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolume indicates an expected call of GetVolume. +func (mr *MockAWSAPIMockRecorder) GetVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolume", reflect.TypeOf((*MockAWSAPI)(nil).GetVolume), arg0, arg1) +} + +// GetVolumeByARN mocks base method. +func (m *MockAWSAPI) GetVolumeByARN(arg0 context.Context, arg1 string) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeByARN", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeByARN indicates an expected call of GetVolumeByARN. +func (mr *MockAWSAPIMockRecorder) GetVolumeByARN(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeByARN", reflect.TypeOf((*MockAWSAPI)(nil).GetVolumeByARN), arg0, arg1) +} + +// GetVolumeByID mocks base method. +func (m *MockAWSAPI) GetVolumeByID(arg0 context.Context, arg1 string) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeByID", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeByID indicates an expected call of GetVolumeByID. +func (mr *MockAWSAPIMockRecorder) GetVolumeByID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeByID", reflect.TypeOf((*MockAWSAPI)(nil).GetVolumeByID), arg0, arg1) +} + +// GetVolumeByName mocks base method. +func (m *MockAWSAPI) GetVolumeByName(arg0 context.Context, arg1 string) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeByName", arg0, arg1) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeByName indicates an expected call of GetVolumeByName. +func (mr *MockAWSAPIMockRecorder) GetVolumeByName(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeByName", reflect.TypeOf((*MockAWSAPI)(nil).GetVolumeByName), arg0, arg1) +} + +// GetVolumes mocks base method. +func (m *MockAWSAPI) GetVolumes(arg0 context.Context) (*[]*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumes", arg0) + ret0, _ := ret[0].(*[]*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumes indicates an expected call of GetVolumes. +func (mr *MockAWSAPIMockRecorder) GetVolumes(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumes", reflect.TypeOf((*MockAWSAPI)(nil).GetVolumes), arg0) +} + +// RelabelVolume mocks base method. +func (m *MockAWSAPI) RelabelVolume(arg0 context.Context, arg1 *awsapi.Volume, arg2 map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RelabelVolume", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RelabelVolume indicates an expected call of RelabelVolume. +func (mr *MockAWSAPIMockRecorder) RelabelVolume(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RelabelVolume", reflect.TypeOf((*MockAWSAPI)(nil).RelabelVolume), arg0, arg1, arg2) +} + +// ResizeVolume mocks base method. +func (m *MockAWSAPI) ResizeVolume(arg0 context.Context, arg1 *awsapi.Volume, arg2 uint64) (*awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResizeVolume", arg0, arg1, arg2) + ret0, _ := ret[0].(*awsapi.Volume) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResizeVolume indicates an expected call of ResizeVolume. +func (mr *MockAWSAPIMockRecorder) ResizeVolume(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResizeVolume", reflect.TypeOf((*MockAWSAPI)(nil).ResizeVolume), arg0, arg1, arg2) +} + +// VolumeExists mocks base method. +func (m *MockAWSAPI) VolumeExists(arg0 context.Context, arg1 *storage.VolumeConfig) (bool, *awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeExists", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(*awsapi.Volume) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// VolumeExists indicates an expected call of VolumeExists. +func (mr *MockAWSAPIMockRecorder) VolumeExists(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeExists", reflect.TypeOf((*MockAWSAPI)(nil).VolumeExists), arg0, arg1) +} + +// VolumeExistsByARN mocks base method. +func (m *MockAWSAPI) VolumeExistsByARN(arg0 context.Context, arg1 string) (bool, *awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeExistsByARN", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(*awsapi.Volume) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// VolumeExistsByARN indicates an expected call of VolumeExistsByARN. +func (mr *MockAWSAPIMockRecorder) VolumeExistsByARN(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeExistsByARN", reflect.TypeOf((*MockAWSAPI)(nil).VolumeExistsByARN), arg0, arg1) +} + +// VolumeExistsByID mocks base method. +func (m *MockAWSAPI) VolumeExistsByID(arg0 context.Context, arg1 string) (bool, *awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeExistsByID", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(*awsapi.Volume) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// VolumeExistsByID indicates an expected call of VolumeExistsByID. +func (mr *MockAWSAPIMockRecorder) VolumeExistsByID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeExistsByID", reflect.TypeOf((*MockAWSAPI)(nil).VolumeExistsByID), arg0, arg1) +} + +// VolumeExistsByName mocks base method. +func (m *MockAWSAPI) VolumeExistsByName(arg0 context.Context, arg1 string) (bool, *awsapi.Volume, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeExistsByName", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(*awsapi.Volume) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// VolumeExistsByName indicates an expected call of VolumeExistsByName. +func (mr *MockAWSAPIMockRecorder) VolumeExistsByName(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeExistsByName", reflect.TypeOf((*MockAWSAPI)(nil).VolumeExistsByName), arg0, arg1) +} + +// WaitForVolumeStates mocks base method. +func (m *MockAWSAPI) WaitForVolumeStates(arg0 context.Context, arg1 *awsapi.Volume, arg2, arg3 []string, arg4 time.Duration) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitForVolumeStates", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WaitForVolumeStates indicates an expected call of WaitForVolumeStates. +func (mr *MockAWSAPIMockRecorder) WaitForVolumeStates(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForVolumeStates", reflect.TypeOf((*MockAWSAPI)(nil).WaitForVolumeStates), arg0, arg1, arg2, arg3, arg4) +} diff --git a/persistent_store/crdv1.go b/persistent_store/crdv1.go index a8223232c..2909edd7c 100644 --- a/persistent_store/crdv1.go +++ b/persistent_store/crdv1.go @@ -383,16 +383,28 @@ func (k *CRDClientV1) addSecretToBackend( } var secretName string + var secretType string var err error // Check if user-provided credentials are in use - if secretName, _, err = backendPersistent.GetBackendCredentials(); err != nil { + if secretName, secretType, err = backendPersistent.GetBackendCredentials(); err != nil { Logc(ctx).WithFields(logFields).Errorf("Could determined if credentials field exist; %v", err) return nil, err - } else if secretName == "" { + } + + if secretName == "" { // Credentials field not set, use the default backend secret name secretName = k.backendSecretName(backendPersistent.BackendUUID) } + if secretType == "" { + // Default secret type to K8s secret + secretType = "secret" + } + + // If the secret type isn't that of a Kubernetes secret, return here without retrieving anything. + if secretType != "secret" { + return backendPersistent, nil + } // Before retrieving the secret, ensure it exists. If we find the secret does not exist, we // log a warning but return the backend object unmodified so that Trident can bootstrap itself diff --git a/storage_drivers/azure/api/azure.go b/storage_drivers/azure/api/azure.go index 48ce4b288..d93bc0da3 100644 --- a/storage_drivers/azure/api/azure.go +++ b/storage_drivers/azure/api/azure.go @@ -874,6 +874,7 @@ func (c Client) WaitForVolumeState( return nil } + volumeState = "" return fmt.Errorf("could not get volume status; %v", err) } diff --git a/storage_drivers/config.go b/storage_drivers/config.go index 6b2eca3e8..2d096012d 100644 --- a/storage_drivers/config.go +++ b/storage_drivers/config.go @@ -30,6 +30,7 @@ const ( TopologyLabelPrefix = "topology.kubernetes.io" CredentialStoreK8sSecret CredentialStore = "secret" + CredentialStoreAWSARN CredentialStore = "awsarn" KeyName string = "name" KeyType string = "type" diff --git a/storage_drivers/ontap/aws_common.go b/storage_drivers/ontap/aws_common.go new file mode 100644 index 000000000..9a6a55d3a --- /dev/null +++ b/storage_drivers/ontap/aws_common.go @@ -0,0 +1,252 @@ +// Copyright 2023 NetApp, Inc. All Rights Reserved. + +package ontap + +import ( + "context" + "fmt" + "strings" + + tridentconfig "github.com/netapp/trident/config" + . "github.com/netapp/trident/logging" + "github.com/netapp/trident/storage" + drivers "github.com/netapp/trident/storage_drivers" + "github.com/netapp/trident/storage_drivers/ontap/awsapi" + "github.com/netapp/trident/utils" + "github.com/netapp/trident/utils/errors" +) + +// SetSvmCredentials Pull SVM credentials out of AWS secret store and enter them into the config. +func SetSvmCredentials(ctx context.Context, secretARN string, api awsapi.AWSAPI, config *drivers.OntapStorageDriverConfig) (err error) { + secretMap, secretErr := api.GetSecret(ctx, secretARN) + if secretErr != nil { + return fmt.Errorf("could not retrieve credentials from AWS Secrets Manager; %w", secretErr) + } + + if username, ok := secretMap["username"]; !ok { + return fmt.Errorf("%s driver must include username in the secret referenced by Credentials", + config.StorageDriverName) + } else { + config.Username = username + } + + if password, ok := secretMap["password"]; !ok { + return fmt.Errorf("%s driver must include password in the secret referenced by Credentials", + config.StorageDriverName) + } else { + config.Password = password + } + + return nil +} + +// initializeAWSDriver returns an AWS SDK client. It does all the other initialization needed for +// AWS (FSxN or CVO), such that this is the only method the drivers have to call. +func initializeAWSDriver( + ctx context.Context, config *drivers.OntapStorageDriverConfig, +) (api awsapi.AWSAPI, err error) { + // No AWS config is normal for on-prem ONTAP, so just return + if config.AWSConfig == nil { + return nil, nil + } + + // Create the AWS API client and read ONTAP credentials if so configured + api, err = initializeAWSAPI(ctx, config) + if err != nil { + return nil, err + } + + // Validate FSxN filesystem if so configured + if config.AWSConfig.FSxFilesystemID != "" { + err = validateFSxFilesystem(ctx, api, config) + if err != nil { + return nil, fmt.Errorf("error validating FSx filesystem; %v", err) + } + } + + return +} + +// initializeAWSAPI returns an AWS SDK client. If configured, it detects and pulls ONTAP credentials from +// AWS Secrets Manager. All discovered values are written back to the supplied OntapStorageDriverConfig. +func initializeAWSAPI( + ctx context.Context, config *drivers.OntapStorageDriverConfig, +) (awsapi.AWSAPI, error) { + fields := LogFields{"Method": "initializeAWSAPI", "Type": "ontap_common"} + Logd(ctx, config.StorageDriverName, config.DebugTraceFlags["method"]).WithFields(fields).Trace( + ">>>> initializeAWSAPI") + defer Logd(ctx, config.StorageDriverName, config.DebugTraceFlags["method"]).WithFields(fields).Trace( + "<<<< initializeAWSAPI") + + secretManagerRegion := config.AWSConfig.APIRegion + // Check if the configured credentials refer to a secret in AWS Secrets Manager + secretARN, secretErr := getAWSSecretsManagerARNFromConfig(ctx, config) + if secretErr != nil && !errors.IsNotFoundError(secretErr) { + // We find ARN, but we get an error. + return nil, secretErr + } + if secretARN != "" { + var err error + secretManagerRegion, _, _, err = awsapi.ParseSecretARN(secretARN) + if err != nil { + return nil, err + } + } + + api, err := awsapi.NewClient(ctx, awsapi.ClientConfig{ + APIRegion: config.AWSConfig.APIRegion, + APIKey: config.AWSConfig.APIKey, + SecretKey: config.AWSConfig.SecretKey, + FilesystemID: config.AWSConfig.FSxFilesystemID, + DebugTraceFlags: config.DebugTraceFlags, + SecretManagerRegion: secretManagerRegion, + }) + if err != nil { + return nil, err + } + + if secretErr != nil && errors.IsNotFoundError(secretErr) { + // No ARN found, so continue with any configured explicit credentials + return api, nil + } + + err = SetSvmCredentials(ctx, secretARN, api, config) + if err != nil { + return nil, err + } + + return api, nil +} + +// validateFSxFilesystem validates a configured FSx filesystem, validates or infers the SVM, and optionally +// determines the SVM management LIF. All discovered values are written back to the supplied OntapStorageDriverConfig. +func validateFSxFilesystem( + ctx context.Context, api awsapi.AWSAPI, config *drivers.OntapStorageDriverConfig, +) error { + fields := LogFields{"Method": "validateFSxFilesystem", "Type": "ontap_common"} + Logd(ctx, config.StorageDriverName, config.DebugTraceFlags["method"]).WithFields(fields).Trace( + ">>>> validateFSxFilesystem") + defer Logd(ctx, config.StorageDriverName, config.DebugTraceFlags["method"]).WithFields(fields).Trace( + "<<<< validateFSxFilesystem") + + // Ensure FSx filesystem exists + if config.AWSConfig.FSxFilesystemID == "" { + return errors.New("filesystem ID in config must be specified") + } + _, err := api.GetFilesystemByID(ctx, config.AWSConfig.FSxFilesystemID) + if err != nil { + return fmt.Errorf("filesystem with ID %s not found; %w", config.AWSConfig.FSxFilesystemID, err) + } + + Logc(ctx).WithField("region", config.AWSConfig.APIRegion).Debug("FSxN SDK access OK.") + + // Build list of SVM names + discoveredSVMs, err := api.GetSVMs(ctx) + if err != nil { + return fmt.Errorf("could not retrieve FSxN SVMs; %w", err) + } + discoveredSVMNames := make([]string, 0) + for _, svm := range *discoveredSVMs { + discoveredSVMNames = append(discoveredSVMNames, svm.Name) + } + + // Validate or infer SVM name + if config.SVM == "" { + if len(discoveredSVMNames) == 1 { + config.SVM = discoveredSVMNames[0] + } else { + return fmt.Errorf("no SVM specified and multiple SVMs exist in filesystem %s", + config.AWSConfig.FSxFilesystemID) + } + } else { + if !utils.SliceContainsString(discoveredSVMNames, config.SVM) { + return fmt.Errorf("SVM %s does not exist in filesystem %s", config.SVM, config.AWSConfig.FSxFilesystemID) + } + } + + var svm *awsapi.SVM + for _, discoveredSVM := range *discoveredSVMs { + if discoveredSVM.Name == config.SVM { + svm = discoveredSVM + break + } + } + + // Infer management LIF + if config.ManagementLIF == "" { + if len(svm.MgtEndpoint.IPAddresses) > 0 { + config.ManagementLIF = svm.MgtEndpoint.IPAddresses[0] + } else { + config.ManagementLIF = svm.MgtEndpoint.DNSName + } + } + + return err +} + +// getAWSSecretsManagerARNFromConfig examines an OntapStorageDriverConfig to find the AWS ARN that identifies the +// secret containing a set of SVM credentials. +func getAWSSecretsManagerARNFromConfig(_ context.Context, config *drivers.OntapStorageDriverConfig) (string, error) { + if config.Credentials != nil && config.Credentials[drivers.KeyType] == string(drivers.CredentialStoreAWSARN) { + return config.Credentials[drivers.KeyName], nil + } + + if strings.HasPrefix(config.Username, "arn:aws:secretsmanager:") { + return config.Username, nil + } + + return "", errors.NotFoundError("%s driver with FSxN personality must include Credentials of type %s "+ + "in the configuration", config.StorageDriverName, string(drivers.CredentialStoreAWSARN)) +} + +// destroyFSxVolume discovers and deletes a volume using the FSx SDK. This is needed to delete a volume in the case +// that FSx has created a hidden SnapMirror relationship for the purpose of creating backups. If the volume isn't +// found, it returns a NotFoundError so that the client may choose to delete the volume using the underlying ONTAP +// ZAPI/REST client. +func destroyFSxVolume( + ctx context.Context, fsx awsapi.AWSAPI, volConfig *storage.VolumeConfig, config *drivers.OntapStorageDriverConfig, +) error { + name := volConfig.InternalName + fields := LogFields{ + "Method": "destroyFSxVolume", + "Type": "ontap_common", + "name": name, + } + Logd(ctx, config.StorageDriverName, + config.DebugTraceFlags["method"]).WithFields(fields).Trace(">>>> destroyFSxVolume") + defer Logd(ctx, config.StorageDriverName, + config.DebugTraceFlags["method"]).WithFields(fields).Trace("<<<< destroyFSxVolume") + + if config.AWSConfig == nil || fsx == nil { + return errors.New("FSxN not configured") + } + + // If volume doesn't exist, return success + volumeExists, extantVolume, err := fsx.VolumeExists(ctx, volConfig) + if err != nil { + return fmt.Errorf("error checking for existing FSx volume: %v", err) + } + if !volumeExists { + Logc(ctx).WithField("volume", name).Warn("FSx volume already deleted.") + return errors.NotFoundError("volume %s not found", name) + } else if extantVolume.State == awsapi.StateDeleting { + // This is a retry, so give it more time before giving up again. + _, err = fsx.WaitForVolumeStates( + ctx, extantVolume, []string{awsapi.StateDeleted}, []string{awsapi.StateFailed}, awsapi.RetryDeleteTimeout) + return err + } + + // Delete the volume + if err = fsx.DeleteVolume(ctx, extantVolume); err != nil { + return err + } + + // Wait for deletion to complete + deleteTimeout := awsapi.RetryDeleteTimeout + if tridentconfig.CurrentDriverContext == tridentconfig.ContextDocker { + deleteTimeout = awsapi.DefaultDeleteTimeout + } + _, err = fsx.WaitForVolumeStates( + ctx, extantVolume, []string{awsapi.StateDeleted}, []string{awsapi.StateFailed}, deleteTimeout) + return err +} diff --git a/storage_drivers/ontap/aws_common_test.go b/storage_drivers/ontap/aws_common_test.go new file mode 100644 index 000000000..167fbb5ec --- /dev/null +++ b/storage_drivers/ontap/aws_common_test.go @@ -0,0 +1,297 @@ +package ontap + +import ( + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + mockapi "github.com/netapp/trident/mocks/mock_storage_drivers/mock_ontap" + drivers "github.com/netapp/trident/storage_drivers" + "github.com/netapp/trident/storage_drivers/ontap/api" + "github.com/netapp/trident/storage_drivers/ontap/awsapi" +) + +const ( + SECRET_MANAGER_ARN = "arn:aws:secretsmanager:eu-west-3:111111111111:secret:secret-name-mlNvrF" +) + +func TestFSxFilesystemValidation_Error(t *testing.T) { + fsxId := FSX_ID + svmName := "SVM1" + svm := &awsapi.SVM{ + FSxObject: awsapi.FSxObject{ + Name: svmName, + }, + } + mockCtrl := gomock.NewController(t) + mockAWSAPI := mockapi.NewMockAWSAPI(mockCtrl) + CommonStorageDriverConfig := &drivers.CommonStorageDriverConfig{} + CommonStorageDriverConfig.DebugTraceFlags = make(map[string]bool) + CommonStorageDriverConfig.DebugTraceFlags["method"] = true + CommonStorageDriverConfig.StorageDriverName = "ontap-nas" + tests := []struct { + name string + fsxId string + fileSystemIdError error + svmError error + svm *[]*awsapi.SVM + config *drivers.OntapStorageDriverConfig + error string + }{ + { + "FSx id is is empty", + "", + nil, + nil, + nil, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + AWSConfig: &drivers.AWSConfig{}, + }, + "filesystem ID in config must be specified", + }, + { + "FSx id is api error", + fsxId, api.ApiError("not found"), + nil, + nil, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: fsxId, + }, + }, + fmt.Sprintf("filesystem with ID %s not found", fsxId), + }, + { + "Get svm error", + fsxId, + nil, + api.ApiError("not found"), + nil, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: FSX_ID, + }, + }, + "could not retrieve FSxN SVMs", + }, + { + "SVM does not exist in filesystem", + fsxId, + nil, + nil, + &[]*awsapi.SVM{ + svm, + }, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + SVM: "SVM2", + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: FSX_ID, + }, + }, + "SVM SVM2 does not exist in filesystem " + fsxId, + }, + { + "multiple SVMs exist in filesystem", + fsxId, + nil, + nil, + &[]*awsapi.SVM{ + svm, + svm, + }, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: FSX_ID, + }, + }, + "no SVM specified and multiple SVMs exist in filesystem " + fsxId, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.fsxId != "" { + mockAWSAPI.EXPECT().GetFilesystemByID(ctx, test.fsxId).Return(nil, test.fileSystemIdError) + } + + if test.svm != nil || test.svmError != nil { + mockAWSAPI.EXPECT().GetSVMs(ctx).Return(test.svm, test.svmError) + } + err := validateFSxFilesystem(ctx, mockAWSAPI, test.config) + + assert.Contains(t, err.Error(), test.error) + }) + } +} + +func TestFSxFilesystemValidation_NoError(t *testing.T) { + fsxId := FSX_ID + svmName := "SVM1" + mockCtrl := gomock.NewController(t) + mockAWSAPI := mockapi.NewMockAWSAPI(mockCtrl) + CommonStorageDriverConfig := &drivers.CommonStorageDriverConfig{} + CommonStorageDriverConfig.DebugTraceFlags = make(map[string]bool) + CommonStorageDriverConfig.DebugTraceFlags["method"] = true + CommonStorageDriverConfig.StorageDriverName = "ontap-nas" + tests := []struct { + name string + svm *[]*awsapi.SVM + config *drivers.OntapStorageDriverConfig + }{ + { + "FSX filesystem validation without one IPAddresses and no Svm", + &[]*awsapi.SVM{ + { + FSxObject: awsapi.FSxObject{ + Name: svmName, + }, + MgtEndpoint: &awsapi.Endpoint{ + IPAddresses: []string{"1.1.1.1"}, + }, + }, + }, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: fsxId, + }, + }, + }, + { + "FSX filesystem validation with ManagementLIF and Svm", + &[]*awsapi.SVM{ + { + FSxObject: awsapi.FSxObject{ + Name: svmName, + }, + }, + }, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + SVM: svmName, + ManagementLIF: "1.1.1.1", + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: fsxId, + }, + }, + }, + { + "FSX filesystem validation with DNSName and no ManagementLIF ", + &[]*awsapi.SVM{ + { + FSxObject: awsapi.FSxObject{ + Name: svmName, + }, + MgtEndpoint: &awsapi.Endpoint{ + IPAddresses: []string{}, + DNSName: "dns", + }, + }, + }, + &drivers.OntapStorageDriverConfig{ + CommonStorageDriverConfig: CommonStorageDriverConfig, + SVM: svmName, + AWSConfig: &drivers.AWSConfig{ + FSxFilesystemID: fsxId, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockAWSAPI.EXPECT().GetFilesystemByID(ctx, fsxId).Return(nil, nil) + + if test.svm != nil { + mockAWSAPI.EXPECT().GetSVMs(ctx).Return(test.svm, nil) + } + err := validateFSxFilesystem(ctx, mockAWSAPI, test.config) + + assert.NoError(t, err, nil) + }) + } +} + +func TestInitializeAWSDriver(t *testing.T) { + fsxId := "" + secretArn := SECRET_MANAGER_ARN + config := &drivers.OntapStorageDriverConfig{} + config.CommonStorageDriverConfig = &drivers.CommonStorageDriverConfig{} + config.CommonStorageDriverConfig.DebugTraceFlags = make(map[string]bool) + config.CommonStorageDriverConfig.DebugTraceFlags["method"] = true + config.StorageDriverName = "ontap-nas" + config.ManagementLIF = "1.1.1.1" + config.AWSConfig = &drivers.AWSConfig{} + config.AWSConfig.FSxFilesystemID = fsxId + tests := []struct { + name string + secretName string + secretType string + userName string + error string + }{ + {"Invalid secret ARN value", "secret-manager-arn-value", "awsarn", "", "secret ARN secret-manager-arn-value is invalid"}, + {"Invalid secret ARN value - use username and password", "", "awsarn", "arn:aws:secretsmanager:region", "secret ARN arn:aws:secretsmanager:region is invalid"}, + {"Invalid awsarn secret", secretArn, "awsarn", "", "could not retrieve credentials from AWS Secrets Manager"}, + {"valid aws secret", "", "secret", "", ""}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config.Username = test.userName + if test.secretName == "" { + config.Credentials = map[string]string{} + } else { + config.Credentials = map[string]string{ + "type": test.secretType, + "name": test.secretName, + } + } + + _, err := initializeAWSDriver(ctx, config) + + if test.error == "" { + assert.NoError(t, err, nil) + } else { + assert.Contains(t, err.Error(), test.error) + } + }) + } +} + +func TestSvmCredentials(t *testing.T) { + secretArn := "secret-arn" + mockCtrl := gomock.NewController(t) + mockAWSAPI := mockapi.NewMockAWSAPI(mockCtrl) + config := &drivers.OntapStorageDriverConfig{} + config.CommonStorageDriverConfig = &drivers.CommonStorageDriverConfig{} + config.CommonStorageDriverConfig.DebugTraceFlags = make(map[string]bool) + config.CommonStorageDriverConfig.DebugTraceFlags["method"] = true + config.StorageDriverName = "ontap-nas" + tests := []struct { + name string + secretMap map[string]string + error string + }{ + {"The username key is missing", map[string]string{}, "ontap-nas driver must include username in the secret referenced by Credentials"}, + {"The username key is missing", map[string]string{"username": "username"}, "ontap-nas driver must include password in the secret referenced by Credentials"}, + {"The username key is missing", map[string]string{"username": "username", "password": "password"}, ""}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockAWSAPI.EXPECT().GetSecret(ctx, secretArn).Return(test.secretMap, nil) + err := SetSvmCredentials(ctx, secretArn, mockAWSAPI, config) + if test.error == "" { + assert.NoError(t, err, nil) + } else { + assert.Contains(t, err.Error(), test.error) + } + }) + } +} diff --git a/storage_drivers/ontap/awsapi/aws.go b/storage_drivers/ontap/awsapi/aws.go new file mode 100644 index 000000000..bc5e6a26c --- /dev/null +++ b/storage_drivers/ontap/awsapi/aws.go @@ -0,0 +1,861 @@ +// Copyright 2023 NetApp, Inc. All Rights Reserved. + +// Package awsapi provides a high-level interface to the AWS FSx for NetApp ONTAP API. +package awsapi + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/middleware" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/fsx" + fsxtypes "github.com/aws/aws-sdk-go-v2/service/fsx/types" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/cenkalti/backoff/v4" + + . "github.com/netapp/trident/logging" + "github.com/netapp/trident/storage" + "github.com/netapp/trident/utils" + "github.com/netapp/trident/utils/errors" +) + +const ( + RetryDeleteTimeout = 1 * time.Second + DefaultDeleteTimeout = 60 * time.Second + PaginationLimit = 100 +) + +var ( + volumeARNRegex = regexp.MustCompile(`^arn:aws:fsx:(?P[^:]+):(?P\d{12}):volume/(?P[A-z0-9-]+)/(?P[A-z0-9-]+)$`) + secretARNRegex = regexp.MustCompile(`^arn:aws:secretsmanager:(?P[^:]+):(?P\d{12}):secret:(?P[A-z0-9/_+=.@-]+)-[A-z0-9/_+=.@-]{6}$`) +) + +// ClientConfig holds configuration data for the API driver object. +type ClientConfig struct { + // Optional AWS SDK authentication parameters + APIRegion string + APIKey string + SecretKey string + + // FSx Filesystem + FilesystemID string + // Secret Manager Region + SecretManagerRegion string + + // Options + DebugTraceFlags map[string]bool +} + +type Client struct { + config *ClientConfig + fsxClient *fsx.Client + secretsClient *secretsmanager.Client +} + +func createAWSConfig(ctx context.Context, region, apiKey, secretKey string) (aws.Config, error) { + var cfg aws.Config + var err error + + if apiKey != "" { + // Explicit credentials + if region != "" { + cfg, err = awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(apiKey, secretKey, ""), + ), + awsconfig.WithRegion(region), + ) + } else { + cfg, err = awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(apiKey, secretKey, ""), + ), + ) + } + } else { + // Implicit credentials + if region != "" { + cfg, err = awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + } else { + cfg, err = awsconfig.LoadDefaultConfig(ctx) + } + } + + return cfg, err +} + +// NewClient is a factory method for creating a new instance. +func NewClient(ctx context.Context, config ClientConfig) (*Client, error) { + fsxCfg, err := createAWSConfig(ctx, config.APIRegion, config.APIKey, config.SecretKey) + if err != nil { + return nil, err + } + + secMngCfg, err := createAWSConfig(ctx, config.SecretManagerRegion, config.APIKey, config.SecretKey) + if err != nil { + return nil, err + } + + client := &Client{ + config: &config, + fsxClient: fsx.NewFromConfig(fsxCfg), + secretsClient: secretsmanager.NewFromConfig(secMngCfg), + } + + return client, nil +} + +func (d *Client) GetSecret(ctx context.Context, secretARN string) (map[string]string, error) { + input := &secretsmanager.GetSecretValueInput{ + SecretId: utils.Ptr(secretARN), + VersionStage: utils.Ptr("AWSCURRENT"), + } + + secretData, err := d.secretsClient.GetSecretValue(ctx, input) + if err != nil { + return nil, err + } + + var secretMap map[string]string + if err = json.Unmarshal([]byte(DerefString(secretData.SecretString)), &secretMap); err != nil { + return nil, err + } + + return secretMap, nil +} + +// ParseVolumeARN parses the AWS-style ARN for a volume. +func ParseVolumeARN(volumeARN string) (region, accountID, filesystemID, volumeID string, err error) { + match := volumeARNRegex.FindStringSubmatch(volumeARN) + + if match == nil { + err = fmt.Errorf("volume ARN %s is invalid", volumeARN) + return + } + + paramsMap := make(map[string]string) + for i, name := range volumeARNRegex.SubexpNames() { + if i > 0 && i <= len(match) { + paramsMap[name] = match[i] + } + } + + region = paramsMap["region"] + accountID = paramsMap["accountID"] + filesystemID = paramsMap["filesystemID"] + volumeID = paramsMap["volumeID"] + + return +} + +// ParseSecretARN parses the AWS-style ARN for a secret. +func ParseSecretARN(secretARN string) (region, accountID, secretName string, err error) { + match := secretARNRegex.FindStringSubmatch(secretARN) + + if match == nil { + err = fmt.Errorf("secret ARN %s is invalid", secretARN) + return + } + + paramsMap := make(map[string]string) + for i, name := range secretARNRegex.SubexpNames() { + if i > 0 && i <= len(match) { + paramsMap[name] = match[i] + } + } + + region = paramsMap["region"] + accountID = paramsMap["accountID"] + secretName = paramsMap["secretName"] + + return +} + +func (d *Client) GetFilesystems(ctx context.Context) (*[]*Filesystem, error) { + logFields := LogFields{"API": "DescribeFileSystemsPaginator.NextPage"} + + input := &fsx.DescribeFileSystemsInput{} + + paginator := fsx.NewDescribeFileSystemsPaginator(d.fsxClient, input, + func(o *fsx.DescribeFileSystemsPaginatorOptions) { o.Limit = PaginationLimit }) + + var filesystems []*Filesystem + + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate filesystems.") + return nil, fmt.Errorf("error iterating filesystems; %w", err) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + + for _, f := range output.FileSystems { + filesystems = append(filesystems, d.getFilesystemFromFSxFilesystem(f)) + } + } + + logFields["count"] = len(filesystems) + Logc(ctx).WithFields(logFields).Debug("Read FSx filesystems.") + + return &filesystems, nil +} + +func (d *Client) GetFilesystemByID(ctx context.Context, filesystemID string) (*Filesystem, error) { + logFields := LogFields{ + "API": "DescribeFileSystems", + "filesystemID": filesystemID, + } + + input := &fsx.DescribeFileSystemsInput{ + FileSystemIds: []string{filesystemID}, + } + + output, err := d.fsxClient.DescribeFileSystems(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate filesystems.") + return nil, fmt.Errorf("error iterating filesystems; %w", err) + } + if len(output.FileSystems) == 0 { + return nil, errors.NotFoundError(fmt.Sprintf("filesystem %s not found", filesystemID)) + } + if len(output.FileSystems) > 1 { + return nil, errors.NotFoundError(fmt.Sprintf("multiple filesystems with ID '%s' found", filesystemID)) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + Logc(ctx).WithFields(logFields).Debug("Found FSx filesystem by ID.") + + return d.getFilesystemFromFSxFilesystem(output.FileSystems[0]), nil +} + +func (d *Client) getFilesystemFromFSxFilesystem(f fsxtypes.FileSystem) *Filesystem { + filesystem := &Filesystem{ + FSxObject: FSxObject{ + ARN: DerefString(f.ResourceARN), + ID: DerefString(f.FileSystemId), + Name: DerefString(f.DNSName), + }, + Created: DerefTime(f.CreationTime), + OwnerID: DerefString(f.OwnerId), + State: string(f.Lifecycle), + VPCID: DerefString(f.VpcId), + } + + return filesystem +} + +func (d *Client) GetSVMs(ctx context.Context) (*[]*SVM, error) { + logFields := LogFields{"API": "DescribeStorageVirtualMachinesPaginator.NextPage"} + + input := &fsx.DescribeStorageVirtualMachinesInput{ + Filters: []fsxtypes.StorageVirtualMachineFilter{ + { + Name: fsxtypes.StorageVirtualMachineFilterNameFileSystemId, + Values: []string{d.config.FilesystemID}, + }, + }, + } + + paginator := fsx.NewDescribeStorageVirtualMachinesPaginator(d.fsxClient, input, + func(o *fsx.DescribeStorageVirtualMachinesPaginatorOptions) { o.Limit = PaginationLimit }) + + var svms []*SVM + + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate SVMs.") + return nil, fmt.Errorf("error iterating SVMs; %w", err) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + + for _, svm := range output.StorageVirtualMachines { + svms = append(svms, d.getSVMFromFSxSVM(svm)) + } + } + + logFields["count"] = len(svms) + Logc(ctx).WithFields(logFields).Debug("Read FSx SVMs.") + + return &svms, nil +} + +func (d *Client) GetSVMByID(ctx context.Context, svmID string) (*SVM, error) { + logFields := LogFields{ + "API": "DescribeStorageVirtualMachines", + "svmID": svmID, + } + + input := &fsx.DescribeStorageVirtualMachinesInput{ + Filters: []fsxtypes.StorageVirtualMachineFilter{ + { + Name: fsxtypes.StorageVirtualMachineFilterNameFileSystemId, + Values: []string{d.config.FilesystemID}, + }, + }, + StorageVirtualMachineIds: []string{svmID}, + } + + output, err := d.fsxClient.DescribeStorageVirtualMachines(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate SVMs.") + return nil, fmt.Errorf("error iterating SVMs; %w", err) + } + if len(output.StorageVirtualMachines) == 0 { + return nil, errors.NotFoundError(fmt.Sprintf("SVM %s not found", svmID)) + } + if len(output.StorageVirtualMachines) > 1 { + return nil, errors.NotFoundError(fmt.Sprintf("multiple SVMs with ID '%s' found", svmID)) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + Logc(ctx).WithFields(logFields).Debug("Found FSx SVM by ID.") + + return d.getSVMFromFSxSVM(output.StorageVirtualMachines[0]), nil +} + +func (d *Client) getSVMFromFSxSVM(s fsxtypes.StorageVirtualMachine) *SVM { + svm := &SVM{ + FSxObject: FSxObject{ + ARN: DerefString(s.ResourceARN), + ID: DerefString(s.StorageVirtualMachineId), + Name: DerefString(s.Name), + }, + Created: DerefTime(s.CreationTime), + FilesystemID: DerefString(s.FileSystemId), + State: string(s.Lifecycle), + Subtype: string(s.Subtype), + UUID: DerefString(s.UUID), + } + + if s.Endpoints != nil { + if s.Endpoints.Iscsi != nil { + svm.IscsiEndpoint = &Endpoint{ + DNSName: DerefString(s.Endpoints.Iscsi.DNSName), + IPAddresses: s.Endpoints.Iscsi.IpAddresses, + } + } + if s.Endpoints.Management != nil { + svm.MgtEndpoint = &Endpoint{ + DNSName: DerefString(s.Endpoints.Management.DNSName), + IPAddresses: s.Endpoints.Management.IpAddresses, + } + } + if s.Endpoints.Nfs != nil { + svm.NFSEndpoint = &Endpoint{ + DNSName: DerefString(s.Endpoints.Nfs.DNSName), + IPAddresses: s.Endpoints.Nfs.IpAddresses, + } + } + if s.Endpoints.Smb != nil { + svm.SMBEndpoint = &Endpoint{ + DNSName: DerefString(s.Endpoints.Smb.DNSName), + IPAddresses: s.Endpoints.Smb.IpAddresses, + } + } + } + + return svm +} + +func (d *Client) GetVolumes(ctx context.Context) (*[]*Volume, error) { + logFields := LogFields{"API": "DescribeVolumesPaginator.NextPage"} + + input := &fsx.DescribeVolumesInput{ + Filters: []fsxtypes.VolumeFilter{ + { + Name: fsxtypes.VolumeFilterNameFileSystemId, + Values: []string{d.config.FilesystemID}, + }, + }, + } + + paginator := fsx.NewDescribeVolumesPaginator(d.fsxClient, input, + func(o *fsx.DescribeVolumesPaginatorOptions) { o.Limit = PaginationLimit }) + + var volumes []*Volume + + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate volumes.") + return nil, fmt.Errorf("error iterating volumes; %w", err) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + + for _, v := range output.Volumes { + volumes = append(volumes, d.getVolumeFromFSxVolume(v)) + } + } + + logFields["count"] = len(volumes) + Logc(ctx).WithFields(logFields).Debug("Read FSx volumes.") + + return &volumes, nil +} + +func (d *Client) GetVolumeByName(ctx context.Context, name string) (*Volume, error) { + logFields := LogFields{ + "API": "DescribeVolumesPaginator.NextPage", + "volumeName": name, + } + + input := &fsx.DescribeVolumesInput{ + Filters: []fsxtypes.VolumeFilter{ + { + Name: fsxtypes.VolumeFilterNameFileSystemId, + Values: []string{d.config.FilesystemID}, + }, + }, + } + + paginator := fsx.NewDescribeVolumesPaginator(d.fsxClient, input, + func(o *fsx.DescribeVolumesPaginatorOptions) { o.Limit = PaginationLimit }) + + matchingVolumes := make([]fsxtypes.Volume, 0) + + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate volumes.") + return nil, fmt.Errorf("error iterating volumes; %w", err) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + + for _, v := range output.Volumes { + if DerefString(v.Name) == name { + matchingVolumes = append(matchingVolumes, v) + } + } + } + + if len(matchingVolumes) == 0 { + return nil, errors.NotFoundError("volume with name %s not found", name) + } else if len(matchingVolumes) > 1 { + return nil, fmt.Errorf("multiple volumes with name %s found", name) + } + + Logc(ctx).WithFields(logFields).Debug("Found FSx volume by name.") + + return d.getVolumeFromFSxVolume(matchingVolumes[0]), nil +} + +func (d *Client) GetVolumeByARN(ctx context.Context, volumeARN string) (*Volume, error) { + _, _, _, volumeID, err := ParseVolumeARN(volumeARN) + if err != nil { + return nil, err + } + + return d.GetVolumeByID(ctx, volumeID) +} + +func (d *Client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) { + logFields := LogFields{ + "API": "DescribeVolumes", + "volumeID": volumeID, + } + + input := &fsx.DescribeVolumesInput{ + Filters: []fsxtypes.VolumeFilter{ + { + Name: fsxtypes.VolumeFilterNameFileSystemId, + Values: []string{d.config.FilesystemID}, + }, + }, + VolumeIds: []string{volumeID}, + } + + output, err := d.fsxClient.DescribeVolumes(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not iterate volumes.") + return nil, fmt.Errorf("error iterating volumes; %w", err) + } + if len(output.Volumes) == 0 { + return nil, errors.NotFoundError(fmt.Sprintf("volume %s not found", volumeID)) + } + if len(output.Volumes) > 1 { + return nil, errors.NotFoundError(fmt.Sprintf("multiple volumes with ID '%s' found", volumeID)) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + Logc(ctx).WithFields(logFields).Debug("Found FSx volume by ID.") + + return d.getVolumeFromFSxVolume(output.Volumes[0]), nil +} + +// GetVolume uses a volume config record to look for a Filesystem by the most efficient means. +func (d *Client) GetVolume(ctx context.Context, volConfig *storage.VolumeConfig) (*Volume, error) { + // When we know the internal ID, use that as it is vastly more efficient + if volConfig.InternalID != "" { + return d.GetVolumeByARN(ctx, volConfig.InternalID) + } + + // Fall back to the name + return d.GetVolumeByName(ctx, volConfig.InternalName) +} + +func (d *Client) getVolumeFromFSxVolume(v fsxtypes.Volume) *Volume { + volume := &Volume{ + FSxObject: FSxObject{ + ARN: DerefString(v.ResourceARN), + ID: DerefString(v.VolumeId), + Name: DerefString(v.Name), + }, + Created: DerefTime(v.CreationTime), + FilesystemID: DerefString(v.FileSystemId), + Labels: make(map[string]string), + State: string(v.Lifecycle), + } + + if v.OntapConfiguration != nil { + volume.JunctionPath = DerefString(v.OntapConfiguration.JunctionPath) + volume.SecurityStyle = string(v.OntapConfiguration.SecurityStyle) + volume.Size = uint64(DerefInt32(v.OntapConfiguration.SizeInMegabytes)) * 1048576 + volume.SnapshotPolicy = DerefString(v.OntapConfiguration.SnapshotPolicy) + volume.SVMID = DerefString(v.OntapConfiguration.StorageVirtualMachineId) + volume.UUID = DerefString(v.OntapConfiguration.UUID) + } + + for _, tag := range v.Tags { + volume.Labels[DerefString(tag.Key)] = DerefString(tag.Value) + } + + return volume +} + +func (d *Client) VolumeExistsByName(ctx context.Context, name string) (bool, *Volume, error) { + volume, err := d.GetVolumeByName(ctx, name) + if err != nil { + if errors.IsNotFoundError(err) { + return false, nil, nil + } + return false, nil, err + } + return true, volume, nil +} + +func (d *Client) VolumeExistsByARN(ctx context.Context, volumeARN string) (bool, *Volume, error) { + _, _, _, volumeID, err := ParseVolumeARN(volumeARN) + if err != nil { + return false, nil, err + } + + return d.VolumeExistsByID(ctx, volumeID) +} + +func (d *Client) VolumeExistsByID(ctx context.Context, volumeID string) (bool, *Volume, error) { + volume, err := d.GetVolumeByID(ctx, volumeID) + if err != nil { + if IsVolumeNotFoundError(err) { + return false, nil, nil + } + return false, nil, err + } + return true, volume, nil +} + +// VolumeExists uses a volume config record to look for a Filesystem by the most efficient means. +func (d *Client) VolumeExists(ctx context.Context, volConfig *storage.VolumeConfig) (bool, *Volume, error) { + // When we know the internal ID, use that as it is vastly more efficient + if volConfig.InternalID != "" { + return d.VolumeExistsByARN(ctx, volConfig.InternalID) + } + + // Fall back to the name + return d.VolumeExistsByName(ctx, volConfig.InternalName) +} + +func (d *Client) WaitForVolumeStates( + ctx context.Context, volume *Volume, desiredStates, abortStates []string, maxElapsedTime time.Duration, +) (string, error) { + volumeState := "" + + checkVolumeState := func() error { + v, err := d.GetVolumeByID(ctx, volume.ID) + if err != nil { + + // There is no 'Deleted' state in FSx -- the volume just vanishes. If we failed to query + // the volume info, and we're trying to transition to StateDeleted, and we get back a 404, + // then return success. Otherwise, log the error as usual. + if utils.SliceContainsString(desiredStates, StateDeleted) && IsVolumeNotFoundError(err) { + Logc(ctx).Debugf("Implied deletion for volume %s.", volume.Name) + volumeState = StateDeleted + return nil + } + + volumeState = "" + return fmt.Errorf("could not get volume status; %w", err) + } + + volumeState = v.State + + if utils.SliceContainsString(desiredStates, volumeState) { + return nil + } + + err = fmt.Errorf("volume state is %s, not any of %s", v.State, desiredStates) + + // Return a permanent error to stop retrying if we reached one of the abort states + for _, abortState := range abortStates { + if volumeState == abortState { + return backoff.Permanent(TerminalState(err)) + } + } + + // Override error for a deleting volume + if utils.SliceContainsString(desiredStates, StateDeleted) && volumeState == StateDeleting { + err = errors.VolumeDeletingError(err.Error()) + } + + return err + } + stateNotify := func(err error, duration time.Duration) { + Logc(ctx).WithFields(LogFields{ + "increment": duration, + "message": err.Error(), + }).Debugf("Waiting for volume state.") + } + stateBackoff := backoff.NewExponentialBackOff() + stateBackoff.MaxElapsedTime = maxElapsedTime + stateBackoff.MaxInterval = 5 * time.Second + stateBackoff.RandomizationFactor = 0.1 + stateBackoff.InitialInterval = backoff.DefaultInitialInterval + stateBackoff.Multiplier = 1.414 + + Logc(ctx).WithField("desiredStates", desiredStates).Info("Waiting for volume state.") + + if err := backoff.RetryNotify(checkVolumeState, stateBackoff, stateNotify); err != nil { + if terminalStateErr, ok := err.(*TerminalStateError); ok { + Logc(ctx).Errorf("Volume reached terminal state; %w", terminalStateErr) + } else { + Logc(ctx).Errorf("Volume state was not any of %s after %3.2f seconds.", + desiredStates, stateBackoff.MaxElapsedTime.Seconds()) + } + return volumeState, err + } + + Logc(ctx).WithField("desiredStates", desiredStates).Debug("Desired volume state reached.") + + return volumeState, nil +} + +func (d *Client) CreateVolume(ctx context.Context, request *VolumeCreateRequest) (*Volume, error) { + logFields := LogFields{ + "API": "CreateVolume", + "volumeName": request.Name, + } + + ontapConfig := &fsxtypes.CreateOntapVolumeConfiguration{ + SizeInMegabytes: utils.Ptr(int32(request.SizeBytes / 1048576)), + StorageVirtualMachineId: utils.Ptr(request.SVMID), + JunctionPath: utils.Ptr("/" + request.Name), + SecurityStyle: fsxtypes.SecurityStyleUnix, + SnapshotPolicy: utils.Ptr(request.SnapshotPolicy), + StorageEfficiencyEnabled: utils.Ptr(true), + TieringPolicy: &fsxtypes.TieringPolicy{ + Name: fsxtypes.TieringPolicyNameNone, + }, + } + + tags := make([]fsxtypes.Tag, 0) + for k, v := range request.Labels { + tags = append(tags, fsxtypes.Tag{ + Key: utils.Ptr(k), + Value: utils.Ptr(v), + }) + } + + input := &fsx.CreateVolumeInput{ + Name: utils.Ptr(request.Name), + VolumeType: fsxtypes.VolumeTypeOntap, + OntapConfiguration: ontapConfig, + Tags: tags, + } + + output, err := d.fsxClient.CreateVolume(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not create volume.") + return nil, fmt.Errorf("error creating volume; %w", err) + } + + newVolume := d.getVolumeFromFSxVolume(*output.Volume) + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + logFields["volumeID"] = newVolume.ID + Logc(ctx).WithFields(logFields).Info("Volume create request issued.") + + return newVolume, nil +} + +func (d *Client) RelabelVolume(ctx context.Context, volume *Volume, labels map[string]string) error { + logFields := LogFields{ + "API": "TagResource", + "volumeID": volume.ID, + "volumeName": volume.Name, + } + + tags := make([]fsxtypes.Tag, 0) + for k, v := range labels { + tags = append(tags, fsxtypes.Tag{ + Key: utils.Ptr(k), + Value: utils.Ptr(v), + }) + } + + input := &fsx.TagResourceInput{ + ResourceARN: utils.Ptr(volume.ARN), + Tags: tags, + } + + output, err := d.fsxClient.TagResource(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not relabel volume.") + return fmt.Errorf("error relabeling volume; %w", err) + } + + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + Logc(ctx).WithFields(logFields).Info("Volume relabeled.") + + return nil +} + +func (d *Client) ResizeVolume(ctx context.Context, volume *Volume, newSizeBytes uint64) (*Volume, error) { + logFields := LogFields{ + "API": "UpdateVolume", + "volumeID": volume.ID, + "volumeName": volume.Name, + } + + ontapConfig := &fsxtypes.UpdateOntapVolumeConfiguration{ + SizeInMegabytes: utils.Ptr(int32(newSizeBytes / 1048576)), + } + + input := &fsx.UpdateVolumeInput{ + VolumeId: utils.Ptr(volume.ID), + OntapConfiguration: ontapConfig, + } + + output, err := d.fsxClient.UpdateVolume(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not resize volume.") + return nil, fmt.Errorf("error resizing volume; %w", err) + } + + logFields["newSize"] = newSizeBytes + logFields["requestID"], _ = middleware.GetRequestIDMetadata(output.ResultMetadata) + Logc(ctx).WithFields(logFields).Info("Volume resized.") + + return d.getVolumeFromFSxVolume(*output.Volume), nil +} + +func (d *Client) DeleteVolume(ctx context.Context, volume *Volume) error { + logFields := LogFields{ + "API": "DeleteVolume", + "volumeID": volume.ID, + "volumeName": volume.Name, + } + + ontapConfig := &fsxtypes.DeleteVolumeOntapConfiguration{ + SkipFinalBackup: utils.Ptr(true), + } + + input := &fsx.DeleteVolumeInput{ + VolumeId: utils.Ptr(volume.ID), + OntapConfiguration: ontapConfig, + } + + _, err := d.fsxClient.DeleteVolume(ctx, input) + if err != nil { + logFields["requestID"] = GetRequestIDFromError(err) + Logc(ctx).WithFields(logFields).WithError(err).Error("Could not delete volume.") + return fmt.Errorf("error deleting volume; %w", err) + } + + Logc(ctx).WithFields(logFields).Info("Volume deleted.") + + return nil +} + +// /////////////////////////////////////////////////////////////////////////////// +// Miscellaneous utility functions and error types +// /////////////////////////////////////////////////////////////////////////////// + +// IsVolumeNotFoundError checks whether an error returned from the AWS SDK contains a VolumeNotFound error. +func IsVolumeNotFoundError(err error) bool { + if err == nil { + return false + } + var vnf *fsxtypes.VolumeNotFound + return errors.As(err, &vnf) +} + +func GetRequestIDFromError(err error) (requestID string) { + if err != nil { + var re *awshttp.ResponseError + if errors.As(err, &re) { + requestID = re.ServiceRequestID() + } + } + return +} + +// DerefString accepts a string pointer and returns the value of the string, or "" if the pointer is nil. +func DerefString(s *string) string { + if s != nil { + return *s + } + return "" +} + +func DerefInt32(i *int32) int32 { + if i != nil { + return *i + } + return 0 +} + +// DerefTime accepts a time pointer and returns the value of the timestamp, or nil. +func DerefTime(t *time.Time) time.Time { + if t != nil { + return *t + } + return time.Time{} +} + +// TerminalStateError signals that the object is in a terminal state. This is used to stop waiting on +// an object to change state. +type TerminalStateError struct { + Err error +} + +func (e *TerminalStateError) Error() string { + return e.Err.Error() +} + +// TerminalState wraps the given err in a *TerminalStateError. +func TerminalState(err error) *TerminalStateError { + return &TerminalStateError{ + Err: err, + } +} diff --git a/storage_drivers/ontap/awsapi/aws_structs.go b/storage_drivers/ontap/awsapi/aws_structs.go new file mode 100644 index 000000000..b06260753 --- /dev/null +++ b/storage_drivers/ontap/awsapi/aws_structs.go @@ -0,0 +1,79 @@ +// Copyright 2023 NetApp, Inc. All Rights Reserved. + +package awsapi + +import "time" + +const ( + StateCreating = "CREATING" + StateCreated = "CREATED" + StateDeleting = "DELETING" + StateFailed = "FAILED" + StateMisconfigured = "MISCONFIGURED" + StatePending = "PENDING" + StateAvailable = "AVAILABLE" + StateDeleted = "DELETED" +) + +type FSxObject struct { + ARN string `json:"arn"` + ID string `json:"id"` + Name string `json:"name"` +} + +type Filesystem struct { + FSxObject + Created time.Time `json:"created"` + OwnerID string `json:"ownerID"` + State string `json:"state"` + VPCID string `json:"vpcID"` +} + +type SVM struct { + FSxObject + Created time.Time `json:"created"` + FilesystemID string `json:"filesystemID"` + State string `json:"state"` + Subtype string `json:"subtype"` + UUID string `json:"uuid"` + + IscsiEndpoint *Endpoint `json:"iscsiEndpoint"` + MgtEndpoint *Endpoint `json:"mgtEndpoint"` + NFSEndpoint *Endpoint `json:"nfsEndpoint"` + SMBEndpoint *Endpoint `json:"smbEndpoint"` +} + +type Endpoint struct { + DNSName string `json:"dnsName"` + IPAddresses []string `json:"ipAddresses"` +} + +type Volume struct { + FSxObject + Created time.Time `json:"created"` + FilesystemID string `json:"filesystemID"` + State string `json:"state"` + JunctionPath string `json:"junctionPath"` + Labels map[string]string `json:"labels"` + SecurityStyle string `json:"securityStyle"` + Size uint64 `json:"size"` + SnapshotPolicy string `json:"snapshotPolicy"` + SVMID string `json:"svmID"` + UUID string `json:"uuid"` +} + +type VolumeCreateRequest struct { + SVMID string `json:"svmID"` + Name string `json:"name"` + Zone string `json:"zone,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ProtocolTypes []string `json:"protocolTypes"` + SizeBytes int64 `json:"sizeBytes"` + SecurityStyle string `json:"securityStyle"` + SnapReserve *int64 `json:"snapReserve,omitempty"` + SnapshotDirectory bool `json:"snapshotDirectory"` + SnapshotPolicy string `json:"snapshotPolicy,omitempty"` + UnixPermissions string `json:"unixPermissions,omitempty"` + BackupID string `json:"backupId,omitempty"` + SnapshotID string `json:"snapshotId,omitempty"` +} diff --git a/storage_drivers/ontap/awsapi/types.go b/storage_drivers/ontap/awsapi/types.go new file mode 100644 index 000000000..3bd4dc39e --- /dev/null +++ b/storage_drivers/ontap/awsapi/types.go @@ -0,0 +1,39 @@ +// Copyright 2023 NetApp, Inc. All Rights Reserved. + +package awsapi + +//go:generate mockgen -package mock_api -destination=../../../mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go github.com/netapp/trident/storage_drivers/ontap/awsapi AWSAPI + +import ( + "context" + "time" + + "github.com/netapp/trident/storage" +) + +type AWSAPI interface { + GetSecret(ctx context.Context, secretARN string) (map[string]string, error) + + GetFilesystems(ctx context.Context) (*[]*Filesystem, error) + GetFilesystemByID(ctx context.Context, ID string) (*Filesystem, error) + + GetSVMs(ctx context.Context) (*[]*SVM, error) + GetSVMByID(ctx context.Context, ID string) (*SVM, error) + + GetVolumes(ctx context.Context) (*[]*Volume, error) + GetVolumeByName(ctx context.Context, name string) (*Volume, error) + GetVolumeByARN(ctx context.Context, volumeARN string) (*Volume, error) + GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) + GetVolume(ctx context.Context, volConfig *storage.VolumeConfig) (*Volume, error) + VolumeExistsByName(ctx context.Context, name string) (bool, *Volume, error) + VolumeExistsByARN(ctx context.Context, volumeARN string) (bool, *Volume, error) + VolumeExistsByID(ctx context.Context, volumeID string) (bool, *Volume, error) + VolumeExists(ctx context.Context, volConfig *storage.VolumeConfig) (bool, *Volume, error) + WaitForVolumeStates( + ctx context.Context, volume *Volume, desiredStates, abortStates []string, maxElapsedTime time.Duration, + ) (string, error) + CreateVolume(ctx context.Context, request *VolumeCreateRequest) (*Volume, error) + RelabelVolume(ctx context.Context, volume *Volume, labels map[string]string) error + ResizeVolume(ctx context.Context, volume *Volume, newSizeBytes uint64) (*Volume, error) + DeleteVolume(ctx context.Context, volume *Volume) error +} diff --git a/storage_drivers/ontap/ontap_common_test.go b/storage_drivers/ontap/ontap_common_test.go index 8eec2235e..597a78b9e 100644 --- a/storage_drivers/ontap/ontap_common_test.go +++ b/storage_drivers/ontap/ontap_common_test.go @@ -844,7 +844,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Positive case: Test no pools, ontap NAS storageDriver := newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools := map[string]storage.Pool{} virtualPools := map[string]storage.Pool{} storageDriver.virtualPools = virtualPools @@ -857,7 +857,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Positive case: Test one valid virtual pool with NASType = NFS storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -872,7 +872,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Positive case: Test one valid virtual pool with NASType = SMB storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -887,7 +887,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Negative case: Test one valid virtual pool with invalid securityStyle storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -902,7 +902,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Positive case: Test two valid virtual pools storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool(), "test2": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -916,7 +916,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with snapshotpolicy empty storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -929,7 +929,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with Invalid value of Encryption storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -942,7 +942,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with empty snapshot dir storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools storageDriver.physicalPools = physicalPools @@ -954,7 +954,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test Invalid value of snapshot dir storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools storageDriver.physicalPools = physicalPools @@ -966,7 +966,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test one valid virtual with invalid value for label in pool storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -979,7 +979,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Negative case: Test one valid virtual with Invalid value of SecurityStyle attribute storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -992,7 +992,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test one valid virtual with empty Export Policy storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1005,7 +1005,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test one valid virtual with Unix Permission empty storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) storageDriver.Config.NASType = sa.NFS physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} @@ -1019,7 +1019,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test one valid virtual with Invalid value of Tiering policy storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1036,7 +1036,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { mockAPI.EXPECT().SVMName().AnyTimes().Return("SVM1") mockAPI.EXPECT().SupportsFeature(ctx, api.QosPolicies).AnyTimes().Return(false) storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) storageDriver.API = mockAPI physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} @@ -1056,7 +1056,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { mockAPI.EXPECT().SVMName().AnyTimes().Return("SVM1") mockAPI.EXPECT().SupportsFeature(ctx, api.QosPolicies).AnyTimes().Return(true) storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) storageDriver.API = mockAPI physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} @@ -1090,7 +1090,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Positive case: Test with media type set storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1105,7 +1105,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Positive case: Test with Invalid media type storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1120,7 +1120,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with less size of the pool storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1135,7 +1135,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with invalid value for size in pool storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1151,7 +1151,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with invalid value for splitOnClone storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1166,7 +1166,7 @@ func TestValidateStoragePools_Valid_OntapNAS(t *testing.T) { // Negative case: Test with empty value for splitOnClone storageDriver = newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, - tridentconfig.DriverContext("CSI"), false) + tridentconfig.DriverContext("CSI"), false, nil) physicalPools = map[string]storage.Pool{} virtualPools = map[string]storage.Pool{"test": getValidOntapNASPool()} storageDriver.virtualPools = virtualPools @@ -1284,7 +1284,7 @@ func TestValidateStoragePools_LUKS(t *testing.T) { // //////////////////////////////////////////////////////////////////////////////////////////////////////////// // Negative case: Test one virtual pool, LUKS and NAS not allowed storageDriver := newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, tridentconfig.DriverContext("CSI"), - false) + false, nil) pool = getValidOntapNASPool() pool.InternalAttributes()[LUKSEncryption] = "true" physicalPools = map[string]storage.Pool{} @@ -5130,7 +5130,7 @@ func TestNewOntapTelemetry(t *testing.T) { vserverAggrName := ONTAPTEST_VSERVER_AGGR_NAME storageDriver := newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, tridentconfig.DriverContext("CSI"), - false) + false, nil) // Test-1 : Valid value of UsageHeartBeat storageDriver.Config.UsageHeartbeat = "3.0" diff --git a/storage_drivers/ontap/ontap_nas.go b/storage_drivers/ontap/ontap_nas.go index 53c9e835e..cb944e923 100644 --- a/storage_drivers/ontap/ontap_nas.go +++ b/storage_drivers/ontap/ontap_nas.go @@ -18,6 +18,7 @@ import ( sa "github.com/netapp/trident/storage_attribute" drivers "github.com/netapp/trident/storage_drivers" "github.com/netapp/trident/storage_drivers/ontap/api" + "github.com/netapp/trident/storage_drivers/ontap/awsapi" "github.com/netapp/trident/utils" "github.com/netapp/trident/utils/errors" ) @@ -47,6 +48,7 @@ type NASStorageDriver struct { initialized bool Config drivers.OntapStorageDriverConfig API api.OntapAPI + AWSAPI awsapi.AWSAPI telemetry *Telemetry physicalPools map[string]storage.Pool @@ -97,18 +99,26 @@ func (d *NASStorageDriver) Initialize( // Parse the config config, err := InitializeOntapConfig(ctx, driverContext, configJSON, commonConfig, backendSecret) if err != nil { - return fmt.Errorf("error initializing %s driver: %v", d.Name(), err) + return fmt.Errorf("error initializing %s driver; %v", d.Name(), err) + } + + // Initialize AWS API if applicable. + // Unit tests mock the API layer, so we only use the real API interface if it doesn't already exist. + if d.AWSAPI == nil { + d.AWSAPI, err = initializeAWSDriver(ctx, config) + if err != nil { + return fmt.Errorf("error initializing %s AWS driver; %v", d.Name(), err) + } } - d.Config = *config + // Initialize the ONTAP API. // Unit tests mock the API layer, so we only use the real API interface if it doesn't already exist. if d.API == nil { d.API, err = InitializeOntapDriver(ctx, config) if err != nil { - return fmt.Errorf("error initializing %s driver: %v", d.Name(), err) + return fmt.Errorf("error initializing %s driver; %v", d.Name(), err) } } - d.Config = *config d.physicalPools, d.virtualPools, err = InitializeStoragePoolsCommon(ctx, d, @@ -532,6 +542,16 @@ func (d *NASStorageDriver) Destroy(ctx context.Context, volConfig *storage.Volum return nil } + // If volume exists and this is FSx, try the FSx SDK first so that any backup mirror relationship + // is cleaned up. If the volume isn't found, then FSx may not know about it yet, so just try the + // underlying ONTAP delete call. Any race condition with FSx will be resolved on a retry. + if d.AWSAPI != nil { + err = destroyFSxVolume(ctx, d.AWSAPI, volConfig, &d.Config) + if err == nil || !errors.IsNotFoundError(err) { + return err + } + } + // If flexvol has been a snapmirror destination if err := d.API.SnapmirrorDeleteViaDestination(ctx, name, d.API.SVMName()); err != nil { if !api.IsNotFoundError(err) { diff --git a/storage_drivers/ontap/ontap_nas_test.go b/storage_drivers/ontap/ontap_nas_test.go index 4efa5791e..e2a50bff1 100644 --- a/storage_drivers/ontap/ontap_nas_test.go +++ b/storage_drivers/ontap/ontap_nas_test.go @@ -21,6 +21,7 @@ import ( sa "github.com/netapp/trident/storage_attribute" drivers "github.com/netapp/trident/storage_drivers" "github.com/netapp/trident/storage_drivers/ontap/api" + "github.com/netapp/trident/storage_drivers/ontap/awsapi" "github.com/netapp/trident/utils" "github.com/netapp/trident/utils/errors" ) @@ -54,6 +55,7 @@ const ( BackendUUID = "deadbeef-03af-4394-ace4-e177cdbcaf28" ONTAPTEST_LOCALHOST = "127.0.0.1" ONTAPTEST_VSERVER_AGGR_NAME = "data" + FSX_ID = "fsx-1234" ) func TestOntapNasStorageDriverConfigString(t *testing.T) { @@ -63,9 +65,9 @@ func TestOntapNasStorageDriverConfigString(t *testing.T) { ontapNasDrivers := []NASStorageDriver{ *newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", true), + "CSI", true, nil), *newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", false), + "CSI", false, nil), } sensitiveIncludeList := map[string]string{ @@ -101,7 +103,7 @@ func TestOntapNasStorageDriverConfigString(t *testing.T) { } func newTestOntapNASDriver( - vserverAdminHost, vserverAdminPort, vserverAggrName string, driverContext tridentconfig.DriverContext, useREST bool, + vserverAdminHost, vserverAdminPort, vserverAggrName string, driverContext tridentconfig.DriverContext, useREST bool, fsxId *string, ) *NASStorageDriver { config := &drivers.OntapStorageDriverConfig{} sp := func(s string) *string { return &s } @@ -119,6 +121,11 @@ func newTestOntapNASDriver( config.DriverContext = driverContext config.UseREST = useREST + if fsxId != nil { + config.AWSConfig = &drivers.AWSConfig{} + config.AWSConfig.FSxFilesystemID = *fsxId + } + nasDriver := &NASStorageDriver{} nasDriver.Config = *config @@ -141,7 +148,7 @@ func TestInitializeStoragePoolsLabels(t *testing.T) { mockCtrl := gomock.NewController(t) mockAPI := mockapi.NewMockOntapAPI(mockCtrl) - d := newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, "CSI", false) + d := newTestOntapNASDriver(vserverAdminHost, "443", vserverAggrName, "CSI", false, nil) d.API = mockAPI d.Config.Storage = []drivers.OntapStorageDriverPool{ { @@ -252,7 +259,7 @@ func TestOntapNasStorageDriverInitialize_WithTwoAuthMethods(t *testing.T) { "clientprivatekey": "dummy-client-private-key" }` ontapNasDriver := newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", false) + "CSI", false, nil) result := ontapNasDriver.Initialize(ctx, "CSI", configJSON, commonConfig, map[string]string{}, BackendUUID) @@ -289,7 +296,7 @@ func TestOntapNasStorageDriverInitialize_WithTwoAuthMethodsWithSecrets(t *testin "clientcertificate": "dummy-certificate", } ontapNasDriver := newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", false) + "CSI", false, nil) result := ontapNasDriver.Initialize(ctx, "CSI", configJSON, commonConfig, secrets, BackendUUID) @@ -326,7 +333,7 @@ func TestOntapNasStorageDriverInitialize_WithTwoAuthMethodsWithConfigAndSecrets( "clientcertificate": "dummy-certificate", } ontapNasDriver := newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", false) + "CSI", false, nil) result := ontapNasDriver.Initialize(ctx, "CSI", configJSON, commonConfig, secrets, BackendUUID) @@ -335,6 +342,14 @@ func TestOntapNasStorageDriverInitialize_WithTwoAuthMethodsWithConfigAndSecrets( assert.Contains(t, result.Error(), "more than one authentication method", "expected error string not found") } +func newMockAWSOntapNASDriver(t *testing.T) (*mockapi.MockOntapAPI, *mockapi.MockAWSAPI, *NASStorageDriver) { + mockCtrl := gomock.NewController(t) + mockAPI, driver := newMockOntapNASDriver(t) + mockAWSAPI := mockapi.NewMockAWSAPI(mockCtrl) + driver.AWSAPI = mockAWSAPI + return mockAPI, mockAWSAPI, driver +} + func newMockOntapNASDriver(t *testing.T) (*mockapi.MockOntapAPI, *NASStorageDriver) { mockCtrl := gomock.NewController(t) mockAPI := mockapi.NewMockOntapAPI(mockCtrl) @@ -342,9 +357,10 @@ func newMockOntapNASDriver(t *testing.T) (*mockapi.MockOntapAPI, *NASStorageDriv vserverAdminHost := ONTAPTEST_LOCALHOST vserverAdminPort := "0" vserverAggrName := ONTAPTEST_VSERVER_AGGR_NAME + fsxId := FSX_ID driver := newTestOntapNASDriver(vserverAdminHost, vserverAdminPort, vserverAggrName, - "CSI", false) + "CSI", false, &fsxId) driver.API = mockAPI return mockAPI, driver } @@ -961,6 +977,69 @@ func TestOntapNasStorageDriverVolumeClone_SMBShareCreateFail(t *testing.T) { assert.Error(t, result) } +func TestOntapNasStorageDriverVolumeDestroy_FSx(t *testing.T) { + svmName := "SVM1" + volName := "testVol" + volNameInternal := volName + "Internal" + mockAPI, mockAWSAPI, driver := newMockAWSOntapNASDriver(t) + + volConfig := &storage.VolumeConfig{ + Size: "1g", + Name: volName, + InternalName: volNameInternal, + Encryption: "false", + FileSystem: "xfs", + } + + assert.NotNil(t, mockAPI) + + tests := []struct { + message string + nasType string + smbShare string + state string + }{ + {"Test NFS volume in FSx in available state", "nfs", "", "AVAILABLE"}, + {"Test NFS volume in FSx in deleting state", "nfs", "", "DELETING"}, + {"Test NFS volume does not exist in FSx", "nfs", "", ""}, + {"Test SMB volume does not exist in FSx", "smb", volConfig.InternalName, ""}, + } + + for _, test := range tests { + t.Run(test.message, func(t *testing.T) { + vol := awsapi.Volume{ + State: test.state, + } + isVolumeExists := vol.State != "" + mockAPI.EXPECT().VolumeExists(ctx, volConfig.InternalName).Return(true, nil) + mockAWSAPI.EXPECT().VolumeExists(ctx, volConfig).Return(isVolumeExists, &vol, nil) + if isVolumeExists { + mockAWSAPI.EXPECT().WaitForVolumeStates( + ctx, &vol, []string{awsapi.StateDeleted}, []string{awsapi.StateFailed}, awsapi.RetryDeleteTimeout).Return("", nil) + if vol.State == awsapi.StateAvailable { + mockAWSAPI.EXPECT().DeleteVolume(ctx, &vol).Return(nil) + } + } else { + mockAPI.EXPECT().SVMName().AnyTimes().Return(svmName) + mockAPI.EXPECT().SnapmirrorDeleteViaDestination(ctx, volConfig.InternalName, svmName).Return(nil) + mockAPI.EXPECT().SnapmirrorRelease(ctx, volConfig.InternalName, svmName).Return(nil) + mockAPI.EXPECT().VolumeDestroy(ctx, volConfig.InternalName, true).Return(nil) + if test.nasType == sa.SMB { + if test.smbShare == "" { + mockAPI.EXPECT().SMBShareExists(ctx, volConfig.InternalName).Return(true, nil) + mockAPI.EXPECT().SMBShareDestroy(ctx, volConfig.InternalName).Return(nil) + } + driver.Config.NASType = sa.SMB + driver.Config.SMBShare = test.smbShare + } + } + result := driver.Destroy(ctx, volConfig) + + assert.NoError(t, result) + }) + } +} + func TestOntapNasStorageDriverVolumeDestroy(t *testing.T) { svmName := "SVM1" volName := "testVol" @@ -1867,7 +1946,7 @@ func TestOntapNasStorageDriverGetUpdateType(t *testing.T) { } newDriver := newTestOntapNASDriver(ONTAPTEST_LOCALHOST, "0", ONTAPTEST_VSERVER_AGGR_NAME, - "CSI", false) + "CSI", false, nil) newDriver.API = mockAPI prefix2 := "storage_" newDriver.Config.StoragePrefix = &prefix2 @@ -1902,7 +1981,7 @@ func TestOntapNasStorageDriverGetUpdateType_Failure(t *testing.T) { // Created a SAN driver newDriver := newTestOntapNASDriver(ONTAPTEST_LOCALHOST, "0", ONTAPTEST_VSERVER_AGGR_NAME, - "CSI", false) + "CSI", false, nil) newDriver.API = mockAPI prefix2 := "storage_" newDriver.Config.StoragePrefix = &prefix2 diff --git a/storage_drivers/ontap/ontap_san_economy_test.go b/storage_drivers/ontap/ontap_san_economy_test.go index 37327fbdd..7d4f06266 100644 --- a/storage_drivers/ontap/ontap_san_economy_test.go +++ b/storage_drivers/ontap/ontap_san_economy_test.go @@ -3347,7 +3347,7 @@ func TestGetUpdateType_OtherChanges(t *testing.T) { func TestGetUpdateType_Failure(t *testing.T) { mockCtrl := gomock.NewController(t) mockAPI := mockapi.NewMockOntapAPI(mockCtrl) - oldDriver := newTestOntapNASDriver(ONTAPTEST_LOCALHOST, "0", ONTAPTEST_VSERVER_AGGR_NAME, "CSI", false) + oldDriver := newTestOntapNASDriver(ONTAPTEST_LOCALHOST, "0", ONTAPTEST_VSERVER_AGGR_NAME, "CSI", false, nil) newDriver := newTestOntapSanEcoDriver(ONTAPTEST_LOCALHOST, "0", ONTAPTEST_VSERVER_AGGR_NAME, true, mockAPI) expectedBitmap := &roaring.Bitmap{} expectedBitmap.Add(storage.InvalidUpdate) diff --git a/storage_drivers/types.go b/storage_drivers/types.go index 373b0a904..8371a2e40 100644 --- a/storage_drivers/types.go +++ b/storage_drivers/types.go @@ -104,24 +104,25 @@ func (d *CommonStorageDriverConfig) SetBackendName(backendName string) { // OntapStorageDriverConfig holds settings for OntapStorageDrivers type OntapStorageDriverConfig struct { - *CommonStorageDriverConfig // embedded types replicate all fields - ManagementLIF string `json:"managementLIF"` - DataLIF string `json:"dataLIF"` - IgroupName string `json:"igroupName"` - SVM string `json:"svm"` - Username string `json:"username"` - Password string `json:"password"` - Aggregate string `json:"aggregate"` - UsageHeartbeat string `json:"usageHeartbeat"` // in hours, default to 24.0 - QtreePruneFlexvolsPeriod string `json:"qtreePruneFlexvolsPeriod"` // in seconds, default to 600 - QtreeQuotaResizePeriod string `json:"qtreeQuotaResizePeriod"` // in seconds, default to 60 - QtreesPerFlexvol string `json:"qtreesPerFlexvol"` // default to 200 - LUNsPerFlexvol string `json:"lunsPerFlexvol"` // default to 100 - EmptyFlexvolDeferredDeletePeriod string `json:"emptyFlexvolDeferredDeletePeriod"` // in seconds, default to 28800 - NfsMountOptions string `json:"nfsMountOptions"` - LimitAggregateUsage string `json:"limitAggregateUsage"` - AutoExportPolicy bool `json:"autoExportPolicy"` - AutoExportCIDRs []string `json:"autoExportCIDRs"` + *CommonStorageDriverConfig // embedded types replicate all fields + AWSConfig *AWSConfig `json:"aws,omitEmpty"` // AWS-specific config attributes + ManagementLIF string `json:"managementLIF"` + DataLIF string `json:"dataLIF"` + IgroupName string `json:"igroupName"` + SVM string `json:"svm"` + Username string `json:"username"` + Password string `json:"password"` + Aggregate string `json:"aggregate"` + UsageHeartbeat string `json:"usageHeartbeat"` // in hours, default to 24.0 + QtreePruneFlexvolsPeriod string `json:"qtreePruneFlexvolsPeriod"` // in seconds, default to 600 + QtreeQuotaResizePeriod string `json:"qtreeQuotaResizePeriod"` // in seconds, default to 60 + QtreesPerFlexvol string `json:"qtreesPerFlexvol"` // default to 200 + LUNsPerFlexvol string `json:"lunsPerFlexvol"` // default to 100 + EmptyFlexvolDeferredDeletePeriod string `json:"emptyFlexvolDeferredDeletePeriod"` // in seconds, default to 28800 + NfsMountOptions string `json:"nfsMountOptions"` + LimitAggregateUsage string `json:"limitAggregateUsage"` + AutoExportPolicy bool `json:"autoExportPolicy"` + AutoExportCIDRs []string `json:"autoExportCIDRs"` OntapStorageDriverPool Storage []OntapStorageDriverPool `json:"storage"` UseCHAP bool `json:"useCHAP"` @@ -149,6 +150,14 @@ type OntapStorageDriverPool struct { OntapStorageDriverConfigDefaults `json:"defaults"` } +// AWSConfig holds settings for ONTAP drivers used with AWS, including FSx for NetApp ONTAP. +type AWSConfig struct { + APIRegion string `json:"apiRegion"` + FSxFilesystemID string `json:"fsxFilesystemID"` + APIKey string `json:"apiKey"` + SecretKey string `json:"secretKey"` +} + // StorageBackendPool is a type constraint that enables drivers to generically report non-overlapping storage pools // within a backend. type StorageBackendPool interface { @@ -720,7 +729,7 @@ func (d FakeStorageDriverConfig) GoString() string { } // InjectSecrets function replaces sensitive fields in the config with the field values in the map -func (d *FakeStorageDriverConfig) InjectSecrets(secretMap map[string]string) error { +func (d *FakeStorageDriverConfig) InjectSecrets(_ map[string]string) error { // Nothing to do return nil @@ -752,7 +761,7 @@ func (d *FakeStorageDriverConfig) HideSensitiveWithSecretName(secretName string) // GetAndHideSensitive function builds a map of any sensitive fields it contains (credentials, etc.), // replaces those fields with secretName and returns the the map. -func (d *FakeStorageDriverConfig) GetAndHideSensitive(secretName string) map[string]string { +func (d *FakeStorageDriverConfig) GetAndHideSensitive(_ string) map[string]string { return map[string]string{} } @@ -885,7 +894,12 @@ func getCredentialNameAndType(credentials map[string]string) (string, string, er secretStore = string(CredentialStoreK8sSecret) } - if secretStore != string(CredentialStoreK8sSecret) { + allowedCredentialTypes := []string{ + string(CredentialStoreK8sSecret), + string(CredentialStoreAWSARN), + } + + if !utils.SliceContainsString(allowedCredentialTypes, secretStore) { return "", "", fmt.Errorf("credentials field does not support type '%s'", secretStore) }