Added infrastructure for virtual pools

core/orchestrator_core_test.go

@@ -18,8 +18,8 @@ import (
 	"github.com/netapp/trident/storage/fake"
 	sa "github.com/netapp/trident/storage_attribute"
 	"github.com/netapp/trident/storage_class"
-	tu "github.com/netapp/trident/storage_class/test_utils"
 	fakedriver "github.com/netapp/trident/storage_drivers/fake"
+	tu "github.com/netapp/trident/storage_drivers/fake/test_utils"
 )
 
 var (
@@ -108,8 +108,8 @@ func cleanupStoreVersion(t *testing.T, etcd string) {
 func diffConfig(expected, got interface{}, fieldToSkip string) []string {
 
 	diffs := make([]string, 0, 0)
-	expectedStruct := reflect.ValueOf(expected).Elem()
-	gotStruct := reflect.ValueOf(got).Elem()
+	expectedStruct := reflect.Indirect(reflect.ValueOf(expected))
+	gotStruct := reflect.Indirect(reflect.ValueOf(got))
 
 	for i := 0; i < expectedStruct.NumField(); i++ {
 
@@ -149,11 +149,17 @@ func diffExternalBackends(t *testing.T, expected, got *storage.BackendExternal)
 	expectedConfig := expected.Config
 	gotConfig := got.Config
 
-	expectedValElem := reflect.ValueOf(expected.Config).Elem()
-	gotValElem := reflect.ValueOf(got.Config).Elem()
+	expectedConfigTypeName := reflect.TypeOf(expectedConfig).Name()
+	gotConfigTypeName := reflect.TypeOf(gotConfig).Name()
+	if expectedConfigTypeName != gotConfigTypeName {
+		t.Errorf("Config type mismatch: %v != %v", expectedConfigTypeName, gotConfigTypeName)
+	}
+
+	expectedConfigValue := reflect.ValueOf(expectedConfig)
+	gotConfigValue := reflect.ValueOf(gotConfig)
 
-	expectedCSDCIntf := expectedValElem.FieldByName("CommonStorageDriverConfigExternal").Interface()
-	gotCSDCIntf := gotValElem.FieldByName("CommonStorageDriverConfigExternal").Interface()
+	expectedCSDCIntf := expectedConfigValue.FieldByName("CommonStorageDriverConfig").Interface()
+	gotCSDCIntf := gotConfigValue.FieldByName("CommonStorageDriverConfig").Interface()
 
 	var configDiffs []string
 
@@ -1344,8 +1350,7 @@ func TestBackendUpdateAndDelete(t *testing.T) {
 		persistentBackend, err := orchestrator.storeClient.GetBackend(backendName)
 		if err != nil {
 			t.Error("Unable to retrieve backend from store client:  ", err)
-		} else if !reflect.DeepEqual(newBackend.ConstructPersistent(),
-			persistentBackend) {
+		} else if !reflect.DeepEqual(newBackend.ConstructPersistent(), persistentBackend) {
 			t.Errorf("Backend not correctly updated in persistent store.")
 		}
 		previousBackends = append(previousBackends, newBackend)
@@ -1416,8 +1421,7 @@ func TestBackendUpdateAndDelete(t *testing.T) {
 		t.Error("Unable to find backend after bootstrapping.")
 	} else if !reflect.DeepEqual(bootstrappedBackend,
 		backend.ConstructExternal()) {
-		diffExternalBackends(t, backend.ConstructExternal(),
-			bootstrappedBackend)
+		diffExternalBackends(t, backend.ConstructExternal(), bootstrappedBackend)
 	}
 	orchestrator.mutex.Unlock()
 

persistent_store/passthrough_test.go

@@ -523,13 +523,30 @@ func TestPassthroughClient_DeleteVolumeIgnoreNotFoundNonexistent(t *testing.T) {
 func TestPassthroughClient_GetVolumes(t *testing.T) {
 	p := newPassthroughClient()
 	fakeBackend := getFakeBackend()
-	createOpts := map[string]string{"pool": "pool-0"}
-	fakeBackend.Driver.Create("fake_volume_1", 1000000000, createOpts)
-	fakeBackend.Driver.Create("fake_volume_2", 2000000000, createOpts)
+
+	volConfig := &storage.VolumeConfig{
+		Name:         "fake_volume_1",
+		InternalName: "trident_fake_volume_1",
+		Size:         "1000000000",
+	}
+	err := fakeBackend.Driver.Create(volConfig, fakeBackend.Storage["pool-0"], make(map[string]sa.Request))
+	if err != nil {
+		t.Error(err)
+	}
+
+	volConfig = &storage.VolumeConfig{
+		Name:         "fake_volume_2",
+		InternalName: "trident_fake_volume_2",
+		Size:         "2000000000",
+	}
+	err = fakeBackend.Driver.Create(volConfig, fakeBackend.Storage["pool-0"], make(map[string]sa.Request))
+	if err != nil {
+		t.Error(err)
+	}
+
 	p.AddBackend(fakeBackend)
 
 	result, err := p.GetVolumes()
-
 	if err != nil {
 		t.Error("Could not get volumes from passthrough client!")
 	}

storage/backend.go

@@ -14,7 +14,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	tridentconfig "github.com/netapp/trident/config"
-	"github.com/netapp/trident/storage_attribute"
+	sa "github.com/netapp/trident/storage_attribute"
 	drivers "github.com/netapp/trident/storage_drivers"
 	"github.com/netapp/trident/utils"
 )
@@ -26,8 +26,8 @@ type Driver interface {
 	Initialized() bool
 	// Terminate tells the driver to clean up, as it won't be called again.
 	Terminate()
-	Create(name string, sizeBytes uint64, opts map[string]string) error
-	CreateClone(name, source, snapshot string, opts map[string]string) error
+	Create(volConfig *VolumeConfig, storagePool *Pool, volAttributes map[string]sa.Request) error
+	CreateClone(volConfig *VolumeConfig) error
 	Destroy(name string) error
 	Publish(name string, publishInfo *utils.VolumePublishInfo) error
 	SnapshotList(name string) ([]Snapshot, error)
@@ -42,11 +42,6 @@ type Driver interface {
 	// value of CommonStorageDriver.SnapshotPrefix to the name.
 	GetInternalVolumeName(name string) string
 	GetStorageBackendSpecs(backend *Backend) error
-	GetVolumeOpts(
-		volConfig *VolumeConfig,
-		pool *Pool,
-		requests map[string]storageattribute.Request,
-	) (map[string]string, error)
 	GetProtocol() tridentconfig.Protocol
 	StoreConfig(b *PersistentStorageBackendConfig)
 	// GetExternalConfig returns a version of the driver configuration that
@@ -99,26 +94,16 @@ func (b *Backend) GetProtocol() tridentconfig.Protocol {
 }
 
 func (b *Backend) AddVolume(
-	volConfig *VolumeConfig,
-	storagePool *Pool,
-	volumeAttributes map[string]storageattribute.Request,
+	volConfig *VolumeConfig, storagePool *Pool, volAttributes map[string]sa.Request,
 ) (*Volume, error) {
 
-	// Determine volume size in bytes
-	requestedSize, err := utils.ConvertSizeToBytes(volConfig.Size)
-	if err != nil {
-		return nil, fmt.Errorf("could not convert volume size %s: %v", volConfig.Size, err)
-	}
-	volSize, err := strconv.ParseUint(requestedSize, 10, 64)
-	if err != nil {
-		return nil, fmt.Errorf("%v is an invalid volume size: %v", volConfig.Size, err)
-	}
+	var err error
 
 	log.WithFields(log.Fields{
 		"backend":       b.Name,
 		"volume":        volConfig.InternalName,
 		"storage_pool":  storagePool.Name,
-		"size":          volSize,
+		"size":          volConfig.Size,
 		"storage_class": volConfig.StorageClass,
 	}).Debug("Attempting volume create.")
 
@@ -127,25 +112,16 @@ func (b *Backend) AddVolume(
 	// 2. Ensure no volume with the same name exists on that backend
 	if b.Driver.CreatePrepare(volConfig) {
 
-		// add volume to the backend
-		args, err := b.Driver.GetVolumeOpts(volConfig, storagePool,
-			volumeAttributes)
-		if err != nil {
-			// An error on GetVolumeOpts is almost certainly going to indicate
-			// a formatting mistake, so go ahead and return an error, rather
-			// than just log a warning.
-			return nil, err
-		}
-
-		if err := b.Driver.Create(volConfig.InternalName, volSize, args); err != nil {
+		// Add volume to the backend
+		if err = b.Driver.Create(volConfig, storagePool, volAttributes); err != nil {
 			// Implement idempotency at the Trident layer
 			// Ignore the error if the volume exists already
 			if b.Driver.Get(volConfig.InternalName) != nil {
 				return nil, err
 			}
 		}
 
-		if err = b.Driver.CreateFollowup(volConfig); err != nil {
+		if err := b.Driver.CreateFollowup(volConfig); err != nil {
 			errDestroy := b.Driver.Destroy(volConfig.InternalName)
 			if errDestroy != nil {
 				log.WithFields(log.Fields{
@@ -165,7 +141,7 @@ func (b *Backend) AddVolume(
 		log.WithFields(log.Fields{
 			"backend":       b.Name,
 			"storage_pool":  storagePool.Name,
-			"size":          volSize,
+			"size":          volConfig.Size,
 			"storage_class": volConfig.StorageClass,
 		}).Debug("Storage pool does not match volume request.")
 	}
@@ -189,18 +165,7 @@ func (b *Backend) CloneVolume(volConfig *VolumeConfig) (*Volume, error) {
 		return nil, errors.New("failed to prepare clone create")
 	}
 
-	nilAttributes := make(map[string]storageattribute.Request)
-	args, err := b.Driver.GetVolumeOpts(volConfig, nil, nilAttributes)
-	if err != nil {
-		// An error on GetVolumeOpts is almost certainly going to indicate
-		// a formatting mistake, so go ahead and return an error, rather
-		// than just log a warning.
-		return nil, err
-	}
-
-	err = b.Driver.CreateClone(volConfig.InternalName,
-		volConfig.CloneSourceVolumeInternal, volConfig.CloneSourceSnapshot,
-		args)
+	err := b.Driver.CreateClone(volConfig)
 	if err != nil {
 		return nil, err
 	}

storage/fake/volume.go

@@ -1,7 +1,8 @@
 package fake
 
 type Volume struct {
-	Name      string
-	PoolName  string
-	SizeBytes uint64
+	Name          string
+	RequestedPool string
+	PhysicalPool  string
+	SizeBytes     uint64
 }

storage/storage_pool.go

@@ -10,19 +10,20 @@ import (
 
 type Pool struct {
 	Name string
-	// A Trident storage pool can potentially satisfy more than one storage
-	// class.
-	StorageClasses []string
-	Backend        *Backend
-	Attributes     map[string]sa.Offer
+	// A Trident storage pool can potentially satisfy more than one storage class.
+	StorageClasses     []string
+	Backend            *Backend
+	Attributes         map[string]sa.Offer // These attributes are used to match storage classes
+	InternalAttributes map[string]string   // These attributes are defined & used internally by storage drivers
 }
 
 func NewStoragePool(backend *Backend, name string) *Pool {
 	return &Pool{
-		Name:           name,
-		StorageClasses: make([]string, 0),
-		Backend:        backend,
-		Attributes:     make(map[string]sa.Offer),
+		Name:               name,
+		StorageClasses:     make([]string, 0),
+		Backend:            backend,
+		Attributes:         make(map[string]sa.Offer),
+		InternalAttributes: make(map[string]string),
 	}
 }
 

storage_attribute/bool.go

@@ -12,6 +12,30 @@ func NewBoolOffer(offer bool) Offer {
 	}
 }
 
+func NewBoolOfferFromOffers(offers ...Offer) Offer {
+
+	anyTrueOffer := false
+
+	for _, offer := range offers {
+		if bOffer, ok := offer.(*boolOffer); ok {
+			if bOffer.Offer {
+				anyTrueOffer = true
+			}
+		}
+	}
+
+	// A boolOffer must hold either a true or false value.  If any of the
+	// supplied offers are true, the combined result must be true.  Otherwise,
+	// the supplied offers were all false, in which case the combined result
+	// must be false.
+
+	if anyTrueOffer {
+		return &boolOffer{Offer: true}
+	} else {
+		return &boolOffer{Offer: false}
+	}
+}
+
 // Matches is a boolean offer of true matches any request; a boolean offer of false
 // only matches a false request.  This assumes that the requested parameter
 // will be passed into the driver.

storage_attribute/common_attributes.go

@@ -15,6 +15,12 @@ const (
 	ProvisioningType = "provisioningType"
 	BackendType      = "backendType"
 	Media            = "media"
+	Region           = "region"
+	Zone             = "zone"
+
+	// Constants for label attributes
+	Labels   = "labels"
+	Selector = "selector"
 
 	// Testing constants
 	RecoveryTest     = "recoveryTest"
@@ -41,6 +47,10 @@ var attrTypes = map[string]Type{
 	ProvisioningType: stringType,
 	BackendType:      stringType,
 	Media:            stringType,
+	Region:           stringType,
+	Zone:             stringType,
+	Labels:           labelType,
+	Selector:         labelType,
 	RecoveryTest:     boolType,
 	UniqueOptions:    stringType,
 	TestingAttribute: boolType,