Feature/dpr/etl

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect

go.sum

@@ -55,13 +55,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
+github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
@@ -124,6 +127,7 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/api/api.go

@@ -1,26 +1,35 @@
 package api
 
-//go:generate mockgen -source api.go -package test -destination ../../test/api.go
-
 import (
 	"github.com/go-resty/resty/v2"
 )
 
+// The VSphereProxyApi interface describes the API available to the endpoints
 type VSphereProxyApi interface {
 
 	// GetSession returns the vmware session id to be used by other requests
 	GetSession(username string, password string) (string, error)
 
 	// GetVMs returns all VMs from the VM endpoint
-	GetVMs(username string, password string) ([]VMResponse, error)
+	GetVMs(username string, password string) ([]VM, error)
 
 	// GetVMTags retrieves a list of tags associated with the given vm
 	GetVMTags(username string, password string, VMID string) ([]VMTag, error)
 
 	// GetFQDN uses the VMware guest tools to get the fqdn of a VM (if possible)
 	GetFQDN(username string, password string, VMID string) (string, error)
+
+	// GetHosts retrieves a list of ESXi hosts from the vCenter
+	GetHosts(username string, password string) ([]Host, error)
+
+	// GetDatastores retrieves a list of datastores from the vCenter
+	GetDatastores(username string, password string) ([]Datastore, error)
+
+	// GetVMInfo creates a statistical overview of the given vm
+	GetVMInfo(username string, password string, VMID string) (VMInfo, error)
 }
 
+// DefaultVSphereProxyApi is the default API implementation
 type DefaultVSphereProxyApi struct {
 	Resty *resty.Client
 }

internal/api/datastores.go

@@ -0,0 +1,30 @@
+package api
+
+import (
+	"fmt"
+	"github.com/sirupsen/logrus"
+)
+
+func (d DefaultVSphereProxyApi) GetDatastores(username string, password string) ([]Datastore, error) {
+	if s, err := d.GetSession(username, password); err != nil {
+		return []Datastore{}, err
+	} else {
+		logrus.Debugf("Fetching all datastores from %s for %s", d.Resty.BaseURL, username)
+		var datastores []Datastore
+		if r, err := d.Resty.
+			R().
+			SetHeader("vmware-api-session-id", s).
+			SetResult(&datastores).
+			Get("/api/vcenter/datastore"); err != nil {
+			logrus.Errorf("Error fetching datastores: %s", err)
+			return []Datastore{}, err
+		} else {
+			if r.IsError() {
+				err := fmt.Errorf("error getting datastores (%s): %s", r.Status(), r.Body())
+				logrus.Error(err)
+				return []Datastore{}, err
+			}
+			return datastores, nil
+		}
+	}
+}

internal/api/hosts.go

@@ -0,0 +1,30 @@
+package api
+
+import (
+	"fmt"
+	"github.com/sirupsen/logrus"
+)
+
+func (d DefaultVSphereProxyApi) GetHosts(username string, password string) ([]Host, error) {
+	if s, err := d.GetSession(username, password); err != nil {
+		return []Host{}, err
+	} else {
+		logrus.Debugf("Fetching all hosts from %s for %s", d.Resty.BaseURL, username)
+		var hostsResponse []Host
+		if r, err := d.Resty.
+			R().
+			SetHeader("vmware-api-session-id", s).
+			SetResult(&hostsResponse).
+			Get("/api/vcenter/host"); err != nil {
+			logrus.Errorf("Error fetching hosts: %s", err)
+			return []Host{}, err
+		} else {
+			if r.IsError() {
+				err := fmt.Errorf("error getting hosts (%s): %s", r.Status(), r.Body())
+				logrus.Error(err)
+				return []Host{}, err
+			}
+			return hostsResponse, nil
+		}
+	}
+}

internal/api/models.go

@@ -0,0 +1,41 @@
+package api
+
+// VM represents a virtual machine in the vCenter as described in https://developer.vmware.com/apis/vsphere-automation/v8.0U1/vcenter/data-structures/VM/Summary/
+type VM struct {
+	VM         string `json:"vm"`
+	Name       string `json:"name"`
+	PowerState string `json:"power_state"`
+}
+
+// VMTag holds a tag from vSphere
+type VMTag struct {
+	// Value holds the value of the tag
+	Value string `json:"value"`
+	// Category holds the tag category
+	Category string `json:"category"`
+}
+
+// Host represents a host in the vCenter as described in https://developer.vmware.com/apis/vsphere-automation/v8.0U1/vcenter/data-structures/Host/Summary/
+type Host struct {
+	Host            string `json:"host"`
+	Name            string `json:"name"`
+	PowerState      string `json:"power_state"`
+	ConnectionState string `json:"connection_state"`
+}
+
+// Datastore represents a host in the vCenter as described in https://developer.vmware.com/apis/vsphere-automation/v8.0U1/vcenter/data-structures/Datastore/Summary/
+type Datastore struct {
+	Datastore string `json:"datastore"`
+	Name      string `json:"name"`
+	Type      string `json:"type"`
+	Capacity  int    `json:"capacity"`
+	FreeSpace int    `json:"free_space"`
+}
+
+type VMInfo struct {
+	Name               string `json:"name"`
+	CPUCores           int    `json:"cpu_cores"`
+	ProvisionedRAM     int    `json:"provisioned_ram"`
+	ProvisionedStorage int    `json:"provisioned_storage"`
+	UsedStorage        int    `json:"used_storage"`
+}

internal/api/vms.go

@@ -4,45 +4,34 @@ import (
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
+	"github.com/thoas/go-funk"
 )
 
-// VMResponse is the value in response from the VM endpoint
-type VMResponse struct {
-	VM   string `json:"vm"`
-	Name string `json:"name"`
-}
-
 // GetVMs returns all VMs from the VM endpoint
-func (d DefaultVSphereProxyApi) GetVMs(username string, password string) ([]VMResponse, error) {
+func (d DefaultVSphereProxyApi) GetVMs(username string, password string) ([]VM, error) {
 	if s, err := d.GetSession(username, password); err != nil {
-		return []VMResponse{}, err
+		return []VM{}, err
 	} else {
 		logrus.Debugf("Fetching all VMs from %s for %s", d.Resty.BaseURL, username)
-		var vmsResponse []VMResponse
+		var vmsResponse []VM
 		if r, err := d.Resty.
 			R().
 			SetHeader("vmware-api-session-id", s).
 			SetResult(&vmsResponse).
 			Get("/api/vcenter/vm"); err != nil {
 			logrus.Errorf("Error fetching VMs: %s", err)
-			return []VMResponse{}, err
+			return []VM{}, err
 		} else {
 			if r.IsError() {
 				err := fmt.Errorf("error getting vms (%s): %s", r.Status(), r.Body())
 				logrus.Error(err)
-				return []VMResponse{}, err
+				return []VM{}, err
 			}
 			return vmsResponse, nil
 		}
 	}
 }
 
-// VMTag holds a tag from vSphere
-type VMTag struct {
-	Value    string `json:"value"`
-	Category string `json:"category"`
-}
-
 // GetVMTags retrieves a list of tags associated with the given vm
 func (d DefaultVSphereProxyApi) GetVMTags(username string, password string, VMID string) ([]VMTag, error) {
 	var tags []VMTag
@@ -157,3 +146,61 @@ func (d DefaultVSphereProxyApi) GetFQDN(username string, password string, VMID s
 		}
 	}
 }
+
+func (d DefaultVSphereProxyApi) GetVMInfo(username string, password string, VMID string) (VMInfo, error) {
+	v := VMInfo{}
+	if s, err := d.GetSession(username, password); err != nil {
+		return v, err
+	} else {
+		logrus.Debugf("Getting basic information for vm %s from %s for %s", VMID, d.Resty.BaseURL, username)
+		var vR struct {
+			Name string `json:"name"`
+			CPU  struct {
+				CoresPerSocket int `json:"cores_per_socket"`
+				Count          int `json:"count"`
+			} `json:"cpu"`
+			Memory struct {
+				SizeMiB int `json:"size_MiB"`
+			} `json:"memory"`
+		}
+		if r, err := d.Resty.
+			R().
+			SetHeader("vmware-api-session-id", s).
+			SetResult(&vR).
+			SetPathParam("vm", VMID).
+			Get("/api/vcenter/vm/{vm}"); err != nil {
+			logrus.Error(err)
+			return v, err
+		} else {
+			if r.IsError() {
+				return v, fmt.Errorf("can not get vm information (%s): %s", r.Status(), r.Body())
+			}
+			v.Name = vR.Name
+			v.CPUCores = vR.CPU.CoresPerSocket * vR.CPU.Count
+			v.ProvisionedRAM = vR.Memory.SizeMiB
+
+			logrus.Debugf("Getting local filesystem information for vm %s from %s for %s", VMID, d.Resty.BaseURL, username)
+			type fs struct {
+				FreeSpace int `json:"free_space"`
+				Capacity  int `json:"capacity"`
+			}
+			var fR map[string]fs
+			if r, err := d.Resty.
+				R().
+				SetHeader("vmware-api-session-id", s).
+				SetResult(&fR).
+				SetPathParam("vm", VMID).
+				Get("/api/vcenter/vm/{vm}/guest/local-filesystem"); err != nil {
+				logrus.Error(err)
+				return v, err
+			} else {
+				if r.IsError() {
+					return v, fmt.Errorf("can not get info about local filesystems (%s): %s", r.Status(), r.Body())
+				}
+				v.ProvisionedStorage = funk.Reduce(funk.Values(fR), func(acc int, f fs) int { return acc + f.Capacity }, 0).(int)
+				v.UsedStorage = funk.Reduce(funk.Values(fR), func(acc int, f fs) int { return acc + f.Capacity - f.FreeSpace }, 0).(int)
+				return v, nil
+			}
+		}
+	}
+}

internal/endpoints/datastores.go

@@ -0,0 +1,32 @@
+package endpoints
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"vmware-rest-proxy/internal/api"
+)
+
+type DataStoreEndpoint struct {
+	API api.VSphereProxyApi
+}
+
+func (d DataStoreEndpoint) Register(engine *gin.Engine) {
+	engine.GET("/datastores", d.getDatastores)
+}
+
+func (d DataStoreEndpoint) getDatastores(context *gin.Context) {
+	if r, ok := HandleRequest(context); ok {
+		if datastores, err := d.API.GetDatastores(r.Username, r.Password); err != nil {
+			context.AbortWithStatusJSON(500, gin.H{
+				"error": fmt.Sprintf("Error getting datastores: %s", err),
+			})
+		} else {
+			context.JSON(200, gin.H{
+				"datastores": gin.H{
+					"count":      len(datastores),
+					"datastores": datastores,
+				},
+			})
+		}
+	}
+}

internal/endpoints/hosts.go

@@ -0,0 +1,32 @@
+package endpoints
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"vmware-rest-proxy/internal/api"
+)
+
+type HostsEndpoint struct {
+	API api.VSphereProxyApi
+}
+
+func (H HostsEndpoint) Register(engine *gin.Engine) {
+	engine.GET("/hosts", H.getHosts)
+}
+
+func (H HostsEndpoint) getHosts(context *gin.Context) {
+	if r, ok := HandleRequest(context); ok {
+		if hosts, err := H.API.GetHosts(r.Username, r.Password); err != nil {
+			context.AbortWithStatusJSON(500, gin.H{
+				"error": fmt.Sprintf("Error getting hosts: %s", err),
+			})
+		} else {
+			context.JSON(200, gin.H{
+				"hosts": gin.H{
+					"count": len(hosts),
+					"hosts": hosts,
+				},
+			})
+		}
+	}
+}

internal/endpoints/vms.go

@@ -22,6 +22,7 @@ func (V *VMSEndpoint) Register(engine *gin.Engine) {
 	engine.GET("/vms", V.getVMS)
 	engine.GET("/vms/:vm/tags", V.getVMTags)
 	engine.GET("/vms/:vm/fqdn", V.getFQDN)
+	engine.GET("/vms/:vm/info", V.getVMInfo)
 }
 
 // getVMS exposes all VMS of the vCenter at /VMS
@@ -87,3 +88,22 @@ func (V *VMSEndpoint) getFQDN(context *gin.Context) {
 		}
 	}
 }
+
+func (V *VMSEndpoint) getVMInfo(context *gin.Context) {
+	if r, ok := HandleRequest(context); ok {
+		var vm VMBinding
+		if err := context.ShouldBindUri(&vm); err != nil {
+			context.AbortWithStatusJSON(400, gin.H{
+				"error": fmt.Sprintf("Missing VM id in path: %s", err),
+			})
+			return
+		}
+		if vmInfo, err := V.API.GetVMInfo(r.Username, r.Password, vm.ID); err != nil {
+			context.AbortWithStatusJSON(500, gin.H{
+				"error": fmt.Sprintf("Error getting vm info: %s", err),
+			})
+		} else {
+			context.JSON(200, vmInfo)
+		}
+	}
+}