Skip to content

Commit

Permalink
Merge pull request #20 from kdeng3849/ENG-7511-custom-endpoints
Browse files Browse the repository at this point in the history 
ENG-7511 custom endpoints
  • Loading branch information
@kqdeng
kqdeng authored Jul 28, 2020
2 parents 834ce73 + 25ee506 commit 4569351
Show file tree
Hide file tree
Showing 8 changed files with 407 additions and 188 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/golang/protobuf v1.4.2
github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/itchyny/gojq v0.11.0
github.com/packethost/cacher v0.0.0-20200319200613-5dc1cac4fd33
github.com/packethost/pkg v0.0.0-20190715213007-7c3a64b4b5e3
github.com/pkg/errors v0.9.1
Expand Down
103 changes: 43 additions & 60 deletions go.sum
View file

Large diffs are not rendered by default.

96 changes: 45 additions & 51 deletions grpc_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net"
Expand All @@ -13,6 +12,9 @@ import (
"strings"
"unicode"

"github.com/itchyny/gojq"
"github.com/tinkerbell/tink/util"

"github.com/packethost/cacher/protos/cacher"
"github.com/packethost/hegel/grpc/hegel"
"github.com/packethost/hegel/metrics"
Expand All @@ -26,8 +28,6 @@ import (

type watchClient interface{}

type exportedHardware interface{}

// exportedHardwareCacher is the structure in which hegel returns to clients using the old cacher data model
// exposes only certain fields of the hardware data returned by cacher
type exportedHardwareCacher struct {
Expand All @@ -44,13 +44,6 @@ type exportedHardwareCacher struct {
BondingMode int `json:"bonding_mode"`
}

// exportedHardwareTinkerbell is the structure in which hegel returns to clients using the new tinkerbell data model
// exposes only certain fields of the hardware data returned by tinkerbell
type exportedHardwareTinkerbell struct {
ID string `json:"id"`
Metadata interface{} `json:"metadata"`
}

type instance struct {
ID string `json:"id"`
State string `json:"state"`
Expand Down Expand Up @@ -172,17 +165,9 @@ func convertSuffix(s string) (int, error) {
return -1, errors.New("invalid suffix")
}

// exportedHardware transforms hardware that is returned from cacher/tink into what we want to expose to clients
// exportedHardware transforms hardware that is returned from cacher into what we want to expose to clients
func exportHardware(hw []byte) ([]byte, error) {
var exported exportedHardware

dataModelVersion := os.Getenv("DATA_MODEL_VERSION")
switch dataModelVersion {
case "1":
exported = &exportedHardwareTinkerbell{}
default:
exported = &exportedHardwareCacher{}
}
exported := &exportedHardwareCacher{}

err := json.Unmarshal(hw, exported)
if err != nil {
Expand All @@ -191,6 +176,38 @@ func exportHardware(hw []byte) ([]byte, error) {
return json.Marshal(exported)
}

func filterMetadata(hw []byte, filter string) ([]byte, error) {
var result interface{}
query, err := gojq.Parse(filter)
if err != nil {
return nil, err
}
input := make(map[string]interface{})
err = json.Unmarshal(hw, &input)
if err != nil {
return nil, err
}
iter := query.Run(input)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
return nil, err
}
result = v
}

if resultString, ok := result.(string); ok { // if already a string, don't marshal
return []byte(resultString), nil
}
if result != nil { // if nil, don't marshal (json.Marshal(nil) returns "null")
return json.Marshal(result)
}
return nil, nil
}

// UnmarshalJSON implements the json.Unmarshaler interface for custom unmarshalling of exportedHardwareCacher
func (eh *exportedHardwareCacher) UnmarshalJSON(b []byte) error {
type ehj exportedHardwareCacher
Expand All @@ -210,29 +227,6 @@ func (eh *exportedHardwareCacher) UnmarshalJSON(b []byte) error {
return nil
}

// UnmarshalJSON implements the json.Unmarshaler interface for custom unmarshalling of exportedHardwareTinkerbell
// transforms the metadata from a string (as defined in the hardware returned by tink) into a map for cleaner printing
func (eh *exportedHardwareTinkerbell) UnmarshalJSON(b []byte) error {
type ehj exportedHardwareTinkerbell
var tmp ehj
err := json.Unmarshal(b, &tmp)
if err != nil {
return err
}

if md, ok := tmp.Metadata.(string); ok { // won't run block if unable to cast into string (including if nil)
metadata := make(map[string]interface{})
err = json.Unmarshal([]byte(md), &metadata) // metadata is now a map

if err != nil {
fmt.Println(err)
}
tmp.Metadata = metadata
}
*eh = exportedHardwareTinkerbell(tmp)
return nil
}

func (s *server) Get(ctx context.Context, in *hegel.GetRequest) (*hegel.GetResponse, error) {
p, ok := peer.FromContext(ctx)
if !ok {
Expand All @@ -241,7 +235,11 @@ func (s *server) Get(ctx context.Context, in *hegel.GetRequest) (*hegel.GetRespo
s.log.With("client", p.Addr, "op", "get").Info()
userIP := p.Addr.(*net.TCPAddr).IP.String()

ehw, err := getByIP(ctx, s, userIP)
hw, err := getByIP(ctx, s, userIP)
if err != nil {
return nil, err
}
ehw, err := exportHardware(hw)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -354,7 +352,7 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr
close(ehws)
return
}
hw, err = json.Marshal(resp)
hw, err = json.Marshal(util.HardwareWrapper{Hardware: resp})
if err != nil {
errs <- errors.New("could not marshal hardware")
close(ehws)
Expand Down Expand Up @@ -427,7 +425,7 @@ func getByIP(ctx context.Context, s *server, userIP string) ([]byte, error) {
return nil, errors.New("could not find hardware")
}

hw, err = json.Marshal(resp)
hw, err = json.Marshal(util.HardwareWrapper{Hardware: resp.(*tink.Hardware)})
if err != nil {
return nil, errors.New("could not marshal hardware")
}
Expand All @@ -448,9 +446,5 @@ func getByIP(ctx context.Context, s *server, userIP string) ([]byte, error) {
hw = []byte(resp.(*cacher.Hardware).JSON)
}

ehw, err := exportHardware(hw)
if err != nil {
return nil, err
}
return ehw, nil
return hw, nil
}
75 changes: 35 additions & 40 deletions grpc_server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"os"
"reflect"
"testing"

"github.com/tinkerbell/tink/protos/packet"
Expand All @@ -19,17 +20,17 @@ func TestGetByIPCacher(t *testing.T) {
log: logger,
hardwareClient: hardwareGetterMock{test.json},
}
ehw, err := getByIP(context.Background(), hegelTestServer, test.remote)
ehw, err := getByIP(context.Background(), hegelTestServer, test.remote) // returns hardware data as []byte
if err != nil {
if err.Error() != test.error {
t.Fatalf("unexpected error in getByIP, want: %v, got: %v\n", test.error, err.Error())
}
continue
t.Fatal("unexpected error while getting hardware by ip:", err)
}
hw := exportedHardwareCacher{}
err = json.Unmarshal(ehw, &hw)
if err != nil {
t.Fatal("Error in unmarshalling hardware", err)
if err.Error() != test.error {
t.Fatalf("unexpected error while unmarshalling, want: %v, got: %v\n", test.error, err.Error())
}
continue
}

if hw.State != test.state {
Expand Down Expand Up @@ -81,68 +82,62 @@ func TestGetByIPTinkerbell(t *testing.T) {
log: logger,
hardwareClient: hardwareGetterMock{test.json},
}
ehw, err := getByIP(context.Background(), hegelTestServer, test.remote)
ehw, err := getByIP(context.Background(), hegelTestServer, test.remote) // returns hardware data as []byte
if err != nil {
if err.Error() != test.error {
t.Fatalf("unexpected error in getByIP, want: %v, got: %v\n", test.error, err.Error())
}
continue
}
hw := exportedHardwareTinkerbell{}

hw := struct {
ID string `json:"id"`
Metadata packet.Metadata `json:"metadata"`
}{}
err = json.Unmarshal(ehw, &hw)
if err != nil {
t.Error("Error in unmarshalling hardware:", err)
t.Error("Error in unmarshalling hardware metadata", err)
}

if hw.ID != test.id {
t.Errorf("handler returned unexpected id: got %v want %v",
hw.ID, test.id)
}

if hw.Metadata == nil {
if reflect.DeepEqual(hw.Metadata, packet.Metadata{}) { // return if metadata is empty
return
}

var metadata packet.Metadata
md, err := json.Marshal(hw.Metadata)
if err != nil {
t.Error("Error in unmarshalling hardware", err)
}
err = json.Unmarshal(md, &metadata)
if err != nil {
t.Error("Error in unmarshalling hardware metadata", err)
}

if metadata.State != test.state {
t.Fatalf("unexpected state, want: %v, got: %v\n", test.state, metadata.State)
if hw.Metadata.State != test.state {
t.Fatalf("unexpected state, want: %v, got: %v\n", test.state, hw.Metadata.State)
}
if metadata.BondingMode != test.bondingMode {
t.Fatalf("unexpected bonding mode, want: %v, got: %v\n", test.bondingMode, metadata.BondingMode)
if hw.Metadata.BondingMode != test.bondingMode {
t.Fatalf("unexpected bonding mode, want: %v, got: %v\n", test.bondingMode, hw.Metadata.BondingMode)
}
if len(metadata.Instance.Storage.Disks) > 0 {
if metadata.Instance.Storage.Disks[0].Device != test.diskDevice {
t.Fatalf("unexpected disk device, want: %v, got: %v\n", test.diskDevice, metadata.Instance.Storage.Disks[0].Device)
if len(hw.Metadata.Instance.Storage.Disks) > 0 {
if hw.Metadata.Instance.Storage.Disks[0].Device != test.diskDevice {
t.Fatalf("unexpected disk device, want: %v, got: %v\n", test.diskDevice, hw.Metadata.Instance.Storage.Disks[0].Device)
}
if metadata.Instance.Storage.Disks[0].WipeTable != test.wipeTable {
t.Fatalf("unexpected wipe table, want: %v, got: %v\n", test.wipeTable, metadata.Instance.Storage.Disks[0].WipeTable)
if hw.Metadata.Instance.Storage.Disks[0].WipeTable != test.wipeTable {
t.Fatalf("unexpected wipe table, want: %v, got: %v\n", test.wipeTable, hw.Metadata.Instance.Storage.Disks[0].WipeTable)
}
if metadata.Instance.Storage.Disks[0].Partitions[0].Size != test.partitionSize {
t.Fatalf("unexpected partition size, want: %v, got: %v\n", test.partitionSize, metadata.Instance.Storage.Disks[0].Partitions[0].Size)
if hw.Metadata.Instance.Storage.Disks[0].Partitions[0].Size != test.partitionSize {
t.Fatalf("unexpected partition size, want: %v, got: %v\n", test.partitionSize, hw.Metadata.Instance.Storage.Disks[0].Partitions[0].Size)
}
}
if len(metadata.Instance.Storage.Filesystems) > 0 {
if metadata.Instance.Storage.Filesystems[0].Mount.Device != test.filesystemDevice {
t.Fatalf("unexpected filesystem mount device, want: %v, got: %v\n", test.filesystemDevice, metadata.Instance.Storage.Filesystems[0].Mount.Device)
if len(hw.Metadata.Instance.Storage.Filesystems) > 0 {
if hw.Metadata.Instance.Storage.Filesystems[0].Mount.Device != test.filesystemDevice {
t.Fatalf("unexpected filesystem mount device, want: %v, got: %v\n", test.filesystemDevice, hw.Metadata.Instance.Storage.Filesystems[0].Mount.Device)
}
if metadata.Instance.Storage.Filesystems[0].Mount.Format != test.filesystemFormat {
t.Fatalf("unexpected filesystem mount format, want: %v, got: %v\n", test.filesystemFormat, metadata.Instance.Storage.Filesystems[0].Mount.Format)
if hw.Metadata.Instance.Storage.Filesystems[0].Mount.Format != test.filesystemFormat {
t.Fatalf("unexpected filesystem mount format, want: %v, got: %v\n", test.filesystemFormat, hw.Metadata.Instance.Storage.Filesystems[0].Mount.Format)
}
}
if metadata.Instance.OperatingSystemVersion.OsSlug != test.osSlug {
t.Fatalf("unexpected os slug, want: %v, got: %v\n", test.osSlug, metadata.Instance.OperatingSystemVersion.OsSlug)
if hw.Metadata.Instance.OperatingSystemVersion.OsSlug != test.osSlug {
t.Fatalf("unexpected os slug, want: %v, got: %v\n", test.osSlug, hw.Metadata.Instance.OperatingSystemVersion.OsSlug)
}
if metadata.Facility.PlanSlug != test.planSlug {
t.Fatalf("unexpected os slug, want: %v, got: %v\n", test.planSlug, metadata.Facility.PlanSlug)
if hw.Metadata.Facility.PlanSlug != test.planSlug {
t.Fatalf("unexpected os slug, want: %v, got: %v\n", test.planSlug, hw.Metadata.Facility.PlanSlug)
}
}
}
Expand Down
54 changes: 40 additions & 14 deletions http_handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package main
import (
"context"
"encoding/json"
"errors"
"net"
"net/http"
"os"
"runtime"
"strings"
"time"
Expand Down Expand Up @@ -52,26 +54,50 @@ func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
}
}

func getMetadata(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
func getMetadata(filter string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
return
}

logger.Debug("Calling getMetadata ")
userIP := getIPFromRequest(r)
if userIP != "" {
metrics.MetadataRequests.Inc()
logger.With("userIP", userIP).Info("Actual IP is : ")
ehw, err := getByIP(context.Background(), hegelServer, userIP)
if userIP == "" {
return
}

metrics.MetadataRequests.Inc()
logger.With("userIP", userIP).Info("Actual IP is: ")
hw, err := getByIP(context.Background(), hegelServer, userIP) // returns hardware data as []byte
if err != nil {
metrics.Errors.WithLabelValues("metadata", "lookup").Inc()
logger.Info("Error in finding hardware: ", err)
w.WriteHeader(http.StatusInternalServerError)
return
}

var resp []byte
dataModelVersion := os.Getenv("DATA_MODEL_VERSION")
switch dataModelVersion {
case "":
resp, err = exportHardware(hw) // in cacher mode, the "filter" is the exportedHardwareCacher type
if err != nil {
metrics.Errors.WithLabelValues("metadata", "lookup").Inc()
logger.Info("Error in finding or exporting hardware: ", err)
w.WriteHeader(http.StatusInternalServerError)
return
logger.Info("Error in exporting hardware: ", err)
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(ehw)
case "1":
resp, err = filterMetadata(hw, filter)
if err != nil {
logger.Error(err, "failed to write Metadata")
logger.Info("Error in filtering metadata: ", err)
}
default:
logger.Fatal(errors.New("unknown DATA_MODEL_VERSION"))

}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(resp)
if err != nil {
logger.Error(err, "failed to write Metadata")
}
}
}
Expand Down
Loading

0 comments on commit 4569351

Please sign in to comment.