From 62002cbc26b69b840d4771cda6962aeb2a55e5fc Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 09:55:05 -0400 Subject: [PATCH 01/10] Check SpiceDB connection for readyz Signed-off-by: Jonathan Marcantonio --- api/health/v1/health.pb.go | 88 +++++++++++++++++++++-------- api/health/v1/health.pb.validate.go | 8 +++ api/health/v1/health.proto | 10 +++- internal/service/health.go | 29 +++++++++- 4 files changed, 107 insertions(+), 28 deletions(-) diff --git a/api/health/v1/health.pb.go b/api/health/v1/health.pb.go index a8a4f37..690f4a1 100644 --- a/api/health/v1/health.pb.go +++ b/api/health/v1/health.pb.go @@ -63,6 +63,9 @@ type GetLivezReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + Code uint32 `protobuf:"varint,2,opt,name=code,proto3" json:"code,omitempty"` } func (x *GetLivezReply) Reset() { @@ -97,6 +100,20 @@ func (*GetLivezReply) Descriptor() ([]byte, []int) { return file_health_v1_health_proto_rawDescGZIP(), []int{1} } +func (x *GetLivezReply) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *GetLivezReply) GetCode() uint32 { + if x != nil { + return x.Code + } + return 0 +} + type GetReadyzRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -139,6 +156,9 @@ type GetReadyzReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + Code uint32 `protobuf:"varint,2,opt,name=code,proto3" json:"code,omitempty"` } func (x *GetReadyzReply) Reset() { @@ -173,6 +193,20 @@ func (*GetReadyzReply) Descriptor() ([]byte, []int) { return file_health_v1_health_proto_rawDescGZIP(), []int{3} } +func (x *GetReadyzReply) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *GetReadyzReply) GetCode() uint32 { + if x != nil { + return x.Code + } + return 0 +} + var File_health_v1_health_proto protoreflect.FileDescriptor var file_health_v1_health_proto_rawDesc = []byte{ @@ -181,31 +215,37 @@ var file_health_v1_health_proto_rawDesc = []byte{ 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x11, 0x0a, 0x0f, 0x47, - 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0f, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x10, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xde, 0x01, 0x0a, 0x0c, 0x4b, 0x65, 0x73, 0x73, 0x65, 0x6c, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x64, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, - 0x65, 0x7a, 0x12, 0x24, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, - 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, + 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, + 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x47, + 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0x3c, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x52, 0x65, 0x70, 0x6c, + 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x32, 0xde, 0x01, + 0x0a, 0x0c, 0x4b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x64, + 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x12, 0x24, 0x2e, 0x6b, 0x65, 0x73, + 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x22, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x08, 0x12, 0x06, 0x2f, 0x6c, + 0x69, 0x76, 0x65, 0x7a, 0x12, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x7a, 0x12, 0x25, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x4c, 0x69, 0x76, 0x65, 0x7a, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0e, 0x82, 0xd3, - 0xe4, 0x93, 0x02, 0x08, 0x12, 0x06, 0x2f, 0x6c, 0x69, 0x76, 0x65, 0x7a, 0x12, 0x68, 0x0a, 0x09, - 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x12, 0x25, 0x2e, 0x6b, 0x65, 0x73, 0x73, - 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x23, 0x2e, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x09, 0x12, 0x07, 0x2f, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x42, 0x5e, 0x0a, 0x23, 0x6f, 0x72, 0x67, 0x2e, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x50, 0x01, 0x5a, - 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x2d, 0x6b, 0x65, 0x73, 0x73, 0x65, 0x6c, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x2d, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x82, + 0xd3, 0xe4, 0x93, 0x02, 0x09, 0x12, 0x07, 0x2f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x42, 0x5e, + 0x0a, 0x23, 0x6f, 0x72, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6b, 0x65, + 0x73, 0x73, 0x65, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x2e, 0x76, 0x31, 0x50, 0x01, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2d, 0x6b, 0x65, 0x73, 0x73, + 0x65, 0x6c, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2d, 0x61, 0x70, 0x69, + 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/health/v1/health.pb.validate.go b/api/health/v1/health.pb.validate.go index 76547cd..67c0015 100644 --- a/api/health/v1/health.pb.validate.go +++ b/api/health/v1/health.pb.validate.go @@ -157,6 +157,10 @@ func (m *GetLivezReply) validate(all bool) error { var errors []error + // no validation rules for Status + + // no validation rules for Code + if len(errors) > 0 { return GetLivezReplyMultiError(errors) } @@ -357,6 +361,10 @@ func (m *GetReadyzReply) validate(all bool) error { var errors []error + // no validation rules for Status + + // no validation rules for Code + if len(errors) > 0 { return GetReadyzReplyMultiError(errors) } diff --git a/api/health/v1/health.proto b/api/health/v1/health.proto index 2d7e9d6..4cac9dd 100644 --- a/api/health/v1/health.proto +++ b/api/health/v1/health.proto @@ -22,7 +22,13 @@ service KesselHealth { } message GetLivezRequest {} -message GetLivezReply {} +message GetLivezReply { + string status = 1; + uint32 code = 2; +} message GetReadyzRequest {} -message GetReadyzReply {} +message GetReadyzReply { + string status = 1; + uint32 code = 2; +} diff --git a/internal/service/health.go b/internal/service/health.go index 3993e85..952243d 100644 --- a/internal/service/health.go +++ b/internal/service/health.go @@ -4,6 +4,9 @@ import ( "context" pb "github.com/project-kessel/relations-api/api/health/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" ) type HealthService struct { @@ -15,8 +18,30 @@ func NewHealthService() *HealthService { } func (s *HealthService) GetLivez(ctx context.Context, req *pb.GetLivezRequest) (*pb.GetLivezReply, error) { - return &pb.GetLivezReply{}, nil + return &pb.GetLivezReply{Status: "OK", Code: 200}, nil } func (s *HealthService) GetReadyz(ctx context.Context, req *pb.GetReadyzRequest) (*pb.GetReadyzReply, error) { - return &pb.GetReadyzReply{}, nil + check := checkSpiceDBReadyz() + if check { + return &pb.GetReadyzReply{Status: "OK", Code: 200}, nil + } + return &pb.GetReadyzReply{Status: "Unavailable", Code: 503}, nil +} + +func checkSpiceDBReadyz() bool { + conn, err := grpc.NewClient( + "spicedb:50051", + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return false + } + + client := grpc_health_v1.NewHealthClient(conn) + resp, err := client.Check(context.Background(), &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return false + } + + return resp.Status == grpc_health_v1.HealthCheckResponse_SERVING } From cb3242ff4014f86ce06a00f03b4efb2c888a030f Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 10:09:25 -0400 Subject: [PATCH 02/10] Add 5 second timeout Signed-off-by: Jonathan Marcantonio --- internal/service/health.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/service/health.go b/internal/service/health.go index 952243d..aafd985 100644 --- a/internal/service/health.go +++ b/internal/service/health.go @@ -2,6 +2,7 @@ package service import ( "context" + "time" pb "github.com/project-kessel/relations-api/api/health/v1" "google.golang.org/grpc" @@ -21,8 +22,8 @@ func (s *HealthService) GetLivez(ctx context.Context, req *pb.GetLivezRequest) ( return &pb.GetLivezReply{Status: "OK", Code: 200}, nil } func (s *HealthService) GetReadyz(ctx context.Context, req *pb.GetReadyzRequest) (*pb.GetReadyzReply, error) { - check := checkSpiceDBReadyz() - if check { + ready := checkSpiceDBReadyz() + if ready { return &pb.GetReadyzReply{Status: "OK", Code: 200}, nil } return &pb.GetReadyzReply{Status: "Unavailable", Code: 503}, nil @@ -38,7 +39,10 @@ func checkSpiceDBReadyz() bool { } client := grpc_health_v1.NewHealthClient(conn) - resp, err := client.Check(context.Background(), &grpc_health_v1.HealthCheckRequest{}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := client.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) if err != nil { return false } From 7f944e7662dc385545d49c7a5f4a7dd51d6620b0 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 14:33:20 -0400 Subject: [PATCH 03/10] Generalize readyz logic Signed-off-by: Jonathan Marcantonio --- cmd/kessel-relations/wire_gen.go | 3 ++- internal/biz/biz.go | 2 +- internal/biz/relationships.go | 14 +++++++++++ internal/conf/conf.pb.go | 1 - internal/data/spicedb.go | 29 +++++++++++++++++++++-- internal/service/health.go | 40 ++++++++------------------------ 6 files changed, 54 insertions(+), 35 deletions(-) diff --git a/cmd/kessel-relations/wire_gen.go b/cmd/kessel-relations/wire_gen.go index a6578b1..d9f78b5 100644 --- a/cmd/kessel-relations/wire_gen.go +++ b/cmd/kessel-relations/wire_gen.go @@ -32,7 +32,8 @@ func wireApp(confServer *conf.Server, confData *conf.Data, logger log.Logger) (* readRelationshipsUsecase := biz.NewReadRelationshipsUsecase(spiceDbRepository, logger) deleteRelationshipsUsecase := biz.NewDeleteRelationshipsUsecase(spiceDbRepository, logger) relationshipsService := service.NewRelationshipsService(logger, createRelationshipsUsecase, readRelationshipsUsecase, deleteRelationshipsUsecase) - healthService := service.NewHealthService() + isBackendAvaliableUsecase := biz.NewIsBackendAvailableUsecase(spiceDbRepository) + healthService := service.NewHealthService(isBackendAvaliableUsecase) checkUsecase := biz.NewCheckUsecase(spiceDbRepository, logger) checkService := service.NewCheckService(logger, checkUsecase) getSubjectsUsecase := biz.NewGetSubjectsUseCase(spiceDbRepository) diff --git a/internal/biz/biz.go b/internal/biz/biz.go index 033bec6..e41d884 100644 --- a/internal/biz/biz.go +++ b/internal/biz/biz.go @@ -5,4 +5,4 @@ import ( ) // ProviderSet is biz providers. -var ProviderSet = wire.NewSet(NewCreateRelationshipsUsecase, NewReadRelationshipsUsecase, NewDeleteRelationshipsUsecase, NewCheckUsecase, NewGetSubjectsUseCase, NewGetResourcesUseCase) +var ProviderSet = wire.NewSet(NewCreateRelationshipsUsecase, NewReadRelationshipsUsecase, NewDeleteRelationshipsUsecase, NewCheckUsecase, NewGetSubjectsUseCase, NewGetResourcesUseCase, NewIsBackendAvailableUsecase) diff --git a/internal/biz/relationships.go b/internal/biz/relationships.go index 0dfb4bb..9751781 100644 --- a/internal/biz/relationships.go +++ b/internal/biz/relationships.go @@ -2,6 +2,7 @@ package biz import ( "context" + v0 "github.com/project-kessel/relations-api/api/relations/v0" "github.com/go-kratos/kratos/v2/log" @@ -32,6 +33,7 @@ type ZanzibarRepository interface { DeleteRelationships(context.Context, *v0.RelationTupleFilter) error LookupSubjects(ctx context.Context, subjectType *v0.ObjectType, subject_relation, relation string, resource *v0.ObjectReference, limit uint32, continuation ContinuationToken) (chan *SubjectResult, chan error, error) LookupResources(ctx context.Context, resouce_type *v0.ObjectType, relation string, subject *v0.SubjectReference, limit uint32, continuation ContinuationToken) (chan *ResourceResult, chan error, error) + IsBackendAvaliable() error } type CheckUsecase struct { @@ -47,6 +49,18 @@ func (rc *CheckUsecase) Check(ctx context.Context, check *v0.CheckRequest) (*v0. return rc.repo.Check(ctx, check) } +type IsBackendAvaliableUsecase struct { + repo ZanzibarRepository +} + +func NewIsBackendAvailableUsecase(repo ZanzibarRepository) *IsBackendAvaliableUsecase { + return &IsBackendAvaliableUsecase{repo: repo} +} + +func (rc *IsBackendAvaliableUsecase) IsBackendAvailable() error { + return rc.repo.IsBackendAvaliable() +} + type CreateRelationshipsUsecase struct { repo ZanzibarRepository log *log.Helper diff --git a/internal/conf/conf.pb.go b/internal/conf/conf.pb.go index e6f4fc3..098868a 100644 --- a/internal/conf/conf.pb.go +++ b/internal/conf/conf.pb.go @@ -1,7 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc-gen-go v1.34.1 // protoc v4.25.1 // source: conf/conf.proto diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 5688526..3fac6fa 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -7,6 +7,11 @@ import ( "io" "os" "strings" + "time" + + apiV0 "github.com/project-kessel/relations-api/api/relations/v0" + "github.com/project-kessel/relations-api/internal/biz" + "github.com/project-kessel/relations-api/internal/conf" apiV0 "github.com/project-kessel/relations-api/api/relations/v0" "github.com/project-kessel/relations-api/internal/biz" @@ -18,11 +23,13 @@ import ( "github.com/go-kratos/kratos/v2/log" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" ) // SpiceDbRepository . type SpiceDbRepository struct { - client *authzed.Client + client *authzed.Client + healthClient grpc_health_v1.HealthClient } // NewSpiceDbRepository . @@ -66,11 +73,21 @@ func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository, return nil, nil, fmt.Errorf("error creating spicedb client: %w", err) } + // Create health client for readyz + conn, err := grpc.NewClient( + c.SpiceDb.Endpoint, + opts..., + ) + if err != nil { + return nil, nil, fmt.Errorf("error creating grpc health client: %w", err) + } + healthClient := grpc_health_v1.NewHealthClient(conn) + cleanup := func() { log.NewHelper(logger).Info("spicedb connection cleanup requested (nothing to clean up)") } - return &SpiceDbRepository{client}, cleanup, nil + return &SpiceDbRepository{client, healthClient}, cleanup, nil } func (s *SpiceDbRepository) LookupSubjects(ctx context.Context, subject_type *apiV0.ObjectType, subject_relation, relation string, object *apiV0.ObjectReference, limit uint32, continuation biz.ContinuationToken) (chan *biz.SubjectResult, chan error, error) { @@ -326,6 +343,14 @@ func (s *SpiceDbRepository) Check(ctx context.Context, check *apiV0.CheckRequest return &apiV0.CheckResponse{Allowed: apiV0.CheckResponse_ALLOWED_FALSE}, nil } +func (s *SpiceDbRepository) IsBackendAvaliable() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := s.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + return err +} + func createSpiceDbRelationshipFilter(filter *apiV0.RelationTupleFilter) *v1.RelationshipFilter { spiceDbRelationshipFilter := &v1.RelationshipFilter{ ResourceType: filter.GetResourceType(), diff --git a/internal/service/health.go b/internal/service/health.go index aafd985..053154a 100644 --- a/internal/service/health.go +++ b/internal/service/health.go @@ -2,50 +2,30 @@ package service import ( "context" - "time" pb "github.com/project-kessel/relations-api/api/health/v1" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/health/grpc_health_v1" + "github.com/project-kessel/relations-api/internal/biz" ) type HealthService struct { pb.UnimplementedKesselHealthServer + backendUseCase *biz.IsBackendAvaliableUsecase } -func NewHealthService() *HealthService { - return &HealthService{} +func NewHealthService(backendUsecase *biz.IsBackendAvaliableUsecase) *HealthService { + return &HealthService{ + backendUseCase: backendUsecase, + } } func (s *HealthService) GetLivez(ctx context.Context, req *pb.GetLivezRequest) (*pb.GetLivezReply, error) { return &pb.GetLivezReply{Status: "OK", Code: 200}, nil } -func (s *HealthService) GetReadyz(ctx context.Context, req *pb.GetReadyzRequest) (*pb.GetReadyzReply, error) { - ready := checkSpiceDBReadyz() - if ready { - return &pb.GetReadyzReply{Status: "OK", Code: 200}, nil - } - return &pb.GetReadyzReply{Status: "Unavailable", Code: 503}, nil -} - -func checkSpiceDBReadyz() bool { - conn, err := grpc.NewClient( - "spicedb:50051", - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - return false - } - - client := grpc_health_v1.NewHealthClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - resp, err := client.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) +func (s *HealthService) GetReadyz(ctx context.Context, req *pb.GetReadyzRequest) (*pb.GetReadyzReply, error) { + err := s.backendUseCase.IsBackendAvailable() if err != nil { - return false + return &pb.GetReadyzReply{Status: "Unavailable", Code: 503}, nil } - - return resp.Status == grpc_health_v1.HealthCheckResponse_SERVING + return &pb.GetReadyzReply{Status: "OK", Code: 200}, nil } From d78d7c7b768005c2de235752f1d6f3ddb6dcf811 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 15:07:01 -0400 Subject: [PATCH 04/10] Add tests Signed-off-by: Jonathan Marcantonio --- internal/data/spicedb_test.go | 27 +++++++++++++-- internal/service/health_test.go | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 internal/service/health_test.go diff --git a/internal/data/spicedb_test.go b/internal/data/spicedb_test.go index 0e1942c..ac8449f 100644 --- a/internal/data/spicedb_test.go +++ b/internal/data/spicedb_test.go @@ -3,11 +3,12 @@ package data import ( "context" "fmt" - apiV0 "github.com/project-kessel/relations-api/api/relations/v0" - "github.com/project-kessel/relations-api/internal/biz" "os" "testing" + apiV0 "github.com/project-kessel/relations-api/api/relations/v0" + "github.com/project-kessel/relations-api/internal/biz" + "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/middleware/tracing" "github.com/stretchr/testify/assert" @@ -117,6 +118,28 @@ func TestSecondCreateRelationshipSucceedsWithTouchTrue(t *testing.T) { assert.True(t, exists) } +func TestIsBackendAvailable(t *testing.T) { + t.Parallel() + + spiceDbrepo, err := container.CreateSpiceDbRepository() + assert.NoError(t, err) + + err = spiceDbrepo.IsBackendAvaliable() + assert.NoError(t, err) +} + +func TestIsBackendAvailableFailsWithError(t *testing.T) { + t.Parallel() + + spiceDbrepo, err := container.CreateSpiceDbRepository() + assert.NoError(t, err) + + container.Close() + + err = spiceDbrepo.IsBackendAvaliable() + assert.Error(t, err) +} + func TestCreateRelationshipFailsWithBadSubjectType(t *testing.T) { t.Parallel() diff --git a/internal/service/health_test.go b/internal/service/health_test.go new file mode 100644 index 0000000..d252124 --- /dev/null +++ b/internal/service/health_test.go @@ -0,0 +1,60 @@ +package service + +import ( + "context" + "testing" + + pb "github.com/project-kessel/relations-api/api/health/v1" + "github.com/project-kessel/relations-api/internal/biz" + "github.com/project-kessel/relations-api/internal/data" + + "github.com/stretchr/testify/assert" +) + +func TestHealthService_GetLivez(t *testing.T) { + t.Parallel() + + ctx := context.TODO() + spicedb, err := container.CreateSpiceDbRepository() + assert.NoError(t, err) + + service := createHealthService(spicedb) + resp, err := service.GetLivez(ctx, &pb.GetLivezRequest{}) + + assert.NoError(t, err) + assert.Equal(t, resp, &pb.GetLivezReply{Status: "OK", Code: 200}) +} + +func TestHealthService_GetReadyz_SpiceDBAvailable(t *testing.T) { + t.Parallel() + + ctx := context.TODO() + spicedb, err := container.CreateSpiceDbRepository() + assert.NoError(t, err) + + service := createHealthService(spicedb) + resp, err := service.GetReadyz(ctx, &pb.GetReadyzRequest{}) + + assert.NoError(t, err) + assert.Equal(t, resp, &pb.GetReadyzReply{Status: "OK", Code: 200}) +} + +func TestHealthService_GetReadyz_SpiceDBUnavailable(t *testing.T) { + t.Parallel() + + ctx := context.TODO() + spicedb, err := container.CreateSpiceDbRepository() + assert.NoError(t, err) + + container.Close() + + service := createHealthService(spicedb) + resp, err := service.GetReadyz(ctx, &pb.GetReadyzRequest{}) + + assert.NoError(t, err) + assert.Equal(t, resp, &pb.GetReadyzReply{Status: "Unavailable", Code: 503}) +} + +func createHealthService(spicedb *data.SpiceDbRepository) *HealthService { + return NewHealthService(biz.NewIsBackendAvailableUsecase(spicedb)) +} From 963112d9ac6f4908a1a283bc3aa7ba4959f4c6dd Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 15:35:07 -0400 Subject: [PATCH 05/10] Remove problem tests Signed-off-by: Jonathan Marcantonio --- internal/data/spicedb_test.go | 12 ------------ internal/service/health_test.go | 16 ---------------- 2 files changed, 28 deletions(-) diff --git a/internal/data/spicedb_test.go b/internal/data/spicedb_test.go index ac8449f..4518214 100644 --- a/internal/data/spicedb_test.go +++ b/internal/data/spicedb_test.go @@ -128,18 +128,6 @@ func TestIsBackendAvailable(t *testing.T) { assert.NoError(t, err) } -func TestIsBackendAvailableFailsWithError(t *testing.T) { - t.Parallel() - - spiceDbrepo, err := container.CreateSpiceDbRepository() - assert.NoError(t, err) - - container.Close() - - err = spiceDbrepo.IsBackendAvaliable() - assert.Error(t, err) -} - func TestCreateRelationshipFailsWithBadSubjectType(t *testing.T) { t.Parallel() diff --git a/internal/service/health_test.go b/internal/service/health_test.go index d252124..ca2ba41 100644 --- a/internal/service/health_test.go +++ b/internal/service/health_test.go @@ -39,22 +39,6 @@ func TestHealthService_GetReadyz_SpiceDBAvailable(t *testing.T) { assert.Equal(t, resp, &pb.GetReadyzReply{Status: "OK", Code: 200}) } -func TestHealthService_GetReadyz_SpiceDBUnavailable(t *testing.T) { - t.Parallel() - - ctx := context.TODO() - spicedb, err := container.CreateSpiceDbRepository() - assert.NoError(t, err) - - container.Close() - - service := createHealthService(spicedb) - resp, err := service.GetReadyz(ctx, &pb.GetReadyzRequest{}) - - assert.NoError(t, err) - assert.Equal(t, resp, &pb.GetReadyzReply{Status: "Unavailable", Code: 503}) -} - func createHealthService(spicedb *data.SpiceDbRepository) *HealthService { return NewHealthService(biz.NewIsBackendAvailableUsecase(spicedb)) } From a8559727861513d7e1dbf428bea8d14385bfebe1 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 16:28:05 -0400 Subject: [PATCH 06/10] Check response status Signed-off-by: Jonathan Marcantonio --- internal/data/spicedb.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 3fac6fa..6beae2b 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -347,7 +347,17 @@ func (s *SpiceDbRepository) IsBackendAvaliable() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err := s.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + resp, err := s.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + + switch resp.Status { + case grpc_health_v1.HealthCheckResponse_NOT_SERVING, grpc_health_v1.HealthCheckResponse_SERVICE_UNKNOWN: + return err + case grpc_health_v1.HealthCheckResponse_SERVING: + return nil + } return err } From b51bb2444f2f331331a8ebc6767bcf0109134ec8 Mon Sep 17 00:00:00 2001 From: wscalf Date: Wed, 26 Jun 2024 17:36:53 -0400 Subject: [PATCH 07/10] Add waits for ready probes in e2e tests (#112) * Add waits for ready probes in e2e tests * Added max wait for containers to come up and removed redundant container purge --- internal/data/spicedb.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 6beae2b..2261738 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -82,6 +82,10 @@ func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository, return nil, nil, fmt.Errorf("error creating grpc health client: %w", err) } healthClient := grpc_health_v1.NewHealthClient(conn) + _, err = client.ReadSchema(context.TODO(), &v1.ReadSchemaRequest{}) + if err != nil { + return nil, nil, fmt.Errorf("error testing connection to SpiceDB: %w", err) + } cleanup := func() { log.NewHelper(logger).Info("spicedb connection cleanup requested (nothing to clean up)") From 29a49946be92627917f9655874b175ec1f180405 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Wed, 26 Jun 2024 14:33:20 -0400 Subject: [PATCH 08/10] Generalize readyz logic Signed-off-by: Jonathan Marcantonio --- internal/data/spicedb.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 2261738..44bdff8 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -87,6 +87,16 @@ func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository, return nil, nil, fmt.Errorf("error testing connection to SpiceDB: %w", err) } + // Create health client for readyz + conn, err := grpc.NewClient( + c.SpiceDb.Endpoint, + opts..., + ) + if err != nil { + return nil, nil, fmt.Errorf("error creating grpc health client: %w", err) + } + healthClient := grpc_health_v1.NewHealthClient(conn) + cleanup := func() { log.NewHelper(logger).Info("spicedb connection cleanup requested (nothing to clean up)") } From 94ffacd9bb6d84efbe6097db36b57b79c8fa2c62 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Thu, 27 Jun 2024 11:59:59 -0400 Subject: [PATCH 09/10] Move Health biz logic & update tests Signed-off-by: Jonathan Marcantonio --- internal/biz/health.go | 13 +++++++++++++ internal/biz/relationships.go | 14 +------------- internal/data/spicedb.go | 24 +++--------------------- internal/data/spicedb_test.go | 18 +++++++++++++++++- internal/service/health_test.go | 26 ++++++++++++++++++++++++++ openapi.yaml | 14 ++++++++++++-- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 internal/biz/health.go diff --git a/internal/biz/health.go b/internal/biz/health.go new file mode 100644 index 0000000..f802a5d --- /dev/null +++ b/internal/biz/health.go @@ -0,0 +1,13 @@ +package biz + +type IsBackendAvaliableUsecase struct { + repo ZanzibarRepository +} + +func NewIsBackendAvailableUsecase(repo ZanzibarRepository) *IsBackendAvaliableUsecase { + return &IsBackendAvaliableUsecase{repo: repo} +} + +func (rc *IsBackendAvaliableUsecase) IsBackendAvailable() error { + return rc.repo.IsBackendAvailable() +} diff --git a/internal/biz/relationships.go b/internal/biz/relationships.go index 9751781..684d8ef 100644 --- a/internal/biz/relationships.go +++ b/internal/biz/relationships.go @@ -33,7 +33,7 @@ type ZanzibarRepository interface { DeleteRelationships(context.Context, *v0.RelationTupleFilter) error LookupSubjects(ctx context.Context, subjectType *v0.ObjectType, subject_relation, relation string, resource *v0.ObjectReference, limit uint32, continuation ContinuationToken) (chan *SubjectResult, chan error, error) LookupResources(ctx context.Context, resouce_type *v0.ObjectType, relation string, subject *v0.SubjectReference, limit uint32, continuation ContinuationToken) (chan *ResourceResult, chan error, error) - IsBackendAvaliable() error + IsBackendAvailable() error } type CheckUsecase struct { @@ -49,18 +49,6 @@ func (rc *CheckUsecase) Check(ctx context.Context, check *v0.CheckRequest) (*v0. return rc.repo.Check(ctx, check) } -type IsBackendAvaliableUsecase struct { - repo ZanzibarRepository -} - -func NewIsBackendAvailableUsecase(repo ZanzibarRepository) *IsBackendAvaliableUsecase { - return &IsBackendAvaliableUsecase{repo: repo} -} - -func (rc *IsBackendAvaliableUsecase) IsBackendAvailable() error { - return rc.repo.IsBackendAvaliable() -} - type CreateRelationshipsUsecase struct { repo ZanzibarRepository log *log.Helper diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 44bdff8..7af0bb5 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -13,10 +13,6 @@ import ( "github.com/project-kessel/relations-api/internal/biz" "github.com/project-kessel/relations-api/internal/conf" - apiV0 "github.com/project-kessel/relations-api/api/relations/v0" - "github.com/project-kessel/relations-api/internal/biz" - "github.com/project-kessel/relations-api/internal/conf" - v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" "github.com/authzed/authzed-go/v1" "github.com/authzed/grpcutil" @@ -73,20 +69,6 @@ func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository, return nil, nil, fmt.Errorf("error creating spicedb client: %w", err) } - // Create health client for readyz - conn, err := grpc.NewClient( - c.SpiceDb.Endpoint, - opts..., - ) - if err != nil { - return nil, nil, fmt.Errorf("error creating grpc health client: %w", err) - } - healthClient := grpc_health_v1.NewHealthClient(conn) - _, err = client.ReadSchema(context.TODO(), &v1.ReadSchemaRequest{}) - if err != nil { - return nil, nil, fmt.Errorf("error testing connection to SpiceDB: %w", err) - } - // Create health client for readyz conn, err := grpc.NewClient( c.SpiceDb.Endpoint, @@ -357,7 +339,7 @@ func (s *SpiceDbRepository) Check(ctx context.Context, check *apiV0.CheckRequest return &apiV0.CheckResponse{Allowed: apiV0.CheckResponse_ALLOWED_FALSE}, nil } -func (s *SpiceDbRepository) IsBackendAvaliable() error { +func (s *SpiceDbRepository) IsBackendAvailable() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -368,11 +350,11 @@ func (s *SpiceDbRepository) IsBackendAvaliable() error { switch resp.Status { case grpc_health_v1.HealthCheckResponse_NOT_SERVING, grpc_health_v1.HealthCheckResponse_SERVICE_UNKNOWN: - return err + return fmt.Errorf("error connecting to backend: %v", resp.Status.Descriptor()) case grpc_health_v1.HealthCheckResponse_SERVING: return nil } - return err + return fmt.Errorf("error connecting to backend") } func createSpiceDbRelationshipFilter(filter *apiV0.RelationTupleFilter) *v1.RelationshipFilter { diff --git a/internal/data/spicedb_test.go b/internal/data/spicedb_test.go index 4518214..55e9189 100644 --- a/internal/data/spicedb_test.go +++ b/internal/data/spicedb_test.go @@ -8,6 +8,7 @@ import ( apiV0 "github.com/project-kessel/relations-api/api/relations/v0" "github.com/project-kessel/relations-api/internal/biz" + "github.com/project-kessel/relations-api/internal/conf" "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/middleware/tracing" @@ -124,10 +125,25 @@ func TestIsBackendAvailable(t *testing.T) { spiceDbrepo, err := container.CreateSpiceDbRepository() assert.NoError(t, err) - err = spiceDbrepo.IsBackendAvaliable() + err = spiceDbrepo.IsBackendAvailable() assert.NoError(t, err) } +func TestIsBackendUnavailable(t *testing.T) { + t.Parallel() + + spiceDBRepo, _, err := NewSpiceDbRepository(&conf.Data{ + SpiceDb: &conf.Data_SpiceDb{ + Endpoint: "-1", + Token: "foobar", + UseTLS: true, + }}, log.GetLogger()) + assert.NoError(t, err) + + err = spiceDBRepo.IsBackendAvailable() + assert.Error(t, err) +} + func TestCreateRelationshipFailsWithBadSubjectType(t *testing.T) { t.Parallel() diff --git a/internal/service/health_test.go b/internal/service/health_test.go index ca2ba41..e5e9100 100644 --- a/internal/service/health_test.go +++ b/internal/service/health_test.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "testing" pb "github.com/project-kessel/relations-api/api/health/v1" @@ -39,6 +40,31 @@ func TestHealthService_GetReadyz_SpiceDBAvailable(t *testing.T) { assert.Equal(t, resp, &pb.GetReadyzReply{Status: "OK", Code: 200}) } +func TestHealthService_GetReadyz_SpiceDBUnavailable(t *testing.T) { + t.Parallel() + + ctx := context.TODO() + + d := &DummyZanzibar{} + service := createDummyHealthService(d) + resp, err := service.GetReadyz(ctx, &pb.GetReadyzRequest{}) + + assert.NoError(t, err) + assert.Equal(t, resp, &pb.GetReadyzReply{Status: "Unavailable", Code: 503}) +} + +type DummyZanzibar struct { + biz.ZanzibarRepository +} + +func (dz *DummyZanzibar) IsBackendAvailable() error { + return fmt.Errorf("Unavailable") +} + +func createDummyHealthService(d *DummyZanzibar) *HealthService { + return NewHealthService(biz.NewIsBackendAvailableUsecase(d)) +} + func createHealthService(spicedb *data.SpiceDbRepository) *HealthService { return NewHealthService(biz.NewIsBackendAvailableUsecase(spicedb)) } diff --git a/openapi.yaml b/openapi.yaml index e660a4e..c88cc1f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -363,10 +363,20 @@ components: description: A reference to a Subject or, if a `relation` is provided, a Subject Set. kessel.relations.v1.GetLivezReply: type: object - properties: {} + properties: + status: + type: string + code: + type: integer + format: uint32 kessel.relations.v1.GetReadyzReply: type: object - properties: {} + properties: + status: + type: string + code: + type: integer + format: uint32 tags: - name: KesselCheckService - name: KesselHealth From 174d332819ffd76791d32a6c7655f72be907ebf4 Mon Sep 17 00:00:00 2001 From: Jonathan Marcantonio Date: Thu, 27 Jun 2024 14:18:54 -0400 Subject: [PATCH 10/10] Error when exceeding timeout Signed-off-by: Jonathan Marcantonio --- internal/data/spicedb.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/data/spicedb.go b/internal/data/spicedb.go index 7af0bb5..d45be59 100644 --- a/internal/data/spicedb.go +++ b/internal/data/spicedb.go @@ -348,11 +348,16 @@ func (s *SpiceDbRepository) IsBackendAvailable() error { return err } - switch resp.Status { - case grpc_health_v1.HealthCheckResponse_NOT_SERVING, grpc_health_v1.HealthCheckResponse_SERVICE_UNKNOWN: - return fmt.Errorf("error connecting to backend: %v", resp.Status.Descriptor()) - case grpc_health_v1.HealthCheckResponse_SERVING: - return nil + select { + case <-ctx.Done(): + return fmt.Errorf("timeout connecting to backend") + default: + switch resp.Status { + case grpc_health_v1.HealthCheckResponse_NOT_SERVING, grpc_health_v1.HealthCheckResponse_SERVICE_UNKNOWN: + return fmt.Errorf("error connecting to backend: %v", resp.Status.String()) + case grpc_health_v1.HealthCheckResponse_SERVING: + return nil + } } return fmt.Errorf("error connecting to backend") }