From 3f5bcd847db3edf7d0ab2da6459584a9f42ae160 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 2 Dec 2023 13:07:06 +0530 Subject: [PATCH 01/27] feat: bulksubscribe http Signed-off-by: sadath-12 --- examples/pubsub/sub/sub.go | 17 +- service/common/service.go | 3 + service/common/type.go | 18 ++ service/grpc/topic.go | 8 + service/http/topic.go | 241 +++++++++++++++++++++----- service/internal/topicregistrar.go | 61 ++++++- service/internal/topicsubscription.go | 21 +++ 7 files changed, 315 insertions(+), 54 deletions(-) diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index c9089d61..29216d8e 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -46,11 +46,13 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") - if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } + // if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { + // log.Fatalf("error adding topic subscription: %v", err) + // } - if err := s.AddTopicEventHandler(importantSubscription, importantEventHandler); err != nil { + + + if err := s.AddBulkTopicEventHandler(defaultSubscription, bulkeventHandler,10,1000); err != nil { log.Fatalf("error adding topic subscription: %v", err) } @@ -64,6 +66,13 @@ func eventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err er return false, nil } +func bulkeventHandler(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { + for _, event := range e { + log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", event.PubsubName, event.Topic, event.ID, event.Data) + } + return false, nil +} + func importantEventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { log.Printf("important event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) return false, nil diff --git a/service/common/service.go b/service/common/service.go index 7f306e4d..3c9c0ff7 100644 --- a/service/common/service.go +++ b/service/common/service.go @@ -35,6 +35,8 @@ type Service interface { // AddTopicEventHandler appends provided event handler with its topic and optional metadata to the service. // Note, retries are only considered when there is an error. Lack of error is considered as a success AddTopicEventHandler(sub *Subscription, fn TopicEventHandler) error + + AddBulkTopicEventHandler(sub *Subscription, fn BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error // AddBindingInvocationHandler appends provided binding invocation handler with its name to the service. AddBindingInvocationHandler(name string, fn BindingInvocationHandler) error // RegisterActorImplFactory Register a new actor to actor runtime of go sdk @@ -53,6 +55,7 @@ type Service interface { type ( ServiceInvocationHandler func(ctx context.Context, in *InvocationEvent) (out *Content, err error) TopicEventHandler func(ctx context.Context, e *TopicEvent) (retry bool, err error) + BulkTopicEventHandler func(ctx context.Context, e []BulkTopic) (retry bool, err error) BindingInvocationHandler func(ctx context.Context, in *BindingEvent) (out []byte, err error) HealthCheckHandler func(context.Context) error ) diff --git a/service/common/type.go b/service/common/type.go index 1883b3c3..5db6767c 100644 --- a/service/common/type.go +++ b/service/common/type.go @@ -47,6 +47,24 @@ type TopicEvent struct { PubsubName string `json:"pubsubname"` } +type BulkTopic struct { + ContentType string `json:"contentType"` + EntryID string `json:"entryId"` + Event map[string]string `json:"event"` + Data json.RawMessage `json:"data"` + DataContentType string `json:"datacontenttype"` + ID string `json:"id"` + PubsubName string `json:"pubsubname"` + Source string `json:"source"` + SpecVersion string `json:"specversion"` + Time string `json:"time"` + Topic string `json:"topic"` + TraceID string `json:"traceid"` + TraceParent string `json:"traceparent"` + TraceState string `json:"tracestate"` + Metadata interface{} `json:"metadata"` +} + func (e *TopicEvent) Struct(target interface{}) error { // TODO: Enhance to inspect DataContentType for the best // deserialization method. diff --git a/service/grpc/topic.go b/service/grpc/topic.go index da0c093c..078808aa 100644 --- a/service/grpc/topic.go +++ b/service/grpc/topic.go @@ -37,6 +37,14 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return s.topicRegistrar.AddSubscription(sub, fn) } +func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.BulkTopicEventHandler,maxMessagesCount,maxAwaitDurationMs int32) error { + if sub == nil { + return errors.New("subscription required") + } + + return s.topicRegistrar.AddBulkSubscription(sub, fn,maxMessagesCount,maxAwaitDurationMs) +} + // ListTopicSubscriptions is called by Dapr to get the list of topics in a pubsub component the app wants to subscribe to. func (s *Server) ListTopicSubscriptions(ctx context.Context, in *empty.Empty) (*runtimev1pb.ListTopicSubscriptionsResponse, error) { subs := make([]*runtimev1pb.TopicSubscription, 0) diff --git a/service/http/topic.go b/service/http/topic.go index 74376a89..910c0568 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -42,9 +42,10 @@ const ( // topicEventJSON is identical to `common.TopicEvent` // except for it treats `data` as a json.RawMessage so it can // be used as bytes or interface{}. +// Merge itemMap into topicEventJSON type topicEventJSON struct { // ID identifies the event. - ID string `json:"id"` + ID string `json:"id"` // y // The version of the CloudEvents specification. SpecVersion string `json:"specversion"` // The type of event related to the originating occurrence. @@ -65,52 +66,20 @@ type topicEventJSON struct { Topic string `json:"topic"` // PubsubName is name of the pub/sub this message came from PubsubName string `json:"pubsubname"` -} - -func (in topicEventJSON) getData() (data any, rawData []byte) { - var ( - err error - v any - ) - if len(in.Data) > 0 { - rawData = []byte(in.Data) - data = rawData - // We can assume that rawData is valid JSON - // without checking in.DataContentType == "application/json". - if err = json.Unmarshal(rawData, &v); err == nil { - data = v - // Handling of JSON base64 encoded or escaped in a string. - if str, ok := v.(string); ok { - // This is the path that will most likely succeed. - var ( - vString any - decoded []byte - ) - if err = json.Unmarshal([]byte(str), &vString); err == nil { - data = vString - } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { - // Decoded Base64 encoded JSON does not seem to be in the spec - // but it is in existing unit tests so this handles that case. - var vBase64 any - if err = json.Unmarshal(decoded, &vBase64); err == nil { - data = vBase64 - } - } - } - } - } else if in.DataBase64 != "" { - rawData, err = base64.StdEncoding.DecodeString(in.DataBase64) - if err == nil { - data = rawData - if in.DataContentType == "application/json" { - if err = json.Unmarshal(rawData, &v); err == nil { - data = v - } - } - } - } - - return data, rawData + // EntryID is the unique identifier of the entry + EntryID string `json:"entryId"` + // Event is a map of strings that contains additional information about the event + Event map[string]string `json:"event"` + // Time is the timestamp of the event + Time string `json:"time"` + // TraceID is the identifier of the trace + TraceID string `json:"traceid"` + // TraceParent is the identifier of the parent span + TraceParent string `json:"traceparent"` + // TraceState is the state of the trace + TraceState string `json:"tracestate"` + // Metadata is an interface that contains any other metadata associated with the event + Metadata interface{} `json:"metadata"` } func (s *Server) registerBaseHandler() { @@ -301,6 +270,184 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return nil } + + +func (in topicEventJSON) getData() (data any, rawData []byte) { + var ( + err error + v any + ) + if len(in.Data) > 0 { + rawData = []byte(in.Data) + data = rawData + // We can assume that rawData is valid JSON + // without checking in.DataContentType == "application/json". + if err = json.Unmarshal(rawData, &v); err == nil { + data = v + // Handling of JSON base64 encoded or escaped in a string. + if str, ok := v.(string); ok { + // This is the path that will most likely succeed. + var ( + vString any + decoded []byte + ) + if err = json.Unmarshal([]byte(str), &vString); err == nil { + data = vString + } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { + // Decoded Base64 encoded JSON does not seem to be in the spec + // but it is in existing unit tests so this handles that case. + var vBase64 any + if err = json.Unmarshal(decoded, &vBase64); err == nil { + data = vBase64 + } + } + } + } + } else if in.DataBase64 != "" { + rawData, err = base64.StdEncoding.DecodeString(in.DataBase64) + if err == nil { + data = rawData + if in.DataContentType == "application/json" { + if err = json.Unmarshal(rawData, &v); err == nil { + data = v + } + } + } + } + + return data, rawData +} + +type BulkTopicJson struct { + ContentType string `json:"contentType"` + EntryID string `json:"entryId"` + Event map[string]string `json:"event"` + Data json.RawMessage `json:"data"` + DataContentType string `json:"datacontenttype"` + ID string `json:"id"` + PubsubName string `json:"pubsubname"` + Source string `json:"source"` + SpecVersion string `json:"specversion"` + Time string `json:"time"` + Topic string `json:"topic"` + TraceID string `json:"traceid"` + TraceParent string `json:"traceparent"` + TraceState string `json:"tracestate"` + Metadata interface{} `json:"metadata"` +} + +// == APP == the item is {application/cloudevents+json 5e582fd2-f1c4-47bd-81b1-803fb7e86552 map[data:multi-pong datacontenttype:text/plain id:92fc5348-097d-4b9f-b093-7e6fcda77add pubsubname:messages source:pub specversion:1.0 time:2023-12-02T11:42:31+05:30 topic:neworder traceid:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 traceparent:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 tracestate: type:com.dapr.event.sent] [] } + +func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { + if sub == nil { + return errors.New("subscription required") + } + // Route is only required for HTTP but should be specified for the + // app protocol to be interchangeable. + if sub.Route == "" { + return errors.New("handler route name") + } + if err := s.topicRegistrar.AddBulkSubscription(sub, fn, maxMessagesCount, maxAwaitDurationMs); err != nil { + return err + } + + s.mux.Handle(sub.Route, optionsHandler(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + // check for post with no data + var ( + body []byte + err error + ) + if r.Body != nil { + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + return + } + } + if len(body) == 0 { + http.Error(w, "nil content", PubSubHandlerDropStatusCode) + return + } + + // deserialize the event + var ins map[string]interface{} + if err = json.Unmarshal(body, &ins); err != nil { + http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + return + } + + entriesInterface, ok := ins["entries"].([]interface{}) + if !ok { + // Handle the error or return an error response + http.Error(w, "Entries format error", PubSubHandlerDropStatusCode) + return + } + + var messages []common.BulkTopic + for _, entry := range entriesInterface { + itemMap, ok := entry.(map[string]interface{}) + if !ok { + http.Error(w, "Entry format error", PubSubHandlerDropStatusCode) + return + } + + itemJSON, err := json.Marshal(itemMap) + if err != nil { + http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + return + } + var item BulkTopicJson + + if err := json.Unmarshal(itemJSON, &item); err != nil { + http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + return + } + + + newItem := common.BulkTopic{ + ContentType: item.ContentType, + EntryID: item.EntryID, + Event: item.Event, + Data: item.Data, + DataContentType: item.DataContentType, + ID: item.ID, + PubsubName: item.PubsubName, + Source: item.Source, + SpecVersion: item.SpecVersion, + Metadata: item.Metadata, + Time: item.Time, + Topic: item.Topic, + TraceID: item.TraceID, + TraceParent: item.TraceParent, + TraceState: item.TraceState, + } + + messages = append(messages, newItem) + } + + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + // execute user handler + retry, err := fn(r.Context(), messages) + if err == nil { + writeStatus(w, common.SubscriptionResponseStatusSuccess) + return + } + + if retry { + writeStatus(w, common.SubscriptionResponseStatusRetry) + return + } + + writeStatus(w, common.SubscriptionResponseStatusDrop) + }))) + + return nil +} + + func writeStatus(w http.ResponseWriter, s string) { status := &common.SubscriptionResponse{Status: s} if err := json.NewEncoder(w).Encode(status); err != nil { diff --git a/service/internal/topicregistrar.go b/service/internal/topicregistrar.go index ec4efb9e..ce6a1607 100644 --- a/service/internal/topicregistrar.go +++ b/service/internal/topicregistrar.go @@ -1,6 +1,7 @@ package internal import ( + "context" "errors" "fmt" @@ -14,9 +15,11 @@ type TopicRegistrar map[string]*TopicRegistration // TopicRegistration encapsulates the subscription and handlers. type TopicRegistration struct { - Subscription *TopicSubscription - DefaultHandler common.TopicEventHandler - RouteHandlers map[string]common.TopicEventHandler + Subscription *TopicSubscription + DefaultHandler common.TopicEventHandler + DefaultBulkHandler common.BulkTopicEventHandler + RouteHandlers map[string]common.TopicEventHandler + BulkRouteHandlers map[string]common.BulkTopicEventHandler } func (m TopicRegistrar) AddSubscription(sub *common.Subscription, fn common.TopicEventHandler) error { @@ -62,3 +65,55 @@ func (m TopicRegistrar) AddSubscription(sub *common.Subscription, fn common.Topi return nil } + +func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common.BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { + if sub.Topic == "" { + return errors.New("topic name required") + } + if sub.PubsubName == "" { + return errors.New("pub/sub name required") + } + if fn == nil { + return fmt.Errorf("topic handler required") + } + + var key string + if !sub.DisableTopicValidation { + key = sub.PubsubName + "-" + sub.Topic + } else { + key = sub.PubsubName + } + + ts, ok := m[key] + if !ok { + ts = &TopicRegistration{ + Subscription: NewTopicSubscription(sub.PubsubName, sub.Topic), + RouteHandlers: make(map[string]common.TopicEventHandler), + DefaultHandler: nil, + } + ts.Subscription.SetMetadata(sub.Metadata) + m[key] = ts + } + + ts.Subscription.SetBulkSubscribe(maxMessagesCount, maxAwaitDurationMs) + + if sub.Match != "" { + if err := ts.Subscription.AddRoutingRule(sub.Route, sub.Match, sub.Priority); err != nil { + return err + } + } else { + if err := ts.Subscription.SetDefaultRoute(sub.Route); err != nil { + return err + } + // ts.DefaultBulkHandler = fn + ts.DefaultHandler = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + return false,nil + } + } + // ts.BulkRouteHandlers[sub.Route] = fn + ts.RouteHandlers[sub.Route] = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + return false,nil + } + + return nil +} diff --git a/service/internal/topicsubscription.go b/service/internal/topicsubscription.go index ca628993..7a37b2ef 100644 --- a/service/internal/topicsubscription.go +++ b/service/internal/topicsubscription.go @@ -18,6 +18,14 @@ type TopicSubscription struct { Routes *TopicRoutes `json:"routes,omitempty"` // Metadata is the subscription metadata. Metadata map[string]string `json:"metadata,omitempty"` + // bulksubsribe + BulkSubscribe *BulkSubscribe `json:"bulkSubscribe,omitempty"` +} + +type BulkSubscribe struct { + Enabled bool `json:"enabled"` + MaxMessagesCount int32 `json:"maxMessagesCount,omitempty"` + MaxAwaitDurationMs int32 `json:"maxAwaitDurationMs,omitempty"` } // TopicRoutes encapsulates the default route and multiple routing rules. @@ -60,6 +68,19 @@ func (s *TopicSubscription) SetMetadata(metadata map[string]string) error { return nil } +func (s *TopicSubscription) SetBulkSubscribe(maxMessagesCount,maxAwaitDurationMs int32) error { + if s.BulkSubscribe != nil { + return fmt.Errorf("subscription for topic %s on pubsub %s already has bulkSubscribe set", s.Topic, s.PubsubName) + } + s.BulkSubscribe = &BulkSubscribe{ + Enabled: true, + MaxMessagesCount: maxMessagesCount, + MaxAwaitDurationMs: maxAwaitDurationMs, + } + + return nil +} + // SetDefaultRoute sets the default route if not already set. // An error is returned if it is already set. func (s *TopicSubscription) SetDefaultRoute(path string) error { From 1c45f4e56d769bf3f39bcd95ac5fcd4bcdb898d6 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 2 Dec 2023 13:51:30 +0530 Subject: [PATCH 02/27] minor changes Signed-off-by: sadath-12 --- service/common/type.go | 3 +- service/http/topic.go | 68 ++++++++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/service/common/type.go b/service/common/type.go index 5db6767c..52902e4c 100644 --- a/service/common/type.go +++ b/service/common/type.go @@ -51,7 +51,8 @@ type BulkTopic struct { ContentType string `json:"contentType"` EntryID string `json:"entryId"` Event map[string]string `json:"event"` - Data json.RawMessage `json:"data"` + RawData []byte `json:"-"` + Data interface{} `json:"data"` DataContentType string `json:"datacontenttype"` ID string `json:"id"` PubsubName string `json:"pubsubname"` diff --git a/service/http/topic.go b/service/http/topic.go index 910c0568..aeeff206 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -66,20 +66,6 @@ type topicEventJSON struct { Topic string `json:"topic"` // PubsubName is name of the pub/sub this message came from PubsubName string `json:"pubsubname"` - // EntryID is the unique identifier of the entry - EntryID string `json:"entryId"` - // Event is a map of strings that contains additional information about the event - Event map[string]string `json:"event"` - // Time is the timestamp of the event - Time string `json:"time"` - // TraceID is the identifier of the trace - TraceID string `json:"traceid"` - // TraceParent is the identifier of the parent span - TraceParent string `json:"traceparent"` - // TraceState is the state of the trace - TraceState string `json:"tracestate"` - // Metadata is an interface that contains any other metadata associated with the event - Metadata interface{} `json:"metadata"` } func (s *Server) registerBaseHandler() { @@ -270,8 +256,6 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return nil } - - func (in topicEventJSON) getData() (data any, rawData []byte) { var ( err error @@ -318,6 +302,42 @@ func (in topicEventJSON) getData() (data any, rawData []byte) { return data, rawData } +func (in BulkTopicJson) getData() (data any, rawData []byte) { + var ( + err error + v any + ) + if len(in.Data) > 0 { + rawData = []byte(in.Data) + data = rawData + // We can assume that rawData is valid JSON + // without checking in.DataContentType == "application/json". + if err = json.Unmarshal(rawData, &v); err == nil { + data = v + // Handling of JSON base64 encoded or escaped in a string. + if str, ok := v.(string); ok { + // This is the path that will most likely succeed. + var ( + vString any + decoded []byte + ) + if err = json.Unmarshal([]byte(str), &vString); err == nil { + data = vString + } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { + // Decoded Base64 encoded JSON does not seem to be in the spec + // but it is in existing unit tests so this handles that case. + var vBase64 any + if err = json.Unmarshal(decoded, &vBase64); err == nil { + data = vBase64 + } + } + } + } + } + + return data, rawData +} + type BulkTopicJson struct { ContentType string `json:"contentType"` EntryID string `json:"entryId"` @@ -404,14 +424,25 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu return } + data, rawData := item.getData() + + if item.PubsubName == "" { + item.PubsubName = sub.PubsubName + } + + if item.Topic == "" { + item.Topic = sub.Topic + } + newItem := common.BulkTopic{ ContentType: item.ContentType, EntryID: item.EntryID, Event: item.Event, - Data: item.Data, + Data: data, + RawData: rawData, DataContentType: item.DataContentType, - ID: item.ID, + ID: item.EntryID, PubsubName: item.PubsubName, Source: item.Source, SpecVersion: item.SpecVersion, @@ -447,7 +478,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu return nil } - func writeStatus(w http.ResponseWriter, s string) { status := &common.SubscriptionResponse{Status: s} if err := json.NewEncoder(w).Encode(status); err != nil { From 2193f475c5f646b38fbee1ec6199c51e114ddddb Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 2 Dec 2023 18:21:45 +0530 Subject: [PATCH 03/27] data retrive fix Signed-off-by: sadath-12 --- service/http/topic.go | 25 +++++++++++++++++++++---- service/internal/topicregistrar.go | 10 ++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/service/http/topic.go b/service/http/topic.go index aeeff206..ed5eac4f 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -68,6 +68,25 @@ type topicEventJSON struct { PubsubName string `json:"pubsubname"` } +type status string + +const ( + // SubscriptionResponseStatusSuccess indicates that the subscription event was processed successfully. + SubscriptionResponseStatusSuccess status = "SUCCESS" +) + +type BulkSubscribeResponseEntry struct { + // The id of the bulk subscribe entry + entryId string + + // The response status of the bulk subscribe entry + status status +} + +type BulkSubscribeResponse struct { + statuses []BulkSubscribeResponseEntry +} + func (s *Server) registerBaseHandler() { // register subscribe handler f := func(w http.ResponseWriter, r *http.Request) { @@ -397,7 +416,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu return } - entriesInterface, ok := ins["entries"].([]interface{}) + entriesInterface ,ok := ins["entries"].([]interface{}) if !ok { // Handle the error or return an error response http.Error(w, "Entries format error", PubSubHandlerDropStatusCode) @@ -411,8 +430,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu http.Error(w, "Entry format error", PubSubHandlerDropStatusCode) return } - - itemJSON, err := json.Marshal(itemMap) + itemJSON, err := json.Marshal(itemMap["event"]) if err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return @@ -423,7 +441,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } - data, rawData := item.getData() if item.PubsubName == "" { diff --git a/service/internal/topicregistrar.go b/service/internal/topicregistrar.go index ce6a1607..a538f2d2 100644 --- a/service/internal/topicregistrar.go +++ b/service/internal/topicregistrar.go @@ -90,6 +90,8 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. Subscription: NewTopicSubscription(sub.PubsubName, sub.Topic), RouteHandlers: make(map[string]common.TopicEventHandler), DefaultHandler: nil, + BulkRouteHandlers: make(map[string]common.BulkTopicEventHandler), + DefaultBulkHandler: nil, } ts.Subscription.SetMetadata(sub.Metadata) m[key] = ts @@ -105,12 +107,16 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. if err := ts.Subscription.SetDefaultRoute(sub.Route); err != nil { return err } - // ts.DefaultBulkHandler = fn + ts.DefaultBulkHandler = func(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { + return false,nil + } ts.DefaultHandler = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { return false,nil } } - // ts.BulkRouteHandlers[sub.Route] = fn + ts.BulkRouteHandlers[sub.Route] = func(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { + return false,nil + } ts.RouteHandlers[sub.Route] = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { return false,nil } From 5d4e498eb72ddc31bb27ff1cd1fe08652a69598b Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 2 Dec 2023 23:22:30 +0530 Subject: [PATCH 04/27] bulkresponse Signed-off-by: sadath-12 --- service/http/topic.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/service/http/topic.go b/service/http/topic.go index ed5eac4f..82c02b8a 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -416,13 +416,15 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu return } - entriesInterface ,ok := ins["entries"].([]interface{}) + entriesInterface, ok := ins["entries"].([]interface{}) if !ok { // Handle the error or return an error response http.Error(w, "Entries format error", PubSubHandlerDropStatusCode) return } + statuses := make([]BulkSubscribeResponseEntry, 0, len(entriesInterface)) + var messages []common.BulkTopic for _, entry := range entriesInterface { itemMap, ok := entry.(map[string]interface{}) @@ -430,6 +432,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu http.Error(w, "Entry format error", PubSubHandlerDropStatusCode) return } + itemJSON, err := json.Marshal(itemMap["event"]) if err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) @@ -443,6 +446,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu } data, rawData := item.getData() + if item.PubsubName == "" { item.PubsubName = sub.PubsubName } @@ -451,6 +455,11 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu item.Topic = sub.Topic } + statuses = append(statuses, BulkSubscribeResponseEntry{ + entryId: item.EntryID, + status: SubscriptionResponseStatusSuccess, + }, + ) newItem := common.BulkTopic{ ContentType: item.ContentType, @@ -473,11 +482,17 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu messages = append(messages, newItem) } - + resp := BulkSubscribeResponse{ + statuses: statuses, + } + responseJSON, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + return + } w.Header().Add("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) + w.Write(responseJSON) - // execute user handler retry, err := fn(r.Context(), messages) if err == nil { writeStatus(w, common.SubscriptionResponseStatusSuccess) @@ -488,7 +503,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu writeStatus(w, common.SubscriptionResponseStatusRetry) return } - writeStatus(w, common.SubscriptionResponseStatusDrop) }))) From f19a50588b8b0ddbd9e051facf856a57eb98f425 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sun, 3 Dec 2023 01:15:12 +0530 Subject: [PATCH 05/27] reformat Signed-off-by: sadath-12 --- examples/pubsub/pub/pub.go | 8 +- examples/pubsub/sub/sub.go | 2 +- service/common/service.go | 2 +- service/http/topic.go | 138 +++++++++-------------------- service/internal/topicregistrar.go | 4 +- 5 files changed, 49 insertions(+), 105 deletions(-) diff --git a/examples/pubsub/pub/pub.go b/examples/pubsub/pub/pub.go index 60bce409..b42852af 100644 --- a/examples/pubsub/pub/pub.go +++ b/examples/pubsub/pub/pub.go @@ -29,7 +29,7 @@ var ( func main() { ctx := context.Background() - publishEventData := []byte("ping") + // publishEventData := []byte("ping") publishEventsData := []interface{}{"multi-ping", "multi-pong"} client, err := dapr.NewClient() @@ -39,9 +39,9 @@ func main() { defer client.Close() // Publish a single event - if err := client.PublishEvent(ctx, pubsubName, topicName, publishEventData); err != nil { - panic(err) - } + // if err := client.PublishEvent(ctx, pubsubName, topicName, publishEventData); err != nil { + // panic(err) + // } // Publish multiple events if res := client.PublishEvents(ctx, pubsubName, topicName, publishEventsData); res.Error != nil { diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index 29216d8e..865a22c8 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -66,7 +66,7 @@ func eventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err er return false, nil } -func bulkeventHandler(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { +func bulkeventHandler(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { for _, event := range e { log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", event.PubsubName, event.Topic, event.ID, event.Data) } diff --git a/service/common/service.go b/service/common/service.go index 3c9c0ff7..34ecb53a 100644 --- a/service/common/service.go +++ b/service/common/service.go @@ -55,7 +55,7 @@ type Service interface { type ( ServiceInvocationHandler func(ctx context.Context, in *InvocationEvent) (out *Content, err error) TopicEventHandler func(ctx context.Context, e *TopicEvent) (retry bool, err error) - BulkTopicEventHandler func(ctx context.Context, e []BulkTopic) (retry bool, err error) + BulkTopicEventHandler func(ctx context.Context, e []TopicEvent) (retry bool, err error) BindingInvocationHandler func(ctx context.Context, in *BindingEvent) (out []byte, err error) HealthCheckHandler func(context.Context) error ) diff --git a/service/http/topic.go b/service/http/topic.go index 82c02b8a..f56b49eb 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -321,58 +321,20 @@ func (in topicEventJSON) getData() (data any, rawData []byte) { return data, rawData } -func (in BulkTopicJson) getData() (data any, rawData []byte) { - var ( - err error - v any - ) - if len(in.Data) > 0 { - rawData = []byte(in.Data) - data = rawData - // We can assume that rawData is valid JSON - // without checking in.DataContentType == "application/json". - if err = json.Unmarshal(rawData, &v); err == nil { - data = v - // Handling of JSON base64 encoded or escaped in a string. - if str, ok := v.(string); ok { - // This is the path that will most likely succeed. - var ( - vString any - decoded []byte - ) - if err = json.Unmarshal([]byte(str), &vString); err == nil { - data = vString - } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { - // Decoded Base64 encoded JSON does not seem to be in the spec - // but it is in existing unit tests so this handles that case. - var vBase64 any - if err = json.Unmarshal(decoded, &vBase64); err == nil { - data = vBase64 - } - } - } - } - } - - return data, rawData +type BulkSubscribeMessageItem struct { + EntryId string `json:"entryId"` //nolint:stylecheck + Event interface{} `json:"event"` + Metadata map[string]string `json:"metadata"` + ContentType string `json:"contentType,omitempty"` } -type BulkTopicJson struct { - ContentType string `json:"contentType"` - EntryID string `json:"entryId"` - Event map[string]string `json:"event"` - Data json.RawMessage `json:"data"` - DataContentType string `json:"datacontenttype"` - ID string `json:"id"` - PubsubName string `json:"pubsubname"` - Source string `json:"source"` - SpecVersion string `json:"specversion"` - Time string `json:"time"` - Topic string `json:"topic"` - TraceID string `json:"traceid"` - TraceParent string `json:"traceparent"` - TraceState string `json:"tracestate"` - Metadata interface{} `json:"metadata"` +type BulkSubscribeEnvelope struct { + ID string + Entries []BulkSubscribeMessageItem + Metadata map[string]string + Topic string + Pubsub string + EventType string } // == APP == the item is {application/cloudevents+json 5e582fd2-f1c4-47bd-81b1-803fb7e86552 map[data:multi-pong datacontenttype:text/plain id:92fc5348-097d-4b9f-b093-7e6fcda77add pubsubname:messages source:pub specversion:1.0 time:2023-12-02T11:42:31+05:30 topic:neworder traceid:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 traceparent:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 tracestate: type:com.dapr.event.sent] [] } @@ -410,88 +372,70 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu } // deserialize the event - var ins map[string]interface{} + var ins BulkSubscribeEnvelope if err = json.Unmarshal(body, &ins); err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } - entriesInterface, ok := ins["entries"].([]interface{}) - if !ok { - // Handle the error or return an error response - http.Error(w, "Entries format error", PubSubHandlerDropStatusCode) - return - } - - statuses := make([]BulkSubscribeResponseEntry, 0, len(entriesInterface)) - - var messages []common.BulkTopic - for _, entry := range entriesInterface { - itemMap, ok := entry.(map[string]interface{}) - if !ok { - http.Error(w, "Entry format error", PubSubHandlerDropStatusCode) - return - } + statuses := make([]BulkSubscribeResponseEntry, 0, len(ins.Entries)) - itemJSON, err := json.Marshal(itemMap["event"]) + var messages []common.TopicEvent + for _, entry := range ins.Entries { + itemJSON, err := json.Marshal(entry.Event) if err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } - var item BulkTopicJson + var in topicEventJSON - if err := json.Unmarshal(itemJSON, &item); err != nil { + if err := json.Unmarshal(itemJSON, &in); err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } - data, rawData := item.getData() - - - if item.PubsubName == "" { - item.PubsubName = sub.PubsubName + if in.PubsubName == "" { + in.Topic = sub.PubsubName } - - if item.Topic == "" { - item.Topic = sub.Topic + if in.Topic == "" { + in.Topic = sub.Topic } + data, rawData := in.getData() statuses = append(statuses, BulkSubscribeResponseEntry{ - entryId: item.EntryID, + entryId: in.ID, status: SubscriptionResponseStatusSuccess, }, ) - newItem := common.BulkTopic{ - ContentType: item.ContentType, - EntryID: item.EntryID, - Event: item.Event, + te := common.TopicEvent{ + ID: in.ID, + SpecVersion: in.SpecVersion, + Type: in.Type, + Source: in.Source, + DataContentType: in.DataContentType, Data: data, RawData: rawData, - DataContentType: item.DataContentType, - ID: item.EntryID, - PubsubName: item.PubsubName, - Source: item.Source, - SpecVersion: item.SpecVersion, - Metadata: item.Metadata, - Time: item.Time, - Topic: item.Topic, - TraceID: item.TraceID, - TraceParent: item.TraceParent, - TraceState: item.TraceState, + DataBase64: in.DataBase64, + Subject: in.Subject, + PubsubName: in.PubsubName, + Topic: in.Topic, } - messages = append(messages, newItem) + messages = append(messages, te) } resp := BulkSubscribeResponse{ statuses: statuses, } - responseJSON, err := json.Marshal(resp) if err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } w.Header().Add("Content-Type", "application/json") - w.Write(responseJSON) + if err := json.NewEncoder(w).Encode(resp); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) retry, err := fn(r.Context(), messages) if err == nil { diff --git a/service/internal/topicregistrar.go b/service/internal/topicregistrar.go index a538f2d2..0ccfe7b7 100644 --- a/service/internal/topicregistrar.go +++ b/service/internal/topicregistrar.go @@ -107,14 +107,14 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. if err := ts.Subscription.SetDefaultRoute(sub.Route); err != nil { return err } - ts.DefaultBulkHandler = func(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { + ts.DefaultBulkHandler = func(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { return false,nil } ts.DefaultHandler = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { return false,nil } } - ts.BulkRouteHandlers[sub.Route] = func(ctx context.Context, e []common.BulkTopic) (retry bool, err error) { + ts.BulkRouteHandlers[sub.Route] = func(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { return false,nil } ts.RouteHandlers[sub.Route] = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { From fdd425a0f42645cc7a0b02c2ca74e5360b6dab5d Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sun, 3 Dec 2023 01:40:20 +0530 Subject: [PATCH 06/27] bulksub success Signed-off-by: sadath-12 --- examples/pubsub/pub/pub.go | 8 +- examples/pubsub/sub/bulksub.go | 62 ++++++++++++++++ examples/pubsub/sub/sub.go | 8 +- service/common/type.go | 19 ----- service/http/topic.go | 130 +++++++++++++++++---------------- 5 files changed, 135 insertions(+), 92 deletions(-) create mode 100644 examples/pubsub/sub/bulksub.go diff --git a/examples/pubsub/pub/pub.go b/examples/pubsub/pub/pub.go index b42852af..60bce409 100644 --- a/examples/pubsub/pub/pub.go +++ b/examples/pubsub/pub/pub.go @@ -29,7 +29,7 @@ var ( func main() { ctx := context.Background() - // publishEventData := []byte("ping") + publishEventData := []byte("ping") publishEventsData := []interface{}{"multi-ping", "multi-pong"} client, err := dapr.NewClient() @@ -39,9 +39,9 @@ func main() { defer client.Close() // Publish a single event - // if err := client.PublishEvent(ctx, pubsubName, topicName, publishEventData); err != nil { - // panic(err) - // } + if err := client.PublishEvent(ctx, pubsubName, topicName, publishEventData); err != nil { + panic(err) + } // Publish multiple events if res := client.PublishEvents(ctx, pubsubName, topicName, publishEventsData); res.Error != nil { diff --git a/examples/pubsub/sub/bulksub.go b/examples/pubsub/sub/bulksub.go new file mode 100644 index 00000000..58e2ab27 --- /dev/null +++ b/examples/pubsub/sub/bulksub.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "net/http" + + "github.com/dapr/go-sdk/service/common" + daprd "github.com/dapr/go-sdk/service/http" +) + + +var defaultSubscription = &common.Subscription{ + PubsubName: "messages", + Topic: "neworder", + Route: "/orders", +} + +var importantSubscription = &common.Subscription{ + PubsubName: "messages", + Topic: "neworder", + Route: "/important", + Match: `event.type == "important"`, + Priority: 1, +} + +func main() { + s := daprd.NewService(":8080") + + if err := s.AddBulkTopicEventHandler(defaultSubscription, bulkeventHandler,10,1000); err != nil { + log.Fatalf("error adding topic subscription: %v", err) + } + + if err := s.Start(); err != nil && err != http.ErrServerClosed { + log.Fatalf("error listenning: %v", err) + } +} + +func bulkeventHandler(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { + for _, event := range e { + log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", event.PubsubName, event.Topic, event.ID, event.Data) + } + return false, nil +} + +func importantEventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + log.Printf("important event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) + return false, nil +} diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index 865a22c8..9d103478 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -46,13 +46,7 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") - // if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { - // log.Fatalf("error adding topic subscription: %v", err) - // } - - - - if err := s.AddBulkTopicEventHandler(defaultSubscription, bulkeventHandler,10,1000); err != nil { + if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { log.Fatalf("error adding topic subscription: %v", err) } diff --git a/service/common/type.go b/service/common/type.go index 52902e4c..1883b3c3 100644 --- a/service/common/type.go +++ b/service/common/type.go @@ -47,25 +47,6 @@ type TopicEvent struct { PubsubName string `json:"pubsubname"` } -type BulkTopic struct { - ContentType string `json:"contentType"` - EntryID string `json:"entryId"` - Event map[string]string `json:"event"` - RawData []byte `json:"-"` - Data interface{} `json:"data"` - DataContentType string `json:"datacontenttype"` - ID string `json:"id"` - PubsubName string `json:"pubsubname"` - Source string `json:"source"` - SpecVersion string `json:"specversion"` - Time string `json:"time"` - Topic string `json:"topic"` - TraceID string `json:"traceid"` - TraceParent string `json:"traceparent"` - TraceState string `json:"tracestate"` - Metadata interface{} `json:"metadata"` -} - func (e *TopicEvent) Struct(target interface{}) error { // TODO: Enhance to inspect DataContentType for the best // deserialization method. diff --git a/service/http/topic.go b/service/http/topic.go index f56b49eb..80d5b227 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -68,23 +68,73 @@ type topicEventJSON struct { PubsubName string `json:"pubsubname"` } -type status string +func (in topicEventJSON) getData() (data any, rawData []byte) { + var ( + err error + v any + ) + if len(in.Data) > 0 { + rawData = []byte(in.Data) + data = rawData + // We can assume that rawData is valid JSON + // without checking in.DataContentType == "application/json". + if err = json.Unmarshal(rawData, &v); err == nil { + data = v + // Handling of JSON base64 encoded or escaped in a string. + if str, ok := v.(string); ok { + // This is the path that will most likely succeed. + var ( + vString any + decoded []byte + ) + if err = json.Unmarshal([]byte(str), &vString); err == nil { + data = vString + } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { + // Decoded Base64 encoded JSON does not seem to be in the spec + // but it is in existing unit tests so this handles that case. + var vBase64 any + if err = json.Unmarshal(decoded, &vBase64); err == nil { + data = vBase64 + } + } + } + } + } else if in.DataBase64 != "" { + rawData, err = base64.StdEncoding.DecodeString(in.DataBase64) + if err == nil { + data = rawData + if in.DataContentType == "application/json" { + if err = json.Unmarshal(rawData, &v); err == nil { + data = v + } + } + } + } + + return data, rawData +} + +type AppResponseStatus string const ( - // SubscriptionResponseStatusSuccess indicates that the subscription event was processed successfully. - SubscriptionResponseStatusSuccess status = "SUCCESS" + // Success means the message is received and processed correctly. + Success AppResponseStatus = "SUCCESS" + // Retry means the message is received but could not be processed and must be retried. + Retry AppResponseStatus = "RETRY" + // Drop means the message is received but should not be processed. + Drop AppResponseStatus = "DROP" ) type BulkSubscribeResponseEntry struct { // The id of the bulk subscribe entry - entryId string + EntryId string `json:"entryId"` // The response status of the bulk subscribe entry - status status + Status AppResponseStatus `json:"status"` } type BulkSubscribeResponse struct { - statuses []BulkSubscribeResponseEntry + Statuses []BulkSubscribeResponseEntry `json:"statuses"` } func (s *Server) registerBaseHandler() { @@ -275,52 +325,6 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return nil } -func (in topicEventJSON) getData() (data any, rawData []byte) { - var ( - err error - v any - ) - if len(in.Data) > 0 { - rawData = []byte(in.Data) - data = rawData - // We can assume that rawData is valid JSON - // without checking in.DataContentType == "application/json". - if err = json.Unmarshal(rawData, &v); err == nil { - data = v - // Handling of JSON base64 encoded or escaped in a string. - if str, ok := v.(string); ok { - // This is the path that will most likely succeed. - var ( - vString any - decoded []byte - ) - if err = json.Unmarshal([]byte(str), &vString); err == nil { - data = vString - } else if decoded, err = base64.StdEncoding.DecodeString(str); err == nil { - // Decoded Base64 encoded JSON does not seem to be in the spec - // but it is in existing unit tests so this handles that case. - var vBase64 any - if err = json.Unmarshal(decoded, &vBase64); err == nil { - data = vBase64 - } - } - } - } - } else if in.DataBase64 != "" { - rawData, err = base64.StdEncoding.DecodeString(in.DataBase64) - if err == nil { - data = rawData - if in.DataContentType == "application/json" { - if err = json.Unmarshal(rawData, &v); err == nil { - data = v - } - } - } - } - - return data, rawData -} - type BulkSubscribeMessageItem struct { EntryId string `json:"entryId"` //nolint:stylecheck Event interface{} `json:"event"` @@ -402,8 +406,8 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu data, rawData := in.getData() statuses = append(statuses, BulkSubscribeResponseEntry{ - entryId: in.ID, - status: SubscriptionResponseStatusSuccess, + EntryId: entry.EntryId, + Status: Success, }, ) @@ -424,30 +428,26 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu messages = append(messages, te) } resp := BulkSubscribeResponse{ - statuses: statuses, + Statuses: statuses, } if err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } w.Header().Add("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } w.WriteHeader(http.StatusOK) retry, err := fn(r.Context(), messages) if err == nil { - writeStatus(w, common.SubscriptionResponseStatusSuccess) + writeBulkStatus(w, resp) return } if retry { - writeStatus(w, common.SubscriptionResponseStatusRetry) + writeBulkStatus(w, resp) return } - writeStatus(w, common.SubscriptionResponseStatusDrop) + writeBulkStatus(w, resp) }))) return nil @@ -459,3 +459,9 @@ func writeStatus(w http.ResponseWriter, s string) { http.Error(w, err.Error(), PubSubHandlerRetryStatusCode) } } + +func writeBulkStatus(w http.ResponseWriter, s BulkSubscribeResponse) { + if err := json.NewEncoder(w).Encode(s); err != nil { + http.Error(w, err.Error(), PubSubHandlerRetryStatusCode) + } +} From cb26d03d82f711e8bd6f0be9d637c9d899e934d2 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sun, 3 Dec 2023 12:41:26 +0530 Subject: [PATCH 07/27] feat: bulksubscribe support Signed-off-by: sadath-12 --- examples/pubsub/sub/bulksub.go | 62 --------------------------- examples/pubsub/sub/sub.go | 5 +++ service/common/service.go | 5 +-- service/grpc/topic.go | 24 ++++++++--- service/http/topic.go | 61 ++++++++++---------------- service/internal/topicregistrar.go | 23 +++------- service/internal/topicsubscription.go | 17 ++++++++ 7 files changed, 70 insertions(+), 127 deletions(-) delete mode 100644 examples/pubsub/sub/bulksub.go diff --git a/examples/pubsub/sub/bulksub.go b/examples/pubsub/sub/bulksub.go deleted file mode 100644 index 58e2ab27..00000000 --- a/examples/pubsub/sub/bulksub.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "log" - "net/http" - - "github.com/dapr/go-sdk/service/common" - daprd "github.com/dapr/go-sdk/service/http" -) - - -var defaultSubscription = &common.Subscription{ - PubsubName: "messages", - Topic: "neworder", - Route: "/orders", -} - -var importantSubscription = &common.Subscription{ - PubsubName: "messages", - Topic: "neworder", - Route: "/important", - Match: `event.type == "important"`, - Priority: 1, -} - -func main() { - s := daprd.NewService(":8080") - - if err := s.AddBulkTopicEventHandler(defaultSubscription, bulkeventHandler,10,1000); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } - - if err := s.Start(); err != nil && err != http.ErrServerClosed { - log.Fatalf("error listenning: %v", err) - } -} - -func bulkeventHandler(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { - for _, event := range e { - log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", event.PubsubName, event.Topic, event.ID, event.Data) - } - return false, nil -} - -func importantEventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - log.Printf("important event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) - return false, nil -} diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index 9d103478..a43cae28 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -46,10 +46,15 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") + // for single event subscribing if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { log.Fatalf("error adding topic subscription: %v", err) } + // if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler,10,100); err != nil { + // log.Fatalf("error adding topic subscription: %v", err) + // } + if err := s.Start(); err != nil && err != http.ErrServerClosed { log.Fatalf("error listenning: %v", err) } diff --git a/service/common/service.go b/service/common/service.go index 34ecb53a..20194501 100644 --- a/service/common/service.go +++ b/service/common/service.go @@ -35,8 +35,8 @@ type Service interface { // AddTopicEventHandler appends provided event handler with its topic and optional metadata to the service. // Note, retries are only considered when there is an error. Lack of error is considered as a success AddTopicEventHandler(sub *Subscription, fn TopicEventHandler) error - - AddBulkTopicEventHandler(sub *Subscription, fn BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error + // AddBulkTopicEventHandler appends provided event handler with its topic along with configuring maxMessagesCount, maxAwaitDurationMs for bulk handling and optional metadata to the service. + AddBulkTopicEventHandler(sub *Subscription, fn TopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error // AddBindingInvocationHandler appends provided binding invocation handler with its name to the service. AddBindingInvocationHandler(name string, fn BindingInvocationHandler) error // RegisterActorImplFactory Register a new actor to actor runtime of go sdk @@ -55,7 +55,6 @@ type Service interface { type ( ServiceInvocationHandler func(ctx context.Context, in *InvocationEvent) (out *Content, err error) TopicEventHandler func(ctx context.Context, e *TopicEvent) (retry bool, err error) - BulkTopicEventHandler func(ctx context.Context, e []TopicEvent) (retry bool, err error) BindingInvocationHandler func(ctx context.Context, in *BindingEvent) (out []byte, err error) HealthCheckHandler func(context.Context) error ) diff --git a/service/grpc/topic.go b/service/grpc/topic.go index 078808aa..4a82dcb4 100644 --- a/service/grpc/topic.go +++ b/service/grpc/topic.go @@ -37,12 +37,12 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return s.topicRegistrar.AddSubscription(sub, fn) } -func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.BulkTopicEventHandler,maxMessagesCount,maxAwaitDurationMs int32) error { +func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.TopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { if sub == nil { return errors.New("subscription required") } - return s.topicRegistrar.AddBulkSubscription(sub, fn,maxMessagesCount,maxAwaitDurationMs) + return s.topicRegistrar.AddBulkSubscription(sub, fn, maxMessagesCount, maxAwaitDurationMs) } // ListTopicSubscriptions is called by Dapr to get the list of topics in a pubsub component the app wants to subscribe to. @@ -51,10 +51,11 @@ func (s *Server) ListTopicSubscriptions(ctx context.Context, in *empty.Empty) (* for _, v := range s.topicRegistrar { s := v.Subscription sub := &runtimev1pb.TopicSubscription{ - PubsubName: s.PubsubName, - Topic: s.Topic, - Metadata: s.Metadata, - Routes: convertRoutes(s.Routes), + PubsubName: s.PubsubName, + Topic: s.Topic, + Metadata: s.Metadata, + Routes: convertRoutes(s.Routes), + BulkSubscribe: convertBulkSubscribe(s.BulkSubscribe), } subs = append(subs, sub) } @@ -81,6 +82,17 @@ func convertRoutes(routes *internal.TopicRoutes) *runtimev1pb.TopicRoutes { } } +func convertBulkSubscribe(bulkSubscribe *internal.BulkSubscribe) *runtimev1pb.BulkSubscribeConfig { + if bulkSubscribe == nil { + return nil + } + return &runtimev1pb.BulkSubscribeConfig{ + Enabled: bulkSubscribe.Enabled, + MaxMessagesCount: bulkSubscribe.MaxMessagesCount, + MaxAwaitDurationMs: bulkSubscribe.MaxAwaitDurationMs, + } +} + // OnTopicEvent fired whenever a message has been published to a topic that has been subscribed. // Dapr sends published messages in a CloudEvents v1.0 envelope. func (s *Server) OnTopicEvent(ctx context.Context, in *runtimev1pb.TopicEventRequest) (*runtimev1pb.TopicEventResponse, error) { diff --git a/service/http/topic.go b/service/http/topic.go index 80d5b227..2c7c22e6 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -325,25 +325,7 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE return nil } -type BulkSubscribeMessageItem struct { - EntryId string `json:"entryId"` //nolint:stylecheck - Event interface{} `json:"event"` - Metadata map[string]string `json:"metadata"` - ContentType string `json:"contentType,omitempty"` -} - -type BulkSubscribeEnvelope struct { - ID string - Entries []BulkSubscribeMessageItem - Metadata map[string]string - Topic string - Pubsub string - EventType string -} - -// == APP == the item is {application/cloudevents+json 5e582fd2-f1c4-47bd-81b1-803fb7e86552 map[data:multi-pong datacontenttype:text/plain id:92fc5348-097d-4b9f-b093-7e6fcda77add pubsubname:messages source:pub specversion:1.0 time:2023-12-02T11:42:31+05:30 topic:neworder traceid:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 traceparent:00-a0373ef078e14e8db358c06e0ec18b27-d8dae6b5080eb9da-01 tracestate: type:com.dapr.event.sent] [] } - -func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { +func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.TopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { if sub == nil { return errors.New("subscription required") } @@ -376,7 +358,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu } // deserialize the event - var ins BulkSubscribeEnvelope + var ins internal.BulkSubscribeEnvelope if err = json.Unmarshal(body, &ins); err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return @@ -384,7 +366,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu statuses := make([]BulkSubscribeResponseEntry, 0, len(ins.Entries)) - var messages []common.TopicEvent for _, entry := range ins.Entries { itemJSON, err := json.Marshal(entry.Event) if err != nil { @@ -405,12 +386,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu } data, rawData := in.getData() - statuses = append(statuses, BulkSubscribeResponseEntry{ - EntryId: entry.EntryId, - Status: Success, - }, - ) - te := common.TopicEvent{ ID: in.ID, SpecVersion: in.SpecVersion, @@ -425,8 +400,28 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu Topic: in.Topic, } - messages = append(messages, te) + retry, err := fn(r.Context(), &te) + if err == nil { + statuses = append(statuses, BulkSubscribeResponseEntry{ + EntryId: entry.EntryId, + Status: Success, + }, + ) + } else if retry { + statuses = append(statuses, BulkSubscribeResponseEntry{ + EntryId: entry.EntryId, + Status: Retry, + }, + ) + } else { + statuses = append(statuses, BulkSubscribeResponseEntry{ + EntryId: entry.EntryId, + Status: Drop, + }, + ) + } } + resp := BulkSubscribeResponse{ Statuses: statuses, } @@ -437,16 +432,6 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.Bu w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - retry, err := fn(r.Context(), messages) - if err == nil { - writeBulkStatus(w, resp) - return - } - - if retry { - writeBulkStatus(w, resp) - return - } writeBulkStatus(w, resp) }))) diff --git a/service/internal/topicregistrar.go b/service/internal/topicregistrar.go index 0ccfe7b7..fe672af2 100644 --- a/service/internal/topicregistrar.go +++ b/service/internal/topicregistrar.go @@ -1,7 +1,6 @@ package internal import ( - "context" "errors" "fmt" @@ -17,9 +16,7 @@ type TopicRegistrar map[string]*TopicRegistration type TopicRegistration struct { Subscription *TopicSubscription DefaultHandler common.TopicEventHandler - DefaultBulkHandler common.BulkTopicEventHandler RouteHandlers map[string]common.TopicEventHandler - BulkRouteHandlers map[string]common.BulkTopicEventHandler } func (m TopicRegistrar) AddSubscription(sub *common.Subscription, fn common.TopicEventHandler) error { @@ -66,7 +63,7 @@ func (m TopicRegistrar) AddSubscription(sub *common.Subscription, fn common.Topi return nil } -func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common.BulkTopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { +func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common.TopicEventHandler, maxMessagesCount, maxAwaitDurationMs int32) error { if sub.Topic == "" { return errors.New("topic name required") } @@ -90,8 +87,6 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. Subscription: NewTopicSubscription(sub.PubsubName, sub.Topic), RouteHandlers: make(map[string]common.TopicEventHandler), DefaultHandler: nil, - BulkRouteHandlers: make(map[string]common.BulkTopicEventHandler), - DefaultBulkHandler: nil, } ts.Subscription.SetMetadata(sub.Metadata) m[key] = ts @@ -107,19 +102,11 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. if err := ts.Subscription.SetDefaultRoute(sub.Route); err != nil { return err } - ts.DefaultBulkHandler = func(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { - return false,nil - } - ts.DefaultHandler = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - return false,nil - } - } - ts.BulkRouteHandlers[sub.Route] = func(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { - return false,nil - } - ts.RouteHandlers[sub.Route] = func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - return false,nil + + ts.DefaultHandler = fn } + ts.RouteHandlers[sub.Route] = fn + return nil } diff --git a/service/internal/topicsubscription.go b/service/internal/topicsubscription.go index 7a37b2ef..837f377b 100644 --- a/service/internal/topicsubscription.go +++ b/service/internal/topicsubscription.go @@ -22,6 +22,23 @@ type TopicSubscription struct { BulkSubscribe *BulkSubscribe `json:"bulkSubscribe,omitempty"` } +type BulkSubscribeMessageItem struct { + EntryId string `json:"entryId"` //nolint:stylecheck + Event interface{} `json:"event"` + Metadata map[string]string `json:"metadata"` + ContentType string `json:"contentType,omitempty"` +} + +type BulkSubscribeEnvelope struct { + ID string + Entries []BulkSubscribeMessageItem + Metadata map[string]string + Topic string + Pubsub string + EventType string +} + + type BulkSubscribe struct { Enabled bool `json:"enabled"` MaxMessagesCount int32 `json:"maxMessagesCount,omitempty"` From 597c099706ed9c4e22d97b63ebcb4eeed897e474 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Mon, 4 Dec 2023 17:29:50 +0530 Subject: [PATCH 08/27] lint fix Signed-off-by: sadath-12 --- service/http/topic.go | 2 +- service/internal/topicregistrar.go | 8 ++++---- service/internal/topicsubscription.go | 15 +++++++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/service/http/topic.go b/service/http/topic.go index 2c7c22e6..e5954f17 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -45,7 +45,7 @@ const ( // Merge itemMap into topicEventJSON type topicEventJSON struct { // ID identifies the event. - ID string `json:"id"` // y + ID string `json:"id"` // The version of the CloudEvents specification. SpecVersion string `json:"specversion"` // The type of event related to the originating occurrence. diff --git a/service/internal/topicregistrar.go b/service/internal/topicregistrar.go index fe672af2..8294c6b2 100644 --- a/service/internal/topicregistrar.go +++ b/service/internal/topicregistrar.go @@ -14,9 +14,9 @@ type TopicRegistrar map[string]*TopicRegistration // TopicRegistration encapsulates the subscription and handlers. type TopicRegistration struct { - Subscription *TopicSubscription - DefaultHandler common.TopicEventHandler - RouteHandlers map[string]common.TopicEventHandler + Subscription *TopicSubscription + DefaultHandler common.TopicEventHandler + RouteHandlers map[string]common.TopicEventHandler } func (m TopicRegistrar) AddSubscription(sub *common.Subscription, fn common.TopicEventHandler) error { @@ -102,7 +102,7 @@ func (m TopicRegistrar) AddBulkSubscription(sub *common.Subscription, fn common. if err := ts.Subscription.SetDefaultRoute(sub.Route); err != nil { return err } - + ts.DefaultHandler = fn } diff --git a/service/internal/topicsubscription.go b/service/internal/topicsubscription.go index 837f377b..d8dc444c 100644 --- a/service/internal/topicsubscription.go +++ b/service/internal/topicsubscription.go @@ -30,15 +30,14 @@ type BulkSubscribeMessageItem struct { } type BulkSubscribeEnvelope struct { - ID string - Entries []BulkSubscribeMessageItem - Metadata map[string]string - Topic string - Pubsub string - EventType string + ID string `json:"id"` + Entries []BulkSubscribeMessageItem `json:"entries"` + Metadata map[string]string `json:"metadata"` + Topic string `json:"topic"` + Pubsub string `json:"pubsub"` + EventType string `json:"eventType"` } - type BulkSubscribe struct { Enabled bool `json:"enabled"` MaxMessagesCount int32 `json:"maxMessagesCount,omitempty"` @@ -85,7 +84,7 @@ func (s *TopicSubscription) SetMetadata(metadata map[string]string) error { return nil } -func (s *TopicSubscription) SetBulkSubscribe(maxMessagesCount,maxAwaitDurationMs int32) error { +func (s *TopicSubscription) SetBulkSubscribe(maxMessagesCount, maxAwaitDurationMs int32) error { if s.BulkSubscribe != nil { return fmt.Errorf("subscription for topic %s on pubsub %s already has bulkSubscribe set", s.Topic, s.PubsubName) } From 10d2c6705edefa9dd0d714138bb730ef59939dab Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Mon, 4 Dec 2023 17:43:26 +0530 Subject: [PATCH 09/27] lint fix Signed-off-by: sadath-12 --- service/http/topic.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/service/http/topic.go b/service/http/topic.go index e5954f17..48073e02 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -127,7 +127,7 @@ const ( type BulkSubscribeResponseEntry struct { // The id of the bulk subscribe entry - EntryId string `json:"entryId"` + EntryId string `json:"entryId"` //nolint:stylecheck // The response status of the bulk subscribe entry Status AppResponseStatus `json:"status"` @@ -367,14 +367,14 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.To statuses := make([]BulkSubscribeResponseEntry, 0, len(ins.Entries)) for _, entry := range ins.Entries { - itemJSON, err := json.Marshal(entry.Event) - if err != nil { - http.Error(w, err.Error(), PubSubHandlerDropStatusCode) + itemJSON, entryErr := json.Marshal(entry.Event) + if entryErr != nil { + http.Error(w, entryErr.Error(), PubSubHandlerDropStatusCode) return } var in topicEventJSON - if err := json.Unmarshal(itemJSON, &in); err != nil { + if err = json.Unmarshal(itemJSON, &in); err != nil { http.Error(w, err.Error(), PubSubHandlerDropStatusCode) return } @@ -400,8 +400,8 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.To Topic: in.Topic, } - retry, err := fn(r.Context(), &te) - if err == nil { + retry, funcErr := fn(r.Context(), &te) + if funcErr == nil { statuses = append(statuses, BulkSubscribeResponseEntry{ EntryId: entry.EntryId, Status: Success, From e2181306a60432b150c8f3f684001658a9736ef9 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Wed, 6 Dec 2023 12:35:55 +0530 Subject: [PATCH 10/27] bulksubscribe unit tests Signed-off-by: sadath-12 --- service/http/topic_test.go | 322 +++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) diff --git a/service/http/topic_test.go b/service/http/topic_test.go index 2d236383..b101147b 100644 --- a/service/http/topic_test.go +++ b/service/http/topic_test.go @@ -60,6 +60,18 @@ func TestEventNilHandler(t *testing.T) { assert.Errorf(t, err, "expected error adding event handler") } +func TestBulkEventNilHandler(t *testing.T) { + s := newServer("", nil) + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/", + Metadata: map[string]string{}, + } + err := s.AddBulkTopicEventHandler(sub, nil, 10, 1000) + assert.Errorf(t, err, "expected error adding event handler") +} + func TestEventHandler(t *testing.T) { data := `{ "specversion" : "1.0", @@ -146,6 +158,92 @@ func TestEventHandler(t *testing.T) { makeEventRequest(t, s, "/errors", data, http.StatusOK) } +func TestBulkEventHandler(t *testing.T) { + data := `{ + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" + }` + + s := newServer("", nil) + + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/", + Metadata: map[string]string{}, + } + err := s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) + assert.NoErrorf(t, err, "error adding event handler") + + sub2 := &common.Subscription{ + PubsubName: "messages", + Topic: "errors", + Route: "/errors", + Metadata: map[string]string{}, + } + err = s.AddBulkTopicEventHandler(sub2, testErrorTopicFunc, 10, 1000) + assert.NoErrorf(t, err, "error adding error event handler") + + sub3 := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/other", + Match: `event.type == "other"`, + Priority: 1, + } + err = s.AddBulkTopicEventHandler(sub3, testTopicFunc, 10, 1000) + assert.NoErrorf(t, err, "error adding error event handler") + + s.registerBaseHandler() + + req, err := http.NewRequest(http.MethodGet, "/dapr/subscribe", nil) + require.NoErrorf(t, err, "error creating request: %s", data) + req.Header.Set("Accept", "application/json") + rr := httptest.NewRecorder() + s.mux.ServeHTTP(rr, req) + resp := rr.Result() + defer resp.Body.Close() + payload, err := io.ReadAll(resp.Body) + require.NoErrorf(t, err, "error reading response") + var subs []internal.TopicSubscription + require.NoErrorf(t, json.Unmarshal(payload, &subs), "could not decode subscribe response") + + sort.Slice(subs, func(i, j int) bool { + less := strings.Compare(subs[i].PubsubName, subs[j].PubsubName) + if less != 0 { + return less < 0 + } + return strings.Compare(subs[i].Topic, subs[j].Topic) <= 0 + }) + + if assert.Lenf(t, subs, 2, "unexpected subscription count") { + assert.Equal(t, "messages", subs[0].PubsubName) + assert.Equal(t, "errors", subs[0].Topic) + + assert.Equal(t, "messages", subs[1].PubsubName) + assert.Equal(t, "test", subs[1].Topic) + assert.Equal(t, "", subs[1].Route) + assert.Equal(t, "/", subs[1].Routes.Default) + if assert.Lenf(t, subs[1].Routes.Rules, 1, "unexpected rules count") { + assert.Equal(t, `event.type == "other"`, subs[1].Routes.Rules[0].Match) + assert.Equal(t, "/other", subs[1].Routes.Rules[0].Path) + } + } + + makeEventRequest(t, s, "/", data, http.StatusOK) + makeEventRequest(t, s, "/", "", http.StatusSeeOther) + makeEventRequest(t, s, "/", "not JSON", http.StatusSeeOther) + makeEventRequest(t, s, "/errors", data, http.StatusOK) +} + func TestEventDataHandling(t *testing.T) { tests := map[string]struct { data string @@ -269,6 +367,176 @@ func TestEventDataHandling(t *testing.T) { } } +func TestBulkEventDataHandling(t *testing.T) { + tests := map[string]struct { + data string + results []interface{} + }{ + "JSON Values": { + data: `{ + "id": "unique_id", + "entries": [ + { + "entryId": "entry_id_1", + "event": { + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : { + "message":"hello" + } + }, + "metadata": { + "meta_key_1": "meta_value_1", + "meta_key_2": "meta_value_2" + }, + "contentType": "application/json" + }, + { + "entryId": "entry_id_2", + "event": { + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" + }, + "metadata": { + "meta_key_3": "meta_value_3", + "meta_key_4": "meta_value_4" + }, + "contentType": "application/json" + }, + { + "entryId": "entry_id_2", + "event": { + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data_base64" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" + }, + "metadata": { + "meta_key_3": "meta_value_3", + "meta_key_4": "meta_value_4" + }, + "contentType": "application/json" + }, + { + "entryId": "entry_id_3", + "event": { + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/octet-stream", + "data_base64" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" + }, + "metadata": { + "meta_key_3": "meta_value_3", + "meta_key_4": "meta_value_4" + }, + "contentType": "application/json" + }, + { + "entryId": "entry_id_4", + "event": { + "specversion" : "1.0", + "type" : "com.github.pull.create", + "source" : "https://github.com/cloudevents/spec/pull", + "subject" : "123", + "id" : "A234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : "{\"message\":\"hello\"}" + }, + "metadata": { + "meta_key_3": "meta_value_3", + "meta_key_4": "meta_value_4" + }, + "contentType": "application/json" + } + ], + "metadata": { + "envelope_key_1": "envelope_value_1", + "envelope_key_2": "envelope_value_2" + }, + "topic": "overall_topic", + "pubsub": "overall_pubsub", + "eventType": "overall_eventType" + }`, + results: []interface{}{ + map[string]interface{}{ + "message": "hello", + }, + map[string]interface{}{ + "message": "hello", + }, + map[string]interface{}{ + "message": "hello", + }, + []byte(`{"message":"hello"}`), + map[string]interface{}{ + "message": "hello", + }, + }, + }, + } + + s := newServer("", nil) + + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/test", + Metadata: map[string]string{}, + } + + recv := make(chan struct{}, 5) + var topicEvents []common.TopicEvent + handler := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + topicEvents = append(topicEvents, *e) + recv <- struct{}{} + return false, nil + } + err := s.AddBulkTopicEventHandler(sub, handler, 5, 1000) + assert.NoErrorf(t, err, "error adding event handler") + + s.registerBaseHandler() + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + makeEventRequest(t, s, "/test", tt.data, http.StatusOK) + <-recv + for i, tdata := range topicEvents { + assert.Equal(t, tt.results[i], tdata.Data) + } + }) + } +} + func TestHealthCheck(t *testing.T) { s := newServer("", nil) s.registerBaseHandler() @@ -375,6 +643,24 @@ func TestAddingInvalidEventHandlers(t *testing.T) { assert.Errorf(t, err, "expected error adding sub without route event handler") } +func TestAddingInvalidBulkEventHandlers(t *testing.T) { + s := newServer("", nil) + err := s.AddBulkTopicEventHandler(nil, testTopicFunc, 10, 1000) + assert.Errorf(t, err, "expected error adding no sub event handler") + + sub := &common.Subscription{Metadata: map[string]string{}} + err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) + assert.Errorf(t, err, "expected error adding empty sub event handler") + + sub.Topic = "test" + err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) + assert.Errorf(t, err, "expected error adding sub without component event handler") + + sub.PubsubName = "messages" + err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) + assert.Errorf(t, err, "expected error adding sub without route event handler") +} + func TestRawPayloadDecode(t *testing.T) { testRawTopicFunc := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { if e.DataContentType != "application/octet-stream" { @@ -410,3 +696,39 @@ func TestRawPayloadDecode(t *testing.T) { s.registerBaseHandler() makeEventRequest(t, s, "/raw", rawData, http.StatusOK) } + +func TestBulkRawPayloadDecode(t *testing.T) { + testRawTopicFunc := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + if e.DataContentType != "application/octet-stream" { + err = fmt.Errorf("invalid content type: %s", e.DataContentType) + } + if e.DataBase64 != "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" { + err = errors.New("error decode data_base64") + } + if err != nil { + assert.NoErrorf(t, err, "error rawPayload decode") + } + return + } + + const rawData = `{ + "datacontenttype" : "application/octet-stream", + "data_base64" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" + }` + + s := newServer("", nil) + + sub3 := &common.Subscription{ + PubsubName: "messages", + Topic: "testRaw", + Route: "/raw", + Metadata: map[string]string{ + "rawPayload": "true", + }, + } + err := s.AddBulkTopicEventHandler(sub3, testRawTopicFunc, 10, 1000) + assert.NoErrorf(t, err, "error adding raw event handler") + + s.registerBaseHandler() + makeEventRequest(t, s, "/raw", rawData, http.StatusOK) +} From 513138724a40b9975ac7d196410e387b67639728 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 7 Dec 2023 23:46:59 +0530 Subject: [PATCH 11/27] enhancements Signed-off-by: sadath-12 --- examples/pubsub/sub/sub.go | 24 +++++------ service/grpc/topic.go | 2 +- service/http/topic.go | 3 +- service/http/topic_test.go | 60 +++++++++++++-------------- service/internal/topicsubscription.go | 8 ++-- 5 files changed, 46 insertions(+), 51 deletions(-) diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index a43cae28..ec705495 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -46,14 +46,17 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") - // for single event subscribing - if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } + bulkSubscribe := true - // if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler,10,100); err != nil { - // log.Fatalf("error adding topic subscription: %v", err) - // } + if bulkSubscribe { + if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { + log.Fatalf("error adding topic subscription: %v", err) + } + } else { + if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { + log.Fatalf("error adding topic subscription: %v", err) + } + } if err := s.Start(); err != nil && err != http.ErrServerClosed { log.Fatalf("error listenning: %v", err) @@ -65,13 +68,6 @@ func eventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err er return false, nil } -func bulkeventHandler(ctx context.Context, e []common.TopicEvent) (retry bool, err error) { - for _, event := range e { - log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", event.PubsubName, event.Topic, event.ID, event.Data) - } - return false, nil -} - func importantEventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { log.Printf("important event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) return false, nil diff --git a/service/grpc/topic.go b/service/grpc/topic.go index 4a82dcb4..0fddfc38 100644 --- a/service/grpc/topic.go +++ b/service/grpc/topic.go @@ -82,7 +82,7 @@ func convertRoutes(routes *internal.TopicRoutes) *runtimev1pb.TopicRoutes { } } -func convertBulkSubscribe(bulkSubscribe *internal.BulkSubscribe) *runtimev1pb.BulkSubscribeConfig { +func convertBulkSubscribe(bulkSubscribe *internal.BulkSubscribeOptions) *runtimev1pb.BulkSubscribeConfig { if bulkSubscribe == nil { return nil } diff --git a/service/http/topic.go b/service/http/topic.go index 48073e02..ab407c95 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -42,7 +42,6 @@ const ( // topicEventJSON is identical to `common.TopicEvent` // except for it treats `data` as a json.RawMessage so it can // be used as bytes or interface{}. -// Merge itemMap into topicEventJSON type topicEventJSON struct { // ID identifies the event. ID string `json:"id"` @@ -250,7 +249,7 @@ func (s *Server) AddTopicEventHandler(sub *common.Subscription, fn common.TopicE // Route is only required for HTTP but should be specified for the // app protocol to be interchangeable. if sub.Route == "" { - return errors.New("handler route name") + return errors.New("missing route for this subscription") } if err := s.topicRegistrar.AddSubscription(sub, fn); err != nil { return err diff --git a/service/http/topic_test.go b/service/http/topic_test.go index b101147b..ad0b8a63 100644 --- a/service/http/topic_test.go +++ b/service/http/topic_test.go @@ -57,7 +57,7 @@ func TestEventNilHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error adding event handler") + assert.Error(t, err, "expected error adding event handler") } func TestBulkEventNilHandler(t *testing.T) { @@ -69,7 +69,7 @@ func TestBulkEventNilHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddBulkTopicEventHandler(sub, nil, 10, 1000) - assert.Errorf(t, err, "expected error adding event handler") + assert.Error(t, err, "expected error adding event handler") } func TestEventHandler(t *testing.T) { @@ -95,7 +95,7 @@ func TestEventHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddTopicEventHandler(sub, testTopicFunc) - assert.NoErrorf(t, err, "error adding event handler") + assert.NoError(t, err, "error adding event handler") sub2 := &common.Subscription{ PubsubName: "messages", @@ -104,7 +104,7 @@ func TestEventHandler(t *testing.T) { Metadata: map[string]string{}, } err = s.AddTopicEventHandler(sub2, testErrorTopicFunc) - assert.NoErrorf(t, err, "error adding error event handler") + assert.NoError(t, err, "error adding error event handler") sub3 := &common.Subscription{ PubsubName: "messages", @@ -114,7 +114,7 @@ func TestEventHandler(t *testing.T) { Priority: 1, } err = s.AddTopicEventHandler(sub3, testTopicFunc) - assert.NoErrorf(t, err, "error adding error event handler") + assert.NoError(t, err, "error adding error event handler") s.registerBaseHandler() @@ -126,9 +126,9 @@ func TestEventHandler(t *testing.T) { resp := rr.Result() defer resp.Body.Close() payload, err := io.ReadAll(resp.Body) - require.NoErrorf(t, err, "error reading response") + require.NoError(t, err, "error reading response") var subs []internal.TopicSubscription - require.NoErrorf(t, json.Unmarshal(payload, &subs), "could not decode subscribe response") + require.NoError(t, json.Unmarshal(payload, &subs), "could not decode subscribe response") sort.Slice(subs, func(i, j int) bool { less := strings.Compare(subs[i].PubsubName, subs[j].PubsubName) @@ -138,7 +138,7 @@ func TestEventHandler(t *testing.T) { return strings.Compare(subs[i].Topic, subs[j].Topic) <= 0 }) - if assert.Lenf(t, subs, 2, "unexpected subscription count") { + if assert.Len(t, subs, 2, "unexpected subscription count") { assert.Equal(t, "messages", subs[0].PubsubName) assert.Equal(t, "errors", subs[0].Topic) @@ -146,7 +146,7 @@ func TestEventHandler(t *testing.T) { assert.Equal(t, "test", subs[1].Topic) assert.Equal(t, "", subs[1].Route) assert.Equal(t, "/", subs[1].Routes.Default) - if assert.Lenf(t, subs[1].Routes.Rules, 1, "unexpected rules count") { + if assert.Len(t, subs[1].Routes.Rules, 1, "unexpected rules count") { assert.Equal(t, `event.type == "other"`, subs[1].Routes.Rules[0].Match) assert.Equal(t, "/other", subs[1].Routes.Rules[0].Path) } @@ -181,7 +181,7 @@ func TestBulkEventHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.NoErrorf(t, err, "error adding event handler") + assert.NoError(t, err, "error adding event handler") sub2 := &common.Subscription{ PubsubName: "messages", @@ -190,7 +190,7 @@ func TestBulkEventHandler(t *testing.T) { Metadata: map[string]string{}, } err = s.AddBulkTopicEventHandler(sub2, testErrorTopicFunc, 10, 1000) - assert.NoErrorf(t, err, "error adding error event handler") + assert.NoError(t, err, "error adding error event handler") sub3 := &common.Subscription{ PubsubName: "messages", @@ -200,7 +200,7 @@ func TestBulkEventHandler(t *testing.T) { Priority: 1, } err = s.AddBulkTopicEventHandler(sub3, testTopicFunc, 10, 1000) - assert.NoErrorf(t, err, "error adding error event handler") + assert.NoError(t, err, "error adding error event handler") s.registerBaseHandler() @@ -212,9 +212,9 @@ func TestBulkEventHandler(t *testing.T) { resp := rr.Result() defer resp.Body.Close() payload, err := io.ReadAll(resp.Body) - require.NoErrorf(t, err, "error reading response") + require.NoError(t, err, "error reading response") var subs []internal.TopicSubscription - require.NoErrorf(t, json.Unmarshal(payload, &subs), "could not decode subscribe response") + require.NoError(t, json.Unmarshal(payload, &subs), "could not decode subscribe response") sort.Slice(subs, func(i, j int) bool { less := strings.Compare(subs[i].PubsubName, subs[j].PubsubName) @@ -224,7 +224,7 @@ func TestBulkEventHandler(t *testing.T) { return strings.Compare(subs[i].Topic, subs[j].Topic) <= 0 }) - if assert.Lenf(t, subs, 2, "unexpected subscription count") { + if assert.Len(t, subs, 2, "unexpected subscription count") { assert.Equal(t, "messages", subs[0].PubsubName) assert.Equal(t, "errors", subs[0].Topic) @@ -232,7 +232,7 @@ func TestBulkEventHandler(t *testing.T) { assert.Equal(t, "test", subs[1].Topic) assert.Equal(t, "", subs[1].Route) assert.Equal(t, "/", subs[1].Routes.Default) - if assert.Lenf(t, subs[1].Routes.Rules, 1, "unexpected rules count") { + if assert.Len(t, subs[1].Routes.Rules, 1, "unexpected rules count") { assert.Equal(t, `event.type == "other"`, subs[1].Routes.Rules[0].Match) assert.Equal(t, "/other", subs[1].Routes.Rules[0].Path) } @@ -354,7 +354,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddTopicEventHandler(sub, handler) - assert.NoErrorf(t, err, "error adding event handler") + assert.NoError(t, err, "error adding event handler") s.registerBaseHandler() @@ -522,7 +522,7 @@ func TestBulkEventDataHandling(t *testing.T) { return false, nil } err := s.AddBulkTopicEventHandler(sub, handler, 5, 1000) - assert.NoErrorf(t, err, "error adding event handler") + assert.NoError(t, err, "error adding event handler") s.registerBaseHandler() @@ -628,37 +628,37 @@ func makeEventRequest(t *testing.T, s *Server, route, data string, expectedStatu func TestAddingInvalidEventHandlers(t *testing.T) { s := newServer("", nil) err := s.AddTopicEventHandler(nil, testTopicFunc) - assert.Errorf(t, err, "expected error adding no sub event handler") + assert.Error(t, err, "expected error adding no sub event handler") sub := &common.Subscription{Metadata: map[string]string{}} err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Errorf(t, err, "expected error adding empty sub event handler") + assert.Error(t, err, "expected error adding empty sub event handler") sub.Topic = "test" err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Errorf(t, err, "expected error adding sub without component event handler") + assert.Error(t, err, "expected error adding sub without component event handler") sub.PubsubName = "messages" err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Errorf(t, err, "expected error adding sub without route event handler") + assert.Error(t, err, "expected error adding sub without route event handler") } func TestAddingInvalidBulkEventHandlers(t *testing.T) { s := newServer("", nil) err := s.AddBulkTopicEventHandler(nil, testTopicFunc, 10, 1000) - assert.Errorf(t, err, "expected error adding no sub event handler") + assert.Error(t, err, "expected error adding no sub event handler") sub := &common.Subscription{Metadata: map[string]string{}} err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Errorf(t, err, "expected error adding empty sub event handler") + assert.Error(t, err, "expected error adding empty sub event handler") sub.Topic = "test" err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Errorf(t, err, "expected error adding sub without component event handler") + assert.Error(t, err, "expected error adding sub without component event handler") sub.PubsubName = "messages" err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Errorf(t, err, "expected error adding sub without route event handler") + assert.Error(t, err, "expected error adding sub without route event handler") } func TestRawPayloadDecode(t *testing.T) { @@ -670,7 +670,7 @@ func TestRawPayloadDecode(t *testing.T) { err = errors.New("error decode data_base64") } if err != nil { - assert.NoErrorf(t, err, "error rawPayload decode") + assert.NoError(t, err, "error rawPayload decode") } return } @@ -691,7 +691,7 @@ func TestRawPayloadDecode(t *testing.T) { }, } err := s.AddTopicEventHandler(sub3, testRawTopicFunc) - assert.NoErrorf(t, err, "error adding raw event handler") + assert.NoError(t, err, "error adding raw event handler") s.registerBaseHandler() makeEventRequest(t, s, "/raw", rawData, http.StatusOK) @@ -706,7 +706,7 @@ func TestBulkRawPayloadDecode(t *testing.T) { err = errors.New("error decode data_base64") } if err != nil { - assert.NoErrorf(t, err, "error rawPayload decode") + assert.NoError(t, err, "error rawPayload decode") } return } @@ -727,7 +727,7 @@ func TestBulkRawPayloadDecode(t *testing.T) { }, } err := s.AddBulkTopicEventHandler(sub3, testRawTopicFunc, 10, 1000) - assert.NoErrorf(t, err, "error adding raw event handler") + assert.NoError(t, err, "error adding raw event handler") s.registerBaseHandler() makeEventRequest(t, s, "/raw", rawData, http.StatusOK) diff --git a/service/internal/topicsubscription.go b/service/internal/topicsubscription.go index d8dc444c..c40a368b 100644 --- a/service/internal/topicsubscription.go +++ b/service/internal/topicsubscription.go @@ -18,8 +18,8 @@ type TopicSubscription struct { Routes *TopicRoutes `json:"routes,omitempty"` // Metadata is the subscription metadata. Metadata map[string]string `json:"metadata,omitempty"` - // bulksubsribe - BulkSubscribe *BulkSubscribe `json:"bulkSubscribe,omitempty"` + // options for customizing the bulksubscribe behaviour + BulkSubscribe *BulkSubscribeOptions `json:"bulkSubscribe,omitempty"` } type BulkSubscribeMessageItem struct { @@ -38,7 +38,7 @@ type BulkSubscribeEnvelope struct { EventType string `json:"eventType"` } -type BulkSubscribe struct { +type BulkSubscribeOptions struct { Enabled bool `json:"enabled"` MaxMessagesCount int32 `json:"maxMessagesCount,omitempty"` MaxAwaitDurationMs int32 `json:"maxAwaitDurationMs,omitempty"` @@ -88,7 +88,7 @@ func (s *TopicSubscription) SetBulkSubscribe(maxMessagesCount, maxAwaitDurationM if s.BulkSubscribe != nil { return fmt.Errorf("subscription for topic %s on pubsub %s already has bulkSubscribe set", s.Topic, s.PubsubName) } - s.BulkSubscribe = &BulkSubscribe{ + s.BulkSubscribe = &BulkSubscribeOptions{ Enabled: true, MaxMessagesCount: maxMessagesCount, MaxAwaitDurationMs: maxAwaitDurationMs, From 81b72ef50118cbf93a3860fbcfece300be163fed Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Fri, 8 Dec 2023 00:51:29 +0530 Subject: [PATCH 12/27] test: added unit tests Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 658 +++++++++++++++------ service/http/topic_test.go | 120 ++-- service/internal/topicregistrar_test.go | 74 ++- service/internal/topicsubscription_test.go | 8 + 4 files changed, 576 insertions(+), 284 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index a716cee9..762f2376 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -28,168 +28,233 @@ import ( func TestTopicErrors(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(nil, nil) - assert.Errorf(t, err, "expected error on nil sub") + assert.Errorf(t, err, "expected error on nil sub with AddTopicEventHandler") + + err = server.AddBulkTopicEventHandler(nil, nil, 0, 0) + assert.Errorf(t, err, "expected error on nil sub with AddBulkTopicEventHandler") sub := &common.Subscription{} err = server.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error on invalid sub") + assert.Errorf(t, err, "expected error on invalid sub with AddTopicEventHandler") + err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) + assert.Errorf(t, err, "expected error on invalid sub with AddBulkTopicEventHandler") sub.PubsubName = "messages" err = server.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error on sub without topic") + assert.Errorf(t, err, "expected error on sub without topic with AddTopicEventHandler") + sub.PubsubName = "messages" + err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) + assert.Errorf(t, err, "expected error on sub without topic with AddBulkTopicEventHandler") sub.Topic = "test" err = server.AddTopicEventHandler(sub, nil) assert.Errorf(t, err, "expected error on sub without handler") + err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) + assert.Errorf(t, err, "expected error on sub without handler") } func TestTopicSubscriptionList(t *testing.T) { - server := getTestServer() + t.Run("With single event handling", func(t *testing.T) { + server := getTestServer() - // Add default route. - sub1 := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - Route: "/test", - } - err := server.AddTopicEventHandler(sub1, eventHandler) - assert.Nil(t, err) - resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) - assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) - assert.Equal(t, "test", sub.Topic) - assert.Nil(t, sub.Routes) - } + sub1 := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/test", + } + err := server.AddTopicEventHandler(sub1, eventHandler) + assert.Nil(t, err) + resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) + assert.NoError(t, err) + assert.NotNil(t, resp) + if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + sub := resp.Subscriptions[0] + assert.Equal(t, "messages", sub.PubsubName) + assert.Equal(t, "test", sub.Topic) + assert.Nil(t, sub.Routes) + } - // Add routing rule. - sub2 := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - Route: "/other", - Match: `event.type == "other"`, - } - err = server.AddTopicEventHandler(sub2, eventHandler) - assert.Nil(t, err) - resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) - assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) - assert.Equal(t, "test", sub.Topic) - if assert.NotNil(t, sub.Routes) { - assert.Equal(t, "/test", sub.Routes.Default) - if assert.Len(t, sub.Routes.Rules, 1) { - rule := sub.Routes.Rules[0] - assert.Equal(t, "/other", rule.Path) - assert.Equal(t, `event.type == "other"`, rule.Match) + // Add routing rule. + sub2 := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/other", + Match: `event.type == "other"`, + } + err = server.AddTopicEventHandler(sub2, eventHandler) + assert.Nil(t, err) + resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) + assert.NoError(t, err) + assert.NotNil(t, resp) + if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + sub := resp.Subscriptions[0] + assert.Equal(t, "messages", sub.PubsubName) + assert.Equal(t, "test", sub.Topic) + if assert.NotNil(t, sub.Routes) { + assert.Equal(t, "/test", sub.Routes.Default) + if assert.Len(t, sub.Routes.Rules, 1) { + rule := sub.Routes.Rules[0] + assert.Equal(t, "/other", rule.Path) + assert.Equal(t, `event.type == "other"`, rule.Match) + } } } - } + }) + t.Run("With bulk event handling", func(t *testing.T) { + server := getTestServer() + sub1 := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/test", + } + err := server.AddBulkTopicEventHandler(sub1, eventHandler, 10, 1000) + assert.Nil(t, err) + resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) + assert.NoError(t, err) + assert.NotNil(t, resp) + if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + sub := resp.Subscriptions[0] + assert.Equal(t, "messages", sub.PubsubName) + assert.Equal(t, "test", sub.Topic) + assert.Nil(t, sub.Routes) + } + + // Add routing rule. + sub2 := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/other", + Match: `event.type == "other"`, + } + err = server.AddBulkTopicEventHandler(sub2, eventHandler, 10, 1000) + assert.Nil(t, err) + resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) + assert.NoError(t, err) + assert.NotNil(t, resp) + if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + sub := resp.Subscriptions[0] + assert.Equal(t, "messages", sub.PubsubName) + assert.Equal(t, "test", sub.Topic) + if assert.NotNil(t, sub.Routes) { + assert.Equal(t, "/test", sub.Routes.Default) + if assert.Len(t, sub.Routes.Rules, 1) { + rule := sub.Routes.Rules[0] + assert.Equal(t, "/other", rule.Path) + assert.Equal(t, `event.type == "other"`, rule.Match) + } + } + } + }) } // go test -timeout 30s ./service/grpc -count 1 -run ^TestTopic$ func TestTopic(t *testing.T) { - ctx := context.Background() + t.Run("With single event handling", func(t *testing.T) { + ctx := context.Background() - sub := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - } - server := getTestServer() + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + } + server := getTestServer() - err := server.AddTopicEventHandler(sub, eventHandler) - assert.Nil(t, err) + err := server.AddTopicEventHandler(sub, eventHandler) + assert.Nil(t, err) - startTestServer(server) + startTestServer(server) - t.Run("topic event without request", func(t *testing.T) { - _, err := server.OnTopicEvent(ctx, nil) - assert.Error(t, err) - }) + t.Run("topic event without request", func(t *testing.T) { + _, err := server.OnTopicEvent(ctx, nil) + assert.Error(t, err) + }) - t.Run("topic event for wrong topic", func(t *testing.T) { - in := &runtime.TopicEventRequest{ - Topic: "invalid", - } - _, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) - }) + t.Run("topic event for wrong topic", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Topic: "invalid", + } + _, err := server.OnTopicEvent(ctx, in) + assert.Error(t, err) + }) - t.Run("topic event for valid topic", func(t *testing.T) { - in := &runtime.TopicEventRequest{ - Id: "a123", - Source: "test", - Type: "test", - SpecVersion: "v1.0", - DataContentType: "text/plain", - Data: []byte("test"), - Topic: sub.Topic, - PubsubName: sub.PubsubName, - } - _, err := server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + t.Run("topic event for valid topic", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub.Topic, + PubsubName: sub.PubsubName, + } + _, err := server.OnTopicEvent(ctx, in) + assert.NoError(t, err) + }) + + stopTestServer(t, server) }) + t.Run("With bulk event handling", func(t *testing.T) { + ctx := context.Background() - stopTestServer(t, server) -} + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + } + server := getTestServer() -func TestTopicWithValidationDisabled(t *testing.T) { - ctx := context.Background() + err := server.AddBulkTopicEventHandler(sub, eventHandler, 10, 1000) + assert.Nil(t, err) - sub := &common.Subscription{ - PubsubName: "messages", - Topic: "*", - DisableTopicValidation: true, - } - server := getTestServer() + startTestServer(server) - err := server.AddTopicEventHandler(sub, eventHandler) - assert.Nil(t, err) + t.Run("topic event without request", func(t *testing.T) { + _, err := server.OnTopicEvent(ctx, nil) + assert.Error(t, err) + }) - startTestServer(server) + t.Run("topic event for wrong topic", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Topic: "invalid", + } + _, err := server.OnTopicEvent(ctx, in) + assert.Error(t, err) + }) - in := &runtime.TopicEventRequest{ - Id: "a123", - Source: "test", - Type: "test", - SpecVersion: "v1.0", - DataContentType: "text/plain", - Data: []byte("test"), - Topic: "test", - PubsubName: sub.PubsubName, - } + t.Run("topic event for valid topic", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub.Topic, + PubsubName: sub.PubsubName, + } + _, err := server.OnTopicEvent(ctx, in) + assert.NoError(t, err) + }) - _, err = server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + stopTestServer(t, server) + }) } -func TestTopicWithErrors(t *testing.T) { - ctx := context.Background() - - sub1 := &common.Subscription{ - PubsubName: "messages", - Topic: "test1", - } - - sub2 := &common.Subscription{ - PubsubName: "messages", - Topic: "test2", - } - server := getTestServer() +func TestTopicWithValidationDisabled(t *testing.T) { + t.Run("With single event handling", func(t *testing.T) { + ctx := context.Background() - err := server.AddTopicEventHandler(sub1, eventHandlerWithRetryError) - assert.Nil(t, err) + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "*", + DisableTopicValidation: true, + } + server := getTestServer() - err = server.AddTopicEventHandler(sub2, eventHandlerWithError) - assert.Nil(t, err) + err := server.AddTopicEventHandler(sub, eventHandler) + assert.Nil(t, err) - startTestServer(server) + startTestServer(server) - t.Run("topic event for retry error", func(t *testing.T) { in := &runtime.TopicEventRequest{ Id: "a123", Source: "test", @@ -197,15 +262,28 @@ func TestTopicWithErrors(t *testing.T) { SpecVersion: "v1.0", DataContentType: "text/plain", Data: []byte("test"), - Topic: sub1.Topic, - PubsubName: sub1.PubsubName, + Topic: "test", + PubsubName: sub.PubsubName, } - resp, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) + + _, err = server.OnTopicEvent(ctx, in) + assert.NoError(t, err) }) + t.Run("With bulk event handling", func(t *testing.T) { + ctx := context.Background() + + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "*", + DisableTopicValidation: true, + } + server := getTestServer() + + err := server.AddBulkTopicEventHandler(sub, eventHandler, 10, 1000) + assert.Nil(t, err) + + startTestServer(server) - t.Run("topic event for error", func(t *testing.T) { in := &runtime.TopicEventRequest{ Id: "a123", Source: "test", @@ -213,15 +291,128 @@ func TestTopicWithErrors(t *testing.T) { SpecVersion: "v1.0", DataContentType: "text/plain", Data: []byte("test"), - Topic: sub2.Topic, - PubsubName: sub2.PubsubName, + Topic: "test", + PubsubName: sub.PubsubName, } - resp, err := server.OnTopicEvent(ctx, in) + + _, err = server.OnTopicEvent(ctx, in) assert.NoError(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) }) +} + +func TestTopicWithErrors(t *testing.T) { + t.Run("With single event handling", func(t *testing.T) { + ctx := context.Background() + + sub1 := &common.Subscription{ + PubsubName: "messages", + Topic: "test1", + } + + sub2 := &common.Subscription{ + PubsubName: "messages", + Topic: "test2", + } + server := getTestServer() + + err := server.AddTopicEventHandler(sub1, eventHandlerWithRetryError) + assert.Nil(t, err) + + err = server.AddTopicEventHandler(sub2, eventHandlerWithError) + assert.Nil(t, err) + + startTestServer(server) + + t.Run("topic event for retry error", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub1.Topic, + PubsubName: sub1.PubsubName, + } + resp, err := server.OnTopicEvent(ctx, in) + assert.Error(t, err) + assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) + }) + + t.Run("topic event for error", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub2.Topic, + PubsubName: sub2.PubsubName, + } + resp, err := server.OnTopicEvent(ctx, in) + assert.NoError(t, err) + assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) + }) + + stopTestServer(t, server) + }) + t.Run("With bulk event handling", func(t *testing.T) { + ctx := context.Background() + + sub1 := &common.Subscription{ + PubsubName: "messages", + Topic: "test1", + } + + sub2 := &common.Subscription{ + PubsubName: "messages", + Topic: "test2", + } + server := getTestServer() + + err := server.AddBulkTopicEventHandler(sub1, eventHandlerWithRetryError, 10, 1000) + assert.Nil(t, err) + + err = server.AddBulkTopicEventHandler(sub2, eventHandlerWithError, 10, 1000) + assert.Nil(t, err) + + startTestServer(server) + + t.Run("topic event for retry error", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub1.Topic, + PubsubName: sub1.PubsubName, + } + resp, err := server.OnTopicEvent(ctx, in) + assert.Error(t, err) + assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) + }) + + t.Run("topic event for error", func(t *testing.T) { + in := &runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: "text/plain", + Data: []byte("test"), + Topic: sub2.Topic, + PubsubName: sub2.PubsubName, + } + resp, err := server.OnTopicEvent(ctx, in) + assert.NoError(t, err) + assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) + }) - stopTestServer(t, server) + stopTestServer(t, server) + }) } func eventHandler(ctx context.Context, event *common.TopicEvent) (retry bool, err error) { @@ -240,77 +431,152 @@ func eventHandlerWithError(ctx context.Context, event *common.TopicEvent) (retry } func TestEventDataHandling(t *testing.T) { - ctx := context.Background() - - tests := map[string]struct { - contentType string - data string - value interface{} - }{ - "JSON bytes": { - contentType: "application/json", - data: `{"message":"hello"}`, - value: map[string]interface{}{ - "message": "hello", + t.Run("With single event handling", func(t *testing.T) { + ctx := context.Background() + tests := map[string]struct { + contentType string + data string + value interface{} + }{ + "JSON bytes": { + contentType: "application/json", + data: `{"message":"hello"}`, + value: map[string]interface{}{ + "message": "hello", + }, }, - }, - "JSON entension media type bytes": { - contentType: "application/extension+json", - data: `{"message":"hello"}`, - value: map[string]interface{}{ - "message": "hello", + "JSON entension media type bytes": { + contentType: "application/extension+json", + data: `{"message":"hello"}`, + value: map[string]interface{}{ + "message": "hello", + }, }, - }, - "Test": { - contentType: "text/plain", - data: `message = hello`, - value: `message = hello`, - }, - "Other": { - contentType: "application/octet-stream", - data: `message = hello`, - value: []byte(`message = hello`), - }, - } + "Test": { + contentType: "text/plain", + data: `message = hello`, + value: `message = hello`, + }, + "Other": { + contentType: "application/octet-stream", + data: `message = hello`, + value: []byte(`message = hello`), + }, + } - s := getTestServer() + s := getTestServer() - sub := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - Route: "/test", - Metadata: map[string]string{}, - } + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/test", + Metadata: map[string]string{}, + } - recv := make(chan struct{}, 1) - var topicEvent *common.TopicEvent - handler := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - topicEvent = e - recv <- struct{}{} + recv := make(chan struct{}, 1) + var topicEvent *common.TopicEvent + handler := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + topicEvent = e + recv <- struct{}{} - return false, nil - } - err := s.AddTopicEventHandler(sub, handler) - assert.NoErrorf(t, err, "error adding event handler") + return false, nil + } + err := s.AddTopicEventHandler(sub, handler) + assert.NoErrorf(t, err, "error adding event handler") + + startTestServer(s) + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + in := runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: tt.contentType, + Data: []byte(tt.data), + Topic: sub.Topic, + PubsubName: sub.PubsubName, + } + + s.OnTopicEvent(ctx, &in) + <-recv + assert.Equal(t, tt.value, topicEvent.Data) + }) + } + }) + t.Run("With bulk event handling", func(t *testing.T) { + ctx := context.Background() + tests := map[string]struct { + contentType string + data string + value interface{} + }{ + "JSON bytes": { + contentType: "application/json", + data: `{"message":"hello"}`, + value: map[string]interface{}{ + "message": "hello", + }, + }, + "JSON entension media type bytes": { + contentType: "application/extension+json", + data: `{"message":"hello"}`, + value: map[string]interface{}{ + "message": "hello", + }, + }, + "Test": { + contentType: "text/plain", + data: `message = hello`, + value: `message = hello`, + }, + "Other": { + contentType: "application/octet-stream", + data: `message = hello`, + value: []byte(`message = hello`), + }, + } - startTestServer(s) + s := getTestServer() - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - in := runtime.TopicEventRequest{ - Id: "a123", - Source: "test", - Type: "test", - SpecVersion: "v1.0", - DataContentType: tt.contentType, - Data: []byte(tt.data), - Topic: sub.Topic, - PubsubName: sub.PubsubName, - } + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/test", + Metadata: map[string]string{}, + } - s.OnTopicEvent(ctx, &in) - <-recv - assert.Equal(t, tt.value, topicEvent.Data) - }) - } + recv := make(chan struct{}, 1) + var topicEvent *common.TopicEvent + handler := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { + topicEvent = e + recv <- struct{}{} + + return false, nil + } + err := s.AddBulkTopicEventHandler(sub, handler, 10, 1000) + assert.NoErrorf(t, err, "error adding event handler") + + startTestServer(s) + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + in := runtime.TopicEventRequest{ + Id: "a123", + Source: "test", + Type: "test", + SpecVersion: "v1.0", + DataContentType: tt.contentType, + Data: []byte(tt.data), + Topic: sub.Topic, + PubsubName: sub.PubsubName, + } + + s.OnTopicEvent(ctx, &in) + <-recv + assert.Equal(t, tt.value, topicEvent.Data) + }) + } + }) } diff --git a/service/http/topic_test.go b/service/http/topic_test.go index ad0b8a63..0fef1324 100644 --- a/service/http/topic_test.go +++ b/service/http/topic_test.go @@ -49,27 +49,28 @@ func testErrorTopicFunc(ctx context.Context, e *common.TopicEvent) (retry bool, } func TestEventNilHandler(t *testing.T) { - s := newServer("", nil) - sub := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - Route: "/", - Metadata: map[string]string{}, - } - err := s.AddTopicEventHandler(sub, nil) - assert.Error(t, err, "expected error adding event handler") -} - -func TestBulkEventNilHandler(t *testing.T) { - s := newServer("", nil) - sub := &common.Subscription{ - PubsubName: "messages", - Topic: "test", - Route: "/", - Metadata: map[string]string{}, - } - err := s.AddBulkTopicEventHandler(sub, nil, 10, 1000) - assert.Error(t, err, "expected error adding event handler") + t.Run("With single event handling", func(t *testing.T) { + s := newServer("", nil) + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/", + Metadata: map[string]string{}, + } + err := s.AddTopicEventHandler(sub, nil) + assert.Error(t, err, "expected error adding event handler") + }) + t.Run("With bulk event handling", func(t *testing.T) { + s := newServer("", nil) + sub := &common.Subscription{ + PubsubName: "messages", + Topic: "test", + Route: "/", + Metadata: map[string]string{}, + } + err := s.AddBulkTopicEventHandler(sub, nil, 10, 1000) + assert.Error(t, err, "expected error adding event handler") + }) } func TestEventHandler(t *testing.T) { @@ -680,55 +681,38 @@ func TestRawPayloadDecode(t *testing.T) { "data_base64" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" }` - s := newServer("", nil) - - sub3 := &common.Subscription{ - PubsubName: "messages", - Topic: "testRaw", - Route: "/raw", - Metadata: map[string]string{ - "rawPayload": "true", - }, - } - err := s.AddTopicEventHandler(sub3, testRawTopicFunc) - assert.NoError(t, err, "error adding raw event handler") - - s.registerBaseHandler() - makeEventRequest(t, s, "/raw", rawData, http.StatusOK) -} + t.Run("With single event handling", func(t *testing.T) { + s := newServer("", nil) -func TestBulkRawPayloadDecode(t *testing.T) { - testRawTopicFunc := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - if e.DataContentType != "application/octet-stream" { - err = fmt.Errorf("invalid content type: %s", e.DataContentType) - } - if e.DataBase64 != "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" { - err = errors.New("error decode data_base64") - } - if err != nil { - assert.NoError(t, err, "error rawPayload decode") + sub3 := &common.Subscription{ + PubsubName: "messages", + Topic: "testRaw", + Route: "/raw", + Metadata: map[string]string{ + "rawPayload": "true", + }, } - return - } - - const rawData = `{ - "datacontenttype" : "application/octet-stream", - "data_base64" : "eyJtZXNzYWdlIjoiaGVsbG8ifQ==" - }` + err := s.AddTopicEventHandler(sub3, testRawTopicFunc) + assert.NoError(t, err, "error adding raw event handler") - s := newServer("", nil) - - sub3 := &common.Subscription{ - PubsubName: "messages", - Topic: "testRaw", - Route: "/raw", - Metadata: map[string]string{ - "rawPayload": "true", - }, - } - err := s.AddBulkTopicEventHandler(sub3, testRawTopicFunc, 10, 1000) - assert.NoError(t, err, "error adding raw event handler") + s.registerBaseHandler() + makeEventRequest(t, s, "/raw", rawData, http.StatusOK) + }) + t.Run("With bulk event handling", func(t *testing.T) { + s := newServer("", nil) + + sub3 := &common.Subscription{ + PubsubName: "messages", + Topic: "testRaw", + Route: "/raw", + Metadata: map[string]string{ + "rawPayload": "true", + }, + } + err := s.AddBulkTopicEventHandler(sub3, testRawTopicFunc, 10, 1000) + assert.NoError(t, err, "error adding raw event handler") - s.registerBaseHandler() - makeEventRequest(t, s, "/raw", rawData, http.StatusOK) + s.registerBaseHandler() + makeEventRequest(t, s, "/raw", rawData, http.StatusOK) + }) } diff --git a/service/internal/topicregistrar_test.go b/service/internal/topicregistrar_test.go index 05a55458..5653bd73 100644 --- a/service/internal/topicregistrar_test.go +++ b/service/internal/topicregistrar_test.go @@ -60,37 +60,71 @@ func TestTopicRegistrarValidation(t *testing.T) { }, fn, "", }, } - for name, tt := range tests { - tt := tt // dereference loop var - t.Run(name, func(t *testing.T) { - m := internal.TopicRegistrar{} - if tt.err != "" { - assert.EqualError(t, m.AddSubscription(&tt.sub, tests[name].fn), tt.err) - } else { - assert.NoError(t, m.AddSubscription(&tt.sub, tt.fn)) - } - }) - } + t.Run("with subscription", func(t *testing.T) { + for name, tt := range tests { + tt := tt // dereference loop var + t.Run(name, func(t *testing.T) { + m := internal.TopicRegistrar{} + if tt.err != "" { + assert.EqualError(t, m.AddSubscription(&tt.sub, tests[name].fn), tt.err) + } else { + assert.NoError(t, m.AddSubscription(&tt.sub, tt.fn)) + } + }) + } + }) + t.Run("with bulk subscription", func(t *testing.T) { + for name, tt := range tests { + tt := tt // dereference loop var + t.Run(name, func(t *testing.T) { + m := internal.TopicRegistrar{} + if tt.err != "" { + assert.EqualError(t, m.AddBulkSubscription(&tt.sub, tests[name].fn, 10, 1000), tt.err) + } else { + assert.NoError(t, m.AddBulkSubscription(&tt.sub, tt.fn, 10, 1000)) + } + }) + } + }) } func TestTopicAddSubscriptionMetadata(t *testing.T) { handler := func(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { return false, nil } - topicRegistrar := internal.TopicRegistrar{} sub := &common.Subscription{ PubsubName: "pubsubname", Topic: "topic", Metadata: map[string]string{"key": "value"}, } - assert.NoError(t, topicRegistrar.AddSubscription(sub, handler)) + t.Run("with subscription", func(t *testing.T) { + topicRegistrar := internal.TopicRegistrar{} + assert.NoError(t, topicRegistrar.AddSubscription(sub, handler)) - actual := topicRegistrar["pubsubname-topic"].Subscription - expected := &internal.TopicSubscription{ - PubsubName: sub.PubsubName, - Topic: sub.Topic, - Metadata: sub.Metadata, - } - assert.Equal(t, expected, actual) + actual := topicRegistrar["pubsubname-topic"].Subscription + expected := &internal.TopicSubscription{ + PubsubName: sub.PubsubName, + Topic: sub.Topic, + Metadata: sub.Metadata, + } + assert.Equal(t, expected, actual) + }) + t.Run("with bulk subscription", func(t *testing.T) { + topicRegistrar := internal.TopicRegistrar{} + assert.NoError(t, topicRegistrar.AddBulkSubscription(sub, handler, 10, 1000)) + + actual := topicRegistrar["pubsubname-topic"].Subscription + expected := &internal.TopicSubscription{ + PubsubName: sub.PubsubName, + Topic: sub.Topic, + Metadata: sub.Metadata, + BulkSubscribe: &internal.BulkSubscribeOptions{ + Enabled: true, + MaxMessagesCount: 10, + MaxAwaitDurationMs: 1000, + }, + } + assert.Equal(t, expected, actual) + }) } diff --git a/service/internal/topicsubscription_test.go b/service/internal/topicsubscription_test.go index 7e30366c..fc62601d 100644 --- a/service/internal/topicsubscription_test.go +++ b/service/internal/topicsubscription_test.go @@ -69,4 +69,12 @@ func TestTopicSubscripiton(t *testing.T) { assert.Equal(t, `event.type == "100"`, sub.Routes.Rules[2].Match) } }) + + t.Run("enabling bulk subscription", func(t *testing.T) { + sub := internal.NewTopicSubscription("test", "mytopic") + assert.NoError(t, sub.SetBulkSubscribe(10, 1000)) + assert.Equal(t, true, sub.BulkSubscribe.Enabled) + assert.Equal(t, int32(10), sub.BulkSubscribe.MaxMessagesCount) + assert.Equal(t, int32(1000), sub.BulkSubscribe.MaxAwaitDurationMs) + }) } From 203d31c8fa6d1bee7e3607920c9baf781a55b679 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Fri, 8 Dec 2023 20:59:44 +0530 Subject: [PATCH 13/27] minor advances Signed-off-by: sadath-12 --- examples/pubsub/pub/pub.go | 7 ++++--- examples/pubsub/sub/sub.go | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/examples/pubsub/pub/pub.go b/examples/pubsub/pub/pub.go index 60bce409..cfd33882 100644 --- a/examples/pubsub/pub/pub.go +++ b/examples/pubsub/pub/pub.go @@ -23,8 +23,9 @@ import ( var ( // set the environment as instructions. - pubsubName = os.Getenv("DAPR_PUBSUB_NAME") - topicName = "neworder" + pubsubName = os.Getenv("DAPR_PUBSUB_NAME") + topicName = "neworder" + bulkTopicName = "newbulkorder" ) func main() { @@ -44,7 +45,7 @@ func main() { } // Publish multiple events - if res := client.PublishEvents(ctx, pubsubName, topicName, publishEventsData); res.Error != nil { + if res := client.PublishEvents(ctx, pubsubName, bulkTopicName, publishEventsData); res.Error != nil { panic(err) } diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index ec705495..b55059a2 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -35,6 +35,12 @@ var defaultSubscription = &common.Subscription{ Route: "/orders", } +var bulkSubscription = &common.Subscription{ + PubsubName: "bulkmessages", + Topic: "newbulkorder", + Route: "/bulkorders", +} + var importantSubscription = &common.Subscription{ PubsubName: "messages", Topic: "neworder", @@ -46,16 +52,12 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") - bulkSubscribe := true + if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { + log.Fatalf("error adding topic subscription: %v", err) + } - if bulkSubscribe { - if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } - } else { - if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } + if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { + log.Fatalf("error adding topic subscription: %v", err) } if err := s.Start(); err != nil && err != http.ErrServerClosed { From 36b78f5d03ff1d846d61d7562ab5fe6e543914f3 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Mon, 18 Dec 2023 23:41:45 +0530 Subject: [PATCH 14/27] minor adv Signed-off-by: sadath-12 --- service/http/topic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/http/topic.go b/service/http/topic.go index ab407c95..31aa55f5 100644 --- a/service/http/topic.go +++ b/service/http/topic.go @@ -331,7 +331,7 @@ func (s *Server) AddBulkTopicEventHandler(sub *common.Subscription, fn common.To // Route is only required for HTTP but should be specified for the // app protocol to be interchangeable. if sub.Route == "" { - return errors.New("handler route name") + return errors.New("missing route for bulk subscription") } if err := s.topicRegistrar.AddBulkSubscription(sub, fn, maxMessagesCount, maxAwaitDurationMs); err != nil { return err From 38bc15444ad16ae8b6dd8e7382ca87f7e97d559a Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Tue, 19 Dec 2023 21:57:33 +0530 Subject: [PATCH 15/27] require.Error Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 53 +++++++++++----------- service/http/topic_test.go | 48 ++++++++++---------- service/internal/topicregistrar_test.go | 13 +++--- service/internal/topicsubscription_test.go | 2 +- 4 files changed, 59 insertions(+), 57 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index 762f2376..d28e0999 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -20,6 +20,7 @@ import ( "github.com/golang/protobuf/ptypes/empty" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" runtime "github.com/dapr/dapr/pkg/proto/runtime/v1" "github.com/dapr/go-sdk/service/common" @@ -28,29 +29,29 @@ import ( func TestTopicErrors(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(nil, nil) - assert.Errorf(t, err, "expected error on nil sub with AddTopicEventHandler") + require.Errorf(t, err, "expected error on nil sub with AddTopicEventHandler") err = server.AddBulkTopicEventHandler(nil, nil, 0, 0) - assert.Errorf(t, err, "expected error on nil sub with AddBulkTopicEventHandler") + require.Errorf(t, err, "expected error on nil sub with AddBulkTopicEventHandler") sub := &common.Subscription{} err = server.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error on invalid sub with AddTopicEventHandler") + require.Errorf(t, err, "expected error on invalid sub with AddTopicEventHandler") err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - assert.Errorf(t, err, "expected error on invalid sub with AddBulkTopicEventHandler") + require.Errorf(t, err, "expected error on invalid sub with AddBulkTopicEventHandler") sub.PubsubName = "messages" err = server.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error on sub without topic with AddTopicEventHandler") + require.Errorf(t, err, "expected error on sub without topic with AddTopicEventHandler") sub.PubsubName = "messages" err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - assert.Errorf(t, err, "expected error on sub without topic with AddBulkTopicEventHandler") + require.Errorf(t, err, "expected error on sub without topic with AddBulkTopicEventHandler") sub.Topic = "test" err = server.AddTopicEventHandler(sub, nil) - assert.Errorf(t, err, "expected error on sub without handler") + require.Errorf(t, err, "expected error on sub without handler") err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - assert.Errorf(t, err, "expected error on sub without handler") + require.Errorf(t, err, "expected error on sub without handler") } func TestTopicSubscriptionList(t *testing.T) { @@ -65,7 +66,7 @@ func TestTopicSubscriptionList(t *testing.T) { err := server.AddTopicEventHandler(sub1, eventHandler) assert.Nil(t, err) resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, resp) if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] @@ -84,7 +85,7 @@ func TestTopicSubscriptionList(t *testing.T) { err = server.AddTopicEventHandler(sub2, eventHandler) assert.Nil(t, err) resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, resp) if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] @@ -110,7 +111,7 @@ func TestTopicSubscriptionList(t *testing.T) { err := server.AddBulkTopicEventHandler(sub1, eventHandler, 10, 1000) assert.Nil(t, err) resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, resp) if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] @@ -129,7 +130,7 @@ func TestTopicSubscriptionList(t *testing.T) { err = server.AddBulkTopicEventHandler(sub2, eventHandler, 10, 1000) assert.Nil(t, err) resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, resp) if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] @@ -165,7 +166,7 @@ func TestTopic(t *testing.T) { t.Run("topic event without request", func(t *testing.T) { _, err := server.OnTopicEvent(ctx, nil) - assert.Error(t, err) + require.Error(t, err) }) t.Run("topic event for wrong topic", func(t *testing.T) { @@ -173,7 +174,7 @@ func TestTopic(t *testing.T) { Topic: "invalid", } _, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) + require.Error(t, err) }) t.Run("topic event for valid topic", func(t *testing.T) { @@ -188,7 +189,7 @@ func TestTopic(t *testing.T) { PubsubName: sub.PubsubName, } _, err := server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) }) stopTestServer(t, server) @@ -209,7 +210,7 @@ func TestTopic(t *testing.T) { t.Run("topic event without request", func(t *testing.T) { _, err := server.OnTopicEvent(ctx, nil) - assert.Error(t, err) + require.Error(t, err) }) t.Run("topic event for wrong topic", func(t *testing.T) { @@ -217,7 +218,7 @@ func TestTopic(t *testing.T) { Topic: "invalid", } _, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) + require.Error(t, err) }) t.Run("topic event for valid topic", func(t *testing.T) { @@ -232,7 +233,7 @@ func TestTopic(t *testing.T) { PubsubName: sub.PubsubName, } _, err := server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) }) stopTestServer(t, server) @@ -267,7 +268,7 @@ func TestTopicWithValidationDisabled(t *testing.T) { } _, err = server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) }) t.Run("With bulk event handling", func(t *testing.T) { ctx := context.Background() @@ -296,7 +297,7 @@ func TestTopicWithValidationDisabled(t *testing.T) { } _, err = server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) }) } @@ -335,7 +336,7 @@ func TestTopicWithErrors(t *testing.T) { PubsubName: sub1.PubsubName, } resp, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) + require.Error(t, err) assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) }) @@ -351,7 +352,7 @@ func TestTopicWithErrors(t *testing.T) { PubsubName: sub2.PubsubName, } resp, err := server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) }) @@ -391,7 +392,7 @@ func TestTopicWithErrors(t *testing.T) { PubsubName: sub1.PubsubName, } resp, err := server.OnTopicEvent(ctx, in) - assert.Error(t, err) + require.Error(t, err) assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) }) @@ -407,7 +408,7 @@ func TestTopicWithErrors(t *testing.T) { PubsubName: sub2.PubsubName, } resp, err := server.OnTopicEvent(ctx, in) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) }) @@ -482,7 +483,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddTopicEventHandler(sub, handler) - assert.NoErrorf(t, err, "error adding event handler") + require.NoErrorf(t, err, "error adding event handler") startTestServer(s) @@ -556,7 +557,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddBulkTopicEventHandler(sub, handler, 10, 1000) - assert.NoErrorf(t, err, "error adding event handler") + require.NoErrorf(t, err, "error adding event handler") startTestServer(s) diff --git a/service/http/topic_test.go b/service/http/topic_test.go index 0fef1324..af4d5bbf 100644 --- a/service/http/topic_test.go +++ b/service/http/topic_test.go @@ -58,7 +58,7 @@ func TestEventNilHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddTopicEventHandler(sub, nil) - assert.Error(t, err, "expected error adding event handler") + require.Error(t, err, "expected error adding event handler") }) t.Run("With bulk event handling", func(t *testing.T) { s := newServer("", nil) @@ -69,7 +69,7 @@ func TestEventNilHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddBulkTopicEventHandler(sub, nil, 10, 1000) - assert.Error(t, err, "expected error adding event handler") + require.Error(t, err, "expected error adding event handler") }) } @@ -96,7 +96,7 @@ func TestEventHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddTopicEventHandler(sub, testTopicFunc) - assert.NoError(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") sub2 := &common.Subscription{ PubsubName: "messages", @@ -105,7 +105,7 @@ func TestEventHandler(t *testing.T) { Metadata: map[string]string{}, } err = s.AddTopicEventHandler(sub2, testErrorTopicFunc) - assert.NoError(t, err, "error adding error event handler") + require.NoError(t, err, "error adding error event handler") sub3 := &common.Subscription{ PubsubName: "messages", @@ -115,7 +115,7 @@ func TestEventHandler(t *testing.T) { Priority: 1, } err = s.AddTopicEventHandler(sub3, testTopicFunc) - assert.NoError(t, err, "error adding error event handler") + require.NoError(t, err, "error adding error event handler") s.registerBaseHandler() @@ -182,7 +182,7 @@ func TestBulkEventHandler(t *testing.T) { Metadata: map[string]string{}, } err := s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.NoError(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") sub2 := &common.Subscription{ PubsubName: "messages", @@ -191,7 +191,7 @@ func TestBulkEventHandler(t *testing.T) { Metadata: map[string]string{}, } err = s.AddBulkTopicEventHandler(sub2, testErrorTopicFunc, 10, 1000) - assert.NoError(t, err, "error adding error event handler") + require.NoError(t, err, "error adding error event handler") sub3 := &common.Subscription{ PubsubName: "messages", @@ -201,7 +201,7 @@ func TestBulkEventHandler(t *testing.T) { Priority: 1, } err = s.AddBulkTopicEventHandler(sub3, testTopicFunc, 10, 1000) - assert.NoError(t, err, "error adding error event handler") + require.NoError(t, err, "error adding error event handler") s.registerBaseHandler() @@ -355,7 +355,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddTopicEventHandler(sub, handler) - assert.NoError(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") s.registerBaseHandler() @@ -523,7 +523,7 @@ func TestBulkEventDataHandling(t *testing.T) { return false, nil } err := s.AddBulkTopicEventHandler(sub, handler, 5, 1000) - assert.NoError(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") s.registerBaseHandler() @@ -605,7 +605,7 @@ func makeRequest(t *testing.T, s *Server, route, data, method string, expectedSt t.Helper() req, err := http.NewRequest(method, route, strings.NewReader(data)) - assert.NoErrorf(t, err, "error creating request: %s", data) + require.NoErrorf(t, err, "error creating request: %s", data) testRequest(t, s, req, expectedStatusCode) } @@ -613,7 +613,7 @@ func makeRequestWithExpectedBody(t *testing.T, s *Server, route, data, method st t.Helper() req, err := http.NewRequest(method, route, strings.NewReader(data)) - assert.NoErrorf(t, err, "error creating request: %s", data) + require.NoErrorf(t, err, "error creating request: %s", data) testRequestWithResponseBody(t, s, req, expectedStatusCode, expectedBody) } @@ -621,7 +621,7 @@ func makeEventRequest(t *testing.T, s *Server, route, data string, expectedStatu t.Helper() req, err := http.NewRequest(http.MethodPost, route, strings.NewReader(data)) - assert.NoErrorf(t, err, "error creating request: %s", data) + require.NoErrorf(t, err, "error creating request: %s", data) req.Header.Set("Content-Type", "application/json") testRequest(t, s, req, expectedStatusCode) } @@ -629,37 +629,37 @@ func makeEventRequest(t *testing.T, s *Server, route, data string, expectedStatu func TestAddingInvalidEventHandlers(t *testing.T) { s := newServer("", nil) err := s.AddTopicEventHandler(nil, testTopicFunc) - assert.Error(t, err, "expected error adding no sub event handler") + require.Error(t, err, "expected error adding no sub event handler") sub := &common.Subscription{Metadata: map[string]string{}} err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Error(t, err, "expected error adding empty sub event handler") + require.Error(t, err, "expected error adding empty sub event handler") sub.Topic = "test" err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Error(t, err, "expected error adding sub without component event handler") + require.Error(t, err, "expected error adding sub without component event handler") sub.PubsubName = "messages" err = s.AddTopicEventHandler(sub, testTopicFunc) - assert.Error(t, err, "expected error adding sub without route event handler") + require.Error(t, err, "expected error adding sub without route event handler") } func TestAddingInvalidBulkEventHandlers(t *testing.T) { s := newServer("", nil) err := s.AddBulkTopicEventHandler(nil, testTopicFunc, 10, 1000) - assert.Error(t, err, "expected error adding no sub event handler") + require.Error(t, err, "expected error adding no sub event handler") sub := &common.Subscription{Metadata: map[string]string{}} err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Error(t, err, "expected error adding empty sub event handler") + require.Error(t, err, "expected error adding empty sub event handler") sub.Topic = "test" err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Error(t, err, "expected error adding sub without component event handler") + require.Error(t, err, "expected error adding sub without component event handler") sub.PubsubName = "messages" err = s.AddBulkTopicEventHandler(sub, testTopicFunc, 10, 1000) - assert.Error(t, err, "expected error adding sub without route event handler") + require.Error(t, err, "expected error adding sub without route event handler") } func TestRawPayloadDecode(t *testing.T) { @@ -671,7 +671,7 @@ func TestRawPayloadDecode(t *testing.T) { err = errors.New("error decode data_base64") } if err != nil { - assert.NoError(t, err, "error rawPayload decode") + require.NoError(t, err, "error rawPayload decode") } return } @@ -693,7 +693,7 @@ func TestRawPayloadDecode(t *testing.T) { }, } err := s.AddTopicEventHandler(sub3, testRawTopicFunc) - assert.NoError(t, err, "error adding raw event handler") + require.NoError(t, err, "error adding raw event handler") s.registerBaseHandler() makeEventRequest(t, s, "/raw", rawData, http.StatusOK) @@ -710,7 +710,7 @@ func TestRawPayloadDecode(t *testing.T) { }, } err := s.AddBulkTopicEventHandler(sub3, testRawTopicFunc, 10, 1000) - assert.NoError(t, err, "error adding raw event handler") + require.NoError(t, err, "error adding raw event handler") s.registerBaseHandler() makeEventRequest(t, s, "/raw", rawData, http.StatusOK) diff --git a/service/internal/topicregistrar_test.go b/service/internal/topicregistrar_test.go index 5653bd73..a2a3e517 100644 --- a/service/internal/topicregistrar_test.go +++ b/service/internal/topicregistrar_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dapr/go-sdk/service/common" "github.com/dapr/go-sdk/service/internal" @@ -66,9 +67,9 @@ func TestTopicRegistrarValidation(t *testing.T) { t.Run(name, func(t *testing.T) { m := internal.TopicRegistrar{} if tt.err != "" { - assert.EqualError(t, m.AddSubscription(&tt.sub, tests[name].fn), tt.err) + require.EqualError(t, m.AddSubscription(&tt.sub, tests[name].fn), tt.err) } else { - assert.NoError(t, m.AddSubscription(&tt.sub, tt.fn)) + require.NoError(t, m.AddSubscription(&tt.sub, tt.fn)) } }) } @@ -79,9 +80,9 @@ func TestTopicRegistrarValidation(t *testing.T) { t.Run(name, func(t *testing.T) { m := internal.TopicRegistrar{} if tt.err != "" { - assert.EqualError(t, m.AddBulkSubscription(&tt.sub, tests[name].fn, 10, 1000), tt.err) + require.EqualError(t, m.AddBulkSubscription(&tt.sub, tests[name].fn, 10, 1000), tt.err) } else { - assert.NoError(t, m.AddBulkSubscription(&tt.sub, tt.fn, 10, 1000)) + require.NoError(t, m.AddBulkSubscription(&tt.sub, tt.fn, 10, 1000)) } }) } @@ -100,7 +101,7 @@ func TestTopicAddSubscriptionMetadata(t *testing.T) { t.Run("with subscription", func(t *testing.T) { topicRegistrar := internal.TopicRegistrar{} - assert.NoError(t, topicRegistrar.AddSubscription(sub, handler)) + require.NoError(t, topicRegistrar.AddSubscription(sub, handler)) actual := topicRegistrar["pubsubname-topic"].Subscription expected := &internal.TopicSubscription{ @@ -112,7 +113,7 @@ func TestTopicAddSubscriptionMetadata(t *testing.T) { }) t.Run("with bulk subscription", func(t *testing.T) { topicRegistrar := internal.TopicRegistrar{} - assert.NoError(t, topicRegistrar.AddBulkSubscription(sub, handler, 10, 1000)) + require.NoError(t, topicRegistrar.AddBulkSubscription(sub, handler, 10, 1000)) actual := topicRegistrar["pubsubname-topic"].Subscription expected := &internal.TopicSubscription{ diff --git a/service/internal/topicsubscription_test.go b/service/internal/topicsubscription_test.go index 2850a8ee..d6c33d47 100644 --- a/service/internal/topicsubscription_test.go +++ b/service/internal/topicsubscription_test.go @@ -74,7 +74,7 @@ func TestTopicSubscripiton(t *testing.T) { t.Run("enabling bulk subscription", func(t *testing.T) { sub := internal.NewTopicSubscription("test", "mytopic") - assert.NoError(t, sub.SetBulkSubscribe(10, 1000)) + require.NoError(t, sub.SetBulkSubscribe(10, 1000)) assert.Equal(t, true, sub.BulkSubscribe.Enabled) assert.Equal(t, int32(10), sub.BulkSubscribe.MaxMessagesCount) assert.Equal(t, int32(1000), sub.BulkSubscribe.MaxAwaitDurationMs) From 35c37bccfc6867d18b41728a0a64167f15b04184 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 23 Dec 2023 12:49:44 +0530 Subject: [PATCH 16/27] Errorf--> Error appropriate change Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index d28e0999..f4e3a483 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -29,29 +29,29 @@ import ( func TestTopicErrors(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(nil, nil) - require.Errorf(t, err, "expected error on nil sub with AddTopicEventHandler") + require.Error(t, err, "expected error on nil sub with AddTopicEventHandler") err = server.AddBulkTopicEventHandler(nil, nil, 0, 0) - require.Errorf(t, err, "expected error on nil sub with AddBulkTopicEventHandler") + require.Error(t, err, "expected error on nil sub with AddBulkTopicEventHandler") sub := &common.Subscription{} err = server.AddTopicEventHandler(sub, nil) - require.Errorf(t, err, "expected error on invalid sub with AddTopicEventHandler") + require.Error(t, err, "expected error on invalid sub with AddTopicEventHandler") err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - require.Errorf(t, err, "expected error on invalid sub with AddBulkTopicEventHandler") + require.Error(t, err, "expected error on invalid sub with AddBulkTopicEventHandler") sub.PubsubName = "messages" err = server.AddTopicEventHandler(sub, nil) - require.Errorf(t, err, "expected error on sub without topic with AddTopicEventHandler") + require.Error(t, err, "expected error on sub without topic with AddTopicEventHandler") sub.PubsubName = "messages" err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - require.Errorf(t, err, "expected error on sub without topic with AddBulkTopicEventHandler") + require.Error(t, err, "expected error on sub without topic with AddBulkTopicEventHandler") sub.Topic = "test" err = server.AddTopicEventHandler(sub, nil) - require.Errorf(t, err, "expected error on sub without handler") + require.Error(t, err, "expected error on sub without handler") err = server.AddBulkTopicEventHandler(sub, nil, 0, 0) - require.Errorf(t, err, "expected error on sub without handler") + require.Error(t, err, "expected error on sub without handler") } func TestTopicSubscriptionList(t *testing.T) { @@ -64,7 +64,7 @@ func TestTopicSubscriptionList(t *testing.T) { Route: "/test", } err := server.AddTopicEventHandler(sub1, eventHandler) - assert.Nil(t, err) + require.NoError(t, err) resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) @@ -83,7 +83,7 @@ func TestTopicSubscriptionList(t *testing.T) { Match: `event.type == "other"`, } err = server.AddTopicEventHandler(sub2, eventHandler) - assert.Nil(t, err) + require.NoError(t, err) resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) @@ -109,7 +109,7 @@ func TestTopicSubscriptionList(t *testing.T) { Route: "/test", } err := server.AddBulkTopicEventHandler(sub1, eventHandler, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) @@ -128,7 +128,7 @@ func TestTopicSubscriptionList(t *testing.T) { Match: `event.type == "other"`, } err = server.AddBulkTopicEventHandler(sub2, eventHandler, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) @@ -160,7 +160,7 @@ func TestTopic(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(sub, eventHandler) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -204,7 +204,7 @@ func TestTopic(t *testing.T) { server := getTestServer() err := server.AddBulkTopicEventHandler(sub, eventHandler, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -252,7 +252,7 @@ func TestTopicWithValidationDisabled(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(sub, eventHandler) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -281,7 +281,7 @@ func TestTopicWithValidationDisabled(t *testing.T) { server := getTestServer() err := server.AddBulkTopicEventHandler(sub, eventHandler, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -317,10 +317,10 @@ func TestTopicWithErrors(t *testing.T) { server := getTestServer() err := server.AddTopicEventHandler(sub1, eventHandlerWithRetryError) - assert.Nil(t, err) + require.NoError(t, err) err = server.AddTopicEventHandler(sub2, eventHandlerWithError) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -373,10 +373,10 @@ func TestTopicWithErrors(t *testing.T) { server := getTestServer() err := server.AddBulkTopicEventHandler(sub1, eventHandlerWithRetryError, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) err = server.AddBulkTopicEventHandler(sub2, eventHandlerWithError, 10, 1000) - assert.Nil(t, err) + require.NoError(t, err) startTestServer(server) @@ -483,7 +483,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddTopicEventHandler(sub, handler) - require.NoErrorf(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") startTestServer(s) @@ -557,7 +557,7 @@ func TestEventDataHandling(t *testing.T) { return false, nil } err := s.AddBulkTopicEventHandler(sub, handler, 10, 1000) - require.NoErrorf(t, err, "error adding event handler") + require.NoError(t, err, "error adding event handler") startTestServer(s) From fb4138acee1498c8fe968b0b43ae29d6ba998468 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sat, 23 Dec 2023 12:52:30 +0530 Subject: [PATCH 17/27] assert.len Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index f4e3a483..1d77dee7 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -68,7 +68,7 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] assert.Equal(t, "messages", sub.PubsubName) assert.Equal(t, "test", sub.Topic) @@ -87,7 +87,7 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] assert.Equal(t, "messages", sub.PubsubName) assert.Equal(t, "test", sub.Topic) @@ -113,7 +113,7 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] assert.Equal(t, "messages", sub.PubsubName) assert.Equal(t, "test", sub.Topic) @@ -132,7 +132,7 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Lenf(t, resp.Subscriptions, 1, "expected 1 handlers") { + if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { sub := resp.Subscriptions[0] assert.Equal(t, "messages", sub.PubsubName) assert.Equal(t, "test", sub.Topic) From 0b41a28dffa700b035967b4bdd8bf52ccc488959 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Sun, 24 Dec 2023 02:10:32 +0530 Subject: [PATCH 18/27] updated readme.md Signed-off-by: sadath-12 --- examples/pubsub/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/pubsub/README.md b/examples/pubsub/README.md index d3e92315..980aa19e 100644 --- a/examples/pubsub/README.md +++ b/examples/pubsub/README.md @@ -23,6 +23,8 @@ background: true sleep: 15 --> +#### Note: sub/sub.go contains both AddTopicEventHandler (used for subscribe of messages) and AddBulkTopicEventHandler (used for bulksubscribe of messages) + ```bash dapr run --app-id sub \ --app-protocol http \ @@ -44,10 +46,12 @@ expected_stdout_lines: background: true sleep: 15 --> +#### Note: pub/pub.go contains both PublishEvents (used for publish of messages) and PublishEvent (used for bulkPublish of messages) ```bash export DAPR_PUBSUB_NAME=messages + dapr run --app-id pub \ --log-level debug \ --resources-path ./config \ From d05856f20d417c21de67c51f42ef77048f9cad6a Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 10:17:37 +0530 Subject: [PATCH 19/27] fix-lint Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 52 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index 1d77dee7..98da0a5e 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -68,11 +68,11 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) + if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { + sub := resp.GetSubscriptions()[0] + assert.Equal(t, "messages", sub.GetPubsubName()) assert.Equal(t, "test", sub.Topic) - assert.Nil(t, sub.Routes) + assert.Nil(t, sub.GetRoutes()) } // Add routing rule. @@ -87,16 +87,16 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) + if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { + sub := resp.GetSubscriptions()[0] + assert.Equal(t, "messages", sub.GetPubsubName()) assert.Equal(t, "test", sub.Topic) - if assert.NotNil(t, sub.Routes) { - assert.Equal(t, "/test", sub.Routes.Default) - if assert.Len(t, sub.Routes.Rules, 1) { - rule := sub.Routes.Rules[0] - assert.Equal(t, "/other", rule.Path) - assert.Equal(t, `event.type == "other"`, rule.Match) + if assert.NotNil(t, sub.GetRoutes()) { + assert.Equal(t, "/test", sub.GetRoutes().Default) + if assert.Len(t, sub.GetRoutes().Rules, 1) { + rule := sub.GetRoutes().Rules[0] + assert.Equal(t, "/other", rule.GetPath()) + assert.Equal(t, `event.type == "other"`, rule.GetMatch()) } } } @@ -113,11 +113,11 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err := server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) + if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { + sub := resp.GetSubscriptions()[0] + assert.Equal(t, "messages", sub.GetPubsubName()) assert.Equal(t, "test", sub.Topic) - assert.Nil(t, sub.Routes) + assert.Nil(t, sub.GetRoutes()) } // Add routing rule. @@ -132,16 +132,16 @@ func TestTopicSubscriptionList(t *testing.T) { resp, err = server.ListTopicSubscriptions(context.Background(), &empty.Empty{}) require.NoError(t, err) assert.NotNil(t, resp) - if assert.Len(t, resp.Subscriptions, 1, "expected 1 handlers") { - sub := resp.Subscriptions[0] - assert.Equal(t, "messages", sub.PubsubName) + if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { + sub := resp.GetSubscriptions()[0] + assert.Equal(t, "messages", sub.GetPubsubName()) assert.Equal(t, "test", sub.Topic) - if assert.NotNil(t, sub.Routes) { - assert.Equal(t, "/test", sub.Routes.Default) - if assert.Len(t, sub.Routes.Rules, 1) { - rule := sub.Routes.Rules[0] - assert.Equal(t, "/other", rule.Path) - assert.Equal(t, `event.type == "other"`, rule.Match) + if assert.NotNil(t, sub.GetRoutes()) { + assert.Equal(t, "/test", sub.GetRoutes().Default) + if assert.Len(t, sub.GetRoutes().Rules, 1) { + rule := sub.GetRoutes().Rules[0] + assert.Equal(t, "/other", rule.GetPath()) + assert.Equal(t, `event.type == "other"`, rule.GetMatch()) } } } From 92e7eef6a6506ce00306da09f436cf2aade9c96c Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 10:31:16 +0530 Subject: [PATCH 20/27] lintfix Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 20 ++++++++++---------- service/internal/topicsubscription_test.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index 98da0a5e..a78dc4f8 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -71,7 +71,7 @@ func TestTopicSubscriptionList(t *testing.T) { if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { sub := resp.GetSubscriptions()[0] assert.Equal(t, "messages", sub.GetPubsubName()) - assert.Equal(t, "test", sub.Topic) + assert.Equal(t, "test", sub.GetTopic()) assert.Nil(t, sub.GetRoutes()) } @@ -90,11 +90,11 @@ func TestTopicSubscriptionList(t *testing.T) { if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { sub := resp.GetSubscriptions()[0] assert.Equal(t, "messages", sub.GetPubsubName()) - assert.Equal(t, "test", sub.Topic) + assert.Equal(t, "test", sub.GetTopic()) if assert.NotNil(t, sub.GetRoutes()) { - assert.Equal(t, "/test", sub.GetRoutes().Default) - if assert.Len(t, sub.GetRoutes().Rules, 1) { - rule := sub.GetRoutes().Rules[0] + assert.Equal(t, "/test", sub.GetRoutes().GetDefault()) + if assert.Len(t, sub.GetRoutes().GetRules(), 1) { + rule := sub.GetRoutes().GetRules()[0] assert.Equal(t, "/other", rule.GetPath()) assert.Equal(t, `event.type == "other"`, rule.GetMatch()) } @@ -116,7 +116,7 @@ func TestTopicSubscriptionList(t *testing.T) { if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { sub := resp.GetSubscriptions()[0] assert.Equal(t, "messages", sub.GetPubsubName()) - assert.Equal(t, "test", sub.Topic) + assert.Equal(t, "test", sub.GetTopic()) assert.Nil(t, sub.GetRoutes()) } @@ -135,11 +135,11 @@ func TestTopicSubscriptionList(t *testing.T) { if assert.Len(t, resp.GetSubscriptions(), 1, "expected 1 handlers") { sub := resp.GetSubscriptions()[0] assert.Equal(t, "messages", sub.GetPubsubName()) - assert.Equal(t, "test", sub.Topic) + assert.Equal(t, "test", sub.GetTopic()) if assert.NotNil(t, sub.GetRoutes()) { - assert.Equal(t, "/test", sub.GetRoutes().Default) - if assert.Len(t, sub.GetRoutes().Rules, 1) { - rule := sub.GetRoutes().Rules[0] + assert.Equal(t, "/test", sub.GetRoutes().GetDefault()) + if assert.Len(t, sub.GetRoutes().GetRules(), 1) { + rule := sub.GetRoutes().GetRules()[0] assert.Equal(t, "/other", rule.GetPath()) assert.Equal(t, `event.type == "other"`, rule.GetMatch()) } diff --git a/service/internal/topicsubscription_test.go b/service/internal/topicsubscription_test.go index d6c33d47..45f17118 100644 --- a/service/internal/topicsubscription_test.go +++ b/service/internal/topicsubscription_test.go @@ -75,7 +75,7 @@ func TestTopicSubscripiton(t *testing.T) { t.Run("enabling bulk subscription", func(t *testing.T) { sub := internal.NewTopicSubscription("test", "mytopic") require.NoError(t, sub.SetBulkSubscribe(10, 1000)) - assert.Equal(t, true, sub.BulkSubscribe.Enabled) + assert.True(t, sub.BulkSubscribe.Enabled) assert.Equal(t, int32(10), sub.BulkSubscribe.MaxMessagesCount) assert.Equal(t, int32(1000), sub.BulkSubscribe.MaxAwaitDurationMs) }) From f315c9e21a2109981d4ef667193e91ade1fa9bba Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 10:36:08 +0530 Subject: [PATCH 21/27] fixlint Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index a78dc4f8..94187b47 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -337,7 +337,7 @@ func TestTopicWithErrors(t *testing.T) { } resp, err := server.OnTopicEvent(ctx, in) require.Error(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) + assert.Equal(t, runtime.TopicEventResponse_RETRY, resp.GetStatus()) }) t.Run("topic event for error", func(t *testing.T) { @@ -353,7 +353,7 @@ func TestTopicWithErrors(t *testing.T) { } resp, err := server.OnTopicEvent(ctx, in) require.NoError(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) + assert.Equal(t, runtime.TopicEventResponse_DROP, resp.GetStatus()) }) stopTestServer(t, server) @@ -393,7 +393,7 @@ func TestTopicWithErrors(t *testing.T) { } resp, err := server.OnTopicEvent(ctx, in) require.Error(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_RETRY) + assert.Equal(t, runtime.TopicEventResponse_RETRY, resp.GetStatus()) }) t.Run("topic event for error", func(t *testing.T) { @@ -409,7 +409,7 @@ func TestTopicWithErrors(t *testing.T) { } resp, err := server.OnTopicEvent(ctx, in) require.NoError(t, err) - assert.Equal(t, resp.GetStatus(), runtime.TopicEventResponse_DROP) + assert.Equal(t, runtime.TopicEventResponse_DROP, resp.GetStatus()) }) stopTestServer(t, server) From 0f2d9bb44ee415ef99e46e608f624d634b0e5132 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 10:46:06 +0530 Subject: [PATCH 22/27] correct sub.go Signed-off-by: sadath-12 --- examples/pubsub/sub/sub.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index b55059a2..32628c26 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -52,9 +52,10 @@ var importantSubscription = &common.Subscription{ func main() { s := daprd.NewService(":8080") - if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { - log.Fatalf("error adding topic subscription: %v", err) - } + // use this for bulk subscription + // if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { + // log.Fatalf("error adding topic subscription: %v", err) + // } if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { log.Fatalf("error adding topic subscription: %v", err) From f4a3b6557aceddd1e8e3df70c9d1b6cc28c4a4b5 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 11:03:38 +0530 Subject: [PATCH 23/27] examples correction Signed-off-by: sadath-12 --- examples/pubsub/README.md | 8 +++----- examples/pubsub/pub/pub.go | 8 ++++---- examples/pubsub/sub/sub.go | 24 +++++------------------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/examples/pubsub/README.md b/examples/pubsub/README.md index 980aa19e..4c6aca73 100644 --- a/examples/pubsub/README.md +++ b/examples/pubsub/README.md @@ -18,6 +18,7 @@ This folder contains two Go files that use the Go SDK to invoke the Dapr Pub/Sub name: Run Subscriber Server output_match_mode: substring expected_stdout_lines: + - 'event - PubsubName: messages, Topic: newbulkorder' - 'event - PubsubName: messages, Topic: neworder' background: true sleep: 15 @@ -49,9 +50,6 @@ sleep: 15 #### Note: pub/pub.go contains both PublishEvents (used for publish of messages) and PublishEvent (used for bulkPublish of messages) ```bash -export DAPR_PUBSUB_NAME=messages - - dapr run --app-id pub \ --log-level debug \ --resources-path ./config \ @@ -80,6 +78,6 @@ dapr stop --app-id sub ```shell == APP == 2023/03/29 21:36:07 event - PubsubName: messages, Topic: neworder, ID: 82427280-1c18-4fab-b901-c7e68d295d31, Data: ping -== APP == 2023/03/29 21:36:07 event - PubsubName: messages, Topic: neworder, ID: cc13829c-af77-4303-a4d7-55cdc0b0fa7d, Data: multi-pong -== APP == 2023/03/29 21:36:07 event - PubsubName: messages, Topic: neworder, ID: 0147f10a-d6c3-4b16-ad5a-6776956757dd, Data: multi-ping +== APP == 2023/03/29 21:36:07 event - PubsubName: messages, Topic: newbulkorder, ID: cc13829c-af77-4303-a4d7-55cdc0b0fa7d, Data: multi-pong +== APP == 2023/03/29 21:36:07 event - PubsubName: messages, Topic: newbulkorder, ID: 0147f10a-d6c3-4b16-ad5a-6776956757dd, Data: multi-ping ``` diff --git a/examples/pubsub/pub/pub.go b/examples/pubsub/pub/pub.go index cfd33882..089229f2 100644 --- a/examples/pubsub/pub/pub.go +++ b/examples/pubsub/pub/pub.go @@ -16,16 +16,16 @@ package main import ( "context" "fmt" - "os" dapr "github.com/dapr/go-sdk/client" ) var ( // set the environment as instructions. - pubsubName = os.Getenv("DAPR_PUBSUB_NAME") - topicName = "neworder" - bulkTopicName = "newbulkorder" + pubsubName = "messages" + bulkpubsubName = "bulkmessages" + topicName = "neworder" + bulkTopicName = "newbulkorder" ) func main() { diff --git a/examples/pubsub/sub/sub.go b/examples/pubsub/sub/sub.go index 32628c26..43e973d2 100644 --- a/examples/pubsub/sub/sub.go +++ b/examples/pubsub/sub/sub.go @@ -36,31 +36,22 @@ var defaultSubscription = &common.Subscription{ } var bulkSubscription = &common.Subscription{ - PubsubName: "bulkmessages", + PubsubName: "messages", Topic: "newbulkorder", Route: "/bulkorders", } -var importantSubscription = &common.Subscription{ - PubsubName: "messages", - Topic: "neworder", - Route: "/important", - Match: `event.type == "important"`, - Priority: 1, -} - func main() { s := daprd.NewService(":8080") - // use this for bulk subscription - // if err := s.AddBulkTopicEventHandler(defaultSubscription, eventHandler, 10, 100); err != nil { - // log.Fatalf("error adding topic subscription: %v", err) - // } - if err := s.AddTopicEventHandler(defaultSubscription, eventHandler); err != nil { log.Fatalf("error adding topic subscription: %v", err) } + if err := s.AddBulkTopicEventHandler(bulkSubscription, eventHandler, 10, 100); err != nil { + log.Fatalf("error adding topic subscription: %v", err) + } + if err := s.Start(); err != nil && err != http.ErrServerClosed { log.Fatalf("error listenning: %v", err) } @@ -70,8 +61,3 @@ func eventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err er log.Printf("event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) return false, nil } - -func importantEventHandler(ctx context.Context, e *common.TopicEvent) (retry bool, err error) { - log.Printf("important event - PubsubName: %s, Topic: %s, ID: %s, Data: %s", e.PubsubName, e.Topic, e.ID, e.Data) - return false, nil -} From 35770810bb15f5de6fde872bb7374bde49cee212 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Thu, 28 Dec 2023 13:18:08 +0530 Subject: [PATCH 24/27] markdown Signed-off-by: sadath-12 --- examples/pubsub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pubsub/README.md b/examples/pubsub/README.md index 4c6aca73..e4bc0e14 100644 --- a/examples/pubsub/README.md +++ b/examples/pubsub/README.md @@ -18,8 +18,8 @@ This folder contains two Go files that use the Go SDK to invoke the Dapr Pub/Sub name: Run Subscriber Server output_match_mode: substring expected_stdout_lines: - - 'event - PubsubName: messages, Topic: newbulkorder' - 'event - PubsubName: messages, Topic: neworder' + - 'event - PubsubName: messages, Topic: newbulkorder' background: true sleep: 15 --> From 5e1f64d5d7e6f3e075d78affb5ea42cc73837672 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Fri, 29 Dec 2023 00:44:04 +0530 Subject: [PATCH 25/27] remove unused Signed-off-by: sadath-12 --- examples/pubsub/pub/pub.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/pubsub/pub/pub.go b/examples/pubsub/pub/pub.go index 089229f2..e4928f45 100644 --- a/examples/pubsub/pub/pub.go +++ b/examples/pubsub/pub/pub.go @@ -22,10 +22,9 @@ import ( var ( // set the environment as instructions. - pubsubName = "messages" - bulkpubsubName = "bulkmessages" - topicName = "neworder" - bulkTopicName = "newbulkorder" + pubsubName = "messages" + topicName = "neworder" + bulkTopicName = "newbulkorder" ) func main() { From de720d95c2ae30251b59e43e4351c71cdc61f020 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Fri, 29 Dec 2023 23:37:33 +0530 Subject: [PATCH 26/27] remove alias Signed-off-by: sadath-12 --- examples/pubsub/README.md | 1 + service/grpc/topic_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/pubsub/README.md b/examples/pubsub/README.md index e4bc0e14..cfa522cb 100644 --- a/examples/pubsub/README.md +++ b/examples/pubsub/README.md @@ -20,6 +20,7 @@ output_match_mode: substring expected_stdout_lines: - 'event - PubsubName: messages, Topic: neworder' - 'event - PubsubName: messages, Topic: newbulkorder' + - 'event - PubsubName: messages, Topic: newbulkorder' background: true sleep: 15 --> diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index 94187b47..718638f5 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - runtime "github.com/dapr/dapr/pkg/proto/runtime/v1" + "github.com/dapr/dapr/pkg/proto/runtime/v1" "github.com/dapr/go-sdk/service/common" ) From 300e5ace0054d9856279a5ffd0191279c60c8265 Mon Sep 17 00:00:00 2001 From: sadath-12 Date: Tue, 14 May 2024 19:03:05 +0530 Subject: [PATCH 27/27] lint fix Signed-off-by: sadath-12 --- service/grpc/topic_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/service/grpc/topic_test.go b/service/grpc/topic_test.go index 1d82fe4d..95bbdec2 100644 --- a/service/grpc/topic_test.go +++ b/service/grpc/topic_test.go @@ -99,7 +99,6 @@ func TestTopicSubscriptionList(t *testing.T) { assert.Equal(t, "/other", rule.GetPath()) assert.Equal(t, `event.type == "other"`, rule.GetMatch()) } - } } }) @@ -266,7 +265,6 @@ func TestTopic(t *testing.T) { stopTestServer(t, server) }) - } func TestTopicWithValidationDisabled(t *testing.T) {