diff --git a/backend/api/event/event.go b/backend/api/event/event.go index 0df56eb41..b0a593285 100644 --- a/backend/api/event/event.go +++ b/backend/api/event/event.go @@ -51,6 +51,9 @@ const ( maxNavigationFromChars = 128 maxNavigationSourceChars = 128 maxScreenViewNameChars = 128 + maxUserDefAttrsCount = 100 + maxUserDefAttrsKeyChars = 256 + maxUserDefAttrsValsChars = 256 ) const TypeANR = "anr" @@ -365,38 +368,39 @@ type ScreenView struct { } type EventField struct { - ID uuid.UUID `json:"id"` - IPv4 net.IP `json:"inet_ipv4"` - IPv6 net.IP `json:"inet_ipv6"` - CountryCode string `json:"inet_country_code"` - AppID uuid.UUID `json:"app_id"` - SessionID uuid.UUID `json:"session_id" binding:"required"` - Timestamp time.Time `json:"timestamp" binding:"required"` - Type string `json:"type" binding:"required"` - UserTriggered bool `json:"user_triggered" binding:"required"` - Attribute Attribute `json:"attribute" binding:"required"` - Attachments []Attachment `json:"attachments" binding:"required"` - ANR *ANR `json:"anr,omitempty"` - Exception *Exception `json:"exception,omitempty"` - AppExit *AppExit `json:"app_exit,omitempty"` - LogString *LogString `json:"string,omitempty"` - GestureLongClick *GestureLongClick `json:"gesture_long_click,omitempty"` - GestureScroll *GestureScroll `json:"gesture_scroll,omitempty"` - GestureClick *GestureClick `json:"gesture_click,omitempty"` - LifecycleActivity *LifecycleActivity `json:"lifecycle_activity,omitempty"` - LifecycleFragment *LifecycleFragment `json:"lifecycle_fragment,omitempty"` - LifecycleApp *LifecycleApp `json:"lifecycle_app,omitempty"` - ColdLaunch *ColdLaunch `json:"cold_launch,omitempty"` - WarmLaunch *WarmLaunch `json:"warm_launch,omitempty"` - HotLaunch *HotLaunch `json:"hot_launch,omitempty"` - NetworkChange *NetworkChange `json:"network_change,omitempty"` - Http *Http `json:"http,omitempty"` - MemoryUsage *MemoryUsage `json:"memory_usage,omitempty"` - LowMemory *LowMemory `json:"low_memory,omitempty"` - TrimMemory *TrimMemory `json:"trim_memory,omitempty"` - CPUUsage *CPUUsage `json:"cpu_usage,omitempty"` - Navigation *Navigation `json:"navigation,omitempty"` - ScreenView *ScreenView `json:"screen_view,omitempty"` + ID uuid.UUID `json:"id"` + IPv4 net.IP `json:"inet_ipv4"` + IPv6 net.IP `json:"inet_ipv6"` + CountryCode string `json:"inet_country_code"` + AppID uuid.UUID `json:"app_id"` + SessionID uuid.UUID `json:"session_id" binding:"required"` + Timestamp time.Time `json:"timestamp" binding:"required"` + Type string `json:"type" binding:"required"` + UserTriggered bool `json:"user_triggered" binding:"required"` + Attribute Attribute `json:"attribute" binding:"required"` + UserDefinedAttribute UDAttribute `json:"user_defined_attribute" binding:"required"` + Attachments []Attachment `json:"attachments" binding:"required"` + ANR *ANR `json:"anr,omitempty"` + Exception *Exception `json:"exception,omitempty"` + AppExit *AppExit `json:"app_exit,omitempty"` + LogString *LogString `json:"string,omitempty"` + GestureLongClick *GestureLongClick `json:"gesture_long_click,omitempty"` + GestureScroll *GestureScroll `json:"gesture_scroll,omitempty"` + GestureClick *GestureClick `json:"gesture_click,omitempty"` + LifecycleActivity *LifecycleActivity `json:"lifecycle_activity,omitempty"` + LifecycleFragment *LifecycleFragment `json:"lifecycle_fragment,omitempty"` + LifecycleApp *LifecycleApp `json:"lifecycle_app,omitempty"` + ColdLaunch *ColdLaunch `json:"cold_launch,omitempty"` + WarmLaunch *WarmLaunch `json:"warm_launch,omitempty"` + HotLaunch *HotLaunch `json:"hot_launch,omitempty"` + NetworkChange *NetworkChange `json:"network_change,omitempty"` + Http *Http `json:"http,omitempty"` + MemoryUsage *MemoryUsage `json:"memory_usage,omitempty"` + LowMemory *LowMemory `json:"low_memory,omitempty"` + TrimMemory *TrimMemory `json:"trim_memory,omitempty"` + CPUUsage *CPUUsage `json:"cpu_usage,omitempty"` + Navigation *Navigation `json:"navigation,omitempty"` + ScreenView *ScreenView `json:"screen_view,omitempty"` } // Compute computes the most accurate cold launch timing diff --git a/backend/api/event/userdefattr.go b/backend/api/event/userdefattr.go new file mode 100644 index 000000000..cdf1b8208 --- /dev/null +++ b/backend/api/event/userdefattr.go @@ -0,0 +1,542 @@ +package event + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/leporo/sqlf" +) + +// attrKeyPattern defines the regular +// expression pattern for validating +// attribute keys. +const attrKeyPattern = "^[a-z0-9_-]+$" + +// maxAllowedDegree defines the maximum +// nesting depth a recursive nested +// expression is allowed. +const maxAllowedDegree = 2 + +const ( + AttrUnknown AttrType = iota + AttrString + AttrInt64 + AttrFloat64 + AttrBool +) + +type AttrType int + +// String returns a string representation of the +// attribute type. +func (a AttrType) String() string { + switch a { + default: + return "unknown" + case AttrString: + return "string" + case AttrInt64: + return "int64" + case AttrFloat64: + return "float64" + case AttrBool: + return "bool" + } +} + +func (a *AttrType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + s = strings.ToLower(s) + + switch s { + case AttrString.String(): + *a = AttrString + case AttrInt64.String(): + *a = AttrInt64 + case AttrFloat64.String(): + *a = AttrFloat64 + case AttrBool.String(): + *a = AttrBool + default: + return fmt.Errorf("invalid attribute type: %s", s) + } + + return nil +} + +const ( + OpEq AttrOp = iota + OpNeq + OpContains + OpStartsWith + OpGt + OpLt + OpGte + OpLte +) + +type AttrOp int + +// String returns a string representation of the +// attribute operator. +func (o AttrOp) String() string { + switch o { + default: + return "unknown" + case OpEq: + return "eq" + case OpNeq: + return "neq" + case OpContains: + return "contains" + case OpStartsWith: + return "startsWith" + case OpGt: + return "gt" + case OpLt: + return "lt" + case OpGte: + return "gte" + case OpLte: + return "lte" + } +} + +// Sql returns the SQL compatible +// operator to be consumed directly +// in a SQL query. +func (o AttrOp) Sql() string { + switch o { + default: + return "=" + case OpEq: + return "=" + case OpNeq: + return "!=" + case OpContains: + return "ilike" + case OpStartsWith: + return "ilike" + case OpGt: + return ">" + case OpLt: + return "<" + case OpGte: + return ">=" + case OpLte: + return "<=" + } +} + +// getValidOperators provides a slice of valid attribute +// operators for the requested attribute type. +func getValidOperators(ty AttrType) (ops []AttrOp) { + switch ty { + case AttrBool: + ops = []AttrOp{OpEq, OpNeq} + case AttrString: + ops = []AttrOp{OpEq, OpNeq, OpContains, OpStartsWith} + case AttrInt64, AttrFloat64: + ops = []AttrOp{OpEq, OpNeq, OpGt, OpGte, OpLt, OpLte} + } + return +} + +func (o *AttrOp) UnmarshalJSON(b []byte) error { + // extract the value inside double quotes + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + switch s { + case OpEq.String(): + *o = OpEq + case OpNeq.String(): + *o = OpNeq + case OpContains.String(): + *o = OpContains + case OpStartsWith.String(): + *o = OpStartsWith + case OpGt.String(): + *o = OpGt + case OpLt.String(): + *o = OpLt + case OpGte.String(): + *o = OpGte + case OpLte.String(): + *o = OpLte + } + + return nil +} + +// UDKeyType represents a single pair +// of user defined attribute key and +// its type. +type UDKeyType struct { + Key string `json:"key"` + Type string `json:"type"` +} + +// UDExpression represents a self-referential +// composite expression used for querying with +// user defined attributes. +type UDExpression struct { + And []UDExpression `json:"and,omitempty"` + Or []UDExpression `json:"or,omitempty"` + Cmp UDComparison `json:"cmp,omitempty"` +} + +// Degree computes the maximum nesting depth +// of a recursive user defined expression. +func (u *UDExpression) Degree() (degree int) { + type stackItem struct { + expr *UDExpression + degree int + } + + stack := []stackItem{{expr: u, degree: 1}} + maxDegree := 1 + + // process expressions until stack + // is empty + for len(stack) > 0 { + // pop last item from stack + n := len(stack) - 1 + current := stack[n] + stack = stack[:n] + + if current.degree > maxDegree { + maxDegree = current.degree + } + + // add all AND expressions to the stack + for _, andExpr := range current.expr.And { + stack = append(stack, stackItem{ + expr: &andExpr, + degree: current.degree + 1, + }) + } + + // add all OR expressions to the stack + for _, orExpr := range current.expr.Or { + stack = append(stack, stackItem{ + expr: &orExpr, + degree: current.degree + 1, + }) + } + } + + degree = maxDegree + + return +} + +// Empty returns true if the user defined +// expression does not contain any usable +// and meaningful values. +func (e *UDExpression) Empty() bool { + return len(e.And) == 0 && len(e.Or) == 0 && e.Cmp.Empty() +} + +// Left returns true if the expression does +// not contain any further `And` or `Or` +// expressions. +func (u *UDExpression) Leaf() bool { + return len(u.And) == 0 && len(u.Or) == 0 && !u.Cmp.Empty() +} + +// HasAnd returns true if expression contains +// at least 1 `And` expression. +func (u *UDExpression) HasAnd() bool { + return len(u.And) > 0 +} + +// HasOr returns true if expression contains +// at least 1 `Or` expression. +func (u *UDExpression) HasOr() bool { + return len(u.Or) > 0 +} + +// Validate validates the user defined expression +// and returns error if not valid. +func (u *UDExpression) Validate() (err error) { + // should not be empty + if u.Empty() { + err = errors.New("user defined expression cannot be empty") + } + + // should not contain `and` and `or` both + // expression at the same time + if u.HasAnd() && u.HasOr() { + err = errors.New("user defined expression cannot contain both `and` and `or` expressions") + } + + // should not exceed maximum allowed nesting + // level + if u.Degree() > maxAllowedDegree { + err = fmt.Errorf("user defined expression exceeds maximum allowed degree of %d. a degree is the maximum depth of nesting of the expression.", maxAllowedDegree) + } + + // validate each comparison expression + comparisons := u.getComparisons() + fmt.Println("comparisons", comparisons) + for _, cmp := range comparisons { + if err = cmp.Validate(); err != nil { + return + } + } + + return +} + +// Augment augments the sql statement with fully +// qualified `where` expressions. +func (u *UDExpression) Augment(stmt *sqlf.Stmt) { + if u.HasAnd() { + stmt.Where("(") + for i, andExpr := range u.And { + if i > 0 { + stmt.Where("and") + } + andExpr.Augment(stmt) + } + stmt.Where(")") + } else if u.HasOr() { + stmt.Where("(") + for i, orExpr := range u.Or { + if i > 0 { + stmt.Where("or") + } + orExpr.Augment(stmt) + } + stmt.Where(")") + } else if !u.Cmp.Empty() { + u.Cmp.Augment(stmt) + } +} + +// getComparisons extracts all comparison expressions +// from the a user defined expression. +func (u *UDExpression) getComparisons() (cmps []UDComparison) { + stack := []UDExpression{*u} + + // process expressions until stack is empty + for len(stack) > 0 { + n := len(stack) - 1 + current := stack[n] + stack = stack[:n] + + // if expression has a comparison, add it + if !current.Cmp.Empty() { + cmps = append(cmps, current.Cmp) + } + + // add all AND expressions + for i := len(current.And) - 1; i >= 0; i -= 1 { + stack = append(stack, current.And[i]) + } + + // add all OR expressions + for i := len(current.Or) - 1; i >= 0; i -= 1 { + stack = append(stack, current.Or[i]) + } + } + + return +} + +// UDComparison represnts comparison +// expressions. +type UDComparison struct { + Key string `json:"key"` + Type AttrType `json:"type"` + Op AttrOp `json:"op"` + Value string `json:"value"` +} + +// Empty returns true if comparison expression +// lacks a usable key. +func (c UDComparison) Empty() bool { + return c.Key == "" +} + +// Augment augments the sql statement with fully +// qualified sql expressions. +func (c *UDComparison) Augment(stmt *sqlf.Stmt) { + if c.Empty() { + fmt.Printf("warning: not augmenting user defined comparison to statement %q because comparison expression is empty", stmt.String()) + return + } + + opSymbol := c.Op.Sql() + + // TODO: Figure out type casting of non-string values + // TODO: Figure out escaping % characters for `ilike` operator + switch c.Op { + case OpEq, OpNeq, OpGt, OpGte, OpLt, OpLte: + stmt.Where(fmt.Sprintf("key = ? and type = ? and value %s ?", opSymbol), c.Key, c.Type.String(), c.Value) + case OpContains: + stmt.Where(fmt.Sprintf("key = ? and type = ? and value %s %%?%%", opSymbol), c.Key, c.Type.String(), c.Value) + case OpStartsWith: + stmt.Where(fmt.Sprintf("key = ? and type = ? and value %s ?%%", opSymbol), c.Key, c.Type.String(), c.Value) + } +} + +// Validate validates the user defined comparison +// and returns error if not valid. +func (c *UDComparison) Validate() (err error) { + validOps := getValidOperators(c.Type) + if !slices.Contains(validOps, c.Op) { + err = fmt.Errorf("%q operator is not valid for type: %q", c.Op.String(), c.Type) + } + return +} + +// UDAttribute represents user defined +// attributes. +// +// User Defined Attributes are used in +// various entities like event or span. +type UDAttribute struct { + rawAttrs map[string]any + keyTypes map[string]AttrType +} + +// MarshalJSON marshals UDAttribute type of user +// defined attributes to JSON. +func (u UDAttribute) MarshalJSON() (data []byte, err error) { + return json.Marshal(u.rawAttrs) +} + +// UnmarshalJSON unmarshalls bytes resembling user defined +// attributes to UDAttribute type. +func (u *UDAttribute) UnmarshalJSON(data []byte) (err error) { + return json.Unmarshal(data, &u.rawAttrs) +} + +// Validate validates user defined attributes bag. +func (u *UDAttribute) Validate() (err error) { + if u.rawAttrs == nil { + return errors.New("user defined attributes must not be empty") + } + + re := regexp.MustCompile(attrKeyPattern) + + count := len(u.rawAttrs) + + if count > maxUserDefAttrsCount { + return fmt.Errorf("user defined attributes must not exceed %d items", maxUserDefAttrsCount) + } + + if u.keyTypes == nil { + u.keyTypes = make(map[string]AttrType) + } + + for k, v := range u.rawAttrs { + if len(k) > maxUserDefAttrsKeyChars { + return fmt.Errorf("user defined attribute keys must not exceed %d characters", maxUserDefAttrsKeyChars) + } + + if !re.MatchString(k) { + return fmt.Errorf("user defined attribute keys must only contain lowercase alphabets, numbers, hyphens and underscores") + } + + switch value := v.(type) { + case string: + if len(value) > maxUserDefAttrsValsChars { + return fmt.Errorf("user defined attributes string values must not exceed %d characters", maxUserDefAttrsValsChars) + } + + u.keyTypes[k] = AttrString + continue + case bool: + u.keyTypes[k] = AttrBool + case float64: + if reflect.TypeOf(v).Kind() == reflect.Float64 { + if v == float64(int(value)) { + u.keyTypes[k] = AttrInt64 + } else { + u.keyTypes[k] = AttrFloat64 + } + } + continue + default: + return fmt.Errorf("user defined attribute values can be only string, number or boolean") + } + } + + return +} + +// HasItems returns true if user defined +// attribute is not empty. +func (u *UDAttribute) HasItems() bool { + return len(u.rawAttrs) > 0 +} + +func (u *UDAttribute) Parameterize() (attr map[string]any) { + attr = map[string]any{} + + val := "" + + for k, v := range u.rawAttrs { + switch v := v.(type) { + case bool: + val = strconv.FormatBool(v) + case float64: + val = strconv.FormatFloat(v, 'g', -1, 64) + case int64: + val = strconv.FormatInt(v, 10) + case string: + val = v + } + + attr[k] = fmt.Sprintf("('%s', '%s')", u.keyTypes[k].String(), val) + } + + return +} + +// GetUDAttrsOpMap provides a type wise list of operators +// for each type of user defined attribute keys. +func GetUDAttrsOpMap() (opmap map[string][]string) { + opmap = map[string][]string{ + AttrBool.String(): {OpEq.String(), OpNeq.String()}, + AttrString.String(): { + OpEq.String(), + OpNeq.String(), + OpContains.String(), + OpStartsWith.String(), + }, + AttrInt64.String(): { + OpEq.String(), + OpNeq.String(), + OpGt.String(), + OpLt.String(), + OpGte.String(), + OpLte.String(), + }, + AttrFloat64.String(): { + OpEq.String(), + OpNeq.String(), + OpGt.String(), + OpLt.String(), + OpGte.String(), + OpLte.String(), + }, + } + + return +} diff --git a/backend/api/event/userdefattr_test.go b/backend/api/event/userdefattr_test.go new file mode 100644 index 000000000..835f9c28a --- /dev/null +++ b/backend/api/event/userdefattr_test.go @@ -0,0 +1,541 @@ +package event + +import ( + "reflect" + "testing" + + "github.com/leporo/sqlf" +) + +func TestEmpty(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + cmpEmpty := UDComparison{} + + { + expected := true + got := exprEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty expression to be %v, got %v", expected, got) + } + } + + { + expected := false + got := exprNotEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty expression to be %v, got %v", expected, got) + } + } + + { + expected := true + got := cmpEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty comparison to be %v, got %v", expected, got) + } + } +} + +func TestLeaf(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + exprSingleAnd := UDExpression{ + And: []UDExpression{ + { + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + { + Cmp: UDComparison{ + Key: "ca_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + + { + expected := false + got := exprEmpty.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } + + { + expected := true + got := exprNotEmpty.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } + + { + expected := false + got := exprSingleAnd.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } +} + +func TestDegree(t *testing.T) { + exprEmpty := UDExpression{} + exprSingleAnd := UDExpression{ + And: []UDExpression{ + { + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + { + Cmp: UDComparison{ + Key: "ca_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + exprSingleCmp := UDExpression{ + Cmp: UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpGte, + Value: "1000", + }, + } + + { + expected := 1 + got := exprEmpty.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } + + { + expected := 2 + got := exprSingleAnd.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } + + { + expected := 1 + got := exprSingleCmp.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } +} + +func TestAugmentComparison(t *testing.T) { + cmpEmpty := UDComparison{} + cmpBoolEq := UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + } + cmpBoolNeq := UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpNeq, + Value: "false", + } + cmpInt64Gt := UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpGt, + Value: "1000", + } + cmpInt64Lt := UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpLt, + Value: "1000", + } + cmpFloat64Lt := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpLt, + Value: "1000.00", + } + cmpFloat64Lte := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpLte, + Value: "1000.00", + } + cmpFloat64Gt := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpGt, + Value: "999.99", + } + cmpFloat64Gte := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpGte, + Value: "1000.00", + } + cmpStringContains := UDComparison{ + Key: "preference", + Type: AttrString, + Op: OpContains, + Value: "spicy", + } + cmpStringStartsWith := UDComparison{ + Key: "name", + Type: AttrString, + Op: OpStartsWith, + Value: "Dr", + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpEmpty.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ?" + expectedArgs := []any{true} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %v args, got %v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpBoolEq.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value = ?" + expectedArgs := []any{true, "us_resident", "bool", "true"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpBoolNeq.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value != ?" + expectedArgs := []any{true, "us_resident", "bool", "false"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpInt64Gt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value > ?" + expectedArgs := []any{true, "credit_balance", "int64", "1000"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpInt64Lt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value < ?" + expectedArgs := []any{true, "credit_balance", "int64", "1000"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Lt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value < ?" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Lte.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value <= ?" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Gt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value > ?" + expectedArgs := []any{true, "invested_amount", "float64", "999.99"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Gte.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value >= ?" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpStringContains.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value ilike %?%" + expectedArgs := []any{true, "preference", "string", "spicy"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpStringStartsWith.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value ilike ?%" + expectedArgs := []any{true, "name", "string", "Dr"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } +} + +func TestAugmentExpression(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprEmpty.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ?" + gotStmt := stmt.String() + expectedArgs := []any{true} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprNotEmpty.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND key = ? and type = ? and value = ?" + gotStmt := stmt.String() + expectedArgs := []any{true, "us_resident", "bool", "true"} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } +} diff --git a/backend/api/filter/appfilter.go b/backend/api/filter/appfilter.go index 5721e2671..57cdec4a3 100644 --- a/backend/api/filter/appfilter.go +++ b/backend/api/filter/appfilter.go @@ -1,6 +1,7 @@ package filter import ( + "backend/api/event" "backend/api/pairs" "backend/api/server" "context" @@ -29,6 +30,15 @@ const MaxPaginationLimit = 1000 // as default for paginating items. const DefaultPaginationLimit = 10 +// Operator represents a comparison operator +// like `eq` for `equal` or `gte` for `greater +// than or equal` used for filtering various +// entities. +type Operator struct { + Code string + Type event.AttrType +} + // AppFilter represents various app filtering // operations and its parameters to query app's // issue journey map, metrics, exceptions and @@ -108,6 +118,19 @@ type AppFilter struct { // consider ANR events. ANR bool `form:"anr"` + // UDAttrKeys indicates a request to receive + // list of user defined attribute key & + // types. + UDAttrKeys bool `form:"ud_attr_keys"` + + // UDExpressionRaw contains the raw user defined + // attribute expression as string. + UDExpressionRaw string `form:"ud_expression"` + + // UDExpression contains the parsed user defined + // attribute expression. + UDExpression *event.UDExpression + // KeyID is the anchor point for keyset // pagination. KeyID string `form:"key_id"` @@ -138,17 +161,18 @@ type AppFilter struct { // used in filtering operations of app's issue journey map, // metrics, exceptions and ANRs. type FilterList struct { - Versions []string `json:"versions"` - VersionCodes []string `json:"version_codes"` - OsNames []string `json:"os_names"` - OsVersions []string `json:"os_versions"` - Countries []string `json:"countries"` - NetworkProviders []string `json:"network_providers"` - NetworkTypes []string `json:"network_types"` - NetworkGenerations []string `json:"network_generations"` - DeviceLocales []string `json:"locales"` - DeviceManufacturers []string `json:"device_manufacturers"` - DeviceNames []string `json:"device_names"` + Versions []string `json:"versions"` + VersionCodes []string `json:"version_codes"` + OsNames []string `json:"os_names"` + OsVersions []string `json:"os_versions"` + Countries []string `json:"countries"` + NetworkProviders []string `json:"network_providers"` + NetworkTypes []string `json:"network_types"` + NetworkGenerations []string `json:"network_generations"` + DeviceLocales []string `json:"locales"` + DeviceManufacturers []string `json:"device_manufacturers"` + DeviceNames []string `json:"device_names"` + UDKeyTypes []event.UDKeyType `json:"ud_keytypes"` } // Hash generates an MD5 hash of the FilterList struct. @@ -209,6 +233,16 @@ func (af *AppFilter) Validate() error { return fmt.Errorf("`limit` cannot be more than %d", MaxPaginationLimit) } + if af.UDExpressionRaw != "" { + if err := af.parseUDExpression(); err != nil { + return fmt.Errorf("`ud_expresssion` is invalid") + } + + if err := af.UDExpression.Validate(); err != nil { + return fmt.Errorf("`ud_expression` is invalid. %s", err.Error()) + } + } + return nil } @@ -255,6 +289,13 @@ func (af *AppFilter) OSVersionPairs() (osVersions *pairs.Pairs[string, string], // Expand expands comma separated fields to slice // of strings func (af *AppFilter) Expand() { + // expand user defined attribute expression + // if af.UDExpressionRaw != "" { + // if err := af.parseUDExpression(); err != nil { + // fmt.Println(err) + // } + // } + filters, err := GetFiltersFromFilterShortCode(af.FilterShortCode, af.AppID) if err != nil { return @@ -295,6 +336,16 @@ func (af *AppFilter) Expand() { } } +// parseUDExpression parses the raw user defined +// attribute expression value. +func (af *AppFilter) parseUDExpression() (err error) { + af.UDExpression = &event.UDExpression{} + if err = json.Unmarshal([]byte(af.UDExpressionRaw), af.UDExpression); err != nil { + return + } + return +} + // HasTimeRange checks if the time values are // appropriately set. func (af *AppFilter) HasTimeRange() bool { @@ -470,6 +521,21 @@ func (af *AppFilter) GetGenericFilters(ctx context.Context, fl *FilterList) erro return nil } +// GetUserDefinedAttrKeys provides list of unique user defined +// attribute keys and its data types by matching a subset of +// filters. +func (af *AppFilter) GetUserDefinedAttrKeys(ctx context.Context, fl *FilterList) (err error) { + if af.UDAttrKeys { + keytypes, err := af.getUDAttrKeys(ctx) + if err != nil { + return err + } + fl.UDKeyTypes = append(fl.UDKeyTypes, keytypes...) + } + + return +} + // hasKeyID checks if key id is a valid non-empty // value. func (af *AppFilter) hasKeyID() bool { @@ -835,6 +901,44 @@ func (af *AppFilter) getDeviceNames(ctx context.Context) (deviceNames []string, return } +// getUDAttrKeys finds distinct user defined attribute +// key and its types. +func (af *AppFilter) getUDAttrKeys(ctx context.Context) (keytypes []event.UDKeyType, err error) { + stmt := sqlf.From("user_def_attrs"). + Select("distinct key"). + Select("toString(type) type"). + Clause("prewhere app_id = toUUID(?) and end_of_month <= ?", af.AppID, af.To). + OrderBy("key") + + defer stmt.Close() + + if af.Crash { + stmt.Where("exception = true") + } + + if af.ANR { + stmt.Where("anr = true") + } + + rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) + if err != nil { + return + } + + for rows.Next() { + var keytype event.UDKeyType + if err = rows.Scan(&keytype.Key, &keytype.Type); err != nil { + return + } + + keytypes = append(keytypes, keytype) + } + + err = rows.Err() + + return +} + // GetExcludedVersions computes list of app version // and version codes that are excluded from app filter. func (af *AppFilter) GetExcludedVersions(ctx context.Context) (versions Versions, err error) { diff --git a/backend/api/filter/appfilter_test.go b/backend/api/filter/appfilter_test.go new file mode 100644 index 000000000..12fe6547d --- /dev/null +++ b/backend/api/filter/appfilter_test.go @@ -0,0 +1,87 @@ +package filter + +import ( + "backend/api/event" + "testing" +) + +func TestParseRawUDExpression(t *testing.T) { + afOne := &AppFilter{ + UDExpressionRaw: `{"and":[{"cmp":{"key":"paid_user","type":"bool","op":"eq","value":"true"}},{"cmp":{"key":"credit_balance","type":"int64","op":"gte","value":"1000"}}]}`, + } + + afOne.parseUDExpression() + + // assert expression exists + if afOne.UDExpression == nil { + t.Error("Expected parsed user defined expression, got nil") + } + + // assert count of expressions + { + expectedAndLen := 2 + gotAndLen := len(afOne.UDExpression.And) + + if expectedAndLen != gotAndLen { + t.Errorf("Expected %d And expressions, got %d", expectedAndLen, gotAndLen) + } + } + + // assert expression structure and type + // for item 0 + { + expectedKeyName := "paid_user" + expectedKeyType := event.AttrBool + expectedOp := event.OpEq + expectedValue := "true" + gotKeyName := afOne.UDExpression.And[0].Cmp.Key + gotKeyType := afOne.UDExpression.And[0].Cmp.Type + gotOp := afOne.UDExpression.And[0].Cmp.Op + gotValue := afOne.UDExpression.And[0].Cmp.Value + + if expectedKeyName != gotKeyName { + t.Errorf("Expected %v key name, got %v", expectedKeyName, gotKeyName) + } + + if expectedKeyType != gotKeyType { + t.Errorf("Expected %v key type, got %v", expectedKeyType, gotKeyType) + } + + if expectedOp != gotOp { + t.Errorf("Expected %v operator, got %v", expectedOp, gotOp) + } + + if expectedValue != gotValue { + t.Errorf("Expected %v value, got %v", expectedValue, gotValue) + } + } + + // assert expression structure and type + // for item 1 + { + expectedKeyName := "credit_balance" + expectedKeyType := event.AttrInt64 + expectedOp := event.OpGte + expectedValue := "1000" + gotKeyName := afOne.UDExpression.And[1].Cmp.Key + gotKeyType := afOne.UDExpression.And[1].Cmp.Type + gotOp := afOne.UDExpression.And[1].Cmp.Op + gotValue := afOne.UDExpression.And[1].Cmp.Value + + if expectedKeyName != gotKeyName { + t.Errorf("Expected %v key name, got %v", expectedKeyName, gotKeyName) + } + + if expectedKeyType != gotKeyType { + t.Errorf("Expected %v key type, got %v", expectedKeyType, gotKeyType) + } + + if expectedOp != gotOp { + t.Errorf("Expected %v operator, got %v", expectedOp, gotOp) + } + + if expectedValue != gotValue { + t.Errorf("Expected %v value, got %v", expectedValue, gotValue) + } + } +} diff --git a/backend/api/measure/app.go b/backend/api/measure/app.go index ea24ec636..6a097461f 100644 --- a/backend/api/measure/app.go +++ b/backend/api/measure/app.go @@ -2234,7 +2234,9 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `id invalid or missing` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2248,14 +2250,20 @@ func GetAppFilters(c *gin.Context) { if err := c.ShouldBindQuery(&af); err != nil { msg := `failed to parse query parameters` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } if err := af.Validate(); err != nil { msg := "app filters request validation failed" fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } @@ -2267,12 +2275,16 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := "failed to get team from app id" fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if team == nil { msg := fmt.Sprintf("no team exists for app [%s]", app.ID) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2281,7 +2293,9 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2289,13 +2303,17 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if !okTeam || !okApp { msg := `you are not authorized to access this app` - c.JSON(http.StatusForbidden, gin.H{"error": msg}) + c.JSON(http.StatusForbidden, gin.H{ + "error": msg, + }) return } @@ -2304,7 +2322,9 @@ func GetAppFilters(c *gin.Context) { if err := af.GetGenericFilters(ctx, &fl); err != nil { msg := `failed to query app filters` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2322,6 +2342,24 @@ func GetAppFilters(c *gin.Context) { osVersions = append(osVersions, osVersion) } + udAttrs := gin.H{ + "operator_types": nil, + "key_types": nil, + } + + if af.UDAttrKeys { + if err := af.GetUserDefinedAttrKeys(ctx, &fl); err != nil { + msg := `failed to query user defined attribute keys` + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) + return + } + udAttrs["operator_types"] = event.GetUDAttrsOpMap() + udAttrs["key_types"] = fl.UDKeyTypes + } + c.JSON(http.StatusOK, gin.H{ "versions": versions, "os_versions": osVersions, @@ -2332,6 +2370,7 @@ func GetAppFilters(c *gin.Context) { "locales": fl.DeviceLocales, "device_manufacturers": fl.DeviceManufacturers, "device_names": fl.DeviceNames, + "ud_attrs": udAttrs, }) } @@ -2357,6 +2396,8 @@ func GetCrashOverview(c *gin.Context) { return } + fmt.Println("ud expression raw:", af.UDExpressionRaw) + af.Expand() msg := "crash overview request validation failed" @@ -2366,6 +2407,10 @@ func GetCrashOverview(c *gin.Context) { return } + if af.UDExpression != nil { + fmt.Println("ud expression:", af.UDExpression) + } + if len(af.Versions) > 0 || len(af.VersionCodes) > 0 { if err := af.ValidateVersions(); err != nil { fmt.Println(msg, err) diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index e46582c1e..1b797f4e7 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -493,6 +493,9 @@ func (e eventreq) validate() error { if err := e.events[i].Attribute.Validate(); err != nil { return err } + if err := e.events[i].UserDefinedAttribute.Validate(); err != nil { + return err + } if e.hasAttachments() { for j := range e.events[i].Attachments { @@ -600,6 +603,9 @@ func (e eventreq) ingest(ctx context.Context) error { Set(`attribute.network_generation`, e.events[i].Attribute.NetworkGeneration). Set(`attribute.network_provider`, e.events[i].Attribute.NetworkProvider). + // user defined attribute + Set(`user_defined_attribute`, e.events[i].UserDefinedAttribute.Parameterize()). + // attachments Set(`attachments`, attachments) diff --git a/self-host/clickhouse/20241112203748_alter_events_table.sql b/self-host/clickhouse/20241112203748_alter_events_table.sql new file mode 100644 index 000000000..87834a69e --- /dev/null +++ b/self-host/clickhouse/20241112203748_alter_events_table.sql @@ -0,0 +1,15 @@ +-- migrate:up +alter table events + add column if not exists user_defined_attribute Map(LowCardinality(String), Tuple(Enum('string' = 1, 'int64', 'float64', 'bool'), String)) codec(ZSTD(3)) after `attribute.network_provider`, + comment column if exists user_defined_attribute 'user defined attributes', + add index if not exists user_defined_attribute_key_bloom_idx mapKeys(user_defined_attribute) type bloom_filter(0.01) granularity 16, + add index if not exists user_defined_attribute_key_minmax_idx mapKeys(user_defined_attribute) type minmax granularity 16, + materialize index if exists user_defined_attribute_key_bloom_idx, + materialize index if exists user_defined_attribute_key_minmax_idx; + + +-- migrate:down +alter table events + drop column if exists user_defined_attribute, + drop index if exists user_defined_attribute_key_bloom_idx, + drop index if exists user_defined_attribute_key_minmax_idx; diff --git a/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql b/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql new file mode 100644 index 000000000..bf9b6b000 --- /dev/null +++ b/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql @@ -0,0 +1,32 @@ +-- migrate:up +create table if not exists user_def_attrs +( + `app_id` UUID not null comment 'associated app id' codec(LZ4), + `event_id` UUID not null comment 'id of the event' codec(LZ4), + `session_id` UUID not null comment 'id of the session' codec(LZ4), + `end_of_month` DateTime not null comment 'last day of the month' codec(DoubleDelta, ZSTD(3)), + `app_version` Tuple(LowCardinality(String), LowCardinality(String)) not null comment 'composite app version' codec(ZSTD(3)), + `os_version` Tuple(LowCardinality(String), LowCardinality(String)) comment 'composite os version' codec (ZSTD(3)), + `exception` Bool comment 'true if source is exception event' codec (ZSTD(3)), + `anr` Bool comment 'true if source is anr event' codec (ZSTD(3)), + `key` LowCardinality(String) comment 'key of the user defined attribute' codec (ZSTD(3)), + `type` Enum('string' = 1, 'int64', 'float64', 'bool') comment 'type of the user defined attribute' codec (ZSTD(3)), + `value` String comment 'value of the user defined attribute' codec (ZSTD(3)), + index end_of_month_minmax_idx end_of_month type minmax granularity 2, + index exception_bloom_idx exception type bloom_filter granularity 2, + index anr_bloom_idx anr type bloom_filter granularity 2, + index key_bloom_idx key type bloom_filter(0.05) granularity 1, + index key_set_idx key type set(1000) granularity 2, + index session_bloom_idx session_id type bloom_filter granularity 2 +) +engine = ReplacingMergeTree +partition by toYYYYMM(end_of_month) +order by (app_id, end_of_month, app_version, os_version, exception, anr, + key, type, value, event_id, session_id) +settings index_granularity = 8192 +comment 'derived user defined attributes'; + + +-- migrate:down +drop table if exists user_def_attrs; + diff --git a/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql b/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql new file mode 100644 index 000000000..498db6b4a --- /dev/null +++ b/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql @@ -0,0 +1,28 @@ +-- migrate:up +create materialized view user_def_attrs_mv to user_def_attrs as +select distinct app_id, + id as event_id, + session_id, + toLastDayOfMonth(timestamp) as end_of_month, + (toString(attribute.app_version), + toString(attribute.app_build)) as app_version, + (toString(attribute.os_name), + toString(attribute.os_version)) as os_version, + if(events.type = 'exception' and exception.handled = false, + true, false) as exception, + if(events.type = 'anr', true, false) as anr, + arr_key as key, + tupleElement(arr_val, 1) as type, + tupleElement(arr_val, 2) as value +from events + array join + mapKeys(user_defined_attribute) as arr_key, + mapValues(user_defined_attribute) as arr_val +where length(user_defined_attribute) > 0 +group by app_id, end_of_month, app_version, os_version, events.type, + exception.handled, key, type, value, event_id, session_id +order by app_id; + + +-- migrate:down +drop view if exists user_def_attrs_mv; diff --git a/self-host/clickhouse/20241115225810_create_user_def_attrs_dict.sql b/self-host/clickhouse/20241115225810_create_user_def_attrs_dict.sql new file mode 100644 index 000000000..d9352a7ea --- /dev/null +++ b/self-host/clickhouse/20241115225810_create_user_def_attrs_dict.sql @@ -0,0 +1,18 @@ +-- migrate:up +create or replace dictionary if not exists user_def_attrs_dict +( + event_id UUID, + key String, + type String, + value String +) +primary key event_id, key +source (CLICKHOUSE(query + 'select event_id, key, type, value from user_def_attrs')) +lifetime (min 1 max 100) +layout (COMPLEX_KEY_CACHE(size_in_cells 1000)) +comment 'event mapped user defined attribute dictionary'; + + +-- migrate:down +drop dictionary if exists user_def_attrs_dict;