From 86e278285f1a3f626eaad601187d06b685202132 Mon Sep 17 00:00:00 2001 From: apoorvyadav1111 Date: Thu, 12 Dec 2024 23:27:38 -0500 Subject: [PATCH] [REFACTOR] remove sql and query_manager --- integration_tests/commands/http/setup.go | 9 +- integration_tests/commands/websocket/setup.go | 9 - internal/clientio/resp.go | 11 - internal/eval/eval.go | 92 --- internal/eval/execute.go | 4 - internal/querymanager/query_manager.go | 412 ---------- internal/sql/constants.go | 30 - internal/sql/dsql.go | 291 ------- internal/sql/dsql_test.go | 533 ------------- internal/sql/executerbechmark_test.go | 424 ---------- internal/sql/executor.go | 523 ------------- internal/sql/executor_test.go | 736 ------------------ internal/sql/fingerprint.go | 121 --- internal/sql/fingerprint_test.go | 418 ---------- internal/store/store.go | 29 - 15 files changed, 1 insertion(+), 3641 deletions(-) delete mode 100644 internal/querymanager/query_manager.go delete mode 100644 internal/sql/constants.go delete mode 100644 internal/sql/dsql.go delete mode 100644 internal/sql/dsql_test.go delete mode 100644 internal/sql/executerbechmark_test.go delete mode 100644 internal/sql/executor.go delete mode 100644 internal/sql/executor_test.go delete mode 100644 internal/sql/fingerprint.go delete mode 100644 internal/sql/fingerprint_test.go diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index 2e257026d..7f8e73767 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -32,7 +32,6 @@ import ( "github.com/dicedb/dice/config" derrors "github.com/dicedb/dice/internal/errors" - "github.com/dicedb/dice/internal/querymanager" "github.com/dicedb/dice/internal/shard" dstore "github.com/dicedb/dice/internal/store" ) @@ -131,7 +130,7 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption globalErrChannel := make(chan error) watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) - queryWatcherLocal := querymanager.NewQueryManager() + config.DiceConfig.HTTP.Port = opt.Port // Initialize the HTTPServer testServer := httpws.NewHTTPServer(shardManager, nil) @@ -144,12 +143,6 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption shardManager.Run(shardManagerCtx) }() - wg.Add(1) - go func() { - defer wg.Done() - queryWatcherLocal.Run(ctx, watchChan) - }() - // Start the server in a goroutine wg.Add(1) go func() { diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index e9ed697f7..fb2b0b800 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -31,7 +31,6 @@ import ( "github.com/dicedb/dice/config" derrors "github.com/dicedb/dice/internal/errors" - "github.com/dicedb/dice/internal/querymanager" "github.com/dicedb/dice/internal/shard" dstore "github.com/dicedb/dice/internal/store" "github.com/gorilla/websocket" @@ -132,7 +131,6 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO globalErrChannel := make(chan error) watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) - queryWatcherLocal := querymanager.NewQueryManager() config.DiceConfig.WebSocket.Port = opt.Port testServer := httpws.NewWebSocketServer(shardManager, testPort1, nil) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) @@ -144,13 +142,6 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO shardManager.Run(shardManagerCtx) }() - // run query manager - wg.Add(1) - go func() { - defer wg.Done() - queryWatcherLocal.Run(ctx, watchChan) - }() - // start websocket server wg.Add(1) go func() { diff --git a/internal/clientio/resp.go b/internal/clientio/resp.go index d15119392..8224bc81f 100644 --- a/internal/clientio/resp.go +++ b/internal/clientio/resp.go @@ -24,8 +24,6 @@ import ( "github.com/dicedb/dice/internal/object" - "github.com/dicedb/dice/internal/sql" - "github.com/dicedb/dice/internal/server/utils" dstore "github.com/dicedb/dice/internal/store" ) @@ -298,15 +296,6 @@ func Encode(value interface{}, isSimple bool) []byte { buf.Write(Encode(fmt.Sprintf("key:%s", we.Key), false)) buf.Write(Encode(fmt.Sprintf("op:%s", we.Operation), false)) return []byte(fmt.Sprintf("*2\r\n%s", buf.Bytes())) - case []sql.QueryResultRow: - var b []byte - buf := bytes.NewBuffer(b) // Create a buffer for accumulating encoded rows. - for _, row := range value.([]sql.QueryResultRow) { - buf.WriteString("*2\r\n") // Start a new array for each row. - buf.Write(Encode(row.Key, false)) // Encode the row key. - buf.Write(Encode(row.Value.Value, false)) // Encode the row value. - } - return []byte(fmt.Sprintf("*%d\r\n%s", len(v), buf.Bytes())) // Return the encoded response. // Handle map[string]bool and return a nil response indicating unsupported types. case map[string]bool: diff --git a/internal/eval/eval.go b/internal/eval/eval.go index 0caa2cb2a..d794cb24f 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -21,15 +21,10 @@ import ( "strconv" "time" - "github.com/dicedb/dice/internal/object" - - "github.com/dicedb/dice/internal/sql" - "github.com/dicedb/dice/config" "github.com/dicedb/dice/internal/clientio" "github.com/dicedb/dice/internal/comm" diceerrors "github.com/dicedb/dice/internal/errors" - "github.com/dicedb/dice/internal/querymanager" dstore "github.com/dicedb/dice/internal/store" ) @@ -189,90 +184,3 @@ func evalSLEEP(args []string, store *dstore.Store) []byte { time.Sleep(time.Duration(durationSec) * time.Second) return clientio.RespOK } - -// EvalQWATCH adds the specified key to the watch list for the caller client. -// Every time a key in the watch list is modified, the client will be sent a response -// containing the new value of the key along with the operation that was performed on it. -// Contains only one argument, the query to be watched. -func EvalQWATCH(args []string, httpOp, websocketOp bool, client *comm.Client, store *dstore.Store) []byte { - if len(args) != 1 { - return diceerrors.NewErrArity("Q.WATCH") - } - - // Parse and get the selection from the query. - query, e := sql.ParseQuery( /*sql=*/ args[0]) - - if e != nil { - return clientio.Encode(e, false) - } - - // use an unbuffered channel to ensure that we only proceed to query execution once the query watcher has built the cache - cacheChannel := make(chan *[]struct { - Key string - Value *object.Obj - }) - var watchSubscription querymanager.QuerySubscription - - if httpOp || websocketOp { - watchSubscription = querymanager.QuerySubscription{ - Subscribe: true, - Query: query, - CacheChan: cacheChannel, - QwatchClientChan: client.HTTPQwatchResponseChan, - ClientIdentifierID: client.ClientIdentifierID, - } - } else { - watchSubscription = querymanager.QuerySubscription{ - Subscribe: true, - Query: query, - ClientFD: client.Fd, - CacheChan: cacheChannel, - } - } - - querymanager.QuerySubscriptionChan <- watchSubscription - store.CacheKeysForQuery(query.Where, cacheChannel) - - // Return the result of the query. - responseChan := make(chan querymanager.AdhocQueryResult) - querymanager.AdhocQueryChan <- querymanager.AdhocQuery{ - Query: query, - ResponseChan: responseChan, - } - - queryResult := <-responseChan - if queryResult.Err != nil { - return clientio.Encode(queryResult.Err, false) - } - - // TODO: We should return the list of all queries being watched by the client. - return clientio.Encode(querymanager.GenericWatchResponse(sql.Qwatch, query.String(), *queryResult.Result), false) -} - -// EvalQUNWATCH removes the specified key from the watch list for the caller client. -func EvalQUNWATCH(args []string, httpOp bool, client *comm.Client) []byte { - if len(args) != 1 { - return diceerrors.NewErrArity("Q.UNWATCH") - } - query, e := sql.ParseQuery( /*sql=*/ args[0]) - if e != nil { - return clientio.Encode(e, false) - } - - if httpOp { - querymanager.QuerySubscriptionChan <- querymanager.QuerySubscription{ - Subscribe: false, - Query: query, - QwatchClientChan: client.HTTPQwatchResponseChan, - ClientIdentifierID: client.ClientIdentifierID, - } - } else { - querymanager.QuerySubscriptionChan <- querymanager.QuerySubscription{ - Subscribe: false, - Query: query, - ClientFD: client.Fd, - } - } - - return clientio.RespOK -} diff --git a/internal/eval/execute.go b/internal/eval/execute.go index 1d1750ab0..a7bc48e9f 100644 --- a/internal/eval/execute.go +++ b/internal/eval/execute.go @@ -86,10 +86,6 @@ func (e *Eval) ExecuteCommand() *EvalResponse { switch diceCmd.Name { // Old implementation kept as it is, but we will be moving // to the new implementation soon for all commands - case "SUBSCRIBE", "Q.WATCH": - return &EvalResponse{Result: EvalQWATCH(e.cmd.Args, e.isHTTPOperation, e.isWebSocketOperation, e.client, e.store), Error: nil} - case "UNSUBSCRIBE", "Q.UNWATCH": - return &EvalResponse{Result: EvalQUNWATCH(e.cmd.Args, e.isHTTPOperation, e.client), Error: nil} case auth.Cmd: return &EvalResponse{Result: EvalAUTH(e.cmd.Args, e.client), Error: nil} case "ABORT": diff --git a/internal/querymanager/query_manager.go b/internal/querymanager/query_manager.go deleted file mode 100644 index e9fa64cee..000000000 --- a/internal/querymanager/query_manager.go +++ /dev/null @@ -1,412 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package querymanager - -import ( - "context" - "errors" - "fmt" - "log/slog" - "sync" - "syscall" - "time" - - "github.com/dicedb/dice/internal/comm" - "github.com/dicedb/dice/internal/common" - - "github.com/ohler55/ojg/jp" - - "github.com/dicedb/dice/internal/object" - - "github.com/dicedb/dice/internal/sql" - - "github.com/dicedb/dice/internal/clientio" - dstore "github.com/dicedb/dice/internal/store" -) - -type ( - CacheStore common.ITable[string, *object.Obj] - - // QuerySubscription represents a subscription to watch a query. - QuerySubscription struct { - Subscribe bool // true for subscribe, false for unsubscribe - Query sql.DSQLQuery // query to watch - ClientFD int // client file descriptor - CacheChan chan *[]struct { - Key string - Value *object.Obj - } // channel to receive cache data for this query - QwatchClientChan chan comm.QwatchResponse // Generic channel for HTTP/Websockets etc. - ClientIdentifierID uint32 // Helps identify qwatch client on httpserver side - } - - // AdhocQueryResult represents the result of an adhoc query. - AdhocQueryResult struct { - Result *[]sql.QueryResultRow - Fingerprint string - Err error - } - - // AdhocQuery represents an adhoc query request. - AdhocQuery struct { - Query sql.DSQLQuery - ResponseChan chan AdhocQueryResult - } - - // Manager watches for changes in keys and notifies clients. - Manager struct { - WatchList sync.Map // WatchList is a map of query string to their respective clients, type: map[string]*sync.Map[int]struct{} - QueryCache common.ITable[string, CacheStore] // QueryCache is a map of fingerprints to their respective data caches - QueryCacheMu sync.RWMutex - } - - HTTPQwatchResponse struct { - Cmd string `json:"cmd"` - Query string `json:"query"` - Data []any `json:"data"` - } - - ClientIdentifier struct { - ClientIdentifierID int - IsHTTPClient bool - } -) - -var ( - // QuerySubscriptionChan is the channel to receive updates about query subscriptions. - QuerySubscriptionChan chan QuerySubscription - - // AdhocQueryChan is the channel to receive adhoc queries. - AdhocQueryChan chan AdhocQuery -) - -func NewClientIdentifier(clientIdentifierID int, isHTTPClient bool) ClientIdentifier { - return ClientIdentifier{ - ClientIdentifierID: clientIdentifierID, - IsHTTPClient: isHTTPClient, - } -} - -func NewQueryCacheStoreRegMap() common.ITable[string, CacheStore] { - return &common.RegMap[string, CacheStore]{ - M: make(map[string]CacheStore), - } -} - -func NewQueryCacheStore() common.ITable[string, CacheStore] { - return NewQueryCacheStoreRegMap() -} - -func NewCacheStoreRegMap() CacheStore { - return &common.RegMap[string, *object.Obj]{ - M: make(map[string]*object.Obj), - } -} - -func NewCacheStore() CacheStore { - return NewCacheStoreRegMap() -} - -// NewQueryManager initializes a new Manager. -func NewQueryManager() *Manager { - QuerySubscriptionChan = make(chan QuerySubscription) - AdhocQueryChan = make(chan AdhocQuery, 1000) - return &Manager{ - WatchList: sync.Map{}, - QueryCache: NewQueryCacheStore(), - } -} - -// Run starts the Manager's main loops. -func (m *Manager) Run(ctx context.Context, watchChan <-chan dstore.QueryWatchEvent) { - var wg sync.WaitGroup - - wg.Add(3) - go func() { - defer wg.Done() - m.listenForSubscriptions(ctx) - }() - - go func() { - defer wg.Done() - m.watchKeys(ctx, watchChan) - }() - - go func() { - defer wg.Done() - m.serveAdhocQueries(ctx) - }() - - <-ctx.Done() - wg.Wait() -} - -// listenForSubscriptions listens for query subscriptions and unsubscriptions. -func (m *Manager) listenForSubscriptions(ctx context.Context) { - for { - select { - case event := <-QuerySubscriptionChan: - var client ClientIdentifier - if event.QwatchClientChan != nil { - client = NewClientIdentifier(int(event.ClientIdentifierID), true) - } else { - client = NewClientIdentifier(event.ClientFD, false) - } - - if event.Subscribe { - m.addWatcher(&event.Query, client, event.QwatchClientChan, event.CacheChan) - } else { - m.removeWatcher(&event.Query, client, event.QwatchClientChan) - } - case <-ctx.Done(): - return - } - } -} - -// watchKeys watches for changes in keys and notifies clients. -func (m *Manager) watchKeys(ctx context.Context, watchChan <-chan dstore.QueryWatchEvent) { - for { - select { - case event := <-watchChan: - m.processWatchEvent(event) - case <-ctx.Done(): - return - } - } -} - -// processWatchEvent processes a single watch event. -func (m *Manager) processWatchEvent(event dstore.QueryWatchEvent) { - // Iterate over the watchlist to go through the query string - // and the corresponding client connections to that query string - m.WatchList.Range(func(key, value interface{}) bool { - queryString := key.(string) - clients := value.(*sync.Map) - - query, err := sql.ParseQuery(queryString) - if err != nil { - slog.Error( - "error parsing query", - slog.String("query", queryString), - ) - return true - } - - // Check if the key matches the regex - if query.Where != nil { - matches, err := sql.EvaluateWhereClause(query.Where, sql.QueryResultRow{Key: event.Key, Value: event.Value}, make(map[string]jp.Expr)) - if err != nil || !matches { - return true - } - } - - m.updateQueryCache(query.Fingerprint, event) - - queryResult, err := m.runQuery(&query) - if err != nil { - slog.Error(err.Error()) - return true - } - - m.notifyClients(&query, clients, queryResult) - return true - }) -} - -// updateQueryCache updates the query cache based on the watch event. -func (m *Manager) updateQueryCache(queryFingerprint string, event dstore.QueryWatchEvent) { - m.QueryCacheMu.Lock() - defer m.QueryCacheMu.Unlock() - - store, ok := m.QueryCache.Get(queryFingerprint) - if !ok { - slog.Warn("Fingerprint not found in CacheStore", slog.String("fingerprint", queryFingerprint)) - return - } - - switch event.Operation { - case dstore.Set: - store.Put(event.Key, &event.Value) - case dstore.Del: - store.Delete(event.Key) - default: - slog.Warn("Unknown operation", slog.String("operation", event.Operation)) - } -} - -func (m *Manager) notifyClients(query *sql.DSQLQuery, clients *sync.Map, queryResult *[]sql.QueryResultRow) { - encodedResult := clientio.Encode(GenericWatchResponse(sql.Qwatch, query.String(), *queryResult), false) - - clients.Range(func(clientKey, clientVal interface{}) bool { - // Identify the type of client and respond accordingly - switch clientIdentifier := clientKey.(ClientIdentifier); { - case clientIdentifier.IsHTTPClient: - qwatchClientResponseChannel := clientVal.(chan comm.QwatchResponse) - qwatchClientResponseChannel <- comm.QwatchResponse{ - ClientIdentifierID: uint32(clientIdentifier.ClientIdentifierID), - Result: encodedResult, - Error: nil, - } - case !clientIdentifier.IsHTTPClient: - // We use a retry mechanism here as the client's socket may be temporarily unavailable for writes due to the - // high number of writes that are possible in qwatch. Without this mechanism, the client may be removed from the - // watchlist prematurely. - // TODO: - // 1. Replace with thread pool to prevent launching an unbounded number of goroutines. - // 2. Each client's writes should be sent in a serialized manner, maybe a per-client queue should be maintained - // here. A single queue-per-client is also helpful when the client file descriptor is closed and the queue can - // just be destroyed. - clientFD := clientIdentifier.ClientIdentifierID - // This is a regular client, use clientFD to send the response - go m.sendWithRetry(query, clientFD, encodedResult) - default: - slog.Warn("Invalid Client, response channel invalid.") - } - - return true - }) -} - -// sendWithRetry writes data to a client file descriptor with retries. It writes with an exponential backoff. -func (m *Manager) sendWithRetry(query *sql.DSQLQuery, clientFD int, data []byte) { - maxRetries := 20 - retryDelay := 20 * time.Millisecond - - for i := 0; i < maxRetries; i++ { - _, err := syscall.Write(clientFD, data) - if err == nil { - return - } - - if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - time.Sleep(retryDelay) - retryDelay *= 2 // exponential backoff - continue - } - - slog.Error( - "error writing to client", - slog.Int("client", clientFD), - slog.Any("error", err), - ) - m.removeWatcher(query, NewClientIdentifier(clientFD, false), nil) - return - } -} - -// serveAdhocQueries listens for adhoc queries, executes them, and sends the result back to the client. -func (m *Manager) serveAdhocQueries(ctx context.Context) { - for { - select { - case query := <-AdhocQueryChan: - result, err := m.runQuery(&query.Query) - query.ResponseChan <- AdhocQueryResult{ - Result: result, - Fingerprint: query.Query.Fingerprint, - Err: err, - } - case <-ctx.Done(): - return - } - } -} - -// addWatcher adds a client as a watcher to a query. -func (m *Manager) addWatcher(query *sql.DSQLQuery, clientIdentifier ClientIdentifier, - qwatchClientChan chan comm.QwatchResponse, cacheChan chan *[]struct { - Key string - Value *object.Obj - }) { - queryString := query.String() - - clients, _ := m.WatchList.LoadOrStore(queryString, &sync.Map{}) - if qwatchClientChan != nil { - clients.(*sync.Map).Store(clientIdentifier, qwatchClientChan) - } else { - clients.(*sync.Map).Store(clientIdentifier, struct{}{}) - } - - m.QueryCacheMu.Lock() - defer m.QueryCacheMu.Unlock() - - cache := NewCacheStore() - // Hydrate the cache with data from all shards. - // TODO: We need to ensure we receive cache data from all shards once we have multithreading in place. - // For now we only expect one update. - kvs := <-cacheChan - for _, kv := range *kvs { - cache.Put(kv.Key, kv.Value) - } - - m.QueryCache.Put(query.Fingerprint, cache) -} - -// removeWatcher removes a client from the watchlist for a query. -func (m *Manager) removeWatcher(query *sql.DSQLQuery, clientIdentifier ClientIdentifier, - qwatchClientChan chan comm.QwatchResponse) { - queryString := query.String() - if clients, ok := m.WatchList.Load(queryString); ok { - if qwatchClientChan != nil { - clients.(*sync.Map).Delete(clientIdentifier) - slog.Debug("HTTP client no longer watching query", - slog.Any("clientIdentifierId", clientIdentifier.ClientIdentifierID), - slog.Any("queryString", queryString)) - } else { - clients.(*sync.Map).Delete(clientIdentifier) - slog.Debug("client no longer watching query", - slog.Int("client", clientIdentifier.ClientIdentifierID), - slog.String("query", queryString)) - } - - // If no more clients for this query, remove the query from WatchList - if m.clientCount(clients.(*sync.Map)) == 0 { - m.WatchList.Delete(queryString) - - // Remove this Query's cached data. - m.QueryCacheMu.Lock() - m.QueryCache.Delete(query.Fingerprint) - m.QueryCacheMu.Unlock() - - slog.Debug("no longer watching query", slog.String("query", queryString)) - } - } -} - -// clientCount returns the number of clients watching a query. -func (m *Manager) clientCount(clients *sync.Map) int { - count := 0 - clients.Range(func(_, _ interface{}) bool { - count++ - return true - }) - return count -} - -// runQuery executes a query on its respective cache. -func (m *Manager) runQuery(query *sql.DSQLQuery) (*[]sql.QueryResultRow, error) { - m.QueryCacheMu.RLock() - defer m.QueryCacheMu.RUnlock() - - store, ok := m.QueryCache.Get(query.Fingerprint) - if !ok { - return nil, fmt.Errorf("fingerprint was not found in the cache: %s", query.Fingerprint) - } - - result, err := sql.ExecuteQuery(query, store) - return &result, err -} diff --git a/internal/sql/constants.go b/internal/sql/constants.go deleted file mode 100644 index 422e6bcff..000000000 --- a/internal/sql/constants.go +++ /dev/null @@ -1,30 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -const ( - Asc string = "asc" - Desc string = "desc" - - String string = "string" - Int64 string = "int64" - Float string = "float" - Bool string = "bool" - Nil string = "nil" - - Qwatch string = "q.watch" -) diff --git a/internal/sql/dsql.go b/internal/sql/dsql.go deleted file mode 100644 index 0296810dc..000000000 --- a/internal/sql/dsql.go +++ /dev/null @@ -1,291 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -import ( - "fmt" - "strconv" - "strings" - - "github.com/xwb1989/sqlparser" -) - -// Constants for custom syntax replacements -const ( - CustomKey = "$key" - CustomValue = "$value" - TempPrefix = "_" - TempKey = TempPrefix + "key" - TempValue = TempPrefix + "value" -) - -// UnsupportedDSQLStatementError is returned when a DSQL statement is not supported -type UnsupportedDSQLStatementError struct { - Stmt sqlparser.Statement -} - -func (e *UnsupportedDSQLStatementError) Error() string { - return fmt.Sprintf("unsupported DSQL statement: %T", e.Stmt) -} - -func newUnsupportedSQLStatementError(stmt sqlparser.Statement) *UnsupportedDSQLStatementError { - return &UnsupportedDSQLStatementError{Stmt: stmt} -} - -// QuerySelection represents the SELECT expressions in the query -type QuerySelection struct { - KeySelection bool - ValueSelection bool -} - -type QueryOrder struct { - OrderBy string - Order string -} - -type DSQLQuery struct { - Selection QuerySelection - Where sqlparser.Expr - OrderBy QueryOrder - Limit int - Fingerprint string -} - -// replacePlaceholders replaces temporary placeholders with custom ones -func replacePlaceholders(s string) string { - replacer := strings.NewReplacer( - TempKey, CustomKey, - TempValue, CustomValue, - ) - return replacer.Replace(s) -} - -// parseSelectExpressions parses the SELECT expressions in the query - -func (q DSQLQuery) String() string { - var parts []string - - // Selection - var selectionParts []string - if q.Selection.KeySelection { - selectionParts = append(selectionParts, CustomKey) - } - if q.Selection.ValueSelection { - selectionParts = append(selectionParts, CustomValue) - } - if len(selectionParts) > 0 { - parts = append(parts, fmt.Sprintf("SELECT %s", strings.Join(selectionParts, ", "))) - } else { - parts = append(parts, "SELECT *") - } - - // Where - if q.Where != nil { - whereClause := sqlparser.String(q.Where) - whereClause = replacePlaceholders(whereClause) - parts = append(parts, fmt.Sprintf("WHERE %s", whereClause)) - } - - // OrderBy - if q.OrderBy.OrderBy != "" { - orderByClause := replacePlaceholders(q.OrderBy.OrderBy) - parts = append(parts, fmt.Sprintf("ORDER BY %s %s", orderByClause, q.OrderBy.Order)) - } - - // Limit - if q.Limit > 0 { - parts = append(parts, fmt.Sprintf("LIMIT %d", q.Limit)) - } - - return strings.Join(parts, " ") -} - -// Utility functions for custom syntax handling -func replaceCustomSyntax(sql string) string { - replacer := strings.NewReplacer(CustomKey, TempKey, CustomValue, TempValue) - - // Add implicit `FROM dual` if no table name is provided - return replacer.Replace(sql) -} - -// ParseQuery takes a SQL query string and returns a DSQLQuery struct -func ParseQuery(sql string) (DSQLQuery, error) { - // Replace custom syntax before parsing - sql = replaceCustomSyntax(sql) - - stmt, err := sqlparser.Parse(sql) - if err != nil { - return DSQLQuery{}, fmt.Errorf("error parsing SQL statement: %v", err) - } - - selectStmt, ok := stmt.(*sqlparser.Select) - if !ok { - return DSQLQuery{}, newUnsupportedSQLStatementError(stmt) - } - - // Ensure no unsupported clauses are present - if err := checkUnsupportedClauses(selectStmt); err != nil { - return DSQLQuery{}, err - } - - querySelection, err := parseSelectExpressions(selectStmt) - if err != nil { - return DSQLQuery{}, err - } - - if err := parseTableName(selectStmt); err != nil { - return DSQLQuery{}, err - } - - where := parseWhere(selectStmt) - - orderBy, err := parseOrderBy(selectStmt) - if err != nil { - return DSQLQuery{}, err - } - - limit, err := parseLimit(selectStmt) - if err != nil { - return DSQLQuery{}, err - } - - return DSQLQuery{ - Selection: querySelection, - Where: where, - OrderBy: orderBy, - Limit: limit, - Fingerprint: generateFingerprint(where), - }, nil -} - -// Function to validate unsupported clauses such as GROUP BY and HAVING -func checkUnsupportedClauses(selectStmt *sqlparser.Select) error { - if selectStmt.GroupBy != nil || selectStmt.Having != nil { - return fmt.Errorf("HAVING and GROUP BY clauses are not supported") - } - return nil -} - -// Function to parse SELECT expressions -func parseSelectExpressions(selectStmt *sqlparser.Select) (QuerySelection, error) { - if len(selectStmt.SelectExprs) < 1 { - return QuerySelection{}, fmt.Errorf("no fields selected in result set") - } else if len(selectStmt.SelectExprs) > 2 { - return QuerySelection{}, fmt.Errorf("only $key and $value are supported in SELECT expressions") - } - - keySelection := false - valueSelection := false - for _, expr := range selectStmt.SelectExprs { - aliasedExpr, ok := expr.(*sqlparser.AliasedExpr) - if !ok { - return QuerySelection{}, fmt.Errorf("error parsing SELECT expression: %v", expr) - } - colName, ok := aliasedExpr.Expr.(*sqlparser.ColName) - if !ok { - return QuerySelection{}, fmt.Errorf("only column names are supported in SELECT") - } - switch colName.Name.String() { - case TempKey: - keySelection = true - case TempValue: - valueSelection = true - default: - return QuerySelection{}, fmt.Errorf("only $key and $value are supported in SELECT expressions") - } - } - - return QuerySelection{KeySelection: keySelection, ValueSelection: valueSelection}, nil -} - -// Function to parse table name -func parseTableName(selectStmt *sqlparser.Select) error { - tableExpr, ok := selectStmt.From[0].(*sqlparser.AliasedTableExpr) - if !ok { - return fmt.Errorf("error parsing table name") - } - - // Remove backticks from table name if present. - tableName := strings.Trim(sqlparser.String(tableExpr.Expr), "`") - - // Ensure table name is not dual, which means no table name was provided. - if tableName != "dual" { - return fmt.Errorf("FROM clause is not supported") - } - - return nil -} - -// Function to parse ORDER BY clause -func parseOrderBy(selectStmt *sqlparser.Select) (QueryOrder, error) { - orderBy := QueryOrder{} - - // Support only one ORDER BY clause - if len(selectStmt.OrderBy) > 1 { - return QueryOrder{}, fmt.Errorf("only one ORDER BY clause is supported") - } - - if len(selectStmt.OrderBy) == 0 { - // No ORDER BY clause, return empty order - return orderBy, nil - } - - // Extract the ORDER BY expression - orderExpr := strings.Trim(sqlparser.String(selectStmt.OrderBy[0].Expr), "`") - orderExpr = trimQuotesOrBackticks(orderExpr) - - // Validate that ORDER BY is either $key or $value - if orderExpr != TempKey && orderExpr != TempValue && !strings.HasPrefix(orderExpr, TempValue) { - return QueryOrder{}, fmt.Errorf("only $key and $value expressions are supported in ORDER BY clause") - } - - // Assign values to QueryOrder - orderBy.OrderBy = orderExpr - orderBy.Order = selectStmt.OrderBy[0].Direction - - return orderBy, nil -} - -// Helper function to trim both single and double quotes/backticks -func trimQuotesOrBackticks(input string) string { - if len(input) > 1 && ((input[0] == '\'' && input[len(input)-1] == '\'') || - (input[0] == '`' && input[len(input)-1] == '`')) { - return input[1 : len(input)-1] - } - return input -} - -// Function to parse LIMIT clause -func parseLimit(selectStmt *sqlparser.Select) (int, error) { - limit := 0 - if selectStmt.Limit != nil { - limitVal, err := strconv.Atoi(sqlparser.String(selectStmt.Limit.Rowcount)) - if err != nil { - return 0, fmt.Errorf("invalid LIMIT value") - } - limit = limitVal - } - return limit, nil -} - -// Function to parse WHERE clause -func parseWhere(selectStmt *sqlparser.Select) sqlparser.Expr { - if selectStmt.Where == nil { - return nil - } - return selectStmt.Where.Expr -} diff --git a/internal/sql/dsql_test.go b/internal/sql/dsql_test.go deleted file mode 100644 index 5d8b98879..000000000 --- a/internal/sql/dsql_test.go +++ /dev/null @@ -1,533 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -import ( - "testing" - - "github.com/dicedb/dice/internal/server/utils" - "github.com/stretchr/testify/assert" - "github.com/xwb1989/sqlparser" -) - -func TestParseQuery(t *testing.T) { - tests := []struct { - name string - sql string - want DSQLQuery - wantErr bool - error string - }{ - { - name: "valid select key and value with order and limit", - sql: "SELECT $key, $value WHERE $key like `match:100:*` ORDER BY $value DESC LIMIT 10", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Where: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("match:100:*")}, - }, - OrderBy: QueryOrder{OrderBy: "_value", Order: "desc"}, - Limit: 10, - }, - wantErr: false, - }, - { - name: "valid select with where clause", - sql: "SELECT $key, $value WHERE $key like `match:100:*` AND $value = 'test' ORDER BY $key LIMIT 5", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Where: &sqlparser.AndExpr{ - Left: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("match:100:*")}, - }, - Right: &sqlparser.ComparisonExpr{ - Operator: "=", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Right: &sqlparser.SQLVal{Val: []uint8("test")}, - }, - }, - OrderBy: QueryOrder{OrderBy: "_key", Order: Asc}, - Limit: 5, - }, - wantErr: false, - }, - { - name: "complex where clause", - sql: "SELECT $key WHERE $key like `user:*` AND $value > 25 AND $key LIKE 'user:1%'", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: false}, - Where: &sqlparser.AndExpr{ - Left: &sqlparser.AndExpr{ - Left: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("user:*")}, - }, - Right: &sqlparser.ComparisonExpr{ - Operator: ">", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Right: sqlparser.NewIntVal([]byte("25")), - }, - }, - Right: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.SQLVal{Val: []uint8("user:1%")}, - }, - }, - }, - wantErr: false, - }, - { - name: "invalid order by expression", - sql: "SELECT $key WHERE $key like `match:100:*` ORDER BY invalid_key LIMIT 5", - want: DSQLQuery{}, - wantErr: true, - error: "only $key and $value expressions are supported in ORDER BY clause", - }, - { - name: "invalid multiple fields", - sql: "SELECT field1, field2 WHERE $key like `test`", - want: DSQLQuery{}, - wantErr: true, - error: "only $key and $value are supported in SELECT expressions", - }, - { - name: "invalid non-select statement", - sql: "INSERT INTO table_name (field_name) values ('value')", - want: DSQLQuery{}, - wantErr: true, - error: "unsupported DSQL statement: *sqlparser.Insert", - }, - { - name: "empty invalid statement", - sql: utils.EmptyStr, - want: DSQLQuery{}, - wantErr: true, - error: "error parsing SQL statement: syntax error at position 1", - }, - { - name: "unsupported having clause", - sql: "SELECT $key WHERE $key like `match:100:*` HAVING $key > 1", - want: DSQLQuery{}, - wantErr: true, - error: "HAVING and GROUP BY clauses are not supported", - }, - { - name: "unsupported group by clause", - sql: "SELECT $key WHERE $key like `match:100:*` GROUP BY $key", - want: DSQLQuery{}, - wantErr: true, - error: "HAVING and GROUP BY clauses are not supported", - }, - { - name: "invalid limit value", - sql: "SELECT $key WHERE $key like `match:100:*` LIMIT abc", - want: DSQLQuery{}, - wantErr: true, - error: "invalid LIMIT value", - }, - { - name: "select only value", - sql: "SELECT $value WHERE $key like `test:*`", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: false, ValueSelection: true}, - Where: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("test:*")}, - }, - }, - wantErr: false, - }, - { - name: "order by key ascending", - sql: "SELECT $key, $value WHERE $key like `test:*` ORDER BY $key ASC", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Where: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("test:*")}, - }, - OrderBy: QueryOrder{OrderBy: "_key", Order: "asc"}, - }, - wantErr: false, - }, - { - name: "invalid table name", - sql: "SELECT $key FROM 123", - want: DSQLQuery{}, - wantErr: true, - error: "error parsing SQL statement: syntax error at position 21 near '123'", - }, - { - name: "Banned FROM clause", - sql: "SELECT $key FROM tablename", - want: DSQLQuery{}, - wantErr: true, - error: "FROM clause is not supported", - }, - { - name: "where clause with NULL comparison", - sql: "SELECT $key, $value WHERE $key like `test:*` AND $value IS NULL", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Where: &sqlparser.AndExpr{ - Left: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("test:*")}, - }, - Right: &sqlparser.IsExpr{Operator: "is null", Expr: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}}, - }, - }, - wantErr: false, - }, - { - name: "where clause with multiple conditions", - sql: "SELECT $key WHERE ($key LIKE `test:*`) AND ($value > 10 OR $value < 5)", - want: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: false}, - Where: &sqlparser.AndExpr{ - Left: &sqlparser.ParenExpr{ - Expr: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("test:*")}, - }, - }, - Right: &sqlparser.ParenExpr{ - Expr: &sqlparser.OrExpr{ - Left: &sqlparser.ComparisonExpr{ - Operator: ">", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Right: sqlparser.NewIntVal([]byte("10")), - }, - Right: &sqlparser.ComparisonExpr{ - Operator: "<", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Right: sqlparser.NewIntVal([]byte("5")), - }, - }, - }, - }, - }, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseQuery(tt.sql) - if tt.wantErr { - assert.Error(t, err, tt.error) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want.Selection, got.Selection) - assert.Equal(t, tt.want.OrderBy, got.OrderBy) - assert.Equal(t, tt.want.Limit, got.Limit) - - //if tt.want.Where == nil { - // assert.Assert(t, got.Where == nil) - //} else { - assert.True(t, got.Where != nil) - assert.Equal(t, tt.want.Where, got.Where) - //} - } - }) - } -} - -func TestParseSelectExpressions(t *testing.T) { - tests := []struct { - name string - sql string - want QuerySelection - wantErr bool - }{ - { - name: "select key and value", - sql: "SELECT $key, $value WHERE $key like `test`", - want: QuerySelection{KeySelection: true, ValueSelection: true}, - }, - { - name: "select only key", - sql: "SELECT $key WHERE $key like `test`", - want: QuerySelection{KeySelection: true, ValueSelection: false}, - }, - { - name: "select only value", - sql: "SELECT $value WHERE $key like `test`", - want: QuerySelection{KeySelection: false, ValueSelection: true}, - }, - { - name: "select invalid field", - sql: "SELECT invalid WHERE $key like `test`", - wantErr: true, - }, - { - name: "select too many fields", - sql: "SELECT $key, $value, extra WHERE $key like `test`", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - stmt, err := sqlparser.Parse(replaceCustomSyntax(tt.sql)) - assert.Nil(t, err) - - selectStmt, ok := stmt.(*sqlparser.Select) - assert.True(t, ok) - - got, err := parseSelectExpressions(selectStmt) - if tt.wantErr { - assert.True(t, err != nil) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} - -func TestParseOrderBy(t *testing.T) { - tests := []struct { - name string - sql string - want QueryOrder - wantErr bool - }{ - { - name: "order by key asc", - sql: "SELECT $key WHERE $key like `test` ORDER BY $key ASC", - want: QueryOrder{OrderBy: "_key", Order: Asc}, - }, - { - name: "order by key desc", - sql: "SELECT $key WHERE $key like `test` ORDER BY $key DESC", - want: QueryOrder{OrderBy: "_key", Order: "desc"}, - }, - { - name: "order by value asc", - sql: "SELECT $value WHERE $key like `test` ORDER BY $value ASC", - want: QueryOrder{OrderBy: "_value", Order: "asc"}, - }, - { - name: "order by value desc", - sql: "SELECT $value WHERE $key like `test` ORDER BY $value DESC", - want: QueryOrder{OrderBy: "_value", Order: "desc"}, - }, - { - name: "order by json path asc", - sql: "SELECT $value WHERE $key like `test` ORDER BY $value.name ASC", - want: QueryOrder{OrderBy: "_value.name", Order: "asc"}, - }, - { - name: "order by nested json path desc", - sql: "SELECT $value WHERE $key like `test` ORDER BY $value.address.city DESC", - want: QueryOrder{OrderBy: "_value.address.city", Order: "desc"}, - }, - { - name: "order by json path with array index", - sql: "SELECT $value WHERE $key like `test` ORDER BY `$value.items[0].price`", - want: QueryOrder{OrderBy: "_value.items[0].price", Order: "asc"}, - }, - { - name: "order by complex json path", - sql: "SELECT $value WHERE $key like `test` ORDER BY `$value.users[*].contacts[0].email`", - want: QueryOrder{OrderBy: "_value.users[*].contacts[0].email", Order: "asc"}, - }, - { - name: "no order by clause", - sql: "SELECT $key WHERE $key like `test`", - want: QueryOrder{}, - }, - { - name: "invalid order by field", - sql: "SELECT $key WHERE $key like `test` ORDER BY invalid", - wantErr: true, - }, - { - name: "no order by clause", - sql: "SELECT $key WHERE $key like `test`", - want: QueryOrder{}, - }, - { - name: "multiple order by clauses", - sql: "SELECT $key WHERE $key like `test` ORDER BY $key ASC, $value DESC", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stmt, err := sqlparser.Parse(replaceCustomSyntax(tt.sql)) - assert.Nil(t, err) - - selectStmt, ok := stmt.(*sqlparser.Select) - assert.True(t, ok) - - got, err := parseOrderBy(selectStmt) - if tt.wantErr { - assert.True(t, err != nil) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} - -func TestParseLimit(t *testing.T) { - tests := []struct { - name string - sql string - want int - wantErr bool - }{ - { - name: "valid limit", - sql: "SELECT $key WHERE $key like `test` LIMIT 10", - want: 10, - }, - { - name: "no limit clause", - sql: "SELECT $key WHERE $key like `test`", - want: 0, - }, - { - name: "invalid limit value", - sql: "SELECT $key WHERE $key like `test` LIMIT abc", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stmt, err := sqlparser.Parse(replaceCustomSyntax(tt.sql)) - assert.Nil(t, err) - - selectStmt, ok := stmt.(*sqlparser.Select) - assert.True(t, ok) - - got, err := parseLimit(selectStmt) - if tt.wantErr { - assert.True(t, err != nil) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} - -func TestDSQLQueryString(t *testing.T) { - tests := []struct { - name string - query DSQLQuery - expected string - }{ - { - name: "Key Selection Only", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true}, - }, - expected: "SELECT $key", - }, - { - name: "Value Selection Only", - query: DSQLQuery{ - Selection: QuerySelection{ValueSelection: true}, - }, - expected: "SELECT $value", - }, - { - name: "Both Key and Value Selection", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - }, - expected: "SELECT $key, $value", - }, - { - query: DSQLQuery{ - Selection: QuerySelection{ValueSelection: true}, - }, - expected: "SELECT $value", - }, - { - name: "With Where Clause", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true}, - Where: sqlparser.NewStrVal([]byte("$value > 10")), - }, - expected: "SELECT $key WHERE '$value > 10'", - }, - { - name: "With OrderBy", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - OrderBy: QueryOrder{OrderBy: "_key", Order: "DESC"}, - }, - expected: "SELECT $key, $value ORDER BY $key DESC", - }, - { - name: "With Limit", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Limit: 5, - }, - expected: "SELECT $key, $value LIMIT 5", - }, - { - name: "Full Query", - query: DSQLQuery{ - Selection: QuerySelection{KeySelection: true, ValueSelection: true}, - Where: &sqlparser.AndExpr{ - Left: &sqlparser.ComparisonExpr{ - Operator: "like", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_key")}, - Right: &sqlparser.ColName{Name: sqlparser.NewColIdent("match:100:*")}, - }, - Right: &sqlparser.ComparisonExpr{ - Operator: "=", - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Right: &sqlparser.SQLVal{Val: []uint8("test")}, - }, - }, - OrderBy: QueryOrder{OrderBy: "_key", Order: "DESC"}, - Limit: 5, - }, - expected: "SELECT $key, $value WHERE $key like `match:100:*` and $value = 'test' ORDER BY $key DESC LIMIT 5", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.query.String() - if result != tt.expected { - t.Errorf("Expected %q, but got %q", tt.expected, result) - } - }) - } -} diff --git a/internal/sql/executerbechmark_test.go b/internal/sql/executerbechmark_test.go deleted file mode 100644 index 9f88b8775..000000000 --- a/internal/sql/executerbechmark_test.go +++ /dev/null @@ -1,424 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql_test - -import ( - "fmt" - "testing" - - "github.com/dicedb/dice/internal/object" - "github.com/dicedb/dice/internal/sql" - - "github.com/bytedance/sonic" - "github.com/dicedb/dice/config" - dstore "github.com/dicedb/dice/internal/store" -) - -var benchmarkDataSizes = []int{100, 1000, 10000, 100000, 1000000} -var benchmarkDataSizesStackQueue = []int{100, 1000, 10000} -var benchmarkDataSizesJSON = []int{100, 1000, 10000, 100000} - -var jsonList = map[string]string{ - "smallJSON": `{"score":10,"id":%d,"field1":{"field2":{"field3":{"score":10.36}}}}`, - "largeJSON": `{"score":10,"id":%d,"field1":{"field2":{"field3":{"score":10.36}}},"inventory":{"mountain_bikes":[{"id":"bike:1","model":"Phoebe","price":1920,"specs":{"material":"carbon","weight":13.1},"colors":["black","silver"]},{"id":"bike:2","model":"Quaoar","price":2072,"specs":{"material":"aluminium","weight":7.9},"colors":["black","white"]},{"id":"bike:3","model":"Weywot","price":3264,"specs":{"material":"alloy","weight":13.8}}],"commuter_bikes":[{"id":"bike:4","model":"Salacia","price":1475,"specs":{"material":"aluminium","weight":16.6},"colors":["black","silver"]},{"id":"bike:5","model":"Mimas","price":3941,"specs":{"material":"alloy","weight":11.6}}]}}`, -} - -func generateBenchmarkData(count int, store *dstore.Store) { - config.DiceConfig.Memory.KeysLimit = 2000000 // Set a high limit for benchmarking - store.ResetStore() - - data := make(map[string]*object.Obj, count) - for i := 0; i < count; i++ { - key := fmt.Sprintf("k%d", i) - value := fmt.Sprintf("v%d", i) - data[key] = store.NewObj(value, -1, object.ObjTypeString) - } - store.PutAll(data) -} - -func BenchmarkExecuteQueryOrderBykey(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - // Reset the timer to exclude the setup time from the benchmark - b.ResetTimer() - - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryBasicOrderByValue(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key like 'k*' ORDER BY $value ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryLimit(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := fmt.Sprintf("SELECT $key, $value WHERE $key like 'k*' ORDER BY $key ASC LIMIT %d", v/3) - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryNoMatch(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key like 'x*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithBasicWhere(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $value = 'v3' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithComplexWhere(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $value > 'v2' AND $value < 'v100' AND $key like 'k*' ORDER BY $value DESC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithCompareWhereKeyandValue(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key = $value AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithBasicWhereNoMatch(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $value = 'nonexistent' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithCaseSesnsitivity(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $value = 'V9' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithClauseOnKey(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key > 'k3' AND $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func BenchmarkExecuteQueryWithAllMatchingKeyRegex(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizes { - generateBenchmarkData(v, store) - - queryStr := "SELECT $key, $value WHERE $key like '*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("keys_%d", v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } -} - -func generateBenchmarkJSONData(b *testing.B, count int, json string, store *dstore.Store) { - config.DiceConfig.Memory.KeysLimit = 2000000 // Set a high limit for benchmarking - store.ResetStore() - - data := make(map[string]*object.Obj, count) - for i := 0; i < count; i++ { - key := fmt.Sprintf("k%d", i) - value := fmt.Sprintf(json, i) - - var jsonValue interface{} - if err := sonic.UnmarshalString(value, &jsonValue); err != nil { - b.Fatalf("Failed to unmarshal JSON: %v", err) - } - - data[key] = store.NewObj(jsonValue, -1, object.ObjTypeJSON) - } - store.PutAll(data) -} - -func BenchmarkExecuteQueryWithJSON(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizesJSON { - for jsonSize, json := range jsonList { - generateBenchmarkJSONData(b, v, json, store) - - queryStr := "SELECT $key, $value WHERE $key like 'k*' AND '$value.id' = 3 ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("%s_keys_%d", jsonSize, v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } - } -} - -func BenchmarkExecuteQueryWithNestedJSON(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizesJSON { - for jsonSize, json := range jsonList { - generateBenchmarkJSONData(b, v, json, store) - - queryStr := "SELECT $key, $value WHERE $key like 'k*' AND '$value.field1.field2.field3.score' > 10.1 ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("%s_keys_%d", jsonSize, v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } - } -} - -func BenchmarkExecuteQueryWithJsonInLeftAndRightExpressions(b *testing.B) { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for _, v := range benchmarkDataSizesJSON { - for jsonSize, json := range jsonList { - generateBenchmarkJSONData(b, v, json, store) - - queryStr := "SELECT $key, $value WHERE '$value.id' = '$value.score' AND $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("%s_keys_%d", jsonSize, v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } - } -} - -func BenchmarkExecuteQueryWithJsonNoMatch(b *testing.B) { - for _, v := range benchmarkDataSizesJSON { - store := dstore.NewStore(nil, nil, dstore.NewBatchEvictionLRU(config.DefaultKeysLimit, config.DefaultEvictionRatio)) - for jsonSize, json := range jsonList { - generateBenchmarkJSONData(b, v, json, store) - - queryStr := "SELECT $key, $value WHERE '$value.id' = 3 AND $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.Run(fmt.Sprintf("%s_keys_%d", jsonSize, v), func(b *testing.B) { - for i := 0; i < b.N; i++ { - if _, err := sql.ExecuteQuery(&query, store.GetStore()); err != nil { - b.Fatal(err) - } - } - }) - store.ResetStore() - } - } -} diff --git a/internal/sql/executor.go b/internal/sql/executor.go deleted file mode 100644 index c538fce56..000000000 --- a/internal/sql/executor.go +++ /dev/null @@ -1,523 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -import ( - "errors" - "fmt" - "sort" - "strconv" - "strings" - - "github.com/dicedb/dice/internal/object" - - "github.com/bytedance/sonic" - "github.com/dicedb/dice/internal/common" - "github.com/dicedb/dice/internal/regex" - "github.com/dicedb/dice/internal/server/utils" - "github.com/ohler55/ojg/jp" - "github.com/xwb1989/sqlparser" -) - -var ErrNoResultsFound = errors.New("ERR No results found") -var ErrInvalidJSONPath = errors.New("ERR invalid JSONPath") - -type QueryResultRow struct { - Key string - Value object.Obj // Use pointer to avoid copying -} - -type QueryResultRowWithOrder struct { - Row QueryResultRow - OrderByValue interface{} - OrderByType string -} - -func ExecuteQuery(query *DSQLQuery, store common.ITable[string, *object.Obj]) ([]QueryResultRow, error) { - var result []QueryResultRow - var err error - jsonPathCache := make(map[string]jp.Expr) // Cache for parsed JSON paths - - store.All(func(key string, value *object.Obj) bool { - row := QueryResultRow{ - Key: key, - Value: *value, // Use pointer to avoid copying - } - - if query.Where != nil { - match, evalErr := EvaluateWhereClause(query.Where, row, jsonPathCache) - if errors.Is(evalErr, ErrNoResultsFound) { - // if no result found error - // and continue with the next iteration - return true - } - if evalErr != nil { - err = evalErr - // stop iteration if any other error - return false - } - if !match { - // if did not match, continue the iteration - return true - } - } - - result = append(result, row) - - // Early termination if limit is reached and no sorting is required - if query.Limit > 0 && len(result) >= query.Limit && query.OrderBy.OrderBy == utils.EmptyStr { - return false - } - - return true - }) - - if err != nil { - return nil, err - } - - // Precompute order-by values and sort if necessary - if query.OrderBy.OrderBy != utils.EmptyStr { - resultWithOrder, err := precomputeOrderByValues(query, result, jsonPathCache) - if err != nil { - return nil, err - } - sortResults(query, resultWithOrder) - // Extract sorted rows - for i := range resultWithOrder { - result[i] = resultWithOrder[i].Row - } - } - - // Apply limit after sorting - if query.Limit > 0 && query.Limit < len(result) { - result = result[:query.Limit] - } - - // Marshal JSON results if necessary - for i := range result { - if err := MarshalResultIfJSON(&result[i]); err != nil { - return nil, err - } - } - - if !query.Selection.KeySelection { - for i := range result { - result[i].Key = utils.EmptyStr - } - } - - if !query.Selection.ValueSelection { - for i := range result { - result[i].Value = object.Obj{} - } - } - - return result, nil -} - -func MarshalResultIfJSON(row *QueryResultRow) error { - // if the row contains JSON field then convert the json object into string representation so it can be encoded - // before being returned to the client - if row.Value.Type == object.ObjTypeJSON { - marshaledData, err := sonic.MarshalString(row.Value.Value) - if err != nil { - return err - } - - row.Value.Value = marshaledData - } - return nil -} - -func precomputeOrderByValues(query *DSQLQuery, result []QueryResultRow, jsonPathCache map[string]jp.Expr) ([]QueryResultRowWithOrder, error) { - resultWithOrder := make([]QueryResultRowWithOrder, len(result)) - for i, row := range result { - val, valType, err := getOrderByValue(query.OrderBy.OrderBy, row, jsonPathCache) - if err != nil { - return nil, err - } - resultWithOrder[i] = QueryResultRowWithOrder{ - Row: row, - OrderByValue: val, - OrderByType: valType, - } - } - return resultWithOrder, nil -} - -func sortResults(query *DSQLQuery, result []QueryResultRowWithOrder) { - sort.Slice(result, func(i, j int) bool { - valI := result[i].OrderByValue - valJ := result[j].OrderByValue - typeI := result[i].OrderByType - typeJ := result[j].OrderByType - - // Handle Nil types - if typeI == Nil && typeJ == Nil { - return false // They are equal - } - if typeI == Nil { - return query.OrderBy.Order == Desc // Place Nil values at the beginning in Descending order - } - if typeJ == Nil { - return query.OrderBy.Order == Asc // Place Nil values at the end in Ascending order - } - - // Only compare values if types are the same - if typeI != typeJ { - return false // Types differ; cannot compare - } - - comparison, err := compareOrderByValues(valI, valJ, typeI, typeJ, query.OrderBy.Order) - if err != nil { - return false // Cannot compare; treat as equal - } - - return comparison - }) -} - -func getOrderByValue(orderBy string, row QueryResultRow, jsonPathCache map[string]jp.Expr) (value interface{}, valueType string, err error) { - switch orderBy { - case TempKey: - return row.Key, String, nil - case TempValue: - return getValueAndType(&row.Value) - default: - if isJSONField(&sqlparser.SQLVal{Val: []byte(orderBy)}, &row.Value) { - return retrieveValueFromJSON(orderBy, &row.Value, jsonPathCache) - } - } - return nil, "", fmt.Errorf("invalid ORDER BY clause: %s", orderBy) -} - -func compareOrderByValues(valI, valJ interface{}, typeI, typeJ, order string) (bool, error) { - // If types differ, define consistent ordering based on type names - if typeI != typeJ { - if order == Asc { - return typeI < typeJ, nil // Ascending order based on type names - } - return typeI > typeJ, nil // Descending order based on type names - } - - // Types are the same, proceed with comparison - switch typeI { - case String: - return compareStringValues(order, valI.(string), valJ.(string)), nil - case Int64: - return compareInt64Values(order, valI.(int64), valJ.(int64)), nil - case Float: - return compareFloatValues(order, valI.(float64), valJ.(float64)), nil - case Bool: - return compareBoolValues(order, valI.(bool), valJ.(bool)), nil - default: - return false, fmt.Errorf("unsupported type for comparison: %s", typeI) - } -} - -func compareStringValues(order, valI, valJ string) bool { - if order == Asc { - return valI < valJ - } - return valI > valJ -} - -func compareInt64Values(order string, valI, valJ int64) bool { - if order == Asc { - return valI < valJ - } - return valI > valJ -} - -func compareFloatValues(order string, valI, valJ float64) bool { - if order == Asc { - return valI < valJ - } - return valI > valJ -} - -func compareBoolValues(order string, valI, valJ bool) bool { - if order == Asc { - return !valI && valJ - } - return valI && !valJ -} - -func EvaluateWhereClause(expr sqlparser.Expr, row QueryResultRow, jsonPathCache map[string]jp.Expr) (bool, error) { - switch expr := expr.(type) { - case *sqlparser.ParenExpr: - return EvaluateWhereClause(expr.Expr, row, jsonPathCache) - case *sqlparser.ComparisonExpr: - return evaluateComparison(expr, row, jsonPathCache) - case *sqlparser.AndExpr: - left, err := EvaluateWhereClause(expr.Left, row, jsonPathCache) - if err != nil || !left { - return false, err - } - return EvaluateWhereClause(expr.Right, row, jsonPathCache) - case *sqlparser.OrExpr: - left, err := EvaluateWhereClause(expr.Left, row, jsonPathCache) - if err != nil { - return false, err - } - if left { - return true, nil - } - return EvaluateWhereClause(expr.Right, row, jsonPathCache) - default: - return false, fmt.Errorf("unsupported expression type: %T", expr) - } -} - -func evaluateComparison(expr *sqlparser.ComparisonExpr, row QueryResultRow, jsonPathCache map[string]jp.Expr) (bool, error) { - left, leftType, err := getExprValueAndType(expr.Left, row, jsonPathCache) - if err != nil { - if errors.Is(err, ErrNoResultsFound) { - return false, nil - } - return false, err - } - right, rightType, err := getExprValueAndType(expr.Right, row, jsonPathCache) - if err != nil { - if errors.Is(err, ErrNoResultsFound) { - return false, nil - } - return false, err - } - - // Handle Nil types - if leftType == Nil || rightType == Nil { - // Comparisons with NULL result in FALSE - return false, nil - } - - // Check if types are the same - if leftType != rightType { - return false, fmt.Errorf("incompatible types in comparison: %s and %s", leftType, rightType) - } - - switch leftType { - case String: - return compareStrings(left.(string), right.(string), expr.Operator) - case Int64: - return compareInt64s(left.(int64), right.(int64), expr.Operator) - case Float: - return compareFloats(left.(float64), right.(float64), expr.Operator) - default: - return false, fmt.Errorf("unsupported type for comparison: %s", leftType) - } -} - -func getExprValueAndType(expr sqlparser.Expr, row QueryResultRow, jsonPathCache map[string]jp.Expr) (value interface{}, valueType string, err error) { - switch expr := expr.(type) { - case *sqlparser.ColName: - switch expr.Name.String() { - case TempKey: - return row.Key, String, nil - case TempValue: - return getValueAndType(&row.Value) - default: - return nil, "", fmt.Errorf("unknown column: %s", expr.Name.String()) - } - case *sqlparser.SQLVal: - // we currently treat JSON query expression as a string value so we will need to differentiate between JSON and - // SQL strings - if isJSONField(expr, &row.Value) { - return retrieveValueFromJSON(string(expr.Val), &row.Value, jsonPathCache) - } - return sqlValToGoValue(expr) - case *sqlparser.NullVal: - return nil, Nil, nil - default: - return nil, "", fmt.Errorf("unsupported expression type: %T", expr) - } -} - -func isJSONField(expr *sqlparser.SQLVal, obj *object.Obj) bool { - if err := object.AssertType(obj.Type, object.ObjTypeJSON); err != nil { - return false - } - - // We convert the $key and $value fields to _key, _value before querying. hence fields starting with _ are - // considered to be stored values - return expr.Type == sqlparser.StrVal && - strings.HasPrefix(string(expr.Val), TempPrefix) -} - -func retrieveValueFromJSON(path string, jsonData *object.Obj, jsonPathCache map[string]jp.Expr) (value interface{}, valueType string, err error) { - // path is in the format '_value.field1.field2'. We need to remove _value reference from the prefix to get the json - // path. - jsonPath := strings.Split(path, ".") - if len(jsonPath) < 2 { - return nil, "", ErrInvalidJSONPath - } - - pathKey := "$." + strings.Join(jsonPath[1:], ".") - - expr, exists := jsonPathCache[pathKey] - if !exists { - exprParsed, err := jp.ParseString(pathKey) - if err != nil { - return nil, "", ErrInvalidJSONPath - } - jsonPathCache[pathKey] = exprParsed - expr = exprParsed - } - - results := expr.Get(jsonData.Value) - if len(results) == 0 { - return nil, Nil, nil // Return nil value with Nil type - } - - return inferTypeAndConvert(results[0]) -} - -// inferTypeAndConvert infers the type of the value and converts it to the appropriate type. currently the only data -// types we support are strings, floats, ints, booleans and nil. -func inferTypeAndConvert(val interface{}) (value interface{}, valueType string, err error) { - switch v := val.(type) { - case string: - return v, String, nil - case float64: - if isInt64(v) { - return int64(v), Int64, nil - } - return v, Float, nil - case bool: - return v, Bool, nil - case nil: - return nil, Nil, nil - default: - return nil, utils.EmptyStr, fmt.Errorf("unsupported JSONPath result type: %T", v) - } -} - -// isInt64 checks if a float is an integer. When we unmarshal JSON data into an interface it sets all numbers as -// floats, https://forum.golangbridge.org/t/type-problem-in-json-conversion/19420. -// This function does not handle the edge case where user enters a floating point number with trailing zeros in the -// fractional part of a decimal number (e.g 10.0) then our code treats that as an integer rather than float. -// One way to solve this is as follows (credit https://github.com/YahyaHaq): -// floatStr := strconv.FormatFloat(val, 'f', -1, 64) -// if strings.Contains(floatStr, ".") { -// return val, "float", nil -// } -// return int(val), "int", nil -// -// However, the string conversion is expensive and we are trying to avoid it. We can assume this to be a limitation of -// using the JSON data type. -// TODO: handle the edge case where the integer is too large for float64. -func isInt64(f float64) bool { - return f == float64(int64(f)) -} - -// getValueAndType returns the type-casted value and type of the object -func getValueAndType(obj *object.Obj) (val interface{}, s string, e error) { - switch v := obj.Value.(type) { - case string: - return v, String, nil - case int64: - return v, Int64, nil - case float64: - return v, Float, nil - default: - return nil, utils.EmptyStr, fmt.Errorf("unsupported value type: %T", v) - } -} - -// sqlValToGoValue converts SQLVal to Go value, and returns the type of the value. -func sqlValToGoValue(sqlVal *sqlparser.SQLVal) (val interface{}, s string, e error) { - switch sqlVal.Type { - case sqlparser.StrVal: - return string(sqlVal.Val), String, nil - case sqlparser.IntVal: - i, err := strconv.ParseInt(string(sqlVal.Val), 10, 64) - if err != nil { - return nil, utils.EmptyStr, err - } - return i, Int64, nil - case sqlparser.FloatVal: - f, err := strconv.ParseFloat(string(sqlVal.Val), 64) - if err != nil { - return nil, utils.EmptyStr, err - } - return f, Float, nil - default: - return nil, utils.EmptyStr, fmt.Errorf("unsupported SQLVal type: %v", sqlVal.Type) - } -} - -func compareStrings(left, right, operator string) (bool, error) { - switch strings.ToLower(operator) { - case sqlparser.EqualStr: - return left == right, nil - case sqlparser.NotEqualStr: - return left != right, nil - case sqlparser.LessThanStr: - return left < right, nil - case sqlparser.LessEqualStr: - return left <= right, nil - case sqlparser.GreaterThanStr: - return left > right, nil - case sqlparser.GreaterEqualStr: - return left >= right, nil - case sqlparser.LikeStr: - return regex.WildCardMatch(right, left), nil - case sqlparser.NotLikeStr: - return !regex.WildCardMatch(right, left), nil - default: - return false, fmt.Errorf("unsupported operator for strings: %s", operator) - } -} - -func compareInt64s(left, right int64, operator string) (bool, error) { - switch operator { - case sqlparser.EqualStr: - return left == right, nil - case sqlparser.NotEqualStr: - return left != right, nil - case sqlparser.LessThanStr: - return left < right, nil - case sqlparser.LessEqualStr: - return left <= right, nil - case sqlparser.GreaterThanStr: - return left > right, nil - case sqlparser.GreaterEqualStr: - return left >= right, nil - default: - return false, fmt.Errorf("unsupported operator for integers: %s", operator) - } -} - -func compareFloats(left, right float64, operator string) (bool, error) { - switch operator { - case sqlparser.EqualStr: - return left == right, nil - case sqlparser.NotEqualStr: - return left != right, nil - case sqlparser.LessThanStr: - return left < right, nil - case sqlparser.LessEqualStr: - return left <= right, nil - case sqlparser.GreaterThanStr: - return left > right, nil - case sqlparser.GreaterEqualStr: - return left >= right, nil - default: - return false, fmt.Errorf("unsupported operator for floats: %s", operator) - } -} diff --git a/internal/sql/executor_test.go b/internal/sql/executor_test.go deleted file mode 100644 index e4af51528..000000000 --- a/internal/sql/executor_test.go +++ /dev/null @@ -1,736 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql_test - -import ( - "sort" - "testing" - - "github.com/bytedance/sonic" - "github.com/dicedb/dice/internal/object" - "github.com/dicedb/dice/internal/server/utils" - "github.com/dicedb/dice/internal/sql" - dstore "github.com/dicedb/dice/internal/store" - "github.com/stretchr/testify/assert" - "github.com/xwb1989/sqlparser" -) - -type keyValue struct { - key string - value string -} - -var ( - simpleKVDataset = []keyValue{ - {"k2", "v4"}, - {"k4", "v2"}, - {"k3", "v3"}, - {"k5", "v1"}, - {"k1", "v5"}, - {"k", "k"}, - } -) - -func setup(store *dstore.Store, dataset []keyValue) { - // delete all keys - for _, data := range dataset { - store.Del(data.key) - } - - for _, data := range dataset { - store.Put(data.key, &object.Obj{Value: data.value}) - } -} - -func TestExecuteQueryOrderBykey(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - queryString := "SELECT $key, $value WHERE $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryString) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), len(simpleKVDataset)) - - sortedDataset := make([]keyValue, len(simpleKVDataset)) - copy(sortedDataset, simpleKVDataset) - - // Sort the new dataset by the "key" field - sort.Slice(sortedDataset, func(i, j int) bool { - return sortedDataset[i].key < sortedDataset[j].key - }) - - for i, data := range sortedDataset { - assert.Equal(t, result[i].Key, data.key) - assert.Equal(t, result[i].Value.Value, data.value) - } -} - -func TestExecuteQueryBasicOrderByValue(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - queryStr := "SELECT $key, $value WHERE $key like 'k*' ORDER BY $value ASC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), len(simpleKVDataset)) - - sortedDataset := make([]keyValue, len(simpleKVDataset)) - copy(sortedDataset, simpleKVDataset) - - // Sort the new dataset by the "value" field - sort.Slice(sortedDataset, func(i, j int) bool { - return sortedDataset[i].value < sortedDataset[j].value - }) - - for i, data := range sortedDataset { - assert.Equal(t, result[i].Key, data.key) - assert.Equal(t, result[i].Value.Value, data.value) - } -} - -func TestExecuteQueryLimit(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - queryStr := "SELECT $value WHERE $key like 'k*' ORDER BY $key ASC LIMIT 3" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 3) // Checks if limit is respected - - sortedDataset := make([]keyValue, len(simpleKVDataset)) - copy(sortedDataset, simpleKVDataset) - - // Sort the new dataset by the "key" field - sort.Slice(sortedDataset, func(i, j int) bool { - return sortedDataset[i].key < sortedDataset[j].key - }) - - for i, data := range sortedDataset[:3] { - assert.Equal(t, result[i].Key, utils.EmptyStr) - assert.Equal(t, result[i].Value.Value, data.value) - } -} - -func TestExecuteQueryNoMatch(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - queryStr := "SELECT $key, $value WHERE $key like 'x*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 0) // No keys match "x*" -} - -func TestExecuteQueryWithWhere(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - t.Run("BasicWhereClause", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $value = 'v3' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for WHERE clause") - assert.Equal(t, result[0].Key, "k3") - assert.Equal(t, result[0].Value.Value, "v3") - }) - - t.Run("EmptyResult", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $value = 'nonexistent' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 0, "Expected empty result for non-matching WHERE clause") - }) - - t.Run("ComplexWhereClause", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $value > 'v2' AND $value < 'v5' AND $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 2, "Expected 2 results for complex WHERE clause") - assert.Equal(t, []string{result[0].Key, result[1].Key}, []string{"k2", "k3"}) - }) - - t.Run("ComparingKeyWithValue", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key = $value" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for comparison between key and value") - assert.Equal(t, result[0].Key, "k") - assert.Equal(t, result[0].Value.Value, "k") - }) -} - -func TestExecuteQueryWithIncompatibleTypes(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - t.Run("ComparingStrWithInt", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $value = 42 AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - _, err = sql.ExecuteQuery(&query, store.GetStore()) - - assert.Error(t, err, "incompatible types in comparison: string and int64") - }) -} - -func TestExecuteQueryWithEdgeCases(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, simpleKVDataset) - - t.Run("CaseSensitivity", func(t *testing.T) { - query := sql.DSQLQuery{ - Selection: sql.QuerySelection{ - KeySelection: true, - ValueSelection: true, - }, - Where: &sqlparser.ComparisonExpr{ - Left: &sqlparser.ColName{Name: sqlparser.NewColIdent("_value")}, - Operator: "=", - Right: sqlparser.NewStrVal([]byte("V3")), // Uppercase V3 - }, - } - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 0, "Expected 0 results due to case sensitivity") - }) - - t.Run("WhereClauseOnKey", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key > 'k3' AND $key like 'k*' ORDER BY $key ASC" - query, err := sql.ParseQuery(queryStr) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 2, "Expected 2 results for WHERE clause on key") - assert.Equal(t, []string{result[0].Key, result[1].Key}, []string{"k4", "k5"}) - }) - - t.Run("UnsupportedOperator", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $value regexp '%3' AND $key like 'k*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - _, err = sql.ExecuteQuery(&query, store.GetStore()) - - assert.ErrorContains(t, err, "unsupported operator") - }) - - t.Run("EmptyKeyRegex", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like ''" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 0, "Expected no keys to be returned for empty regex") - }) -} - -var jsonWhereClauseDataset = []keyValue{ - {"json1", `{"name":"Tom"}`}, - {"json2", `{"name":"Bob","score":18.1}`}, - {"json3", `{"scoreInt":20}`}, - {"json4", `{"field1":{"field2":{"field3":{"score":2}}}}`}, - {"json5", `{"field1":{"field2":{"field3":{"score":18}},"score2":5}}`}, -} - -func setupJSON(t *testing.T, store *dstore.Store, dataset []keyValue) { - t.Helper() - for _, data := range dataset { - store.Del(data.key) - } - - for _, data := range dataset { - var jsonValue interface{} - if err := sonic.UnmarshalString(data.value, &jsonValue); err != nil { - t.Fatalf("Failed to unmarshal value: %v", err) - } - - store.Put(data.key, store.NewObj(jsonValue, -1, object.ObjTypeJSON)) - } -} - -func TestExecuteQueryWithJsonExpressionInWhere(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setupJSON(t, store, jsonWhereClauseDataset) - - t.Run("BasicWhereClauseWithJSON", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.name' = 'Tom' AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 results for WHERE clause") - assert.Equal(t, result[0].Key, "json1") - - var expected, actual interface{} - assert.Nil(t, sonic.UnmarshalString(`{"name":"Tom"}`, &expected)) - assert.Nil(t, sonic.UnmarshalString(result[0].Value.Value.(string), &actual)) - assert.Equal(t, actual, expected) - }) - - t.Run("EmptyResult", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.name' = 'Bill' AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 0, "Expected empty result for non-matching WHERE clause") - }) - - t.Run("WhereClauseWithFloats", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.score' > 13.15 AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for WHERE clause with floating point values") - assert.Equal(t, result[0].Key, "json2") - - var expected, actual interface{} - assert.Nil(t, sonic.UnmarshalString(`{"name":"Bob","score":18.1}`, &expected)) - assert.Nil(t, sonic.UnmarshalString(result[0].Value.Value.(string), &actual)) - assert.Equal(t, actual, expected) - }) - - t.Run("WhereClauseWithInteger", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.scoreInt' > 13 AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for WHERE clause with integer values") - assert.Equal(t, result[0].Key, "json3") - - var expected, actual interface{} - assert.Nil(t, sonic.UnmarshalString(`{"scoreInt":20}`, &expected)) - assert.Nil(t, sonic.UnmarshalString(result[0].Value.Value.(string), &actual)) - assert.Equal(t, actual, expected) - }) - - t.Run("NestedWhereClause", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.field1.field2.field3.score' < 13 AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for WHERE clause with nested json") - assert.Equal(t, result[0].Key, "json4") - - var expected, actual interface{} - assert.Nil(t, sonic.UnmarshalString(`{"field1":{"field2":{"field3":{"score":2}}}}`, &expected)) - assert.Nil(t, sonic.UnmarshalString(result[0].Value.Value.(string), &actual)) - assert.Equal(t, actual, expected) - }) - - t.Run("ComplexWhereClause", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE '$value.field1.field2.field3.score' > '$value.field1.score2' AND $key like 'json*'" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), 1, "Expected 1 result for Complex WHERE clause expression") - assert.Equal(t, result[0].Key, "json5") - - var expected, actual interface{} - assert.Nil(t, sonic.UnmarshalString(`{"field1":{"field2":{"field3":{"score":18}},"score2":5}}`, &expected)) - assert.Nil(t, sonic.UnmarshalString(result[0].Value.Value.(string), &actual)) - assert.Equal(t, actual, expected) - }) -} - -var jsonOrderDataset = []keyValue{ - {"json3", `{"name":"Alice", "age":35, "scoreInt":20, "nested":{"field":{"value":15}}}`}, - {"json2", `{"name":"Bob", "age":25, "score":18.1, "nested":{"field":{"value":40}}}`}, - {"json1", `{"name":"Tom", "age":30, "nested":{"field":{"value":20}}}`}, - {"json5", `{"name":"Charlie", "age":50, "nested":{"field":{"value":19}}}`}, - {"json4", `{"name":"Eve", "age":32, "nested":{"field":{"value":60}}}`}, -} - -func TestExecuteQueryWithJsonOrderBy(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setupJSON(t, store, jsonOrderDataset) - - t.Run("OrderBySimpleJSONField", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' ORDER BY $value.name ASC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, 5, len(result), "Expected 5 results") - - assert.Equal(t, "json3", result[0].Key) // Alice - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[0].value, result[0].Value.Value.(string)) - - assert.Equal(t, "json2", result[1].Key) // Bob - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[1].value, result[1].Value.Value.(string)) - - assert.Equal(t, "json5", result[2].Key) // Charlie - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[3].value, result[2].Value.Value.(string)) - - assert.Equal(t, "json4", result[3].Key) // Eve - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[4].value, result[3].Value.Value.(string)) - - assert.Equal(t, "json1", result[4].Key) // Tom - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[2].value, result[4].Value.Value.(string)) - }) - - t.Run("OrderByNumericJSONField", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' ORDER BY $value.age DESC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, 5, len(result)) - - assert.Equal(t, "json5", result[0].Key) // Charlie, age 50 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[3].value, result[0].Value.Value.(string)) - - assert.Equal(t, "json3", result[1].Key) // Alice, age 35 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[0].value, result[1].Value.Value.(string)) - - assert.Equal(t, "json4", result[2].Key) // Eve, age 32 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[4].value, result[2].Value.Value.(string)) - - assert.Equal(t, "json1", result[3].Key) // Tom, age 30 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[2].value, result[3].Value.Value.(string)) - - assert.Equal(t, "json2", result[4].Key) // Bob, age 25 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[1].value, result[4].Value.Value.(string)) - }) - - t.Run("OrderByNestedJSONField", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' ORDER BY '$value.nested.field.value' ASC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, 5, len(result)) - assert.Equal(t, "json3", result[0].Key) // Alice, nested.field.value: 15 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[0].value, result[0].Value.Value.(string)) - - assert.Equal(t, "json5", result[1].Key) // Charlie, nested.field.value: 19 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[3].value, result[1].Value.Value.(string)) - - assert.Equal(t, "json1", result[2].Key) // Tom, nested.field.value: 20 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[2].value, result[2].Value.Value.(string)) - - assert.Equal(t, "json2", result[3].Key) // Bob, nested.field.value: 40 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[1].value, result[3].Value.Value.(string)) - - assert.Equal(t, "json4", result[4].Key) // Eve, nested.field.value: 60 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[4].value, result[4].Value.Value.(string)) - }) - - t.Run("OrderByMixedTypes", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' ORDER BY $value.score DESC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - // No ordering guarantees for mixed types. - assert.Equal(t, 5, len(result)) - }) - - t.Run("OrderByWithWhereClause", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' AND '$value.age' > 30 ORDER BY $value.name DESC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, 3, len(result), "Expected 3 results (age > 30, ordered by name)") - assert.Equal(t, "json4", result[0].Key) // Eve, age 32 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[4].value, result[0].Value.Value.(string)) - - assert.Equal(t, "json5", result[1].Key) // Charlie, age 50 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[3].value, result[1].Value.Value.(string)) - - assert.Equal(t, "json3", result[2].Key) // Alice, age 35 - validateJSONStringRepresentationsAreEqual(t, jsonOrderDataset[0].value, result[2].Value.Value.(string)) - }) - - t.Run("OrderByNonExistentField", func(t *testing.T) { - queryStr := "SELECT $key, $value WHERE $key like 'json*' ORDER BY $value.nonexistent ASC" - query, err := sql.ParseQuery(queryStr) - assert.Nil(t, err) - - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - // No ordering guarantees for non-existent field references. - assert.Equal(t, 5, len(result), "Expected 5 results") - }) -} - -// validateJSONStringRepresentationsAreEqual unmarshals the expected and actual JSON strings and performs a deep comparison. -func validateJSONStringRepresentationsAreEqual(t *testing.T, expectedJSONString, actualJSONString string) { - t.Helper() - var expectedValue, actualValue interface{} - assert.Nil(t, sonic.UnmarshalString(expectedJSONString, &expectedValue)) - assert.Nil(t, sonic.UnmarshalString(actualJSONString, &actualValue)) - assert.Equal(t, actualValue, expectedValue) -} - -// Dataset will be used for LIKE comparisons -var stringComparisonDataset = []keyValue{ - {"user:1", "Alice Smith"}, - {"user:2", "Bob Johnson"}, - {"user:3", "Charlie Brown"}, - {"user:4", "David Lee"}, - {"user:5", "Eve Wilson"}, - {"product:1", "Red Apple"}, - {"product:2", "Green Banana"}, - {"product:3", "Yellow Lemon"}, - {"product:4", "Orange Orange"}, - {"product:5", "Purple Grape"}, - {"email:1", "alice@example.com"}, - {"email:2", "bob@test.org"}, - {"email:3", "charlie@gmail.com"}, - {"email:4", "david@company.net"}, - {"email:5", "eve@domain.io"}, - {"desc:1", "This is a short description"}, - {"desc:2", "A slightly longer description with more words"}, - {"desc:3", "Description containing numbers 123 and symbols !@#"}, - {"desc:4", "UPPERCASE DESCRIPTION"}, - {"desc:5", "mixed CASE DeScRiPtIoN"}, - {"tag:1", "important"}, - {"tag:2", "urgent"}, - {"tag:3", "low-priority"}, - {"tag:4", "follow-up"}, - {"tag:5", "archived"}, -} - -func TestExecuteQueryWithLikeStringComparisons(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, stringComparisonDataset) - - testCases := []struct { - name string - query string - expectLen int - expectKeys []string - }{ - { - name: "NamesStartingWithA", - query: "SELECT $key, $value WHERE $value LIKE 'A*' AND $key LIKE 'user:*'", - expectLen: 1, - expectKeys: []string{"user:1"}, - }, - { - name: "EmailsWithGmailDomain", - query: "SELECT $key, $value WHERE $value LIKE '*@gmail.com' AND $key LIKE 'email:*'", - expectLen: 1, - expectKeys: []string{"email:3"}, - }, - { - name: "DescriptionsContainingWord", - query: "SELECT $key, $value WHERE $value LIKE '*description*' AND $key LIKE 'desc:*' ORDER BY $key ASC", - expectLen: 2, - expectKeys: []string{"desc:1", "desc:2"}, - }, - { - name: "CaseInsensitiveMatching", - query: "SELECT $key, $value WHERE $value LIKE '*UPPERCASE*' AND $key LIKE 'desc:*'", - expectLen: 1, - expectKeys: []string{"desc:4"}, - }, - { - name: "MatchingSpecialCharacters", - query: "SELECT $key, $value WHERE $value LIKE '*!@#*' AND $key LIKE 'desc:*'", - expectLen: 1, - expectKeys: []string{"desc:3"}, - }, - { - name: "MatchingNumbers", - query: "SELECT $key, $value WHERE $value LIKE '*123*' AND $key LIKE 'desc:*'", - expectLen: 1, - expectKeys: []string{"desc:3"}, - }, - { - name: "ProductsContainingColor", - query: "SELECT $key, $value WHERE $value LIKE '*Red*' AND $key LIKE 'product:*'", - expectLen: 1, - expectKeys: []string{"product:1"}, - }, - { - name: "TagsEndingWithPriority", - query: "SELECT $key, $value WHERE $value LIKE '*priority' AND $key LIKE 'tag:*'", - expectLen: 1, - expectKeys: []string{"tag:3"}, - }, - { - name: "NamesWith5Characters", - query: "SELECT $key, $value WHERE $value LIKE '???????????' AND $key LIKE 'user:*' ORDER BY $key ASC", - expectLen: 2, - expectKeys: []string{"user:1", "user:2"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - query, err := sql.ParseQuery(tc.query) - assert.Nil(t, err) - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), tc.expectLen, "Expected %d results, got %d", tc.expectLen, len(result)) - - resultKeys := make([]string, len(result)) - for i, r := range result { - resultKeys[i] = r.Key - } - - assert.Equal(t, resultKeys, tc.expectKeys) - }) - } -} - -func TestExecuteQueryWithStringNotLikeComparisons(t *testing.T) { - store := dstore.NewStore(nil, nil, nil) - setup(store, stringComparisonDataset) - - testCases := []struct { - name string - query string - expectLen int - expectKeys []string - }{ - { - name: "NamesNotStartingWithA", - query: "SELECT $key, $value WHERE $value NOT LIKE 'A*' AND $key LIKE 'user:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"user:2", "user:3", "user:4", "user:5"}, - }, - { - name: "EmailsNotWithGmailDomain", - query: "SELECT $key, $value WHERE $value NOT LIKE '*@gmail.com' AND $key LIKE 'email:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"email:1", "email:2", "email:4", "email:5"}, - }, - { - name: "DescriptionsNotContainingWord", - query: "SELECT $key, $value WHERE $value NOT LIKE '*description*' AND $key LIKE 'desc:*' ORDER BY $key ASC", - expectLen: 3, - expectKeys: []string{"desc:3", "desc:4", "desc:5"}, - }, - { - name: "NotCaseInsensitiveMatching", - query: "SELECT $key, $value WHERE $value NOT LIKE '*UPPERCASE*' AND $key LIKE 'desc:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"desc:1", "desc:2", "desc:3", "desc:5"}, - }, - { - name: "NotMatchingSpecialCharacters", - query: "SELECT $key, $value WHERE $value NOT LIKE '*!@#*' AND $key LIKE 'desc:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"desc:1", "desc:2", "desc:4", "desc:5"}, - }, - { - name: "ProductsNotContainingColor", - query: "SELECT $key, $value WHERE $value NOT LIKE '*Red*' AND $key LIKE 'product:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"product:2", "product:3", "product:4", "product:5"}, - }, - { - name: "TagsNotEndingWithPriority", - query: "SELECT $key, $value WHERE $value NOT LIKE '*priority' AND $key LIKE 'tag:*' ORDER BY $key ASC", - expectLen: 4, - expectKeys: []string{"tag:1", "tag:2", "tag:4", "tag:5"}, - }, - { - name: "NamesNotWith5Characters", - query: "SELECT $key, $value WHERE $value NOT LIKE '???????????' AND $key LIKE 'user:*' ORDER BY $key ASC", - expectLen: 3, - expectKeys: []string{"user:3", "user:4", "user:5"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - query, err := sql.ParseQuery(tc.query) - assert.Nil(t, err) - result, err := sql.ExecuteQuery(&query, store.GetStore()) - - assert.Nil(t, err) - assert.Equal(t, len(result), tc.expectLen, "Expected %d results, got %d", tc.expectLen, len(result)) - - resultKeys := make([]string, len(result)) - for i, r := range result { - resultKeys[i] = r.Key - } - - assert.Equal(t, resultKeys, tc.expectKeys) - }) - } -} diff --git a/internal/sql/fingerprint.go b/internal/sql/fingerprint.go deleted file mode 100644 index 6ee9150b0..000000000 --- a/internal/sql/fingerprint.go +++ /dev/null @@ -1,121 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -import ( - "fmt" - "sort" - "strings" - - hash "github.com/dgryski/go-farm" - "github.com/xwb1989/sqlparser" -) - -// OR terms containing AND expressions -type expression [][]string - -func (expr expression) String() string { - var orTerms []string - for _, andTerm := range expr { - // Sort AND terms within OR - sort.Strings(andTerm) - orTerms = append(orTerms, strings.Join(andTerm, " AND ")) - } - // Sort the OR terms - sort.Strings(orTerms) - return strings.Join(orTerms, " OR ") -} - -func generateFingerprint(where sqlparser.Expr) string { - expr := parseAstExpression(where) - return fmt.Sprintf("f_%d", hash.Hash64([]byte(expr.String()))) -} - -func parseAstExpression(expr sqlparser.Expr) expression { - switch expr := expr.(type) { - case *sqlparser.AndExpr: - leftExpr := parseAstExpression(expr.Left) - rightExpr := parseAstExpression(expr.Right) - return combineAnd(leftExpr, rightExpr) - case *sqlparser.OrExpr: - leftExpr := parseAstExpression(expr.Left) - rightExpr := parseAstExpression(expr.Right) - return combineOr(leftExpr, rightExpr) - case *sqlparser.ParenExpr: - return parseAstExpression(expr.Expr) - case *sqlparser.ComparisonExpr: - return expression([][]string{{expr.Operator + sqlparser.String(expr.Left) + sqlparser.String(expr.Right)}}) - default: - return expression{} - } -} - -func combineAnd(a, b expression) expression { - result := make(expression, 0, len(a)+len(b)) - for _, termA := range a { - for _, termB := range b { - combined := make([]string, len(termA), len(termA)+len(termB)) - copy(combined, termA) - combined = append(combined, termB...) - uniqueCombined := removeDuplicates(combined) - sort.Strings(uniqueCombined) - result = append(result, uniqueCombined) - } - } - return result -} - -func combineOr(a, b expression) expression { - result := make(expression, 0, len(a)+len(b)) - uniqueTerms := make(map[string]bool) - - // Helper function to add unique terms - addUnique := func(terms []string) { - // Sort the terms for consistent ordering - sort.Strings(terms) - key := strings.Join(terms, ",") - if !uniqueTerms[key] { - result = append(result, terms) - uniqueTerms[key] = true - } - } - - // Add unique terms from a - for _, terms := range a { - addUnique(append([]string(nil), terms...)) - } - - // Add unique terms from b - for _, terms := range b { - addUnique(append([]string(nil), terms...)) - } - - return result -} - -// helper -func removeDuplicates(input []string) []string { - seen := make(map[string]struct{}) - var result []string - for _, v := range input { - if _, exists := seen[v]; !exists { - seen[v] = struct{}{} - result = append(result, v) - } - } - return result -} diff --git a/internal/sql/fingerprint_test.go b/internal/sql/fingerprint_test.go deleted file mode 100644 index b094b003b..000000000 --- a/internal/sql/fingerprint_test.go +++ /dev/null @@ -1,418 +0,0 @@ -// This file is part of DiceDB. -// Copyright (C) 2024 DiceDB (dicedb.io). -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package sql - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/xwb1989/sqlparser" -) - -func TestExpressionString(t *testing.T) { - tests := []struct { - name string - expr expression - expected string - }{ - { - name: "Single AND term", - expr: expression{{"_key LIKE 'match:1:*'", "_value > 10"}}, - expected: "_key LIKE 'match:1:*' AND _value > 10", - }, - { - name: "Multiple AND terms in a single OR term", - expr: expression{{"_key LIKE 'match:1:*'", "_value > 10", "_value < 5"}}, - expected: "_key LIKE 'match:1:*' AND _value < 5 AND _value > 10", - }, - { - name: "Multiple OR terms with single AND terms", - expr: expression{{"_key LIKE 'match:1:*'"}, {"_value < 5"}, {"_value > 10"}}, - expected: "_key LIKE 'match:1:*' OR _value < 5 OR _value > 10", - }, - { - name: "Multiple OR terms with AND combinations", - expr: expression{{"_key LIKE 'match:1:*'", "_value > 10"}, {"_value < 5", "_value > 0"}}, - expected: "_key LIKE 'match:1:*' AND _value > 10 OR _value < 5 AND _value > 0", - }, - { - name: "Unordered terms", - expr: expression{{"_value > 10", "_key LIKE 'match:1:*'"}, {"_value > 0", "_value < 5"}}, - expected: "_key LIKE 'match:1:*' AND _value > 10 OR _value < 5 AND _value > 0", - }, - { - name: "Nested AND and OR terms with duplicates", - expr: expression{{"_key LIKE 'match:1:*'", "_value < 5"}, {"_key LIKE 'match:1:*'", "_value < 5", "_value > 10"}}, - expected: "_key LIKE 'match:1:*' AND _value < 5 OR _key LIKE 'match:1:*' AND _value < 5 AND _value > 10", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, test.expr.String()) - }) - } -} - -func TestCombineOr(t *testing.T) { - tests := []struct { - name string - a expression - b expression - expected expression - }{ - { - name: "Combining two empty expressions", - a: expression([][]string{}), - b: expression([][]string{}), - expected: expression([][]string{}), - }, - { - name: "Identity law", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{}), // equivalent to 0 - expected: expression([][]string{{"_value > 10"}}), - }, - { - name: "Idempotent law", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{{"_value > 10"}}), - expected: expression([][]string{{"_value > 10"}}), - }, - { - name: "Simple OR combination with non-overlapping terms", - a: expression([][]string{{"_key LIKE 'test:*'"}}), - b: expression([][]string{{"_value > 10"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'"}, {"_value > 10"}, - }), - }, - { - name: "Complex OR combination with multiple AND terms", - a: expression([][]string{{"_key LIKE 'test:*'", "_value > 10"}}), - b: expression([][]string{{"_key LIKE 'example:*'", "_value < 5"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value > 10"}, {"_key LIKE 'example:*'", "_value < 5"}, - }), - }, - { - name: "Combining overlapping AND terms", - a: expression([][]string{{"_key LIKE 'test:*'", "_value > 10"}}), - b: expression([][]string{{"_value > 10", "_key LIKE 'test:*'"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value > 10"}, - }), - }, - { - name: "Combining overlapping AND terms in reverse order", - a: expression([][]string{{"_value > 10", "_key LIKE 'test:*'"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value > 10"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value > 10"}, - }), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, combineOr(tt.a, tt.b)) - }) - } -} - -func TestCombineAnd(t *testing.T) { - tests := []struct { - name string - a expression - b expression - expected expression - }{ - { - name: "Combining two empty expressions", - a: expression([][]string{}), - b: expression([][]string{}), - expected: expression([][]string{}), - }, - { - name: "Annulment law", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{}), // equivalent to 0 - expected: expression([][]string{}), - }, - { - name: "Identity law", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{{}}), // equivalent to 1 - expected: expression([][]string{{"_value > 10"}}), - }, - { - name: "Idempotent law", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{{"_value > 10"}}), - expected: expression([][]string{{"_value > 10"}}), - }, - { - name: "Multiple AND terms, no duplicates", - a: expression([][]string{{"_value > 10"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value < 5"}}), - expected: expression([][]string{{"_key LIKE 'test:*'", "_value < 5", "_value > 10"}}), - }, - { - name: "Multiple terms in both expressions with duplicates", - a: expression([][]string{{"_value > 10", "_key LIKE 'test:*'"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value < 5"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value < 5", "_value > 10"}, - }), - }, - { - name: "Terms in different order, no duplicates", - a: expression([][]string{{"_key LIKE 'test:*'", "_value > 10"}}), - b: expression([][]string{{"_value < 5"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value < 5", "_value > 10"}, - }), - }, - { - name: "Terms in different order with duplicates", - a: expression([][]string{{"_value > 10", "_key LIKE 'test:*'"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value < 5"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value < 5", "_value > 10"}, - }), - }, - { - name: "Partial duplicates across expressions", - a: expression([][]string{{"_value > 10", "_key LIKE 'test:*'"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_key = 'abc'"}}), - expected: expression([][]string{ - {"_key = 'abc'", "_key LIKE 'test:*'", "_value > 10"}, - }), - }, - { - name: "Nested AND groups", - a: expression([][]string{{"_key LIKE 'test:*'", "_value > 10"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value < 5"}, {"_value = 7"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value < 5", "_value > 10"}, - {"_key LIKE 'test:*'", "_value = 7", "_value > 10"}, - }), - }, - { - name: "Same terms but in different AND groups", - a: expression([][]string{{"_key LIKE 'test:*'"}}), - b: expression([][]string{{"_key LIKE 'test:*'", "_value < 5"}, {"_key LIKE 'test:*'"}}), - expected: expression([][]string{ - {"_key LIKE 'test:*'", "_value < 5"}, - {"_key LIKE 'test:*'"}, - }), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, combineAnd(tt.a, tt.b)) - }) - } -} - -func TestGenerateFingerprintAndParseAstExpression(t *testing.T) { - tests := []struct { - name string - similarExpr []string // logically same where expressions - expression string - fingerprint string - }{ - { - name: "Terms in different order, OR operator", - similarExpr: []string{ - "_value > 10 OR _value < 5", - "_value < 5 OR _value > 10", - }, - expression: "<_value5 OR >_value10", - fingerprint: "f_5731466836575684070", - }, - { - name: "Terms in different order, AND operator", - similarExpr: []string{ - "_value > 10 AND _value < 5", - "_value < 5 AND _value > 10", - }, - expression: "<_value5 AND >_value10", - fingerprint: "f_8696580727138087340", - }, - { - // ideally this and below test should spit same output - name: "Simple comparison operator (comparison value in backticks)", - similarExpr: []string{ - "_key like `match:1:*`", - }, - expression: "like_key`match:1:*`", - fingerprint: "f_15929225480754059748", - }, - { - name: "Simple comparison operator (comparison value in single quotes)", - similarExpr: []string{ - "_key like 'match:1:*'", - }, - expression: "like_key'match:1:*'", - fingerprint: "f_5313097907453016110", - }, - { - name: "Simple comparison operator with multiple redundant parentheses", - similarExpr: []string{ - "_key like 'match:1:*'", - "(_key like 'match:1:*')", - "((_key like 'match:1:*'))", - "(((_key like 'match:1:*')))", - }, - expression: "like_key'match:1:*'", - fingerprint: "f_5313097907453016110", - }, - { - name: "Expression with duplicate terms (or Idempotent law)", - similarExpr: []string{ - "_key like 'match:1:*' AND _key like 'match:1:*'", - "_key like 'match:1:*'", - }, - expression: "like_key'match:1:*'", - fingerprint: "f_5313097907453016110", - }, - { - name: "expression with exactly 1 term, multiple AND OR (Idempotent law)", - similarExpr: []string{ - "_value > 10 AND _value > 10 OR _value > 10 AND _value > 10", - "_value > 10", - }, - expression: ">_value10", - fingerprint: "f_11845737393789912467", - }, - { - name: "Expression in form 'A AND (B OR C)' which can reduce to 'A AND B OR A AND C' etc (or Distributive law)", - similarExpr: []string{ - "(_key LIKE 'test:*') AND (_value > 10 OR _value < 5)", - "(_value > 10 OR _value < 5) AND (_key LIKE 'test:*')", - "(_value < 5 OR _value > 10) AND (_key LIKE 'test:*')", - "(_key LIKE 'test:*' AND _value > 10) OR (_key LIKE 'test:*' AND _value < 5)", - "((_key LIKE 'test:*') AND _value > 10) OR ((_key LIKE 'test:*') AND _value < 5)", - "(_key LIKE 'test:*') AND ((_value > 10) OR (_value < 5))", - "(_value > 10 AND _key LIKE 'test:*') OR (_value < 5 AND _key LIKE 'test:*')", - "(_value < 5 AND _key LIKE 'test:*') OR (_value > 10 AND _key LIKE 'test:*')", - }, - expression: "<_value5 AND like_key'test:*' OR >_value10 AND like_key'test:*'", - fingerprint: "f_6936111135456499050", - }, - { - // ideally this and below test should spit same output - // but our algorithm is not sophisticated enough yet - name: "Expression in form 'A OR (B AND C)' which can reduce to 'A OR B AND A OR C' etc (or Distributive law)", - similarExpr: []string{ - "_key LIKE 'test:*' OR _value > 10 AND _value < 5", - "(_key LIKE 'test:*') OR (_value > 10 AND _value < 5)", - "(_value > 10 AND _value < 5) OR (_key LIKE 'test:*')", - "(_value < 5 AND _value > 10) OR (_key LIKE 'test:*')", - // "(_key LIKE 'test:*' OR _value > 10) AND (_key LIKE 'test:*' OR _value < 5)", - // "((_key LIKE 'test:*') OR (_value > 10)) AND ((_key LIKE 'test:*') OR (_value < 5))", - }, - expression: "<_value5 AND >_value10 OR like_key'test:*'", - fingerprint: "f_655732287561200780", - }, - { - name: "Complex expression with multiple redundant parentheses", - similarExpr: []string{ - "(_key LIKE 'test:*' OR _value > 10) AND (_key LIKE 'test:*' OR _value < 5)", - "((_key LIKE 'test:*') OR (_value > 10)) AND ((_key LIKE 'test:*') OR (_value < 5))", - }, - expression: "<_value5 AND >_value10 OR <_value5 AND like_key'test:*' OR >_value10 AND like_key'test:*' OR like_key'test:*'", - fingerprint: "f_1509117529358989129", - }, - { - name: "Test Precedence: AND before OR with LIKE and Value Comparison", - similarExpr: []string{ - "_key LIKE 'test:*' AND _value > 10 OR _value < 5", - "(_key LIKE 'test:*' AND _value > 10) OR _value < 5", - }, - expression: "<_value5 OR >_value10 AND like_key'test:*'", - fingerprint: "f_8791273852316817684", - }, - { - name: "Simple JSON expression", - similarExpr: []string{ - "'_value.age' > 30 and _key like 'user:*' and '_value.age' > 30", - "'_value.age' > 30 and _key like 'user:*'", - "_key like 'user:*' and '_value.age' > 30 ", - }, - expression: ">'_value.age'30 AND like_key'user:*'", - fingerprint: "f_5016002712062179335", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - for _, query := range tt.similarExpr { - where, err := parseSQL("SELECT * WHERE " + query) - if err != nil { - t.Fail() - } - assert.Equal(t, tt.expression, parseAstExpression(where).String()) - assert.Equal(t, tt.fingerprint, generateFingerprint(where)) - } - }) - } -} - -// Benchmark for generateFingerprint function -func BenchmarkGenerateFingerprint(b *testing.B) { - queries := []struct { - name string - query string - }{ - {"Simple", "SELECT * WHERE _key LIKE 'match:1:*'"}, - {"OrExpression", "SELECT * WHERE _key LIKE 'match:1:*' OR _value > 10"}, - {"AndExpression", "SELECT * WHERE _key LIKE 'match:1:*' AND _value > 10"}, - {"NestedOrAnd", "SELECT * WHERE _key LIKE 'match:1:*' OR (_value > 10 AND _value < 5)"}, - {"DeepNested", "SELECT * FROM table WHERE _key LIKE 'match:1:*' OR (_value > 10 AND (_value < 5 OR '_value.age' > 18))"}, - } - - for _, tt := range queries { - expr, err := parseSQL(tt.query) - if err != nil { - b.Fail() - } - - b.Run(tt.name, func(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - generateFingerprint(expr) - } - }) - } -} - -// helper -func parseSQL(query string) (sqlparser.Expr, error) { - stmt, err := sqlparser.Parse(query) - if err != nil { - return nil, err - } - - selectStmt, ok := stmt.(*sqlparser.Select) - if !ok { - return nil, err - } - - return selectStmt.Where.Expr, nil -} diff --git a/internal/store/store.go b/internal/store/store.go index f8b2726b6..99b2814a8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -24,9 +24,6 @@ import ( "github.com/dicedb/dice/internal/common" "github.com/dicedb/dice/internal/object" "github.com/dicedb/dice/internal/server/utils" - "github.com/dicedb/dice/internal/sql" - "github.com/ohler55/ojg/jp" - "github.com/xwb1989/sqlparser" ) func NewStoreRegMap() common.ITable[string, *object.Obj] { @@ -363,32 +360,6 @@ func (store *Store) GetStore() common.ITable[string, *object.Obj] { return store.store } -// CacheKeysForQuery scans the store for keys that match the given where clause and sends them to the cache channel. -// This allows the query manager to cache the existing keys that match the query. -func (store *Store) CacheKeysForQuery(whereClause sqlparser.Expr, cacheChannel chan *[]struct { - Key string - Value *object.Obj -}) { - shardCache := make([]struct { - Key string - Value *object.Obj - }, 0) - store.store.All(func(k string, v *object.Obj) bool { - matches, err := sql.EvaluateWhereClause(whereClause, sql.QueryResultRow{Key: k, Value: *v}, make(map[string]jp.Expr)) - if err != nil || !matches { - return true - } - - shardCache = append(shardCache, struct { - Key string - Value *object.Obj - }{Key: k, Value: v}) - - return true - }) - cacheChannel <- &shardCache -} - func (store *Store) evict(evictCount int) bool { store.evictionStrategy.EvictVictims(store, evictCount) return true