Merge branch 'master' into iamskp11/dice-db/issue/486/support-json-ar…

…r-index-command

Makefile

@@ -105,3 +105,7 @@ release: ## build and push the Docker image to Docker Hub with the latest tag an
 	docker build --tag dicedb/dicedb:latest --tag dicedb/dicedb:$(VERSION) .
 	docker push dicedb/dicedb:$(VERSION)
 	docker push dicedb/dicedb:latest
+
+push-binary-remote:
+	$(MAKE) build
+	scp -i ${SSH_PEM_PATH} ./dicedb ubuntu@${REMOTE_HOST}:.

README.md

@@ -5,7 +5,7 @@ DiceDB
 <a href="https://dicedb.io/get-started/installation/">![Docs](https://img.shields.io/badge/docs-00A1FF?style=flat-square)</a>
 <a target="_blank" href="https://discord.gg/6r8uXWtXh7"><img src="https://dcbadge.limes.pink/api/server/6r8uXWtXh7?style=flat" alt="discord community" /></a>
 
-DiceDB is a redis-compliant, reactive, scalable, highly available, unified cache optimized for modern hardware.
+DiceDB is an open source, redis-compliant, reactive, scalable, highly available, unified cache optimized for modern hardware.
 
 We are looking for Early Design Partners, so, if you want to evaluate DiceDB, [block our calendar](https://cal.com/dicedb-arpit). always up for a chat.
 

config/config.go

@@ -1,3 +1,19 @@
+// This file is part of DiceDB.
+// Copyright (C) 2024 DiceDB (dicedb.io).
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 package config
 
 import (
@@ -66,7 +82,7 @@ memory.keys_limit = 200000000
 memory.lfu_log_factor = 10
 
 # Persistence Configuration
-persistence.enabled = true
+persistence.enabled = false
 persistence.aof_file = "./dice-master.aof"
 persistence.persistence_enabled = true
 persistence.write_aof_on_cleanup = false
@@ -84,7 +100,22 @@ auth.password = ""
 
 # Network Configuration
 network.io_buffer_length = 512
-network.io_buffer_length_max = 51200`
+network.io_buffer_length_max = 51200
+
+# WAL Configuration
+LogDir = "tmp/dicedb-wal"
+Enabled = "true"
+WalMode = "buffered"
+WriteMode = "default"
+BufferSizeMB = 1
+RotationMode = "segemnt-size"
+MaxSegmentSizeMB = 16
+MaxSegmentRotationTime = 60s
+BufferSyncInterval = 200ms
+RetentionMode = "num-segments" 
+MaxSegmentCount = 10
+MaxSegmentRetentionDuration = 600s
+RecoveryMode = "strict"`
 )
 
 var (
@@ -104,6 +135,7 @@ type Config struct {
 	Persistence persistence `config:"persistence"`
 	Logging     logging     `config:"logging"`
 	Network     network     `config:"network"`
+	WAL         WALConfig   `config:"WAL"`
 }
 
 type auth struct {
@@ -147,19 +179,47 @@ type memory struct {
 	MaxMemory      int64   `config:"max_memory" default:"0" validate:"min=0"`
 	EvictionPolicy string  `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"`
 	EvictionRatio  float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"`
-	KeysLimit      int     `config:"keys_limit" default:"200000000" validate:"min=0"`
+	KeysLimit      int     `config:"keys_limit" default:"200000000" validate:"min=10"`
 	LFULogFactor   int     `config:"lfu_log_factor" default:"10" validate:"min=0"`
 }
 
 type persistence struct {
 	Enabled           bool   `config:"enabled" default:"false"`
 	AOFFile           string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"`
 	WriteAOFOnCleanup bool   `config:"write_aof_on_cleanup" default:"false"`
-	WALDir            string `config:"wal-dir" default:"./" validate:"dirpath"`
 	RestoreFromWAL    bool   `config:"restore-wal" default:"false"`
 	WALEngine         string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"`
 }
 
+type WALConfig struct {
+	// Directory where WAL log files will be stored
+	LogDir string `config:"log_dir" default:"tmp/dicedb-wal"`
+	// Whether WAL is enabled
+	Enabled bool `config:"enabled" default:"true"`
+	// WAL buffering mode: 'buffered' (writes buffered in memory) or 'unbuffered' (immediate disk writes)
+	WalMode string `config:"wal_mode" default:"buffered" validate:"oneof=buffered unbuffered"`
+	// Write mode: 'default' (OS handles syncing) or 'fsync' (explicit fsync after writes)
+	WriteMode string `config:"write_mode" default:"default" validate:"oneof=default fsync"`
+	// Size of the write buffer in megabytes
+	BufferSizeMB int `config:"buffer_size_mb" default:"1" validate:"min=1"`
+	// How WAL rotation is triggered: 'segment-size' (based on file size) or 'time' (based on duration)
+	RotationMode string `config:"rotation_mode" default:"segemnt-size" validate:"oneof=segment-size time"`
+	// Maximum size of a WAL segment file in megabytes before rotation
+	MaxSegmentSizeMB int `config:"max_segment_size_mb" default:"16" validate:"min=1"`
+	// Time interval in seconds after which WAL segment is rotated when using time-based rotation
+	MaxSegmentRotationTime time.Duration `config:"max_segment_rotation_time" default:"60s" validate:"min=1s"`
+	// Time interval in Milliseconds after which buffered WAL data is synced to disk
+	BufferSyncInterval time.Duration `config:"buffer_sync_interval" default:"200ms" validate:"min=1ms"`
+	// How old segments are removed: 'num-segments' (keep N latest), 'time' (by age), or 'checkpoint' (after checkpoint)
+	RetentionMode string `config:"retention_mode" default:"num-segments" validate:"oneof=num-segments time checkpoint"`
+	// Maximum number of WAL segment files to retain when using num-segments retention
+	MaxSegmentCount int `config:"max_segment_count" default:"10" validate:"min=1"`
+	// Time interval in Seconds till which WAL segments are retained when using time-based retention
+	MaxSegmentRetentionDuration time.Duration `config:"max_segment_retention_duration" default:"600s" validate:"min=1s"`
+	// How to handle WAL corruption on recovery: 'strict' (fail), 'truncate' (truncate at corruption), 'ignore' (skip corrupted)
+	RecoveryMode string `config:"recovery_mode" default:"strict" validate:"oneof=strict truncate ignore"`
+}
+
 type logging struct {
 	LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"`
 	LogDir   string `config:"log_dir" default:"/tmp/dicedb" validate:"dirpath"`

config/parser.go

@@ -1,3 +1,19 @@
+// This file is part of DiceDB.
+// Copyright (C) 2024 DiceDB (dicedb.io).
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 package config
 
 import (

config/validator.go

@@ -1,3 +1,19 @@
+// This file is part of DiceDB.
+// Copyright (C) 2024 DiceDB (dicedb.io).
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 package config
 
 import (
@@ -12,6 +28,7 @@ import (
 func validateConfig(config *Config) error {
 	validate := validator.New()
 	validate.RegisterStructValidation(validateShardCount, Config{})
+	validate.RegisterStructValidation(validateWALConfig, Config{})
 
 	if err := validate.Struct(config); err != nil {
 		validationErrors, ok := err.(validator.ValidationErrors)
@@ -95,3 +112,36 @@ func applyDefaultValuesFromTags(config *Config, fieldName string) error {
 	log.Printf("Setting default value for %s to: %s", fieldName, defaultValue)
 	return nil
 }
+
+func validateWALConfig(sl validator.StructLevel) {
+	config := sl.Current().Interface().(Config)
+
+	// LogDir validation
+	if config.WAL.LogDir == "" {
+		sl.ReportError(config.WAL.LogDir, "LogDir", "LogDir", "required", "cannot be empty")
+	}
+
+	// MaxSegmentSize validation
+	if config.WAL.MaxSegmentSizeMB <= 0 {
+		sl.ReportError(config.WAL.MaxSegmentSizeMB, "MaxSegmentSize", "MaxSegmentSize", "gt", "must be greater than 0")
+	}
+
+	// MaxSegmentCount validation
+	if config.WAL.MaxSegmentCount <= 0 {
+		sl.ReportError(config.WAL.MaxSegmentCount, "MaxSegmentCount", "MaxSegmentCount", "gt", "must be greater than 0")
+	}
+
+	// BufferSize validation
+	if config.WAL.BufferSizeMB <= 0 {
+		sl.ReportError(config.WAL.BufferSizeMB, "BufferSize", "BufferSize", "gt", "must be greater than 0")
+	}
+
+	// WALMode and WriteMode compatibility checks
+	if config.WAL.WalMode == "buffered" && config.WAL.WriteMode == "fsync" {
+		sl.ReportError(config.WAL.WalMode, "WALMode", "WALMode", "incompatible", "walMode 'buffered' cannot be used with writeMode 'fsync'")
+	}
+
+	if config.WAL.WalMode == "unbuffered" && config.WAL.WriteMode == "default" {
+		sl.ReportError(config.WAL.WalMode, "WALMode", "WALMode", "incompatible", "walMode 'unbuffered' cannot have writeMode as 'default'")
+	}
+}

dicedb.conf

@@ -39,7 +39,7 @@ memory.keys_limit = 200000000
 memory.lfu_log_factor = 10
 
 # Persistence Configuration
-persistence.enabled = true
+persistence.enabled = false
 persistence.aof_file = "./dice-master.aof"
 persistence.persistence_enabled = true
 persistence.write_aof_on_cleanup = false
@@ -56,4 +56,19 @@ auth.password = ""
 
 # Network Configuration
 network.io_buffer_length = 512
-network.io_buffer_length_max = 51200
+network.io_buffer_length_max = 51200
+
+# WAL Configuration
+LogDir = "tmp/dicedb-wal"
+Enabled = "true"
+WalMode = "buffered"
+WriteMode = "default"
+BufferSizeMB = 1
+RotationMode = "segemnt-size"
+MaxSegmentSizeMB = 16
+MaxSegmentRotationTime = 60s
+BufferSyncInterval = 200ms
+RetentionMode = "num-segments" 
+MaxSegmentCount = 10
+MaxSegmentRetentionDuration = 600s
+RecoveryMode = "strict"

docs/astro.config.mjs

@@ -15,17 +15,17 @@ export default defineConfig({
       // useStarlightDarkModeSwitch: false,
       favicon: "/favicon.png",
       editLink: {
-        baseUrl: 'https://github.com/DiceDB/dice/edit/master/docs/',
+        baseUrl: "https://github.com/DiceDB/dice/edit/master/docs/",
       },
       lastUpdated: true,
       expressiveCode: {
         textMarkers: true,
-        themes: ['ayu-dark','light-plus'],
+        themes: ["ayu-dark", "light-plus"],
         defaultProps: {
           wrap: true,
         },
-        styleOverrides: { 
-          borderRadius: '0.2rem' 
+        styleOverrides: {
+          borderRadius: "0.2rem",
         },
       },
       sidebar: [
@@ -42,8 +42,8 @@ export default defineConfig({
           autogenerate: { directory: "protocols" },
         },
         {
-          label: 'Tutorials',
-          autogenerate: { directory: 'tutorials' }
+          label: "Tutorials",
+          autogenerate: { directory: "tutorials" },
         },
         {
           label: "Commands",

docs/public/.well-known/funding-manifest-urls

@@ -0,0 +1 @@
+https://dicedb.io/funding.json