Improved performance of journal cleaning

src/Kdyby/Redis/RedisJournal.php

@@ -31,6 +31,9 @@ class RedisJournal extends Nette\Object implements Nette\Caching\Storages\IJourn
 		TAGS = 'tags',
 		KEYS = 'keys';
 
+	/** @internal batch delete size */
+	const BATCH_SIZE = 8000;
+
 	/**
 	 * @var RedisClient
 	 */
@@ -108,34 +111,86 @@ private function cleanEntry($keys)
 	 * Cleans entries from journal.
 	 *
 	 * @param  array  $conds
+	 * @param \Nette\Caching\IStorage $storage
 	 *
 	 * @return array of removed items or NULL when performing a full cleanup
 	 */
-	public function clean(array $conds)
+	public function clean(array $conds, Nette\Caching\IStorage $storage = NULL)
 	{
 		if (!empty($conds[Cache::ALL])) {
 			$all = $this->client->keys(self::NS_NETTE . ':*');
+			if ($storage instanceof RedisStorage) {
+				$all = array_merge($all, $this->client->keys(RedisStorage::NS_NETTE . ':*'));
+			}
 
-			$this->client->multi();
 			call_user_func_array(array($this->client, 'del'), $all);
-			$this->client->exec();
 			return NULL;
 		}
 
 		$entries = array();
 		if (!empty($conds[Cache::TAGS])) {
+			$removingTagKeys = array();
 			foreach ((array)$conds[Cache::TAGS] as $tag) {
-				$this->cleanEntry($found = $this->tagEntries($tag));
+				$found = $this->tagEntries($tag);
+				$removingTagKeys[] = $this->formatKey($tag, self::KEYS);
 				$entries = array_merge($entries, $found);
 			}
+			if ($removingTagKeys) {
+				call_user_func_array(array($this->client, 'del'), $removingTagKeys);
+			}
 		}
 
 		if (isset($conds[Cache::PRIORITY])) {
-			$this->cleanEntry($found = $this->priorityEntries($conds[Cache::PRIORITY]));
+			$found = $this->priorityEntries($conds[Cache::PRIORITY]);
+			call_user_func_array(array($this->client, 'zRemRangeByScore'), array($this->formatKey(self::PRIORITY), 0, (int)$conds[Cache::PRIORITY]));
 			$entries = array_merge($entries, $found);
 		}
 
-		return array_unique($entries);
+		$entries = array_unique($entries);
+
+		$removingKeys = array();
+		$removingKeyTags = array();
+		$removingKeyPriorities = array();
+		foreach ($entries as $key) {
+			if ($storage instanceof RedisStorage) {
+				$removingKeys[] = $key;
+			}
+			$removingKeyTags[] = $this->formatKey($key, self::TAGS);
+			$removingKeyPriorities[] = $this->formatKey($key, self::PRIORITY);
+			if (count($removingKeyTags) >= self::BATCH_SIZE) {
+				$this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
+				$removingKeys = array();
+				$removingKeyTags = array();
+				$removingKeyPriorities = array();
+			}
+		}
+
+		$this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
+
+		return $storage instanceof RedisStorage ? array() : $entries;
+	}
+
+
+
+	private function cleanBatchData(array $removingKeys, array $removingKeyPriorities, array $removingKeyTags, array $keys)
+	{
+		if ($removingKeyTags) {
+			if ($keys) {
+				$affectedTags = call_user_func_array(array($this->client, 'sunion'), array($removingKeyTags));
+				foreach ($affectedTags as $tag) {
+					if ($tag) {
+						call_user_func_array(array($this->client, 'sRem'), array_merge(array($this->formatKey($tag, self::KEYS)), $keys));
+					}
+				}
+			}
+			call_user_func_array(array($this->client, 'del'), $removingKeyTags);
+		}
+		if ($removingKeyPriorities) {
+			call_user_func_array(array($this->client, 'del'), $removingKeyPriorities);
+		}
+		if ($removingKeys) {
+			call_user_func_array(array($this->client, 'del'), $removingKeys);
+		}
 	}
 
 

src/Kdyby/Redis/scripts/common.lua

@@ -29,6 +29,23 @@ local tagEntries = function (tag)
     return redis.call('sMembers', formatKey(tag, "keys"))
 end
 
+local range = function (from, to, step)
+	step = step or 1
+	local f =
+		step > 0 and
+			function(_, lastvalue)
+				local nextvalue = lastvalue + step
+				if nextvalue <= to then return nextvalue end
+			end or
+		step < 0 and
+			function(_, lastvalue)
+				local nextvalue = lastvalue + step
+				if nextvalue >= to then return nextvalue end
+			end or
+			function(_, lastvalue) return lastvalue end
+	return f, nil, from - step
+end
+
 local cleanEntry = function (keys)
     for i, key in pairs(keys) do
         local tags = entryTags(key)
@@ -45,3 +62,34 @@ local cleanEntry = function (keys)
         -- redis.call('exec')
     end
 end
+
+local mergeTables = function (first, second)
+    for i, key in pairs(second) do
+        first[#first + 1] = key
+    end
+    return first
+end
+
+local batch = function (keys, callback)
+    if #keys > 0 then
+        -- redis.call('multi')
+        -- the magic number 7998 becomes from Lua limitations, see http://stackoverflow.com/questions/19202367/how-to-avoid-redis-calls-in-lua-script-limitations
+        local tmp = {}
+        for i,key in pairs(keys) do
+            tmp[#tmp + 1] = key
+            if #tmp >= 7998 then
+                callback(tmp)
+                tmp = {}
+            end
+        end
+        callback(tmp)
+        -- redis.call('exec')
+    end
+end
+
+local batchDelete = function(keys)
+    local delete = function (tmp)
+        redis.call('del', unpack(tmp))
+    end
+    batch(keys, delete)
+end

src/Kdyby/Redis/scripts/journal.clean.lua

@@ -2,11 +2,10 @@
 local conds = cjson.decode(ARGV[1])
 
 if conds["all"] ~= nil then
-    -- redis.call('multi')
-    for i, value in pairs(redis.call('keys', "Nette.Journal:*")) do
-        redis.call('del', value)
+    batchDelete(redis.call('keys', "Nette.Journal:*"))
+    if conds["delete-entries"] ~= nil then
+        batchDelete(redis.call('keys', "Nette.Storage:*"))
     end
-    -- redis.call('exec')
 
     return redis.status_reply("Ok")
 end

tests/KdybyTests/Redis/RedisJournal.phpt

@@ -284,7 +284,7 @@ class RedisJournalTest extends AbstractRedisTestCase
 		Assert::null($result);
 
 		$result2 = $this->journal->clean(array(Cache::TAGS => 'test:all'));
-		Assert::true(empty($result2));
+		Assert::equal(array(), $result2);
 	}
 
 
@@ -362,26 +362,7 @@ LUA;
 	private function cacheGeneratorScripts()
 	{
 		$script = file_get_contents(__DIR__ . '/../../../src/Kdyby/Redis/scripts/common.lua');
-		$script .= <<<LUA
-local range = function (from, to, step)
-	step = step or 1
-	local f =
-		step > 0 and
-			function(_, lastvalue)
-				local nextvalue = lastvalue + step
-				if nextvalue <= to then return nextvalue end
-			end or
-		step < 0 and
-			function(_, lastvalue)
-				local nextvalue = lastvalue + step
-				if nextvalue >= to then return nextvalue end
-			end or
-			function(_, lastvalue) return lastvalue end
-	return f, nil, from - step
-end
 
-
-LUA;
 		return $script;
 	}