From 08a6052d9878079eb290f5653f656ab44d2d75d3 Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Mon, 15 Jul 2024 11:05:45 +0800 Subject: [PATCH 1/7] #2583 support disable rocksdb auto compaction through configuration --- .../apache/hugegraph/rocksdb/access/RocksDBOptions.java | 7 +++++++ .../apache/hugegraph/rocksdb/access/RocksDBSession.java | 3 +++ 2 files changed, 10 insertions(+) diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 9d380b3356..6f5c35f627 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -120,6 +120,13 @@ public class RocksDBOptions extends OptionHolder { disallowEmpty(), "rocksdb-snapshot" ); + public static final ConfigOption DISABLE_AUTO_COMPACTION = + new ConfigOption<>( + "rocksdb.disable_auto_compaction", + "Set disable auto compaction.", + disallowEmpty(), + false + ); public static final ConfigConvOption COMPACTION_STYLE = new ConfigConvOption<>( "rocksdb.compaction_style", diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index 70299f8f09..0002a8c4df 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -430,6 +430,9 @@ private void openRocksDB(String dbDataPath, long version) { List columnFamilyBytes = RocksDB.listColumnFamilies(new Options(), dbPath); ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + if(hugeConfig.get(RocksDBOptions.DISABLE_AUTO_COMPACTION)){ + cfOptions.setDisableAutoCompactions(true); + } RocksDBSession.initOptions(this.hugeConfig, null, null, cfOptions, cfOptions); if (columnFamilyBytes.size() > 0) { From e3377df68fe3f044f74eb97847637fd22995d7f4 Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Tue, 16 Jul 2024 09:50:00 +0800 Subject: [PATCH 2/7] close #2583 --- .../org/apache/hugegraph/rocksdb/access/RocksDBSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index 0002a8c4df..d1f89262a3 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -430,7 +430,7 @@ private void openRocksDB(String dbDataPath, long version) { List columnFamilyBytes = RocksDB.listColumnFamilies(new Options(), dbPath); ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); - if(hugeConfig.get(RocksDBOptions.DISABLE_AUTO_COMPACTION)){ + if (hugeConfig.get(RocksDBOptions.DISABLE_AUTO_COMPACTION)) { cfOptions.setDisableAutoCompactions(true); } RocksDBSession.initOptions(this.hugeConfig, null, null, cfOptions, cfOptions); From 11b5be2a870453e59f9d478824a3eb0e07a09f18 Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Tue, 6 Aug 2024 13:37:01 +0800 Subject: [PATCH 3/7] close #2618 --- .../backend/store/AbstractBackendStoreProvider.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java index 906d795149..4877797e8b 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hugegraph.backend.BackendException; import org.apache.hugegraph.backend.store.raft.StoreSnapshotFile; @@ -44,6 +45,8 @@ public abstract class AbstractBackendStoreProvider private final EventHub storeEventHub = new EventHub("store"); protected Map stores = null; + AtomicBoolean schemaCacheClearListened = new AtomicBoolean(false); + AtomicBoolean vertexEdgeCacheClearListened = new AtomicBoolean(false); protected final void notifyAndWaitEvent(String event) { Future future = this.storeEventHub.notify(event, this); @@ -67,7 +70,10 @@ protected final void checkOpened() { @Override public void listen(EventListener listener) { - this.storeEventHub.listen(EventHub.ANY_EVENT, listener); + if (schemaCacheClearListened.compareAndSet(false, true) || + vertexEdgeCacheClearListened.compareAndSet(false, true)) { + this.storeEventHub.listen(EventHub.ANY_EVENT, listener); + } } @Override From 8c324042766bd942e1a25a33311fa3ced571979b Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Mon, 12 Aug 2024 18:18:48 +0800 Subject: [PATCH 4/7] close #2618 --- .../backend/cache/CachedGraphTransaction.java | 2 +- .../cache/CachedSchemaTransactionV2.java | 3 +- .../store/AbstractBackendStoreProvider.java | 31 ++++++++++++++++--- .../backend/store/BackendStoreProvider.java | 6 ++++ 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedGraphTransaction.java index 83ab7f51ad..29f6cc3f36 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedGraphTransaction.java @@ -133,7 +133,7 @@ private void listenChanges() { } return false; }; - this.store().provider().listen(this.storeEventListener); + this.store().provider().listenDataCacheClear(this.storeEventListener); // Listen cache event: "cache"(invalid cache item) this.cacheEventListener = event -> { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java index e6a5e78533..f5a23ffb35 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java @@ -105,7 +105,8 @@ private void listenChanges() { } return false; }; - this.graphParams().loadGraphStore().provider().listen(this.storeEventListener); + this.graphParams().loadGraphStore().provider() + .listenSchemaCacheClear(this.storeEventListener); // Listen cache event: "cache"(invalid cache item) this.cacheEventListener = event -> { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java index 4877797e8b..73608f1d36 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -45,8 +45,8 @@ public abstract class AbstractBackendStoreProvider private final EventHub storeEventHub = new EventHub("store"); protected Map stores = null; - AtomicBoolean schemaCacheClearListened = new AtomicBoolean(false); - AtomicBoolean vertexEdgeCacheClearListened = new AtomicBoolean(false); + volatile Boolean schemaCacheClearListened = false; + volatile Boolean vertexEdgeCacheClearListened = false; protected final void notifyAndWaitEvent(String event) { Future future = this.storeEventHub.notify(event, this); @@ -70,9 +70,30 @@ protected final void checkOpened() { @Override public void listen(EventListener listener) { - if (schemaCacheClearListened.compareAndSet(false, true) || - vertexEdgeCacheClearListened.compareAndSet(false, true)) { - this.storeEventHub.listen(EventHub.ANY_EVENT, listener); + this.storeEventHub.listen(EventHub.ANY_EVENT, listener); + } + + @Override + public void listenSchemaCacheClear(EventListener listener) { + if (!schemaCacheClearListened) { + synchronized (this) { + if (!schemaCacheClearListened) { + listen(listener); + schemaCacheClearListened = true; + } + } + } + } + + @Override + public void listenDataCacheClear(EventListener listener) { + if (!vertexEdgeCacheClearListened) { + synchronized (this) { + if (!vertexEdgeCacheClearListened) { + listen(listener); + vertexEdgeCacheClearListened = true; + } + } } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreProvider.java index 44e58c2289..8302679ec0 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreProvider.java @@ -67,6 +67,12 @@ public interface BackendStoreProvider { void listen(EventListener listener); + default void listenSchemaCacheClear(EventListener listener) { + } + + default void listenDataCacheClear(EventListener listener) { + } + void unlisten(EventListener listener); EventHub storeEventHub(); From 0b4725f6a9d147fae2083670b0263cdd42ea0c26 Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Tue, 13 Aug 2024 11:20:04 +0800 Subject: [PATCH 5/7] close #2618 --- .../backend/store/AbstractBackendStoreProvider.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java index 73608f1d36..a292cc6cb8 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -45,8 +45,8 @@ public abstract class AbstractBackendStoreProvider private final EventHub storeEventHub = new EventHub("store"); protected Map stores = null; - volatile Boolean schemaCacheClearListened = false; - volatile Boolean vertexEdgeCacheClearListened = false; + private static volatile Boolean schemaCacheClearListened = Boolean.FALSE; + private static volatile Boolean vertexEdgeCacheClearListened = Boolean.FALSE; protected final void notifyAndWaitEvent(String event) { Future future = this.storeEventHub.notify(event, this); @@ -79,7 +79,7 @@ public void listenSchemaCacheClear(EventListener listener) { synchronized (this) { if (!schemaCacheClearListened) { listen(listener); - schemaCacheClearListened = true; + schemaCacheClearListened = Boolean.TRUE; } } } @@ -91,7 +91,7 @@ public void listenDataCacheClear(EventListener listener) { synchronized (this) { if (!vertexEdgeCacheClearListened) { listen(listener); - vertexEdgeCacheClearListened = true; + vertexEdgeCacheClearListened = Boolean.TRUE; } } } From ceba46ce8e83f7a8d65e0486b83e90fe70cb9ee3 Mon Sep 17 00:00:00 2001 From: "alan578.zhao" <956322745@qq.com> Date: Tue, 13 Aug 2024 16:42:11 +0800 Subject: [PATCH 6/7] close #2618 --- .../backend/store/AbstractBackendStoreProvider.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java index a292cc6cb8..9bc78e430f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -45,8 +45,8 @@ public abstract class AbstractBackendStoreProvider private final EventHub storeEventHub = new EventHub("store"); protected Map stores = null; - private static volatile Boolean schemaCacheClearListened = Boolean.FALSE; - private static volatile Boolean vertexEdgeCacheClearListened = Boolean.FALSE; + private static volatile boolean schemaCacheClearListened = false; + private static volatile boolean vertexEdgeCacheClearListened = false; protected final void notifyAndWaitEvent(String event) { Future future = this.storeEventHub.notify(event, this); @@ -79,7 +79,7 @@ public void listenSchemaCacheClear(EventListener listener) { synchronized (this) { if (!schemaCacheClearListened) { listen(listener); - schemaCacheClearListened = Boolean.TRUE; + schemaCacheClearListened = true; } } } @@ -91,7 +91,7 @@ public void listenDataCacheClear(EventListener listener) { synchronized (this) { if (!vertexEdgeCacheClearListened) { listen(listener); - vertexEdgeCacheClearListened = Boolean.TRUE; + vertexEdgeCacheClearListened = true; } } } From b1ec9559d3f3b324daaed88cf03335773c44da50 Mon Sep 17 00:00:00 2001 From: haohao0103 <956322745@qq.com> Date: Thu, 15 Aug 2024 10:07:35 +0800 Subject: [PATCH 7/7] Update hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java Co-authored-by: imbajin --- .../hugegraph/backend/store/AbstractBackendStoreProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java index 9bc78e430f..9310f71b42 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -20,7 +20,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hugegraph.backend.BackendException; import org.apache.hugegraph.backend.store.raft.StoreSnapshotFile;