diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html index 3dfe99724efc..75b64e9068ef 100644 --- a/docs/layouts/shortcodes/generated/core_configuration.html +++ b/docs/layouts/shortcodes/generated/core_configuration.html @@ -339,6 +339,12 @@ Integer The maximal fan-in for external merge sort. It limits the number of file handles. If it is too small, may cause intermediate merging. But if it is too large, it will cause too many files opened at the same time, consume memory and lead to random reading. + +
force-lookup
+ false + Boolean + Whether to force the use of lookup for compaction. +
lookup-wait
true @@ -467,12 +473,6 @@ Integer Total level number, for example, there are 3 levels, including 0,1,2 levels. - -
prefer-compaction-strategy
- universal - Enum - By default, universal compaction strategy is used. Note: This is just a suggestion for paimon to use this compaction strategy. Paimon will automatically decide which compaction strategy to use. For example, when 'changelog-producer' is set to 'lookup', paimon will automatically use the lookup compaction strategy.

Possible values: -
num-sorted-run.compaction-trigger
5 diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java index 0df03792298e..c9109a59a83a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java @@ -454,17 +454,6 @@ public class CoreOptions implements Serializable { text("append table: the default value is 256 MB.")) .build()); - @Immutable - public static final ConfigOption PREFER_COMPACTION_STRATEGY = - key("prefer-compaction-strategy") - .enumType(CompactionStrategy.class) - .defaultValue(CompactionStrategy.UNIVERSAL) - .withDescription( - "By default, universal compaction strategy is used. Note: This is just a suggestion " - + "for paimon to use this compaction strategy. Paimon will automatically decide " - + "which compaction strategy to use. For example, when 'changelog-producer' is set " - + "to 'lookup', paimon will automatically use the lookup compaction strategy."); - public static final ConfigOption NUM_SORTED_RUNS_COMPACTION_TRIGGER = key("num-sorted-run.compaction-trigger") .intType() @@ -1262,6 +1251,13 @@ public class CoreOptions implements Serializable { .noDefaultValue() .withDescription("Specifies the commit user prefix."); + @Immutable + public static final ConfigOption FORCE_LOOKUP = + key("force-lookup") + .booleanType() + .defaultValue(false) + .withDescription("Whether to force the use of lookup for compaction."); + public static final ConfigOption LOOKUP_WAIT = key("lookup-wait") .booleanType() @@ -1703,7 +1699,7 @@ public LookupStrategy lookupStrategy() { mergeEngine().equals(MergeEngine.FIRST_ROW), changelogProducer().equals(ChangelogProducer.LOOKUP), deletionVectorsEnabled(), - preferCompactionStrategy().equals(CompactionStrategy.LOOKUP)); + forLookup()); } public boolean changelogRowDeduplicate() { @@ -2036,8 +2032,8 @@ public boolean metadataIcebergCompatible() { return options.get(METADATA_ICEBERG_COMPATIBLE); } - public CompactionStrategy preferCompactionStrategy() { - return options.get(PREFER_COMPACTION_STRATEGY); + public boolean forLookup() { + return options.get(FORCE_LOOKUP); } /** Specifies the merge engine for table with primary key. */ @@ -2648,34 +2644,4 @@ public InlineElement getDescription() { return text(description); } } - - /** The compaction strategy of the LSM tree. */ - public enum CompactionStrategy implements DescribedEnum { - UNIVERSAL( - "universal", - "Universal compaction strategy, mainly triggered by the number of sorted runs, space amplification, etc."), - - LOOKUP( - "lookup", - "When the L0 file is generated, compaction will be triggered as soon as possible."); - - private final String value; - - private final String description; - - CompactionStrategy(String value, String description) { - this.value = value; - this.description = description; - } - - @Override - public String toString() { - return value; - } - - @Override - public InlineElement getDescription() { - return text(description); - } - } } diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/LookupStrategy.java b/paimon-common/src/main/java/org/apache/paimon/lookup/LookupStrategy.java index 3d37d2d56898..f01c7c967b38 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/LookupStrategy.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/LookupStrategy.java @@ -33,18 +33,18 @@ private LookupStrategy( boolean isFirstRow, boolean produceChangelog, boolean deletionVector, - boolean preferLookup) { + boolean forceLookup) { this.isFirstRow = isFirstRow; this.produceChangelog = produceChangelog; this.deletionVector = deletionVector; - this.needLookup = produceChangelog || deletionVector || isFirstRow || preferLookup; + this.needLookup = produceChangelog || deletionVector || isFirstRow || forceLookup; } public static LookupStrategy from( boolean isFirstRow, boolean produceChangelog, boolean deletionVector, - boolean preferLookup) { - return new LookupStrategy(isFirstRow, produceChangelog, deletionVector, preferLookup); + boolean forceLookup) { + return new LookupStrategy(isFirstRow, produceChangelog, deletionVector, forceLookup); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java index 4fa30f2ab33e..e8ece0779ad1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java @@ -20,7 +20,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.CoreOptions.ChangelogProducer; -import org.apache.paimon.CoreOptions.CompactionStrategy; import org.apache.paimon.CoreOptions.LookupLocalFileType; import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryString; @@ -1681,8 +1680,7 @@ public void testRollbackToTagWithChangelogDecoupled(String changelogProducer) th @ParameterizedTest @EnumSource(CoreOptions.MergeEngine.class) - public void testPreferLookupCompactionStrategy(CoreOptions.MergeEngine mergeEngine) - throws Exception { + public void testForceLookupCompaction(CoreOptions.MergeEngine mergeEngine) throws Exception { Map> testData = new HashMap<>(); testData.put(DEDUPLICATE, Pair.of(50L, 100L)); testData.put(PARTIAL_UPDATE, Pair.of(null, 100L)); @@ -1693,9 +1691,7 @@ public void testPreferLookupCompactionStrategy(CoreOptions.MergeEngine mergeEngi FileStoreTable table = createFileStoreTable( options -> { - options.set( - CoreOptions.PREFER_COMPACTION_STRATEGY, - CompactionStrategy.LOOKUP); + options.set(CoreOptions.FORCE_LOOKUP, true); options.set(MERGE_ENGINE, mergeEngine); if (mergeEngine == AGGREGATE) { options.set("fields.b.aggregate-function", "sum");