test

pylib/cqlshlib/test/test_keyspace_init.cql

@@ -34,7 +34,7 @@ CREATE TABLE has_all_types (
     uuidcol uuid,
     varcharcol varchar,
     varintcol varint
-) WITH compression = {'sstable_compression':'LZ4Compressor'};
+) WITH compression = {'class':'LZ4Compressor'};
 
 INSERT INTO has_all_types (num, intcol, asciicol, bigintcol, blobcol, booleancol,
                            decimalcol, doublecol, durationcol, floatcol, smallintcol, textcol,

test/conf/cassandra-with-sstable-compressor.yaml

@@ -0,0 +1,110 @@
+
+cluster_name: Test Cluster
+# memtable_allocation_type: heap_buffers
+memtable_allocation_type: offheap_objects
+commitlog_sync: batch
+commitlog_sync_batch_window_in_ms: 1.0
+commitlog_segment_size: 5MiB
+commitlog_directory: build/test/cassandra/commitlog
+# commitlog_compression:
+# - class_name: LZ4Compressor
+cdc_raw_directory: build/test/cassandra/cdc_raw
+cdc_enabled: false
+hints_directory: build/test/cassandra/hints
+partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
+listen_address: 127.0.0.1
+storage_port: 7012
+ssl_storage_port: 17012
+start_native_transport: true
+native_transport_port: 9042
+column_index_size: 4KiB
+saved_caches_directory: build/test/cassandra/saved_caches
+data_file_directories:
+    - build/test/cassandra/data
+disk_access_mode: mmap
+seed_provider:
+    - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+      parameters:
+          - seeds: "127.0.0.1:7012"
+endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
+dynamic_snitch: true
+server_encryption_options:
+    internode_encryption: none
+    keystore: conf/.keystore
+    keystore_password: cassandra
+    truststore: conf/.truststore
+    truststore_password: cassandra
+incremental_backups: true
+concurrent_compactors: 4
+compaction_throughput: 0MiB/s
+row_cache_class_name: org.apache.cassandra.cache.OHCProvider
+row_cache_size: 16MiB
+user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: true
+prepared_statements_cache_size: 1MiB
+corrupted_tombstone_strategy: exception
+stream_entire_sstables: true
+stream_throughput_outbound: 23841858MiB/s
+sasi_indexes_enabled: true
+materialized_views_enabled: true
+drop_compact_storage_enabled: true
+file_cache_enabled: true
+auto_hints_cleanup_enabled: true
+
+read_thresholds_enabled: true
+coordinator_read_size_warn_threshold: 1024KiB
+coordinator_read_size_fail_threshold: 4096KiB
+local_read_size_warn_threshold: 4096KiB
+local_read_size_fail_threshold: 8192KiB
+row_index_read_size_warn_threshold: 4096KiB
+row_index_read_size_fail_threshold: 8192KiB
+
+memtable:
+    configurations:
+        skiplist:
+            inherits: default
+            class_name: SkipListMemtable
+        skiplist_sharded:
+            class_name: ShardedSkipListMemtable
+            parameters:
+                serialize_writes: false
+                shards: 4
+        skiplist_sharded_locking:
+            inherits: skiplist_sharded
+            parameters:
+                serialize_writes: true
+        skiplist_remapped:
+            inherits: skiplist
+        test_fullname:
+            inherits: default
+            class_name: org.apache.cassandra.db.memtable.TestMemtable
+        test_shortname:
+            class_name: TestMemtable
+            parameters:
+                skiplist: true  # note: YAML must interpret this as string, not a boolean
+        test_empty_class:
+            class_name: ""
+        test_missing_class:
+            parameters:
+        test_unknown_class:
+            class_name: NotExisting
+        test_invalid_param:
+            class_name: SkipListMemtable
+            parameters:
+                invalid: throw
+        test_invalid_extra_param:
+            inherits: test_shortname
+            parameters:
+                invalid: throw
+        test_invalid_factory_method:
+            class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryMethod
+        test_invalid_factory_field:
+            class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryField
+
+sstable_compression:
+    class_name: lz4
+    parameters:
+        lz4_compressor_type: high
+        lz4_high_compressor_level: "8"
+        chunk_length: 32MiB
+        min_compress_ratio: "1.5"

test/distributed/org/apache/cassandra/distributed/util/PyDtest.java

@@ -160,7 +160,7 @@ public String build()
                 query += String.format(" AND CLUSTERING ORDER BY (%s)", clustering);
 
             if (compression != null)
-                query += String.format(" AND compression = { \'sstable_compression\': \'%sCompressor\' }", compression);
+                query += String.format(" AND compression = { \'class\': \'%sCompressor\' }", compression);
             else
                 query += " AND compression = {}";
 

test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java

@@ -203,7 +203,7 @@ public void testSetup(String compactionClass, String compressorClass, DiskAccess
         DatabaseDescriptor.setFileCacheEnabled(cacheEnabled);
         DatabaseDescriptor.setDiskAccessMode(mode);
         alterTable("ALTER TABLE %s WITH compaction = { 'class' :  '" + compactionClass + "'  };");
-        alterTable("ALTER TABLE %s WITH compression = { 'sstable_compression' : '" + compressorClass + "'  };");
+        alterTable("ALTER TABLE %s WITH compression = { 'class' : '" + compressorClass + "'  };");
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         cfs.disableAutoCompaction();
 

test/unit/org/apache/cassandra/SchemaLoader.java

@@ -770,15 +770,15 @@ private static CompressionParams compressionParams(int chunkLength)
         switch (algo)
         {
             case "deflate":
-                return CompressionParams.deflate(chunkLength);
+                return TestCompressionParamsFactory.deflate(chunkLength);
             case "lz4":
-                return CompressionParams.lz4(chunkLength);
+                return TestCompressionParamsFactory.lz4(chunkLength);
             case "snappy":
-                return CompressionParams.snappy(chunkLength);
+                return TestCompressionParamsFactory.snappy(chunkLength);
             case "noop":
-                return CompressionParams.noop();
+                return TestCompressionParamsFactory.noop();
             case "zstd":
-                return CompressionParams.zstd(chunkLength);
+                return TestCompressionParamsFactory.zstd(chunkLength);
             case "none":
                 return CompressionParams.noCompression();
             default:

test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java

@@ -35,6 +35,7 @@
 
 import org.apache.cassandra.distributed.shared.WithProperties;
 import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.CompressionParams;
 import org.yaml.snakeyaml.error.YAMLException;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES;
@@ -44,6 +45,8 @@
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 
@@ -170,6 +173,23 @@ public void readThresholdsFromConfig()
         assertThat(c.row_index_read_size_fail_threshold).isEqualTo(new DataStorageSpec.LongBytesBound(1 << 13, KIBIBYTES));
     }
 
+    @Test
+    public void readSSTableCompressionFromConfig()
+    {
+        Config c = load("test/conf/cassandra.yaml");
+
+        assertNull(c.sstable_compression);
+
+        c = load("test/conf/cassandra-with-sstable-compressor.yaml");
+
+        assertNotNull(c.sstable_compression);
+        assertThat(c.sstable_compression.class_name).isEqualTo("lz4");
+        assertThat(c.sstable_compression.parameters.remove(CompressionParams.CHUNK_LENGTH)).isEqualTo("32MiB");
+        assertThat(c.sstable_compression.parameters.remove(CompressionParams.MIN_COMPRESS_RATIO)).isEqualTo("1.5");
+        assertThat(c.sstable_compression.parameters.remove(CompressionParams.ENABLED)).isNull();
+        assertThat(c.sstable_compression.parameters.size()).isEqualTo(2);
+    }
+
     @Test
     public void readThresholdsFromMap()
     {

test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java

@@ -36,31 +36,12 @@
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.FBUtilities;
 
-
 public class CrcCheckChanceTest extends CQLTester
 {
-
-
-    @Test
-    public void testChangingCrcCheckChanceNewFormat() throws Throwable
-    {
-        testChangingCrcCheckChance(true);
-    }
-
     @Test
-    public void testChangingCrcCheckChanceOldFormat() throws Throwable
-    {
-        testChangingCrcCheckChance(false);
-    }
-
-
-    public void testChangingCrcCheckChance(boolean newFormat) throws Throwable
+    public void testChangingCrcCheckChance()
     {
-        //Start with crc_check_chance of 99%
-        if (newFormat)
-            createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor'} AND crc_check_chance = 0.99;");
-        else
-            createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
+        createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'class': 'LZ4Compressor'} AND crc_check_chance = 0.99;");
 
         execute("CREATE INDEX foo ON %s(v)");
 
@@ -79,10 +60,7 @@ public void testChangingCrcCheckChance(boolean newFormat) throws Throwable
         Assert.assertEquals(0.99, indexCfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0);
 
         //Test for stack overflow
-        if (newFormat)
-            alterTable("ALTER TABLE %s WITH crc_check_chance = 0.99");
-        else
-            alterTable("ALTER TABLE %s WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance': 0.99}");
+        alterTable("ALTER TABLE %s WITH crc_check_chance = 0.99");
 
         assertRows(execute("SELECT * FROM %s WHERE p=?", "p1"),
                    row("p1", "k1", "sv1", "v1"),
@@ -131,10 +109,7 @@ public void testChangingCrcCheckChance(boolean newFormat) throws Throwable
         );
 
         //Alter again via schema
-        if (newFormat)
-            alterTable("ALTER TABLE %s WITH crc_check_chance = 0.5");
-        else
-            alterTable("ALTER TABLE %s WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance': 0.5}");
+        alterTable("ALTER TABLE %s WITH crc_check_chance = 0.5");
 
         //We should be able to get the new value by accessing directly the schema metadata
         Assert.assertEquals(0.5, cfs.metadata().params.crcCheckChance, 0.0);
@@ -168,12 +143,12 @@ public void testChangingCrcCheckChance(boolean newFormat) throws Throwable
     }
 
     @Test
-    public void testDropDuringCompaction() throws Throwable
+    public void testDropDuringCompaction()
     {
         CompactionManager.instance.disableAutoCompaction();
 
         //Start with crc_check_chance of 99%
-        createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
+        createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'class': 'LZ4Compressor'} AND crc_check_chance = 0.99;");
 
         ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(currentTable());
 

test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java

@@ -635,10 +635,10 @@ public void testAlterTableWithCompression() throws Throwable
         createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))");
         assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor"));
 
-        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length' : '32KiB' };");
         assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"));
 
-        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'chunk_length_in_kb' : 64 };");
+        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'chunk_length' : '64KiB' };");
         assertSchemaOption("compression", map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"));
 
         alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 2 };");
@@ -650,36 +650,20 @@ public void testAlterTableWithCompression() throws Throwable
         alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 0 };");
         assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor"));
 
-        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+        alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length' : '32KiB' };");
         alterTable("ALTER TABLE %s WITH compression = { 'enabled' : 'false'};");
         assertSchemaOption("compression", map("enabled", "false"));
 
-        assertAlterTableThrowsException(ConfigurationException.class,
-                                        "Missing sub-option 'class' for the 'compression' option.",
-                                        "ALTER TABLE %s WITH  compression = {'chunk_length_in_kb' : 32};");
-
         assertAlterTableThrowsException(ConfigurationException.class,
                                         "The 'class' option must not be empty. To disable compression use 'enabled' : false",
                                         "ALTER TABLE %s WITH  compression = { 'class' : ''};");
 
         assertAlterTableThrowsException(ConfigurationException.class,
-                                        "If the 'enabled' option is set to false no other options must be specified",
-                                        "ALTER TABLE %s WITH compression = { 'enabled' : 'false', 'class' : 'SnappyCompressor'};");
-
-        assertAlterTableThrowsException(ConfigurationException.class,
-                                        "The 'sstable_compression' option must not be used if the compression algorithm is already specified by the 'class' option",
-                                        "ALTER TABLE %s WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'class' : 'SnappyCompressor'};");
-
-        assertAlterTableThrowsException(ConfigurationException.class,
-                                        "The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option",
-                                        "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
-
-        assertAlterTableThrowsException(ConfigurationException.class,
-                                        "Invalid negative min_compress_ratio",
+                                        "Invalid 'min_compress_ratio' value for the 'compression' option.  Can either be 0 or greater than or equal to 1: -1.0",
                                         "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };");
 
         assertAlterTableThrowsException(ConfigurationException.class,
-                                        "min_compress_ratio can either be 0 or greater than or equal to 1",
+                                        "Invalid 'min_compress_ratio' value for the 'compression' option.  Can either be 0 or greater than or equal to 1: 0.5",
                                         "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : 0.5 };");
     }