diff --git a/Jenkinsfile b/Jenkinsfile index a85b2753..f0473873 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { ''' } } - stage('Python bwc-hotfix_downgrades tests') { + stage('Python bwc-hotfix-downgrades tests') { agent { label 'medium' } steps { checkout scm @@ -63,6 +63,20 @@ pipeline { ''' } } + stage('Python bwc-recovery tests') { + agent { label 'medium' } + steps { + checkout scm + sh ''' + rm -rf env + /usr/bin/python3 -m venv env + source env/bin/activate + python -m pip install -U -e . + + (cd tests/bwc && python -m unittest -vvvf test_recovery.py) + ''' + } + } stage('Python restart tests') { agent { label 'medium' } steps { diff --git a/tests/bwc/test_recovery.py b/tests/bwc/test_recovery.py index fbd47e9a..8e9c0656 100644 --- a/tests/bwc/test_recovery.py +++ b/tests/bwc/test_recovery.py @@ -8,13 +8,7 @@ from crate.qa.tests import NodeProvider, insert_data, UpgradePath, assert_busy UPGRADE_PATHS = [ - UpgradePath('4.2.x', '4.3.x'), - UpgradePath('4.3.x', '4.4.0'), - UpgradePath('4.4.x', '4.5.x'), - UpgradePath('4.5.x', '4.6.x'), - UpgradePath('4.6.x', '4.7.x'), - UpgradePath('4.7.x', '4.8.x'), - UpgradePath('4.8.x', '5.0.x'), + # UpgradePath('4.0.x', '5.0.x'), UpgradePath('5.0.x', '5.1.x'), UpgradePath('5.1.x', '5.2.x'), UpgradePath('5.2.x', '5.3.x'), @@ -23,7 +17,6 @@ UPGRADE_PATHS_FROM_43 = [UpgradePath('4.3.x', '4.4.x')] -@unittest.skip('Recovery tests are currently flaky, skip them until fixed') class RecoveryTest(NodeProvider, unittest.TestCase): """ In depth testing of the recovery mechanism during a rolling restart. @@ -254,8 +247,8 @@ def _test_recovery(self, path): with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' - create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 1, - "unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0') + create table doc.test(x int) clustered into 1 shards with(number_of_replicas = 1, + "unassigned.node_left.delayed_timeout" = '20s', "allocation.max_retries" = '10') ''') num_docs = random.randint(0, 10) @@ -292,8 +285,8 @@ def _test_recovery_closed_index(self, path): with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' - create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 1, - "unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0') + create table doc.test(x int) clustered into 1 shards with(number_of_replicas = 1, + "unassigned.node_left.delayed_timeout" = '20s', "allocation.max_retries" = '10') ''') assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) @@ -325,7 +318,7 @@ def _test_closed_index_during_rolling_upgrade(self, path): with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' - create table doc.old_cluster(x int) clustered into 1 shards with( number_of_replicas = 0) + create table doc.old_cluster(x int) clustered into 1 shards with(number_of_replicas = 0) ''') self._assert_is_green(conn, 'doc', 'old_cluster') @@ -337,7 +330,7 @@ def _test_closed_index_during_rolling_upgrade(self, path): self._assert_is_closed(conn, 'doc', 'old_cluster') c.execute(''' - create table doc.mixed_cluster(x int) clustered into 1 shards with( number_of_replicas = 0) + create table doc.mixed_cluster(x int) clustered into 1 shards with(number_of_replicas = 0) ''') assert_busy(lambda: self._assert_is_green(conn, 'doc', 'mixed_cluster')) @@ -352,7 +345,7 @@ def _test_closed_index_during_rolling_upgrade(self, path): self._assert_is_closed(conn, 'doc', 'mixed_cluster') c.execute(''' - create table doc.upgraded_cluster(x int) clustered into 1 shards with( number_of_replicas = 0) + create table doc.upgraded_cluster(x int) clustered into 1 shards with(number_of_replicas = 0) ''') assert_busy(lambda: self._assert_is_green(conn, 'doc', 'upgraded_cluster')) @@ -375,7 +368,7 @@ def _test_update_docs(self, path): c = conn.cursor() c.execute(''' create table doc.test(id int primary key, data text) clustered into 1 shards with( - "unassigned.node_left.delayed_timeout" = '100ms', "number_of_replicas" = 2) + "unassigned.node_left.delayed_timeout" = '20s', "number_of_replicas" = 2) ''') inserts = [(i, str(random.randint)) for i in range(0, 100)] @@ -482,7 +475,7 @@ def _test_turnoff_translog_retention_after_upgraded(self, path): c = conn.cursor() number_of_replicas = random.randint(0, 2) c.execute(''' - create table doc.test(x int) clustered into 1 shards with( number_of_replicas =?, + create table doc.test(x int) clustered into 1 shards with(number_of_replicas =?, "soft_deletes.enabled" = true) ''', (number_of_replicas, )) @@ -538,7 +531,7 @@ def _test_auto_expand_indices_during_rolling_upgrade(self, path): node_ids = c.fetchall() self.assertEqual(len(node_ids), number_of_nodes) - c.execute('''create table doc.test(x int) clustered into 1 shards with( "number_of_replicas" = ?)''', + c.execute('''create table doc.test(x int) clustered into 1 shards with("number_of_replicas" = ?)''', (f"0-{number_of_replicas}",)) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) @@ -632,8 +625,8 @@ def _test_retention_leases_established_when_relocating_primary(self, path): '''create table doc.test(x int) clustered into ? shards with( "number_of_replicas" = ?, "soft_deletes.enabled" = false, - "allocation.max_retries" = 0, - "unassigned.node_left.delayed_timeout" = '100ms' + "allocation.max_retries" = 10, + "unassigned.node_left.delayed_timeout" = '20s' )''', (number_of_shards, number_of_replicas,))