Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable recovery bwc tests #252

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Enable recovery bwc tests #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5c50d06
Enable recovery bwc tests
matriv Mar 13, 2023
f7b46a6
relax allocation params
matriv Mar 13, 2023
82159d6
Keep only one 4.0.x->5.0.x ugprade path
matriv May 15, 2023
9397dec
remove 4.0.x
matriv May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ pipeline {
'''
}
}
stage('Python bwc-hotfix_downgrades tests') {
stage('Python bwc-hotfix-downgrades tests') {
agent { label 'medium' }
steps {
checkout scm
Expand Down Expand Up @@ -63,6 +63,20 @@ pipeline {
'''
}
}
stage('Python bwc-recovery tests') {
agent { label 'medium' }
steps {
checkout scm
sh '''
rm -rf env
/usr/bin/python3 -m venv env
source env/bin/activate
python -m pip install -U -e .

(cd tests/bwc && python -m unittest -vvvf test_recovery.py)
'''
}
}
stage('Python restart tests') {
agent { label 'medium' }
steps {
Expand Down
33 changes: 13 additions & 20 deletions tests/bwc/test_recovery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,7 @@
from crate.qa.tests import NodeProvider, insert_data, UpgradePath, assert_busy

UPGRADE_PATHS = [
UpgradePath('4.2.x', '4.3.x'),
UpgradePath('4.3.x', '4.4.0'),
UpgradePath('4.4.x', '4.5.x'),
UpgradePath('4.5.x', '4.6.x'),
UpgradePath('4.6.x', '4.7.x'),
UpgradePath('4.7.x', '4.8.x'),
UpgradePath('4.8.x', '5.0.x'),
# UpgradePath('4.0.x', '5.0.x'),
UpgradePath('5.0.x', '5.1.x'),
UpgradePath('5.1.x', '5.2.x'),
UpgradePath('5.2.x', '5.3.x'),
Expand All @@ -23,7 +17,6 @@
UPGRADE_PATHS_FROM_43 = [UpgradePath('4.3.x', '4.4.x')]


@unittest.skip('Recovery tests are currently flaky, skip them until fixed')
class RecoveryTest(NodeProvider, unittest.TestCase):
"""
In depth testing of the recovery mechanism during a rolling restart.
Expand Down Expand Up @@ -254,8 +247,8 @@ def _test_recovery(self, path):
with connect(cluster.node().http_url, error_trace=True) as conn:
c = conn.cursor()
c.execute('''
create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 1,
"unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0')
create table doc.test(x int) clustered into 1 shards with(number_of_replicas = 1,
"unassigned.node_left.delayed_timeout" = '20s', "allocation.max_retries" = '10')
''')

num_docs = random.randint(0, 10)
Expand Down Expand Up @@ -292,8 +285,8 @@ def _test_recovery_closed_index(self, path):
with connect(cluster.node().http_url, error_trace=True) as conn:
c = conn.cursor()
c.execute('''
create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 1,
"unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0')
create table doc.test(x int) clustered into 1 shards with(number_of_replicas = 1,
"unassigned.node_left.delayed_timeout" = '20s', "allocation.max_retries" = '10')
''')

assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test'))
Expand Down Expand Up @@ -325,7 +318,7 @@ def _test_closed_index_during_rolling_upgrade(self, path):
with connect(cluster.node().http_url, error_trace=True) as conn:
c = conn.cursor()
c.execute('''
create table doc.old_cluster(x int) clustered into 1 shards with( number_of_replicas = 0)
create table doc.old_cluster(x int) clustered into 1 shards with(number_of_replicas = 0)
''')

self._assert_is_green(conn, 'doc', 'old_cluster')
Expand All @@ -337,7 +330,7 @@ def _test_closed_index_during_rolling_upgrade(self, path):
self._assert_is_closed(conn, 'doc', 'old_cluster')

c.execute('''
create table doc.mixed_cluster(x int) clustered into 1 shards with( number_of_replicas = 0)
create table doc.mixed_cluster(x int) clustered into 1 shards with(number_of_replicas = 0)
''')

assert_busy(lambda: self._assert_is_green(conn, 'doc', 'mixed_cluster'))
Expand All @@ -352,7 +345,7 @@ def _test_closed_index_during_rolling_upgrade(self, path):
self._assert_is_closed(conn, 'doc', 'mixed_cluster')

c.execute('''
create table doc.upgraded_cluster(x int) clustered into 1 shards with( number_of_replicas = 0)
create table doc.upgraded_cluster(x int) clustered into 1 shards with(number_of_replicas = 0)
''')

assert_busy(lambda: self._assert_is_green(conn, 'doc', 'upgraded_cluster'))
Expand All @@ -375,7 +368,7 @@ def _test_update_docs(self, path):
c = conn.cursor()
c.execute('''
create table doc.test(id int primary key, data text) clustered into 1 shards with(
"unassigned.node_left.delayed_timeout" = '100ms', "number_of_replicas" = 2)
"unassigned.node_left.delayed_timeout" = '20s', "number_of_replicas" = 2)
''')

inserts = [(i, str(random.randint)) for i in range(0, 100)]
Expand Down Expand Up @@ -482,7 +475,7 @@ def _test_turnoff_translog_retention_after_upgraded(self, path):
c = conn.cursor()
number_of_replicas = random.randint(0, 2)
c.execute('''
create table doc.test(x int) clustered into 1 shards with( number_of_replicas =?,
create table doc.test(x int) clustered into 1 shards with(number_of_replicas =?,
"soft_deletes.enabled" = true)
''', (number_of_replicas, ))

Expand Down Expand Up @@ -538,7 +531,7 @@ def _test_auto_expand_indices_during_rolling_upgrade(self, path):
node_ids = c.fetchall()
self.assertEqual(len(node_ids), number_of_nodes)

c.execute('''create table doc.test(x int) clustered into 1 shards with( "number_of_replicas" = ?)''',
c.execute('''create table doc.test(x int) clustered into 1 shards with("number_of_replicas" = ?)''',
(f"0-{number_of_replicas}",))
assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test'))

Expand Down Expand Up @@ -632,8 +625,8 @@ def _test_retention_leases_established_when_relocating_primary(self, path):
'''create table doc.test(x int) clustered into ? shards with(
"number_of_replicas" = ?,
"soft_deletes.enabled" = false,
"allocation.max_retries" = 0,
"unassigned.node_left.delayed_timeout" = '100ms'
"allocation.max_retries" = 10,
"unassigned.node_left.delayed_timeout" = '20s'
)''',
(number_of_shards, number_of_replicas,))

Expand Down