-
Notifications
You must be signed in to change notification settings - Fork 1
/
tests.py
183 lines (173 loc) · 9.23 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
from __future__ import absolute_import
from mock.mock import patch
from couchdb_cluster_admin.suggest_shard_allocation import suggest_shard_allocation, _NodeAllocation
from couchdb_cluster_admin.doc_models import ShardAllocationDoc
from couchdb_cluster_admin.file_plan import get_missing_files_by_node_and_source, Nodefile
@patch('couchdb_cluster_admin.file_plan.get_node_files', return_value=({'node1': [
Nodefile('db1', 'node2', 'shard3', 'f3'),
Nodefile('db1', 'node3', 'shard1', 'f1'),
Nodefile('db1', 'node3', 'shard2', 'f2'),
Nodefile('db1', 'node3', 'shard4', 'f4'),
]}, None))
@patch('couchdb_cluster_admin.file_plan.get_shard_allocation', return_value=ShardAllocationDoc.from_plan_json(
'db1', {
'shard_suffix': '123132',
'by_range': {
'shard1': ['node1'],
'shard2': ['node1'],
'shard3': ['node2'],
'shard4': ['node2'],
}
}
))
def test_get_missing_files(m1, m2):
"""
from:
node1: shard1, shard2
node2: shard3, shard4
to:
node2: shard3
node3: shard1, shard2, shard4
"""
missing_files = get_missing_files_by_node_and_source(None, None)
assert missing_files == {
'node3': {
'node1': [Nodefile('db1', 'node3', 'shard1', 'f1'), Nodefile('db1', 'node3', 'shard2', 'f2')],
'node2': [Nodefile('db1', 'node3', 'shard4', 'f4')],
}
}
def test_suggest_shard_allocation():
real = suggest_shard_allocation(
shard_sizes=[
(916925224199.5, (u'00000000-1fffffff', u'commcarehq')),
(916925224199.5, (u'20000000-3fffffff', u'commcarehq')),
(916925224199.5, (u'40000000-5fffffff', u'commcarehq')),
(916925224199.5, (u'60000000-7fffffff', u'commcarehq')),
(916925224199.5, (u'80000000-9fffffff', u'commcarehq')),
(916925224199.5, (u'a0000000-bfffffff', u'commcarehq')),
(916925224199.5, (u'c0000000-dfffffff', u'commcarehq')),
(916925224199.5, (u'e0000000-ffffffff', u'commcarehq')),
],
n_nodes=8,
n_copies=3,
)
expected = [
_NodeAllocation(i=0, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'a0000000-bfffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq')
]),
_NodeAllocation(i=1, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq')]),
_NodeAllocation(i=2, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=3, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=4, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=5, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')]),
_NodeAllocation(i=6, size=2750775672598.5,
shards=[(u'a0000000-bfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')]),
_NodeAllocation(i=7, size=2750775672598.5,
shards=[(u'a0000000-bfffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')])
]
assert real == expected
def test_suggest_shard_allocation__no_chnage():
existing_allocation = [
_NodeAllocation(i=0, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'a0000000-bfffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq')
]),
_NodeAllocation(i=1, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq')]),
_NodeAllocation(i=2, size=2750775672598.5,
shards=[(u'e0000000-ffffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=3, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'80000000-9fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=4, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'20000000-3fffffff', u'commcarehq')]),
_NodeAllocation(i=5, size=2750775672598.5,
shards=[(u'c0000000-dfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')]),
_NodeAllocation(i=6, size=2750775672598.5,
shards=[(u'a0000000-bfffffff', u'commcarehq'),
(u'60000000-7fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')]),
_NodeAllocation(i=7, size=2750775672598.5,
shards=[(u'a0000000-bfffffff', u'commcarehq'),
(u'40000000-5fffffff', u'commcarehq'),
(u'00000000-1fffffff', u'commcarehq')])
]
real = suggest_shard_allocation(
shard_sizes=[
(916925224199.5, (u'00000000-1fffffff', u'commcarehq')),
(916925224199.5, (u'20000000-3fffffff', u'commcarehq')),
(916925224199.5, (u'40000000-5fffffff', u'commcarehq')),
(916925224199.5, (u'60000000-7fffffff', u'commcarehq')),
(916925224199.5, (u'80000000-9fffffff', u'commcarehq')),
(916925224199.5, (u'a0000000-bfffffff', u'commcarehq')),
(916925224199.5, (u'c0000000-dfffffff', u'commcarehq')),
(916925224199.5, (u'e0000000-ffffffff', u'commcarehq')),
],
n_nodes=8,
n_copies=3,
existing_allocation=[node.shards for node in existing_allocation],
)
# assert no change
assert real == existing_allocation
def test_suggest_shard_allocation__increase_copies():
# start with 3 copies across 7 nodes
existing_allocation = [
{(u'e0000000-ffffffff', u'commcarehq'), (u'00000000-1fffffff', u'commcarehq'), (u'40000000-5fffffff', u'commcarehq'), (u'20000000-3fffffff', u'commcarehq')},
{(u'c0000000-dfffffff', u'commcarehq'), (u'e0000000-ffffffff', u'commcarehq'), (u'00000000-1fffffff', u'commcarehq'), (u'20000000-3fffffff', u'commcarehq')},
{(u'a0000000-bfffffff', u'commcarehq'), (u'e0000000-ffffffff', u'commcarehq'), (u'00000000-1fffffff', u'commcarehq'), (u'c0000000-dfffffff', u'commcarehq')},
{(u'80000000-9fffffff', u'commcarehq'), (u'c0000000-dfffffff', u'commcarehq'), (u'a0000000-bfffffff', u'commcarehq')},
{(u'60000000-7fffffff', u'commcarehq'), (u'a0000000-bfffffff', u'commcarehq'), (u'80000000-9fffffff', u'commcarehq')},
{(u'40000000-5fffffff', u'commcarehq'), (u'80000000-9fffffff', u'commcarehq'), (u'60000000-7fffffff', u'commcarehq')},
{(u'20000000-3fffffff', u'commcarehq'), (u'60000000-7fffffff', u'commcarehq'), (u'40000000-5fffffff', u'commcarehq')},
{},
]
# and then add an 8th node
new_allocation = suggest_shard_allocation(
shard_sizes=[
(916925224199.5, (u'00000000-1fffffff', u'commcarehq')),
(916925224199.5, (u'20000000-3fffffff', u'commcarehq')),
(916925224199.5, (u'40000000-5fffffff', u'commcarehq')),
(916925224199.5, (u'60000000-7fffffff', u'commcarehq')),
(916925224199.5, (u'80000000-9fffffff', u'commcarehq')),
(916925224199.5, (u'a0000000-bfffffff', u'commcarehq')),
(916925224199.5, (u'c0000000-dfffffff', u'commcarehq')),
(916925224199.5, (u'e0000000-ffffffff', u'commcarehq')),
],
n_nodes=8,
n_copies=3,
existing_allocation=existing_allocation,
)
for i, node in enumerate(new_allocation):
# the nodes should now be properly balanced with 3 shard-copies per node
assert len(node.shards) == 3