Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DB-12133 Fix predicate packing logic for multiple IN lists MultiProbeScan #5671

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

DB-12133 Fix predicate packing logic for multiple IN lists MultiProbeScan #5671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1387e80
DB-12133 Fix predicate packing logic for multiple IN lists MultiProbe…
msirek Jul 14, 2021
3df5a78
DB-12133 Fix InListMultiprobeIT
msirek Jul 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions db-engine/src/main/java/com/splicemachine/db/impl/sql/compile/PredicateList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1374,13 +1374,13 @@ isHashableJoin && isQualifierForHashableJoin(pred, optTable, cd, pushPreds)){
// Pack the remaining useful preds in usefulPredicates so
// there are no gaps.
int j = firstPred + 1;
for (int i = firstPred + 1; i < usefulCount; i++) {
for (int i = firstPred + 1; i < usefulCount && j < usefulCount; i++) {
while(i < usefulCount && foundPred[i])
i++;
if (i != j && i < usefulCount) {
usefulPredicates[j] = usefulPredicates[i];
j++;
}
j++;
}
usefulCount -= (predsForNewInList.size() - 1);
}
Expand Down
7 changes: 3 additions & 4 deletions splice_machine/src/test/java/com/splicemachine/db/impl/sql/compile/InListMultiprobeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -664,13 +664,12 @@ public void testInListWithFloatIT() throws Exception {
/** explain should look like the following:
Cursor(n=3,rows=17,updateMode=READ_ONLY (1),engine=control)
-> ScrollInsensitive(n=2,totalCost=8.199,outputRows=17,outputHeapSize=34 B,partitions=1)
-> ProjectRestrict(n=1,totalCost=4.027,outputRows=1,outputHeapSize=0 B,partitions=1,preds=[(F[0:1] <> 8)])
-> MultiProbeIndexScan[IX_FLOAT(7697)](n=0,totalCost=4.027,scannedRows=17,outputRows=1,outputHeapSize=0 B,partitions=1,baseTable=TS_FLOAT(7680),preds=[((F[0:1],N[0:2],R[0:3]) IN ((1.0,1,4.0),(1.0,3,4.0),(1.0,4,4.0),(1.0,5,4.0),(2.0,1,4.0),(2.0,3,4.0),(2.0,4,4.0),(2.0,5,4.0),(3.0,1,4.0),(3.0,3,4.0),(3.0,4,4.0),(3.0,5,4.0),(4.0,1,4.0),(4.0,3,4.0),(4.0,4,4.0),(4.0,5,4.0))),(F[0:1] <> 6),(F[0:1] <> 7),(R[0:3] <> 55),(R[0:3] <> 33),(R[0:3] <> 44),(R[0:3] <> 55)]) |
-> MultiProbeIndexScan[IX_FLOAT(2625)](n=0,totalCost=4.027,scannedRows=17,outputRows=1,outputHeapSize=0 B,partitions=1,parallelTasks=1,baseTable=TS_FLOAT(2608),keys=[((F[0:1],N[0:2],R[0:3]) IN ((1.0,1,4.0),(1.0,3,4.0),(1.0,4,4.0),(1.0,5,4.0),(2.0,1,4.0),(2.0,3,4.0),(2.0,4,4.0),(2.0,5,4.0),(3.0,1,4.0),(3.0,3,4.0),(3.0,4,4.0),(3.0,5,4.0),(4.0,1,4.0),(4.0,3,4.0),(4.0,4,4.0),(4.0,5,4.0)))],preds=[(F[0:1] <> 6),(F[0:1] <> 7),(F[0:1] <> 8),(R[0:3] <> 33),(R[0:3] <> 44),(R[0:3] <> 55)]) |
*/
level = 1;
while (rs.next()) {
String resultString = rs.getString(1);
if (level == 4) {
if (level == 3) {
Assert.assertTrue("MultiProbeIndexScan is expected", resultString.contains("MultiProbeIndexScan"));
}
level++;
Expand Down Expand Up @@ -953,7 +952,7 @@ public void testInListWithDynamicParamsIT() throws Exception {
level = 1;
while (rs.next()) {
String resultString = rs.getString(1);
if (level == 4) {
if (level == 3) {
Assert.assertTrue("MultiProbeIndexScan is expected", resultString.contains("MultiProbeIndexScan"));
}
level++;
Expand Down
148 changes: 101 additions & 47 deletions ...va/com/splicemachine/derby/impl/sql/execute/operations/MultiProbeTableScanOperatonIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,26 @@ public class MultiProbeTableScanOperatonIT extends SpliceUnitTest {
protected static SpliceTableWatcher t7Watcher = new SpliceTableWatcher("t7",schemaWatcher.schemaName,"(a7 int, b7 int, c7 int)");
protected static SpliceIndexWatcher i7Watcher = new SpliceIndexWatcher("t7",schemaWatcher.schemaName,"ix_t7",schemaWatcher.schemaName,"(b7, c7)");

protected static SpliceTableWatcher t11Watcher = new SpliceTableWatcher("t11",schemaWatcher.schemaName," (a1 int,\n" +
" b1 int,\n" +
" c1 int,\n" +
" d1 int,\n" +
" e1 int,\n" +
" f1 int,\n" +
" g1 int,\n" +
" h1 char(80),\n" +
" primary key (a1, b1, c1))");
protected static SpliceIndexWatcher i11Watcher = new SpliceIndexWatcher("t11",schemaWatcher.schemaName,"idx_t11",schemaWatcher.schemaName,"(g1, f1, e1)");
protected static SpliceTableWatcher t22Watcher = new SpliceTableWatcher("t22",schemaWatcher.schemaName," (a2 int,\n" +
" b2 int,\n" +
" c2 int,\n" +
" d2 int,\n" +
" e2 int,\n" +
" f2 int,\n" +
" g2 int,\n" +
" h2 char(80),\n" +
" primary key (a2, b2, c2))");

private static int size = 10;

@ClassRule
Expand All @@ -59,6 +79,9 @@ public class MultiProbeTableScanOperatonIT extends SpliceUnitTest {
.around(i6Watcher)
.around(t7Watcher)
.around(i7Watcher)
.around(t11Watcher)
.around(i11Watcher)
.around(t22Watcher)
.around(new SpliceDataWatcher() {
@Override
protected void starting(Description description) {
Expand Down Expand Up @@ -139,6 +162,22 @@ protected void starting(Description description) {
}
ps.executeBatch();

ps = spliceClassWatcher.prepareStatement("insert into " + t11Watcher.toString() + " values (?,?,?,?,?,?,?,?)");
for (int i = 0; i < 3; i++) {
ps.setInt(1, i);
ps.setInt(2, i);
ps.setInt(3, i);
ps.setInt(4, i);
ps.setInt(5, i);
ps.setInt(6, i);
ps.setInt(7, i);
ps.setString(8, "a");
ps.addBatch();
}
ps.executeBatch();

spliceClassWatcher.executeUpdate("insert into t22 select * from t11");

} catch (Exception e) {
throw new RuntimeException(e);
} finally {
Expand All @@ -150,58 +189,71 @@ protected void starting(Description description) {

@Rule public SpliceWatcher methodWatcher = new SpliceWatcher(CLASS_NAME);

@Test
public void testMultiColumnMultiProbeWithAdditionalJoinPredicate() throws Exception {
String sql = "select a1, t22.a2 from t11 INNER JOIN t22 on t11.a1=t22.a2 " +
"where a1=(SELECT MAX(a2) FROM t22 ) and a1 in (0,1,2) " +
"and b1 in (0,1,2,3,4,5) and c1 in (0,1,2,3,4,5,6,7,8) order by t22.a2";
String expected = "A1 |A2 |\n" +
"--------\n" +
" 2 | 2 |";
testQuery(sql, expected, methodWatcher);
}

@Test
public void testMultiProbeTableScanScroll() throws Exception {
ResultSet rs = methodWatcher.executeQuery("select user_id from "+t1Watcher+" where segment_id in (1,5,8,12)");
int i = 0;
while (rs.next()) {
i++;
try (ResultSet rs = methodWatcher.executeQuery("select user_id from "+t1Watcher+" where segment_id in (1,5,8,12)")) {
int i = 0;
while (rs.next()) {
i++;
}
Assert.assertEquals("Incorrect count returned!", 3, i);
}
Assert.assertEquals("Incorrect count returned!",3,i);
}

@Test
//DB-2575
public void testMultiProbeTableScanWithEqualPredicate() throws Exception {
ResultSet rs = methodWatcher.executeQuery("select user_id from "+t1Watcher+" where segment_id in (1,5,8,12) and unixtime = 1");
int i = 0;
while (rs.next()) {
i++;
try (ResultSet rs = methodWatcher.executeQuery("select user_id from "+t1Watcher+" where segment_id in (1,5,8,12) and unixtime = 1")) {
int i = 0;
while (rs.next()) {
i++;
}
Assert.assertEquals("Incorrect count returned!", 3, i);
}
Assert.assertEquals("Incorrect count returned!",3,i);
}

@Test
public void testMultiProbeTableScanSink() throws Exception {
ResultSet rs = methodWatcher.executeQuery("select count(user_id) from (" +
try (ResultSet rs = methodWatcher.executeQuery("select count(user_id) from (" +
"select user_id, ("+
"max(case when segment_id = 7 then true else false end) " +
"or " +
"max(case when segment_id = 4 then true else false end)" +
") as in_segment " +
"from "+t1Watcher+ " " +
"where segment_id in (7, 4) " +
"group by user_id) foo where in_segment = true");
int i = 0;
while (rs.next()) {
i++;
Assert.assertEquals("Incorrect Distinct Customers",3,rs.getLong(1));
"group by user_id) foo where in_segment = true")) {
int i = 0;
while (rs.next()) {
i++;
Assert.assertEquals("Incorrect Distinct Customers", 3, rs.getLong(1));
}
Assert.assertEquals("Incorrect records returned!", 1, i);
}
Assert.assertEquals("Incorrect records returned!",1,i);
}

@Test
public void testMultiProbeInSubQueryWithIndex() throws Exception {
/* Regression test for DB-1040 */
SpliceIndexWatcher indexWatcher = new SpliceIndexWatcher(t3Watcher.tableName, t3Watcher.getSchema(),"new_index_3",t3Watcher.getSchema(),"(collid)");
indexWatcher.starting(null);
try{
ResultSet rs = methodWatcher.executeQuery("select count(id) from docs where id > any (select id from colls where collid in (-2,1))");
try (ResultSet rs = methodWatcher.executeQuery("select count(id) from docs where id > any (select id from colls where collid in (-2,1))")) {
Assert.assertTrue("No results returned!",rs.next());
int count = rs.getInt(1);
Assert.assertEquals("Incorrect count returned!",4,count);
Assert.assertFalse("Too many rows returned!",rs.next());
}finally{
} finally{
indexWatcher.drop();
}
}
Expand All @@ -211,13 +263,14 @@ public void testMultiProbeInSubQueryWithIndex() throws Exception {
public void testMultiProbeIntegerValue() throws Exception {
SpliceIndexWatcher indexWatcher = new SpliceIndexWatcher(t4Watcher.tableName, t4Watcher.getSchema(),"idxb",t4Watcher.getSchema(),"(d)");
indexWatcher.starting(null);
ResultSet rs = methodWatcher.executeQuery("select count(*) from b where d in (9,10)");
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 2);

rs = methodWatcher.executeQuery("select count(*) from b where d in (9)");
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1)==1);
try (ResultSet rs = methodWatcher.executeQuery("select count(*) from b where d in (9,10)")) {
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 2);
}
try (ResultSet rs = methodWatcher.executeQuery("select count(*) from b where d in (9)")) {
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 1);
}
}

@Test
Expand All @@ -229,12 +282,13 @@ public void testMultiProbeTableScanWithProbeVariables() throws Exception {
ps.setInt(3,8);
ps.setInt(4,12);
ps.setLong(5,1);
ResultSet rs = ps.executeQuery();
int i = 0;
while (rs.next()) {
i++;
try (ResultSet rs = ps.executeQuery()) {
int i = 0;
while (rs.next()) {
i++;
}
Assert.assertEquals("Incorrect count returned!", 3, i);
}
Assert.assertEquals("Incorrect count returned!",3,i);
}


Expand All @@ -248,22 +302,22 @@ public void testMultiProbeWithComputations() throws Exception {
// DB-1323
@Test
public void testMultiProbeWithSpark() throws Exception {
ResultSet rs = methodWatcher.executeQuery("select count(*) from "+ t6Watcher+ "--splice-properties useSpark=true\n" +
" where j in (1181) and (i = 1 or i = 118 )");
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 3);
rs.close();

rs = methodWatcher.executeQuery("select count(*) from "+ t6Watcher+ "--splice-properties useSpark=true\n" +
" where i in (1, 118)");
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 3);
try (ResultSet rs = methodWatcher.executeQuery("select count(*) from "+ t6Watcher+ "--splice-properties useSpark=true\n" +
" where j in (1181) and (i = 1 or i = 118 )")) {
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 3);
}

try (ResultSet rs = methodWatcher.executeQuery("select count(*) from "+ t6Watcher+ "--splice-properties useSpark=true\n" +
" where i in (1, 118)")) {
Assert.assertTrue(rs.next());
Assert.assertTrue("wrong count", rs.getInt(1) == 3);
}
}

@Test
public void testMultiProbeWithLargeInListThroughSpark() throws Exception {
ResultSet rs = methodWatcher.executeQuery(format("select count(*) from %s --splice-properties useSpark=true, index=%s\n" +
try (ResultSet rs = methodWatcher.executeQuery(format("select count(*) from %s --splice-properties useSpark=true, index=%s\n" +
" where b7 in (0,1,2,3,4,5,6,7,8,9,\n" +
"10,11,12,13,14,15,16,17,18,19,\n" +
"20,21,22,23,24,25,26,27,28,29,\n" +
Expand Down Expand Up @@ -313,9 +367,9 @@ public void testMultiProbeWithLargeInListThroughSpark() throws Exception {
"460,461,462,463,464,465,466,467,468,469,\n" +
"470,471,472,473,474,475,476,477,478,479,\n" +
"480,481,482,483,484,485,486,487,488,489,\n" +
"490,491,492,493,494,495,496,497,498,499)", t7Watcher, "ix_t7"));
Assert.assertTrue(rs.next());
Assert.assertTrue(format("wrong count: expected: %d, actual: %d", 500, rs.getInt(1)), rs.getInt(1) == 500);
rs.close();
"490,491,492,493,494,495,496,497,498,499)", t7Watcher, "ix_t7"))) {
Assert.assertTrue(rs.next());
Assert.assertTrue(format("wrong count: expected: %d, actual: %d", 500, rs.getInt(1)), rs.getInt(1) == 500);
}
}
}