better print for terms filter, expanded support for docset producer, …

…qparser getFilter, bool query conversion

solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java

@@ -191,7 +191,7 @@ public void prepare(ResponseBuilder rb) throws IOException
         for (String fq : fqs) {
           if (fq != null && fq.trim().length()!=0) {
             QParser fqp = QParser.getParser(fq, null, req);
-            filters.add(fqp.getQuery());
+            filters.add(fqp.getFilter());
           }
         }
         // only set the filters if they are not empty otherwise

solr/core/src/java/org/apache/solr/query/FilterQuery.java

@@ -26,12 +26,17 @@
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.search.DocSet;
+import org.apache.solr.search.DocSetProducer;
+import org.apache.solr.search.DocSetUtil;
+import org.apache.solr.search.ExtendedQueryBase;
+import org.apache.solr.search.QueryContext;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.WrappedQuery;
 
 import java.io.IOException;
 import java.util.Set;
 
-public class FilterQuery extends Query {
+public class FilterQuery extends ExtendedQueryBase implements DocSetProducer {
   protected final Query q;
 
   public FilterQuery(Query q) {
@@ -101,4 +106,9 @@ public Weight createWeight(IndexSearcher searcher) throws IOException {
     csq.setBoost( this.getBoost() );
     return csq.createWeight(searcher);
   }
+
+  @Override
+  public DocSet createDocSet(QueryContext queryContext) throws IOException {
+    return DocSetUtil.createDocSet(queryContext, q);
+  }
 }

solr/core/src/java/org/apache/solr/query/TFilter.java

@@ -17,42 +17,55 @@
  * limitations under the License.
  */
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.core.HS;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaUtil;
 import org.apache.solr.search.DedupDocSetCollector;
 import org.apache.solr.search.DocSet;
 import org.apache.solr.search.DocSetProducer;
 import org.apache.solr.search.QueryContext;
 import org.apache.solr.search.field.NativePagedBytes;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-
 // Builder with prefix coding, native array
-class TFilter extends Filter implements DocSetProducer {
+public class TFilter extends Filter implements DocSetProducer {
+  public static long num_creations; // keeps track of the number of creations for testing purposes
+
   private final String field;
   final byte[] termBytes;
   private final int nTerms;
   private final int unsorted;
   private final int hash;
 
   private TFilter(String field, byte[] termBytes, int nTerms, int unsorted, int hash) {
+    this.num_creations++;
     this.field = field;
     this.termBytes = termBytes;
     this.nTerms = nTerms;
@@ -122,6 +135,10 @@ public TFilter build() {
       return new TFilter(field, getBytes(), nTerms, unsorted, hash);
     }
 
+    public TQuery buildQuery() {
+      return new TQuery(build());
+    }
+
     @Override
     public void close() {
       if (termBytes != null) {
@@ -218,7 +235,46 @@ public boolean equals(Object o) {
 
   @Override
   public String toString() {
-    return "terms(f=" + field + ",num="+nTerms+",sorted="+(unsorted>0)+",nbytes="+termBytes.length + ")";
+    StringBuilder sb = new StringBuilder();
+    sb.append("{!terms f="+field+"}");
+
+    boolean needSep = false;
+    boolean truncated = false;
+    int numTerms = 0;
+
+    FieldType ft = SchemaUtil.getFieldTypeNoContext(field);
+
+    CharsRef charsOut = new CharsRef();
+
+    BytesRefIterator iter = iterator();
+    try {
+      for (;;) {
+        if (++numTerms > 20) {
+          truncated = true;
+          break;
+        }
+        BytesRef term = iter.next();
+        if (term == null) break;
+        if (needSep) {
+         sb.append(',');
+        } else {
+         needSep = true;
+        }
+
+        // TODO - make this more efficient
+        charsOut.length=0;
+        ft.indexedToReadable(term, charsOut);
+        sb.append(charsOut);
+      }
+    } catch (IOException e) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+    }
+
+    if (truncated) {
+      sb.append(",...(truncated=true, nTerms="+nTerms+",sorted="+(unsorted>0)+",nbytes="+termBytes.length+")");
+    }
+
+   return sb.toString();
   }
 
 
@@ -262,5 +318,58 @@ public DocSet createDocSet(QueryContext queryContext) throws IOException {
     }
   }
 
+  /** returns null if the BooleanQuery can't be converted. */
+  public static Query convertBooleanQuery(BooleanQuery q) {
+    List<BytesRef> lst = new ArrayList<>(q.clauses().size());
+    String field = getTerms(q, null, lst);
+    if (field == null) {
+      return null;
+    }
+
+    Collections.sort(lst);
+
+    TFilter.Builder builder = new TFilter.Builder(field, lst.size()*8);
+    try {
+      BytesRef prev = new BytesRef();
+      for (BytesRef br : lst) {
+        builder.addTerm(prev, br);
+        prev = br;
+      }
+      return builder.buildQuery();
+    } finally {
+      builder.close();
+    }
+  }
+
+
+  private static String getTerms(BooleanQuery q, String field, List<BytesRef> out) {
+    if (q.getMinimumNumberShouldMatch() != 0) {
+      return null;
+    }
+
+    List<BooleanClause> clauses = q.clauses();
+    for (BooleanClause clause : clauses) {
+      if (clause.isProhibited() || clause.isRequired()) {
+        return null;
+      }
+      Query sub = clause.getQuery();
+      if (sub instanceof TermQuery) {
+        TermQuery tq = (TermQuery)sub;
+        String tqf = tq.getTerm().field();
+        if (field == null) {
+          field = tqf;
+        } else {
+          if (!field.equals(tqf)) {
+            return null;
+          }
+        }
+        out.add( tq.getTerm().bytes() );
+      } else if (sub instanceof BooleanQuery) {
+        return getTerms((BooleanQuery) sub, field, out);
+      }
+    }
+
+    return field;
+  }
 
 }

solr/core/src/java/org/apache/solr/query/TQuery.java

@@ -0,0 +1,33 @@
+package org.apache.solr.query;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.search.ExtendedQueryBase;
+import org.apache.solr.search.SolrConstantScoreQuery;
+
+public class TQuery extends SolrConstantScoreQuery {
+  public TQuery(TFilter filter) {
+    super(filter);
+  }
+
+  @Override
+  public String toString() {
+    String opts = ExtendedQueryBase.getOptionsString(this);
+    return opts + filter.toString();
+  }
+}

solr/core/src/java/org/apache/solr/query/TermsQParserPlugin.java

@@ -88,16 +88,14 @@ public long callback(CharSequence info) {
                builder.addTerm(prev, br);
                prev = br;
              }
-             TFilter tf = builder.build();
-             return new SolrConstantScoreQuery(tf);
+             return builder.buildQuery();
            } finally {
              builder.close();
            }
          }
 
 
          // if not sorting, build incrementally to avoid instantiating entire list
-         TFilter tfilter;
          try (TFilter.Builder builder = new TFilter.Builder(field, termStr.length())) {
 
            CharUtils.splitSmart(termStr, sepChar, true, new Callback<CharSequence>() {
@@ -117,10 +115,9 @@ public long callback(CharSequence info) {
              }
            });
 
-           tfilter = builder.build();
+           return builder.buildQuery();
          }
 
-         return new SolrConstantScoreQuery(tfilter);
        }
 
 

solr/core/src/java/org/apache/solr/schema/SchemaUtil.java

@@ -0,0 +1,39 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.request.SolrRequestInfo;
+
+public class SchemaUtil {
+  public static FieldType GENERIC_STRTYPE = new StrField();
+
+
+  /** Returns a FieldType for the given field name (or defaults to a generic StrField type if the schema can't be found, or if the field doesn't exist in the schema).
+   * This should *only* be used when there is no normal solr request object or other context.
+   */
+  public static FieldType getFieldTypeNoContext(String fieldName) {
+    SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
+    if (reqInfo == null) return GENERIC_STRTYPE;
+
+    FieldType ft = reqInfo.getReq().getSchema().getFieldTypeNoEx(fieldName);
+    return ft == null ? GENERIC_STRTYPE : ft;
+  }
+
+
+}

solr/core/src/java/org/apache/solr/search/BitDocSetNative.java

@@ -521,7 +521,7 @@ public int prevSetBit(int index) {
 
   /** this = this AND other */
   public void intersectMe(BitDocSetNative other) {
-    assert this.wlen == other.wlen;
+    assert this.wlen == other.wlen && getRefCount() == 1; // can't be shared
     long thisArr = this.array;
     long otherArr = other.array;
     // testing against zero can be more efficient
@@ -533,7 +533,7 @@ public void intersectMe(BitDocSetNative other) {
 
   /** this = this OR other */
   public void unionMe(BitDocSetNative other) {
-    assert this.wlen == other.wlen;
+    assert this.wlen == other.wlen && getRefCount() == 1; // can't be shared
     long thisArr = this.array;
     long otherArr = other.array;
     // testing against zero can be more efficient
@@ -545,7 +545,7 @@ public void unionMe(BitDocSetNative other) {
 
   /** Remove all elements set in other. this = this AND_NOT other */
   public void remove(BitDocSetNative other) {
-    assert this.wlen == other.wlen;
+    assert this.wlen == other.wlen && getRefCount() == 1; // can't be shared
     long thisArr = this.array;
     long otherArr = other.array;
     // testing against zero can be more efficient
@@ -557,7 +557,7 @@ public void remove(BitDocSetNative other) {
 
   /** Remove all elements set in other. this = this AND_NOT other */
   public void xorMe(BitDocSetNative other) {
-    assert this.wlen == other.wlen;
+    assert this.wlen == other.wlen && getRefCount() == 1; // can't be shared
     long thisArr = this.array;
     long otherArr = other.array;
     // testing against zero can be more efficient
@@ -642,7 +642,7 @@ public void setBitsOn(BitDocSetNative target) {
 
   @Override
   public void addAllTo(DocSet target) {
-
+    throw new UnsupportedOperationException();
   }
 
   @Override

solr/core/src/java/org/apache/solr/search/DocSetBaseNative.java

@@ -387,6 +387,7 @@ public void setBitsOn(FixedBitSet target) {
 
   @Override
   public void setBitsOn(BitDocSetNative target) {
+    assert target.getRefCount() == 1; // can't be shared
     DocIterator iter = iterator();
     while (iter.hasNext()) {
       target.fastSet(iter.nextDoc());