PCBC-987: Fix consistency vector encoding for FTS

Couchbase/MutationState.php

@@ -60,8 +60,8 @@ public function export(): array
         foreach ($this->tokens as $token) {
             $state[] = [
                 "partitionId" => $token->partitionId(),
-                "partitionUuid" => hexdec($token->partitionUuid()),
-                "sequenceNumber" => hexdec($token->sequenceNumber()),
+                "partitionUuid" => $token->partitionUuid(),
+                "sequenceNumber" => $token->sequenceNumber(),
                 "bucketName" => $token->bucketName(),
             ];
         }

Couchbase/SearchOptions.php

@@ -29,7 +29,7 @@ class SearchOptions implements JsonSerializable
     private ?int $skip = null;
     private ?bool $explain = null;
     private ?bool $disableScoring = null;
-    private ?array $consistentWith = null;
+    private ?MutationState $consistentWith = null;
     private ?array $fields = null;
     private ?array $facets = null;
     private ?array $sort = null;
@@ -132,16 +132,7 @@ public function disableScoring(bool $disabled): SearchOptions
      */
     public function consistentWith(string $index, MutationState $state): SearchOptions
     {
-        $vectors = [];
-        foreach ($state->tokens() as $token) {
-            $vectors[] = [
-                'partitionId' => $token->partitionId(),
-                'partitionUuid' => $token->partitionUuid(),
-                'sequenceNumber' => $token->sequenceNumber(),
-                'bucketName' => $token->bucketName(),
-            ];
-        }
-        $this->consistentWith = $vectors;
+        $this->consistentWith = $state;
         return $this;
     }
 
@@ -322,7 +313,7 @@ public static function export(?SearchOptions $options): array
             'disableScoring' => $options->disableScoring,
             'fields' => $options->fields,
             'sortSpecs' => $sort,
-            'consistentWith' => $options->consistentWith,
+            'consistentWith' => $options->consistentWith == null ? null : $options->consistentWith->export(),
             'facets' => $options->facets,
             'highlightStyle' => $highlightStyle,
             'highlightFields' => $highlightFields,

src/wrapper/conversion_utilities.cxx

@@ -732,16 +732,16 @@ zval_to_common_search_request(const zend_string* index_name, const zend_string*
             std::uint64_t sequence_number;
             std::uint16_t partition_id;
             std::string bucket_name;
-            if (auto e = cb_assign_integer(partition_id, options, "partitionId"); e.ec) {
+            if (auto e = cb_assign_integer(partition_id, item, "partitionId"); e.ec) {
                 return { {}, e };
             }
-            if (auto e = cb_assign_integer(partition_uuid, options, "partitionUuid"); e.ec) {
+            if (auto e = cb_assign_integer(partition_uuid, item, "partitionUuid"); e.ec) {
                 return { {}, e };
             }
-            if (auto e = cb_assign_integer(sequence_number, options, "sequenceNumber"); e.ec) {
+            if (auto e = cb_assign_integer(sequence_number, item, "sequenceNumber"); e.ec) {
                 return { {}, e };
             }
-            if (auto e = cb_assign_string(bucket_name, options, "bucketName"); e.ec) {
+            if (auto e = cb_assign_string(bucket_name, item, "bucketName"); e.ec) {
                 return { {}, e };
             }
             vectors.emplace_back(mutation_token{ partition_uuid, sequence_number, partition_id, bucket_name });

src/wrapper/conversion_utilities.hxx

@@ -32,6 +32,7 @@
 #include <chrono>
 
 #include <fmt/format.h>
+#include <type_traits>
 
 namespace couchbase::transactions
 {
@@ -67,6 +68,44 @@ query_response_to_zval(zval* return_value, const core::operations::query_respons
 void
 search_query_response_to_zval(zval* return_value, const core::operations::search_response& resp);
 
+template <typename Integer>
+static Integer
+parse_integer(const std::string& str, std::size_t* pos = 0, int base = 10)
+{
+    if constexpr (std::is_signed_v<Integer>) {
+        return std::stoll(str, pos, base);
+    } else {
+        return std::stoull(str, pos, base);
+    }
+}
+
+template<typename Integer>
+static std::pair<core_error_info, std::optional<Integer>>
+cb_get_integer_from_hex(const zend_string* value, std::string_view name)
+{
+    auto hex_string = cb_string_new(value);
+
+    if(hex_string.empty()) {
+        return { { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("unexpected empty string for {}", name) }, {} };
+    }
+
+    try {
+        std::size_t pos;
+        auto result = parse_integer<Integer>(hex_string, &pos, 16);
+        if (result < std::numeric_limits<Integer>::min() || result > std::numeric_limits<Integer>::max()) {
+            return { { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("number out of range for {}", name) }, {} };
+        }
+        if (pos != hex_string.length()) {
+            return { { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("trailing garbage in {}", name) }, {} };
+        }
+        return {{}, result};
+    } catch (const std::invalid_argument& e) {
+        return { { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("invalid hex number for {}", name) }, {} };
+    } catch (const std::out_of_range& e) {
+        return { { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("number out of range for {}", name) }, {} };
+    }
+}
+
 template<typename Integer>
 static std::pair<core_error_info, std::optional<Integer>>
 cb_get_integer(const zval* options, std::string_view name)
@@ -87,6 +126,8 @@ cb_get_integer(const zval* options, std::string_view name)
             return {};
         case IS_LONG:
             break;
+        case IS_STRING:
+            return cb_get_integer_from_hex<Integer>(Z_STR_P(value), name);
         default:
             return {
                 { errc::common::invalid_argument, ERROR_LOCATION, fmt::format("expected {} to be a integer value in the options", name) },

tests/KeyValueScanTest.php

@@ -43,6 +43,7 @@ public function setUp(): void
     {
         parent::setUp();
         $this->skipIfProtostellar();
+        $this->skipIfUnsupported($this->version()->supportsCollections());
 
         $this->collection = $this->defaultCollection();
         for ($i = 0; $i < 100; $i++) {

tests/SearchTest.php

@@ -20,11 +20,13 @@
 
 use Couchbase\BooleanSearchQuery;
 use Couchbase\ClusterInterface;
+use Couchbase\CollectionInterface;
 use Couchbase\ConjunctionSearchQuery;
 use Couchbase\DateRangeSearchFacet;
 use Couchbase\DateRangeSearchQuery;
 use Couchbase\DisjunctionSearchQuery;
 use Couchbase\DocIdSearchQuery;
+use Couchbase\DurabilityLevel;
 use Couchbase\Exception\FeatureNotAvailableException;
 use Couchbase\Exception\IndexNotFoundException;
 use Couchbase\GeoBoundingBoxSearchQuery;
@@ -52,6 +54,7 @@
 use Couchbase\TermRangeSearchQuery;
 use Couchbase\TermSearchFacet;
 use Couchbase\TermSearchQuery;
+use Couchbase\UpsertOptions;
 use Couchbase\VectorQuery;
 use Couchbase\VectorQueryCombination;
 use Couchbase\VectorSearch;
@@ -63,34 +66,59 @@
 class SearchTest extends Helpers\CouchbaseTestCase
 {
     private ClusterInterface $cluster;
+    private CollectionInterface $collection;
     private SearchIndexManager $indexManager;
 
+    /**
+     * @return number of the documents in dataset
+     */
+    public function loadDataset(): int
+    {
+        $dataset = json_decode(file_get_contents(__DIR__ . "/beer-data.json"), true);
+
+        $options = UpsertOptions::build()->durabilityLevel(DurabilityLevel::MAJORITY_AND_PERSIST_TO_ACTIVE);
+        foreach ($dataset as $id => $document) {
+            $this->collection->upsert($id, $document, $options);
+        }
+
+        return count($dataset);
+    }
+
+    public function createSearchIndex(int $datasetSize): void
+    {
+        fprintf(STDERR, "Create 'beer-search' to index %d docs\n", $datasetSize);
+        $indexDump = json_decode(file_get_contents(__DIR__ . "/beer-search.json"), true);
+        $index = SearchIndex::build("beer-search", self::env()->bucketName());
+        $index->setParams($indexDump["params"]);
+        $this->indexManager->upsertIndex($index);
+
+        $start = time();
+        while (true) {
+            try {
+                $indexedDocuments = $this->indexManager->getIndexedDocumentsCount("beer-search");
+                fprintf(STDERR, "%ds, Indexing 'beer-search': %d docs\n", time() - $start, $indexedDocuments);
+                if ($indexedDocuments >= $datasetSize) {
+                    break;
+                }
+                sleep(5);
+            } catch (\Couchbase\Exception\IndexNotReadyException $ex) {
+            }
+        }
+    }
+
     public function setUp(): void
     {
         parent::setUp();
 
         $this->cluster = $this->connectCluster();
+        $this->collection = $this->openBucket(self::env()->bucketName())->defaultCollection();
 
         if (self::env()->useCouchbase()) {
             $this->indexManager = $this->cluster->searchIndexes();
             try {
                 $this->indexManager->getIndex("beer-search");
             } catch (IndexNotFoundException $ex) {
-                $indexDump = json_decode(file_get_contents(__DIR__ . "/beer-search.json"), true);
-                $index = SearchIndex::build("beer-search", "beer-sample");
-                $index->setParams($indexDump["params"]);
-                $this->indexManager->upsertIndex($index);
-            }
-            while (true) {
-                try {
-                    $indexedDocuments = $this->indexManager->getIndexedDocumentsCount("beer-search");
-                    fprintf(STDERR, "Indexing 'beer-search': %d docs\n", $indexedDocuments);
-                    if ($indexedDocuments > 7000) {
-                        break;
-                    }
-                    sleep(3);
-                } catch (\Couchbase\Exception\IndexNotReadyException $ex) {
-                }
+                $this->createSearchIndex($this->loadDataset());
             }
         }
     }
@@ -159,6 +187,7 @@ public function testSearchWithNoHits()
         $this->assertEquals(0, $result->metaData()->totalHits());
     }
 
+
     public function testSearchWithConsistency()
     {
         $this->skipIfCaves();
@@ -173,8 +202,7 @@ public function testSearchWithConsistency()
         $this->assertEmpty($result->rows());
         $this->assertEquals(0, $result->metaData()->totalHits());
 
-        $collection = $this->cluster->bucket('beer-sample')->defaultCollection();
-        $result = $collection->upsert($id, ["type" => "beer", "name" => $id]);
+        $result = $this->collection->upsert($id, ["type" => "beer", "name" => $id]);
         $mutationState = new MutationState();
         $mutationState->add($result);
 
@@ -358,7 +386,7 @@ public function testCompoundSearchQueries()
         $disjunctionQuery = new DisjunctionSearchQuery([$nameQuery, $descriptionQuery]);
         $options = SearchOptions::build()->fields(["type", "name", "description"]);
         $result = $this->cluster->searchQuery("beer-search", $disjunctionQuery, $options);
-        $this->assertGreaterThan(1000, $result->metaData()->totalHits());
+        $this->assertGreaterThan(20, $result->metaData()->totalHits());
         $this->assertNotEmpty($result->rows());
         $this->assertMatchesRegularExpression('/green/i', $result->rows()[0]['fields']['name']);
         $this->assertDoesNotMatchRegularExpression('/hop/i', $result->rows()[0]['fields']['name']);
@@ -434,18 +462,18 @@ public function testSearchWithFacets()
         $this->assertNotNull($result->facets()['foo']);
         $this->assertEquals('name', $result->facets()['foo']->field());
         $this->assertEquals('ale', $result->facets()['foo']->terms()[0]->term());
-        $this->assertGreaterThan(1000, $result->facets()['foo']->terms()[0]->count());
+        $this->assertGreaterThan(10, $result->facets()['foo']->terms()[0]->count());
 
         $this->assertNotNull($result->facets()['bar']);
         $this->assertEquals('updated', $result->facets()['bar']->field());
         $this->assertEquals('old', $result->facets()['bar']->dateRanges()[0]->name());
-        $this->assertGreaterThan(5000, $result->facets()['bar']->dateRanges()[0]->count());
+        $this->assertGreaterThan(30, $result->facets()['bar']->dateRanges()[0]->count());
 
         $this->assertNotNull($result->facets()['baz']);
         $this->assertEquals('abv', $result->facets()['baz']->field());
         $this->assertEquals('light', $result->facets()['baz']->numericRanges()[0]->name());
         $this->assertGreaterThan(0, $result->facets()['baz']->numericRanges()[0]->max());
-        $this->assertGreaterThan(100, $result->facets()['baz']->numericRanges()[0]->count());
+        $this->assertGreaterThan(15, $result->facets()['baz']->numericRanges()[0]->count());
     }
 
     public function testNullInNumericRangeFacet()