From 2af0d42c088c4b244329791fe87f75f1f1d0030c Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Wed, 2 Oct 2024 09:57:20 -0700 Subject: [PATCH] Fix tests --- .github/workflows/tests.yml | 14 +-- bin/test.rb | 5 ++ tests/Helpers/ConsistencyUtils.php | 130 +++++++++++++++++++++------- tests/Helpers/CouchbaseTestCase.php | 9 +- tests/KeyValueGetReplicaTest.php | 4 +- tests/QueryIndexManagerTest.php | 101 ++++++++++++++------- 6 files changed, 195 insertions(+), 68 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c5a6c03..b10ef5e3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -316,8 +316,8 @@ jobs: - nts - zts server: - - 7.6.0 - - 7.2.4 + - 7.6.3 + - 7.2.5 - 7.1.6 - 7.0.5 steps: @@ -331,7 +331,7 @@ jobs: - name: Install cbdinocluster run: | mkdir -p "$HOME/bin" - curl -L -o "$HOME/bin/cbdinocluster" https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.35/cbdinocluster-linux + curl -L -o "$HOME/bin/cbdinocluster" https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.52/cbdinocluster-linux-amd64 chmod a+x "$HOME/bin/cbdinocluster" echo "$HOME/bin" >> $GITHUB_PATH - name: Initialize cbdinocluster @@ -341,7 +341,11 @@ jobs: env: CLUSTERCONFIG: | nodes: - - count: 2 + - count: 1 + version: ${{ matrix.server }} + services: + - kv + - count: 1 version: ${{ matrix.server }} services: - kv @@ -358,7 +362,7 @@ jobs: run: | CLUSTER_ID=$(cbdinocluster -v allocate --def="${CLUSTERCONFIG}") CONNECTION_STRING=$(cbdinocluster -v connstr "${CLUSTER_ID}") - cbdinocluster -v buckets add ${CLUSTER_ID} default --ram-quota-mb=100 --flush-enabled=true + cbdinocluster -v buckets add ${CLUSTER_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=1 cbdinocluster -v buckets load-sample ${CLUSTER_ID} travel-sample echo "CLUSTER_ID=${CLUSTER_ID}" >> "$GITHUB_ENV" echo "TEST_CONNECTION_STRING=${CONNECTION_STRING}?dump_configuration=true" >> "$GITHUB_ENV" diff --git a/bin/test.rb b/bin/test.rb index 3a382e35..bcfd382e 100755 --- a/bin/test.rb +++ b/bin/test.rb @@ -68,6 +68,11 @@ def capture(*args) output end +if ENV["CB_VALGRIND"] + ENV["USE_ZEND_ALLOC"] = 0 + ENV["ZEND_DONT_UNLOAD_MODULES"] = 1 +end + project_root = File.expand_path(File.join(__dir__, "..")) build_root = File.join(project_root, "build") diff --git a/tests/Helpers/ConsistencyUtils.php b/tests/Helpers/ConsistencyUtils.php index d12a4106..0e43c743 100644 --- a/tests/Helpers/ConsistencyUtils.php +++ b/tests/Helpers/ConsistencyUtils.php @@ -4,8 +4,13 @@ use Exception; -const MANAGEMENT_PORT = 8091; -const VIEWS_PORT = 8092; +const PORTS = [ + 'mgmt' => 8091, + 'views' => 8092, + 'n1ql' => 8093, + 'fts' => 8094, +]; + class ConsistencyUtils { private string $hostname; @@ -39,7 +44,7 @@ public function waitUntilUserPresent(string $userName, string $domain = 'local') [$this, 'resourceIsPresent'], "User $userName is not present on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -54,7 +59,7 @@ public function waitUntilUserDropped(string $userName, string $domain = 'local') [$this, 'resourceIsDropped'], "User $userName is not dropped on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -69,7 +74,7 @@ public function waitUntilGroupPresent(string $groupName): void [$this, 'resourceIsPresent'], "Group $groupName is not present on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -81,7 +86,7 @@ public function waitUntilGroupDropped(string $groupName): void [$this, 'resourceIsDropped'], "Group $groupName is not dropped on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -93,7 +98,7 @@ public function waitUntilBucketPresent(string $bucketName): void [$this, 'resourceIsPresent'], "Bucket $bucketName is not present on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -105,7 +110,7 @@ public function waitUntilBucketDropped(string $bucketName): void [$this, 'resourceIsDropped'], "Bucket $bucketName is not dropped on all nodes", true, - MANAGEMENT_PORT + 'mgmt' ); } @@ -117,7 +122,7 @@ public function waitUntilDesignDocumentPresent(string $bucketName, string $desig [$this, 'resourceIsPresent'], "Design document $designDocumentName on bucket $bucketName is not present on all nodes", true, - VIEWS_PORT + 'views' ); } @@ -129,7 +134,7 @@ public function waitUntilDesignDocumentDropped(string $bucketName, string $desig [$this, 'resourceIsDropped'], "Design document $designDocumentName on bucket $bucketName is not dropped on all nodes", true, - VIEWS_PORT + 'views' ); } @@ -141,7 +146,7 @@ public function waitUntilViewPresent(string $bucketName, string $designDocumentN [$this, 'resourceIsPresent'], "View $viewName on design document $designDocumentName on bucket $bucketName is not present on all nodes", true, - VIEWS_PORT + 'views' ); } @@ -153,7 +158,7 @@ public function waitUntilViewDropped(string $bucketName, string $designDocumentN [$this, 'resourceIsDropped'], "View $viewName on design document $designDocumentName on bucket $bucketName is not dropped on all nodes", true, - VIEWS_PORT + 'views' ); } @@ -174,7 +179,7 @@ public function waitUntilScopePresent(string $bucketName, string $scopeName): vo $scopePresent, "Scope $scopeName on bucket $bucketName is not present on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } @@ -195,7 +200,7 @@ public function waitUntilScopeDropped(string $bucketName, string $scopeName): vo $scopeDropped, "Scope $scopeName on bucket $bucketName is not dropped on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } @@ -220,7 +225,7 @@ public function waitUntilCollectionPresent(string $bucketName, string $scopeName $collectionPresent, "Collection $collectionName on scope $scopeName on bucket $bucketName is not present on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } @@ -245,7 +250,7 @@ public function waitUntilCollectionDropped(string $bucketName, string $scopeName $collectionDropped, "Collection $collectionName on scope $scopeName on bucket $bucketName is not dropped on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } @@ -258,7 +263,7 @@ public function waitUntilBucketUpdated(string $bucketName, callable $predicate, $predicate, $errorMsg ?? "Bucket $bucketName has not been updated on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } @@ -271,28 +276,29 @@ public function waitUntilCollectionUpdated(string $bucketName, string $scopeName $predicate, $errorMsg ?? "Collection $collectionName has not been updated on all nodes", false, - MANAGEMENT_PORT + 'mgmt' ); } /** * @throws Exception */ - public function waitUntilAllNodesMatchPredicate(string $path, callable $predicate, string $errorMsg, bool $onlyStatusCode, int $port): void + public function waitUntilAllNodesMatchPredicate(string $path, callable $predicate, string $errorMsg, bool $onlyStatusCode, string $service, array $request = []): void { try { - $deadline = $this->currentTimeMillis() + 2000; + $deadline = $this->currentTimeMillis() + 200_000; while ($this->currentTimeMillis() < $deadline) { $predicateMatched = $this->allNodesMatchPredicate( $path, $predicate, $onlyStatusCode, - $port + $service, + $request ); if ($predicateMatched) { return; } - usleep(10_000); + usleep(100_000); } throw new Exception("Timed out waiting for nodes to match predicate"); } catch (Exception $e) { @@ -332,11 +338,70 @@ public function waitForConfig(bool $isMock): void /** * @throws Exception */ - private function allNodesMatchPredicate(string $path, callable $predicate, bool $onlyStatusCode, int $port): bool + public function waitUntilQueryIndexReady(string $bucketName, string $indexName, bool $isPrimary): void { - foreach ($this->nodes as $key => $value) { - $url = "http://" . $key . ":" . $port . $path; - $response = $this->runHttpRequest($url, $onlyStatusCode); + fprintf( + STDERR, + "waiting until query index \"%s\" (is_primary=%s) of bucket \"%s\" present on all nodes\n", + $indexName, + $isPrimary ? "true" : "false", + $bucketName + ); + $indexPresent = function ($response) use ($bucketName, $indexName, $isPrimary) { + foreach ($response->results as $result) { + $index = $result->indexes; + + if ( + $index->state == "online" && + $index->keyspace_id == $bucketName && + $index->name == $indexName && + (isset($index->is_primary) && $index->is_primary) == $isPrimary + ) { + return true; + } + } + return false; + }; + + + $this->waitUntilAllNodesMatchPredicate( + "/query/service", + $indexPresent, + sprintf("index %s (primary=%s) is not present on all nodes\n", $indexName, $isPrimary ? "true" : "false"), + false, + "n1ql", + [ + 'method' => 'POST', + 'header' => ['Content-type: application/json'], + 'content' => json_encode( + [ + 'statement' => 'SELECT * FROM system:indexes' + ] + ) + ] + ); + } + + /** + * @throws Exception + */ + private function allNodesMatchPredicate(string $path, callable $predicate, bool $onlyStatusCode, string $service, array $request): bool + { + foreach ($this->nodes as $hostname => $services) { + if ($service != "mgmt" && $service != "views") { + $found = false; + foreach ($services as $name) { + if ($service == $name) { + $found = true; + break; + } + } + if (!$found) { + continue; + } + } + $url = "http://" . $hostname . ":" . PORTS[$service] . $path; + $response = $this->runHttpRequest($url, $request, $onlyStatusCode); $predicateMatched = $predicate($response); if (!$predicateMatched) { return false; @@ -348,13 +413,18 @@ private function allNodesMatchPredicate(string $path, callable $predicate, bool /** * @throws Exception */ - private function runHttpRequest(string $url, bool $onlyStatusCode) + private function runHttpRequest(string $url, array $request, bool $onlyStatusCode) { + $headers = $request['header'] ?? []; + unset($request['header']); $opts = array('http' => - array( + array_merge( + [ 'method' => 'GET', - 'header' => $this->basicAuthString, + 'header' => array_merge([$this->basicAuthString], $headers), 'ignore_errors' => true, + ], + $request ) ); $context = stream_context_create($opts); @@ -363,7 +433,7 @@ private function runHttpRequest(string $url, bool $onlyStatusCode) if ($onlyStatusCode) { return $statusCode; } elseif ($statusCode != 200) { - throw new Exception("Non 200 status code from response"); + throw new Exception(sprintf("Non 200 status code from response (%d).\n%s", $statusCode, $response)); } return json_decode($response); } diff --git a/tests/Helpers/CouchbaseTestCase.php b/tests/Helpers/CouchbaseTestCase.php index aad05634..2472ec81 100644 --- a/tests/Helpers/CouchbaseTestCase.php +++ b/tests/Helpers/CouchbaseTestCase.php @@ -109,6 +109,9 @@ public function openBucket(string $name = null): BucketInterface public function defaultCollection(string $bucketName = null): CollectionInterface { + if ($bucketName == null) { + $bucketName = self::env()->bucketName(); + } return $this->openBucket($bucketName)->defaultCollection(); } @@ -290,7 +293,11 @@ protected function assertError($type, $code, $ex) protected function assertErrorType($type, $ex) { - $this->assertInstanceOf($type, $ex); + $this->assertInstanceOf( + $type, + $ex, + sprintf("Exception: %s, Message: %s", get_class($ex), $ex->getMessage()) + ); } protected function assertErrorMessage($msg, $ex) diff --git a/tests/KeyValueGetReplicaTest.php b/tests/KeyValueGetReplicaTest.php index b51e0e30..ed885276 100644 --- a/tests/KeyValueGetReplicaTest.php +++ b/tests/KeyValueGetReplicaTest.php @@ -50,9 +50,7 @@ public function testGetAllReplicasReturnCorrectValue() $res = $collection->upsert($id, ["answer" => 42], $opts); $cas = $res->cas(); $this->assertNotNull($cas); - if (self::env()->useCaves()) { - sleep(1); - } + sleep(1); $results = $collection->getAllReplicas($id); $this->assertGreaterThanOrEqual(1, count($results)); $seenActiveVersion = false; diff --git a/tests/QueryIndexManagerTest.php b/tests/QueryIndexManagerTest.php index f7c3f0c8..97b306d8 100644 --- a/tests/QueryIndexManagerTest.php +++ b/tests/QueryIndexManagerTest.php @@ -57,14 +57,28 @@ public function testQueryIndexesCrud() $this->bucketManager->createBucket($settings); $this->consistencyUtil()->waitUntilBucketPresent($bucketName); - $deadline = time() + 5; /* 5 seconds from now */ + $deadline = time() + 20; /* 20 seconds from now */ while (true) { try { - $manager->createPrimaryIndex($bucketName, CreateQueryPrimaryIndexOptions::build()->ignoreIfExists(true)); + try { + $manager->createPrimaryIndex( + $bucketName, + CreateQueryPrimaryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(true) + ); + } catch (\Couchbase\Exception\InternalServerException $ex) { + if (preg_match('/will be retried/', $ex->getMessage())) { + fprintf(STDERR, "Ignoring transient error during primary index creation for '%s': %s, %s\n", $bucketName, $ex->getMessage(), var_export($ex->getContext(), true)); + } else { + throw $ex; + } + } + $this->consistencyUtil()->waitUntilQueryIndexReady($bucketName, "#primary", true); break; } catch (CouchbaseException $ex) { - printf("Error during primary index creation for '%s': %s, %s", $bucketName, $ex->getMessage(), var_export($ex->getContext(), true)); + fprintf(STDERR, "Ignoring error during primary index creation for '%s': %s, %s\n", $bucketName, $ex->getMessage(), var_export($ex->getContext(), true)); if (time() > $deadline) { $this->fail("timed out waiting for create index to succeed"); } @@ -79,38 +93,46 @@ function () use ($manager, $bucketName) { IndexExistsException::class ); + $indexName = $this->uniqueId('testIndex'); $manager->createIndex( $bucketName, - "testIndex", + $indexName, ["field"], - CreateQueryIndexOptions::build()->ignoreIfExists(true) + CreateQueryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(true) ); + $this->consistencyUtil()->waitUntilQueryIndexReady($bucketName, $indexName, false); $this->wrapException( - function () use ($manager, $bucketName) { + function () use ($manager, $bucketName, $indexName) { $manager->createIndex( $bucketName, - "testIndex", + $indexName, ["field"], - CreateQueryIndexOptions::build()->ignoreIfExists(false) + CreateQueryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(false) ); }, IndexExistsException::class ); // We create this first to give it a chance to be created by the time we need it. + $deferredIndexName = $this->uniqueId('testIndexDeferred'); $manager->createIndex( $bucketName, - "testIndexDeferred", + $deferredIndexName, ["field"], CreateQueryIndexOptions::build() + ->timeout(200_000) ->ignoreIfExists(true) ->deferred(true) ); $manager->buildDeferredIndexes($bucketName); - $manager->watchIndexes($bucketName, ["testIndexDeferred"], 60_000); + $manager->watchIndexes($bucketName, [$deferredIndexName], 60_000); $indexes = $manager->getAllIndexes($bucketName); $this->assertGreaterThanOrEqual(3, count($indexes)); @@ -120,13 +142,13 @@ function () use ($manager, $bucketName) { * @var QueryIndex $entry */ foreach ($indexes as $entry) { - if ($entry->name() == "testIndex") { + if ($entry->name() == $indexName) { $index = $entry; } } $this->assertNotNull($index); - $this->assertEquals("testIndex", $index->name()); + $this->assertEquals($indexName, $index->name()); $this->assertFalse($index->isPrimary()); $this->assertEquals(QueryIndexType::GSI, $index->type()); $this->assertEquals("online", $index->state()); @@ -137,11 +159,11 @@ function () use ($manager, $bucketName) { $this->assertNull($index->condition()); $this->assertNull($index->partition()); - $manager->dropIndex($bucketName, "testIndex"); + $manager->dropIndex($bucketName, $indexName); $this->wrapException( - function () use ($manager, $bucketName) { - $manager->dropIndex($bucketName, "testIndex"); + function () use ($manager, $bucketName, $indexName) { + $manager->dropIndex($bucketName, $indexName); }, IndexNotFoundException::class ); @@ -166,10 +188,23 @@ public function testCollectionQueryIndexesCrud() while (true) { try { - $manager->createPrimaryIndex(CreateQueryPrimaryIndexOptions::build()->ignoreIfExists(true)); + try { + $manager->createPrimaryIndex( + CreateQueryPrimaryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(true) + ); + } catch (\Couchbase\Exception\InternalServerException $ex) { + if (preg_match('/will be retried/', $ex->getMessage())) { + fprintf(STDERR, "Ignoring transient error during primary index creation for '%s': %s, %s\n", $bucketName, $ex->getMessage(), var_export($ex->getContext(), true)); + } else { + throw $ex; + } + } + $this->consistencyUtil()->waitUntilQueryIndexReady($this->env()->bucketName(), "#primary", true); break; } catch (CouchbaseException $ex) { - printf("Error during primary index creation: %s, %s", $ex->getMessage(), var_export($ex->getContext(), true)); + fprintf(STDERR, "Ignoring error during primary index creation: %s, %s\n", $ex->getMessage(), var_export($ex->getContext(), true)); if (time() > $deadline) { $this->assertFalse("timed out waiting for create index to succeed"); } @@ -184,35 +219,43 @@ function () use ($manager) { IndexExistsException::class ); + $indexName = $this->uniqueId('testIndex'); $manager->createIndex( - "testIndex", + $indexName, ["field"], - CreateQueryIndexOptions::build()->ignoreIfExists(true) + CreateQueryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(true) ); + $this->consistencyUtil()->waitUntilQueryIndexReady($this->env()->bucketName(), $indexName, false); $this->wrapException( - function () use ($manager) { + function () use ($manager, $indexName) { $manager->createIndex( - "testIndex", + $indexName, ["field"], - CreateQueryIndexOptions::build()->ignoreIfExists(false) + CreateQueryIndexOptions::build() + ->timeout(200_000) + ->ignoreIfExists(false) ); }, IndexExistsException::class ); // We create this first to give it a chance to be created by the time we need it. + $deferredIndexName = $this->uniqueId('testIndexDeferred'); $manager->createIndex( - "testIndexDeferred", + $deferredIndexName, ["field"], CreateQueryIndexOptions::build() + ->timeout(200_000) ->ignoreIfExists(true) ->deferred(true) ); $manager->buildDeferredIndexes(); - $manager->watchIndexes(["testIndexDeferred"], 60_000); + $manager->watchIndexes([$deferredIndexName], 60_000); $indexes = $manager->getAllIndexes(); $this->assertGreaterThanOrEqual(3, count($indexes)); @@ -222,13 +265,13 @@ function () use ($manager) { * @var QueryIndex $entry */ foreach ($indexes as $entry) { - if ($entry->name() == "testIndex") { + if ($entry->name() == $indexName) { $index = $entry; } } $this->assertNotNull($index); - $this->assertEquals("testIndex", $index->name()); + $this->assertEquals($indexName, $index->name()); $this->assertFalse($index->isPrimary()); $this->assertEquals(QueryIndexType::GSI, $index->type()); $this->assertEquals("online", $index->state()); @@ -238,11 +281,11 @@ function () use ($manager) { $this->assertNull($index->condition()); $this->assertNull($index->partition()); - $manager->dropIndex("testIndex"); + $manager->dropIndex($indexName); $this->wrapException( - function () use ($manager) { - $manager->dropIndex("testIndex"); + function () use ($manager, $indexName) { + $manager->dropIndex($indexName); }, IndexNotFoundException::class );