From 79df801b37c2a9db662aafa302c20eeebca8ce42 Mon Sep 17 00:00:00 2001 From: Romain Ruaud Date: Fri, 25 Oct 2024 16:43:51 +0200 Subject: [PATCH 1/2] Implementing AWS Opensearch serverless connection --- .../Client/ClientConfigurationInterface.php | 42 ++++++++++ .../Client/Client.php | 26 +++++++ .../Client/ClientBuilder.php | 76 ++++++++++++++++++- .../Client/ClientConfiguration.php | 45 +++++++++++ .../Index/IndexOperation.php | 7 +- .../etc/adminhtml/system.xml | 54 ++++++++++--- src/module-elasticsuite-core/etc/config.xml | 3 + src/module-elasticsuite-core/i18n/en_US.csv | 8 ++ src/module-elasticsuite-core/i18n/fr_FR.csv | 8 ++ 9 files changed, 254 insertions(+), 15 deletions(-) diff --git a/src/module-elasticsuite-core/Api/Client/ClientConfigurationInterface.php b/src/module-elasticsuite-core/Api/Client/ClientConfigurationInterface.php index f182f43b2..9f2091626 100644 --- a/src/module-elasticsuite-core/Api/Client/ClientConfigurationInterface.php +++ b/src/module-elasticsuite-core/Api/Client/ClientConfigurationInterface.php @@ -93,6 +93,48 @@ public function getMaxParallelHandles(); */ public function getMaxRetries(); + /** + * Check if SSL verification is enabled. + * + * @return bool + */ + public function isVerifyEnabled(); + + /** + * Check if AWS Sig4 verification is enabled. + * + * @return bool + */ + public function isAwsSig4Enabled(); + + /** + * Fetch the AWS Service to be used + * + * @return string + */ + public function getAwsService(); + + /** + * Get the AWS Region + * + * @return string + */ + public function getAwsRegion(); + + /** + * Get the AWS Sig4 Key + * + * @return string + */ + public function getAwsSig4Key(); + + /** + * Get the AWS Sig4 Secret + * + * @return string + */ + public function getAwsSig4Secret(); + /** * Client config options. * diff --git a/src/module-elasticsuite-core/Client/Client.php b/src/module-elasticsuite-core/Client/Client.php index f93459fef..a5d12f979 100644 --- a/src/module-elasticsuite-core/Client/Client.php +++ b/src/module-elasticsuite-core/Client/Client.php @@ -72,6 +72,10 @@ public function __construct( */ public function info() { + if ($this->isAoss()) { + return ['version' => ['number' => '2 ?', 'distribution' => 'AWS OpenSearch Service Serverless']]; + } + return $this->getEsClient()->info(); } @@ -96,6 +100,10 @@ public function cluster() */ public function ping() { + if ($this->isAoss()) { + return true; + } + return $this->getEsClient()->ping(); } @@ -128,6 +136,12 @@ public function indexExists($indexName) */ public function putIndexSettings($indexName, $indexSettings) { + if ($this->isAoss()) { + unset($indexSettings['number_of_replicas']); + unset($indexSettings['refresh_interval']); + unset($indexSettings['translog.durability']); + } + $this->getEsClient()->indices()->putSettings(['index' => $indexName, 'body' => $indexSettings]); } @@ -320,4 +334,16 @@ private function getEsClient(): \OpenSearch\Client return $this->esClient; } + + /** + * Check if using the AWS OpenSerch Service Serverless, because some operations are not permitted or behaving differently : + * - setting number of replicas per index is not allowed + * - client info is empty + * + * @return bool + */ + private function isAoss(): bool + { + return ($this->clientConfiguration->isAwsSig4Enabled() && $this->clientConfiguration->getAwsService() === 'aoss'); + } } diff --git a/src/module-elasticsuite-core/Client/ClientBuilder.php b/src/module-elasticsuite-core/Client/ClientBuilder.php index 27e6363b0..99b780155 100644 --- a/src/module-elasticsuite-core/Client/ClientBuilder.php +++ b/src/module-elasticsuite-core/Client/ClientBuilder.php @@ -14,6 +14,8 @@ namespace Smile\ElasticsuiteCore\Client; +use Aws\Credentials\CredentialProvider; +use Aws\Credentials\Credentials; use OpenSearch\ConnectionPool\Selectors\StickyRoundRobinSelector; /** @@ -48,6 +50,9 @@ class ClientBuilder 'max_parallel_handles' => 100, // As per default Elasticsearch Handler configuration. 'max_retries' => 2, 'verify' => true, + 'enable_aws_sig4' => false, + 'aws_region' => 'eu-central-1', + 'aws_service' => 'es', ]; /** @@ -60,6 +65,11 @@ class ClientBuilder */ private $namespaceBuilders = []; + /** + * @var callable|null + */ + private $sig4CredentialsProvider = null; + /** * Constructor. * @@ -119,10 +129,26 @@ public function build(array $options = []): \OpenSearch\Client $clientBuilder->setTracer($this->logger); } + $handlerParams = []; if ($options['max_parallel_handles']) { $handlerParams = ['max_handles' => (int) $options['max_parallel_handles']]; - $handler = \OpenSearch\ClientBuilder::defaultHandler($handlerParams); - $clientBuilder->setHandler($handler); + } + $handler = \OpenSearch\ClientBuilder::defaultHandler($handlerParams); + $clientBuilder->setHandler($handler); + + if ($this->isAwsSig4Enabled($options)) { + // The ->setSigV4* methods are only available starting from opensearch/opensearch-php 2.0.1. + try { + $clientBuilder->setSigV4Region($options['aws_region']) + ->setSigV4Service($options['aws_service']) + ->setSigV4CredentialProvider($this->getSig4CredentialsProvider($options)); + } catch (\Exception $exception) { + $this->logger->error($exception->getMessage()); + $message = "Cannot configure SigV4 on OpenSearch client. " . + "This can be due to an outdated (<2.0.1) version of the opensearch-project/opensearch-php package."; + + throw new \Exception($message . " Error was : " . $exception->getMessage()); + } } $connectionParams = $this->getConnectionParams($options); @@ -168,11 +194,13 @@ private function getHosts(array $options): array foreach ($options['servers'] as $host) { if (!empty($host)) { - [$hostname, $port] = array_pad(explode(':', trim($host), 2), 2, 9200); + [$hostname, $port] = array_pad(explode(':', trim(preg_replace('/^https?:\/\//', '', $host)), 2), 2, 9200); + preg_match('/^(https?):\/\//', $host, $matches); + $currentHostConfig = [ 'host' => $hostname, 'port' => $port, - 'scheme' => isset($options['enable_https_mode']) ? 'https' : $options['scheme'] ?? 'http', + 'scheme' => isset($options['enable_https_mode']) ? 'https' : $matches[1] ?? 'http', ]; if ($options['enable_http_auth']) { @@ -180,6 +208,11 @@ private function getHosts(array $options): array $currentHostConfig['pass'] = $options['http_auth_pwd']; } + if ($this->isAwsSig4Enabled($options) === true) { + $currentHostConfig['scheme'] = 'https'; // AOSS is alway behind HTTPS url. + $currentHostConfig['host'] = $host; // Use raw host string for AOSS. + } + $hosts[] = $currentHostConfig; } } @@ -210,4 +243,39 @@ private function getConnectionParams(array $options): array ], ] : []; } + + /** + * Check if AWS Signature v4 should be used for signing the requests + * + * @param array $options The Options + * + * @return bool + */ + private function isAwsSig4Enabled($options) + { + return (((bool) $options['enable_aws_sig4']) === true); + } + + /** + * Get CredentialProvider Instance to be used with Sig4 authentication. + * + * @param array $options The client options + * + * @return callable + */ + private function getSig4CredentialsProvider($options) + { + if (null === $this->sig4CredentialsProvider) { + $this->sig4CredentialsProvider = CredentialProvider::fromCredentials( + new Credentials( + $options['aws_key'], + $options['aws_secret'], + isset($options['aws_token']) ? $options['aws_token'] : null, + isset($options['aws_expires']) ? $options['aws_expires'] : null + ) + ); + } + + return $this->sig4CredentialsProvider; + } } diff --git a/src/module-elasticsuite-core/Client/ClientConfiguration.php b/src/module-elasticsuite-core/Client/ClientConfiguration.php index 50c332df5..9c52b99ec 100644 --- a/src/module-elasticsuite-core/Client/ClientConfiguration.php +++ b/src/module-elasticsuite-core/Client/ClientConfiguration.php @@ -138,6 +138,46 @@ public function isVerifyEnabled() return (bool) $this->getElasticsearchClientConfigParam('enable_certificate_validation'); } + /** + * @return bool + */ + public function isAwsSig4Enabled() + { + return (bool) $this->getElasticsearchClientConfigParam('enable_aws_sig4'); + } + + /** + * @return string + */ + public function getAwsService() + { + return (string) $this->getElasticsearchClientConfigParam('aws_service'); + } + + /** + * @return string + */ + public function getAwsRegion() + { + return (string) $this->getElasticsearchClientConfigParam('aws_region'); + } + + /** + * @return string + */ + public function getAwsSig4Key() + { + return (string) $this->getElasticsearchClientConfigParam('aws_key'); + } + + /** + * @return string + */ + public function getAwsSig4Secret() + { + return (string) $this->getElasticsearchClientConfigParam('aws_secret'); + } + /** * {@inheritDoc} */ @@ -154,6 +194,11 @@ public function getOptions() 'max_parallel_handles' => $this->getMaxParallelHandles(), 'max_retries' => $this->getMaxRetries(), 'verify' => $this->isVerifyEnabled(), + 'enable_aws_sig4' => $this->isAwsSig4Enabled(), + 'aws_service' => $this->getAwsService(), + 'aws_region' => $this->getAwsRegion(), + 'aws_key' => $this->getAwsSig4Key(), + 'aws_secret' => $this->getAwsSig4Secret(), ]; return $options; diff --git a/src/module-elasticsuite-core/Index/IndexOperation.php b/src/module-elasticsuite-core/Index/IndexOperation.php index 55fa1b12e..cd1300490 100644 --- a/src/module-elasticsuite-core/Index/IndexOperation.php +++ b/src/module-elasticsuite-core/Index/IndexOperation.php @@ -207,7 +207,12 @@ public function installIndex(\Smile\ElasticsuiteCore\Api\Index\IndexInterface $i $indexName = $index->getName(); $indexAlias = $this->indexSettings->getIndexAliasFromIdentifier($indexIdentifier, $store); - $this->client->forceMerge($indexName); + try { + $this->client->forceMerge($indexName); + } catch (\Exception $e) { + $this->logger->error($e->getMessage()); + } + $this->client->putIndexSettings($indexName, $this->indexSettings->getInstallIndexSettings($indexIdentifier)); $this->proceedIndexInstall($indexName, $indexAlias); diff --git a/src/module-elasticsuite-core/etc/adminhtml/system.xml b/src/module-elasticsuite-core/etc/adminhtml/system.xml index e5aeab176..2a8a785be 100644 --- a/src/module-elasticsuite-core/etc/adminhtml/system.xml +++ b/src/module-elasticsuite-core/etc/adminhtml/system.xml @@ -30,53 +30,87 @@ - + List of servers in [host]:[port] format separated by a comma (e.g. : "es-node1.fqdn:9200, es-node2.fqdn:9200") - + Select yes if you want to connect to your Elasticsearch server over HTTPS. Magento\Config\Model\Config\Source\Yesno - + Select no if you are using self-signed SSL certificate. Magento\Config\Model\Config\Source\Yesno - + Enable this option when your Elasticsearch server use basic HTTP authentication. Magento\Config\Model\Config\Source\Yesno - + Enable this option when you want to base64 encode the Authorization headers. (Open Distro requires this) Magento\Config\Model\Config\Source\Yesno + + 1 + - + 1 - + 1 - + + + AWS Signature V4.]]> + Magento\Config\Model\Config\Source\Yesno + + + + + 1 + + es for AWS OpenSearch Service, and aoss for AWS OpenSearch Service Serverless]]> + + + + + 1 + + Eg : eu-west-1 + + + + + 1 + + + + + + 1 + + + When enabled the module will produce logs through Magento logging system. Magento\Config\Model\Config\Source\Yesno - + In seconds. validate-number - + Maximum number of times to retry connection when there is a connection failure validate-number diff --git a/src/module-elasticsuite-core/etc/config.xml b/src/module-elasticsuite-core/etc/config.xml index cbebaa6ed..bd45897a0 100644 --- a/src/module-elasticsuite-core/etc/config.xml +++ b/src/module-elasticsuite-core/etc/config.xml @@ -26,6 +26,9 @@ 10 2 1 + 0 + aoss + eu-west-1 magento2 diff --git a/src/module-elasticsuite-core/i18n/en_US.csv b/src/module-elasticsuite-core/i18n/en_US.csv index 3c35ff94d..dfe2a65f9 100644 --- a/src/module-elasticsuite-core/i18n/en_US.csv +++ b/src/module-elasticsuite-core/i18n/en_US.csv @@ -126,3 +126,11 @@ Autocomplete,Autocomplete "Custom stemmer","Custom stemmer" "In the list ""[default]"" indicates the stemmer used by Elasticsuite by default for the language, while ""[recommended]"" indicates the stemmer is the one or one of those recommended for that language by Elasticsearch in its online documentation. Those can differ, hence this configuration parameter.","In the list ""[default]"" indicates the stemmer used by Elasticsuite by default for the language, while ""[recommended]"" indicates the stemmer is the one or one of those recommended for that language by Elasticsearch in its online documentation. Those can differ, hence this configuration parameter." "Please select a stemmer for the store","Please select a stemmer for the store" +"Enable AWS Signature V4","Enable AWS Signature V4" +"Enable this option when you are using AWS Opensearch Service (serverless or not) and you want to use AWS Signature V4.","Enable this option when you are using AWS Opensearch Service (serverless or not) and you want to use AWS Signature V4." +"AWS Service","AWS Service" +"es for AWS OpenSearch Service, and aoss for AWS OpenSearch Service Serverless","es for AWS OpenSearch Service, and aoss for AWS OpenSearch Service Serverless" +"AWS Region","AWS Region" +"Eg : eu-west-1","Eg : eu-west-1" +"AWS Signature V4 Key","AWS Signature V4 Key" +"AWS Signature V4 Secret","AWS Signature V4 Secret" diff --git a/src/module-elasticsuite-core/i18n/fr_FR.csv b/src/module-elasticsuite-core/i18n/fr_FR.csv index 8b2fe1b43..fdeeb49a1 100644 --- a/src/module-elasticsuite-core/i18n/fr_FR.csv +++ b/src/module-elasticsuite-core/i18n/fr_FR.csv @@ -126,3 +126,11 @@ General,Général "Custom stemmer","Stemmer spécifique" "In the list ""[default]"" indicates the stemmer used by Elasticsuite by default for the language, while ""[recommended]"" indicates the stemmer is the one or one of those recommended for that language by Elasticsearch in its online documentation. Those can differ, hence this configuration parameter.","Dans la liste, ""[default]"" indique le stemmer utilisé par défaut par Elasticsuite pour la langue, tandis que ""[recommended]"" indique que le stemmer est celui ou l'un de deux recommendés pour cette langue par Elasticsearch dans sa documentation en ligne. Ceux-ci peuvent différer, d'où l'existence de ce paramètre de configuration." "Please select a stemmer for the store","Veuillez sélectionner un stemmer pour le magasin" +"Enable AWS Signature V4","Activer AWS Signature V4" +"Enable this option when you are using AWS Opensearch Service (serverless or not) and you want to use AWS Signature V4.","Activez cette option si vous utilisez AWS Opensearch Service (serverless ou non) et que vous souhaitez utiliser AWS Signature V4." +"AWS Service","Service AWS" +"es for AWS OpenSearch Service, and aoss for AWS OpenSearch Service Serverless","es pour AWS OpenSearch Service, et aoss pour AWS OpenSearch Service Serverless" +"AWS Region","Région AWS" +"Eg : eu-west-1","Par ex : eu-west-1" +"AWS Signature V4 Key","Clé de la Signature AWS V4" +"AWS Signature V4 Secret","Secret de la Signature AWS V4" From d1b46b74c9419f3f79f6d6edea1dece22c6715a5 Mon Sep 17 00:00:00 2001 From: Romain Ruaud Date: Fri, 25 Oct 2024 16:49:11 +0200 Subject: [PATCH 2/2] Code cleaning --- src/module-elasticsuite-core/Client/ClientBuilder.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module-elasticsuite-core/Client/ClientBuilder.php b/src/module-elasticsuite-core/Client/ClientBuilder.php index 99b780155..942cb7f45 100644 --- a/src/module-elasticsuite-core/Client/ClientBuilder.php +++ b/src/module-elasticsuite-core/Client/ClientBuilder.php @@ -258,6 +258,7 @@ private function isAwsSig4Enabled($options) /** * Get CredentialProvider Instance to be used with Sig4 authentication. + * @SuppressWarnings(PHPMD.StaticAccess) * * @param array $options The client options *