From e556984656c9b07fbe2aeb38e6b6b144bdd55f17 Mon Sep 17 00:00:00 2001 From: Peter van der Wal Date: Mon, 27 May 2024 14:44:45 +0200 Subject: [PATCH 1/6] Correct PHP versions in README.md Readme still contained PHP versions that aren't supported anymore --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f92fcb..b3d8505 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Webgriffe ESB is designed to use a single binary which is used as the main entry Requirements ------------ -* PHP 7.2, 7.3 or 7.4 +* PHP 7.4 or 8.0 * Beanstalk * Elasticsearch 7.* From 1aba779ecea97c3f0d8827be351a7383d48ab0c0 Mon Sep 17 00:00:00 2001 From: Peter van der Wal Date: Mon, 27 May 2024 15:38:42 +0200 Subject: [PATCH 2/6] Add PHP 8.3 support --- .github/workflows/ci.yaml | 2 +- Dockerfile | 2 +- README.md | 2 +- composer.json | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 945df14..36becbc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - php: [7.4, 8.0] + php: [7.4, 8.0, 8.1, 8.2, 8.3] env: ESB_CONSOLE_PORT: 8080 diff --git a/Dockerfile b/Dockerfile index c3ab0fa..1183000 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PHP_VERSION=8.0 +ARG PHP_VERSION=8.3 FROM php:${PHP_VERSION}-cli-alpine diff --git a/README.md b/README.md index b3d8505..9425545 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Webgriffe ESB is designed to use a single binary which is used as the main entry Requirements ------------ -* PHP 7.4 or 8.0 +* PHP 7.4, 8.0, 8.1, 8.2, 8.3 * Beanstalk * Elasticsearch 7.* diff --git a/composer.json b/composer.json index 7e7334e..8d6307d 100644 --- a/composer.json +++ b/composer.json @@ -10,7 +10,7 @@ } ], "require": { - "php": "~7.4.0|~8.0.0", + "php": "~7.4.0|~8.0.0|~8.1.0|~8.2.0|~8.3.0", "ext-pcntl": "*", "ext-json": "*", "symfony/dependency-injection": "^4.3", @@ -33,7 +33,7 @@ "symfony/property-info": "^4.3", "doctrine/annotations": "^1.8", "ramsey/uuid": "^3.8", - "webgriffe/amp-elasticsearch": "^2.0", + "webgriffe/amp-elasticsearch": "^2.1", "pagerfanta/pagerfanta": "^2.4", "symfony/deprecation-contracts": "^2.1", "amphp/http-server-form-parser": "^1.1" From 67e5e0a195f9e2e7c422994454d3eedd5f2f0652 Mon Sep 17 00:00:00 2001 From: Peter van der Wal Date: Mon, 27 May 2024 15:42:32 +0200 Subject: [PATCH 3/6] Replace deprecated utf8_encode with mb_convert_encoding Error: Function utf8_encode() is deprecated in /srv/esb/vendor/monolog/monolog/src/Monolog/Utils.php on line 179 Resolved with: Implementing Monolog\Utils::detectAndCleanUtf8() from Monolog version 2.8 --- .github/workflows/ci.yaml | 2 +- composer-require-checker.json | 8 +++++++- composer.json | 3 +++ src/NonUtf8Cleaner.php | 29 ++++++++++++++++++++++++----- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 36becbc..b3ab3ef 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -68,7 +68,7 @@ jobs: run: composer install --no-interaction - name: Composer Require Checker - run: composer-require-checker + run: composer-require-checker --config-file=composer-require-checker.json - name: Run Easy Coding Standard run: vendor/bin/ecs check diff --git a/composer-require-checker.json b/composer-require-checker.json index 6d5bb51..7dfd157 100644 --- a/composer-require-checker.json +++ b/composer-require-checker.json @@ -1,3 +1,9 @@ { - "scan-files" : ["bin/esb"] + "scan-files" : ["bin/esb"], + "symbol-whitelist": [ + "null", "true", "false", "static", "self", "parent", + "array", "string", "int", "float", "bool", "iterable", "callable", "void", "object", "mixed", "never", + + "mb_convert_encoding" + ] } diff --git a/composer.json b/composer.json index 8d6307d..473f144 100644 --- a/composer.json +++ b/composer.json @@ -38,6 +38,9 @@ "symfony/deprecation-contracts": "^2.1", "amphp/http-server-form-parser": "^1.1" }, + "suggest": { + "ext-mbstring": "*" + }, "autoload": { "psr-4": { "Webgriffe\\Esb\\": ["src/"] diff --git a/src/NonUtf8Cleaner.php b/src/NonUtf8Cleaner.php index 88a9b1b..b1e296d 100644 --- a/src/NonUtf8Cleaner.php +++ b/src/NonUtf8Cleaner.php @@ -4,8 +4,6 @@ namespace Webgriffe\Esb; -use Monolog\Utils; - /** * @internal */ @@ -17,13 +15,34 @@ class NonUtf8Cleaner */ public static function clean(array $data): array { - array_walk_recursive($data, [Utils::class, 'detectAndCleanUtf8']); + array_walk_recursive($data, [__CLASS__, 'cleanString']); return $data; } public static function cleanString(string $data): string { - Utils::detectAndCleanUtf8($data); - return $data; + // Implementation borrowed from Monolog\Utils::detectAndCleanUtf8() which is no longer a public method as of version 2 + + if (preg_match('//u', $data)) { + return $data; + } + + $data = preg_replace_callback( + '/[\x80-\xFF]+/', + function ($m) { + return function_exists('mb_convert_encoding') ? mb_convert_encoding($m[0], 'UTF-8', 'ISO-8859-1') : utf8_encode($m[0]); + }, + $data + ); + + if (!is_string($data)) { + throw new \RuntimeException('Failed to preg_replace_callback: ' . preg_last_error()); + } + + return str_replace( + ['¤', '¦', '¨', '´', '¸', '¼', '½', '¾'], + ['€', 'Š', 'š', 'Ž', 'ž', 'Œ', 'œ', 'Ÿ'], + $data + ); } } From 9efe932221b6e43e742527ed48aace39cef87f4c Mon Sep 17 00:00:00 2001 From: Peter van der Wal Date: Mon, 27 May 2024 16:45:21 +0200 Subject: [PATCH 4/6] Install Roave BC Check in separate directory To avoid dependency conflicts with project dependencies, require BC Check tool in a separate directory --- .github/workflows/ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b3ab3ef..7e5cc62 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,10 @@ jobs: run: sudo bash ./upload-textfiles "/var/log/elasticsearch/*.log" - name: Install roave/backward-compatibility-check. - run: composer require -W --dev roave/backward-compatibility-check --no-plugins + run: | + # Install BC check tool in separate directory to avoid dependency conflicts + mkdir -p test-tools/bc-check + composer require --working-dir="test-tools/bc-check" roave/backward-compatibility-check --no-plugins - name: Run roave/backward-compatibility-check. - run: vendor/bin/roave-backward-compatibility-check --format=github-actions + run: test-tools/bc-check/vendor/bin/roave-backward-compatibility-check --format=github-actions From 8933545a22f718138b3c60cf1fde1901bdbc9dc5 Mon Sep 17 00:00:00 2001 From: Manuele Menozzi Date: Mon, 28 Oct 2024 15:37:50 +0100 Subject: [PATCH 5/6] Fix composer require checker --- composer-require-checker.json | 8 +------- composer.json | 4 +--- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/composer-require-checker.json b/composer-require-checker.json index 7dfd157..6d5bb51 100644 --- a/composer-require-checker.json +++ b/composer-require-checker.json @@ -1,9 +1,3 @@ { - "scan-files" : ["bin/esb"], - "symbol-whitelist": [ - "null", "true", "false", "static", "self", "parent", - "array", "string", "int", "float", "bool", "iterable", "callable", "void", "object", "mixed", "never", - - "mb_convert_encoding" - ] + "scan-files" : ["bin/esb"] } diff --git a/composer.json b/composer.json index 473f144..579c430 100644 --- a/composer.json +++ b/composer.json @@ -13,6 +13,7 @@ "php": "~7.4.0|~8.0.0|~8.1.0|~8.2.0|~8.3.0", "ext-pcntl": "*", "ext-json": "*", + "ext-mbstring": "*", "symfony/dependency-injection": "^4.3", "symfony/config": "^4.3", "symfony/yaml": "^4.3", @@ -38,9 +39,6 @@ "symfony/deprecation-contracts": "^2.1", "amphp/http-server-form-parser": "^1.1" }, - "suggest": { - "ext-mbstring": "*" - }, "autoload": { "psr-4": { "Webgriffe\\Esb\\": ["src/"] From 4dac01b164b06b02e1924b2974a071958db84f98 Mon Sep 17 00:00:00 2001 From: Manuele Menozzi Date: Mon, 28 Oct 2024 14:53:19 +0100 Subject: [PATCH 6/6] Allow to configure ElasticSearch index in esb.yml --- esb.yml.sample | 5 +++++ src/FlowConfiguration.php | 1 + src/Model/FlowConfig.php | 8 ++++++++ src/Service/ElasticSearch.php | 8 ++++++++ src/Service/QueueManager.php | 13 +++++++++++++ 5 files changed, 35 insertions(+) diff --git a/esb.yml.sample b/esb.yml.sample index 432348b..67e5ef6 100644 --- a/esb.yml.sample +++ b/esb.yml.sample @@ -30,6 +30,11 @@ services: flows: sample_flow: # The flow "code" and will be the Beanstalkd tube name description: Sample Flow # The flow description + es_index: # Optional: the create/update ElasticSearch index API body (see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html). This is useful if you want to control index mapping, settings and aliases. + settings: # For example you can set the total_fields limit to an higher (or lower) value: + index: + total_fields: + limit: 2000 producer: service: My\Esb\Producer # A producer service ID defined above batch_size: 1200 # Jobs are produced in batches of 1200 jobs. Optional: default is 1000 diff --git a/src/FlowConfiguration.php b/src/FlowConfiguration.php index 47ea2a6..5caa211 100644 --- a/src/FlowConfiguration.php +++ b/src/FlowConfiguration.php @@ -24,6 +24,7 @@ public function getConfigTreeBuilder(): TreeBuilder ->arrayPrototype() ->children() ->scalarNode('description')->isRequired()->cannotBeEmpty()->end() + ->variableNode('es_index')->defaultNull()->end() ->arrayNode('producer') ->children() ->scalarNode('service')->isRequired()->end() diff --git a/src/Model/FlowConfig.php b/src/Model/FlowConfig.php index 874174a..50f1066 100644 --- a/src/Model/FlowConfig.php +++ b/src/Model/FlowConfig.php @@ -102,4 +102,12 @@ public function getProducerBatchSize(): int { return $this->config['producer']['batch_size']; } + + /** + * @return array|null + */ + public function getElasticSearchIndexCreateOrUpdateBody(): ?array + { + return $this->config['es_index'] ?? null; + } } diff --git a/src/Service/ElasticSearch.php b/src/Service/ElasticSearch.php index b614443..9d32ca2 100644 --- a/src/Service/ElasticSearch.php +++ b/src/Service/ElasticSearch.php @@ -91,6 +91,14 @@ public function getClient(): Client return $this->client; } + /** + * @param array|null $createOrUpdateBody + */ + public function setElasticSearchIndex(string $indexName, array $createOrUpdateBody = null): Amp\Promise + { + return $this->client->createOrUpdateIndex($indexName, $createOrUpdateBody); + } + /** * @param JobInterface $job * @param string $indexName diff --git a/src/Service/QueueManager.php b/src/Service/QueueManager.php index f4206ec..ec6248f 100644 --- a/src/Service/QueueManager.php +++ b/src/Service/QueueManager.php @@ -72,6 +72,19 @@ public function __construct( public function boot(): Promise { return call(function () { + if ($this->flowConfig->getElasticSearchIndexCreateOrUpdateBody() !== null) { + yield $this->elasticSearch->setElasticSearchIndex( + $this->flowConfig->getTube(), + $this->flowConfig->getElasticSearchIndexCreateOrUpdateBody() + ); + $this->logger->debug( + 'Successfully set ElasticSearch index', + [ + 'index' => $this->flowConfig->getTube(), + 'body' => $this->flowConfig->getElasticSearchIndexCreateOrUpdateBody() + ] + ); + } //Producer yield $this->beanstalkClient->use($this->flowConfig->getTube());