From 5002ffbefcaa9ab73824076f465c66135353b174 Mon Sep 17 00:00:00 2001 From: David Grudl Date: Wed, 27 Oct 2021 01:21:23 +0200 Subject: [PATCH] Strings: added support for UTF8 offsets in regexp [WIP] --- src/Utils/Strings.php | 48 ++++++++++++++++++++++++++--- tests/Utils/Strings.match().phpt | 12 ++++++-- tests/Utils/Strings.matchAll().phpt | 15 +++++++++ tests/Utils/Strings.replace().phpt | 3 +- tests/Utils/Strings.split().phpt | 44 ++++++++++++++++++-------- 5 files changed, 102 insertions(+), 20 deletions(-) diff --git a/src/Utils/Strings.php b/src/Utils/Strings.php index 868680ecb..02f12330f 100644 --- a/src/Utils/Strings.php +++ b/src/Utils/Strings.php @@ -474,11 +474,16 @@ public static function split( string $pattern, bool|int $captureOffset = false, bool $skipEmpty = false, + bool $utf8Offset = false, ): array { $flags = is_int($captureOffset) // back compatibility ? $captureOffset : ($captureOffset ? PREG_SPLIT_OFFSET_CAPTURE : 0) | ($skipEmpty ? PREG_SPLIT_NO_EMPTY : 0); - return self::pcre('preg_split', [$pattern, $subject, -1, $flags | PREG_SPLIT_DELIM_CAPTURE]); + $m = self::pcre('preg_split', [$pattern, $subject, -1, $flags | PREG_SPLIT_DELIM_CAPTURE]); + if ($utf8Offset && ($flags & PREG_SPLIT_OFFSET_CAPTURE)) { + return self::bytesToChars($subject, [$m])[0]; + } + return $m; } @@ -491,16 +496,24 @@ public static function match( bool|int $captureOffset = false, int $offset = 0, bool $unmatchedAsNull = false, + bool $utf8Offset = false, ): ?array { $flags = is_int($captureOffset) // back compatibility ? $captureOffset : ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0); + if ($utf8Offset) { + $offset = strlen(self::substring($subject, 0, $offset)); + } if ($offset > strlen($subject)) { return null; } - return self::pcre('preg_match', [$pattern, $subject, &$m, $flags, $offset]) - ? $m - : null; + if (!self::pcre('preg_match', [$pattern, $subject, &$m, $flags, $offset])) { + return null; + } + if ($utf8Offset && ($flags & PREG_OFFSET_CAPTURE)) { + return self::bytesToChars($subject, [$m])[0]; + } + return $m; } @@ -515,10 +528,14 @@ public static function matchAll( int $offset = 0, bool $unmatchedAsNull = false, bool $patternOrder = false, + bool $utf8Offset = false, ): array { $flags = is_int($captureOffset) // back compatibility ? $captureOffset : ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0) | ($patternOrder ? PREG_PATTERN_ORDER : 0); + if ($utf8Offset) { + $offset = strlen(self::substring($subject, 0, $offset)); + } if ($offset > strlen($subject)) { return []; } @@ -527,6 +544,9 @@ public static function matchAll( ($flags & PREG_PATTERN_ORDER) ? $flags : ($flags | PREG_SET_ORDER), $offset, ]); + if ($utf8Offset && ($flags & PREG_OFFSET_CAPTURE)) { + return self::bytesToChars($subject, $m); + } return $m; } @@ -541,12 +561,16 @@ public static function replace( int $limit = -1, bool $captureOffset = false, bool $unmatchedAsNull = false, + bool $utf8Offset = false, ): string { if (is_object($replacement) || is_array($replacement)) { if (!is_callable($replacement, false, $textual)) { throw new Nette\InvalidStateException("Callback '$textual' is not callable."); } $flags = ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0); + if ($utf8Offset && $captureOffset) { + $replacement = fn($m) => $replacement(self::bytesToChars($subject, [$m])[0]); + } return self::pcre('preg_replace_callback', [$pattern, $replacement, $subject, $limit, 0, $flags]); } elseif (is_array($pattern) && is_string(key($pattern))) { @@ -558,6 +582,22 @@ public static function replace( } + private static function bytesToChars(string $s, array $groups): array + { + $lastBytes = $lastChars = 0; + foreach ($groups as &$matches) { + foreach ($matches as &$match) { + if ($match[1] > $lastBytes) { + $lastChars += self::length(substr($s, $lastBytes, $match[1] - $lastBytes)); + $lastBytes = $match[1]; + } + $match[1] = $lastChars; + } + } + return $groups; + } + + /** @internal */ public static function pcre(string $func, array $args) { diff --git a/tests/Utils/Strings.match().phpt b/tests/Utils/Strings.match().phpt index 314472c01..9050f8851 100644 --- a/tests/Utils/Strings.match().phpt +++ b/tests/Utils/Strings.match().phpt @@ -19,10 +19,16 @@ Assert::same(['hell', 'l'], Strings::match('hello world!', '#([e-l])+#')); Assert::same(['hell'], Strings::match('hello world!', '#[e-l]+#')); -Assert::same([['hell', 0]], Strings::match('hello world!', '#[e-l]+#', PREG_OFFSET_CAPTURE)); -Assert::same([['hell', 0]], Strings::match('hello world!', '#[e-l]+#', captureOffset: true)); +Assert::same([['l', 2]], Strings::match('žluťoučký kůň', '#[e-l]+#u', PREG_OFFSET_CAPTURE)); +Assert::same([['l', 2]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true)); -Assert::same(['ll'], Strings::match('hello world!', '#[e-l]+#', 0, 2)); +Assert::same([['l', 1]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true, utf8Offset: true)); + +Assert::same(['l'], Strings::match('žluťoučký kůň', '#[e-l]+#u', 0, 2)); + +Assert::same(['k'], Strings::match('žluťoučký kůň', '#[e-l]+#u', utf8Offset: true, offset: 2)); + +Assert::same([['k', 7]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true, utf8Offset: true, offset: 2)); Assert::null(Strings::match('hello world!', '', 0, 50)); Assert::null(Strings::match('', '', 0, 1)); diff --git a/tests/Utils/Strings.matchAll().phpt b/tests/Utils/Strings.matchAll().phpt index bc81c4bee..1596c35a0 100644 --- a/tests/Utils/Strings.matchAll().phpt +++ b/tests/Utils/Strings.matchAll().phpt @@ -45,14 +45,29 @@ Assert::same([ [['u', 3], ['u', 7], ['', 11], ['', 15]], ], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', PREG_OFFSET_CAPTURE | PREG_PATTERN_ORDER)); +Assert::same([ + [['lu', 1], ['l', 1], ['u', 2]], + [['ou', 4], ['o', 4], ['u', 5]], + [['k', 7], ['k', 7], ['', 8]], + [['k', 10], ['k', 10], ['', 11]], +], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, utf8Offset: true)); + Assert::same([ [['lu', 2], ['ou', 6], ['k', 10], ['k', 14]], [['l', 2], ['o', 6], ['k', 10], ['k', 14]], [['u', 3], ['u', 7], ['', 11], ['', 15]], ], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, patternOrder: true)); +Assert::same([ + [['lu', 1], ['ou', 4], ['k', 7], ['k', 10]], + [['l', 10], ['o', 10], ['k', 10], ['k', 10]], + [['u', 10], ['u', 10], ['', 10], ['', 11]], +], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, utf8Offset: true, patternOrder: true)); + Assert::same([['l'], ['k'], ['k']], Strings::matchAll('žluťoučký kůň', '#[e-l]+#u', 0, 2)); +Assert::same([['k'], ['k']], Strings::matchAll('žluťoučký kůň', '#[e-l]+#u', utf8Offset: true, offset: 2)); + Assert::same([['ll', 'l']], Strings::matchAll('hello world!', '#[e-l]+#', PREG_PATTERN_ORDER, 2)); Assert::same([['ll', 'l']], Strings::matchAll('hello world!', '#[e-l]+#', patternOrder: true, offset: 2)); diff --git a/tests/Utils/Strings.replace().phpt b/tests/Utils/Strings.replace().phpt index f64a2edb6..79c0fe520 100644 --- a/tests/Utils/Strings.replace().phpt +++ b/tests/Utils/Strings.replace().phpt @@ -34,5 +34,6 @@ Assert::same('#@ @@@#d!', Strings::replace('hello world!', [ ])); Assert::same(' !', Strings::replace('hello world!', '#\w#')); Assert::same(' !', Strings::replace('hello world!', ['#\w#'])); -Assert::same('hell0o worl9d!', Strings::replace('hello world!', '#[e-l]+#', fn($m) => implode($m[0]), captureOffset: true)); +Assert::same('žl2uťoučk10ý k14ůň!', Strings::replace('žluťoučký kůň!', '#[e-l]+#u', fn($m) => implode($m[0]), captureOffset: true)); +Assert::same('žl1uťoučk7ý k10ůň!', Strings::replace('žluťoučký kůň!', '#[e-l]+#u', fn($m) => implode($m[0]), captureOffset: true, utf8Offset: true)); Strings::replace('hello world!', '#e(x)*#', fn($m) => Assert::null($m[1]), unmatchedAsNull: true); diff --git a/tests/Utils/Strings.split().phpt b/tests/Utils/Strings.split().phpt index 3a57a4c4e..f90447bbf 100644 --- a/tests/Utils/Strings.split().phpt +++ b/tests/Utils/Strings.split().phpt @@ -38,17 +38,37 @@ Assert::same([ ], Strings::split('a, b, c', '#(,)\s*#', skipEmpty: true)); Assert::same([ - ['a', 0], - [',', 1], - ['b', 3], - [',', 4], - ['c', 6], -], Strings::split('a, b, c', '#(,)\s*#', PREG_SPLIT_OFFSET_CAPTURE)); + ['ž', 0], + ['lu', 2], + ['ť', 4], + ['ou', 6], + ['č', 8], + ['k', 10], + ['ý ', 11], + ['k', 14], + ['ůň', 15], +], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', PREG_SPLIT_OFFSET_CAPTURE)); Assert::same([ - ['a', 0], - [',', 1], - ['b', 3], - [',', 4], - ['c', 6], -], Strings::split('a, b, c', '#(,)\s*#', captureOffset: true)); + ['ž', 0], + ['lu', 2], + ['ť', 4], + ['ou', 6], + ['č', 8], + ['k', 10], + ['ý ', 11], + ['k', 14], + ['ůň', 15], +], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', captureOffset: true)); + +Assert::same([ + ['ž', 0], + ['lu', 1], + ['ť', 3], + ['ou', 4], + ['č', 6], + ['k', 7], + ['ý ', 8], + ['k', 10], + ['ůň', 11], +], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', captureOffset: true, utf8Offset: true));