Skip to content

Commit

Permalink
Strings: added support for UTF8 offsets in regexp [WIP]
Browse files Browse the repository at this point in the history
  • Loading branch information
@dg
dg committed Oct 26, 2021
1 parent 8015d13 commit 5002ffb
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 20 deletions.
48 changes: 44 additions & 4 deletions src/Utils/Strings.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,11 +474,16 @@ public static function split(
string $pattern,
bool|int $captureOffset = false,
bool $skipEmpty = false,
bool $utf8Offset = false,
): array {
$flags = is_int($captureOffset) // back compatibility
? $captureOffset
: ($captureOffset ? PREG_SPLIT_OFFSET_CAPTURE : 0) | ($skipEmpty ? PREG_SPLIT_NO_EMPTY : 0);
return self::pcre('preg_split', [$pattern, $subject, -1, $flags | PREG_SPLIT_DELIM_CAPTURE]);
$m = self::pcre('preg_split', [$pattern, $subject, -1, $flags | PREG_SPLIT_DELIM_CAPTURE]);
if ($utf8Offset && ($flags & PREG_SPLIT_OFFSET_CAPTURE)) {
return self::bytesToChars($subject, [$m])[0];
}
return $m;
}


Expand All @@ -491,16 +496,24 @@ public static function match(
bool|int $captureOffset = false,
int $offset = 0,
bool $unmatchedAsNull = false,
bool $utf8Offset = false,
): ?array {
$flags = is_int($captureOffset) // back compatibility
? $captureOffset
: ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0);
if ($utf8Offset) {
$offset = strlen(self::substring($subject, 0, $offset));
}
if ($offset > strlen($subject)) {
return null;
}
return self::pcre('preg_match', [$pattern, $subject, &$m, $flags, $offset])
? $m
: null;
if (!self::pcre('preg_match', [$pattern, $subject, &$m, $flags, $offset])) {
return null;
}
if ($utf8Offset && ($flags & PREG_OFFSET_CAPTURE)) {
return self::bytesToChars($subject, [$m])[0];
}
return $m;
}


Expand All @@ -515,10 +528,14 @@ public static function matchAll(
int $offset = 0,
bool $unmatchedAsNull = false,
bool $patternOrder = false,
bool $utf8Offset = false,
): array {
$flags = is_int($captureOffset) // back compatibility
? $captureOffset
: ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0) | ($patternOrder ? PREG_PATTERN_ORDER : 0);
if ($utf8Offset) {
$offset = strlen(self::substring($subject, 0, $offset));
}
if ($offset > strlen($subject)) {
return [];
}
Expand All @@ -527,6 +544,9 @@ public static function matchAll(
($flags & PREG_PATTERN_ORDER) ? $flags : ($flags | PREG_SET_ORDER),
$offset,
]);
if ($utf8Offset && ($flags & PREG_OFFSET_CAPTURE)) {
return self::bytesToChars($subject, $m);
}
return $m;
}

Expand All @@ -541,12 +561,16 @@ public static function replace(
int $limit = -1,
bool $captureOffset = false,
bool $unmatchedAsNull = false,
bool $utf8Offset = false,
): string {
if (is_object($replacement) || is_array($replacement)) {
if (!is_callable($replacement, false, $textual)) {
throw new Nette\InvalidStateException("Callback '$textual' is not callable.");
}
$flags = ($captureOffset ? PREG_OFFSET_CAPTURE : 0) | ($unmatchedAsNull ? PREG_UNMATCHED_AS_NULL : 0);
if ($utf8Offset && $captureOffset) {
$replacement = fn($m) => $replacement(self::bytesToChars($subject, [$m])[0]);
}
return self::pcre('preg_replace_callback', [$pattern, $replacement, $subject, $limit, 0, $flags]);

} elseif (is_array($pattern) && is_string(key($pattern))) {
Expand All @@ -558,6 +582,22 @@ public static function replace(
}


private static function bytesToChars(string $s, array $groups): array
{
$lastBytes = $lastChars = 0;
foreach ($groups as &$matches) {
foreach ($matches as &$match) {
if ($match[1] > $lastBytes) {
$lastChars += self::length(substr($s, $lastBytes, $match[1] - $lastBytes));
$lastBytes = $match[1];
}
$match[1] = $lastChars;
}
}
return $groups;
}


/** @internal */
public static function pcre(string $func, array $args)
{
Expand Down
12 changes: 9 additions & 3 deletions tests/Utils/Strings.match().phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,16 @@ Assert::same(['hell', 'l'], Strings::match('hello world!', '#([e-l])+#'));

Assert::same(['hell'], Strings::match('hello world!', '#[e-l]+#'));

Assert::same([['hell', 0]], Strings::match('hello world!', '#[e-l]+#', PREG_OFFSET_CAPTURE));
Assert::same([['hell', 0]], Strings::match('hello world!', '#[e-l]+#', captureOffset: true));
Assert::same([['l', 2]], Strings::match('žluťoučký kůň', '#[e-l]+#u', PREG_OFFSET_CAPTURE));
Assert::same([['l', 2]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true));

Assert::same(['ll'], Strings::match('hello world!', '#[e-l]+#', 0, 2));
Assert::same([['l', 1]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true, utf8Offset: true));

Assert::same(['l'], Strings::match('žluťoučký kůň', '#[e-l]+#u', 0, 2));

Assert::same(['k'], Strings::match('žluťoučký kůň', '#[e-l]+#u', utf8Offset: true, offset: 2));

Assert::same([['k', 7]], Strings::match('žluťoučký kůň', '#[e-l]+#u', captureOffset: true, utf8Offset: true, offset: 2));

Assert::null(Strings::match('hello world!', '', 0, 50));
Assert::null(Strings::match('', '', 0, 1));
15 changes: 15 additions & 0 deletions tests/Utils/Strings.matchAll().phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,29 @@ Assert::same([
[['u', 3], ['u', 7], ['', 11], ['', 15]],
], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', PREG_OFFSET_CAPTURE | PREG_PATTERN_ORDER));

Assert::same([
[['lu', 1], ['l', 1], ['u', 2]],
[['ou', 4], ['o', 4], ['u', 5]],
[['k', 7], ['k', 7], ['', 8]],
[['k', 10], ['k', 10], ['', 11]],
], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, utf8Offset: true));

Assert::same([
[['lu', 2], ['ou', 6], ['k', 10], ['k', 14]],
[['l', 2], ['o', 6], ['k', 10], ['k', 14]],
[['u', 3], ['u', 7], ['', 11], ['', 15]],
], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, patternOrder: true));

Assert::same([
[['lu', 1], ['ou', 4], ['k', 7], ['k', 10]],
[['l', 10], ['o', 10], ['k', 10], ['k', 10]],
[['u', 10], ['u', 10], ['', 10], ['', 11]],
], Strings::matchAll('žluťoučký kůň!', '#([a-z])([a-z]*)#u', captureOffset: true, utf8Offset: true, patternOrder: true));

Assert::same([['l'], ['k'], ['k']], Strings::matchAll('žluťoučký kůň', '#[e-l]+#u', 0, 2));

Assert::same([['k'], ['k']], Strings::matchAll('žluťoučký kůň', '#[e-l]+#u', utf8Offset: true, offset: 2));

Assert::same([['ll', 'l']], Strings::matchAll('hello world!', '#[e-l]+#', PREG_PATTERN_ORDER, 2));
Assert::same([['ll', 'l']], Strings::matchAll('hello world!', '#[e-l]+#', patternOrder: true, offset: 2));

Expand Down
3 changes: 2 additions & 1 deletion tests/Utils/Strings.replace().phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,6 @@ Assert::same('#@ @@@#d!', Strings::replace('hello world!', [
]));
Assert::same(' !', Strings::replace('hello world!', '#\w#'));
Assert::same(' !', Strings::replace('hello world!', ['#\w#']));
Assert::same('hell0o worl9d!', Strings::replace('hello world!', '#[e-l]+#', fn($m) => implode($m[0]), captureOffset: true));
Assert::same('žl2uťoučk10ý k14ůň!', Strings::replace('žluťoučký kůň!', '#[e-l]+#u', fn($m) => implode($m[0]), captureOffset: true));
Assert::same('žl1uťoučk7ý k10ůň!', Strings::replace('žluťoučký kůň!', '#[e-l]+#u', fn($m) => implode($m[0]), captureOffset: true, utf8Offset: true));
Strings::replace('hello world!', '#e(x)*#', fn($m) => Assert::null($m[1]), unmatchedAsNull: true);
44 changes: 32 additions & 12 deletions tests/Utils/Strings.split().phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,37 @@ Assert::same([
], Strings::split('a, b, c', '#(,)\s*#', skipEmpty: true));

Assert::same([
['a', 0],
[',', 1],
['b', 3],
[',', 4],
['c', 6],
], Strings::split('a, b, c', '#(,)\s*#', PREG_SPLIT_OFFSET_CAPTURE));
['ž', 0],
['lu', 2],
['ť', 4],
['ou', 6],
['č', 8],
['k', 10],
['ý ', 11],
['k', 14],
['ůň', 15],
], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', PREG_SPLIT_OFFSET_CAPTURE));

Assert::same([
['a', 0],
[',', 1],
['b', 3],
[',', 4],
['c', 6],
], Strings::split('a, b, c', '#(,)\s*#', captureOffset: true));
['ž', 0],
['lu', 2],
['ť', 4],
['ou', 6],
['č', 8],
['k', 10],
['ý ', 11],
['k', 14],
['ůň', 15],
], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', captureOffset: true));

Assert::same([
['ž', 0],
['lu', 1],
['ť', 3],
['ou', 4],
['č', 6],
['k', 7],
['ý ', 8],
['k', 10],
['ůň', 11],
], Strings::split('žluťoučký kůň', '#([a-z]+)\s*#u', captureOffset: true, utf8Offset: true));

0 comments on commit 5002ffb

Please sign in to comment.