From c8dd8f7822a9be484532079521d77d33c0e4549d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gamez?= Date: Thu, 7 Apr 2022 01:15:06 +0200 Subject: [PATCH 01/33] Update `halaxa/json-machine` dependency to `^0.8|^1.0` --- CHANGELOG.md | 4 ++-- composer.json | 2 +- src/Handlers/Filename.php | 4 ++-- src/Handlers/IterableSource.php | 4 ++-- src/Handlers/JsonString.php | 4 ++-- src/Handlers/Resource.php | 4 ++-- tests/Handlers/EndpointTest.php | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faba229..754b118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip ## NEXT - YYYY-MM-DD ### Added -- Nothing +- Support for `halaxa/json-machine: ^0.8|^1.0` ### Deprecated - Nothing @@ -16,7 +16,7 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip - Nothing ### Removed -- Nothing +- Support for `halaxa/json-machine: ^0.7` ### Security - Nothing diff --git a/composer.json b/composer.json index a2d8b5d..4aee4b5 100644 --- a/composer.json +++ b/composer.json @@ -20,7 +20,7 @@ }], "require": { "php": "^7.2||^8.0", - "halaxa/json-machine": "^0.7", + "halaxa/json-machine": "^0.8|^1.0", "illuminate/support": ">=6.0" }, "suggest": { diff --git a/src/Handlers/Filename.php b/src/Handlers/Filename.php index 811caee..268d965 100644 --- a/src/Handlers/Filename.php +++ b/src/Handlers/Filename.php @@ -3,7 +3,7 @@ namespace Cerbero\LazyJson\Handlers; use Cerbero\LazyJson\Concerns\JsonPointerAware; -use JsonMachine\JsonMachine; +use JsonMachine\Items; use Traversable; /** @@ -34,6 +34,6 @@ public function handles($source): bool */ public function handle($source, string $path): Traversable { - return JsonMachine::fromFile($source, $this->toJsonPointer($path)); + return Items::fromFile($source, ['pointer' => $this->toJsonPointer($path)]); } } diff --git a/src/Handlers/IterableSource.php b/src/Handlers/IterableSource.php index bdfbb32..9aec694 100644 --- a/src/Handlers/IterableSource.php +++ b/src/Handlers/IterableSource.php @@ -3,7 +3,7 @@ namespace Cerbero\LazyJson\Handlers; use Cerbero\LazyJson\Concerns\JsonPointerAware; -use JsonMachine\JsonMachine; +use JsonMachine\Items; use Traversable; /** @@ -34,6 +34,6 @@ public function handles($source): bool */ public function handle($source, string $path): Traversable { - return JsonMachine::fromIterable($source, $this->toJsonPointer($path)); + return Items::fromIterable($source, ['pointer' => $this->toJsonPointer($path)]); } } diff --git a/src/Handlers/JsonString.php b/src/Handlers/JsonString.php index 9e7e505..93202ba 100644 --- a/src/Handlers/JsonString.php +++ b/src/Handlers/JsonString.php @@ -4,7 +4,7 @@ use Cerbero\LazyJson\Concerns\EndpointAware; use Cerbero\LazyJson\Concerns\JsonPointerAware; -use JsonMachine\JsonMachine; +use JsonMachine\Items; use Traversable; /** @@ -36,6 +36,6 @@ public function handles($source): bool */ public function handle($source, string $path): Traversable { - return JsonMachine::fromString($source, $this->toJsonPointer($path)); + return Items::fromString($source, ['pointer' => $this->toJsonPointer($path)]); } } diff --git a/src/Handlers/Resource.php b/src/Handlers/Resource.php index f82eee4..6040b0b 100644 --- a/src/Handlers/Resource.php +++ b/src/Handlers/Resource.php @@ -3,7 +3,7 @@ namespace Cerbero\LazyJson\Handlers; use Cerbero\LazyJson\Concerns\JsonPointerAware; -use JsonMachine\JsonMachine; +use JsonMachine\Items; use Traversable; /** @@ -34,6 +34,6 @@ public function handles($source): bool */ public function handle($source, string $path): Traversable { - return JsonMachine::fromStream($source, $this->toJsonPointer($path)); + return Items::fromStream($source, ['pointer' => $this->toJsonPointer($path)]); } } diff --git a/tests/Handlers/EndpointTest.php b/tests/Handlers/EndpointTest.php index e973e36..ca4eab9 100644 --- a/tests/Handlers/EndpointTest.php +++ b/tests/Handlers/EndpointTest.php @@ -6,7 +6,7 @@ use Cerbero\LazyJson\Handlers\Endpoint; use GuzzleHttp\Client; use GuzzleHttp\Psr7\Response as Psr7Response; -use JsonMachine\JsonMachine; +use JsonMachine\Items; use Mockery as m; use PHPUnit\Framework\TestCase; @@ -54,7 +54,7 @@ public function handleEndpoint() $handled = (new Endpoint())->handle('https://endpoint.test', ''); - $this->assertInstanceOf(JsonMachine::class, $handled); + $this->assertInstanceOf(Items::class, $handled); foreach ($handled as $key => $value) { $this->assertSame('end', $key); @@ -73,7 +73,7 @@ public function extractsJsonSubtrees() $handled = (new Endpoint())->handle('https://endpoint.test', 'foo.bar'); - $this->assertInstanceOf(JsonMachine::class, $handled); + $this->assertInstanceOf(Items::class, $handled); foreach ($handled as $key => $value) { $this->assertSame('bar', $key); From 3363f886a232c62149f8dcd5cdb66528afe63f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gamez?= Date: Thu, 7 Apr 2022 01:22:36 +0200 Subject: [PATCH 02/33] Prevent code coverage uploads in forks --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eda588e..8b37f10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,6 +70,7 @@ jobs: run: vendor/bin/phpunit --coverage-text --coverage-clover=coverage.clover - name: Upload coverage + if: github.repository_owner == 'cerbero90' run: | wget https://scrutinizer-ci.com/ocular.phar php ocular.phar code-coverage:upload --format=php-clover coverage.clover From f616146855798d11de81459420603b34d49bc9ae Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 29 Jun 2023 12:45:41 +0200 Subject: [PATCH 03/33] Modernize package --- .github/FUNDING.yml | 2 +- .github/workflows/build.yml | 55 +++------ CHANGELOG.md | 4 +- README.md | 49 ++++---- composer.json | 40 +++--- duster.json | 13 ++ phpstan-baseline.neon | 1 + phpstan.neon | 6 + phpunit.xml.dist | 47 ++++--- pint.json | 19 +++ src/Concerns/EndpointAware.php | 25 ---- src/Concerns/JsonPointerAware.php | 25 ---- src/Exceptions/LazyJsonException.php | 14 --- src/Handlers/Endpoint.php | 61 --------- src/Handlers/Filename.php | 39 ------ src/Handlers/Handler.php | 29 ----- src/Handlers/IterableSource.php | 39 ------ src/Handlers/JsonString.php | 41 ------ src/Handlers/LaravelClientResponse.php | 36 ------ src/Handlers/Psr7Message.php | 36 ------ src/Handlers/Psr7Stream.php | 45 ------- src/Handlers/Resource.php | 39 ------ src/Macro.php | 32 ----- src/Providers/LazyJsonServiceProvider.php | 24 ---- src/Source.php | 82 ------------ src/StreamWrapper.php | 75 ----------- tests/Handlers/EndpointTest.php | 101 --------------- tests/LazyJsonTest.php | 144 ---------------------- tests/StreamWrapperTest.php | 126 ------------------- tests/stub.json | 1 - 30 files changed, 119 insertions(+), 1131 deletions(-) create mode 100644 duster.json create mode 100644 phpstan-baseline.neon create mode 100644 phpstan.neon create mode 100644 pint.json delete mode 100644 src/Concerns/EndpointAware.php delete mode 100644 src/Concerns/JsonPointerAware.php delete mode 100644 src/Exceptions/LazyJsonException.php delete mode 100644 src/Handlers/Endpoint.php delete mode 100644 src/Handlers/Filename.php delete mode 100644 src/Handlers/Handler.php delete mode 100644 src/Handlers/IterableSource.php delete mode 100644 src/Handlers/JsonString.php delete mode 100644 src/Handlers/LaravelClientResponse.php delete mode 100644 src/Handlers/Psr7Message.php delete mode 100644 src/Handlers/Psr7Stream.php delete mode 100644 src/Handlers/Resource.php delete mode 100644 src/Macro.php delete mode 100644 src/Providers/LazyJsonServiceProvider.php delete mode 100644 src/Source.php delete mode 100644 src/StreamWrapper.php delete mode 100644 tests/Handlers/EndpointTest.php delete mode 100644 tests/LazyJsonTest.php delete mode 100644 tests/StreamWrapperTest.php delete mode 100644 tests/stub.json diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index ceeb344..af174ad 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1,2 @@ github: cerbero90 - +ko_fi: cerbero90 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b37f10..14f8a58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,8 @@ name: build on: - push: - pull_request: + push: + pull_request: jobs: tests: @@ -11,20 +11,11 @@ jobs: strategy: fail-fast: false matrix: - php: [7.2, 7.3, 7.4, 8.0] - laravel: [6.*, 7.*, 8.*] + php: [8.1, 8.2] dependency-version: [prefer-lowest, prefer-stable] - os: [ubuntu-latest, windows-latest] - exclude: - - laravel: 6.* - php: 8.0 - dependency-version: prefer-lowest - - laravel: 8.* - php: 7.2 - - os: windows-latest - php: 8.0 - - name: PHP ${{ matrix.php }} - Laravel ${{ matrix.laravel }} - ${{ matrix.dependency-version }} - ${{ matrix.os }} + os: [ubuntu-latest] + + name: PHP ${{ matrix.php }} - ${{ matrix.dependency-version }} - ${{ matrix.os }} steps: - name: Checkout code @@ -34,17 +25,15 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php }} - extensions: dom, curl, libxml, mbstring, zip tools: composer:v2 coverage: none - name: Install dependencies run: | - composer require "illuminate/contracts=${{ matrix.laravel }}" --no-interaction --no-update composer update --${{ matrix.dependency-version }} --prefer-dist --no-interaction - name: Execute tests - run: vendor/bin/phpunit --verbose + run: vendor/bin/pest coverage: runs-on: ubuntu-latest @@ -54,12 +43,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: 7.4 - extensions: dom, curl, libxml, mbstring, zip + php-version: 8.1 tools: composer:v2 coverage: xdebug @@ -67,29 +57,20 @@ jobs: run: composer update --prefer-stable --prefer-dist --no-interaction - name: Execute tests - run: vendor/bin/phpunit --coverage-text --coverage-clover=coverage.clover + run: vendor/bin/pest --coverage-text --coverage-clover=coverage.clover - name: Upload coverage - if: github.repository_owner == 'cerbero90' run: | - wget https://scrutinizer-ci.com/ocular.phar - php ocular.phar code-coverage:upload --format=php-clover coverage.clover + vendor/bin/ocular code-coverage:upload --format=php-clover coverage.clover - style: + linting: runs-on: ubuntu-latest - name: Coding style + name: Linting steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Setup PHP - uses: shivammathur/setup-php@v2 + - uses: actions/checkout@v3 + - name: Duster Lint + uses: tighten/duster-action@v1 with: - php-version: 8.0 - tools: phpcs - coverage: none - - - name: Execute check - run: phpcs --standard=psr12 src/ tests/ + args: lint -u tlint,phpcodesniffer,pint,phpstan diff --git a/CHANGELOG.md b/CHANGELOG.md index 754b118..faba229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip ## NEXT - YYYY-MM-DD ### Added -- Support for `halaxa/json-machine: ^0.8|^1.0` +- Nothing ### Deprecated - Nothing @@ -16,7 +16,7 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip - Nothing ### Removed -- Support for `halaxa/json-machine: ^0.7` +- Nothing ### Security - Nothing diff --git a/README.md b/README.md index 5e676f7..655259c 100644 --- a/README.md +++ b/README.md @@ -7,35 +7,28 @@ [![Build Status][ico-actions]][link-actions] [![Coverage Status][ico-scrutinizer]][link-scrutinizer] [![Quality Score][ico-code-quality]][link-code-quality] +[![PHPStan Level][ico-phpstan]][link-phpstan] [![Latest Version][ico-version]][link-packagist] [![Software License][ico-license]](LICENSE.md) -[![PSR-7][ico-psr7]][link-psr7] -[![PSR-12][ico-psr12]][link-psr12] +[![PER][ico-per]][link-per] [![Total Downloads][ico-downloads]][link-downloads] -Framework agnostic package to load heavy JSON in [lazy collections](https://laravel.com/docs/collections#lazy-collections). Under the hood, the brilliant [JSON Machine](https://github.com/halaxa/json-machine) by [@halaxa](https://github.com/halaxa) is used as a lexer and parser. +Framework-agnostic package to load JSONs of any dimension and from any source into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). -Need to load paginated items of JSON APIs? Consider using [Lazy JSON Pages](https://github.com/cerbero90/lazy-json-pages) instead. +Under the hood, [🧩 JSON Parser](https://github.com/cerbero90/json-parser) is used to parse and extract sub-trees from any JSON. +Need to lazy load items from paginated JSON APIs? Consider using [🐼 Lazy JSON Pages](https://github.com/cerbero90/lazy-json-pages) instead. -## Install -In a Laravel application, all you need to do is requiring the package: +## 📦 Install + +Via Composer: ``` bash composer require cerbero/lazy-json ``` -Otherwise, you also need to register the lazy collection macro manually: - -```php -use Cerbero\LazyJson\Macro; -use Illuminate\Support\LazyCollection; - -LazyCollection::macro('fromJson', new Macro()); -``` - -## Usage +## 🔮 Usage Loading JSON in lazy collections is possible by using the collection itself or the included helper: @@ -94,44 +87,44 @@ $ids = lazyJson($source, 'data.*.users.*.id') ->all(); ``` -## Change log +## 📆 Change log Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. -## Testing +## 🧪 Testing ``` bash composer test ``` -## Contributing +## 💞 Contributing Please see [CONTRIBUTING](CONTRIBUTING.md) and [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) for details. -## Security +## 🧯 Security If you discover any security related issues, please email andrea.marco.sartori@gmail.com instead of using the issue tracker. -## Credits +## 🏅 Credits - [Andrea Marco Sartori][link-author] - [All Contributors][link-contributors] -## License +## ⚖️ License The MIT License (MIT). Please see [License File](LICENSE.md) for more information. [ico-author]: https://img.shields.io/static/v1?label=author&message=cerbero90&color=50ABF1&logo=twitter&style=flat-square [ico-php]: https://img.shields.io/packagist/php-v/cerbero/lazy-json?color=%234F5B93&logo=php&style=flat-square -[ico-laravel]: https://img.shields.io/static/v1?label=laravel&message=%E2%89%A56.0&color=ff2d20&logo=laravel&style=flat-square +[ico-laravel]: https://img.shields.io/static/v1?label=laravel&message=%E2%89%A56.20&color=ff2d20&logo=laravel&style=flat-square [ico-octane]: https://img.shields.io/static/v1?label=octane&message=compatible&color=ff2d20&logo=laravel&style=flat-square [ico-version]: https://img.shields.io/packagist/v/cerbero/lazy-json.svg?label=version&style=flat-square -[ico-actions]: https://img.shields.io/github/workflow/status/cerbero90/lazy-json/build?style=flat-square&logo=github +[ico-actions]: https://img.shields.io/github/actions/workflow/status/cerbero90/json-parser/build.yml?branch=master&style=flat-square&logo=github [ico-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square -[ico-psr7]: https://img.shields.io/static/v1?label=compliance&message=PSR-7&color=blue&style=flat-square -[ico-psr12]: https://img.shields.io/static/v1?label=compliance&message=PSR-12&color=blue&style=flat-square +[ico-per]: https://img.shields.io/static/v1?label=compliance&message=PER&color=blue&style=flat-square [ico-scrutinizer]: https://img.shields.io/scrutinizer/coverage/g/cerbero90/lazy-json.svg?style=flat-square&logo=scrutinizer [ico-code-quality]: https://img.shields.io/scrutinizer/g/cerbero90/lazy-json.svg?style=flat-square&logo=scrutinizer +[ico-phpstan]: https://img.shields.io/badge/level-max-success?style=flat-square&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAGb0lEQVR42u1Xe1BUZRS/y4Kg8oiR3FCCBUySESZBRCiaBnmEsOzeSzsg+KxYYO9dEEftNRqZjx40FRZkTpqmOz5S2LsXlEZBciatkQnHDGYaGdFy1EpGMHl/p/PdFlt2rk5O+J9n5nA/vtf5ned3lnlISpRhafBlLRLHCtJGVrB/ZBDsaw2lUqzReGAC46DstTYfnSCGUjaaDvgxACo6j3vUenNdImeRXqdnWV5az5rrnzeZznj8J+E5Ftsclhf3s4J4CS/oRx5Bvon8ZU65FGYQxAwcf85a7CeRz+C41THejueydCZ7AAK34nwv3kHP/oUKdOL4K7258fF7Cud427O48RQeGkIGJ77N8fZqlrcfRP4d/x90WQfHXLeBt9dTrSlwl3V65ynWLM1SEA2qbNQckbe4Xmww10Hmy3shid0CMcmlEJtSDsl5VZBdfAgMvI3uuR+moJqN6LaxmpsOBeLCDmTifCB92RcQmbAUJvtqALc5sQr8p86gYBCcFdBq9wOin7NQax6ewlB6rqLZHf23FP10y3lj6uJtEBg2HxiVCtzd3SEwMBCio6Nh9uzZ4O/vLwOZ4OUNM2NyIGPFrvuzBG//lRPs+VQ2k1ki+ePkd84bskz7YFpYgizEz88P8vPzYffu3dDS0gJNTU1QXV0NqampRK1WIwgfiE4qhOyig0rC+pCvK8QUoML7uJVHA5kcQUp3DSpqWjc3d/Dy8oKioiLo6uqCoaEhuHb1KvT09AAhBFpbW4lOpyMyyIBQSCmoUQLQzgniNvz+obB2HS2RwBgE6dOxCyJogmNkP2u1Wrhw4QJ03+iGrR9XEd3CTNBn6eCbo40wPDwMdXV1BF1DVG5qiEtboxSUP6J71+D3NwUAhLOIRQzm7lnnhYUv7QFv/yDZ/Lm5ubK2DVI9iZ8bR8JDtEB57lNzENQN6OjoIGlpabIVZsYaMTO+hrikRRA1JxmSX9hE7/sJtVyF38tKsUCVZxBhz9jI3wGT/QJlADzPAyXrnj0kInzGHQCRMyOg/ed2uHjxIuE4TgYQHq2DLJqumashY+lnsMC4GVC5do6XVuK9l+4SkN8y+GfYeVJn2g++U7QygPT0dBgYGIDvT58mnF5PQcjC83PzSF9fH7S1tZGEhAQZQOT8JaA317oIkM6jS8uVLSDzOQqg23Uh+MlkOf00Gg0cP34c+vv74URzM9n41gby/rvvkc7OThlATU3NCGYJUXt4QaLuTYwBcTSOBmj1RD7D4Tsix4ByOjZRF/zgupDEbgZ3j4ly/qekpND0o5aQ44HS4OAgsVqtI1gTZO01IbG0aP1bknnxCDUvArHi+B0lJSlzglTFYO2udF3Ql9TCrHn5oEIreHp6QlRUFJSUlJCqqipSWVlJ8vLyCGYIFS7HS3zGa87mv4lcjLwLlStlLTKYYUUAlvrlDGcW45wKxXX6aqHZNutM+1oQBHFTewAKkoH4+vqCj48PYAGS5yb5amjNoO+CU2SL53NKpDD0vxHHmOJir7L5xUvZgm0us2R142ScOIyVqYvlpWU4XoHIP8DXL2b+wjdWeXh6U2FjmIIKmbWAYPFRMus62h/geIvjOQYlpuDysQrLL6Ger49HgW8jqvXUhI7UvDb9iaSTDqHtyItiF5Suw5ewF/Nd8VJ6zlhsn06bEhwX4NyfCvuGEeRpTmh4mkG68yDpyuzB9EUcjU5awbAgncPlAeSdAQER0zCndzqVbeXC4qDsMpvGEYBXRnsDx4N3Auf1FCTjTIaVtY/QTmd0I8bBVm1kejEubUfO01vqImn3c49X7qpeqI9inIgtbpxK3YrKfIJCt+OeV2nfUVFR4ca4EkVENyA7gkYcMfB1R5MMmxZ7ez/2KF5SSN1yV+158UPsJT0ZBcI2bRLtIXGoYu5FerOUiJe1OfsL3XEWH43l2KS+iJF9+S4FpcNgsc+j8cT8H4o1bfPg/qkLt50uJ1RzdMsGg0UqwfEN114Pwb1CtWTGg+Y9U5ClK9x7xUWI7BI5VQVp0AVcQ3bZkQhmnEgdHhKyNSZe16crtBIlc7sIb6cRLft2PCgoKGjijBDtjrAQ7a3EdMsxzIRflAFIhPb6mHYmYwX+WBlPQgskhgVryyJCQyNyBLsBQdQ6fgsQhyt6MSOOsWZ7gbH8wETmgRKAijatNL8Ngm0xx4tLcsps0Wzx4al0jXlI40B/A3pa144MDtSgAAAAAElFTkSuQmCC [ico-downloads]: https://img.shields.io/packagist/dt/cerbero/lazy-json.svg?style=flat-square [link-author]: https://twitter.com/cerbero90 @@ -140,9 +133,9 @@ The MIT License (MIT). Please see [License File](LICENSE.md) for more informatio [link-octane]: https://github.com/laravel/octane [link-packagist]: https://packagist.org/packages/cerbero/lazy-json [link-actions]: https://github.com/cerbero90/lazy-json/actions?query=workflow%3Abuild -[link-psr7]: https://www.php-fig.org/psr/psr-7/ -[link-psr12]: https://www.php-fig.org/psr/psr-12/ +[link-per]: https://www.php-fig.org/per/coding-style/ [link-scrutinizer]: https://scrutinizer-ci.com/g/cerbero90/lazy-json/code-structure [link-code-quality]: https://scrutinizer-ci.com/g/cerbero90/lazy-json [link-downloads]: https://packagist.org/packages/cerbero/lazy-json +[link-phpstan]: https://phpstan.org/ [link-contributors]: ../../contributors diff --git a/composer.json b/composer.json index 4aee4b5..8c906a6 100644 --- a/composer.json +++ b/composer.json @@ -1,14 +1,13 @@ { "name": "cerbero/lazy-json", "type": "library", - "description": "Load heavy JSON in Laravel lazy collections.", + "description": "Framework-agnostic package to load JSONs of any dimension and from any source into Laravel lazy collections.", "keywords": [ - "laravel", "json", "lazy", "collection", "parser", - "lexer" + "laravel" ], "homepage": "https://github.com/cerbero90/lazy-json", "license": "MIT", @@ -19,19 +18,16 @@ "role": "Developer" }], "require": { - "php": "^7.2||^8.0", - "halaxa/json-machine": "^0.8|^1.0", - "illuminate/support": ">=6.0" - }, - "suggest": { - "guzzlehttp/guzzle": "Required to load JSON from endpoints (^7.0)." + "php": "^8.1", + "cerbero/json-parser": "^1.0", + "illuminate/support": ">=6.20" }, "require-dev": { - "guzzlehttp/guzzle": "^7.0", - "mockery/mockery": "^1.3.4", - "orchestra/testbench": ">=3.9", - "phpunit/phpunit": ">=8.0", - "squizlabs/php_codesniffer": "^3.0" + "pestphp/pest": "^2.0", + "phpstan/phpstan": "^1.9", + "scrutinizer/ocular": "^1.8", + "squizlabs/php_codesniffer": "^3.0", + "tightenco/duster": "^2.0" }, "autoload": { "psr-4": { @@ -47,21 +43,19 @@ } }, "scripts": { - "test": "phpunit", - "check-style": "phpcs --standard=PSR12 src tests", - "fix-style": "phpcbf --standard=PSR12 src tests" + "fix": "duster fix -u tlint,phpcodesniffer,pint", + "lint": "duster lint -u tlint,phpcodesniffer,pint,phpstan", + "test": "pest" }, "extra": { "branch-alias": { "dev-master": "1.0-dev" - }, - "laravel": { - "providers": [ - "Cerbero\\LazyJson\\Providers\\LazyJsonServiceProvider" - ] } }, "config": { - "sort-packages": true + "sort-packages": true, + "allow-plugins": { + "pestphp/pest-plugin": true + } } } diff --git a/duster.json b/duster.json new file mode 100644 index 0000000..e3834c7 --- /dev/null +++ b/duster.json @@ -0,0 +1,13 @@ +{ + "include": [ + "src" + ], + "exclude": [ + "tests" + ], + "scripts": { + "lint": { + "phpstan": ["./vendor/bin/phpstan", "analyse"] + } + } +} diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/phpstan-baseline.neon @@ -0,0 +1 @@ + diff --git a/phpstan.neon b/phpstan.neon new file mode 100644 index 0000000..5209c3e --- /dev/null +++ b/phpstan.neon @@ -0,0 +1,6 @@ +parameters: + level: max + paths: + - src +includes: + - phpstan-baseline.neon diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 917e093..990c7b1 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -1,28 +1,23 @@ - - - - tests - - - - - src/ - - - - - - - - + + + + + + + + + + + tests + + + + + + + + src/ + + diff --git a/pint.json b/pint.json new file mode 100644 index 0000000..03e092d --- /dev/null +++ b/pint.json @@ -0,0 +1,19 @@ +{ + "preset": "per", + "rules": { + "align_multiline_comment": true, + "combine_consecutive_issets": true, + "combine_consecutive_unsets": true, + "concat_space": {"spacing": "one"}, + "explicit_string_variable": true, + "ordered_imports": { + "sort_algorithm": "alpha", + "imports_order": [ + "class", + "function", + "const" + ] + }, + "simple_to_complex_string_variable": true + } +} diff --git a/src/Concerns/EndpointAware.php b/src/Concerns/EndpointAware.php deleted file mode 100644 index 69c8972..0000000 --- a/src/Concerns/EndpointAware.php +++ /dev/null @@ -1,25 +0,0 @@ -isEndpoint($source); - } - - /** - * Handle the given source - * - * @param mixed $source - * @param string $path - * @return Traversable - */ - public function handle($source, string $path): Traversable - { - if (!$this->guzzleIsLoaded()) { - throw new LazyJsonException('Guzzle is required to load JSON from endpoints'); - } - - $response = (new Client())->get($source, [ - 'headers' => [ - 'Accept' => 'application/json', - 'Content-Type' => 'application/json', - ], - ]); - - return parent::handle($response, $path); - } - - /** - * Determine whether Guzzle is loaded, useful for testing - * - * @return bool - */ - protected function guzzleIsLoaded(): bool - { - return class_exists(Client::class); - } -} diff --git a/src/Handlers/Filename.php b/src/Handlers/Filename.php deleted file mode 100644 index 268d965..0000000 --- a/src/Handlers/Filename.php +++ /dev/null @@ -1,39 +0,0 @@ - $this->toJsonPointer($path)]); - } -} diff --git a/src/Handlers/Handler.php b/src/Handlers/Handler.php deleted file mode 100644 index 59a9966..0000000 --- a/src/Handlers/Handler.php +++ /dev/null @@ -1,29 +0,0 @@ - $this->toJsonPointer($path)]); - } -} diff --git a/src/Handlers/JsonString.php b/src/Handlers/JsonString.php deleted file mode 100644 index 93202ba..0000000 --- a/src/Handlers/JsonString.php +++ /dev/null @@ -1,41 +0,0 @@ -isEndpoint($source); - } - - /** - * Handle the given source - * - * @param mixed $source - * @param string $path - * @return Traversable - */ - public function handle($source, string $path): Traversable - { - return Items::fromString($source, ['pointer' => $this->toJsonPointer($path)]); - } -} diff --git a/src/Handlers/LaravelClientResponse.php b/src/Handlers/LaravelClientResponse.php deleted file mode 100644 index 34fa6dd..0000000 --- a/src/Handlers/LaravelClientResponse.php +++ /dev/null @@ -1,36 +0,0 @@ -toPsrResponse(), $path); - } -} diff --git a/src/Handlers/Psr7Message.php b/src/Handlers/Psr7Message.php deleted file mode 100644 index 8d67a28..0000000 --- a/src/Handlers/Psr7Message.php +++ /dev/null @@ -1,36 +0,0 @@ -getBody(), $path); - } -} diff --git a/src/Handlers/Psr7Stream.php b/src/Handlers/Psr7Stream.php deleted file mode 100644 index 7ce57ea..0000000 --- a/src/Handlers/Psr7Stream.php +++ /dev/null @@ -1,45 +0,0 @@ - ['stream' => $source], - ])); - - return parent::handle($stream, $path); - } -} diff --git a/src/Handlers/Resource.php b/src/Handlers/Resource.php deleted file mode 100644 index 6040b0b..0000000 --- a/src/Handlers/Resource.php +++ /dev/null @@ -1,39 +0,0 @@ - $this->toJsonPointer($path)]); - } -} diff --git a/src/Macro.php b/src/Macro.php deleted file mode 100644 index 0470b3a..0000000 --- a/src/Macro.php +++ /dev/null @@ -1,32 +0,0 @@ -getMessage(), 0, $e); - } - }); - } -} diff --git a/src/Providers/LazyJsonServiceProvider.php b/src/Providers/LazyJsonServiceProvider.php deleted file mode 100644 index 5fea13e..0000000 --- a/src/Providers/LazyJsonServiceProvider.php +++ /dev/null @@ -1,24 +0,0 @@ -traversable = $this->toTraversable($source, $path); - } - - /** - * Turn the given JSON source into a traversable instance - * - * @param mixed $source - * @param string $path - * @return Traversable - * - * @throws LazyJsonException - */ - protected function toTraversable($source, string $path): Traversable - { - foreach ($this->handlers as $class) { - /** @var Handlers\Handler $handler */ - $handler = new $class(); - - if ($handler->handles($source)) { - return $handler->handle($source, $path); - } - } - - throw new LazyJsonException('Unable to load the JSON from the provided source.'); - } - - /** - * Retrieve the traversable JSON - * - * @return Traversable - */ - public function getIterator(): Traversable - { - return $this->traversable; - } -} diff --git a/src/StreamWrapper.php b/src/StreamWrapper.php deleted file mode 100644 index 0027c0f..0000000 --- a/src/StreamWrapper.php +++ /dev/null @@ -1,75 +0,0 @@ -context); - - $this->stream = $options[static::NAME]['stream'] ?? null; - - return $this->stream instanceof StreamInterface && $this->stream->isReadable(); - } - - /** - * Determine whether the pointer is at the end of the stream - * - * @return bool - */ - public function stream_eof(): bool - { - return $this->stream->eof(); - } - - /** - * Read from the stream - * - * @param int $count - * @return string - */ - public function stream_read(int $count): string - { - return $this->stream->read($count); - } -} diff --git a/tests/Handlers/EndpointTest.php b/tests/Handlers/EndpointTest.php deleted file mode 100644 index ca4eab9..0000000 --- a/tests/Handlers/EndpointTest.php +++ /dev/null @@ -1,101 +0,0 @@ -assertTrue($handler->handles('http://endpoint.test')); - $this->assertTrue($handler->handles('https://endpoint.test')); - $this->assertFalse($handler->handles(123)); - $this->assertFalse($handler->handles('http://')); - $this->assertFalse($handler->handles('https://')); - $this->assertFalse($handler->handles('ftp://foo.test')); - } - - /** - * @test - */ - public function handleEndpoint() - { - m::mock('overload:' . Client::class, [ - 'get' => new Psr7Response(200, [], '{"end":"point"}'), - ]); - - $handled = (new Endpoint())->handle('https://endpoint.test', ''); - - $this->assertInstanceOf(Items::class, $handled); - - foreach ($handled as $key => $value) { - $this->assertSame('end', $key); - $this->assertSame('point', $value); - } - } - - /** - * @test - */ - public function extractsJsonSubtrees() - { - m::mock('overload:' . Client::class, [ - 'get' => new Psr7Response(200, [], '{"foo":{"bar":1,"baz":2}}'), - ]); - - $handled = (new Endpoint())->handle('https://endpoint.test', 'foo.bar'); - - $this->assertInstanceOf(Items::class, $handled); - - foreach ($handled as $key => $value) { - $this->assertSame('bar', $key); - $this->assertSame(1, $value); - } - } - - /** - * @test - */ - public function failsIfGuzzleIsNotLoaded() - { - $double = m::mock(Endpoint::class) - ->makePartial() - ->shouldAllowMockingProtectedMethods() - ->shouldReceive('guzzleIsLoaded') - ->once() - ->andReturn(false) - ->getMock(); - - $this->expectExceptionObject(new LazyJsonException('Guzzle is required to load JSON from endpoints')); - - $double->handle('https://endpoint.test', 'foo.bar'); - } -} diff --git a/tests/LazyJsonTest.php b/tests/LazyJsonTest.php deleted file mode 100644 index 3827869..0000000 --- a/tests/LazyJsonTest.php +++ /dev/null @@ -1,144 +0,0 @@ -each(function ($value, $key) { - $this->assertSame('key', $key); - $this->assertSame('JSON file value', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromIterable() - { - lazyJson(['{"foo":"bar"}'])->each(function ($value, $key) { - $this->assertSame('foo', $key); - $this->assertSame('bar', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromString() - { - lazyJson('{"bar":"baz"}')->each(function ($value, $key) { - $this->assertSame('bar', $key); - $this->assertSame('baz', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromLaravelClientResponse() - { - if (!class_exists(Response::class)) { - $this->markTestSkipped('The Laravel HTTP client is not loaded.'); - } - - $response = new Response(new Psr7Response(200, [], '{"status":"success"}')); - - lazyJson($response)->each(function ($value, $key) { - $this->assertSame('status', $key); - $this->assertSame('success', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromPsr7Message() - { - $response = new Psr7Response(200, [], '{"one":"two"}'); - - lazyJson($response)->each(function ($value, $key) { - $this->assertSame('one', $key); - $this->assertSame('two', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromPsr7Stream() - { - $stream = new Stream(fopen(__DIR__ . '/stub.json', 'rb')); - - lazyJson($stream)->each(function ($value, $key) { - $this->assertSame('key', $key); - $this->assertSame('JSON file value', $value); - }); - } - - /** - * @test - */ - public function lazyLoadsJsonFromResource() - { - $resource = fopen(__DIR__ . '/stub.json', 'rb'); - - lazyJson($resource)->each(function ($value, $key) { - $this->assertSame('key', $key); - $this->assertSame('JSON file value', $value); - }); - } - - /** - * @test - */ - public function failsWithInvalidJsonSource() - { - $this->expectExceptionObject(new LazyJsonException('Unable to load the JSON from the provided source.')); - - lazyJson(123)->all(); - } - - /** - * @test - */ - public function trowsPackageExceptionWhenAnyExceptionOccursDuringJsonLoading() - { - try { - lazyJson('{}}')->all(); - } catch (Throwable $e) { - $this->assertInstanceOf(LazyJsonException::class, $e); - $this->assertSame($e->getPrevious()->getMessage(), $e->getMessage()); - } - } -} diff --git a/tests/StreamWrapperTest.php b/tests/StreamWrapperTest.php deleted file mode 100644 index 51f3b51..0000000 --- a/tests/StreamWrapperTest.php +++ /dev/null @@ -1,126 +0,0 @@ - true, - ]); - - $resource = $this->openStreamWith($double); - - $this->assertTrue(is_resource($resource)); - } - - /** - * Open the stream with the given wrapper - * - * @param mixed $stream - * @return resource|bool - */ - protected function openStreamWith($stream) - { - return @fopen(StreamWrapper::NAME . '://stream', 'rb', false, stream_context_create([ - StreamWrapper::NAME => compact('stream'), - ])); - } - - /** - * @test - */ - public function cannotOpenInvalidStream() - { - $bool = $this->openStreamWith(new \stdClass()); - - $this->assertFalse($bool); - } - - /** - * @test - */ - public function cannotOpenUnreadableStream() - { - $double = m::mock(StreamInterface::class, [ - 'isReadable' => false, - ]); - - $bool = $this->openStreamWith($double); - - $this->assertFalse($bool); - } - - /** - * @test - */ - public function canReadEof() - { - $double = m::mock(StreamInterface::class, [ - 'isReadable' => true, - 'eof' => true, - ]); - - $resource = $this->openStreamWith($double); - - $this->assertTrue(is_resource($resource)); - $this->assertTrue(feof($resource)); - } - - /** - * @test - */ - public function canRead() - { - $double = m::mock(StreamInterface::class, [ - 'isReadable' => true, - 'eof' => true, - ]) - ->shouldReceive('read') - ->andReturn('abc') - ->getMock(); - - $resource = $this->openStreamWith($double); - - $this->assertTrue(is_resource($resource)); - $this->assertSame('abc', fread($resource, 3)); - } -} diff --git a/tests/stub.json b/tests/stub.json deleted file mode 100644 index 7c13e8a..0000000 --- a/tests/stub.json +++ /dev/null @@ -1 +0,0 @@ -{"key":"JSON file value"} From 710c01239fbcb5f5a238c3ac3fe2f52b94eb46b5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 29 Jun 2023 15:04:44 +0200 Subject: [PATCH 04/33] feat: create source --- src/Sources/Source.php | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/Sources/Source.php diff --git a/src/Sources/Source.php b/src/Sources/Source.php new file mode 100644 index 0000000..fea891c --- /dev/null +++ b/src/Sources/Source.php @@ -0,0 +1,10 @@ + Date: Thu, 29 Jun 2023 15:05:03 +0200 Subject: [PATCH 05/33] feat: create exception --- src/Exceptions/LazyJsonException.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/Exceptions/LazyJsonException.php diff --git a/src/Exceptions/LazyJsonException.php b/src/Exceptions/LazyJsonException.php new file mode 100644 index 0000000..d4218da --- /dev/null +++ b/src/Exceptions/LazyJsonException.php @@ -0,0 +1,12 @@ + Date: Thu, 29 Jun 2023 15:06:03 +0200 Subject: [PATCH 06/33] feat: autoload lazy collection macro --- bootstrap.php | 8 ++++++++ composer.json | 1 + src/Macro.php | 24 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 bootstrap.php create mode 100644 src/Macro.php diff --git a/bootstrap.php b/bootstrap.php new file mode 100644 index 0000000..8e423c7 --- /dev/null +++ b/bootstrap.php @@ -0,0 +1,8 @@ +getMessage()); + } + }); + } +} From 046e8c93f15ad9d48eb9d97b0bcf4bab5dfbfe14 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 29 Jun 2023 15:06:13 +0200 Subject: [PATCH 07/33] test: macro autoload --- tests/Feature/InstanceTest.php | 7 ++++++ tests/Pest.php | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/Feature/InstanceTest.php create mode 100644 tests/Pest.php diff --git a/tests/Feature/InstanceTest.php b/tests/Feature/InstanceTest.php new file mode 100644 index 0000000..18159d9 --- /dev/null +++ b/tests/Feature/InstanceTest.php @@ -0,0 +1,7 @@ +toBeInstanceOf(LazyCollection::class); +}); diff --git a/tests/Pest.php b/tests/Pest.php new file mode 100644 index 0000000..baddc84 --- /dev/null +++ b/tests/Pest.php @@ -0,0 +1,45 @@ +in('Feature'); + +/* +|-------------------------------------------------------------------------- +| Expectations +|-------------------------------------------------------------------------- +| +| When you're writing tests, you often need to check that values meet certain conditions. The +| "expect()" function gives you access to a set of "expectations" methods that you can use +| to assert different things. Of course, you may extend the Expectation API at any time. +| +*/ + +// expect()->extend('toBeOne', function () { +// return $this->toBe(1); +// }); + +/* +|-------------------------------------------------------------------------- +| Functions +|-------------------------------------------------------------------------- +| +| While Pest is very powerful out-of-the-box, you may have some testing code specific to your +| project that you don't want to repeat in every file. Here you can also expose helpers as +| global functions to help you to reduce the number of lines of code in your test files. +| +*/ + +// function something() +// { +// // .. +// } From 3a8b2382f4b4a2e7ac6e87904e6ff48258a5a392 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:14:10 +0200 Subject: [PATCH 08/33] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7ae6add..bd2b70e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ vendor phpcs.xml phpunit.xml .phpunit.result.cache +.DS_Store From 6ef80139b4f105d6ab09a828a7c036b4122e2d19 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:18:27 +0200 Subject: [PATCH 09/33] Update dependencies --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index fef19ad..db09fff 100644 --- a/composer.json +++ b/composer.json @@ -19,8 +19,8 @@ }], "require": { "php": "^8.1", - "cerbero/json-parser": "^1.0", - "illuminate/support": ">=6.20" + "cerbero/json-parser": "^1.1", + "illuminate/collections": ">=8.12" }, "require-dev": { "pestphp/pest": "^2.0", From 4a9f8c5322aa9c9c194e77948fa71f155ed0b1dc Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:20:31 +0200 Subject: [PATCH 10/33] Provide different ways to use Lazy JSON --- bootstrap.php | 4 ++-- helpers.php | 20 +++++++--------- src/LazyJson.php | 54 ++++++++++++++++++++++++++++++++++++++++++ src/Macro.php | 24 ------------------- src/Sources/Source.php | 10 -------- 5 files changed, 64 insertions(+), 48 deletions(-) create mode 100644 src/LazyJson.php delete mode 100644 src/Macro.php delete mode 100644 src/Sources/Source.php diff --git a/bootstrap.php b/bootstrap.php index 8e423c7..3893e29 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -1,8 +1,8 @@ $dot + */ +function lazyJson(mixed $source, string|array $dot = ''): LazyCollection +{ + return LazyJson::from($source, $dot); } diff --git a/src/LazyJson.php b/src/LazyJson.php new file mode 100644 index 0000000..3019baa --- /dev/null +++ b/src/LazyJson.php @@ -0,0 +1,54 @@ + + */ +final class LazyJson implements IteratorAggregate +{ + private JsonParser $parser; + + /** + * @param string|string[]|array $dot + * @return LazyCollection + */ + public static function from(mixed $source, string|array $dot = '*'): LazyCollection + { + return new LazyCollection(fn () => yield from new self($source, (array) $dot)); + } + + /** + * @param string[]|array $dots + */ + private function __construct(mixed $source, array $dots) + { + $this->parser = JsonParser::parse($source) + ->lazyPointers(DotsConverter::toPointers($dots)) + ->wrap(fn (Parser $parser) => new LazyCollection(fn () => yield from $parser)); + } + + /** + * @return Traversable + */ + public function getIterator(): Traversable + { + try { + yield from $this->parser; + } catch (Throwable $e) { + throw new LazyJsonException($e); + } + } +} diff --git a/src/Macro.php b/src/Macro.php deleted file mode 100644 index 4233d60..0000000 --- a/src/Macro.php +++ /dev/null @@ -1,24 +0,0 @@ -getMessage()); - } - }); - } -} diff --git a/src/Sources/Source.php b/src/Sources/Source.php deleted file mode 100644 index fea891c..0000000 --- a/src/Sources/Source.php +++ /dev/null @@ -1,10 +0,0 @@ - Date: Mon, 7 Aug 2023 20:21:09 +0200 Subject: [PATCH 11/33] Wrap thrown exceptions --- src/Exceptions/LazyJsonException.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Exceptions/LazyJsonException.php b/src/Exceptions/LazyJsonException.php index d4218da..1682ffa 100644 --- a/src/Exceptions/LazyJsonException.php +++ b/src/Exceptions/LazyJsonException.php @@ -5,8 +5,12 @@ namespace Cerbero\LazyJson\Exceptions; use Exception; +use Throwable; final class LazyJsonException extends Exception { - // + public function __construct(public readonly Throwable $exception) + { + parent::__construct($exception->getMessage()); + } } From 1b3a3464b4dc5a578e9c2ba7ed100341d7e28cb2 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:21:41 +0200 Subject: [PATCH 12/33] Create class to turn dots into JSON pointers --- src/Pointers/DotsConverter.php | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/Pointers/DotsConverter.php diff --git a/src/Pointers/DotsConverter.php b/src/Pointers/DotsConverter.php new file mode 100644 index 0000000..8d8f7fa --- /dev/null +++ b/src/Pointers/DotsConverter.php @@ -0,0 +1,37 @@ + $dots + * @return string[]|array + */ + public static function toPointers(array $dots): array + { + $pointers = []; + + foreach ($dots as $dot => $callback) { + if ($callback instanceof Closure) { + $pointers[self::toPointer($dot)] = $callback; + } else { + $pointers[] = self::toPointer($callback); + } + } + + return $pointers; + } + + public static function toPointer(string $dot): string + { + $search = ['~', '/', '.', '*', '\\', '"']; + $replace = ['~0', '~1', '/', '-', '\\\\', '\"']; + + return $dot == '*' ? '' : '/' . str_replace($search, $replace, $dot); + } +} From fe3620a622c9c7c535a8fb177da876ac8d0266e7 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:22:06 +0200 Subject: [PATCH 13/33] Add helpers and expectations --- tests/Pest.php | 103 ++++++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 43 deletions(-) diff --git a/tests/Pest.php b/tests/Pest.php index baddc84..835fd95 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,45 +1,62 @@ in('Feature'); - -/* -|-------------------------------------------------------------------------- -| Expectations -|-------------------------------------------------------------------------- -| -| When you're writing tests, you often need to check that values meet certain conditions. The -| "expect()" function gives you access to a set of "expectations" methods that you can use -| to assert different things. Of course, you may extend the Expectation API at any time. -| -*/ - -// expect()->extend('toBeOne', function () { -// return $this->toBe(1); -// }); - -/* -|-------------------------------------------------------------------------- -| Functions -|-------------------------------------------------------------------------- -| -| While Pest is very powerful out-of-the-box, you may have some testing code specific to your -| project that you don't want to repeat in every file. Here you can also expose helpers as -| global functions to help you to reduce the number of lines of code in your test files. -| -*/ - -// function something() -// { -// // .. -// } +use Cerbero\JsonParser\Tokens\Parser; +use Illuminate\Support\LazyCollection; +use Pest\Expectation; + +if (!function_exists('fixture')) { + function fixture(string $fixture): string + { + return __DIR__ . "/fixtures/{$fixture}"; + } +} + +/** + * Expect the given sequence from a Traversable + * Temporary fix to sequence() until this PR is merged: https://github.com/pestphp/pest/pull/895 + * + * @param mixed ...$callbacks + * @return Expectation + */ +expect()->extend('traverse', function (mixed ...$callbacks) { + if (! is_iterable($this->value)) { + throw new BadMethodCallException('Expectation value is not iterable.'); + } + + if (empty($callbacks)) { + throw new InvalidArgumentException('No sequence expectations defined.'); + } + + $index = $valuesCount = 0; + + foreach ($this->value as $key => $value) { + $valuesCount++; + + if ($callbacks[$index] instanceof Closure) { + $callbacks[$index](new self($value), new self($key)); + } else { + (new self($value))->toEqual($callbacks[$index]); + } + + $index = isset($callbacks[$index + 1]) ? $index + 1 : 0; + } + + if (count($callbacks) > $valuesCount) { + throw new OutOfRangeException('Sequence expectations are more than the iterable items'); + } + + return $this; +}); + +/** + * Expect that all Parser instances are wrapped recursively into lazy collections + * + * @return Expectation + */ +expect()->extend('toBeWrappedIntoLazyCollection', function () { + return $this->when(is_object($this->value), fn (Expectation $value) => $value + ->toBeInstanceOf(LazyCollection::class) + ->not->toBeInstanceOf(Parser::class) + ->traverse(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()) + ); +}); From 3ed8ef219feeb2ca3d7c3aaedb0b51508e0b0bea Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:22:30 +0200 Subject: [PATCH 14/33] Update tests --- tests/Feature/InstanceTest.php | 7 ------ tests/Feature/LazyJsonTest.php | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) delete mode 100644 tests/Feature/InstanceTest.php create mode 100644 tests/Feature/LazyJsonTest.php diff --git a/tests/Feature/InstanceTest.php b/tests/Feature/InstanceTest.php deleted file mode 100644 index 18159d9..0000000 --- a/tests/Feature/InstanceTest.php +++ /dev/null @@ -1,7 +0,0 @@ -toBeInstanceOf(LazyCollection::class); -}); diff --git a/tests/Feature/LazyJsonTest.php b/tests/Feature/LazyJsonTest.php new file mode 100644 index 0000000..25bad36 --- /dev/null +++ b/tests/Feature/LazyJsonTest.php @@ -0,0 +1,45 @@ +toBeInstanceOf(LazyCollection::class); +}); + +it('can be used via static method', function () { + expect(LazyJson::from('{"foo":123}'))->toBeInstanceOf(LazyCollection::class); +}); + +it('can be used via namespaced helper', function () { + expect(lazyJson('{"foo":123}'))->toBeInstanceOf(LazyCollection::class); +}); + +it('wraps the thrown exception when an error occurs', function () { + expect(fn () => LazyJson::from('/foo')->all())->toThrow(fn (LazyJsonException $e) => expect($e) + ->getMessage()->toBe("Syntax error: unexpected '/' at position 0") + ->exception->toBeInstanceOf(SyntaxException::class) + ); +}); + +it('iterates through keys and values', function () { + expect(LazyJson::from('{"foo":123,"bar":321}'))->sequence( + fn (Expectation $value, Expectation $key) => $key->toBe('foo')->and($value)->toBe(123), + fn (Expectation $value, Expectation $key) => $key->toBe('bar')->and($value)->toBe(321), + ); +}); + +it('wraps JSON objects and arrays into lazy collections', function () { + expect(LazyJson::from('{"foo":{"one":1,"two":2},"bar":[3,4]}')) + ->traverse(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()); +}); + +// it('sets a JSON pointer by using the dot notation syntax', function (string $source, string|int $dot, mixed $sequence) { +// expect(LazyJson::from($source, (string) $dot))->sequence($sequence); +// })->with(Dataset::forDots()); From 2dcda1936f5bfbbef3c6f4c520b586e4cd319c5d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 20:22:42 +0200 Subject: [PATCH 15/33] Introduce fixtures and dataset --- tests/Dataset.php | 43 +++++ tests/fixtures/complex_array.json | 70 +++++++++ tests/fixtures/complex_array.php | 132 ++++++++++++++++ tests/fixtures/complex_object.json | 26 +++ tests/fixtures/complex_object.php | 58 +++++++ tests/fixtures/empty_array.json | 1 + tests/fixtures/empty_array.php | 3 + tests/fixtures/empty_object.json | 1 + tests/fixtures/empty_object.php | 3 + tests/fixtures/simple_array.json | 1 + tests/fixtures/simple_array.php | 3 + tests/fixtures/simple_object.json | 22 +++ tests/fixtures/simple_object.php | 26 +++ tests/fixtures/single_dot.php | 244 +++++++++++++++++++++++++++++ 14 files changed, 633 insertions(+) create mode 100644 tests/Dataset.php create mode 100644 tests/fixtures/complex_array.json create mode 100644 tests/fixtures/complex_array.php create mode 100644 tests/fixtures/complex_object.json create mode 100644 tests/fixtures/complex_object.php create mode 100644 tests/fixtures/empty_array.json create mode 100644 tests/fixtures/empty_array.php create mode 100644 tests/fixtures/empty_object.json create mode 100644 tests/fixtures/empty_object.php create mode 100644 tests/fixtures/simple_array.json create mode 100644 tests/fixtures/simple_array.php create mode 100644 tests/fixtures/simple_object.json create mode 100644 tests/fixtures/simple_object.php create mode 100644 tests/fixtures/single_dot.php diff --git a/tests/Dataset.php b/tests/Dataset.php new file mode 100644 index 0000000..402f788 --- /dev/null +++ b/tests/Dataset.php @@ -0,0 +1,43 @@ + $value) { + yield [$source, $key, $value]; + } + + $singleDot = require fixture('single_dot.php'); + + foreach ($singleDot as $fixture => $subtreeByDot) { + $source = fixture("{$fixture}.json"); + + foreach ($subtreeByDot as $dot => $subtree) { + $values = (array) reset($subtree); + + foreach ($values as $expected) { + yield [ + $source, + $dot, + fn ($value, $key) => $key->toBe(key($subtree)) + ->and($value) + ->when(is_array($expected), fn ($value) => $value->toBeInstanceOf(LazyCollection::class)) + ->and(is_array($expected) ? $value->value->toArray() : $value) + ->toBe($expected), + ]; + } + } + } + } +} diff --git a/tests/fixtures/complex_array.json b/tests/fixtures/complex_array.json new file mode 100644 index 0000000..70ddac0 --- /dev/null +++ b/tests/fixtures/complex_array.json @@ -0,0 +1,70 @@ +[ + { + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" }, + { "id": "1003", "type": "Blueberry" }, + { "id": "1004", "type": "Devil's Food" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5007", "type": "Powdered Sugar" }, + { "id": "5006", "type": "Chocolate with Sprinkles" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + }, + { + "id": "0002", + "type": "donut", + "name": "Raised", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + }, + { + "id": "0003", + "type": "donut", + "name": "Old Fashioned", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + } +] diff --git a/tests/fixtures/complex_array.php b/tests/fixtures/complex_array.php new file mode 100644 index 0000000..3d4875f --- /dev/null +++ b/tests/fixtures/complex_array.php @@ -0,0 +1,132 @@ + "0001", + "type" => "donut", + "name" => "Cake", + "ppu" => 0.55, + "batters" => [ + "batter" => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + "topping" => [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5005", + "type" => "Sugar", + ], + [ + "id" => "5007", + "type" => "Powdered Sugar", + ], + [ + "id" => "5006", + "type" => "Chocolate with Sprinkles", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + ], + [ + "id" => "0002", + "type" => "donut", + "name" => "Raised", + "ppu" => 0.55, + "batters" => [ + "batter" => [ + [ + "id" => "1001", + "type" => "Regular", + ], + ], + ], + "topping" => [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5005", + "type" => "Sugar", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + ], + [ + "id" => "0003", + "type" => "donut", + "name" => "Old Fashioned", + "ppu" => 0.55, + "batters" => [ + "batter" => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + ], + ], + "topping" => [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + ], +]; diff --git a/tests/fixtures/complex_object.json b/tests/fixtures/complex_object.json new file mode 100644 index 0000000..ef2b654 --- /dev/null +++ b/tests/fixtures/complex_object.json @@ -0,0 +1,26 @@ +{ + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" }, + { "id": "1003", "type": "Blueberry" }, + { "id": "1004", "type": "Devil's Food" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5007", "type": "Powdered Sugar" }, + { "id": "5006", "type": "Chocolate with Sprinkles" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] +} diff --git a/tests/fixtures/complex_object.php b/tests/fixtures/complex_object.php new file mode 100644 index 0000000..06b2a6d --- /dev/null +++ b/tests/fixtures/complex_object.php @@ -0,0 +1,58 @@ + '0001', + 'type' => 'donut', + 'name' => 'Cake', + 'ppu' => 0.55, + 'batters' => [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', + ], + [ + 'id' => '1003', + 'type' => 'Blueberry', + ], + [ + 'id' => '1004', + 'type' => 'Devil\'s Food', + ], + ], + ], + 'topping' => [ + [ + 'id' => '5001', + 'type' => 'None', + ], + [ + 'id' => '5002', + 'type' => 'Glazed', + ], + [ + 'id' => '5005', + 'type' => 'Sugar', + ], + [ + 'id' => '5007', + 'type' => 'Powdered Sugar', + ], + [ + 'id' => '5006', + 'type' => 'Chocolate with Sprinkles', + ], + [ + 'id' => '5003', + 'type' => 'Chocolate', + ], + [ + 'id' => '5004', + 'type' => 'Maple', + ], + ], +]; diff --git a/tests/fixtures/empty_array.json b/tests/fixtures/empty_array.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/fixtures/empty_array.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/empty_array.php b/tests/fixtures/empty_array.php new file mode 100644 index 0000000..0b67a5f --- /dev/null +++ b/tests/fixtures/empty_array.php @@ -0,0 +1,3 @@ + 1, + 'empty_string' => '', + 'string' => 'foo', + 'escaped_string' => '"bar"', + '"escaped_key"' => 'baz', + "unicode" => "hej då", + 'float' => 3.14, + 'bool' => false, + 'null' => null, + 'empty_array' => new LazyCollection(function () {}), + 'empty_object' => new LazyCollection(function () {}), + '' => 0, + 'a/b' => 1, + 'c%d' => 2, + 'e^f' => 3, + 'g|h' => 4, + 'i\\j' => 5, + 'k"l' => 6, + ' ' => 7, + 'm~n' => 8 +]; diff --git a/tests/fixtures/single_dot.php b/tests/fixtures/single_dot.php new file mode 100644 index 0000000..3fd06ea --- /dev/null +++ b/tests/fixtures/single_dot.php @@ -0,0 +1,244 @@ + [ + '*' => require fixture('complex_array.php'), + '*.id' => ['id' => ['0001', '0002', '0003']], + '*.batters' => [ + 'batters' => [ + [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + ], + ], + [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + ], + ], + ], + ], + '*.batters.batter' => [ + 'batter' => [ + [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + [ + [ + "id" => "1001", + "type" => "Regular", + ], + ], + [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + ], + ], + ], + '*.batters.batter.*' => [ + 0 => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1001", + "type" => "Regular", + ], + ], + 1 => [ + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + ], + 2 => [ + "id" => "1003", + "type" => "Blueberry", + ], + 3 => [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + '*.batters.batter.*.id' => ['id' => ["1001", "1002", "1003", "1004", "1001", "1001", "1002"]], + ], + 'complex_object' => [ + '*' => require fixture('complex_object.php'), + 'id' => ['id' => '0001'], + 'batters' => [ + 'batters' => [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + ], + 'batters.batter' => [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + 'batters.batter.*' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + 'batters.batter.*.id' => ['id' => ["1001", "1002", "1003", "1004"]], + ], + 'empty_array' => [ + '*' => [], + '-1' => [], + '0' => [], + 'foo' => [], + ], + 'empty_object' => [ + '*' => [], + '-1' => [], + '0' => [], + 'foo' => [], + ], + 'simple_array' => [ + '*' => require fixture('simple_array.php'), + '-1' => [], + '0' => [0 => 1], + '1' => [1 => ''], + '2' => [2 => 'foo'], + '3' => [3 => '"bar"'], + '4' => [4 => 'hej då'], + '5' => [5 => 3.14], + '6' => [6 => false], + '7' => [7 => null], + '8' => [8 => []], + '9' => [9 => []], + '10' => [], + 'foo' => [], + ], + 'simple_object' => [ + '*' => require fixture('simple_object.php'), + '-1' => [], + 'int' => ['int' => 1], + 'empty_string' => ['empty_string' => ''], + 'string' => ['string' => 'foo'], + 'escaped_string' => ['escaped_string' => '"bar"'], + '"escaped_key"' => ['"escaped_key"' => 'baz'], + 'unicode' => ['unicode' => "hej då"], + 'float' => ['float' => 3.14], + 'bool' => ['bool' => false], + 'null' => ['null' => null], + 'empty_array' => ['empty_array' => []], + 'empty_object' => ['empty_object' => []], + '10' => [], + 'foo' => [], + '' => ['' => 0], + 'a/b' => ['a/b' => 1], + 'c%d' => ['c%d' => 2], + 'e^f' => ['e^f' => 3], + 'g|h' => ['g|h' => 4], + 'i\\j' => ['i\\j' => 5], + 'k"l' => ['k"l' => 6], + ' ' => [' ' => 7], + 'm~n' => ['m~n' => 8], + ], +]; From 3733b90f85f8f2b6289f14adcc9fcc3efb9401c8 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 7 Aug 2023 21:07:31 +0200 Subject: [PATCH 16/33] Update readme --- README.md | 83 +++++++++++++++++++------------------------------------ 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 655259c..3bafb01 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ [![Author][ico-author]][link-author] [![PHP Version][ico-php]][link-php] -[![Laravel Version][ico-laravel]][link-laravel] -[![Octane Compatibility][ico-octane]][link-octane] [![Build Status][ico-actions]][link-actions] [![Coverage Status][ico-scrutinizer]][link-scrutinizer] [![Quality Score][ico-code-quality]][link-code-quality] @@ -15,7 +13,11 @@ Framework-agnostic package to load JSONs of any dimension and from any source into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). -Under the hood, [🧩 JSON Parser](https://github.com/cerbero90/json-parser) is used to parse and extract sub-trees from any JSON. +Lazy JSON recursively turns any JSON array and object into a lazy collection, consuming only a few KB of memory while parsing JSON of any size. + +It optionally allows to extract only some sub-trees, instead of the whole JSON, with a simple dot-notation syntax. + +Under the hood, [🧩 JSON Parser](https://github.com/cerbero90/json-parser) is used to parse any JSON and extract sub-trees. Need to lazy load items from paginated JSON APIs? Consider using [🐼 Lazy JSON Pages](https://github.com/cerbero90/lazy-json-pages) instead. @@ -30,62 +32,39 @@ composer require cerbero/lazy-json ## 🔮 Usage -Loading JSON in lazy collections is possible by using the collection itself or the included helper: +Depending on our code style, we can call Lazy JSON in 3 different ways: ```php -LazyCollection::fromJson($source); +use Cerbero\LazyJson\LazyJson; +use Illuminate\Support\LazyCollection; +use function Cerbero\LazyJson\lazyJson; -lazyJson($source); -``` +// auto-registered lazy collection macro +$lazyCollection = LazyCollection::fromJson($source); -The following are the supported JSON sources: +// static method +$lazyCollection = LazyJson::from($source); -```php -$source = '{"foo":"bar"}'; // JSON string -$source = ['{"foo":"bar"}']; // any iterable containing JSON, i.e. array or Traversable -$source = 'https://foo.test/endpoint'; // endpoint -$source = Http::get('https://foo.test/endpoint'); // Laravel HTTP client response -$source = '/path/to/file.json'; // JSON file -$source = fopen('/path/to/file.json', 'rb'); // any resource -$source = ; // any PSR-7 message, e.g. Guzzle response -$source = ; // any PSR-7 stream +// namespaced helper +$lazyCollection = lazyJson($source); ``` -Optionally, you can define a dot-noted path to extract only a sub-tree of the JSON. For example, given the following JSON: - -```json -{ - "data": [ - { - "name": "Team 1", - "users": [ - { - "id": 1 - }, - { - "id": 2 - } - ] - }, - { - "name": "Team 2", - "users": [ - { - "id": 3 - } - ] - } - ] -} -``` +The variable `$source` in the above examples indicates any data point that provides a JSON. A wide range of sources are supported by default: +- **strings**, e.g. `{"foo":"bar"}` +- **iterables**, i.e. arrays or instances of `Traversable` +- **file paths**, e.g. `/path/to/large.json` +- **resources**, e.g. streams +- **API endpoint URLs**, e.g. `https://endpoint.json` or any instance of `Psr\Http\Message\UriInterface` +- **PSR-7 requests**, i.e. any instance of `Psr\Http\Message\RequestInterface` +- **PSR-7 messages**, i.e. any instance of `Psr\Http\Message\MessageInterface` +- **PSR-7 streams**, i.e. any instance of `Psr\Http\Message\StreamInterface` +- **Laravel HTTP client requests**, i.e. any instance of `Illuminate\Http\Client\Request` +- **Laravel HTTP client responses**, i.e. any instance of `Illuminate\Http\Client\Response` +- **user-defined sources**, i.e. any instance of `Cerbero\JsonParser\Sources\Source` -defining the path `data.*.users.*.id` would iterate only user IDs: +For more information about JSON sources, please consult the [🧩 JSON Parser documentation](https://github.com/cerbero90/json-parser). -```php -$ids = lazyJson($source, 'data.*.users.*.id') - ->filter(fn ($id) => $id % 2 == 0) - ->all(); -``` +// @todo: complete README ## 📆 Change log @@ -116,8 +95,6 @@ The MIT License (MIT). Please see [License File](LICENSE.md) for more informatio [ico-author]: https://img.shields.io/static/v1?label=author&message=cerbero90&color=50ABF1&logo=twitter&style=flat-square [ico-php]: https://img.shields.io/packagist/php-v/cerbero/lazy-json?color=%234F5B93&logo=php&style=flat-square -[ico-laravel]: https://img.shields.io/static/v1?label=laravel&message=%E2%89%A56.20&color=ff2d20&logo=laravel&style=flat-square -[ico-octane]: https://img.shields.io/static/v1?label=octane&message=compatible&color=ff2d20&logo=laravel&style=flat-square [ico-version]: https://img.shields.io/packagist/v/cerbero/lazy-json.svg?label=version&style=flat-square [ico-actions]: https://img.shields.io/github/actions/workflow/status/cerbero90/json-parser/build.yml?branch=master&style=flat-square&logo=github [ico-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square @@ -129,8 +106,6 @@ The MIT License (MIT). Please see [License File](LICENSE.md) for more informatio [link-author]: https://twitter.com/cerbero90 [link-php]: https://www.php.net -[link-laravel]: https://laravel.com -[link-octane]: https://github.com/laravel/octane [link-packagist]: https://packagist.org/packages/cerbero/lazy-json [link-actions]: https://github.com/cerbero90/lazy-json/actions?query=workflow%3Abuild [link-per]: https://www.php-fig.org/per/coding-style/ From 3b586ef76e46cad924c7e9adda63a6540c22e121 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 16 Aug 2023 20:00:13 +0200 Subject: [PATCH 17/33] Remove unneeded fixtures --- tests/fixtures/complex_array.php | 132 ------------------------------ tests/fixtures/complex_object.php | 58 ------------- tests/fixtures/empty_array.json | 1 - tests/fixtures/empty_array.php | 3 - tests/fixtures/empty_object.json | 1 - tests/fixtures/empty_object.php | 3 - tests/fixtures/simple_array.json | 1 - tests/fixtures/simple_array.php | 3 - 8 files changed, 202 deletions(-) delete mode 100644 tests/fixtures/complex_array.php delete mode 100644 tests/fixtures/complex_object.php delete mode 100644 tests/fixtures/empty_array.json delete mode 100644 tests/fixtures/empty_array.php delete mode 100644 tests/fixtures/empty_object.json delete mode 100644 tests/fixtures/empty_object.php delete mode 100644 tests/fixtures/simple_array.json delete mode 100644 tests/fixtures/simple_array.php diff --git a/tests/fixtures/complex_array.php b/tests/fixtures/complex_array.php deleted file mode 100644 index 3d4875f..0000000 --- a/tests/fixtures/complex_array.php +++ /dev/null @@ -1,132 +0,0 @@ - "0001", - "type" => "donut", - "name" => "Cake", - "ppu" => 0.55, - "batters" => [ - "batter" => [ - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1002", - "type" => "Chocolate", - ], - [ - "id" => "1003", - "type" => "Blueberry", - ], - [ - "id" => "1004", - "type" => "Devil's Food", - ], - ], - ], - "topping" => [ - [ - "id" => "5001", - "type" => "None", - ], - [ - "id" => "5002", - "type" => "Glazed", - ], - [ - "id" => "5005", - "type" => "Sugar", - ], - [ - "id" => "5007", - "type" => "Powdered Sugar", - ], - [ - "id" => "5006", - "type" => "Chocolate with Sprinkles", - ], - [ - "id" => "5003", - "type" => "Chocolate", - ], - [ - "id" => "5004", - "type" => "Maple", - ], - ], - ], - [ - "id" => "0002", - "type" => "donut", - "name" => "Raised", - "ppu" => 0.55, - "batters" => [ - "batter" => [ - [ - "id" => "1001", - "type" => "Regular", - ], - ], - ], - "topping" => [ - [ - "id" => "5001", - "type" => "None", - ], - [ - "id" => "5002", - "type" => "Glazed", - ], - [ - "id" => "5005", - "type" => "Sugar", - ], - [ - "id" => "5003", - "type" => "Chocolate", - ], - [ - "id" => "5004", - "type" => "Maple", - ], - ], - ], - [ - "id" => "0003", - "type" => "donut", - "name" => "Old Fashioned", - "ppu" => 0.55, - "batters" => [ - "batter" => [ - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1002", - "type" => "Chocolate", - ], - ], - ], - "topping" => [ - [ - "id" => "5001", - "type" => "None", - ], - [ - "id" => "5002", - "type" => "Glazed", - ], - [ - "id" => "5003", - "type" => "Chocolate", - ], - [ - "id" => "5004", - "type" => "Maple", - ], - ], - ], -]; diff --git a/tests/fixtures/complex_object.php b/tests/fixtures/complex_object.php deleted file mode 100644 index 06b2a6d..0000000 --- a/tests/fixtures/complex_object.php +++ /dev/null @@ -1,58 +0,0 @@ - '0001', - 'type' => 'donut', - 'name' => 'Cake', - 'ppu' => 0.55, - 'batters' => [ - 'batter' => [ - [ - 'id' => '1001', - 'type' => 'Regular', - ], - [ - 'id' => '1002', - 'type' => 'Chocolate', - ], - [ - 'id' => '1003', - 'type' => 'Blueberry', - ], - [ - 'id' => '1004', - 'type' => 'Devil\'s Food', - ], - ], - ], - 'topping' => [ - [ - 'id' => '5001', - 'type' => 'None', - ], - [ - 'id' => '5002', - 'type' => 'Glazed', - ], - [ - 'id' => '5005', - 'type' => 'Sugar', - ], - [ - 'id' => '5007', - 'type' => 'Powdered Sugar', - ], - [ - 'id' => '5006', - 'type' => 'Chocolate with Sprinkles', - ], - [ - 'id' => '5003', - 'type' => 'Chocolate', - ], - [ - 'id' => '5004', - 'type' => 'Maple', - ], - ], -]; diff --git a/tests/fixtures/empty_array.json b/tests/fixtures/empty_array.json deleted file mode 100644 index fe51488..0000000 --- a/tests/fixtures/empty_array.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/fixtures/empty_array.php b/tests/fixtures/empty_array.php deleted file mode 100644 index 0b67a5f..0000000 --- a/tests/fixtures/empty_array.php +++ /dev/null @@ -1,3 +0,0 @@ - Date: Wed, 16 Aug 2023 20:09:04 +0200 Subject: [PATCH 18/33] Remove unneeded use statement --- src/LazyJson.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LazyJson.php b/src/LazyJson.php index 3019baa..61c34b9 100644 --- a/src/LazyJson.php +++ b/src/LazyJson.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJson; -use Cerbero\JsonParser\Exceptions\JsonParserException; use Cerbero\JsonParser\JsonParser; use Cerbero\JsonParser\Tokens\Parser; use Cerbero\LazyJson\Exceptions\LazyJsonException; From 6544af7bf7d044b9c445448a3f47f10b5f0d3be5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 13:23:13 +0200 Subject: [PATCH 19/33] Update tests --- tests/Dataset.php | 33 +++--- tests/Feature/LazyJsonTest.php | 50 ++++++++- tests/Pest.php | 5 - tests/fixtures/multiple_dots.php | 172 +++++++++++++++++++++++++++++++ tests/fixtures/single_dot.php | 158 +++++----------------------- 5 files changed, 262 insertions(+), 156 deletions(-) create mode 100644 tests/fixtures/multiple_dots.php diff --git a/tests/Dataset.php b/tests/Dataset.php index 402f788..423ff88 100644 --- a/tests/Dataset.php +++ b/tests/Dataset.php @@ -5,7 +5,6 @@ namespace Cerbero\LazyJson; use Generator; -use Illuminate\Support\LazyCollection; final class Dataset { @@ -17,26 +16,30 @@ public static function forDots(): Generator foreach ($simpleObject as $key => $value) { yield [$source, $key, $value]; } + } + public static function forSingleDots(): Generator + { $singleDot = require fixture('single_dot.php'); foreach ($singleDot as $fixture => $subtreeByDot) { $source = fixture("{$fixture}.json"); - foreach ($subtreeByDot as $dot => $subtree) { - $values = (array) reset($subtree); - - foreach ($values as $expected) { - yield [ - $source, - $dot, - fn ($value, $key) => $key->toBe(key($subtree)) - ->and($value) - ->when(is_array($expected), fn ($value) => $value->toBeInstanceOf(LazyCollection::class)) - ->and(is_array($expected) ? $value->value->toArray() : $value) - ->toBe($expected), - ]; - } + foreach ($subtreeByDot as $dot => $expectedValuesByKey) { + yield [$source, $dot, $expectedValuesByKey]; + } + } + } + + public static function forMultipleDots(): Generator + { + $singleDot = require fixture('multiple_dots.php'); + + foreach ($singleDot as $fixture => $subtreeByDots) { + $source = fixture("{$fixture}.json"); + + foreach ($subtreeByDots as $dots => $expectedValuesByKey) { + yield [$source, explode(',', $dots), $expectedValuesByKey]; } } } diff --git a/tests/Feature/LazyJsonTest.php b/tests/Feature/LazyJsonTest.php index 25bad36..935c89e 100644 --- a/tests/Feature/LazyJsonTest.php +++ b/tests/Feature/LazyJsonTest.php @@ -40,6 +40,50 @@ ->traverse(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()); }); -// it('sets a JSON pointer by using the dot notation syntax', function (string $source, string|int $dot, mixed $sequence) { -// expect(LazyJson::from($source, (string) $dot))->sequence($sequence); -// })->with(Dataset::forDots()); +it('turns dot notation into JSON pointers correctly', function (string $source, string|int $dot, mixed $sequence) { + expect(LazyJson::from($source, (string) $dot))->sequence($sequence); +})->with(Dataset::forDots()); + +it('sets a callable JSON pointer by using the dot notation syntax', function () { + $dots = [ + 'foo' => function ($value, $key) { + expect($key)->toBe('foo')->and($value)->toBeInstanceOf(LazyCollection::class)->values()->all()->toBe([1, 2]); + return 'foo closure was run'; + }, + 'bar' => function ($value, $key) { + expect($key)->toBe('bar')->and($value)->toBeInstanceOf(LazyCollection::class)->all()->toBe([3, 4]); + return 'bar closure was run'; + }, + ]; + + expect(LazyJson::from('{"foo":{"one":1,"two":2},"bar":[3,4]}', $dots))->traverse( + fn (Expectation $value, Expectation $key) => $key->toBe('foo')->and($value)->toBe('foo closure was run'), + fn (Expectation $value, Expectation $key) => $key->toBe('bar')->and($value)->toBe('bar closure was run'), + ); +}); + +it('sets a JSON pointer by using the dot notation syntax', function (string $source, string $dot, array $expectedValuesByKey) { + $actualValues = []; + $expectedKey = key($expectedValuesByKey); + $expectedValues = reset($expectedValuesByKey); + + expect(LazyJson::from($source, $dot)) + ->traverse(function (Expectation $value, Expectation $key) use (&$actualValues, $expectedKey) { + $key->toBe($expectedKey)->and($value)->toBeInstanceOf(LazyCollection::class); + $actualValues[] = $value->value->toArray(); + }); + + expect($actualValues)->toBe($expectedValues); +})->with(Dataset::forSingleDots()); + +it('sets JSON pointers by using the dot notation syntax', function (string $source, array $dots, array $expectedValuesByKey) { + $actualValues = []; + + expect(LazyJson::from($source, $dots)) + ->traverse(function (Expectation $value, Expectation $key) use (&$actualValues) { + $value->toBeInstanceOf(LazyCollection::class); + $actualValues[$key->value][] = $value->value->toArray(); + }); + + expect($actualValues)->toBe($expectedValuesByKey); +})->with(Dataset::forMultipleDots()); diff --git a/tests/Pest.php b/tests/Pest.php index 835fd95..be703db 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -14,9 +14,6 @@ function fixture(string $fixture): string /** * Expect the given sequence from a Traversable * Temporary fix to sequence() until this PR is merged: https://github.com/pestphp/pest/pull/895 - * - * @param mixed ...$callbacks - * @return Expectation */ expect()->extend('traverse', function (mixed ...$callbacks) { if (! is_iterable($this->value)) { @@ -50,8 +47,6 @@ function fixture(string $fixture): string /** * Expect that all Parser instances are wrapped recursively into lazy collections - * - * @return Expectation */ expect()->extend('toBeWrappedIntoLazyCollection', function () { return $this->when(is_object($this->value), fn (Expectation $value) => $value diff --git a/tests/fixtures/multiple_dots.php b/tests/fixtures/multiple_dots.php new file mode 100644 index 0000000..e17e46e --- /dev/null +++ b/tests/fixtures/multiple_dots.php @@ -0,0 +1,172 @@ + [ + '*.batters.batter,*.topping' => [ + 'batter' => [ + [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + [ + [ + "id" => "1001", + "type" => "Regular", + ], + ], + [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + ], + ], + 'topping' => [ + [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5005", + "type" => "Sugar", + ], + [ + "id" => "5007", + "type" => "Powdered Sugar", + ], + [ + "id" => "5006", + "type" => "Chocolate with Sprinkles", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5005", + "type" => "Sugar", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + ], + ], + ], + 'complex_object' => [ + 'batters.batter,topping' => [ + 'batter' => [ + [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + 'topping' => [ + [ + [ + "id" => "5001", + "type" => "None", + ], + [ + "id" => "5002", + "type" => "Glazed", + ], + [ + "id" => "5005", + "type" => "Sugar", + ], + [ + "id" => "5007", + "type" => "Powdered Sugar", + ], + [ + "id" => "5006", + "type" => "Chocolate with Sprinkles", + ], + [ + "id" => "5003", + "type" => "Chocolate", + ], + [ + "id" => "5004", + "type" => "Maple", + ], + ], + ], + ], + ], +]; diff --git a/tests/fixtures/single_dot.php b/tests/fixtures/single_dot.php index 3fd06ea..a645fdc 100644 --- a/tests/fixtures/single_dot.php +++ b/tests/fixtures/single_dot.php @@ -2,8 +2,6 @@ return [ 'complex_array' => [ - '*' => require fixture('complex_array.php'), - '*.id' => ['id' => ['0001', '0002', '0003']], '*.batters' => [ 'batters' => [ [ @@ -86,48 +84,35 @@ ], ], ], - '*.batters.batter.*' => [ - 0 => [ - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1001", - "type" => "Regular", - ], - ], - 1 => [ - [ - "id" => "1002", - "type" => "Chocolate", - ], - [ - "id" => "1002", - "type" => "Chocolate", - ], - ], - 2 => [ - "id" => "1003", - "type" => "Blueberry", - ], - 3 => [ - "id" => "1004", - "type" => "Devil's Food", - ], - ], - '*.batters.batter.*.id' => ['id' => ["1001", "1002", "1003", "1004", "1001", "1001", "1002"]], ], 'complex_object' => [ - '*' => require fixture('complex_object.php'), - 'id' => ['id' => '0001'], 'batters' => [ 'batters' => [ - 'batter' => [ + [ + 'batter' => [ + [ + "id" => "1001", + "type" => "Regular", + ], + [ + "id" => "1002", + "type" => "Chocolate", + ], + [ + "id" => "1003", + "type" => "Blueberry", + ], + [ + "id" => "1004", + "type" => "Devil's Food", + ], + ], + ], + ], + ], + 'batters.batter' => [ + 'batter' => [ + [ [ "id" => "1001", "type" => "Regular", @@ -147,98 +132,5 @@ ], ], ], - 'batters.batter' => [ - 'batter' => [ - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1002", - "type" => "Chocolate", - ], - [ - "id" => "1003", - "type" => "Blueberry", - ], - [ - "id" => "1004", - "type" => "Devil's Food", - ], - ], - ], - 'batters.batter.*' => [ - [ - "id" => "1001", - "type" => "Regular", - ], - [ - "id" => "1002", - "type" => "Chocolate", - ], - [ - "id" => "1003", - "type" => "Blueberry", - ], - [ - "id" => "1004", - "type" => "Devil's Food", - ], - ], - 'batters.batter.*.id' => ['id' => ["1001", "1002", "1003", "1004"]], - ], - 'empty_array' => [ - '*' => [], - '-1' => [], - '0' => [], - 'foo' => [], - ], - 'empty_object' => [ - '*' => [], - '-1' => [], - '0' => [], - 'foo' => [], - ], - 'simple_array' => [ - '*' => require fixture('simple_array.php'), - '-1' => [], - '0' => [0 => 1], - '1' => [1 => ''], - '2' => [2 => 'foo'], - '3' => [3 => '"bar"'], - '4' => [4 => 'hej då'], - '5' => [5 => 3.14], - '6' => [6 => false], - '7' => [7 => null], - '8' => [8 => []], - '9' => [9 => []], - '10' => [], - 'foo' => [], - ], - 'simple_object' => [ - '*' => require fixture('simple_object.php'), - '-1' => [], - 'int' => ['int' => 1], - 'empty_string' => ['empty_string' => ''], - 'string' => ['string' => 'foo'], - 'escaped_string' => ['escaped_string' => '"bar"'], - '"escaped_key"' => ['"escaped_key"' => 'baz'], - 'unicode' => ['unicode' => "hej då"], - 'float' => ['float' => 3.14], - 'bool' => ['bool' => false], - 'null' => ['null' => null], - 'empty_array' => ['empty_array' => []], - 'empty_object' => ['empty_object' => []], - '10' => [], - 'foo' => [], - '' => ['' => 0], - 'a/b' => ['a/b' => 1], - 'c%d' => ['c%d' => 2], - 'e^f' => ['e^f' => 3], - 'g|h' => ['g|h' => 4], - 'i\\j' => ['i\\j' => 5], - 'k"l' => ['k"l' => 6], - ' ' => [' ' => 7], - 'm~n' => ['m~n' => 8], ], ]; From c7bb8ec46752a37659cb192239ad4cca5f86b1f7 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 16:11:37 +0200 Subject: [PATCH 20/33] Upgrade Duster action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 14f8a58..88b04d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,6 @@ jobs: steps: - uses: actions/checkout@v3 - name: Duster Lint - uses: tighten/duster-action@v1 + uses: tighten/duster-action@v2 with: args: lint -u tlint,phpcodesniffer,pint,phpstan From 792f97daa9f55dd69cf6f2dddf6310c4506489de Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 16:21:16 +0200 Subject: [PATCH 21/33] Update linting job --- .github/workflows/build.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 88b04d6..c10ea6e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,8 +69,19 @@ jobs: name: Linting steps: - - uses: actions/checkout@v3 - - name: Duster Lint - uses: tighten/duster-action@v2 + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 with: - args: lint -u tlint,phpcodesniffer,pint,phpstan + php-version: ${{ matrix.php }} + tools: composer:v2 + coverage: none + + - name: Install dependencies + run: | + composer update --prefer-stable --prefer-dist --no-interaction + + - name: Execute Duster + run: vendor/bin/duster lint -u tlint,phpcodesniffer,pint,phpstan From 07650f94efdfe87dddb0be26c64c244ffb42c903 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 17:34:26 +0200 Subject: [PATCH 22/33] Update readme --- README.md | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3bafb01..fa0ff0b 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,22 @@ [![PER][ico-per]][link-per] [![Total Downloads][ico-downloads]][link-downloads] -Framework-agnostic package to load JSONs of any dimension and from any source into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). +```php +LazyCollection::fromJson($source, 'data.*.users.*') + ->map($this->mapToUser(...)) + ->filter($this->filterUser(...)) + ->values() + ->chunk(1_000) + ->each($this->storeUsersChunk(...)); +``` -Lazy JSON recursively turns any JSON array and object into a lazy collection, consuming only a few KB of memory while parsing JSON of any size. +Framework-agnostic package to load JSON of any size and from any source into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). -It optionally allows to extract only some sub-trees, instead of the whole JSON, with a simple dot-notation syntax. +Lazy JSON recursively turns any JSON array and object into a lazy collection, consuming only a few KB of memory while parsing JSON of any dimension. -Under the hood, [🧩 JSON Parser](https://github.com/cerbero90/json-parser) is used to parse any JSON and extract sub-trees. +It optionally allows to extract only some sub-trees, instead of the whole JSON, with an easy dot-notation syntax. + +Under the hood, [🧩 JSON Parser](https://github.com/cerbero90/json-parser) is used to parse JSONs and extract sub-trees. Need to lazy load items from paginated JSON APIs? Consider using [🐼 Lazy JSON Pages](https://github.com/cerbero90/lazy-json-pages) instead. @@ -32,11 +41,12 @@ composer require cerbero/lazy-json ## 🔮 Usage -Depending on our code style, we can call Lazy JSON in 3 different ways: +Depending on our coding style, we can call Lazy JSON in 3 different ways: ```php use Cerbero\LazyJson\LazyJson; use Illuminate\Support\LazyCollection; + use function Cerbero\LazyJson\lazyJson; // auto-registered lazy collection macro @@ -64,7 +74,20 @@ The variable `$source` in the above examples indicates any data point that provi For more information about JSON sources, please consult the [🧩 JSON Parser documentation](https://github.com/cerbero90/json-parser). -// @todo: complete README +If we only need a sub-tree of a large JSON, we can define the path to extract by using a simple dot-notation syntax. + +Consider [this JSON](https://randomuser.me/api/1.4?seed=json-parser&results=5) for example. To extract only the cities and avoid parsing the rest of the JSON, we can set the `results.*.location.city` dot-notation: + +```php +$source = 'https://randomuser.me/api/1.4?seed=json-parser&results=5'; + +LazyCollection::fromJson($source, 'results.*.location.city')->each(function (string $value, string $key) { + // 1st iteration: $key === 'city', $value === 'Sontra' + // 2nd iteration: $key === 'city', $value === 'San Rafael Tlanalapan' + // 3rd iteration: $key === 'city', $value === 'گرگان' + // ... +}); +``` ## 📆 Change log From a9e1bf4175a8d59bfbf9073f188aacab6a539618 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 18:02:51 +0200 Subject: [PATCH 23/33] Update readme --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fa0ff0b..e154503 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ composer require cerbero/lazy-json ## 🔮 Usage +* [👣 Basics](#-basics) +* [💧 Sources](#-sources) +* [🎯 Dots](#-dots) + + +### 👣 Basics + Depending on our coding style, we can call Lazy JSON in 3 different ways: ```php @@ -59,7 +66,10 @@ $lazyCollection = LazyJson::from($source); $lazyCollection = lazyJson($source); ``` -The variable `$source` in the above examples indicates any data point that provides a JSON. A wide range of sources are supported by default: + +### 💧 Sources + +A JSON source is any data point that provides a JSON. A wide range of sources are supported by default: - **strings**, e.g. `{"foo":"bar"}` - **iterables**, i.e. arrays or instances of `Traversable` - **file paths**, e.g. `/path/to/large.json` @@ -74,6 +84,9 @@ The variable `$source` in the above examples indicates any data point that provi For more information about JSON sources, please consult the [🧩 JSON Parser documentation](https://github.com/cerbero90/json-parser). + +### 🎯 Dots + If we only need a sub-tree of a large JSON, we can define the path to extract by using a simple dot-notation syntax. Consider [this JSON](https://randomuser.me/api/1.4?seed=json-parser&results=5) for example. To extract only the cities and avoid parsing the rest of the JSON, we can set the `results.*.location.city` dot-notation: @@ -89,6 +102,11 @@ LazyCollection::fromJson($source, 'results.*.location.city')->each(function (str }); ``` +The dot-notation syntax is very simple and it can include any of the following 3 elements: +- a key of a JSON object, e.g. `results` +- an asterisk to indicate all items within an array, e.g. `results.*` +- a dot to indicate the nesting level within a JSON, e.g. `results.*.location` + ## 📆 Change log Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. From c807e243538311157037e6e8860ebdd221c1378f Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 18:28:20 +0200 Subject: [PATCH 24/33] Update readme --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e154503..4609bd1 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,15 @@ $lazyCollection = LazyJson::from($source); $lazyCollection = lazyJson($source); ``` +Once the [JSON source](#-sources) is set, we can chain any method of the [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) to process the JSON in a memory-efficient way: + +```php +LazyCollection::fromJson($source) + ->map(/* ... */) + ->where(/* ... */) + ->each(/* ... */); +``` + ### 💧 Sources @@ -87,9 +96,9 @@ For more information about JSON sources, please consult the [🧩 JSON Parser do ### 🎯 Dots -If we only need a sub-tree of a large JSON, we can define the path to extract by using a simple dot-notation syntax. +If we only need a sub-tree of a large JSON, we can use a simple dot-notation syntax to extract the desired path (or **dot**). -Consider [this JSON](https://randomuser.me/api/1.4?seed=json-parser&results=5) for example. To extract only the cities and avoid parsing the rest of the JSON, we can set the `results.*.location.city` dot-notation: +Consider [this JSON](https://randomuser.me/api/1.4?seed=json-parser&results=5) for example. To extract only the cities and avoid parsing the rest of the JSON, we can set the `results.*.location.city` dot: ```php $source = 'https://randomuser.me/api/1.4?seed=json-parser&results=5'; From a86c3b7d8f0ab1cbb3f59a62f4eba01c8ecab8c0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 19:04:25 +0200 Subject: [PATCH 25/33] Update readme --- README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4609bd1..dc26f4c 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,26 @@ LazyCollection::fromJson($source, 'results.*.location.city')->each(function (str }); ``` -The dot-notation syntax is very simple and it can include any of the following 3 elements: +The dot-notation syntax is very simple and it can include any of the following 4 elements: +- a key of a JSON array, e.g. `0` - a key of a JSON object, e.g. `results` +- a dot to indicate the nesting level within a JSON, e.g. `results.0` - an asterisk to indicate all items within an array, e.g. `results.*` -- a dot to indicate the nesting level within a JSON, e.g. `results.*.location` + +If we need to extract several sub-trees, we are in luck as Lazy JSON supports multiple dots: + +```php +$source = 'https://randomuser.me/api/1.4?seed=json-parser&results=5'; +$dots = ['results.*.gender', 'results.*.email']; + +LazyCollection::fromJson($source, $dots)->each(function (string $value, string $key) { + // 1st iteration: $key === 'gender', $value === 'female' + // 2nd iteration: $key === 'email', $value === 'sara.meder@example.com' + // 3rd iteration: $key === 'gender', $value === 'female' + // 4th iteration: $key === 'email', $value === 'andrea.roque@example.com' + // ... +}); +``` ## 📆 Change log From a9b81bb8e7bd56a7f90348fa086d9fb553c76c07 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 19 Aug 2023 19:36:05 +0200 Subject: [PATCH 26/33] Update readme --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc26f4c..0d48bdc 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,9 @@ Consider [this JSON](https://randomuser.me/api/1.4?seed=json-parser&results=5) f ```php $source = 'https://randomuser.me/api/1.4?seed=json-parser&results=5'; -LazyCollection::fromJson($source, 'results.*.location.city')->each(function (string $value, string $key) { +$dot = 'results.*.location.city'; + +LazyCollection::fromJson($source, $dot)->each(function (string $value, string $key) { // 1st iteration: $key === 'city', $value === 'Sontra' // 2nd iteration: $key === 'city', $value === 'San Rafael Tlanalapan' // 3rd iteration: $key === 'city', $value === 'گرگان' @@ -117,10 +119,9 @@ The dot-notation syntax is very simple and it can include any of the following 4 - a dot to indicate the nesting level within a JSON, e.g. `results.0` - an asterisk to indicate all items within an array, e.g. `results.*` -If we need to extract several sub-trees, we are in luck as Lazy JSON supports multiple dots: +If we need to extract several sub-trees, Lazy JSON supports multiple dots: ```php -$source = 'https://randomuser.me/api/1.4?seed=json-parser&results=5'; $dots = ['results.*.gender', 'results.*.email']; LazyCollection::fromJson($source, $dots)->each(function (string $value, string $key) { From 4bbf3e0d6de4932b35fa1004730e1dc8366fa100 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 21 Aug 2023 15:31:42 +0200 Subject: [PATCH 27/33] Upgrade Pest to use the new sequence expectation --- composer.json | 2 +- tests/Feature/LazyJsonTest.php | 8 ++++---- tests/Pest.php | 36 +--------------------------------- 3 files changed, 6 insertions(+), 40 deletions(-) diff --git a/composer.json b/composer.json index db09fff..468c201 100644 --- a/composer.json +++ b/composer.json @@ -23,7 +23,7 @@ "illuminate/collections": ">=8.12" }, "require-dev": { - "pestphp/pest": "^2.0", + "pestphp/pest": "^2.16", "phpstan/phpstan": "^1.9", "scrutinizer/ocular": "^1.8", "squizlabs/php_codesniffer": "^3.0", diff --git a/tests/Feature/LazyJsonTest.php b/tests/Feature/LazyJsonTest.php index 935c89e..28f0c33 100644 --- a/tests/Feature/LazyJsonTest.php +++ b/tests/Feature/LazyJsonTest.php @@ -37,7 +37,7 @@ it('wraps JSON objects and arrays into lazy collections', function () { expect(LazyJson::from('{"foo":{"one":1,"two":2},"bar":[3,4]}')) - ->traverse(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()); + ->sequence(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()); }); it('turns dot notation into JSON pointers correctly', function (string $source, string|int $dot, mixed $sequence) { @@ -56,7 +56,7 @@ }, ]; - expect(LazyJson::from('{"foo":{"one":1,"two":2},"bar":[3,4]}', $dots))->traverse( + expect(LazyJson::from('{"foo":{"one":1,"two":2},"bar":[3,4]}', $dots))->sequence( fn (Expectation $value, Expectation $key) => $key->toBe('foo')->and($value)->toBe('foo closure was run'), fn (Expectation $value, Expectation $key) => $key->toBe('bar')->and($value)->toBe('bar closure was run'), ); @@ -68,7 +68,7 @@ $expectedValues = reset($expectedValuesByKey); expect(LazyJson::from($source, $dot)) - ->traverse(function (Expectation $value, Expectation $key) use (&$actualValues, $expectedKey) { + ->sequence(function (Expectation $value, Expectation $key) use (&$actualValues, $expectedKey) { $key->toBe($expectedKey)->and($value)->toBeInstanceOf(LazyCollection::class); $actualValues[] = $value->value->toArray(); }); @@ -80,7 +80,7 @@ $actualValues = []; expect(LazyJson::from($source, $dots)) - ->traverse(function (Expectation $value, Expectation $key) use (&$actualValues) { + ->sequence(function (Expectation $value, Expectation $key) use (&$actualValues) { $value->toBeInstanceOf(LazyCollection::class); $actualValues[$key->value][] = $value->value->toArray(); }); diff --git a/tests/Pest.php b/tests/Pest.php index be703db..7443a5d 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -11,40 +11,6 @@ function fixture(string $fixture): string } } -/** - * Expect the given sequence from a Traversable - * Temporary fix to sequence() until this PR is merged: https://github.com/pestphp/pest/pull/895 - */ -expect()->extend('traverse', function (mixed ...$callbacks) { - if (! is_iterable($this->value)) { - throw new BadMethodCallException('Expectation value is not iterable.'); - } - - if (empty($callbacks)) { - throw new InvalidArgumentException('No sequence expectations defined.'); - } - - $index = $valuesCount = 0; - - foreach ($this->value as $key => $value) { - $valuesCount++; - - if ($callbacks[$index] instanceof Closure) { - $callbacks[$index](new self($value), new self($key)); - } else { - (new self($value))->toEqual($callbacks[$index]); - } - - $index = isset($callbacks[$index + 1]) ? $index + 1 : 0; - } - - if (count($callbacks) > $valuesCount) { - throw new OutOfRangeException('Sequence expectations are more than the iterable items'); - } - - return $this; -}); - /** * Expect that all Parser instances are wrapped recursively into lazy collections */ @@ -52,6 +18,6 @@ function fixture(string $fixture): string return $this->when(is_object($this->value), fn (Expectation $value) => $value ->toBeInstanceOf(LazyCollection::class) ->not->toBeInstanceOf(Parser::class) - ->traverse(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()) + ->sequence(fn (Expectation $value) => $value->toBeWrappedIntoLazyCollection()) ); }); From 73a7fb75ece8fba7d471fd9a3e58cc40ea523fc6 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 18:38:43 +1000 Subject: [PATCH 28/33] Update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d48bdc..57d0534 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ LazyCollection::fromJson($source, 'data.*.users.*') Framework-agnostic package to load JSON of any size and from any source into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). -Lazy JSON recursively turns any JSON array and object into a lazy collection, consuming only a few KB of memory while parsing JSON of any dimension. +Lazy JSON recursively turns any JSON array or object into a lazy collection, consuming only a few KB of memory while parsing JSON of any dimension. It optionally allows to extract only some sub-trees, instead of the whole JSON, with an easy dot-notation syntax. @@ -66,10 +66,11 @@ $lazyCollection = LazyJson::from($source); $lazyCollection = lazyJson($source); ``` -Once the [JSON source](#-sources) is set, we can chain any method of the [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) to process the JSON in a memory-efficient way: +The variable `$source` in our examples represents any [JSON source](#-sources). Once we define the source, we can chain any method of the [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) to process the JSON in a memory-efficient way: ```php LazyCollection::fromJson($source) + ->values() ->map(/* ... */) ->where(/* ... */) ->each(/* ... */); From 7b3cb16405bc567f9b83a5123a6b0579f739fce3 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 19:00:41 +1000 Subject: [PATCH 29/33] Update workflow --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c10ea6e..96260cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - php: [8.1, 8.2] + php: [8.1, 8.2, 8.3] dependency-version: [prefer-lowest, prefer-stable] os: [ubuntu-latest] @@ -75,7 +75,7 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: ${{ matrix.php }} + php-version: 8.1 tools: composer:v2 coverage: none From 83cb7ccc621c23d7c8eebc6408482454a772e00a Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 19:14:57 +1000 Subject: [PATCH 30/33] Update changelog --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faba229..203b4b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,22 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip - Nothing +## 2.0.0 - 2023-11-29 + +### Added +- Recursive lazy collections for JSON objects and arrays +- Auto-registering macro for lazy collections +- Dependency from [🧩 JSON Parser](https://github.com/cerbero90/json-parser) +- Namespaced helper +- Compatibility with latest versions of PHP +- Pest testing framework +- Tools for static analysis + +### Removed +- Dependency from [JSON Machine](https://github.com/halaxa/json-machine) +- Compatibility with older versions of PHP + + ## 1.1.0 - 2021-05-06 ### Added From 5851c8f288d990129e3f6fcc7265e63d8a2be60c Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 19:19:18 +1000 Subject: [PATCH 31/33] Update build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96260cd..0fe6dd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,4 +84,4 @@ jobs: composer update --prefer-stable --prefer-dist --no-interaction - name: Execute Duster - run: vendor/bin/duster lint -u tlint,phpcodesniffer,pint,phpstan + run: vendor/bin/duster lint -u tlint,phpcodesniffer,pint,phpstan -vvv From 166e0b24ccf1dbe7b5b5a61e358053114ed64080 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 19:25:02 +1000 Subject: [PATCH 32/33] Update build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fe6dd4..3efe43b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,7 +75,7 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 tools: composer:v2 coverage: none From 82a2b4f6e482229cb0ce23b692f6eae48b4b32f6 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 29 Nov 2023 20:08:23 +1000 Subject: [PATCH 33/33] Fix style --- src/LazyJson.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LazyJson.php b/src/LazyJson.php index 61c34b9..99b62e0 100644 --- a/src/LazyJson.php +++ b/src/LazyJson.php @@ -26,7 +26,7 @@ final class LazyJson implements IteratorAggregate */ public static function from(mixed $source, string|array $dot = '*'): LazyCollection { - return new LazyCollection(fn () => yield from new self($source, (array) $dot)); + return new LazyCollection(fn() => yield from new self($source, (array) $dot)); } /** @@ -36,7 +36,7 @@ private function __construct(mixed $source, array $dots) { $this->parser = JsonParser::parse($source) ->lazyPointers(DotsConverter::toPointers($dots)) - ->wrap(fn (Parser $parser) => new LazyCollection(fn () => yield from $parser)); + ->wrap(fn(Parser $parser) => new LazyCollection(fn() => yield from $parser)); } /**