Make AssetQueryBuilder for performance gains (#218)

README.md

@@ -61,6 +61,11 @@ We have provided imports from file based content for each repository, which can
 - Revisions: `php please eloquent:import-revisions`
 - Taxonomies: `php please eloquent:import-taxonomies`
 
+If your assets are eloquent driver and you are managing your assets outside of Statamic, we have provided a sync assets command which will check your container for updates and add database entries for any missing files, while removing any that no longer exist.
+
+`php please eloquent:sync-assets`
+
+
 ## Exporting back to file based content
 
 We have provided exports from eloquent to file based content for each repository, which can be run as follows:

database/migrations/create_asset_table.php.stub

@@ -9,13 +9,19 @@ return new class extends Migration {
     {
         Schema::create($this->prefix('assets_meta'), function (Blueprint $table) {
             $table->id();
-            $table->string('handle')->index();
-            $table->jsonb('data')->nullable();
+            $table->string('container')->index();
+            $table->string('folder')->index();
+            $table->string('basename')->index();
+            $table->string('filename')->index();
+            $table->char('extension', 10)->index();
+            $table->string('path')->index();
+            $table->jsonb('meta')->nullable();
             $table->timestamps();
+
+            $table->index(['container', 'folder', 'basename']);
         });
     }
 
-
     public function down()
     {
         Schema::dropIfExists($this->prefix('assets_meta'));

database/migrations/updates/update_assets_table.php.stub

@@ -0,0 +1,72 @@
+<?php
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Str;
+use Statamic\Eloquent\Assets\AssetModel;
+use Statamic\Eloquent\Database\BaseMigration as Migration;
+
+return new class extends Migration {
+    public function up()
+    {
+        Schema::table($this->prefix('assets_meta'), function (Blueprint $table) {
+            $table->string('container')->after('handle')->index();
+            $table->string('folder')->after('container')->index();
+            $table->string('basename')->after('folder')->index();
+            $table->string('filename')->after('basename')->index();
+            $table->char('extension', 10)->after('filename')->index();
+            $table->string('path')->after('extension')->index();
+            $table->jsonb('meta')->after('path')->nullable();
+
+            $table->index(['container', 'folder', 'basename']);
+        });
+
+        AssetModel::all()
+            ->each(function ($model) {
+                $path = Str::of($model->handle)->after('::')->replace('.meta/', '')->beforeLast('.yaml');
+
+                if ($path->startsWith('./')) {
+                    $path = $path->replaceFirst('./', '');
+                }
+
+                $model->container = Str::before($model->handle, '::');
+                $model->path = $path;
+                $model->folder = $path->contains('/') ? $path->beforeLast('/') : '/';
+                $model->basename = $path->afterLast('/');
+                $model->extension = Str::of($model->basename)->afterLast('.');
+                $model->filename = Str::of($model->basename)->beforeLast('.');
+                $model->meta = $model->data;
+                $model->save();
+            });
+
+        Schema::table($this->prefix('assets_meta'), function (Blueprint $table) {
+            $table->dropColumn('handle');
+        });
+    }
+
+    public function down()
+    {
+        Schema::table($this->prefix('assets_meta'), function (Blueprint $table) {
+            $table->string('handle')->index();
+        });
+
+        AssetModel::all()
+            ->each(function ($model) {
+                $model->handle = $model->container.'::'.$model->folder.'/.meta/'.$model->basename.'.yaml';
+                $model->data = $model->meta;
+                $model->saveQuietly();
+            });
+
+        Schema::table($this->prefix('assets_meta'), function (Blueprint $table) {
+            $table->dropIndex(['container', 'folder', 'basename']);
+
+            $table->dropColumn('meta');
+            $table->dropColumn('path');
+            $table->dropColumn('basename');
+            $table->dropColumn('filename');
+            $table->dropColumn('extension');
+            $table->dropColumn('folder');
+            $table->dropColumn('container');
+        });
+    }
+};

src/Assets/Asset.php

@@ -2,18 +2,27 @@
 
 namespace Statamic\Eloquent\Assets;
 
+use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Facades\Cache;
 use Statamic\Assets\Asset as FileAsset;
 use Statamic\Assets\AssetUploader as Uploader;
-use Statamic\Facades\Blink;
 use Statamic\Facades\Path;
 use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 class Asset extends FileAsset
 {
+    protected $existsOnDisk = false;
     protected $removedData = [];
 
+    public static function fromModel(Model $model)
+    {
+        return (new static())
+            ->container($model->container)
+            ->path(Str::replace('//', '/', $model->folder.'/'.$model->basename))
+            ->hydrateMeta($model->meta);
+    }
+
     public function meta($key = null)
     {
         if (func_num_args() === 1) {
@@ -37,27 +46,37 @@ public function meta($key = null)
 
         return $this->meta = Cache::rememberForever($this->metaCacheKey(), function () {
             $handle = $this->container()->handle().'::'.$this->metaPath();
-            if ($model = app('statamic.eloquent.assets.model')::where('handle', $handle)->first()) {
-                return $model->data;
+            if ($model = app('statamic.eloquent.assets.model')::where([
+                'container' => $this->containerHandle(),
+                'folder' => $this->folder(),
+                'basename' => $this->basename(),
+            ])->first()) {
+                return $model->meta;
             }
 
             $this->writeMeta($meta = $this->generateMeta());
 
+            if (! $meta['data']) {
+                $meta['data'] = [];
+            }
+
             return $meta;
         });
     }
 
     public function exists()
     {
-        $files = Blink::once($this->container()->handle().'::files', function () {
-            return $this->container()->files();
-        });
-
-        if (! $path = $this->path()) {
-            return false;
-        }
+        return $this->existsOnDisk || $this->metaExists();
+    }
 
-        return $files->contains($path);
+    public function metaExists()
+    {
+        return app('statamic.eloquent.assets.model')::query()
+            ->where([
+                'container' => $this->containerHandle(),
+                'folder' => $this->folder(),
+                'basename' => $this->basename(),
+            ])->count() > 0;
     }
 
     private function metaValue($key)
@@ -71,13 +90,38 @@ private function metaValue($key)
         return Arr::get($this->generateMeta(), $key);
     }
 
+    public function generateMeta()
+    {
+        if (! $this->disk()->exists($this->path())) {
+            return ['data' => $this->data->all()];
+        }
+
+        $this->existsOnDisk = true;
+
+        return parent::generateMeta();
+    }
+
+    public function hydrateMeta($meta)
+    {
+        $this->meta = $meta;
+
+        return $this;
+    }
+
     public function writeMeta($meta)
     {
         $meta['data'] = Arr::removeNullValues($meta['data']);
 
         $model = app('statamic.eloquent.assets.model')::firstOrNew([
-            'handle' => $this->container()->handle().'::'.$this->metaPath(),
-        ])->fill(['data' => $meta]);
+            'container' => $this->containerHandle(),
+            'folder' => $this->folder(),
+            'basename' => $this->basename(),
+        ])->fill([
+            'meta' => $meta,
+            'filename' => $this->filename(),
+            'extension' => $this->extension(),
+            'path' => $this->path(),
+        ]);
 
         // Set initial timestamps.
         if (empty($model->created_at) && isset($meta['last_modified'])) {
@@ -88,6 +132,11 @@ public function writeMeta($meta)
         $model->save();
     }
 
+    public function metaPath()
+    {
+        return $this->path();
+    }
+
     /**
      * Move the asset to a different location.
      *
@@ -100,6 +149,8 @@ public function move($folder, $filename = null)
         $filename = Uploader::getSafeFilename($filename ?: $this->filename());
         $oldPath = $this->path();
         $oldMetaPath = $this->metaPath();
+        $oldFolder = $this->folder();
+        $oldBasename = $this->basename();
         $newPath = Str::removeLeft(Path::tidy($folder.'/'.$filename.'.'.pathinfo($oldPath, PATHINFO_EXTENSION)), '/');
 
         $this->hydrate();
@@ -108,11 +159,17 @@ public function move($folder, $filename = null)
         $this->save();
 
         if ($oldMetaPath != $this->metaPath()) {
-            $oldMetaModel = app('statamic.eloquent.assets.model')::whereHandle($this->container()->handle().'::'.$oldMetaPath)->first();
+            $oldMetaModel = app('statamic.eloquent.assets.model')::where([
+                'container' => $this->containerHandle(),
+                'folder' => $oldFolder,
+                'basename' => $oldBasename,
+            ])->first();
 
             if ($oldMetaModel) {
+                $meta = $oldMetaModel->meta;
                 $oldMetaModel->delete();
-                $this->writeMeta($oldMetaModel->data);
+
+                $this->writeMeta($meta);
             }
         }
 

src/Assets/AssetContainer.php

@@ -6,6 +6,7 @@
 use Statamic\Assets\AssetContainer as FileEntry;
 use Statamic\Events\AssetContainerDeleted;
 use Statamic\Events\AssetContainerSaved;
+use Statamic\Support\Str;
 
 class AssetContainer extends FileEntry
 {
@@ -106,4 +107,22 @@ public function delete()
 
         return true;
     }
+
+    public function folders($folder = '/', $recursive = false)
+    {
+        return $this->disk()->getFolders($folder, $recursive);
+    }
+
+    public function metaFiles($folder = '/', $recursive = false)
+    {
+        // When requesting files() as-is, we want all of them.
+        if (func_num_args() === 0) {
+            $recursive = true;
+        }
+
+        return $this->queryAssets()
+            ->when($recursive, fn ($query) => $query->where('folder', $folder), fn ($query) => $query->where('folder', 'like', Str::replaceEnd('/', '', $folder).'/%'))
+            ->get()
+            ->pluck('path');
+    }
 }