Job Batching (#16)

* Batch import jobs

* Sometimes, we won't have a batch ID

* Fix styling

* Get a database working in tests.

* Fix styling

* Translate this string.

* Add `php artisan migrate` to the addon's install steps

* fix indentation

* Revert "Add `php artisan migrate` to the addon's install steps"

This reverts commit 0ffa848.

* Attempt to run migrations automatically, if possible.

* wip

* Fix styling

* wip

* Fix styling

* wip

* Fix styling

* wip

* wip

* Enough now... make the tests pass.

* Simplify.

* Revert "Simplify."

This reverts commit 1d66605.

* tweak messaging

---------

Co-authored-by: duncanmcclean <[email protected]>

Author: duncanmcclean

lang/en/messages.php

@@ -3,5 +3,6 @@
 return [
     'configuration_instructions' => 'You can add or modify your Blueprint fields to customize what data is imported and what fieldtype it will be stored in. You can save, refresh, and come back to this import config later until it\'s ready to run.',
     'mapping_instructions' => 'Map the fields from your import to the fields in your blueprint.',
+    'migrations_needed' => 'In order to keep track of import progress, the importer uses Laravel\'s Job Batching feature. It uses a <code>job_batches</code> table in your database to store information about batches. Before you can run the importer, you will need to run the <code>php artisan migrate</code> command.',
     'unique_field_instructions' => 'Select a "unique field" to determine if an item already exists.',
 ];

phpunit.xml

@@ -11,8 +11,8 @@
     <env name="APP_KEY" value="base64:ybcI9MKuhLnESRSuWDfnJQuohOXMBaynfbTC5Y5i1FE="/>
     <env name="BCRYPT_ROUNDS" value="4"/>
     <env name="CACHE_DRIVER" value="array"/>
-    <!-- <env name="DB_CONNECTION" value="sqlite"/> -->
-    <!-- <env name="DB_DATABASE" value=":memory:"/> -->
+    <env name="DB_CONNECTION" value="sqlite"/>
+    <env name="DB_DATABASE" value=":memory:"/>
     <env name="MAIL_MAILER" value="array"/>
     <env name="QUEUE_CONNECTION" value="sync"/>
     <env name="SESSION_DRIVER" value="array"/>

resources/js/components/EditImportForm.vue

@@ -7,10 +7,15 @@
 
             <div class="flex items-center space-x-4">
                 <button class="btn" :disabled="saving" @click="save()">{{ __('Save') }}</button>
-                <button class="btn-primary" :disabled="saving" @click="save(true)">{{ __('Save & Run') }}</button>
+                <button class="btn-primary" :disabled="saving || batchesTableMissing" @click="save(true)">{{ __('Save & Run') }}</button>
             </div>
         </div>
 
+        <div v-if="batchesTableMissing" class="text-xs border border-yellow-dark rounded p-4 bg-yellow dark:bg-dark-blue-100 dark:border-none">
+            <div class="font-bold mb-2">{{ __('Please run your migrations.') }}</div>
+            <p v-html="__('importer::messages.migrations_needed')"></p>
+        </div>
+
         <div class="mt-3 card overflow-hidden">
             <Mappings
                 :config="config"
@@ -35,6 +40,7 @@ export default {
         title: String,
         initialConfig: Object,
         mappingsUrl: String,
+        batchesTableMissing: Boolean,
     },
 
     data() {

resources/views/edit.blade.php

@@ -1,3 +1,5 @@
+@use('Statamic\Support\Str')
+
 @extends('statamic::layout')
 @section('title', $import->name())
 @section('wrapper_class', 'max-w-3xl')
@@ -9,5 +11,6 @@
         title="{{ $import->name() }}"
         :initial-config='@json($import->config())'
         mappings-url="{{ cp_route('utilities.importer.mappings') }}"
+        :batches-table-missing="{{ Str::bool($batchesTableMissing) }}"
     ></edit-import-form>
 @stop

src/Exceptions/JobBatchesTableMissingException.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace Statamic\Importer\Exceptions;
+
+class JobBatchesTableMissingException extends \Exception
+{
+    public function __construct()
+    {
+        parent::__construct('The job_batches table is missing. Please run `php artisan migrate` to run the required migrations.');
+    }
+}

src/Http/Controllers/ImportController.php

@@ -4,6 +4,8 @@
 
 use Illuminate\Http\Request;
 use Illuminate\Support\Arr;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\Schema;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
 use Statamic\CP\Breadcrumbs;
@@ -94,6 +96,7 @@ public function edit(Request $request, $import)
 
         return view('importer::edit', [
             'import' => $import,
+            'batchesTableMissing' => ! $this->ensureJobBatchesTableExists(),
             'breadcrumbs' => Breadcrumbs::make([
                 ['text' => __('Imports'), 'url' => cp_route('utilities.importer')],
             ]),
@@ -211,4 +214,29 @@ private function getConfigBlueprint(): \Statamic\Fields\Blueprint
 
         return $blueprint;
     }
+
+    private function ensureJobBatchesTableExists(): bool
+    {
+        if (Schema::connection(config('queue.batching.database'))->hasTable(config('queue.batching.table'))) {
+            return true;
+        }
+
+        if (app()->isProduction()) {
+            return false;
+        }
+
+        try {
+            // When this return a non-zero exit code, it doesn't necessarily mean there's an issue.
+            // It could be because the migration has already been published.
+            Artisan::call('make:queue-batches-table');
+
+            if (Artisan::call('migrate') !== 0) {
+                return false;
+            }
+        } catch (\Exception $e) {
+            return false;
+        }
+
+        return true;
+    }
 }

src/Importer.php

@@ -2,6 +2,10 @@
 
 namespace Statamic\Importer;
 
+use Illuminate\Bus\Batch;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\Schema;
+use Statamic\Importer\Exceptions\JobBatchesTableMissingException;
 use Statamic\Importer\Imports\Import;
 use Statamic\Importer\Jobs\ImportItemJob;
 use Statamic\Importer\Sources\Csv;
@@ -13,12 +17,18 @@ class Importer
 
     public static function run(Import $import): void
     {
+        if (! Schema::connection(config('queue.batching.database'))->hasTable(config('queue.batching.table'))) {
+            throw new JobBatchesTableMissingException;
+        }
+
         $items = match ($import->get('type')) {
             'csv' => (new Csv($import->config()->all()))->getItems($import->get('path')),
             'xml' => (new Xml($import->config()->all()))->getItems($import->get('path')),
         };
 
-        $items->each(fn (array $item) => ImportItemJob::dispatch($import, $item));
+        Bus::batch($items->map(fn (array $item) => new ImportItemJob($import, $item)))
+            ->before(fn (Batch $batch) => $import->batchId($batch->id)->save())
+            ->dispatch();
     }
 
     public static function getTransformer(string $fieldtype): ?string

src/Imports/Import.php

@@ -2,6 +2,8 @@
 
 namespace Statamic\Importer\Imports;
 
+use Illuminate\Bus\PendingBatch;
+use Illuminate\Support\Facades\Bus;
 use Statamic\Importer\Facades\Import as ImportFacade;
 use Statamic\Importer\Importer;
 use Statamic\Support\Traits\FluentlyGetsAndSets;
@@ -13,6 +15,7 @@ class Import
     public $id;
     public $name;
     public $config;
+    public $batchId;
 
     public function __construct()
     {
@@ -51,11 +54,26 @@ public function get(string $key, ?string $default = null): mixed
         return data_get($this->config, $key, $default);
     }
 
+    public function batchId($batchId = null)
+    {
+        return $this->fluentlyGetOrSet('batchId')->args(func_get_args());
+    }
+
+    public function batch(): ?PendingBatch
+    {
+        if (! $this->batchId()) {
+            return null;
+        }
+
+        return Bus::batch($this->batchId());
+    }
+
     public function fileData(): array
     {
         return collect([
             'name' => $this->name(),
             'config' => $this->config()->all(),
+            'batch_id' => $this->batchId(),
         ])->filter()->all();
     }
 

src/Imports/ImportRepository.php

@@ -23,6 +23,7 @@ public function all(): Collection
                 return $this->make()
                     ->id($id)
                     ->name(Arr::pull($data, 'name'))
+                    ->batchId(Arr::pull($data, 'batch_id'))
                     ->config(Arr::pull($data, 'config'));
             })
             ->sortBy->name()

src/Jobs/ImportItemJob.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Importer\Jobs;
 
+use Illuminate\Bus\Batchable;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -20,7 +21,7 @@
 
 class ImportItemJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable;
+    use Batchable, Dispatchable, InteractsWithQueue, Queueable;
 
     public function __construct(public Import $import, public array $item) {}
 
@@ -71,9 +72,12 @@ protected function getBlueprint(): Blueprint
 
     protected function findOrCreateEntry(array $data): void
     {
+        $collection = Collection::find($this->import->get('destination.collection'));
+        $site = Site::get($this->import->get('destination.site') ?? Site::selected()->handle());
+
         $entry = Entry::query()
-            ->where('collection', $this->import->get('destination.collection'))
-            ->where('site', $this->import->get('destination.site') ?? Site::selected()->handle())
+            ->where('locale', $site->handle())
+            ->where('collection', $collection->handle())
             ->where($this->import->get('unique_field'), $data[$this->import->get('unique_field')])
             ->first();
 
@@ -82,9 +86,7 @@ protected function findOrCreateEntry(array $data): void
                 return;
             }
 
-            $entry = Entry::make()
-                ->collection($this->import->get('destination.collection'))
-                ->locale($this->import->get('destination.site') ?? Site::selected()->handle());
+            $entry = Entry::make()->collection($collection)->locale($site);
         }
 
         if ($entry->id() && ! $this->import->get('strategy.update', true)) {

tests/ImporterTest.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Importer\Tests;
 
+use Illuminate\Foundation\Testing\DatabaseMigrations;
 use Illuminate\Support\Facades\File;
 use PHPUnit\Framework\Attributes\Test;
 use Statamic\Facades\Collection;
@@ -12,7 +13,7 @@
 
 class ImporterTest extends TestCase
 {
-    use PreventsSavingStacheItemsToDisk;
+    use DatabaseMigrations, PreventsSavingStacheItemsToDisk;
 
     #[Test]
     public function it_can_import_from_csv_files()

tests/TestCase.php

@@ -7,6 +7,8 @@
 use Statamic\Importer\ServiceProvider;
 use Statamic\Testing\AddonTestCase;
 
+use function Orchestra\Testbench\artisan;
+
 abstract class TestCase extends AddonTestCase
 {
     protected string $addonServiceProvider = ServiceProvider::class;
@@ -23,12 +25,30 @@ protected function getEnvironmentSetUp($app)
             'driver' => 'file',
             'path' => storage_path('framework/cache/outpost-data'),
         ]);
+
+        $app['config']->set('queue.batching.database', 'testing');
     }
 
-    protected function setSites($sites)
+    protected function setSites($sites): void
     {
         Site::setSites($sites);
 
         Config::set('statamic.system.multisite', Site::hasMultiple());
     }
+
+    /**
+     * Define database migrations.
+     *
+     * @return void
+     */
+    protected function defineDatabaseMigrations()
+    {
+        artisan($this, 'queue:batches-table');
+
+        artisan($this, 'migrate', ['--database' => 'testing']);
+
+        $this->beforeApplicationDestroyed(
+            fn () => artisan($this, 'migrate:rollback', ['--database' => 'testing'])
+        );
+    }
 }