[Bug]: Maatwebsite\Excel\Jobs\ReadChunk has been attempted too many times

[kstmostofa]

Is the bug applicable and reproducable to the latest version of the package and hasn't it been reported before?

• [X] Yes, it's still reproducable

What version of Laravel Excel are you using?

3.1.30

What version of Laravel are you using?

10

What version of PHP are you using?

8.1

Describe your issue

Illuminate\Queue\MaxAttemptsExceededException: Maatwebsite\Excel\Jobs\ReadChunk has been attempted too many times. in /Volumes/Backup Data/lara-1m/vendor/laravel/framework/src/Illuminate/Queue/Worker.php:785

How can the issue be reproduced?

namespace App\Imports;

use App\Models\Leads;
use Illuminate\Support\Facades\DB;
use Illuminate\Validation\Rule;
use Illuminate\Support\Collection;
use Illuminate\Contracts\Queue\ShouldQueue;
use Maatwebsite\Excel\Concerns\Importable;
use Maatwebsite\Excel\Concerns\SkipsErrors;
use Maatwebsite\Excel\Concerns\SkipsFailures;
use Maatwebsite\Excel\Concerns\SkipsOnError;
use Maatwebsite\Excel\Concerns\SkipsOnFailure;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Imports\HeadingRowFormatter;
use Maatwebsite\Excel\Concerns\RegistersEventListeners;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Session;
use Maatwebsite\Excel\Events\AfterImport;

HeadingRowFormatter::default('none');

class LeadsImport implements
    ToCollection,
    WithHeadingRow,
    SkipsOnError,
    WithValidation,
    SkipsOnFailure,
    WithChunkReading,
    ShouldQueue,
    WithEvents
{
    use Importable, SkipsErrors, SkipsFailures, RegistersEventListeners;

    /**
     * @param array $row
     *
     * @return \Illuminate\Database\Eloquent\Model|null
     */
    public function collection(Collection $rows)
    {
        // dd($rows);
        foreach ($rows as $row) {

            if ($row['Lead Owner'] == "" && $row['Lead Status'] == "") {
                $status_lead = 2;
            } elseif ($row['Lead Owner'] != "" && $row['Lead Status'] == "") {
                $status_lead = 1;
            } else {
                $status_lead = $row['Lead Status'];
            }
            if ($row['Mobile'] != ""  && preg_match('/[0-9]{10}/s', $row['Mobile'])) {
                $this->leadCheck = DB::table('leads')
                    ->where('mobile', $row['Mobile'])
                    ->where('user_id', 1)
                    ->where('client_id', 1)
                    ->count();
                if ($this->leadCheck > 0) {
                } else {
                    $member_type = explode(',', $row['Member Type']);
                    $alt_numbers = explode('-', $row['Alt Number']);
                    $address_type = ['permanent', 'temp'];
                    $address = [$row['Address 1'], $row['Address 2']];
                    $country = [$row['Country 1'], $row['Country 2']];
                    $state = [$row['State 1'], $row['State 2']];
                    $district = [$row['District 1'], $row['District 2']];
                    $pincode = [$row['Pincode 1'], $row['Pincode 2']];

                    $lead = new Leads();
                    $lead->user_id = 1;
                    $lead->client_id = 1;
                    $lead->account_type = $row['Account Type'];
                    $lead->department = $row['Department'];
                    $lead->member_type = json_encode($member_type);

                    $lead->lead_source = $row['Lead Source'];
                    $lead->lead_status = $status_lead;
                    $lead->preferred_language = $row['Preferred Language'];
                    $lead->agent_id = $row['Lead Owner'];
                    $lead->profile_photo = $row['Profile Photo'];
                    $lead->member_id = $row['Member ID'];
                    $lead->name = $row['Name'];

                    $lead->date_of_birth = ($row['Date of Birth'] != '') ? date('Y-m-d', strtotime($row['Date of Birth'])) : NULL;
                    $lead->date_of_joining = ($row['Date of Joining'] != '') ? date('Y-m-d', strtotime($row['Date of Joining'])) : NULL;
                    $lead->date_of_anniversary = ($row['Date Of Anniversary'] != '') ? date('Y-m-d', strtotime($row['Date Of Anniversary'])) : NULL;
                    $lead->rfid = $row['RFID'];
                    $lead->gender = $row['Gender'];
                    $lead->blood_group = $row['Blood Group'];
                    $lead->married_status = $row['Married Status'];
                    $lead->email = $row['Email'];
                    $lead->mobile = $row['Mobile'];

                    $lead->alt_numbers = empty($alt_numbers) ? '' : implode(',', $alt_numbers); //json_encode($alt_numbers);

                    $lead->id_proof = $row['Id Proof'];
                    $lead->created_by = 1;
                    $lead->qualification = $row['Qualification'];
                    $lead->branch = $row['Branch'];
                    $lead->profession = $row['Profession'];
                    $lead->sms_required = $row['Sms Requred'];
                    $lead->call_required = $row['Call Requred'];
                    $lead->sms_language = $row['Sms Language'];
                    $lead->lead_response = $row['Lead Response'];
                    $lead->address_type = serialize($address_type);
                    $lead->address = empty($address) ? '' : serialize($address);
                    $lead->country = empty($country) ? '' : serialize($country);
                    $lead->state = empty($state) ? '' : serialize($state);
                    $lead->district = empty($district) ? '' : serialize($district);
                    $lead->pincode = empty($pincode) ? '' : serialize($pincode);
                    $lead->is_member = $row['Is Member'] ? $row['Is Member'] : 0;

                    $lead->save();
                    Log::alert($lead->id . ' lead created successfully');
                }
            }

        }
    }

    public function rules(): array
    {
        return [
            '*.mobile' => ['Mobile', 'unique:leads,mobile']

        ];
    }

    public function chunkSize(): int
    {
        return 10000;
    }

    public static function afterImport(AfterImport $event)
    {
        Session::flash('import_message',  'Files Imported Successfully!');
    }
}

What should be the expected behaviour?

when i am trying with less entries of file then everything working fine. but when entries goes above 100K or 200K then coming this error

Illuminate\Queue\MaxAttemptsExceededException: Maatwebsite\Excel\Jobs\ReadChunk has been attempted too many times. in /Volumes/Backup Data/lara-1m/vendor/laravel/framework/src/Illuminate/Queue/Worker.php:785

[celeEtionic]

It seems that it has exceeded the execution timeout of the queue where the job is running. Try increasing the timeout in your config files For reference in our project importing 30.000 rows takes a half hour

[stale[bot]]

This bug report has been automatically closed because it has not had recent activity. If this is still an active bug, please comment to reopen. Thank you for your contributions.

[Uhasith]

I'm getting the same error. I did increase max_execution_time = 120000 both inside php.ini and config/queue.php file as well like below. Plus i added public $failOnTimeout = false; to ProductsImport class file as well but nothing seems to be working. I'm trying this on my Loca env as well so no idea what is the problem behind this.

 'database' => [
            'driver' => 'database',
            'connection' => env('DB_QUEUE_CONNECTION'),
            'table' => env('DB_QUEUE_TABLE', 'jobs'),
            'queue' => env('DB_QUEUE', 'default'),
            'retry_after' => (int) env('DB_QUEUE_RETRY_AFTER', 120000),
            'after_commit' => false,
            'max_attempts' => 5,
        ],

[kstmostofa]

Split your jobs into some parts like ProductsImport will only read the file data using chunks

 <?php

namespace App\Imports;

use App\Jobs\ProductsImportJob;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithHeadingRow;

class ProductsImport implements ToCollection, WithHeadingRow, WithChunkReading, ShouldQueue
{

    public function collection(Collection $collection)
    {
        foreach ($collection as $row) {
            ProductsImportJob::dispatch($row->toArray());
        }
    }

    public function chunkSize(): int
    {
        return 1000;
    }
}

Note: You can adjust the chunk size according to the data column size. in ProductsImportJob you can do the database transaction.