[Feature]: uncompress strategy (zip/gzip) support for input data file

[FredoVelcro]

Feature description

Hello,

Sometimes, data feeds are compressed (zip/gzip) and need uncompression before import. It would be great to add an option to configure this ! Best regards, Fred

[FredoVelcro]

implementation sample (works but need to be done a better way i think) :

<?php

namespace App\DataHub\DataImporter\DataSource\Loader;

use App\DataHub\DataImporter\Exception\CompressionException;
use Pimcore\Bundle\DataImporterBundle\DataSource\Loader\HttpLoader;
use Pimcore\Bundle\DataImporterBundle\Exception\InvalidConfigurationException;
use ZipArchive;

class CompressedHttpLoader extends HttpLoader
{
    protected string $compression;

    /**
     * @param array $settings
     * @return void
     * @throws InvalidConfigurationException
     */
    public function setSettings(array $settings): void
    {
        parent::setSettings($settings);
        $this->compression = $settings['compression'];
    }

    /**
     * @return string
     * @throws CompressionException
     * @throws InvalidConfigurationException
     */
    public function loadData(): string
    {
        $importedFile = parent::loadData();
        switch ($this->compression) {
            case 'zip':
                $zip = new ZipArchive;
                if ($zip->open($importedFile) === TRUE) {
                    $zip->extractTo(dirname($importedFile));
                    $this->cleanup();
                    //@todo : below ($zip->getNameIndex(0)...) may not work eveytime, depends on how the archive was compressed, review this 
                    $importedFile = dirname($importedFile) . DIRECTORY_SEPARATOR . $zip->getNameIndex(0);
                    $this->importFilePath = $importedFile;
                    $zip->close();
                } else {
                    throw new CompressionException(sprintf('Could not unzip file : %s (%s)', $importedFile, $this->compression));
                }
                break;
            case 'gzip':
                $decodedData = gzdecode(file_get_contents($importedFile));
                file_put_contents($importedFile, $decodedData);
                break;
            default:
                throw new InvalidConfigurationException('Compression type not supported : ' . $this->compression);
        }

        return $importedFile;
    }
}

    App\DataHub\DataImporter\DataSource\Loader\CompressedHttpLoader:
        tags:
            - { name: "pimcore.datahub.data_importer.loader", type: "http_compressed" }

// TODO : use heritage from pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.http
// or move compression select box to another Ext component hierarchy
pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.http_compressed");
pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.http_compressed = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, {

    type: 'http',

    buildSettingsForm: function() {

        if(!this.form) {
            this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', {
                defaults: {
                    labelWidth: 200,
                    width: 600
                },
                border: false,
                items: [
                    {
                        xtype: 'combo',
                        fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_http_schema'),
                        name: this.dataNamePrefix + 'schema',
                        store: ['https://', 'http://'],
                        forceSelection: true,
                        value: this.data.schema,
                        allowBlank: false,
                        msgTarget: 'under',
                        width: 330
                    },{
                        xtype: 'textfield',
                        fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_http_url'),
                        name: this.dataNamePrefix + 'url',
                        value: this.data.url,
                        allowBlank: false,
                        msgTarget: 'under',
                        width: 900

                    },
                    {
                        xtype: 'combo',
                        fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_compression'),
                        name: this.dataNamePrefix + 'compression',
                        store: ['zip', 'gzip'],
                        forceSelection: true,
                        value: this.data.compression,
                        allowBlank: true,
                        msgTarget: 'under',
                        width: 330
                    }
                ]
            });
        }

        return this.form;
    }

});

[fashxp]

Sounds like a great addition. I would add this to the original adapters though, wdyt?

[FredoVelcro]

I think too ! :) Should be an option on the original http adapter