s3 storage for uploaded images

[toose]

Hi there--

My application is setup to work with django-tinymce4-lite and it works great when using local media storage. What i mean by that is if I choose to upload an image, the image gets uploaded to media/uploads, including several thumbnail 'versions' that get saved to media/_versions.

When I enable s3 as my media storage location, the image files are getting uploaded to the uploads folder correctly (folder is auto created at upload time), however thumbnails are not generated alongside, and in fact, after an upload, the _versions folder does not get created at all.

While I'm able to insert these images after upload using the image insert button, it's a bit frustrating in that unless the image has a descriptive filename, its impossible to tell which image is which without that image preview.

Here are the relevant settings regarding tinymce and s3 configuration:

# Media config
USE_S3 = int(os.environ.get('USE_S3', 0))
if USE_S3:
    AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_ID')
    AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_KEY')
    AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
    AWS_DEFAULT_ACL = 'public-read'
    AWS_S3_CUSTOM_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com'
    AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'}
    AWS_PRELOAD_METADATA = True
    AWS_QUERYSTRING_AUTH = False
    # s3 media config
    MEDIA_LOCATION = 'media'
    MEDIA_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/'
    DEFAULT_FILE_STORAGE = 'config.storage_backends.MediaStorage'
else:
    MEDIA_URL = '/media/'
    MEDIA_ROOT = os.path.join(BASE_DIR, 'media')

# TinyMCE-lite config
TINYMCE_DEFAULT_CONFIG = {
    'selector': 'textarea',
    'theme': 'modern',
    'plugins': 'link image preview codesample contextmenu table code lists colorpicker textcolor',
    'toolbar1': 'formatselect | bold italic underline | forecolor backcolor | alignleft aligncenter alignright alignjustify '
               '| bullist numlist | outdent indent | table | link image | codesample | preview code | fontsizeselect fontsize ',
    'content_css': 'https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css',
    'contextmenu': 'formats | link image',
    'menubar': False,
    'inline': False,
    'statusbar': True,
    'width': 'auto',
    'height': 360,
}

Here is storage_backends.py:

from storages.backends.s3boto3 import S3Boto3Storage
from django.conf import settings
from django.utils import timezone

class MediaStorage(S3Boto3Storage):
    location = settings.MEDIA_LOCATION
    default_acl = 'private'
    file_overwrite = False
    #custom_domain = False
    isfilecached = {}

    def isdir(self, name):
        if not name:  # Empty name is a directory
            return True

        if self.isfile(name):
            return False

        return True

    def isfile(self, name):
        if len(name.split('.')) > 1:
            return True
        try:
            name = self._normalize_name(self._clean_name(name))
            if self.isfilecached.get(name) is not None:
                return self.isfilecached.get(name)

            f = S3Boto3StorageFile(name, 'rb', self)
            if "directory" in f.obj.content_type:
                isfile = False
            else:
                isfile = True
        except Exception:
            isfile = False
        self.isfilecached[name] = isfile
        return isfile

    def move(self, old_file_name, new_file_name, allow_overwrite=False):

        if self.exists(new_file_name):
            if allow_overwrite:
                self.delete(new_file_name)
            else:
                raise "The destination file '%s' exists and allow_overwrite is False" % new_file_name

        old_key_name = self._encode_name(self._normalize_name(self._clean_name(old_file_name)))
        new_key_name = self._encode_name(self._normalize_name(self._clean_name(new_file_name)))

        k = self.bucket.meta.client.copy(
            {
                'Bucket': self.bucket.name,
                'Key': new_key_name
            },
            self.bucket.name,
            old_key_name
        )

        if not k:
            raise "Couldn't copy '%s' to '%s'" % (old_file_name, new_file_name)

        self.delete(old_file_name)

    def makedirs(self, name):
        name = self._normalize_name(self._clean_name(name))
        return self.bucket.meta.client.put_object(Bucket=self.bucket.name, Key=f'{name}/')

    def rmtree(self, name):
        name = self._normalize_name(self._clean_name(name))
        delete_objects = [{'Key': f"{name}/"}]

        dirlist = self.listdir(self._encode_name(name))
        for item in dirlist:
            for obj in item:
                obj_name = f"{name}/{obj}"
                if self.isdir(obj_name):
                    obj_name = f"{obj_name}/"
                delete_objects.append({'Key': obj_name})
        self.bucket.delete_objects(Delete={'Objects': delete_objects})

    def path(self, name):
        return name

    def listdir(self, name):
        directories, files = super().listdir(name)
        if '.' in files:
            files.remove('.')
        return directories, files

    def exists(self, name):
        if self.isdir(name):
            return True
        else:
            return super().exists(name)

    def get_modified_time(self, name):
        try:
            # S3 boto3 library requires that directorys have the trailing slash
            if self.isdir(name):
                name = f'{name}/'
            modified_date = super().get_modified_time(name)
        except Exception:
            modified_date = timezone.now()
        return modified_date

    def size(self, name):
        try:
            # S3 boto3 library requires that directorys have the trailing slash
            if self.isdir(name):
                name = f'{name}/'
            size = super().size(name)
        except Exception:
            size = 0
        return size

[ZeroCoolHacker]

Yes S3 storage should be added

[toose]

@ZeroCoolHacker - unfortunatley your comment doesn't help solve my problem. Any recommendations?

[romanvm]

I'm afraid you have opened this issue in a wrong place. All file management functionality is provided by django-filebrowser-no-grappelli that is a separate optional package from another developer. tinymce4-lite has nothing to do with it except for providing a JS callback for opening a filebrowser window. Please address your issue to django-filebrowser-no-grappelli developer(-s).

P.S. I guess the same happens if you upload an image via Django Admin.

[toose]

Thank you @romanvm for pointing me in the right direction!