UnicodeDecodeError with CharField

[GerardPaligot]

Hello,

I got an UnicodeDecodeError when I used CharField in a serializer. This is the error:

Traceback (most recent call last):
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/core/handlers/base.py", line 111, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 57, in wrapped_view
    return view_func(*args, **kwargs)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/views/generic/base.py", line 69, in view
    return self.dispatch(request, *args, **kwargs)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/views.py", line 452, in dispatch
    response = self.handle_exception(exc)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/views.py", line 449, in dispatch
    response = handler(request, *args, **kwargs)
  File "/Users/Gerard/Documents/workspace-django/zds-site/zds/mp/api/views.py", line 332, in post
    return self.create(request, *args, **kwargs)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/mixins.py", line 21, in create
    headers = self.get_success_headers(serializer.data)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 466, in data
    ret = super(Serializer, self).data
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 213, in data
    self._data = self.to_representation(self.instance)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 435, in to_representation
    ret[field.field_name] = field.to_representation(attribute)
  File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/fields.py", line 593, in to_representation
    return six.text_type(value)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 8: ordinal not in range(128)

This is my code simplify:

views.py

class PrivatePostListAPI(MarkPrivateTopicAsRead, ListCreateAPIView):
    """
    Private post resource to list of a member.
    """

    permission_classes = (IsAuthenticated, IsParticipantFromPrivatePost)
    list_key_func = PagingPrivatePostListKeyConstructor()

    @etag(list_key_func)
    @cache_response(key_func=list_key_func)
    def get(self, request, *args, **kwargs):
        """
        Documentation for Swagger
        """
        return self.list(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        """
        Documentation for Swagger
        """
        return self.create(request, *args, **kwargs) # This is the line 332 in the stacktrace

    def get_serializer_class(self):
        if self.request.method == 'GET':
            return PrivatePostSerializer
        elif self.request.method == 'POST':
            return PrivatePostCreateSerializer

serializers.py

class PrivatePostCreateSerializer(serializers.ModelSerializer):
    """
    Serializer to update the last private post of a private topic.
    """

    class Meta:
        model = PrivatePost
        fields = ('privatetopic', 'author', 'text', 'text_html', 'pubdate', 'update', 'position_in_topic')
        read_only_fields = ('privatetopic', 'author', 'text_html', 'pubdate', 'update', 'position_in_topic')

    def create(self, validated_data):
        # some stuff...
        return instance

    def validate_text(self, value):
        """
        Checks about text.

        :param value: text value
        :return: text value
        """
        msg = None
        if value:
            if value.strip() == '':
                msg = _(u'Le champ text ne peut être vide')
            if msg is not None:
                serializers.ValidationError(message)
        return value

The bug appears when I would like to execute my post request with the value "Une réponse" in the field text of my model PrivatePost (text is a TextField in my model).

I'm using DRF 3.1.1.

[rkirmizi]

this is not about DRF you have to declare your encoding in your serializers.py like:

-- coding: utf-8 --

2015-05-05 23:40 GMT+03:00 Gérard Paligot notifications@github.com:

Hello,

I got an UnicodeDecodeError when I used CharField in a serializer. This is the error:

Traceback (most recent call last): File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/core/handlers/base.py", line 111, in get_response response = wrapped_callback(request, _callback_args, _callback_kwargs) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 57, in wrapped_view return view_func(_args, _kwargs) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/django/views/generic/base.py", line 69, in view return self.dispatch(request, _args, _kwargs) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/views.py", line 452, in dispatch response = self.handle_exception(exc) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/views.py", line 449, in dispatch response = handler(request, _args, _kwargs) File "/Users/Gerard/Documents/workspace-django/zds-site/zds/mp/api/views.py", line 332, in post return self.create(request, _args, *_kwargs) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/mixins.py", line 21, in create headers = self.get_success_headers(serializer.data) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 466, in data ret = super(Serializer, self).data File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 213, in data self._data = self.to_representation(self.instance) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/serializers.py", line 435, in to_representation ret[field.field_name] = field.to_representation(attribute) File "/Users/Gerard/.virtualenvs/zdsenv/lib/python2.7/site-packages/rest_framework/fields.py", line 593, in to_representation return six.text_type(value)UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 8: ordinal not in range(128)

This is my code simplify:

views.py

class PrivatePostListAPI(MarkPrivateTopicAsRead, ListCreateAPIView): """ Private post resource to list of a member. """

permission_classes = (IsAuthenticated, IsParticipantFromPrivatePost)
list_key_func = PagingPrivatePostListKeyConstructor()

@etag(list_key_func)
@cache_response(key_func=list_key_func)
def get(self, request, *args, **kwargs):
    """        Documentation for Swagger        """
    return self.list(request, *args, **kwargs)

def post(self, request, *args, **kwargs):
    """        Documentation for Swagger        """
    return self.create(request, *args, **kwargs) # This is the line 332 in the stacktrace

def get_serializer_class(self):
    if self.request.method == 'GET':
        return PrivatePostSerializer
    elif self.request.method == 'POST':
        return PrivatePostCreateSerializer

serializers.py

class PrivatePostCreateSerializer(serializers.ModelSerializer): """ Serializer to update the last private post of a private topic. """

class Meta:
    model = PrivatePost
    fields = ('privatetopic', 'author', 'text', 'text_html', 'pubdate', 'update', 'position_in_topic')
    read_only_fields = ('privatetopic', 'author', 'text_html', 'pubdate', 'update', 'position_in_topic')

def create(self, validated_data):
    # some stuff...
    return instance

def validate_text(self, value):
    """        Checks about text.        :param value: text value        :return: text value        """
    msg = None
    if value:
        if value.strip() == '':
            msg = _(u'Le champ text ne peut être vide')
        if msg is not None:
            serializers.ValidationError(message)
    return value

The bug appears when I would like to execute my post request with the value "Une réponse" in the field text of my model PrivatePost.

— Reply to this email directly or view it on GitHub https://github.com/tomchristie/django-rest-framework/issues/2912.

ISM Teknoloji

İstanbul Üniversitesi ARGEM Binası Avcılar Istanbul Turkiye 34320

+90 (537) 502 92 94

[GerardPaligot]

The encoding was already specified. https://github.com/GerardPaligot/zds-site/blob/refacto_mp/zds/mp/api/serializers.py#L1

[xordoquy]

@rkirmizi if that was the issue, it would appear during import time. @GerardPaligot your default encoding is set to ascii and char such as "ê" aren't part of this encoding. You need to change the default encoding to utf8. Note that by doing so, you'll probably have issue with the various encoders. They should limit the output to ascii - at least JSON will.

[GerardPaligot]

@xordoquy Thanks for your answer. This is probably not the right place to ask but how can I change this default encoding?

[xordoquy]

Usually, it'll be something like calling sys.setdefaultencoding() and reloading sys. Searching on the web will get you a more detailed answer.

Edit: The proper fix is to set LC_ALL / LANG to utf8. If none is defined, Django will fallback to ascii.

[GerardPaligot]

Thanks a lot. I'll make my research in that way.

[freakypie]

This is the required code:

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

It looks dangerous; Is there another way? If DRF requires this for creating an API, is it documented somewhere? (I couldn't find it.) If not, I'll be happy to help

[xordoquy]

DRF doesn't require it. OP's code most probably does. Alternatively setting the correct locales is prefered, dunno why i didn't mentionned it earlier

[freakypie]

Would you happen to have a reference to what you mean for anyone else (including me) who stumbles on this problem?

[xordoquy]

@freakypie http://stackoverflow.com/questions/4398540/unicodeencodeerror-when-saving-imagefield-containing-non-ascii-characters-in-djan for example.

[theromis]

I had same issue

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 149, in get_response
    response = self.process_exception_by_middleware(e, request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 147, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/viewsets.py", line 87, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 466, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 463, in dispatch
    response = handler(request, *args, **kwargs)
  File "/opt/git/lonje/lonje/account/rest_api.py", line 296, in users
    return Response(serializer.data)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 674, in data
    ret = super(ListSerializer, self).data
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 239, in data
    self._data = self.to_representation(self.instance)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 614, in to_representation
    self.child.to_representation(item) for item in iterable
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 472, in to_representation
    ret[field.field_name] = field.to_representation(attribute)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/fields.py", line 703, in to_representation
    return six.text_type(value)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 5: ordinal not in range(128)

why it using ascii codec here?

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

adding this line fixed it, but looks not clear, is it possible to put some django rest framework option for that?

[xordoquy]

adding this line fixed it, but looks not clear, is it possible to put some django rest framework option for that?

No. The proper fix is done at environment level by setting the correct LANG / LC_ALL. If none is defined, it'll falls back to ascii as shown in the backtrace.

[theromis]

thank you will do that.

[theromis]

even after that I'm gettin same exception uwsgi.ini have

  env = LANG=en_US.UTF-8
  env = LC_ALL=en_US.UTF-8

in code I'm printing this envs and they exists

en_US.UTF-8
en_US.UTF-8
Internal Server Error: /rest/chatroom/main-room/users/
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 149, in get_response
    response = self.process_exception_by_middleware(e, request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 147, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/viewsets.py", line 87, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 466, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 463, in dispatch
    response = handler(request, *args, **kwargs)
  File "/opt/git/lonje/lonje/account/rest_api.py", line 303, in users
    return Response(serializer.data)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 674, in data
    ret = super(ListSerializer, self).data
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 239, in data
    self._data = self.to_representation(self.instance)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 614, in to_representation
    self.child.to_representation(item) for item in iterable
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/serializers.py", line 472, in to_representation
    ret[field.field_name] = field.to_representation(attribute)
  File "/usr/local/lib/python2.7/dist-packages/rest_framework/fields.py", line 703, in to_representation
    return six.text_type(value)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 16: ordinal not in range(128)

[wholeinsoul]

@theromis : Were you ever able to resolve this ? Thanks.

[theromis]

@bishtpradeep Yes, it works perfectly fine

[wholeinsoul]

@theromis : How were you able to solve it ? Did setting env = LANG=en_US.UTF-8 and env = LC_ALL=en_US.UTF-8 help ? Thanks.