ninenines / gun

HTTP/1.1, HTTP/2, Websocket client (and more) for Erlang/OTP.
ISC License
898 stars 231 forks source link

Russian data not fetched properly #95

Closed grizzly-monkey closed 8 years ago

grizzly-monkey commented 8 years ago
-module(con).

-export([req/1]).

req(URL)->
  {ok, ConnPid} = gun:open("www.gazeta.ru", 80),
  MRef = monitor(process, ConnPid),
  print_body(ConnPid, MRef).

print_body(ConnPid, MRef) ->
    StreamRef = gun:get(ConnPid, "/"),
    receive
        {gun_response, ConnPid, StreamRef, fin, Status, Headers} ->
            no_data;
        {gun_response, ConnPid, StreamRef, nofin, Status, Headers} ->
            receive_data(ConnPid, MRef, StreamRef);
        {'DOWN', MRef, process, ConnPid, Reason} ->
            error_logger:error_msg("Oops!"),
            exit(Reason)
    after 1000 ->
        exit(timeout)
    end.

receive_data(ConnPid, MRef, StreamRef) ->
    receive
        {gun_data, ConnPid, StreamRef, nofin, Data} ->
            io:format("~s~n", [Data]),
            receive_data(ConnPid, MRef, StreamRef);
        {gun_data, ConnPid, StreamRef, fin, Data} ->
            io:format("~s~n", [Data]);
        {'DOWN', MRef, process, ConnPid, Reason} ->
            io:format("Oops!",[]),
            exit(Reason)
    after 1000 ->
        exit(timeout)
    end.

meta property="og:title" content="Ãëàâíûå íîâîñòè äíÿ - Ãàçåòà.Ru" meta property="og:type" content="website" meta property="fb:app_id" content="406317839387165" meta property="og:site_name" content="Ãàçåòà.Ru" meta property="og:url" content="http://www.gazeta.ru/"

should be

meta property="og:title" content="Главные новости дня - Газета.Ru" meta property="og:type" content="website" meta property="fb:app_id" content="406317839387165" meta property="og:site_name" content="Газет

essen commented 8 years ago

If you are using io:format to print Unicode text you need to use ~ts instead of ~s.

grizzly-monkey commented 8 years ago

I have to manually convert it , I am good thanks