node-modules / urllib

Request HTTP(s) URLs in a complex world.
MIT License
726 stars 118 forks source link

Parse Error cause callback twice!! #45

Closed fengmk2 closed 9 years ago

fengmk2 commented 9 years ago
'use strict';

var co = require('co');
var urllib = require('../');

co(function* () {
  var result = yield urllib.requestThunk('http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php');
  console.log('status: %s, body size: %d, headers: %j',
    result.res.statusCode, result.data.length, result.res.headers);
})();
$ DEBUG=urllib node --harmony examples/co_urllib.js 
  urllib Request#1 GET http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php with headers {"User-Agent":"node-urllib/2.0.1 node/v0.11.14 (darwin 14.0.0; x64)"}, options.path: /go/market/promotion-act/shengxian1111_mobile.php +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php `req response` event emit: status 200, headers: {"server":"Tengine","date":"Fri, 31 Oct 2014 15:06:04 GMT","content-type":"text/html; charset=GB2312","transfer-encoding":"chunked","connection":"keep-alive","vary":"Accept-Encoding","expires":"Fri, 31 Oct 2014 15:06:04 GMT","cache-control":"max-age=0"} +67ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1179 +3ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 11520 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +6ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 790 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 642 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +4ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 325 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1107 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1210 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 222 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +2ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +3ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 2880 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1440 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 202 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 1231 +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res data` event emit, size 117 +1ms
  urllib Request#1 /go/market/promotion-act/shengxian1111_mobile.php `req error` event emit, RequestError: Parse Error (req "error") +0ms
  urllib [143ms] 0 bytes HTTP -1 GET /go/market/promotion-act/shengxian1111_mobile.php +1ms
  urllib Request#1 /go/market/promotion-act/shengxian1111_mobile.php: `req close` event emit +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res aborted` event emit, total size 115025 +1ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res end` event emit, total size 115025, _dumped: false +0ms
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: Remote socket was terminated before `response.end()` was called +1ms
[urllib:warn] [Fri Oct 31 2014 23:06:04 GMT+0800 (CST)] [worker:3654] GET http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php callback twice!!!
  urllib Request#1 http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php: `res close` event emit, total size 115025 +0ms
/Users/mk2/git/urllib/node_modules/co/index.js:292
    throw err;
          ^
Error: Parse Error
    at Error (native)
    at Socket.socketOnData (_http_client.js:309:20)
    at Socket.emit (events.js:107:17)
    at readableAddChunk (_stream_readable.js:162:16)
    at Socket.Readable.push (_stream_readable.js:125:10)
    at TCP.onread (net.js:514:20)
fengmk2 commented 9 years ago
Error: Parse Error
    at Error (native)
    at Socket.socketOnData (_http_client.js:309:20)
    at Socket.emit (events.js:107:17)
    at readableAddChunk (_stream_readable.js:162:16)
    at Socket.Readable.push (_stream_readable.js:125:10)
    at TCP.onread (net.js:514:20)
fengmk2 commented 9 years ago
var http = require('http');
var size = 0;

var options = {
  method: 'GET',
  hostname: 'www.tmall.com',
  port: 80,
  path: '/go/market/promotion-act/shengxian1111_mobile.php',
  headers: {
    'Accept-Encoding': 'gzip'
  }
};

http.request(options, function (res) {
  console.log(res.statusCode, res.headers);
  res.on('data', function (data) {
    console.log('data size: %d', data.length);
    size += data.length;
  }).on('end', function () {
    console.log('response end: %d', size);
  }).on('close', function () {
    console.log('response close: %d', size);
  }).on('error', function (err) {
    console.log(err.stack);
    console.log('response error: %d', size);
  });
}).on('error', function (err) {
  console.log(err.stack);
  console.log('size: %s', size);
}).end();
$ node examples/req_parse_error.js 
200 { server: 'Tengine',
  date: 'Fri, 31 Oct 2014 15:33:12 GMT',
  'content-type': 'text/html; charset=GB2312',
  'transfer-encoding': 'chunked',
  connection: 'keep-alive',
  vary: 'Accept-Encoding',
  expires: 'Fri, 31 Oct 2014 15:33:12 GMT',
  'cache-control': 'max-age=0',
  'content-encoding': 'gzip' }
data size: 1155
data size: 8640
data size: 1440
data size: 1440
data size: 1440
data size: 4320
data size: 202
response end: 18637
fengmk2 commented 9 years ago

curl -v 'http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php'

("")}for(var f=a?void 0:{},g=document.cookie?document.cookie.split("; "):[],h=0,i=g.length;i>h;h++){var j=g[h].split("="),k=j.shift(),l=j.join("=");if(a&&a===k){f=l;break}a||void 0===l||(f[k]=l)}return f},b.removeCookie=function(a,c){return void 0===b.cookie(a)?!1:(c=c||{},c.expires=-1,b.cookie(a,"",c),!b.cookie(a))}}(window);
* Malformed encoding found in chunked-encoding
* Closing connection 0
curl: (56) Malformed encoding found in chunked-encoding
!function(a,b){var c,d,e=document,f=a.devicePixelRatio||1,g=e.documentElement.clientWidth,h=e.documentElement.clientHeight;screen?Math.abs(screen.width-g*f)<.2*screen.width?(g=screen.width/f,h=screen.height/f,c=screen.width,d=screen.height):(g=screen.width,h=screen.height,c=screen.width*f,d=screen.height*f):(c=g*f,d=h*f);var i="createTouch"in e&&"ontouchstart"in a?1:0,j=["dw:"+g,"dh:"+h,"pw:"+c,"ph:"+d,"ist:"+i].join("&");b.cookie("_med",j,{expires:3650})}(window,_med);</script>
fengmk2 commented 9 years ago

use request also throw parse error

var request = require('request');
request('http://www.tmall.com/go/market/promotion-act/shengxian1111_mobile.php', function (err, response, body) {
  if (err) {
    throw err;
  }
  console.log(response.statusCode, response.headers);
  console.log(body.length);
});
/Users/mk2/git/urllib/examples/co_urllib.js:21
    throw err;
          ^
Error: Parse Error
    at Error (native)
    at Socket.socketOnData (_http_client.js:309:20)
    at Socket.emit (events.js:107:17)
    at readableAddChunk (_stream_readable.js:162:16)
    at Socket.Readable.push (_stream_readable.js:125:10)
    at TCP.onread (net.js:514:20)
fengmk2 commented 9 years ago

err, url was fixed ..

zouyixiong commented 8 years ago

http://dl33.yunpan.360.cn/oss/screenshot/MTExOTQ0NjBfMTEzODA5MzM4MS5qcGc=?scid=33&fhash=e717ff74bf434ef1b889eac2258d9ab1f25974b6&e=1461771499&uid=1366530053&validity=30&s=44975124dc0153386e2c84c7bc85f8eb (this url will only alive in 12 hours)

I get the same error with this URL now. But actually the file is downloaded.

[urllib:warn] [Wed Apr 27 2016 12:02:25 GMT+0800 (CST)] [worker:28418] GET http://dl73.yunpan.360.cn/oss/screenshot/MTExOTE0MjBfMTEzNzU4NzQ1Ni5qcGc=?scid=73&fhash=b10a743 435da799f56fafb1de7fb2be2f7976597&e=1461771499&uid=1366530053&validity=30&s=60853544b50587b81708af8baa923c4f callback twice!!! [Wed Apr 27 2016 12:02:25] [WARN] [urllib:warn] [Wed Apr 27 2016 12:02:25 GMT+0800 (CST)] [worker:28418] ResponseError: Parse Error (req "error") stack: ResponseError: Parse Error (req "error") at Error (native) at Socket.socketOnData (_http_client.js:309:20) at emitOne (events.js:77:13) at Socket.emit (events.js:169:7) at readableAddChunk (_stream_readable.js:146:16) at Socket.Readable.push (_stream_readable.js:110:10) at TCP.onread (net.js:529:20)

Can you help? node:v4.4.0 urllib: 2.9.0