search

cdhigh / KindleEar

Aggregates RSS and web content(Calibre recipe), sends to Kindle, and includes an e-ink optimized online reader.
http://cdhigh.github.io/KindleEar/
MIT License
2.73k stars 630 forks source link

科学美国人403 #585

Closed linux40 closed 5 years ago

linux40 commented 5 years ago

版本 1.26.5

rss地址

[
 {
   "protoPayload": {
     "@type": "type.googleapis.com/google.appengine.logging.v1.RequestLog",
     "appId": "s~appid",
     "versionId": "1",
     "requestId": "5da8fbd000ff05764c4f562e730001737e6b696e646c652d7273732d34320001776f726b65723a31000100",
     "ip": "0.1.0.2",
     "startTime": "2019-10-17T23:40:00.357964Z",
     "endTime": "2019-10-17T23:40:27.644463Z",
     "latency": "27.286499s",
     "megaCycles": "25474",
     "method": "GET",
     "resource": "/worker?u=admin_ce&id=5664922973241344",
     "httpVersion": "HTTP/1.1",
     "status": 200,
     "responseSize": "132",
     "referrer": "http://appid.appspot.com/deliver",
     "userAgent": "AppEngine-Google; (+http://code.google.com/appengine)",
     "urlMapEntry": "apps.module_backend.app",
     "host": "worker.appid.appspot.com",
     "cost": 0.000106053426,
     "taskQueueName": "deliverqueue1",
     "taskName": "93108701081679073911",
     "pendingTime": "4.557524312s",
     "finished": true,
     "instanceId": "00c61b117c5471e952c635aec004c585c8d335a647cc97db78e2f1bff700fada4b740ff06a",
     "line": [
       {
         "time": "2019-10-17T23:40:22.729646Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/olympics-marathon-moved-out-of-tokyo-over-heat-concerns/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:22.793293Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/observations/jim-peebles-richly-deserved-his-nobel-prize/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:22.843436Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/observations/we-have-no-reason-to-believe-5g-is-safe/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:22.920389Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/nasa-just-unveiled-the-space-suit-to-be-worn-by-the-first-woman-on-the-moon/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:22.973157Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/the-u-s-should-tighten-vaccination-mandates/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.025762Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/spacexs-starlink-constellation-could-swell-by-30-000-more-satellites/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.195512Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/observations/doctors-should-care-for-patients-and-social-justice/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.246992Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/video/the-color-of-noise-what-do-hues-have-to-do-with-sound/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.302755Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/why-do-some-people-need-less-sleep-its-in-their-dna/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.354172Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/extreme-snows-in-greenland-caused-ecosystems-reproductive-collapse/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.404845Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/observations/happier-babies-have-an-edge/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.461456Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/vaginal-fluid-transplants-hold-promise-but-raise-safety-concerns/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.514953Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/podcast/episode/tardigrade-protein-protects-dna-from-chemical-attack/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.567256Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/more-grazing-leads-to-fewer-fires-in-the-serengeti/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.621254Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/voices/were-sorry-vera-rubin/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.672159Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/the-exquisite-precision-of-time-crystals/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.724391Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://blogs.scientificamerican.com/observations/is-moral-relativism-really-a-problem/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       },
       {
         "time": "2019-10-17T23:40:23.776173Z",
         "severity": "WARNING",
         "logMessage": "fetch page failed(403 Forbidden):https://www.scientificamerican.com/article/trumps-border-wall-could-cause-flooding-in-texas/.",
         "sourceLocation": {
           "file": "/base/data/home/apps/s~appid/worker:1.418825641977741417/books/base.py",
           "line": "506",
           "functionName": "fetch"
         }
       }
     ],
     "moduleId": "worker",
     "appEngineRelease": "1.9.71",
     "traceId": "9c94bb378b29d476fe228131f40d998c",
     "first": true,
     "traceSampled": true
   },
   "insertId": "5da8fbeb0009d6a274e71754",
   "httpRequest": {
     "status": 200
   },
   "resource": {
     "type": "gae_app",
     "labels": {
       "project_id": "appid",
       "version_id": "1",
       "module_id": "worker",
       "zone": "us16"
     }
   },
   "timestamp": "2019-10-17T23:40:00.357964Z",
   "severity": "WARNING",
   "labels": {
     "clone_id": "00c61b117c5471e952c635aec004c585c8d335a647cc97db78e2f1bff700fada4b740ff06a"
   },
   "logName": "projects/appid/logs/appengine.googleapis.com%2Frequest_log",
   "operation": {
     "id": "5da8fbd000ff05764c4f562e730001737e6b696e646c652d7273732d34320001776f726b65723a31000100",
     "producer": "appengine.googleapis.com/request_id",
     "first": true,
     "last": true
   },
   "trace": "projects/appid/traces/9c94bb378b29d476fe228131f40d998c",
   "receiveTimestamp": "2019-10-17T23:40:27.650163258Z",
   "traceSampled": true
 }
]
cdhigh commented 5 years ago

是通过ip地址封锁的， 只有通过转发器，代理之类的方法才能突破。

linux40 commented 5 years ago

好吧，可能是不准机器人抓去文章内容吧。。。

cdhigh commented 5 years ago

是因为GAE太流行了，很多粗暴没节制的爬虫都部署在GAE上，导致很多网站开始封锁GAE的IP地址。 我使用 https://github.com/cdhigh/Forwarder 发现是能正常获取文章内容的。 如果你确实想看此网站，可以仿照 books/ZhihuDaily.py，创建一个py文件，通过转发器获取内容。 建议在heroku上搭建自己的转发器，内置的默认转发器负荷有点大了，免费额度只够每个月运行多半个月。