mikf / gallery-dl

Command-line program to download image galleries and collections from several image hosting sites
GNU General Public License v2.0
11.4k stars 931 forks source link

The dreaded '429 too many requests' #3709

Closed rossypants closed 1 year ago

rossypants commented 1 year ago

I apologize for bogging down this amazing resource with my ignorance, but I can't seem to figure out the API tokens and ID issue. I keep getting longer and longer request delays from deviantart and I think I've taken all the recommended steps.

I've created the config file in ~/.config/gallery-dl/config.json I've published the app in deviantart and copied and pasted the client-id and client-secret into the config file (with dashes, not underscores). I've cleared the cache and refreshed token When I clear the cache, it reports: Could not parse '/Users/****/.config/gallery-dl/config.json': Expecting property name enclosed in double quotes: line 1 column 2 (char 1) I've ran the config.json contents through https://jsonlint.com/ and it reports it as valid

I should mention I don't know what I'm doing at all. I'm on Macos using terminal and am barely able to figure out what's happening. I don't know code at all, this all feels like magic - especially the formatting in these threads. I'm just using basic textedit to modify the files and then changing the file extensions after it saves in the finder. Inefficient as hell, I'm sure. I have the config file pasted below and would really appreciate some guidance on how to proceed. I feel like I'm maybe 1 or 2 steps away here. It works, but is just slow as all get out. Thanks in advance for any help and for all the hard work!

(Bonus question - Which part of the config file lets me set the default save directory? I've been using '--dest' to set it each time and it's a little cumbersome. Cheers!)

config ``` json { "extractor": { "base-directory": "./gallery-dl/", "parent-directory": false, "postprocessors": null, "archive": null, "cookies": null, "cookies-update": true, "proxy": null, "skip": true, "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", "retries": 4, "timeout": 30.0, "verify": true, "fallback": true, "sleep": 0, "sleep-request": 0, "sleep-extractor": 0, "path-restrict": "auto", "path-replace": "_", "path-remove": "\\u0000-\\u001f\\u007f", "path-strip": "auto", "path-extended": true, "extension-map": { "jpeg": "jpg", "jpe" : "jpg", "jfif": "jpg", "jif" : "jpg", "jfi" : "jpg" }, "artstation": { "external": false, "pro-first": true }, "aryion": { "username": null, "password": null, "recursive": true }, "bbc": { "width": 1920 }, "blogger": { "videos": true }, "cyberdrop": { "domain": null }, "danbooru": { "username": null, "password": null, "external": false, "metadata": false, "ugoira": false }, "derpibooru": { "api-key": null, "filter": 56027 }, "deviantart": { "client-id": “12345”, "client-secret": “123456789012344567890”, "auto-watch": false, "auto-unwatch": false, "comments": false, "extra": false, "flat": true, "folders": false, "group": true, "include": "gallery", "journals": "html", "mature": true, "metadata": false, "original": true, "wait-min": 0 }, "e621": { "username": null, "password": null }, "exhentai": { "username": null, "password": null, "domain": "auto", "limits": true, "metadata": false, "original": true, "sleep-request": 5.0 }, "flickr": { "videos": true, "size-max": null }, "furaffinity": { "descriptions": "text", "external": false, "include": "gallery", "layout": "auto" }, "gelbooru": { "api-key": null, "user-id": null }, "gfycat": { "format": ["mp4", "webm", "mobile", "gif"] }, "gofile": { "api-token": null, "website-token": "12345" }, "hentaifoundry": { "include": "pictures" }, "hitomi": { "format": "webp", "metadata": false }, "idolcomplex": { "username": null, "password": null, "sleep-request": 5.0 }, "imgbb": { "username": null, "password": null }, "imgur": { "mp4": true }, "inkbunny": { "username": null, "password": null, "orderby": "create_datetime" }, "instagram": { "api": "rest", "cookies": null, "include": "posts", "sleep-request": [6.0, 12.0], "videos": true }, "khinsider": { "format": "mp3" }, "luscious": { "gif": false }, "mangadex": { "api-server": "https://api.mangadex.org", "api-parameters": null, "lang": null, "ratings": ["safe", "suggestive", "erotica", "pornographic"] }, "mangoxo": { "username": null, "password": null }, "newgrounds": { "username": null, "password": null, "flash": true, "format": "original", "include": "art" }, "nana": { "favkey": null }, "nijie": { "username": null, "password": null, "include": "illustration,doujin" }, "nitter": { "quoted": false, "retweets": false, "videos": true }, "oauth": { "browser": true, "cache": true, "host": "localhost", "port": 6414 }, "paheal": { "metadata": false }, "pillowfort": { "external": false, "inline": true, "reblogs": false }, "pinterest": { "domain": "auto", "sections": true, "videos": true }, "pixiv": { "refresh-token": null, "include": "artworks", "metadata": false, "metadata-bookmark": false, "tags": "japanese", "ugoira": true }, "reactor": { "gif": false, "sleep-request": 5.0 }, "reddit": { "comments": 0, "morecomments": false, "date-min": 0, "date-max": 253402210800, "date-format": "%Y-%m-%dT%H:%M:%S", "id-min": null, "id-max": null, "recursion": 0, "videos": true }, "redgifs": { "format": ["hd", "sd", "gif"] }, "sankaku": { "username": null, "password": null, "refresh": false }, "sankakucomplex": { "embeds": false, "videos": true }, "skeb": { "article": false, "filters": null, "sent-requests": false, "thumbnails": false }, "smugmug": { "videos": true }, "seiga": { "username": null, "password": null }, "subscribestar": { "username": null, "password": null }, "tsumino": { "username": null, "password": null }, "tumblr": { "avatar": false, "external": false, "inline": true, "posts": "all", "offset": 0, "original": true, "reblogs": true }, "twitter": { "username": null, "password": null, "cards": false, "conversations": false, "pinned": false, "quoted": false, "replies": true, "retweets": false, "strategy": null, "text-tweets": false, "twitpic": false, "unique": true, "users": "timeline", "videos": true }, "unsplash": { "format": "raw" }, "vsco": { "videos": true }, "wallhaven": { "api-key": null, "metadata": false, "include": "uploads" }, "weasyl": { "api-key": null, "metadata": false }, "weibo": { "livephoto": true, "retweets": true, "videos": true }, "ytdl": { "enabled": false, "format": null, "generic": true, "logging": true, "module": null, "raw-options": null }, "zerochan": { "username": null, "password": null, "metadata": false }, "booru": { "tags": false, "notes": false } }, "downloader": { "filesize-min": null, "filesize-max": null, "mtime": true, "part": true, "part-directory": null, "progress": 3.0, "rate": null, "retries": 4, "timeout": 30.0, "verify": true, "http": { "adjust-extensions": true, "chunk-size": 32768, "headers": null, "validate": true }, "ytdl": { "format": null, "forward-cookies": false, "logging": true, "module": null, "outtmpl": null, "raw-options": null } }, "output": { "mode": "auto", "progress": true, "shorten": true, "ansi": false, "colors": { "success": "1;32", "skip" : "2" }, "skip": true, "log": "[{name}][{levelname}] {message}", "logfile": null, "unsupportedfile": null }, "netrc": false } ```
mikf commented 1 year ago

I'm just using basic textedit to modify the files

That seems to be the root cause of your problems. Try a plain-text text editor that doesn't insert any fancy quotes like (JSON expects the standard ASCII quote ") and doesn't write a BOM, which is probably the cause of Expecting property name enclosed in double quotes: line 1 column 2 (char 1).


I copied your config into my own editor, replaced all with ", and now I'm getting nvalid control character at: line 75 column 34 (char 1672).

Just copy these two lines, fill in your values, and save the file without BOM

            "client-id": "12345",
            "client-secret": "1234567890",

(Bonus question - Which part of the config file lets me set the default save directory? I've been using '--dest' to set it each time and it's a little cumbersome. Cheers!)

        "base-directory": "./gallery-dl/",
rossypants commented 1 year ago

I re-copied the provided config file from the DOCS section of this post and re-inserted the client-id and client-secret using an actual code editor 'Sublime Text' and it works great now. Thank you for your time and patience!