mikf / gallery-dl

Command-line program to download image galleries and collections from several image hosting sites
GNU General Public License v2.0
12.08k stars 983 forks source link

[Question] Possible to Ignore Downloads With Reddit? #4042

Closed cheese529 closed 1 year ago

cheese529 commented 1 year ago

I am trying to scrape reddit for as many imgur links as possible before the purge on May 17th. I am using "whitelist": ["imgur"], in order to do this inside my reddit config but it still downloads material from reddit, is there anyway to ignore this behavior and only download stuff from imgur?

taskhawk commented 1 year ago

Perhaps adding "download": false to your Reddit config?

"reddit": {
    ...
    "download": false,
    ...
}
cheese529 commented 1 year ago

"download": false

This works but then gallery-dl also skips the imgur downloads even with "whitelist": ["imgur"],

mikf commented 1 year ago

Or you set image-filter for reddit. and only reddit, to "False".


"download": false

This works but then gallery-dl also skips the imgur downloads even with "whitelist": ["imgur"],

It works if you disable download for reddit only.

cheese529 commented 1 year ago

It works if you disable download for reddit only.

I apologize for being very dumb but can you please elaborate a bit more on this? It should be working because I don't have any other downloads disabled, just imgur whitelisted. Here's my config again so maybe you can just tell me exactly what I should paste here.

Config ```{ "extractor": { "base-directory": "D:/Test/", "parent-directory": false, "postprocessors": null, "archive": null, "cookies": null, "cookies-update": true, "proxy": null, "skip": true, "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", "retries": 4, "timeout": 30.0, "verify": true, "fallback": true, "sleep": 0, "sleep-request": 0, "sleep-extractor": 0, "path-restrict": { "\\": "⧹", "/" : "⧸", "|" : "│", ":" : "꞉", "*" : "∗", "?" : "?", "\"": "″", "<" : "﹤", ">" : "﹥" }, "path-replace": "_", "path-remove": "\u0000-\u001f\u007f", "path-strip": "auto", "path-extended": true, "extension-map": { "jpeg": "jpg", "jpe" : "jpg", "jfif": "jpg", "jif" : "jpg", "jfi" : "jpg" }, "artstation": { "external": false, "pro-first": true }, "aryion": { "username": null, "password": null, "recursive": true }, "bbc": { "width": 1920 }, "blogger": { "videos": true }, "cyberdrop": { "domain": null }, "danbooru": { "username": null, "password": null, "external": false, "metadata": false, "ugoira": false }, "derpibooru": { "api-key": null, "filter": 56027 }, "deviantart": { "filename": "{filename}.{extension}", "client-id": "", "client-secret": "", "refresh-token": "982d400895e87753b32650f95eff52a857d8c39b", "auto-watch": true, "auto-unwatch": true, "comments": false, "extra": true, "flat": true, "folders":false, "group": true, "include": "all", "journals": "html", "mature": true, "metadata": true, "original": true, "wait-min": 0 }, "e621": { "username": null, "password": null }, "exhentai": { "username": null, "password": null, "domain": "auto", "limits": true, "metadata": false, "original": true, "sleep-request": 5.0 }, "flickr": { "videos": true, "size-max": null }, "furaffinity": { "descriptions": "text", "external": false, "include": "gallery", "layout": "auto" }, "gelbooru": { "api-key": null, "user-id": null }, "gfycat": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}.{extension}", "not locals().get('title')": "{gfyName} - {title}.{extension}" }, "format": ["mp4", "webm", "mobile", "gif"] }, "gofile": { "api-token": null, "website-token": "12345" }, "hentaifoundry": { "include": "pictures" }, "hitomi": { "format": "webp", "metadata": false }, "idolcomplex": { "username": null, "password": null, "sleep-request": 5.0 }, "imgbb": { "username": null, "password": null }, "imgur": { "image": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}{num:?_//}.{extension}", "" : "{title:?/ - /}{filename}.{extension}" }, "directory": ["imgur"] }, "album": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}{num:?_//}.{extension}", "" : "{num} - {filename}.{extension}" }, "directory": ["imgur", "{album['title']|'Imgur'} - {album['id']}"] }, "mp4": true, "postprocessors": [ { "name": "metadata", "filter": "description", "format": "{description}" } ] }, "inkbunny": { "username": null, "password": null, "orderby": "create_datetime" }, "instagram": { "filename": "{date:%Y-%m-%d} {post_id}_{owner_id}_{num}.{extension}", "api": "rest", "cookies": null, "include": "posts", "sleep-request": [6.0, 12.0], "videos": true, "saved": { "directory": ["{category}", "Saved"], "highlights": { "directory": ["Highlights", "{highlight_title}"], "filename": "{date:%Y-%m-%d} {post_id}_{num}.{extension}" } } }, "khinsider": { "format": "mp3" }, "luscious": { "gif": false }, "mangadex": { "api-server": "https://api.mangadex.org", "api-parameters": null, "lang": null, "ratings": ["safe", "suggestive", "erotica", "pornographic"] }, "mangoxo": { "username": null, "password": null }, "newgrounds": { "username": null, "password": null, "flash": true, "format": "original", "include": "art" }, "nana": { "favkey": null }, "nijie": { "username": null, "password": null, "include": "illustration,doujin" }, "nitter": { "quoted": false, "retweets": false, "videos": true }, "oauth": { "browser": true, "cache": true, "host": "localhost", "port": 6414 }, "paheal": { "metadata": false }, "pillowfort": { "external": false, "inline": true, "reblogs": false }, "pinterest": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}{num:?_//}.{extension}", "not locals().get('title')": "{id}.{extension}" }, "sections": true, "videos": true }, "pixiv": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}{num:?_//}.{extension}" }, "refresh-token": null, "include": "artworks", "metadata": false, "tags": "japanese", "ugoira": true }, "reactor": { "gif": false, "sleep-request": 5.0 }, "imagefap": { "filename": "{num}-{filename}.{extension}" }, "reddit": { "filename": "{title} {date:%Y-%m-%d} {id}{num:?_//}.{extension}", "comments": 500, "morecomments": false, "date-min": 0, "date-max": 253402210800, "date-format": "%Y-%m-%dT%H:%M:%S", "id-min": null, "id-max": null, "recursion": 0, "videos": "ytdl", "whitelist": ["imgur"], "parent-directory": true, "parent-metadata": "_reddit" }, "redgifs": { "filename": { "'_reddit' in locals()": "{_reddit[title]} {_reddit[date]:%Y-%m-%d} {_reddit[id]}.{extension}", "not locals().get('title')": "{filename}.{extension}" }, "format": ["hd", "sd", "gif"] }, "sankaku": { "username": null, "password": null, "refresh": false }, "sankakucomplex": { "embeds": false, "videos": true }, "skeb": { "article": false, "filters": null, "sent-requests": false, "thumbnails": false }, "smugmug": { "videos": true }, "seiga": { "username": null, "password": null }, "subscribestar": { "username": null, "password": null }, "tsumino": { "username": null, "password": null }, "tumblr": { "api-key": "", "api-secret": "", "avatar":true, "date-min": 0, "date-max": null, "external": true, "inline": true, "posts": "all", "offset": 0, "original": true, "reblogs": true }, "twitter": { "username": null, "password": null, "cards": false, "conversations": false, "pinned": false, "quoted": true, "replies": false, "retweets": true, "logout": false, "filename": "{author[name]} {date:%Y-%m-%d} {tweet_id}_{num}.{extension}", "strategy": null, "text-tweets": true, "twitpic": true, "unique": true, "users": "https://twitter.com/{legacy[screen_name]}", "videos": true, "likes": { "directory": ["{category}", "Likes"] }, "postprocessors":[ { "name": "metadata", "event": "post", "filename": "twitter_{author[name]}_{tweet_id}_main.json" } ] }, "unsplash": { "format": "raw" }, "vsco": { "videos": true }, "wallhaven": { "api-key": null, "metadata": false, "include": "uploads" }, "weasyl": { "api-key": null, "metadata": false }, "weibo": { "livephoto": true, "retweets": true, "videos": true }, "ytdl": { "enabled": false, "format": null, "generic": true, "logging": true, "module": null, "raw-options": null }, "zerochan": { "username": null, "password": null, "metadata": false }, "booru": { "tags": false, "notes": false } }, "downloader": { "filesize-min": null, "filesize-max": null, "mtime": true, "part": true, "part-directory": null, "progress": 3.0, "rate": null, "retries": 4, "timeout": 30.0, "verify": true, "http": { "adjust-extensions": true, "chunk-size": 32768, "headers": null }, "ytdl": { "config-file": "C:/Users/Mohammad Noor/AppData/Roaming/yt-dlp/config.txt", "format": null, "forward-cookies": true, "logging": true, "module": null, "outtmpl": null, "raw-options": null } }, "logfile": { "path": "C:/Logs/gallery-dl/logfile.txt", "mode": "a", "format": { "debug" : "[{asctime}][{levelname}] {message}", "info" : "[{asctime}][{levelname}] {message}", "warning": "[{asctime}][{levelname}] {message} [Source URL: {extractor.url}]", "error" : "[{asctime}][{levelname}] {message} [Source URL: {extractor.url}]" }, "format-date": "%Y-%m-%dT%H:%M:%S", "level": "info" }, "output": { "mode": "auto", "progress": true, "shorten": true, "ansi": false, "colors": { "success": "1;32", "skip" : "2" }, "skip": true, "log": "[{name}][{levelname}] {message}", "logfile": null, "unsupportedfile": null }, "netrc": false }```
taskhawk commented 1 year ago

Unless it was dummy data you exposed your Tumblr API key & secret in your config, you'll need to get new ones.

cheese529 commented 1 year ago

Unless it was dummy data you exposed your Tumblr API key & secret in your config, you'll need to get new ones.

I edited the message, thank you for telling me.

taskhawk commented 1 year ago

People can still see the edited messages, there's an "edited" dropdown menu next to your nick in the message, so it doesn't help much.

cheese529 commented 1 year ago

Oh wow lol I did not know that, still changed my key tho dw.

taskhawk commented 1 year ago

I tested your config and it works as you wanted it, you just need to add the last line to your Reddit config:

"reddit": {
    "filename": "{title} {date:%Y-%m-%d} {id}{num:?_//}.{extension}",
    "comments": 500,
    "morecomments": false,
    "date-min": 0,
    "date-max": 253402210800,
    "date-format": "%Y-%m-%dT%H:%M:%S",
    "id-min": null,
    "id-max": null,
    "recursion": 0,
    "videos": "ytdl",
    "whitelist": ["imgur"],
    "parent-directory": true,
    "parent-metadata": "_reddit",
    "download": false
}
cheese529 commented 1 year ago

This works perfectly now, seems like I had it in the wrong spot, thanks.