search

mikf / gallery-dl

Command-line program to download image galleries and collections from several image hosting sites
GNU General Public License v2.0
12.08k stars 983 forks source link

Miscellaneous Questions about Several Sites #3608

Closed FM-ABADON closed 1 year ago

FM-ABADON commented 1 year ago

...

Hrxn commented 1 year ago

Here, try this (copy with a click on the button at the top right corner in the greyed-background code field);

{
"extractor":
{
    "base-directory": "./gallery-dl/",
    "parent-directory": false,
    "postprocessors": null,
    "archive": null,
    "cookies": null,
    "cookies-update": true,
    "proxy": null,
    "skip": true,

    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
    "retries": 4,
    "timeout": 30.0,
    "verify": true,
    "fallback": true,

    "sleep": 0,
    "sleep-request": 0,
    "sleep-extractor": 0,

    "path-restrict": "auto",
    "path-replace": "_",
    "path-remove": "\\u0000-\\u001f\\u007f",
    "path-strip": "auto",
    "path-extended": true,

    "extension-map": {
        "jpeg": "jpg",
        "jpe" : "jpg",
        "jfif": "jpg",
        "jif" : "jpg",
        "jfi" : "jpg"
    },

    "artstation":
    {
        "external": false,
        "pro-first": true
    },
    "aryion":
    {
        "username": null,
        "password": null,
        "recursive": true
    },
    "bbc": {
        "width": 1920
    },
    "blogger":
    {
        "videos": true
    },
    "cyberdrop":
    {
        "domain": null
    },
    "danbooru":
    {
        "username": null,
        "password": null,
        "external": false,
        "metadata": false,
        "ugoira": false
    },
    "derpibooru":
    {
        "api-key": null,
        "filter": 56027
    },
    "deviantart":
    {
        "client-id": "23328",
        "client-secret": "3b4f803d2089bf579be3379d22802049",
        "auto-watch": false,
        "auto-unwatch": false,
        "comments": false,
        "extra": false,
        "flat": true,
        "folders": false,
        "group": true,
        "include": "gallery",
        "journals": "html",
        "mature": true,
        "metadata": false,
        "original": true,
        "wait-min": 0
    },
    "e621":
    {
        "username": null,
        "password": null
    },
    "exhentai":
    {
        "username": null,
        "password": null,
        "domain": "auto",
        "limits": true,
        "metadata": false,
        "original": true,
        "sleep-request": 5.0
    },
    "flickr":
    {
        "videos": true,
        "size-max": null
    },
    "furaffinity":
    {
        "descriptions": "text",
        "external": false,
        "include": "gallery, scraps",
        "layout": "auto",
        "cookies": {
            "a": "22f0fd67-dcf2-4264-ac99-aea9feea601f",
            "b": "9fb25398-d6e1-44db-8e19-30e06f8e16c3"
        }
    },
    "gelbooru":
    {
        "api-key": null,
        "user-id": null
    },
    "gfycat":
    {
        "format": ["mp4", "webm", "mobile", "gif"]
    },
    "gofile": {
        "api-token": null,
        "website-token": "12345"
    },
    "hentaifoundry":
    {
        "include": "pictures"
    },
    "hitomi":
    {
        "format": "webp",
        "metadata": false
    },
    "idolcomplex":
    {
        "username": null,
        "password": null,
        "sleep-request": 5.0
    },
    "imgbb":
    {
        "username": null,
        "password": null
    },
    "imgur":
    {
        "mp4": true
    },
    "inkbunny":
    {
        "username": null,
        "password": null,
        "orderby": "create_datetime"
    },
    "instagram":
    {
        "api": "rest",
        "cookies": null,
        "include": "posts",
        "sleep-request": [6.0, 12.0],
        "videos": true
    },
    "kemonoparty":
    {
        "archive": "path/to/archive.sqlite",
        "archive-format": "{hash}"
    },
    "khinsider":
    {
        "format": "mp3"
    },
    "luscious":
    {
        "gif": false
    },
    "mangadex":
    {
        "api-server": "https://api.mangadex.org",
        "api-parameters": null,
        "lang": null,
        "ratings": ["safe", "suggestive", "erotica", "pornographic"]
    },
    "mangoxo":
    {
        "username": null,
        "password": null
    },
    "newgrounds":
    {
        "username": null,
        "password": null,
        "flash": true,
        "format": "original",
        "include": "art"
    },
    "nana":
    {
        "favkey": null
    },
    "nijie":
    {
        "username": null,
        "password": null,
        "include": "illustration,doujin"
    },
    "nitter": {
        "quoted": false,
        "retweets": false,
        "videos": true
    },
    "oauth":
    {
        "browser": true,
        "cache": true,
        "host": "localhost",
        "port": 6414
    },
    "paheal":
    {
        "metadata": false
    },
    "pillowfort":
    {
        "external": false,
        "inline": true,
        "reblogs": false
    },
    "pinterest":
    {
        "domain": "auto",
        "sections": true,
        "videos": true
    },
    "pixiv":
    {
        "refresh-token": null,
        "include": "artworks",
        "metadata": false,
        "metadata-bookmark": false,
        "tags": "japanese",
        "ugoira": true
    },
    "reactor":
    {
        "gif": false,
        "sleep-request": 5.0
    },
    "reddit":
    {
        "comments": 0,
        "morecomments": false,
        "date-min": 0,
        "date-max": 253402210800,
        "date-format": "%Y-%m-%dT%H:%M:%S",
        "id-min": null,
        "id-max": null,
        "recursion": 0,
        "videos": true
    },
    "redgifs":
    {
        "format": ["hd", "sd", "gif"]
    },
    "sankaku":
    {
        "username": null,
        "password": null,
        "refresh": false
    },
    "sankakucomplex":
    {
        "embeds": false,
        "videos": true
    },
    "skeb":
    {
        "article": false,
        "filters": null,
        "sent-requests": false,
        "thumbnails": false
    },
    "smugmug":
    {
        "videos": true
    },
    "seiga":
    {
        "username": null,
        "password": null
    },
    "subscribestar":
    {
        "username": null,
        "password": null
    },
    "tsumino":
    {
        "username": null,
        "password": null
    },
    "tumblr":
    {
        "avatar": false,
        "external": false,
        "inline": true,
        "posts": "all",
        "offset": 0,
        "original": true,
        "reblogs": true
    },
    "twitter":
    {
        "username": null,
        "password": null,
        "cards": false,
        "conversations": false,
        "pinned": false,
        "quoted": false,
        "replies": true,
        "retweets": false,
        "strategy": null,
        "text-tweets": false,
        "twitpic": false,
        "unique": true,
        "users": "timeline",
        "videos": true
    },
    "unsplash":
    {
        "format": "raw"
    },
    "vsco":
    {
        "videos": true
    },
    "wallhaven":
    {
        "api-key": null,
        "metadata": false,
        "include": "uploads"
    },
    "weasyl":
    {
        "api-key": null,
        "metadata": false
    },
    "weibo":
    {
        "livephoto": true,
        "retweets": true,
        "videos": true
    },
    "ytdl":
    {
        "enabled": false,
        "format": null,
        "generic": true,
        "logging": true,
        "module": null,
        "raw-options": null
    },
    "zerochan":
    {
        "username": null,
        "password": null,
        "metadata": false
    },
    "booru":
    {
        "tags": false,
        "notes": false
    }
},

"downloader":
{
    "filesize-min": null,
    "filesize-max": null,
    "mtime": true,
    "part": true,
    "part-directory": null,
    "progress": 3.0,
    "rate": null,
    "retries": 4,
    "timeout": 30.0,
    "verify": true,

    "http":
    {
        "adjust-extensions": true,
        "chunk-size": 32768,
        "headers": null,
        "validate": true
    },

    "ytdl":
    {
        "format": null,
        "forward-cookies": false,
        "logging": true,
        "module": null,
        "outtmpl": null,
        "raw-options": null
    }
},

"output":
{
    "mode": "auto",
    "progress": true,
    "shorten": true,
    "ansi": false,
    "colors": {
        "success": "1;32",
        "skip"   : "2"
    },
    "skip": true,
    "log": "[{name}][{levelname}] {message}",
    "logfile": null,
    "unsupportedfile": null
},

"netrc": false
}

And for the future, try something like this:

https://www.jslint.com/

This is a very good and strict linter for JS and JSON.

  1. Click 'Clear Source'
  2. Paste your config into the text area
  3. Click 'JSLint'

In "Report: Warnings (0)" you will see all potential issues, including their line numbers (They are also highlighted in the gutter of the text area)

If your JSON is good, you should see "There are no warnings." there. Also, in "Report: Functions (0)" below, it should show "JSON: good."

FM-ABADON commented 1 year ago

Thanks so much! As a whole, this works much better, but a couple of issues have popped up. Mostly just missing images from certain galleries. Missing twenty here, missing five there, missing a hundred here, that sort of thing. I have looked around but I haven't seemed to find a solution. Also, is there anything for downloading descriptions for Furaffinity or Twitter content, as stories and other things are typically found there I'm wondering if they can be grabbed as well? Tumblr also gives an oauth error every single time despite the key having been refreshed in the last 5 minutes, so that evades me as well. The last thing, despite my config saying otherwise, scraps are not downloaded in Furaffinity.

rautamiekka commented 1 year ago

I'd get rid of those logins from the posts, now.