mikf / gallery-dl

Command-line program to download image galleries and collections from several image hosting sites
GNU General Public License v2.0
11.7k stars 953 forks source link

Could Not Parse Issue #1420

Closed mm125Y closed 3 years ago

mm125Y commented 3 years ago

I keep getting this error message "Could not parse Expecting ',' delimiter: line 171 column 16 (char 3595)". I have no idea why line 171 is the problem when its literally the default setting in the provided config.json file.

        },

"downloader": {
    "filesize-min": null,
    "filesize-max": null,
    "part": true,
    "part-directory": null,

    "http": {
        "adjust-extensions": true,
        "mtime": true,
        "rate": null,
        "retries": 4,
        "timeout": 30.0,
        "verify": true
    },

    "ytdl": {
        "format": null,
        "forward-cookies": false,
        "mtime": true,
        "outtmpl": null,
        "rate": null,
        "retries": 4,
        "timeout": 30.0,
        "verify": true
    }
},

"output": {
    "mode": "auto",
    "progress": true,
    "shorten": true,
    "log": "[{name}][{levelname}] {message}",
    "logfile": null,
    "unsupportedfile": null
},

"netrc": false

}

I also tried testing it on JSONLint but keep getting this error message: Error: Parse error on line 171: ... }, "downloader": { "filesize-mi ---------------------^ Expecting 'EOF', '}', ',', ']', got ':'

Any help with this?

Skyofflad commented 3 years ago

Could you paste the whole file?

mikf commented 3 years ago

The error is usually on the line(s) before 171, probably a , too many after the last line in a block. (The snippet you posted is valid JSON, by the way)

mm125Y commented 3 years ago

Could you paste the whole file?

"extractor": {
    "base-directory": "./gallery-dl/",
    "postprocessors": [{
            "name": "ugoira",
            "whitelist": ["pixiv", "danbooru"],
            "ffmpeg-twopass": true,
            "ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "4M"],
            "archive": null,
            "cookies": null,
            "cookies-update": false,
            "proxy": null,
            "skip": true,
            "sleep": 0,
            "path-restrict": "auto",
            "path-replace": "_",
            "path-remove": "\\u0000-\\u001f\\u007f",
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",

            "artstation": {
                "external": false
            },
            "aryion": {
                "username": null,
                "password": null
            },
            "blogger": {
                "videos": true
            },
            "danbooru": {
                "username": null,
                "password": null,
                "ugoira": false
            },
            "deviantart": {
                "include": "gallery,scraps",
                "refresh-token": "cache",
                "client-id": "####",
                "client-secret": "######",
                "flat": true,
                "folders": false,
                "journals": "html",
                "mature": true,
                "metadata": true,
                "original": true,
                "quality": 100,
                "extra": true,
                "wait-min": 0,
                "cookies": "insert",
                "cookies-update": true
            },
            "exhentai": {
                "username": "insert",
                "password": "insert",
                "domain": "auto",
                "limits": true,
                "original": true,
                "wait-min": 3.0,
                "wait-max": 6.0
            },
            "flickr": {
                "username": "insert",
                "password": "insert",
                "videos": true,
                "size-max": null
            },
            "gelbooru": {
                "api": true
            },
            "gfycat": {
                "format": "mp4"
            },
            "hitomi": {
                "metadata": true
            },
            "idolcomplex": {
                "username": null,
                "password": null,
                "wait-min": 3.0,
                "wait-max": 6.0
            },
            "imgur": {
                "mp4": true
            },
            "instagram": {
                "username": "insert",
                "password": "insert",
                "highlights": true,
                "videos": true
            },
            "nijie": {
                "username": null,
                "password": null
            },
            "oauth": {
                "browser": true,
                "cache": true,
                "port": 6414
            },
            "pixiv": {
                "username": "insert",
                "password": "insert",
                "avatar": true,
                "ugoira": false
            },
            "patreon": {
                "cookies": "insert",
                "username": "insert",
                "password": "insert",
                "avatar": true
            },
            "reactor": {
                "wait-min": 3.0,
                "wait-max": 6.0
            },
            "reddit": {
                "username": "insert",
                "password": "insert",
                "comments": 0,
                "morecomments": false,
                "date-min": 0,
                "date-max": 253402210800,
                "date-format": "%Y-%m-%dT%H:%M:%S",
                "id-min": "0",
                "id-max": "zik0zj",
                "recursion": 0,
                "videos": true,
                "user-agent": "Python:gallery-dl:0.8.4 (by /u/mikf1)"
            },
            "4chan": {
                "filename": "{date:%Y%m%d}_{filename} - {title}.{extension}"
            },
            "redgifs": {
                "format": "mp4"
            },
            "seiga": {
                "username": null,
                "password": null
            },
            "tumblr": {
                "avatar": false,
                "external": false,
                "inline": true,
                "posts": "all",
                "reblogs": true
            },
            "twitter": {
                "cookies": true,
                "cookies-update": true,
                "quoted": true,
                "replies": true,
                "retweets": false,
                "twitpic": true,
                "videos": true
            },
            "vsco": {
                "videos": true
            },
            "wallhaven": {
                "api-key": null
            },
            "weibo": {
                "retweets": true,
                "videos": true
            },
            "booru": {
                "tags": false
            }
        },

        "downloader": {
            "part-directory": "/tmp/.download/",
            "rate": "1M",
            "retries": 3,
            "timeout": 8.5
            }
        },
        "http": {
            "adjust-extensions": true,
            "mtime": true,
            "rate": null,
            "retries": 4,
            "timeout": 30.0,
            "verify": true
        },

        "ytdl": {
            "format": null,
            "forward-cookies": false,
            "mtime": true,
            "outtmpl": null,
            "rate": null,
            "retries": 4,
            "timeout": 30.0,
            "verify": true
        }
    },

    "output": {
        "mode": "auto",
        "progress": true,
        "shorten": true,
        "log": "[{name}][{levelname}] {message}",
        "logfile": null,
        "unsupportedfile": null
    },

    "netrc": false
}
Skyofflad commented 3 years ago

You missed some brackets Line 7 - change from "ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "4M"], to "ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "4M"]}],

mikf commented 3 years ago

There were also a few brackets too many in the downloader section. You can find a working version of your config at https://gist.githubusercontent.com/mikf/298ef85b2daa05f71750d058e7c0816e/raw/49c57412483a31bc75c5b95b7a85b5e12ebad65a/config.json

There are also a lot of username and password entries that aren't recognized by gallery-dl. flickr, reddit, and pixiv use OAuth and patreon needs cookies (login form has a CAPTCHA)

PurpleWii commented 3 years ago

I've tried using the default build, and I'm receiving a similar issue after only putting in my API data for Deviantart/OAuth to bypass 429 request timeouts; I receive a parse error on L 63 Clm 31. Would anyone happen to know the issue of this particular instance?

{ "extractor": { "base-directory": "./gallery-dl/", "parent-directory": false, "postprocessors": null, "archive": null, "cookies": null, "cookies-update": true, "proxy": null, "skip": true,

    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
    "retries": 4,
    "timeout": 30.0,
    "verify": true,

    "sleep": 0,
    "sleep-request": 0,
    "sleep-extractor": 0,

    "path-restrict": "auto",
    "path-replace": "_",
    "path-remove": "\\u0000-\\u001f\\u007f",
    "extension-map": {
        "jpeg": "jpg",
        "jpe" : "jpg",
        "jfif": "jpg",
        "jif" : "jpg",
        "jfi" : "jpg"
    },

    "artstation":
    {
        "external": false
    },
    "aryion":
    {
        "username": null,
        "password": null,
        "recursive": true
    },
    "blogger":
    {
        "videos": true
    },
    "danbooru":
    {
        "username": null,
        "password": null,
        "ugoira": false,
        "metadata": false
    },
    "derpibooru":
    {
        "api-key": null,
        "filter": 56027
    },
    "deviantart":
    {
        "client-id": #####,
        "client-secret": ############,
        "extra": false,
        "flat": true,
        "folders": false,
        "include": "gallery",
        "journals": "html",
        "mature": true,
        "metadata": false,
        "original": true,
        "quality": 100,
        "wait-min": 0
    },
    "e621":
    {
        "username": null,
        "password": null
    },
    "exhentai":
    {
        "username": null,
        "password": null,
        "domain": "auto",
        "limits": true,
        "metadata": false,
        "original": true,
        "sleep-request": 5.0
    },
    "flickr":
    {
        "videos": true,
        "size-max": null
    },
    "furaffinity":
    {
        "descriptions": "text",
        "include": "gallery"
    },
    "gfycat":
    {
        "format": "mp4"
    },
    "hentaifoundry":
    {
        "include": "pictures"
    },
    "hentainexus":
    {
        "original": true
    },
    "hitomi":
    {
        "metadata": true
    },
    "idolcomplex":
    {
        "username": null,
        "password": null,
        "sleep-request": 5.0
    },
    "imgbb":
    {
        "username": null,
        "password": null
    },
    "imgur":
    {
        "mp4": true
    },
    "inkbunny":
    {
        "username": null,
        "password": null,
        "orderby": "create_datetime"
    },
    "instagram":
    {
        "username": null,
        "password": null,
        "include": "posts",
        "sleep-request": 5.0,
        "videos": true
    },
    "khinsider":
    {
        "format": "mp3"
    },
    "mangadex":
    {
        "api-server": "https://api.mangadex.org"
    },
    "mangoxo":
    {
        "username": null,
        "password": null
    },
    "newgrounds":
    {
        "username": null,
        "password": null,
        "flash": true,
        "include": "art"
    },
    "nijie":
    {
        "username": null,
        "password": null,
        "include": "illustration,doujin"
    },
    "oauth":
    {
        "browser": true,
        "cache": true,
        "port": 6414
    },
    "pillowfort":
    {
        "external": false,
        "inline": true,
        "reblogs": false
    },
    "pinterest":
    {
        "sections": true,
        "videos": true
    },
    "pixiv":
    {
        "refresh-token": null,
        "avatar": false,
        "tags": "japanese",
        "ugoira": true
    },
    "reactor":
    {
        "sleep-request": 5.0
    },
    "reddit":
    {
        "comments": 0,
        "morecomments": false,
        "date-min": 0,
        "date-max": 253402210800,
        "date-format": "%Y-%m-%dT%H:%M:%S",
        "id-min": "0",
        "id-max": "zik0zj",
        "recursion": 0,
        "videos": true,
        "user-agent": "Python:gallery-dl:0.8.4 (by /u/mikf1)"
    },
    "redgifs":
    {
        "format": "mp4"
    },
    "sankakucomplex":
    {
        "embeds": false,
        "videos": true
    },
    "sankaku":
    {
        "username": null,
        "password": null
    },
    "smugmug":
    {
        "videos": true
    },
    "seiga":
    {
        "username": null,
        "password": null
    },
    "subscribestar":
    {
        "username": null,
        "password": null
    },
    "tsumino":
    {
        "username": null,
        "password": null
    },
    "tumblr":
    {
        "avatar": false,
        "external": false,
        "inline": true,
        "posts": "all",
        "reblogs": true
    },
    "twitter":
    {
        "username": null,
        "password": null,
        "cards": false,
        "conversations": false,
        "quoted": true,
        "replies": true,
        "retweets": true,
        "text-tweets": false,
        "twitpic": false,
        "users": "timeline",
        "videos": true
    },
    "unsplash":
    {
        "format": "raw"
    },
    "vsco":
    {
        "videos": true
    },
    "wallhaven":
    {
        "api-key": null
    },
    "weasyl":
    {
        "api-key": null
    },
    "weibo":
    {
        "retweets": true,
        "videos": true
    },
    "booru":
    {
        "tags": false,
        "notes": false
    }
},

"downloader":
{
    "filesize-min": null,
    "filesize-max": null,
    "mtime": true,
    "part": true,
    "part-directory": null,
    "rate": null,
    "retries": 4,
    "timeout": 30.0,
    "verify": true,

    "http":
    {
        "adjust-extensions": true,
        "headers": null
    },

    "ytdl":
    {
        "format": null,
        "forward-cookies": false,
        "logging": true,
        "module": "youtube_dl",
        "outtmpl": null,
        "raw-options": null
    }
},

"output":
{
    "mode": "auto",
    "progress": true,
    "shorten": true,
    "skip": true,
    "log": "[{name}][{levelname}] {message}",
    "logfile": null,
    "unsupportedfile": null
},

"netrc": false

}

mikf commented 3 years ago

@PurpleWii Did you input the values for client-id and client-secret as strings, i.e. enclosed in double quotes? ("client-id": "value here")

PurpleWii commented 3 years ago

@PurpleWii Did you input the values for client-id and client-secret as strings, i.e. enclosed in double quotes? ("client-id": "value here")

Inputting the values as strings does not resolve the parse issue. It does however changed the clm number that it implies is the issue to L 63, CLM 13. The JSON file looks like this now.

        "deviantart":
        {
            "client-id": "#####,"
            "client-secret": "############,"
            "extra": false,
            "flat": true,
            "folders": false,
            "include": "gallery",
            "journals": "html",
            "mature": true,
            "metadata": false,
            "original": true,
            "quality": 100,
            "wait-min": 0
        },
mikf commented 3 years ago

The , needs to be at the end of a line, not inside the quotes:

        "deviantart":
        {
            "client-id": "#####",
            "client-secret": "############",
            "extra": false,
            "flat": true,
            "folders": false,
            "include": "gallery",
            "journals": "html",
            "mature": true,
            "metadata": false,
            "original": true,
            "quality": 100,
            "wait-min": 0
        },
PurpleWii commented 3 years ago

Ah, that's where I went wrong. I did what you said, and I'm no longer having issues. Thanks for for going out of your way to help someone (very) technically inept. I greatly appreciate it!