Closed pums974 closed 3 years ago
Hi @pums974,
Thanks for splitting up a separate issue! From my own testing here it appears that this is an aesthetic issue only? And it still works (minus the extra output of fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project
) can you confirm if that is correct or not?
I'm trying to think about the best way to handle this, and I think that distinction is likely very important, because if it's only that it is producing extra command line output, the best thing may be to simply silent that output within isort. While I understand what you expect to happen, because of how this is currently enabled, it simply can't take into account your other skips. To be performant the skip git ignore is running separately against your code base from the other ignores, and collecting the list of files and paths to ignore all at once from git's perspective. Then as it decides one file at a time to process or skip, it does fully take into account any skips you have set up, and will still skip files correctly.
Hi,
I suppose you are right, it does it's job correctly. But it is very slow.
with skip_gitignore = false
(and venv
in extend_skip
):
$> time isort .
Skipped 5 files
isort . 0,52s user 0,08s system 99% cpu 0,601 total
with skip_gitignore = true
(and venv
still in extend_skip
):
$> time isort .
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project'
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project'
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project'
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project'
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project'
... (**a lot** of repetition of this line)
Skipped 5 files
isort . 502,69s user 240,89s system 135% cpu 9:09,91 total
I ran into this when upgrading from isort 5.8.0 to 5.9.2. isort became very slow, taking over 10 minutes to complete with skip_gitignore = true
vs a few seconds with skip_gitignore = false
.
From my own testing here it appears that this is an aesthetic issue only? And it still works (minus the extra output of
fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project
) can you confirm if that is correct or not?
@timothycrosley Unfortunately it is more than aesthetic since it prevents --skip-gitignore
from actually skipping the paths in .gitignore
:
$ git init demo
$ cd demo
$ touch foo.py bar.py
$ echo foo.py > .gitignore
$ isort --vn
5.9.2
# without a submodule present isort correctly skips foo.py since it is in .gitignore
$ isort --show-files --skip-gitignore .
./bar.py
# with a submodule present isort incorrectly checks foo.py despite it being in .gitignore
$ git submodule add https://github.com/PyCQA/meta.git # could be any repo
$ isort --show-files --skip-gitignore .
fatal: Pathspec '/Users/robbie/demo/meta/LICENSE' is in submodule 'meta'
fatal: Pathspec '/Users/robbie/demo/meta/LICENSE' is in submodule 'meta'
./bar.py
fatal: Pathspec '/Users/robbie/demo/meta/LICENSE' is in submodule 'meta'
./foo.py # <--------- this shouldn't appear here since it is in .gitignore
./meta/source/conf.py
In v5.8.0 isort would correctly skip foo.py
with or without a submodule present.
Seems fixed indeed.
Thanks !
Hi, With the version 5.9.2, when
skip_gitignore=True
, I have the error:fatal: Pathspec '/some/path/other_project/setup.py' is in submodule 'other_project
where
other_project
is a git submodule, not in.gitignore
but in extend_skipIf I turn off skip_gitignore, everything seems to be working fine.
`isort . --show-config`:
```python { "_known_patterns": null, "_section_comments": null, "_skips": null, "_skip_globs": null, "_sorting_function": null, "py_version": "py3", "force_to_top": [], "skip": [ "__pypackages__", ".tox", "dist", ".nox", ".venv", "build", ".mypy_cache", ".direnv", ".bzr", "venv", ".eggs", "buck-out", ".git", "node_modules", ".hg", ".svn", "_build", ".pants.d" ], "extend_skip": [ "other_project" ], "skip_glob": [], "extend_skip_glob": [], "skip_gitignore": true, "line_length": 100, "wrap_length": 0, "line_ending": "", "sections": [ "FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER" ], "no_sections": false, "known_future_library": [ "__future__" ], "known_third_party": [], "known_first_party": [], "known_local_folder": [], "known_standard_library": [ "shelve", "formatter", "struct", "pstats", "sys", "imp", "imaplib", "pkgutil", "trace", "pprint", "fcntl", "symtable", "rlcompleter", "msilib", "tempfile", "tarfile", "json", "msvcrt", "pickle", "uu", "grp", "statistics", "keyword", "cmd", "difflib", "string", "hashlib", "argparse", "site", "itertools", "pipes", "tokenize", "faulthandler", "tracemalloc", "cmath", "smtplib", "datetime", "ast", "winreg", "aifc", "netrc", "pydoc", "ntpath", "subprocess", "zipimport", "importlib", "zipapp", "curses", "_dummy_thread", "resource", "multiprocessing", "bdb", "email", "gettext", "fractions", "tkinter", "gc", "select", "unittest", "warnings", "timeit", "sqlite3", "encodings", "zoneinfo", "reprlib", "math", "chunk", "readline", "pyclbr", "binascii", "binhex", "inspect", "random", "zipfile", "imghdr", "xmlrpc", "operator", "sre", "typing", "atexit", "pty", "stat", "ctypes", "compileall", "urllib", "wsgiref", "asyncio", "html", "macpath", "sre_compile", "csv", "crypt", "copyreg", "abc", "types", "zlib", "lib2to3", "linecache", "shlex", "dataclasses", "poplib", "mimetypes", "io", "smtpd", "colorsys", "pwd", "dbm", "contextlib", "glob", "unicodedata", "concurrent", "array", "hmac", "asyncore", "graphlib", "time", "venv", "profile", "builtins", "ensurepip", "sunau", "bz2", "mmap", "sched", "http", "queue", "sre_parse", "posix", "uuid", "contextvars", "mailcap", "cgitb", "telnetlib", "mailbox", "token", "marshal", "doctest", "wave", "functools", "copy", "nntplib", "ossaudiodev", "fnmatch", "termios", "selectors", "pathlib", "tabnanny", "tty", "sndhdr", "fileinput", "posixpath", "locale", "webbrowser", "enum", "dis", "quopri", "codeop", "numbers", "threading", "secrets", "calendar", "ssl", "base64", "heapq", "collections", "turtledemo", "getpass", "stringprep", "ftplib", "xdrlib", "lzma", "_thread", "test", "xml", "sre_constants", "asynchat", "socketserver", "pdb", "nis", "symbol", "codecs", "os", "filecmp", "textwrap", "code", "getopt", "re", "logging", "plistlib", "parser", "cProfile", "dummy_threading", "bisect", "shutil", "py_compile", "socket", "configparser", "sysconfig", "distutils", "gzip", "pickletools", "optparse", "turtle", "runpy", "syslog", "fpectl", "ipaddress", "errno", "modulefinder", "winsound", "cgi", "weakref", "spwd", "audioop", "traceback", "signal", "decimal", "platform" ], "extra_standard_library": [], "known_other": {}, "multi_line_output": "VERTICAL_HANGING_INDENT", "forced_separate": [], "indent": " ", "comment_prefix": " #", "length_sort": false, "length_sort_straight": false, "length_sort_sections": [], "add_imports": [], "remove_imports": [], "append_only": false, "reverse_relative": false, "force_single_line": false, "single_line_exclusions": [], "default_section": "THIRDPARTY", "import_headings": {}, "balanced_wrapping": false, "use_parentheses": true, "order_by_type": true, "atomic": false, "lines_after_imports": -1, "lines_between_sections": 1, "lines_between_types": 0, "combine_as_imports": false, "combine_star": false, "include_trailing_comma": true, "from_first": false, "verbose": false, "quiet": false, "force_adds": false, "force_alphabetical_sort_within_sections": false, "force_alphabetical_sort": false, "force_grid_wrap": 0, "force_sort_within_sections": false, "lexicographical": false, "group_by_package": false, "ignore_whitespace": false, "no_lines_before": [], "no_inline_sort": false, "ignore_comments": false, "case_sensitive": false, "sources": [ { "py_version": "py3", "force_to_top": [], "skip": [ "__pypackages__", ".tox", "dist", ".nox", ".venv", "build", ".mypy_cache", ".direnv", ".bzr", "venv", ".eggs", "buck-out", ".git", "node_modules", ".hg", ".svn", "_build", ".pants.d" ], "extend_skip": [], "skip_glob": [], "extend_skip_glob": [], "skip_gitignore": false, "line_length": 79, "wrap_length": 0, "line_ending": "", "sections": [ "FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER" ], "no_sections": false, "known_future_library": [ "__future__" ], "known_third_party": [], "known_first_party": [], "known_local_folder": [], "known_standard_library": [ "shelve", "formatter", "struct", "pstats", "sys", "imp", "imaplib", "pkgutil", "trace", "pprint", "fcntl", "symtable", "rlcompleter", "msilib", "tempfile", "tarfile", "json", "msvcrt", "pickle", "uu", "grp", "statistics", "keyword", "cmd", "difflib", "string", "hashlib", "argparse", "site", "itertools", "pipes", "tokenize", "faulthandler", "tracemalloc", "cmath", "smtplib", "datetime", "ast", "winreg", "aifc", "netrc", "pydoc", "ntpath", "subprocess", "zipimport", "importlib", "zipapp", "curses", "_dummy_thread", "resource", "multiprocessing", "bdb", "email", "gettext", "fractions", "tkinter", "gc", "select", "unittest", "warnings", "timeit", "sqlite3", "encodings", "zoneinfo", "reprlib", "math", "chunk", "readline", "pyclbr", "binascii", "binhex", "inspect", "random", "zipfile", "imghdr", "xmlrpc", "operator", "sre", "typing", "atexit", "pty", "stat", "ctypes", "compileall", "urllib", "wsgiref", "asyncio", "html", "macpath", "sre_compile", "csv", "crypt", "copyreg", "abc", "types", "zlib", "lib2to3", "linecache", "shlex", "dataclasses", "poplib", "mimetypes", "io", "smtpd", "colorsys", "pwd", "dbm", "contextlib", "glob", "unicodedata", "concurrent", "array", "hmac", "asyncore", "graphlib", "time", "venv", "profile", "builtins", "ensurepip", "sunau", "bz2", "mmap", "sched", "http", "queue", "sre_parse", "posix", "uuid", "contextvars", "mailcap", "cgitb", "telnetlib", "mailbox", "token", "marshal", "doctest", "wave", "functools", "copy", "nntplib", "ossaudiodev", "fnmatch", "termios", "selectors", "pathlib", "tabnanny", "tty", "sndhdr", "fileinput", "posixpath", "locale", "webbrowser", "enum", "dis", "quopri", "codeop", "numbers", "threading", "secrets", "calendar", "ssl", "base64", "heapq", "collections", "turtledemo", "getpass", "stringprep", "ftplib", "xdrlib", "lzma", "_thread", "test", "xml", "sre_constants", "asynchat", "socketserver", "pdb", "nis", "symbol", "codecs", "os", "filecmp", "textwrap", "code", "getopt", "re", "logging", "plistlib", "parser", "cProfile", "dummy_threading", "bisect", "shutil", "py_compile", "socket", "configparser", "sysconfig", "distutils", "gzip", "pickletools", "optparse", "turtle", "runpy", "syslog", "fpectl", "ipaddress", "errno", "modulefinder", "winsound", "cgi", "weakref", "spwd", "audioop", "traceback", "signal", "decimal", "platform" ], "extra_standard_library": [], "known_other": {}, "multi_line_output": "GRID", "forced_separate": [], "indent": " ", "comment_prefix": " #", "length_sort": false, "length_sort_straight": false, "length_sort_sections": [], "add_imports": [], "remove_imports": [], "append_only": false, "reverse_relative": false, "force_single_line": false, "single_line_exclusions": [], "default_section": "THIRDPARTY", "import_headings": {}, "balanced_wrapping": false, "use_parentheses": false, "order_by_type": true, "atomic": false, "lines_after_imports": -1, "lines_between_sections": 1, "lines_between_types": 0, "combine_as_imports": false, "combine_star": false, "include_trailing_comma": false, "from_first": false, "verbose": false, "quiet": false, "force_adds": false, "force_alphabetical_sort_within_sections": false, "force_alphabetical_sort": false, "force_grid_wrap": 0, "force_sort_within_sections": false, "lexicographical": false, "group_by_package": false, "ignore_whitespace": false, "no_lines_before": [], "no_inline_sort": false, "ignore_comments": false, "case_sensitive": false, "sources": [], "virtual_env": "", "conda_env": "", "ensure_newline_before_comments": false, "directory": "", "profile": "", "honor_noqa": false, "src_paths": [], "old_finders": false, "remove_redundant_aliases": false, "float_to_top": false, "filter_files": false, "formatter": "", "formatting_function": null, "color_output": false, "treat_comments_as_code": [], "treat_all_comments_as_code": false, "supported_extensions": [ "py", "pyi", "pxd", "pyx" ], "blocked_extensions": [ "pex" ], "constants": [], "classes": [], "variables": [], "dedup_headings": false, "only_sections": false, "only_modified": false, "combine_straight_imports": false, "auto_identify_namespace_packages": true, "namespace_packages": [], "follow_links": true, "indented_import_headings": true, "honor_case_in_force_sorted_sections": false, "sort_relative_in_force_sorted_sections": false, "overwrite_in_place": false, "reverse_sort": false, "star_first": false, "git_ignore": {}, "format_error": "{error}: {message}", "format_success": "{success}: {message}", "sort_order": "natural", "source": "defaults" }, { "multi_line_output": 3, "include_trailing_comma": true, "force_grid_wrap": 0, "use_parentheses": true, "ensure_newline_before_comments": true, "line_length": 88, "source": "black profile" }, { "profile": "black", "multi_line_output": "VERTICAL_HANGING_INDENT", "line_length": 100, "extend_skip": [ "other_project" ], "skip_gitignore": true, "source": "/some/path/pyproject.toml" } ], "virtual_env": "", "conda_env": "", "ensure_newline_before_comments": true, "directory": "/some/path", "profile": "black", "honor_noqa": false, "src_paths": [ "/some/path/src", "/some/path" ], "old_finders": false, "remove_redundant_aliases": false, "float_to_top": false, "filter_files": false, "formatter": "", "formatting_function": null, "color_output": false, "treat_comments_as_code": [], "treat_all_comments_as_code": false, "supported_extensions": [ "py", "pyi", "pxd", "pyx" ], "blocked_extensions": [ "pex" ], "constants": [], "classes": [], "variables": [], "dedup_headings": false, "only_sections": false, "only_modified": false, "combine_straight_imports": false, "auto_identify_namespace_packages": true, "namespace_packages": [], "follow_links": true, "indented_import_headings": true, "honor_case_in_force_sorted_sections": false, "sort_relative_in_force_sorted_sections": false, "overwrite_in_place": false, "reverse_sort": false, "star_first": false, "git_ignore": {}, "format_error": "{error}: {message}", "format_success": "{success}: {message}", "sort_order": "natural" } ````pyproject.toml`:
```toml [tool.isort] profile = "black" multi_line_output = 3 line_length = 100 extend_skip = [ "other_project", ] skip_gitignore = true ```Originally posted by @pums974 in https://github.com/PyCQA/isort/issues/1762#issuecomment-876414860