Extract Python Dependencies & Imports

[DavidBakerEffendi]

Use Joern to:

• Find a call to setup in /setup.py
• Determine if the argument install_requires is set
• If it is set, tag the associated List as a list of dependencies (Question: Is this always a list? Could this point to a file?)
• Parse all the imports in the project and note which modules are used (excl. test and local imports)
• Use the output format:

{
  "modules": [],
}

Tasks

• [x] Build extraction logic
• [x] Wire logic to CLI and give JSON output
• [x] Refactor unit test to use folders for test data
• [x] Have a separate list for builtin_modules by using a hardcoded list of DEFINES for python similar to jssrc.

[prabhu]

Thanks @DavidBakerEffendi. Could we also refactor the unit test to load the test data from a folder? This way we could add more tests over time.

https://github.com/AppThreat/atom/blob/main/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala

[prabhu]

const PYTHON_STD_MODULES = [
  "_abc",
  "_ast",
  "_codecs",
  "_collections",
  "_functools",
  "_imp",
  "_io",
  "_locale",
  "_operator",
  "_peg_parser",
  "_signal",
  "_sre",
  "_stat",
  "_string",
  "_symtable",
  "_thread",
  "_tracemalloc",
  "_warnings",
  "_weakref",
  "atexit",
  "builtins",
  "errno",
  "faulthandler",
  "gc",
  "itertools",
  "marshal",
  "posix",
  "pwd",
  "sys",
  "time",
  "xxsubtype",
  "__future__",
  "_aix_support",
  "_bootlocale",
  "_bootsubprocess",
  "_collections_abc",
  "_compat_pickle",
  "_compression",
  "_markupbase",
  "_osx_support",
  "_py_abc",
  "_pydecimal",
  "_pyio",
  "_sitebuiltins",
  "_strptime",
  "_sysconfigdata__linux_x86_64-linux-gnu",
  "_sysconfigdata_d_linux_x86_64-linux-gnu",
  "_threading_local",
  "_weakrefset",
  "abc",
  "aifc",
  "antigravity",
  "argparse",
  "ast",
  "asynchat",
  "asyncio",
  "asyncore",
  "base64",
  "bdb",
  "binhex",
  "bisect",
  "bz2",
  "cProfile",
  "calendar",
  "cgi",
  "cgitb",
  "chunk",
  "cmd",
  "code",
  "codecs",
  "codeop",
  "collections",
  "colorsys",
  "compileall",
  "concurrent",
  "configparser",
  "contextlib",
  "contextvars",
  "copy",
  "copyreg",
  "crypt",
  "csv",
  "ctypes",
  "curses",
  "dataclasses",
  "datetime",
  "dbm",
  "decimal",
  "difflib",
  "dis",
  "distutils",
  "doctest",
  "email",
  "encodings",
  "ensurepip",
  "enum",
  "filecmp",
  "fileinput",
  "fnmatch",
  "formatter",
  "fractions",
  "ftplib",
  "functools",
  "genericpath",
  "getopt",
  "getpass",
  "gettext",
  "glob",
  "graphlib",
  "gzip",
  "hashlib",
  "heapq",
  "hmac",
  "html",
  "http",
  "imaplib",
  "imghdr",
  "imp",
  "importlib",
  "inspect",
  "io",
  "ipaddress",
  "json",
  "keyword",
  "lib2to3",
  "linecache",
  "locale",
  "logging",
  "lzma",
  "mailbox",
  "mailcap",
  "mimetypes",
  "modulefinder",
  "multiprocessing",
  "netrc",
  "nntplib",
  "ntpath",
  "nturl2path",
  "numbers",
  "opcode",
  "operator",
  "optparse",
  "os",
  "pathlib",
  "pdb",
  "pickle",
  "pickletools",
  "pipes",
  "pkgutil",
  "platform",
  "plistlib",
  "poplib",
  "posixpath",
  "pprint",
  "profile",
  "pstats",
  "pty",
  "py_compile",
  "pyclbr",
  "pydoc",
  "pydoc_data",
  "queue",
  "quopri",
  "random",
  "re",
  "reprlib",
  "rlcompleter",
  "runpy",
  "sched",
  "secrets",
  "selectors",
  "shelve",
  "shlex",
  "shutil",
  "signal",
  "site",
  "smtpd",
  "smtplib",
  "sndhdr",
  "socket",
  "socketserver",
  "sqlite3",
  "sre_compile",
  "sre_constants",
  "sre_parse",
  "ssl",
  "stat",
  "statistics",
  "string",
  "stringprep",
  "struct",
  "subprocess",
  "sunau",
  "symbol",
  "symtable",
  "sysconfig",
  "tabnanny",
  "tarfile",
  "telnetlib",
  "tempfile",
  "textwrap",
  "this",
  "threading",
  "timeit",
  "token",
  "tokenize",
  "trace",
  "traceback",
  "tracemalloc",
  "tty",
  "types",
  "typing",
  "unittest",
  "urllib",
  "uu",
  "uuid",
  "venv",
  "warnings",
  "wave",
  "weakref",
  "webbrowser",
  "wsgiref",
  "xdrlib",
  "xml",
  "xmlrpc",
  "zipapp",
  "zipfile",
  "zipimport",
  "zoneinfo",
  "_asyncio",
  "_bisect",
  "_blake2",
  "_bz2",
  "_codecs_cn",
  "_codecs_hk",
  "_codecs_iso2022",
  "_codecs_jp",
  "_codecs_kr",
  "_codecs_tw",
  "_contextvars",
  "_crypt",
  "_csv",
  "_ctypes",
  "_curses",
  "_curses_panel",
  "_datetime",
  "_dbm",
  "_decimal",
  "_elementtree",
  "_gdbm",
  "_hashlib",
  "_heapq",
  "_json",
  "_lsprof",
  "_lzma",
  "_multibytecodec",
  "_multiprocessing",
  "_opcode",
  "_pickle",
  "_posixshmem",
  "_posixsubprocess",
  "_queue",
  "_random",
  "_socket",
  "_sqlite3",
  "_ssl",
  "_statistics",
  "_struct",
  "_uuid",
  "_xxsubinterpreters",
  "_zoneinfo",
  "array",
  "audioop",
  "binascii",
  "cmath",
  "fcntl",
  "grp",
  "math",
  "mmap",
  "ossaudiodev",
  "parser",
  "pyexpat",
  "readline",
  "resource",
  "select",
  "spwd",
  "syslog",
  "termios",
  "unicodedata",
  "xxlimited",
  "zlib",
  "_distutils_hack",
  "_virtualenv",
  "pip",
  "pkg_resources",
  "setuptools",
  "wheel"
];