search

17451k / clade

Clade is a tool for extracting information about software build process and source code
Apache License 2.0
19 stars 6 forks source link

Clade sometimes cannot import chardet module #128

Closed IljaZakharov closed 4 years ago

IljaZakharov commented 4 years ago

Clade nondeterministically fails with the following exception:

clade_meta = self.clade.get_meta()

  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 719, in get_meta
    if not self.are_parsed("PidGraph"):
  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 163, in are_parsed
    e = self.__get_ext_obj(ext_name)
  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 145, in __get_ext_obj
    ext_objs = self.__get_ext_obj_list([ext_name])
  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 222, in __get_ext_obj_list
    ext_objs.extend(self.__create_ext_obj_list(ext_name, already_initialized))
  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 245, in __create_ext_obj_list
    e = self.__create_ext_obj(ext_name)
  File "/usr/local/lib/python3.7/dist-packages/clade/__init__.py", line 270, in __create_ext_obj
    Extension._import_extension_modules()
  File "/usr/local/lib/python3.7/dist-packages/clade/extensions/abstract.py", line 514, in _import_extension_modules
    __import__(module_name)
  File "/usr/local/lib/python3.7/dist-packages/clade/extensions/storage.py", line 16, in <module>
    import cchardet
  File "/usr/local/lib/python3.7/dist-packages/cchardet/__init__.py", line 1, in <module>
    from cchardet import _cchardet
ImportError: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: undefined symbol: _Unwind_GetRegionStart, version GCC_3.0
17451k commented 4 years ago

I am not sure that the issue is related to Clade. Maybe something is wrong in your system, or in compiler that was used to build cchardet. But if the issue become prevalent, I can always catch ImportError exception and replace cchardet by much slower chardet.

Also, in future please include meta.json file from the Clade base when reporting a bug. It contains info about the system and installed python packages that can be useful to determine the cause of the issue.

eunovm commented 4 years ago

I confirm that the issue happens very seldom, but for users it is extremely unpleasant since in the production mode they can wast considerable time and computational resources because of it.

I have a bit another environment as @IljaZakharov (Debian 9 rather than modern Ubuntu).

Here is the content of meta.json (GitHub does not allow to attach this file):

{
    "AR":{
        "corrupted":false,
        "time":"0:00:00.237104",
        "version":"1.1.1"
    },
    "CC":{
        "corrupted":false,
        "time":"0:00:00.530955",
        "version":"1.1.1.1"
    },
    "CL":{
        "corrupted":false,
        "time":"0:00:00.017632",
        "version":"1.1.1.1"
    },
    "Callgraph":{
        "corrupted":false,
        "time":"0:00:00.156132",
        "version":"1.1"
    },
    "CmdGraph":{
        "corrupted":false,
        "time":"0:00:00.080383",
        "version":"1.2"
    },
    "CrossRef":{
        "corrupted":false,
        "time":"0:00:00.878629",
        "version":"1.1.1"
    },
    "Functions":{
        "corrupted":false,
        "time":"0:00:00.139769",
        "version":"1.1.1"
    },
    "Info":{
        "corrupted":false,
        "time":"0:00:06.760744",
        "version":"1.2"
    },
    "LD":{
        "corrupted":false,
        "time":"0:00:00.264656",
        "version":"1.1.1"
    },
    "MV":{
        "corrupted":false,
        "time":"0:00:00.264158",
        "version":"1.1.1"
    },
    "Macros":{
        "corrupted":false,
        "time":"0:00:00.413661",
        "version":"1.2"
    },
    "Objcopy":{
        "corrupted":false,
        "time":"0:00:00.238820",
        "version":"1.1.1"
    },
    "Path":{
        "corrupted":false,
        "time":"0:00:00.000098",
        "version":"1.3"
    },
    "PidGraph":{
        "corrupted":false,
        "time":"0:00:00.026677",
        "version":"1.1"
    },
    "SrcGraph":{
        "corrupted":false,
        "time":"0:00:00.106388",
        "version":"1.1"
    },
    "Storage":{
        "corrupted":false,
        "time":"0:00:00.000122",
        "version":"1.1"
    },
    "Typedefs":{
        "corrupted":false,
        "time":"0:00:00.006358",
        "version":"1.1"
    },
    "Variables":{
        "corrupted":true,
        "time":"0:00:00.025741",
        "version":"1.1.1"
    },
    "build_dir":"/home/novikov/work/klever-data/linux-stable",
    "clade_version":"3.2.5",
    "conf":{
        "AR.which_list":[
            "(/|-)ar$"
        ],
        "AS.which_list":[
            "(/|-)as$"
        ],
        "CC.ignore_cc1":true,
        "CC.which_list":[
            "cc$",
            "cc1$",
            "[mg]cc(-?\\d+(\\.\\d+){0,2})?$",
            "clang(-?\\d+(\\.\\d+){0,2})?$",
            "aspectator$"
        ],
        "CC.with_system_header_files":true,
        "CL.deps_encoding":null,
        "CL.pre_encoding":null,
        "CL.which_list":[
            "(cl|CL)\\.exe$"
        ],
        "CmdGraph.as_picture":false,
        "CmdGraph.requires":[
            "AR",
            "CC",
            "LD",
            "MV",
            "Objcopy"
        ],
        "Common.filter":[
            ".*?\\.tmp$"
        ],
        "Common.filter_in":[
            "^-$",
            "/dev/null",
            "scripts/(?!mod/empty\\.c)",
            "kernel/.*?bounds.*?",
            "arch/x86/tools/relocs",
            "arch/x86/kernel/asm-offsets.c",
            "\\.mod\\.c",
            "\\.tmp\\w+\\.s",
            "\\.S"
        ],
        "Common.filter_out":[
            "/dev/null",
            "scripts/(?!mod/.*empty\\.o)",
            "\\.cmd$"
        ],
        "Compiler.deps_encoding":null,
        "Compiler.extra_preprocessor_opts":[

        ],
        "Compiler.preprocess_cmds":false,
        "Compiler.store_deps":true,
        "Copy.which_list":[
            "cmd\\.exe$"
        ],
        "Info.aspectator":null,
        "Info.cif":"cif",
        "Info.extra_CIF_opts":[
            "-D__GNUC__=4",
            "-D__GNUC_MINOR__=6"
        ],
        "Info.use_preprocessed_files":false,
        "LD.which_list":[
            "(/|-)ld$"
        ],
        "Link.which_list":[
            "link\\.exe$"
        ],
        "MV.which_list":[
            "/mv$"
        ],
        "Objcopy.which_list":[
            "objcopy$"
        ],
        "PidGraph.as_picture":false,
        "PidGraph.filter_cmds_by_pid":true,
        "SrcGraph.requires":[
            "CC",
            "CL"
        ],
        "Storage.convert_to_utf8":false,
        "Wrapper.recursive_wrap":false,
        "Wrapper.wrap_list":[

        ],
        "cpu_count":null,
        "extends":"base",
        "extensions":[
            "SrcGraph"
        ],
        "force":false,
        "log_level":"INFO"
    },
    "date":"2019-12-11 14:07",
    "platform":"Linux-4.12.14-lp150.12.79-default-x86_64-with-glibc2.3.4",
    "project attrs":[
        {
            "name":"project",
            "value":[
                {
                    "name":"name",
                    "value":"Linux"
                },
                {
                    "name":"architecture",
                    "value":"x86_64"
                },
                {
                    "name":"version",
                    "value":"v3.14"
                },
                {
                    "name":"configuration",
                    "value":"allmodconfig"
                }
            ]
        }
    ],
    "python_version":"3.6.5",
    "requirements":[
        "urllib3==1.24.1",
        "ujson==1.35",
        "six==1.12.0",
        "simplejson==3.16.0",
        "requests==2.21.0",
        "ply==3.11",
        "idna==2.8",
        "graphviz==0.10.1",
        "cryptography==2.6.1",
        "chardet==3.0.4",
        "certifi==2018.11.29",
        "cchardet==2.1.4",
        "asn1crypto==0.24.0"
    ],
    "target program description":{
        "Git repository version":"v3.14",
        "allow local source trees use":true,
        "architecture":"x86_64",
        "build base":"build bases/testing/5b3d50",
        "configuration":"allmodconfig",
        "external modules":"/home/novikov/work/klever/utils/bin/../../bridge/jobs/presets/linux/testing/decomposition strategies/tests",
        "external modules header files search directory":"/home/novikov/work/klever/utils/bin/../../bridge/jobs/presets/specifications",
        "extra Clade options":{
            "Info.extra_CIF_opts":[
                "-D__GNUC__=4",
                "-D__GNUC_MINOR__=6"
            ]
        },
        "extra headers":[
            "linux/user_namespace.h",
            "linux/tty.h",
            "linux/tty_driver.h",
            "linux/usb.h",
            "linux/usb/serial.h",
            "linux/platform_device.h",
            "linux/netdevice.h",
            "linux/net.h",
            "linux/timer.h",
            "linux/interrupt.h",
            "linux/seq_file.h",
            "linux/i2c.h",
            "linux/mod_devicetable.h",
            "linux/device.h",
            "linux/pm.h",
            "linux/fs.h",
            "linux/rtnetlink.h",
            "net/mac80211.h",
            "linux/iio/iio.h",
            "linux/iio/triggered_buffer.h",
            "linux/cdev.h",
            "linux/miscdevice.h",
            "linux/pci.h",
            "linux/rtc.h",
            "scsi/scsi_host.h",
            "linux/pagemap.h",
            "linux/poll.h",
            "linux/blkdev.h",
            "target/target_core_base.h",
            "target/target_core_backend.h",
            "linux/spi/spi.h",
            "linux/fb.h"
        ],
        "generate makefiles":true,
        "loadable kernel modules":[
            "ext-modules/kernel_library",
            "ext-modules/load_order",
            "ext-modules/multimodule_error",
            "ext-modules/multimodule_false_error",
            "ext-modules/several_groups"
        ],
        "model CC options file":"scripts/mod/empty.c",
        "name":"Linux",
        "source code":"linux-stable"
    },
    "uuid":"1e233170-f756-4f64-a59d-ef64f6b31548",
    "working source trees":[
        "/home/novikov/work/klever-data/linux-stable",
        "/tmp/tmpt7fou3_o"
    ]
}