KeyError: 'shellphish_qemu/bin'

[knnikita]

Hi! I run the example you provided, but get KeyError : 'shellphish_qemu/bin'. What could be the problem?

In [1]: import rex                                                                                                                                                                                                 

In [2]: crash = rex.Crash("./legit_00003", b"\x00\x0b1\xc1\x00\x0c\xeb\xe4\xf1\xf1\x14\r\rM\r\xf3\x1b\r\r\r~\x7f\x1b\xe3\x0c`_222\r\rM\r\xf3\x1b\r\x7f\x002\x7f~\x7f\xe2\xff\x7f\xff\xff\x8b\xc7\xc9\x83\x8b\x0c\xe
   ...: b\x80\x002\xac\xe2\xff\xff\x00t\x8bt\x8bt_o_\x00t\x8b\xc7\xdd\x83\xc2t~n~~\xac\xe2\xff\xff_k_\x00t\x8b\xc7\xdd\x83\xc2t~n~~\xac\xe2\xff\xff\x00t\x8bt\x8b\xac\xf1\x83\xc2t~c\x00\x00\x00~~\x7f\xe2\xff\xff\
   ...: x00t\x9e\xac\xe2\xf1\xf2@\x83\xc3t")                                                                                                                                                                       
WARNING | 2018-12-20 15:01:41,188 | angr.engines.vex.engine | Executing symbolic code at 0xbaaab04c
WARNING | 2018-12-20 15:01:41,666 | angr.engines.vex.engine | Executing symbolic code at 0xbaaab112
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-2-bb2d345df134> in <module>
----> 1 crash = rex.Crash("./legit_00003", b"\x00\x0b1\xc1\x00\x0c\xeb\xe4\xf1\xf1\x14\r\rM\r\xf3\x1b\r\r\r~\x7f\x1b\xe3\x0c`_222\r\rM\r\xf3\x1b\r\x7f\x002\x7f~\x7f\xe2\xff\x7f\xff\xff\x8b\xc7\xc9\x83\x8b\x0c\xeb\x80\x002\xac\xe2\xff\xff\x00t\x8bt\x8bt_o_\x00t\x8b\xc7\xdd\x83\xc2t~n~~\xac\xe2\xff\xff_k_\x00t\x8b\xc7\xdd\x83\xc2t~n~~\xac\xe2\xff\xff\x00t\x8bt\x8b\xac\xf1\x83\xc2t~c\x00\x00\x00~~\x7f\xe2\xff\xff\x00t\x9e\xac\xe2\xf1\xf2@\x83\xc3t")

~/virtenvs/angr/lib/python3.6/site-packages/rex/crash.py in __init__(self, binary, crash, pov_file, aslr, constrained_addrs, crash_state, prev_path, hooks, format_infos, rop_cache_tuple, use_rop, fast_mode, explore_steps, angrop_object, argv, concrete_fs, chroot, rop_cache_path, trace_timeout, input_type, port, use_crash_input, tracer_args, initial_state)
    135             # optimized crash check
    136             if self.os == 'cgc':
--> 137                 if not tracer.QEMURunner(binary, input=self.crash, **tracer_args).crash_mode:
    138                     if not tracer.QEMURunner(binary, input=self.crash, report_bad_args=True, **tracer_args).crash_mode:
    139                         l.warning("input did not cause a crash")

~/virtenvs/angr/lib/python3.6/site-packages/tracer-0.1-py3.6.egg/tracer/qemu_runner.py in __init__(self, binary, input, project, record_trace, record_stdout, record_magic, record_core, seed, memory_limit, bitflip, report_bad_args, use_tiny_core, max_size, qemu, argv, library_path, ld_linux, trace_log_limit, trace_timeout, exec_func)
    142         self.trace_log_limit = trace_log_limit
    143         self.trace_timeout = trace_timeout
--> 144         self.sanity_check()
    145 
    146         l.debug("Accumulating basic block trace...")

~/virtenvs/angr/lib/python3.6/site-packages/tracer-0.1-py3.6.egg/tracer/qemu_runner.py in sanity_check(self)
    172 
    173     def sanity_check(self):
--> 174         self._check_binary()
    175         self._check_qemu_install()
    176 

~/virtenvs/angr/lib/python3.6/site-packages/tracer-0.1-py3.6.egg/tracer/qemu_runner.py in _check_binary(self)
    194 
    195         # try to find the install base
--> 196         self._check_qemu_install()
    197 
    198     def _check_qemu_install(self):

~/virtenvs/angr/lib/python3.6/site-packages/tracer-0.1-py3.6.egg/tracer/qemu_runner.py in _check_qemu_install(self)
    210                 l.warning("Problem accessing forced %s. Using our default %s.", self._trace_source_path, self.trace_source)
    211 
--> 212             self._trace_source_path = shellphish_qemu.qemu_path(self.trace_source)
    213 
    214             if not os.access(self._trace_source_path, os.X_OK):

~/virtenvs/angr/lib/python3.6/site-packages/shellphish_qemu-0.9.10-py3.6.egg/shellphish_qemu/__init__.py in qemu_path(platform)
     10         '%s' % platform,
     11     ):
---> 12         path = os.path.join(qemu_base(), basename)
     13         if os.path.isfile(path):
     14             return path

~/virtenvs/angr/lib/python3.6/site-packages/shellphish_qemu-0.9.10-py3.6.egg/shellphish_qemu/__init__.py in qemu_base()
     17 
     18 def qemu_base():
---> 19     return pkg_resources.resource_filename('shellphish_qemu', 'bin')
     20 
     21 def qemu_list():

~/virtenvs/angr/lib/python3.6/site-packages/pkg_resources/__init__.py in resource_filename(self, package_or_requirement, resource_name)
   1135         """Return a true filesystem path for specified resource"""
   1136         return get_provider(package_or_requirement).get_resource_filename(
-> 1137             self, resource_name
   1138         )
   1139 

~/virtenvs/angr/lib/python3.6/site-packages/pkg_resources/__init__.py in get_resource_filename(self, manager, resource_name)
   1644             for name in eagers:
   1645                 self._extract_resource(manager, self._eager_to_zip(name))
-> 1646         return self._extract_resource(manager, zip_path)
   1647 
   1648     @staticmethod

~/virtenvs/angr/lib/python3.6/site-packages/pkg_resources/__init__.py in _extract_resource(self, manager, zip_path)
   1665             return os.path.dirname(last)
   1666 
-> 1667         timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
   1668 
   1669         if not WRITE_SUPPORT:

KeyError: 'shellphish_qemu/bin'

`

[rhelmot]

Your shellphish-qemu installation seems to be very messed up. How did you install it? It shouldn't be possible to get into a setup like this if you installed it with pip.

[knnikita]

Your shellphish-qemu installation seems to be very messed up. How did you install it? It shouldn't be possible to get into a setup like this if you installed it with pip.

I installed angr from https://hub.docker.com/r/angr/angr. Then packages from repos: https://github.com/shellphish/shellphish-qemu.git https://github.com/mechaphish/povsim.git https://github.com/salls/angrop.git https://github.com/mechaphish/compilerex.git https://github.com/shellphish/rex.git https://github.com/angr/tracer https://github.com/shellphish/shellphish-afl.git

And then

sudo apt-get update
sudo apt-get install -y virtualenvwrapper python2.7-dev build-essential sudo libxml2-dev libxslt1-dev git libffi-dev cmake libreadline-dev libtool debootstrap debian-archive-keyring libglib2.0-dev libpixman-1-dev libpq-dev python-dev libc6:i386 libncurses5:i386 libstdc++6:i386 zlib1g:i386 pkg-config zlib1g-dev libtool libtool-bin wget automake autoconf coreutils bison libacl1-dev qemu-user qemu-kvm socat postgresql-client nasm binutils-multiarch llvm clang

[rhelmot]

Please do not try to build shellphish-qemu and shellphish-afl by yourself. Use the wheels on pypi or in https://github.com/angr/wheels.

[rhelmot]

If you must, install it with pip install . or pip install -e .

[knnikita]

If you must, install it with pip install . or pip install -e .

Now it work! Thanks! Other binary (elf arm32):

WARNING | 2018-12-21 10:34:44,884 | pyvex.lifting.gym.arm_spotter | Spotting an LDM instruction at 0x176ec4.  This is not fully tested.  Prepare for errors.
WARNING | 2018-12-21 10:34:44,884 | pyvex.lifting.gym.arm_spotter | Spotting an LDM instruction at 0x176ec4.  This is not fully tested.  Prepare for errors.
WARNING | 2018-12-21 10:35:01,381 | tracer.qemu_runner | The trace is found to be malformed. it is possible that the log file size exceeds the 1G limit, meaning that there might be infinite loops in the target program.
---------------------------------------------------------------------------
AngrTracerError                           Traceback (most recent call last)
<ipython-input-3-86ac8f8f0c42> in <module>
----> 1 crash = rex.Crash('./tlrun_static', bytes_crash)

~/virtenvs/angr/lib/python3.6/site-packages/rex/crash.py in __init__(self, binary, crash, pov_file, aslr, constrained_addrs, crash_state, prev_path, hooks, format_infos, rop_cache_tuple, use_rop, fast_mode, explore_steps, angrop_object, argv, concrete_fs, chroot, rop_cache_path, trace_timeout, input_type, port, use_crash_input, tracer_args, initial_state)
    223 
    224             self._t = angr.exploration_techniques.Tracer(trace=r.trace, resiliency=False, keep_predecessors=2, crash_addr=r.crash_addr)
--> 225             simgr.use_technique(self._t)
    226             simgr.use_technique(angr.exploration_techniques.Oppologist())
    227 

/home/angr/angr-dev/angr/angr/sim_manager.py in use_technique(self, tech)
    186         # XXX: as promised
    187         tech.project = self._project
--> 188         tech.setup(self)
    189 
    190         HookSet.install_hooks(self, **tech._get_hooks())

/home/angr/angr-dev/angr/angr/exploration_techniques/tracer.py in setup(self, simgr)
     72                 break
     73         else:
---> 74             raise AngrTracerError("Could not identify program entry point in trace!")
     75 
     76         # pylint: disable=undefined-loop-variable

AngrTracerError: Could not identify program entry point in trace!

what does it mean?

[rhelmot]

It means angr can't figure out which block in the trace is the program entry point. That seems like a fairly easy conclusion to draw.

Get a debug shell there and print out the trace, and see if you can identify the entry point in the trace. Should be pretty easy if the program isn't PIE.

It might have to do with the warning message: The trace is found to be malformed. it is possible that the log file size exceeds the 1G limit, meaning that there might be infinite loops in the target program.. I don't really know how to approach that issue though.

[knnikita]

Get a debug shell there and print out the trace, and see if you can identify the entry point in the trace. Should be pretty easy if the program isn't PIE.

How I can get a debug shell? I used logging:

logging.getLogger('rex').setLevel('DEBUG')
logging.getLogger('angr').setLevel('DEBUG')
logging.getLogger('tracer').setLevel('DEBUG')

and get this:

INFO    | 2018-12-25 10:53:17,486 | angr.project | Loading binary ./bin_test
DEBUG   | 2018-12-25 10:53:19,608 | angr.project | hooking 0x2000000 with <SimProcedure LinuxLoader>
DEBUG   | 2018-12-25 10:53:19,608 | angr.project | hooking 0x2000008 with <SimProcedure _dl_rtld_lock_recursive>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0x2000010 with <SimProcedure _dl_rtld_unlock_recursive>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0x2000018 with <SimProcedure _dl_initial_error_catch_tsd>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0x2000020 with <SimProcedure _vsyscall>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0xffff0fe0 with <SimProcedure _kernel_user_helper_get_tls>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0x2000030 with <SimProcedure CallReturn>
DEBUG   | 2018-12-25 10:53:19,609 | angr.project | hooking 0x2000038 with <SimProcedure UnresolvableJumpTarget>
DEBUG   | 2018-12-25 10:53:19,610 | angr.project | hooking 0x2000040 with <SimProcedure UnresolvableCallTarget>
DEBUG   | 2018-12-25 10:53:19,657 | tracer.qemu_runner | Accumulating basic block trace...
DEBUG   | 2018-12-25 10:53:19,657 | tracer.qemu_runner | tracer qemu path: /root/virtenvs/angr/lib/python3.6/site-packages/shellphish_qemu/bin/shellphish-qemu-linux-arm
WARNING | 2018-12-25 10:53:29,773 | tracer.qemu_runner | The trace is found to be malformed. it is possible that the log file size exceeds the 1G limit, meaning that there might be infinite loops in the target program.
DEBUG   | 2018-12-25 10:53:29,776 | angr.state_plugins.solver | Creating SimSolverClaripy.
DEBUG   | 2018-12-25 10:53:29,823 | angr.state_plugins.solver | Creating new unconstrained BV named aeg_stdin_0
DEBUG   | 2018-12-25 10:53:29,824 | angr.state_plugins.preconstrainer | Preconstraint: <Bool aeg_stdin_0_15546_8{UNINITIALIZED} == 5>
DEBUG   | 2018-12-25 10:53:30,155 | angr.state_plugins.solver | Creating new unconstrained BV named aeg_stdin_1
DEBUG   | 2018-12-25 10:53:30,155 | angr.state_plugins.preconstrainer | Preconstraint: <Bool aeg_stdin_1_15547_8{UNINITIALIZED} == 0>
....

Also a get many warning like this:

WARNING | 2018-12-25 10:53:06,036 | pyvex.lifting.gym.arm_spotter | Spotting an LDM instruction at 0x1735e0.  This is not fully tested.  Prepare for errors.
WARNING | 2018-12-25 10:53:06,057 | pyvex.lifting.gym.arm_spotter | Spotting an LDM instruction at 0x173606.  This is not fully tested.  Prepare for errors.
WARNING | 2018-12-25 10:53:06,066 | pyvex.lifting.gym.arm_spotter | Ignoring STMxx ^ instruction at 0x17360e. This mode is not implemented by VEX! See pyvex/lifting/gym/arm_spotter.py
WARNING | 2018-12-25 10:53:06,068 | pyvex.lifting.gym.arm_spotter | Ignoring STMxx ^ instruction at 0x173610. This mode is not implemented by VEX! See pyvex/lifting/gym/arm_spotter.py

Sorry, I don't understand what it means.

[rhelmot]

A shell is an interactive prompt. A debug shell is an interactive prompt embedded in the program that lets you explore its state at a given point.

In python, the two main ways of doing this are to insert import ipdb; ipdb.set_trace() into a source file to effectively set a breakpoint, or to run ipython --pdb instead of python to execute your scripts, which will spawn a debug shell when the program terminates with an exception. Both of these shells will be the pdb shell, which is like the gdb shell but for python. Tutorials for using it should be available online.

[github-actions[bot]]

This issue has been marked as stale because it has no recent activity. Please comment or add the pinned tag to prevent this issue from being closed.

[github-actions[bot]]

This issue has been closed due to inactivity.