Closed cameel closed 6 years ago
This is high priority. It happens quite often (19 times yesterday, possibly more today but Sentry was down) and prevents us from generating valid tokens. Whole use case 2 is probably broken because of this.
This happens in receive/
too:
https://talkback.golem.network/sentry/concent/issues/15337/
AssertionError: null
File "django/core/handlers/base.py", line 185, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "python3.6/contextlib.py", line 52, in inner
return func(*args, **kwds)
File "utils/decorators.py", line 202, in wrapper
return _function(*args, **kwargs)
File "django/views/decorators/csrf.py", line 58, in wrapped_view
return view_func(*args, **kwargs)
File "django/views/decorators/http.py", line 40, in inner
return func(request, *args, **kwargs)
File "utils/decorators.py", line 68, in wrapper
return view(request, auth_message, auth_message.client_public_key, *args, *kwargs)
File "utils/decorators.py", line 124, in wrapper
response_from_view = view(request, client_message, client_public_key, *args, **kwargs)
File "core/views.py", line 50, in receive
response_type = PendingResponse.Queue.Receive,
File "core/message_handlers.py", line 937, in handle_messages_from_database
FileTransferToken.Operation.upload,
File "core/transfer_operations.py", line 157, in create_file_transfer_token_for_golem_client
token_expiration_deadline=calculate_token_expiration_deadline(operation, report_computed_task)
File "core/transfer_operations.py", line 176, in _create_file_transfer_token
assert isinstance(token_expiration_deadline, int) and not isinstance(token_expiration_deadline, bool) or token_expiration_deadline is None
ForceGetTaskResult(header=MessageHeader(type_=4009, timestamp=1528814276, encrypted=False), sig=None, slots=[['report_computed_task', ReportComputedTask(header=MessageHeader(type_=2004, timestamp=1528814276, encrypted=False), sig=b'@\xe6\x1b\x03G\x8a\x91\xef\x0b\\\xb7\x04LU\xc8fcX*/\x8c\x8a\x19\xb6\x9c\x9e\xdb\xfe\xe4\x8d\x17k\t\x00\xa9\xe0\xf7/\xc8\xca\x05\x14g\xc6T\xba\x1d\xe5\xde-C\x8f\n\x81V\xdf8\xbe\xbee\xbf\x9c\xefL\x00', slots=[['result_type', 1], ['node_name', ''], ['address', '10.30.8.141'], ['node_info', {'hyperdrive_prv_port': 3282, 'hyperdrive_pub_port': 3282, 'key': '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27', 'nat_type': 'Symmetric NAT', 'node_name': '', 'p2p_prv_port': 40102, 'p2p_pub_port': 40102, 'port_statuses': {3282: 'timeout', 40102: 'timeout', 40103: 'timeout'}, 'prv_addr': '10.30.8.141', 'prv_addresses': ['10.30.8.141', '192.168.246.1', '192.168.107.1', '172.17.0.1'], 'prv_port': 40103, 'pub_addr': '194.181.80.91', 'pub_port': 40103}], ['port', 40103], ['key_id', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'], ['extra_data', ['test task_10001.png', 'stdout.log', 'stderr.log']], ['eth_account', '0x37bc2d4d565b3fcB557B86E682D2AA255Fde0550'], ['task_to_compute', TaskToCompute(header=MessageHeader(type_=2002, timestamp=1528814265, encrypted=False), sig=b'\x15(\x9fg(\xb2\xef\xba\xd4\xc7*\x9e\xe8hyQ\x08\xb1A\xaau\xb85\x8f\xb4~\x89%<x\xe0AFU\x02\xb2\xfe\x8e\xd1\x83\x0f\xb5c1\xdf\x1f\xa0\xd0C\x14i\xeeP\xda\xc4E\xe2YI\xfb\xb5\x13\x99\xf5\x01', slots=[['requestor_id', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'], ['requestor_public_key', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'], ['requestor_ethereum_public_key', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'], ['provider_id', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'], ['provider_public_key', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'], ['provider_ethereum_public_key', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'], ['compute_task_def', {'deadline': 1528814788.516703, 'docker_images': [{'image_id': None, 'repository': 'golemfactory/blender', 'tag': '1.4'}], 'extra_data': {'end_task': 1, 'frames': [1], 'outfilebasename': 'test task', 'output_format': 'PNG', 'path_root': '/home/blue/PycharmProjects/golem/scripts/concent_node_tests/tasks/test_task_1', 'scene_file': '/golem/resources/wlochaty3.blend', 'script_src': '# This template is rendered by\n# apps.blender.resources.scenefileeditor.generate_blender_crop_file(),\n# written to tempfile and passed as arg to blender.\nimport bpy\n\nclass EngineWarning(bpy.types.Operator):\n bl_idname = "wm.engine_warning"\n bl_label = "Inform about not supported rendering engine"\n\n def execute(self, context):\n self.report({"ERROR"}, "Engine " + bpy.context.scene.render.engine + \\\n " not supported by Golem")\n return {"FINISHED"}\n\nclass ShowInformation(bpy.types.Operator):\n bl_idname = "wm.scene_information"\n bl_label = "Inform user about scene settings"\n\n\n def execute(self, context):\n self.report({"INFO"}, "Engine: " +\n str(bpy.context.scene.render.engine))\n if bpy.context.scene.render.engine == "CYCLES":\n self.report({"INFO"}, "Samples: " + str(bpy.context.scene.cycles.samples))\n self.report({"INFO"}, "Resolution: " +\n str(bpy.context.scene.render.resolution_x) +\n " x " +\n str(bpy.context.scene.render.resolution_y))\n self.report({"INFO"}, "File format: " +\n str(bpy.context.scene.render.file_extension))\n self.report({"INFO"}, "Filepath: " +\n str(bpy.context.scene.render.filepath))\n self.report({"INFO"}, "Frames: " +\n str(bpy.context.scene.frame_start) + "-" +\n str(bpy.context.scene.frame_end) + ";" +\n str(bpy.context.scene.frame_step))\n\n return {"FINISHED"}\n\n\nbpy.utils.register_class(EngineWarning)\nengine = bpy.context.scene.render.engine\nif engine not in ("BLENDER_RENDER", "CYCLES"):\n bpy.ops.wm.engine_warning()\n\nbpy.utils.register_class(ShowInformation)\nbpy.ops.wm.scene_information()\n\nbpy.context.scene.render.tile_x = 0\nbpy.context.scene.render.tile_y = 0\nbpy.context.scene.render.resolution_x = 320\nbpy.context.scene.render.resolution_y = 240\nbpy.context.scene.render.resolution_percentage = 100\nbpy.context.scene.render.use_border = True\nbpy.context.scene.render.use_crop_to_border = True\nbpy.context.scene.render.border_max_x = 1.0\nbpy.context.scene.render.border_min_x = 0.0\nbpy.context.scene.render.border_min_y = 0.0\nbpy.context.scene.render.border_max_y = 1.0\nbpy.context.scene.render.use_compositing = bool(False)\nif engine == "CYCLES":\n samples = 0\n if samples != 0:\n bpy.context.scene.cycles.samples = samples\n\n#and check if additional files aren\'t missing\nbpy.ops.file.report_missing_files()\n', 'start_task': 1, 'total_tasks': 1}, 'performance': 506.1405887496573, 'short_description': 'path_root: /home/blue/PycharmProjects/golem/scripts/concent_node_tests/tasks/test_task_1, start_task: 1, end_task: 1, total_tasks: 1, outfilebasename: test task, scene_file: /golem/resources/wlochaty3.blend', 'src_code': 'from __future__ import print_function\n\nimport os\nimport subprocess\nimport sys\nfrom multiprocessing import cpu_count\n\nimport params # This module is generated before this script is run\n\nBLENDER_COMMAND = "blender"\nWORK_DIR = "/golem/work"\nOUTPUT_DIR = "/golem/output"\n\n\ndef exec_cmd(cmd):\n pc = subprocess.Popen(cmd)\n return pc.wait()\n\n\ndef format_blender_render_cmd(outfilebasename, scene_file, script_file,\n start_task, frame, output_format):\n cmd = [\n "{}".format(BLENDER_COMMAND),\n "-b", "{}".format(scene_file),\n "-y", # enable scripting by default\n "-P", "{}".format(script_file),\n "-o", "{}/{}_{}".format(OUTPUT_DIR, outfilebasename, start_task),\n "-noaudio",\n "-F", "{}".format(output_format.upper()),\n "-t", "{}".format(cpu_count()),\n "-f", "{}".format(frame)\n ]\n return cmd\n\n\ndef run_blender_task(outfilebasename, scene_file, script_src, start_task,\n frames, output_format):\n scene_file = os.path.normpath(scene_file)\n if not os.path.exists(scene_file):\n print("Scene file \'{}\' does not exist".format(scene_file),\n file=sys.stderr)\n sys.exit(1)\n\n blender_script_path = WORK_DIR + "/blenderscript.py"\n with open(blender_script_path, "w") as script_file:\n script_file.write(script_src)\n\n for frame in frames:\n cmd = format_blender_render_cmd(outfilebasename, scene_file,\n script_file.name, start_task, frame, output_format)\n print(cmd, file=sys.stderr)\n exit_code = exec_cmd(cmd)\n if exit_code is not 0:\n sys.exit(exit_code)\n\n\nrun_blender_task(params.outfilebasename, params.scene_file, params.script_src, params.start_task, params.frames,\n params.output_format)\n\n', 'subtask_id': '2bf0e7f6-6e4e-11e8-a545-2c447ed6985d', 'task_id': 'fdd87c8a-6e4d-11e8-8150-2c447ed6985d', 'working_directory': '.'}], ['package_hash', 'sha1:a292f007f33cd183c6625f061cde9526ba9e8612'], ['size', 0], ['concent_enabled', True], ['price', 163888888888888889]])], ['size', 261056], ['package_hash', 'sha1:36d4541390c59b1e92321287a8219407f8d204bb'], ['multihash', '350d8154ce06e86bdfb322f8902fe1fa998202ba3b237500ed92656cb49da1e4'], ['secret', b'\xc3\xbd\x1fF\xd18\xc9V:y5\xdaA\xc4M\x0bu-r\xd8_\xc4\xde{\x11'], ['options', {'client_id': 'hyperg', 'options': {'peers': [{'TCP': ['10.30.8.141', 3282]}, {'TCP': ['10.30.8.141', 3282]}, {'TCP': ['192.168.246.1', 3282]}, {'TCP': ['192.168.107.1', 3282]}, {'TCP': ['194.181.80.91', 3282]}, {'TCP': ['172.17.0.1', 3282]}]}, 'version': 1.1}]])]])
ForceGetTaskResult(
header=MessageHeader(type_=4009, timestamp=1528814276, encrypted=False),
sig=None, slots=[
['report_computed_task',
ReportComputedTask(
header=MessageHeader(type_=2004, timestamp=1528814276, encrypted=False),
sig=b'@\xe6\x1b\x03G\x8a\x91\xef\x0b\\\xb7\x04LU\xc8fcX*/\x8c\x8a\x19\xb6\x9c\x9e\xdb\xfe\xe4\x8d\x17k\t\x00\xa9\xe0\xf7/\xc8\xca\x05\x14g\xc6T\xba\x1d\xe5\xde-C\x8f\n\x81V\xdf8\xbe\xbee\xbf\x9c\xefL\x00',
slots=[
['result_type', 1],
['node_name', ''],
['address', '10.30.8.141'],
['node_info', {'hyperdrive_prv_port': 3282, 'hyperdrive_pub_port': 3282, 'key': '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27', 'nat_type': 'Symmetric NAT', 'node_name': '', 'p2p_prv_port': 40102, 'p2p_pub_port': 40102, 'port_statuses': {3282: 'timeout', 40102: 'timeout', 40103: 'timeout'}, 'prv_addr': '10.30.8.141', 'prv_addresses': ['10.30.8.141', '192.168.246.1', '192.168.107.1', '172.17.0.1'], 'prv_port': 40103, 'pub_addr': '194.181.80.91', 'pub_port': 40103}],
['port', 40103],
['key_id', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'],
['extra_data', ['test task_10001.png', 'stdout.log', 'stderr.log']],
['eth_account', '0x37bc2d4d565b3fcB557B86E682D2AA255Fde0550'],
['task_to_compute', TaskToCompute(
header=MessageHeader(type_=2002, timestamp=1528814265, encrypted=False),
sig=b'\x15(\x9fg(\xb2\xef\xba\xd4\xc7*\x9e\xe8hyQ\x08\xb1A\xaau\xb85\x8f\xb4~\x89%<x\xe0AFU\x02\xb2\xfe\x8e\xd1\x83\x0f\xb5c1\xdf\x1f\xa0\xd0C\x14i\xeeP\xda\xc4E\xe2YI\xfb\xb5\x13\x99\xf5\x01',
slots=[
['requestor_id', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'],
['requestor_public_key', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'],
['requestor_ethereum_public_key', '2c447ed6985d0ab80f2f920767e2df13836ec0a8ff13499c697d6c331b4622ee46cfb981e469c7fc57a5add13ac57968047118c81ae367789198fb96174d014d'],
['provider_id', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'],
['provider_public_key', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'],
['provider_ethereum_public_key', '7bfb2a95726b59c2e92fc7a9d4d4f431850af611417c25f3e3ba2c45f736247233f767ebfc36a38414ad57127129f4bf343bc898de4b5e460d722c328f104d27'],
['compute_task_def', {
'deadline': 1528814788.516703,
'docker_images': [{'image_id': None, 'repository': 'golemfactory/blender', 'tag': '1.4'}],
'extra_data': {
'end_task': 1,
'frames': [1],
'outfilebasename': 'test task',
'output_format': 'PNG',
'path_root': '/home/blue/PycharmProjects/golem/scripts/concent_node_tests/tasks/test_task_1',
'scene_file': '/golem/resources/wlochaty3.blend',
'script_src': '# This template is rendered by\n# apps.blender.resources.scenefileeditor.generate_blender_crop_file(),\n# written to tempfile and passed as arg to blender.\nimport bpy\n\nclass EngineWarning(bpy.types.Operator):\n bl_idname = "wm.engine_warning"\n bl_label = "Inform about not supported rendering engine"\n\n def execute(self, context):\n self.report({"ERROR"}, "Engine " + bpy.context.scene.render.engine + \\\n " not supported by Golem")\n return {"FINISHED"}\n\nclass ShowInformation(bpy.types.Operator):\n bl_idname = "wm.scene_information"\n bl_label = "Inform user about scene settings"\n\n\n def execute(self, context):\n self.report({"INFO"}, "Engine: " +\n str(bpy.context.scene.render.engine))\n if bpy.context.scene.render.engine == "CYCLES":\n self.report({"INFO"}, "Samples: " + str(bpy.context.scene.cycles.samples))\n self.report({"INFO"}, "Resolution: " +\n str(bpy.context.scene.render.resolution_x) +\n " x " +\n str(bpy.context.scene.render.resolution_y))\n self.report({"INFO"}, "File format: " +\n str(bpy.context.scene.render.file_extension))\n self.report({"INFO"}, "Filepath: " +\n str(bpy.context.scene.render.filepath))\n self.report({"INFO"}, "Frames: " +\n str(bpy.context.scene.frame_start) + "-" +\n str(bpy.context.scene.frame_end) + ";" +\n str(bpy.context.scene.frame_step))\n\n return {"FINISHED"}\n\n\nbpy.utils.register_class(EngineWarning)\nengine = bpy.context.scene.render.engine\nif engine not in ("BLENDER_RENDER", "CYCLES"):\n bpy.ops.wm.engine_warning()\n\nbpy.utils.register_class(ShowInformation)\nbpy.ops.wm.scene_information()\n\nbpy.context.scene.render.tile_x = 0\nbpy.context.scene.render.tile_y = 0\nbpy.context.scene.render.resolution_x = 320\nbpy.context.scene.render.resolution_y = 240\nbpy.context.scene.render.resolution_percentage = 100\nbpy.context.scene.render.use_border = True\nbpy.context.scene.render.use_crop_to_border = True\nbpy.context.scene.render.border_max_x = 1.0\nbpy.context.scene.render.border_min_x = 0.0\nbpy.context.scene.render.border_min_y = 0.0\nbpy.context.scene.render.border_max_y = 1.0\nbpy.context.scene.render.use_compositing = bool(False)\nif engine == "CYCLES":\n samples = 0\n if samples != 0:\n bpy.context.scene.cycles.samples = samples\n\n#and check if additional files aren\'t missing\nbpy.ops.file.report_missing_files()\n',
'start_task': 1,
'total_tasks': 1
},
'performance': 506.1405887496573,
'short_description': 'path_root: /home/blue/PycharmProjects/golem/scripts/concent_node_tests/tasks/test_task_1, start_task: 1, end_task: 1, total_tasks: 1, outfilebasename: test task, scene_file: /golem/resources/wlochaty3.blend',
'src_code': 'from __future__ import print_function\n\nimport os\nimport subprocess\nimport sys\nfrom multiprocessing import cpu_count\n\nimport params # This module is generated before this script is run\n\nBLENDER_COMMAND = "blender"\nWORK_DIR = "/golem/work"\nOUTPUT_DIR = "/golem/output"\n\n\ndef exec_cmd(cmd):\n pc = subprocess.Popen(cmd)\n return pc.wait()\n\n\ndef format_blender_render_cmd(outfilebasename, scene_file, script_file,\n start_task, frame, output_format):\n cmd = [\n "{}".format(BLENDER_COMMAND),\n "-b", "{}".format(scene_file),\n "-y", # enable scripting by default\n "-P", "{}".format(script_file),\n "-o", "{}/{}_{}".format(OUTPUT_DIR, outfilebasename, start_task),\n "-noaudio",\n "-F", "{}".format(output_format.upper()),\n "-t", "{}".format(cpu_count()),\n "-f", "{}".format(frame)\n ]\n return cmd\n\n\ndef run_blender_task(outfilebasename, scene_file, script_src, start_task,\n frames, output_format):\n scene_file = os.path.normpath(scene_file)\n if not os.path.exists(scene_file):\n print("Scene file \'{}\' does not exist".format(scene_file),\n file=sys.stderr)\n sys.exit(1)\n\n blender_script_path = WORK_DIR + "/blenderscript.py"\n with open(blender_script_path, "w") as script_file:\n script_file.write(script_src)\n\n for frame in frames:\n cmd = format_blender_render_cmd(outfilebasename, scene_file,\n script_file.name, start_task, frame, output_format)\n print(cmd, file=sys.stderr)\n exit_code = exec_cmd(cmd)\n if exit_code is not 0:\n sys.exit(exit_code)\n\n\nrun_blender_task(params.outfilebasename, params.scene_file, params.script_src, params.start_task, params.frames,\n params.output_format)\n\n',
'subtask_id': '2bf0e7f6-6e4e-11e8-a545-2c447ed6985d',
'task_id': 'fdd87c8a-6e4d-11e8-8150-2c447ed6985d',
'working_directory': '.'
}],
['package_hash', 'sha1:a292f007f33cd183c6625f061cde9526ba9e8612'],
['size', 0],
['concent_enabled', True],
['price', 163888888888888889]
]
)],
['size', 261056],
['package_hash', 'sha1:36d4541390c59b1e92321287a8219407f8d204bb'],
['multihash', '350d8154ce06e86bdfb322f8902fe1fa998202ba3b237500ed92656cb49da1e4'],
['secret', b'\xc3\xbd\x1fF\xd18\xc9V:y5\xdaA\xc4M\x0bu-r\xd8_\xc4\xde{\x11'],
['options', {'client_id': 'hyperg', 'options': {'peers': [{'TCP': ['10.30.8.141', 3282]}, {'TCP': ['10.30.8.141', 3282]}, {'TCP': ['192.168.246.1', 3282]}, {'TCP': ['192.168.107.1', 3282]}, {'TCP': ['194.181.80.91', 3282]}, {'TCP': ['172.17.0.1', 3282]}]}, 'version': 1.1}]
]
)]
]
)
@Jakub89 As agreed on the last retro, please drop some info on how the issue was solved when you closed it.
Issue was solved by adding int()
conversion to report_computed_task.task_to_compute.compute_task_def['deadline']
, when we run calculate_token_expiration_deadline()
function. I forget to add it in previous bugfix #455 .
On top of that I've added some unit tests for this.
We have received the following crash report in Sentry (https://talkback.golem.network/sentry/concent/issues/15278/):
The error happened three times on the testnet. Looks like our code tries to create an invalid token.