search

golemfactory / concent

Repository for Concent Service sources
8 stars 7 forks source link

Use real sample of TaskToCompute in tests #235

Closed rwrzesien closed 6 years ago

rwrzesien commented 6 years ago

We have received a real-world example of TaskToCompute. We might use it in our tests to avoid surprises in the future. While working on this issue please do:

Sample TaskToCompute fully filled:

TaskToCompute(
    header=MessageHeader(
        type_=2002,
        timestamp=1522323925,
        encrypted=False),
    sig=b'T\xbb;\x8e`\xfb6\x84\xeaO)\xa4\x1b\x84\x87@\xa8\xe8\xdeZWO \xea\xd5oQ\xce\x1d\x8f\x15;L\xc0\xd0.t\xe9\xddz\xe9\x90\xf6\xd7\x8f\xec@\x1f\xcb\xbf\xf0\x94\xcb\xc0\x96\xa9\xfe\x9b\xf0\xe5\xe7f\xdd\x9c\x00',
    slots=[
        ['requestor_id', '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['requestor_public_key', '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['requestor_ethereum_public_key', '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['provider_id', 'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['provider_public_key', 'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['provider_ethereum_public_key', 'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['compute_task_def',{
        'deadline': 1522355642.491945,
        'docker_images': [{
            'image_id': None,
         'repository': 'golemfactory/blender',
         'tag': '1.4'
        }],
        'extra_data': {
            'end_task': 6,
            'frames': [1],
            'outfilebasename': 'Heli-cycles(3)',
            'output_format': 'PNG',
            'path_root': '/home/dariusz/Documents/tasks/resources',
            'scene_file': '/golem/resources/scene-Helicopter-27-internal.blend',
            'script_src': '# This template is rendered by\n# apps.blender.resources.scenefileeditor.generate_blender_crop_file(),\n# written to tempfile and passed as arg to blender.\nimport bpy\n\nclass EngineWarning(bpy.types.Operator):\n    bl_idname = "wm.engine_warning"\n    bl_label = "Inform about not supported rendering engine"\n\n    def execute(self, context):\n        self.report({"ERROR"}, "Engine " + bpy.context.scene.render.engine + \\\n                               " not supported by Golem")\n        return {"FINISHED"}\n\nclass ShowInformation(bpy.types.Operator):\n    bl_idname = "wm.scene_information"\n    bl_label = "Inform user about scene settings"\n\n\n    def execute(self, context):\n        self.report({"INFO"}, "Resolution: " +\n                              str(bpy.context.scene.render.resolution_x) +\n                               " x " +\n                               str(bpy.context.scene.render.resolution_y))\n        self.report({"INFO"}, "File format: " +\n                               str(bpy.context.scene.render.file_extension))\n        self.report({"INFO"}, "Filepath: " +\n                              str(bpy.context.scene.render.filepath))\n        self.report({"INFO"}, "Frames: " +\n                              str(bpy.context.scene.frame_start) + "-" +\n                              str(bpy.context.scene.frame_end) + ";" +\n                              str(bpy.context.scene.frame_step))\n\n        return {"FINISHED"}\n\n\nbpy.utils.register_class(EngineWarning)\nengine = bpy.context.scene.render.engine\nif engine not in ("BLENDER_RENDER", "CYCLES"):\n    bpy.ops.wm.engine_warning()\n\nbpy.utils.register_class(ShowInformation)\nbpy.ops.wm.scene_information()\n\n\nfor scene in bpy.data.scenes:\n\n    scene.render.tile_x = 0\n    scene.render.tile_y = 0\n    scene.render.resolution_x = 1000\n    scene.render.resolution_y = 600\n    scene.render.resolution_percentage = 100\n    scene.render.use_border = True\n    scene.render.use_crop_to_border = True\n    scene.render.border_max_x = 1.0\n    scene.render.border_min_x = 0.0\n    scene.render.border_min_y = 0.25\n    scene.render.border_max_y = 0.375\n    scene.render.use_compositing = bool(False)\n\n#and check if additional files aren\'t missing\nbpy.ops.file.report_missing_files()\n',
            'start_task': 6,
            'total_tasks': 8
        },
        'performance': 829.7531773625524,
        'short_description': 'path_root: /home/dariusz/Documents/tasks/resources, start_task: 6, end_task: 6, total_tasks: 8, outfilebasename: Heli-cycles(3), scene_file: /golem/resources/scene-Helicopter-27-internal.blend',
        'src_code': 'from __future__ import print_function\n\nimport os\nimport subprocess\nimport sys\nfrom multiprocessing import cpu_count\n\nimport params  # This module is generated before this script is run\n\nBLENDER_COMMAND = "blender"\nWORK_DIR = "/golem/work"\nOUTPUT_DIR = "/golem/output"\n\n\ndef exec_cmd(cmd):\n    pc = subprocess.Popen(cmd)\n    return pc.wait()\n\n\ndef format_blender_render_cmd(outfilebasename, scene_file, script_file,\n                              start_task, frame, output_format):\n    cmd = [\n        "{}".format(BLENDER_COMMAND),\n        "-b", "{}".format(scene_file),\n        "-y",  # enable scripting by default\n        "-P", "{}".format(script_file),\n        "-o", "{}/{}_{}".format(OUTPUT_DIR, outfilebasename, start_task),\n        "-noaudio",\n        "-F", "{}".format(output_format.upper()),\n        "-t", "{}".format(cpu_count()),\n        "-f", "{}".format(frame)\n    ]\n    return cmd\n\n\ndef run_blender_task(outfilebasename, scene_file, script_src, start_task,\n                     frames, output_format):\n    scene_file = os.path.normpath(scene_file)\n    if not os.path.exists(scene_file):\n        print("Scene file \'{}\' does not exist".format(scene_file),\n              file=sys.stderr)\n        sys.exit(1)\n\n    blender_script_path = WORK_DIR + "/blenderscript.py"\n    with open(blender_script_path, "w") as script_file:\n        script_file.write(script_src)\n\n    for frame in frames:\n        cmd = format_blender_render_cmd(outfilebasename, scene_file,\n                                        script_file.name, start_task, frame, output_format)\n        print(cmd, file=sys.stderr)\n        exit_code = exec_cmd(cmd)\n        if exit_code is not 0:\n            sys.exit(exit_code)\n\n\nrun_blender_task(params.outfilebasename, params.scene_file, params.script_src, params.start_task, params.frames,\n                 params.output_format)\n\n',
        'subtask_id': 'ab86bc80-3346-11e8-aef1-3fa1fe13c163',
        'task_id': 'b2c20fe6-333c-11e8-8056-3fa1fe13c163',
        'working_directory': '.'
    }],
    ['package_hash', 'sha1:230fb0cad8c7ed29810a2183f0ec1d39c9df3f4a'],
    ['concent_enabled', False]
    ]
)

@rwrzesien I have edited your issue to make this message looks more humanly :) //kbeker

kbeker commented 6 years ago

Fer questions at the beginning:

  1. I see requestor_public_key and provider_public_key are normal strings, not bytes, right? And they will always be strings?
  2. requestor_id, requestor_public_key, requestor_ethereum_public_key are completely the same. (same situation with provider) Are you sure that it is OK? As far as I know requestor_ethereum_public_key and requestor_public_key shouldn't be the same? Can someone confirm it?
cameel commented 6 years ago

Here's a cleaner version:

TaskToCompute(
    header = MessageHeader(
        type_     = 2002,
        timestamp = 1522323925,
        encrypted = False,
    ),
    sig    = b'T\xbb;\x8e`\xfb6\x84\xeaO)\xa4\x1b\x84\x87@\xa8\xe8\xdeZWO \xea\xd5oQ\xce\x1d\x8f\x15;L\xc0\xd0.t\xe9\xddz\xe9\x90\xf6\xd7\x8f\xec@\x1f\xcb\xbf\xf0\x94\xcb\xc0\x96\xa9\xfe\x9b\xf0\xe5\xe7f\xdd\x9c\x00',
    slots  = [
        ['requestor_id',                  '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['requestor_public_key',          '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['requestor_ethereum_public_key', '3fa1fe13c1631e157a692317939ce0fe82b39df65336ec1ff17f0e7376aa374858c980cf54261caa5af9dc5939abd7e3455cdaf0888f05e3f4ae964ef8a217ab'],
        ['provider_id',                   'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['provider_public_key',           'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['provider_ethereum_public_key',  'b7da70a8bbb439e8f1cdf7494ce163294a884ccaf9f117023e3cad5e5d4cb60655aec697c859fdeb9880ef94e3705ee0a2dd44eba7d68cabcd90e7972b48d00f'],
        ['package_hash',                  'sha1:230fb0cad8c7ed29810a2183f0ec1d39c9df3f4a'],
        ['concent_enabled',               False],
        ['compute_task_def',              {
            'deadline':      1522355642.491945,
            'docker_images': [
                {
                    'image_id':    None,
                    'repository': 'golemfactory/blender',
                    'tag':        '1.4',
                }
            ],
            'extra_data': {
                'start_task':      6,
                'end_task':        6,
                'total_tasks':     8,
                'frames':          [1],
                'outfilebasename': 'Heli-cycles(3)',
                'output_format':   'PNG',
                'path_root':       '/home/dariusz/Documents/tasks/resources',
                'scene_file':      '/golem/resources/scene-Helicopter-27-internal.blend',
                'script_src':      '# This template is rendered by\n# apps.blender.resources.scenefileeditor.generate_blender_crop_file(),\n# written to tempfile and passed as arg to blender.\nimport bpy\n\nclass EngineWarning(bpy.types.Operator):\n    bl_idname = "wm.engine_warning"\n    bl_label = "Inform about not supported rendering engine"\n\n    def execute(self, context):\n        self.report({"ERROR"}, "Engine " + bpy.context.scene.render.engine + \\\n                               " not supported by Golem")\n        return {"FINISHED"}\n\nclass ShowInformation(bpy.types.Operator):\n    bl_idname = "wm.scene_information"\n    bl_label = "Inform user about scene settings"\n\n\n    def execute(self, context):\n        self.report({"INFO"}, "Resolution: " +\n                              str(bpy.context.scene.render.resolution_x) +\n                               " x " +\n                               str(bpy.context.scene.render.resolution_y))\n        self.report({"INFO"}, "File format: " +\n                               str(bpy.context.scene.render.file_extension))\n        self.report({"INFO"}, "Filepath: " +\n                              str(bpy.context.scene.render.filepath))\n        self.report({"INFO"}, "Frames: " +\n                              str(bpy.context.scene.frame_start) + "-" +\n                              str(bpy.context.scene.frame_end) + ";" +\n                              str(bpy.context.scene.frame_step))\n\n        return {"FINISHED"}\n\n\nbpy.utils.register_class(EngineWarning)\nengine = bpy.context.scene.render.engine\nif engine not in ("BLENDER_RENDER", "CYCLES"):\n    bpy.ops.wm.engine_warning()\n\nbpy.utils.register_class(ShowInformation)\nbpy.ops.wm.scene_information()\n\n\nfor scene in bpy.data.scenes:\n\n    scene.render.tile_x = 0\n    scene.render.tile_y = 0\n    scene.render.resolution_x = 1000\n    scene.render.resolution_y = 600\n    scene.render.resolution_percentage = 100\n    scene.render.use_border = True\n    scene.render.use_crop_to_border = True\n    scene.render.border_max_x = 1.0\n    scene.render.border_min_x = 0.0\n    scene.render.border_min_y = 0.25\n    scene.render.border_max_y = 0.375\n    scene.render.use_compositing = bool(False)\n\n#and check if additional files aren\'t missing\nbpy.ops.file.report_missing_files()\n',
            },
            'task_id':           'b2c20fe6-333c-11e8-8056-3fa1fe13c163',
            'subtask_id':        'ab86bc80-3346-11e8-aef1-3fa1fe13c163',
            'performance':       829.7531773625524,
            'short_description': 'path_root: /home/dariusz/Documents/tasks/resources, start_task: 6, end_task: 6, total_tasks: 8, outfilebasename: Heli-cycles(3), scene_file: /golem/resources/scene-Helicopter-27-internal.blend',
            'src_code':          'from __future__ import print_function\n\nimport os\nimport subprocess\nimport sys\nfrom multiprocessing import cpu_count\n\nimport params  # This module is generated before this script is run\n\nBLENDER_COMMAND = "blender"\nWORK_DIR = "/golem/work"\nOUTPUT_DIR = "/golem/output"\n\n\ndef exec_cmd(cmd):\n    pc = subprocess.Popen(cmd)\n    return pc.wait()\n\n\ndef format_blender_render_cmd(outfilebasename, scene_file, script_file,\n                              start_task, frame, output_format):\n    cmd = [\n        "{}".format(BLENDER_COMMAND),\n        "-b", "{}".format(scene_file),\n        "-y",  # enable scripting by default\n        "-P", "{}".format(script_file),\n        "-o", "{}/{}_{}".format(OUTPUT_DIR, outfilebasename, start_task),\n        "-noaudio",\n        "-F", "{}".format(output_format.upper()),\n        "-t", "{}".format(cpu_count()),\n        "-f", "{}".format(frame)\n    ]\n    return cmd\n\n\ndef run_blender_task(outfilebasename, scene_file, script_src, start_task,\n                     frames, output_format):\n    scene_file = os.path.normpath(scene_file)\n    if not os.path.exists(scene_file):\n        print("Scene file \'{}\' does not exist".format(scene_file),\n              file=sys.stderr)\n        sys.exit(1)\n\n    blender_script_path = WORK_DIR + "/blenderscript.py"\n    with open(blender_script_path, "w") as script_file:\n        script_file.write(script_src)\n\n    for frame in frames:\n        cmd = format_blender_render_cmd(outfilebasename, scene_file,\n                                        script_file.name, start_task, frame, output_format)\n        print(cmd, file=sys.stderr)\n        exit_code = exec_cmd(cmd)\n        if exit_code is not 0:\n            sys.exit(exit_code)\n\n\nrun_blender_task(params.outfilebasename, params.scene_file, params.script_src, params.start_task, params.frames,\n                 params.output_format)\n\n',
            'working_directory': '.',
        }],
    ]
)

Content of script_src field printed with line breaks:

# This template is rendered by
# apps.blender.resources.scenefileeditor.generate_blender_crop_file(),
# written to tempfile and passed as arg to blender.
import bpy

class EngineWarning(bpy.types.Operator):
    bl_idname = "wm.engine_warning"
    bl_label = "Inform about not supported rendering engine"

    def execute(self, context):
        self.report({"ERROR"}, "Engine " + bpy.context.scene.render.engine + \
                               " not supported by Golem")
        return {"FINISHED"}

class ShowInformation(bpy.types.Operator):
    bl_idname = "wm.scene_information"
    bl_label = "Inform user about scene settings"

    def execute(self, context):
        self.report({"INFO"}, "Resolution: " +
                              str(bpy.context.scene.render.resolution_x) +
                               " x " +
                               str(bpy.context.scene.render.resolution_y))
        self.report({"INFO"}, "File format: " +
                               str(bpy.context.scene.render.file_extension))
        self.report({"INFO"}, "Filepath: " +
                              str(bpy.context.scene.render.filepath))
        self.report({"INFO"}, "Frames: " +
                              str(bpy.context.scene.frame_start) + "-" +
                              str(bpy.context.scene.frame_end) + ";" +
                              str(bpy.context.scene.frame_step))

        return {"FINISHED"}

bpy.utils.register_class(EngineWarning)
engine = bpy.context.scene.render.engine
if engine not in ("BLENDER_RENDER", "CYCLES"):
    bpy.ops.wm.engine_warning()

bpy.utils.register_class(ShowInformation)
bpy.ops.wm.scene_information()

for scene in bpy.data.scenes:

    scene.render.tile_x = 0
    scene.render.tile_y = 0
    scene.render.resolution_x = 1000
    scene.render.resolution_y = 600
    scene.render.resolution_percentage = 100
    scene.render.use_border = True
    scene.render.use_crop_to_border = True
    scene.render.border_max_x = 1.0
    scene.render.border_min_x = 0.0
    scene.render.border_min_y = 0.25
    scene.render.border_max_y = 0.375
    scene.render.use_compositing = bool(False)

#and check if additional files aren't missing
bpy.ops.file.report_missing_files()

And the content of src_code printed with line breaks:

from __future__ import print_function

import os
import subprocess
import sys
from multiprocessing import cpu_count

import params  # This module is generated before this script is run

BLENDER_COMMAND = "blender"
WORK_DIR = "/golem/work"
OUTPUT_DIR = "/golem/output"

def exec_cmd(cmd):
    pc = subprocess.Popen(cmd)
    return pc.wait()

def format_blender_render_cmd(outfilebasename, scene_file, script_file,
                              start_task, frame, output_format):
    cmd = [
        "{}".format(BLENDER_COMMAND),
        "-b", "{}".format(scene_file),
        "-y",  # enable scripting by default
        "-P", "{}".format(script_file),
        "-o", "{}/{}_{}".format(OUTPUT_DIR, outfilebasename, start_task),
        "-noaudio",
        "-F", "{}".format(output_format.upper()),
        "-t", "{}".format(cpu_count()),
        "-f", "{}".format(frame)
    ]
    return cmd

def run_blender_task(outfilebasename, scene_file, script_src, start_task,
                     frames, output_format):
    scene_file = os.path.normpath(scene_file)
    if not os.path.exists(scene_file):
        print("Scene file '{}' does not exist".format(scene_file),
              file=sys.stderr)
        sys.exit(1)

    blender_script_path = WORK_DIR + "/blenderscript.py"
    with open(blender_script_path, "w") as script_file:
        script_file.write(script_src)

    for frame in frames:
        cmd = format_blender_render_cmd(outfilebasename, scene_file,
                                        script_file.name, start_task, frame, output_format)
        print(cmd, file=sys.stderr)
        exit_code = exec_cmd(cmd)
        if exit_code is not 0:
            sys.exit(exit_code)

run_blender_task(params.outfilebasename, params.scene_file, params.script_src, params.start_task, params.frames,
                 params.output_format)