test livedata SAXS & WAXS data reduction from Bluesky

#S 98  Flyscan(pos_X=152.0, pos_Y=42.0, thickness=1.1, scan_title=Peter_TEB)
#D Wed Jun 19 15:22:36 2019
#C Wed Jun 19 15:22:36 2019.  plan_type = generator
#C Wed Jun 19 15:22:36 2019.  uid = 77bdfa1a-1d2c-48cd-8359-b1dea5b7dcf5
#C Wed Jun 19 15:22:37 2019.  start USAXS Fly scan
#C Wed Jun 19 15:22:37 2019.  HDF5 configuration file: /share1/AreaDetectorConfig/FlyScan_config/saveFlyData.xml
#MD uid = 77bdfa1a-1d2c-48cd-8359-b1dea5b7dcf5
#MD EPICS_CA_MAX_ARRAY_BYTES = 1280000
#MD EPICS_HOST_ARCH = linux-x86_64
#MD action = USAXSscan
#MD archive = OrderedDict([('text', 'Command file: usaxs.mac\n====== ======================= =====================================\nline # action                  parameters                           \n====== ======================= =====================================\n2      CURRENT_EXPERIMENT_NAME Peter                                \n22     USAXSscan               152.000, 62.500, 0, Peter_EMPTY      \n24     USAXSscan               152.000, 52.400, 1.1, Peter_DI       \n26     USAXSscan               152.000, -7.300, 1.1, Peter_PCEIDTSDS\n28     USAXSscan               152.000, 2.800, 1.1, Peter_PCEIDTTEB \n30     USAXSscan               152.000, 12.200, 1.1, Peter_PCETEB   \n32     USAXSscan               152.000, 22.000, 1.1, Peter_IDTTEB   \n34     USAXSscan               152.000, 32.400, 1.1, Peter_SDS      \n36     USAXSscan               152.000, 42.000, 1.1, Peter_TEB      \n====== ======================= =====================================\n'), ('source', '/home/beams11/USAXS/.ipython/profile_bluesky/startup/50-plans.py'), ('is_file', True), ('line', 423), ('caller', 'execute_command_list'), ('caller_code', '    print(text)\n    archive = instrument_archive(text)\n'), ('source_contents', ['print(__file__)\n', '\n', '"""\n', 'Bluesky plans (scans)\n', '\n', 'PLANS\n', '\n', '    after_plan()\n', '    before_plan()\n', '    after_command_list()\n', '    before_command_list()\n', '    execute_command_list()\n', '    FlyScan()\n', '    preUSAXStune()\n', '    run_command_file()\n', '    SAXS()\n', '    WAXS()\n', '    uascan()\n', '\n', 'UTILITIES\n', '\n', '    command_list_as_table()\n', '    get_command_list()\n', '    makeOrderedDictFromTwoLists()\n', '    parse_Excel_command_file()\n', '    parse_text_command_file()\n', '    postCommandsListfile2WWW()\n', '    split_quoted_line()\n', '    summarize_command_file()\n', '\n', '"""\n', '\n', '\n', 'def makeOrderedDictFromTwoLists(labels, values):\n', '    """return an OrderedDict"""\n', '    if len(values) > len(labels):\n', '        msg = "Too many values for known labels."\n', '        msg += f"  labels={labels}"\n', '        msg += f"  values={values}"\n', '        raise ValueError(msg)\n', '    # only the first len(values) labels will be used!\n', '    return OrderedDict(zip(labels, values))\n', '\n', '\n', 'def postCommandsListfile2WWW(commands):\n', '    """\n', '    post list of commands to WWW and archive the list for posterity\n', '    """\n', '    tbl_file = "commands.txt"\n', '    tbl = command_list_as_table(commands)\n', '    timestamp = datetime.datetime.now().isoformat().replace("T", " ")\n', '    file_contents = "bluesky command sequence\\n"\n', '    file_contents += f"written: {timestamp}\\n"\n', '    file_contents += str(tbl.reST())\n', '\n', '    # post for livedata page\n', '    # path = "/tmp"\n', '    path = "/share1/local_livedata"\n', '    with open(os.path.join(path, tbl_file), "w") as fp:\n', '        fp.write(file_contents)\n', '\n', '    # post to EPICS\n', '    yield from bp.mv(\n', '        user_data.macro_file, os.path.split(tbl_file)[-1],\n', '        user_data.macro_file_time, timestamp,\n', '        )\n', '\n', '    # keep this list for posterity\n', '    timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")\n', '    path = "/share1/log/macros"\n', '    posterity_file = f"{timestamp}-{tbl_file}"\n', '    with open(os.path.join(path, posterity_file), "w") as fp:\n', '        fp.write(file_contents)\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def before_command_list(md={}, commands=None):\n', '    """\n', '    things to be done before a command list is run\n', '    """\n', '    yield from bps.mv(\n', '        user_data.time_stamp, str(datetime.datetime.now()),\n', '        user_data.state, "Starting data collection",\n', '        user_data.collection_in_progress, 1,\n', '        ti_filter_shutter, "close",\n', '    )\n', '    # epics_put ("9idcLAX:collectingSAXS", 0)\n', '    # epics_put ("9idcLAX:collectingWAXS", 0)\n', '    if constants["MEASURE_DARK_CURRENTS"]:\n', '        yield from measure_background(\n', '            [upd_controls, I0_controls, I00_controls, trd_controls],\n', '        )\n', '\n', '    # reset the ranges to be used when tuning optical axes (issue #129)\n', '    # These routines are defined in file: 29-axis-tuning.py\n', '    yield from instrument_default_tune_ranges()\n', '    yield from user_defined_settings()\n', '    yield from update_EPICS_tuning_widths()\n', '\n', '    yield from beforeScanComputeOtherStuff()        # 41-commands.py\n', '\n', '    if terms.preUSAXStune.run_tune_on_qdo.value:\n', '        logger.info("Runing preUSAXStune as requested at start of measurements")\n', '        yield from preUSAXStune(md=md)\n', '\n', '    if constants["SYNC_ORDER_NUMBERS"]:\n', '        order_number = max([\n', '            terms.FlyScan.order_number.value,\n', '            # saxs_det.cam.file_number.value,\n', '            # waxs_det.cam.file_number.value,\n', '        ])\n', '\n', '    try:\n', '        yield from bps.mv(saxs_det.hdf1.file_number, order_number)\n', '    except NameError:\n', '        pass\n', '    try:\n', '        yield from bps.mv(waxs_det.hdf1.file_number, order_number)\n', '    except NameError:\n', '        pass\n', '    yield from bps.mv(terms.FlyScan.order_number, order_number)\n', '\n', '    if commands is not None:\n', '        postCommandsListfile2WWW(commands)\n', '\n', '\n', 'def after_command_list(md={}):\n', '    """\n', '    things to be done after a command list is run\n', '    """\n', '    yield from bps.mv(\n', '        user_data.time_stamp, str(datetime.datetime.now()),\n', '        user_data.state, "Ended data collection",\n', '        user_data.collection_in_progress, 0,\n', '        ti_filter_shutter, "close",\n', '    )\n', '\n', '\n', 'def before_plan(md={}):\n', '    """\n', '    things to be done before every data collection plan\n', '    """\n', '    if terms.preUSAXStune.needed:\n', '        # tune at previous sample position\n', "        # don't overexpose the new sample position\n", '        yield from preUSAXStune(md=md)\n', '\n', '\n', '\n', 'def after_plan(weight=1, md={}):\n', '    """\n', '    things to be done after every data collection plan\n', '    """\n', '    yield from bps.mv(      # increment it\n', '        terms.preUSAXStune.num_scans_last_tune,\n', '        terms.preUSAXStune.num_scans_last_tune.value + weight\n', '        )\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def split_quoted_line(line):\n', '    """\n', '    splits a line into words some of which might be quoted\n', '\n', '    TESTS::\n', '\n', '        FlyScan 0   0   0   blank\n', '        FlyScan 5   2   0   "empty container"\n', '        FlyScan 5   12   0   "even longer name"\n', '        SAXS 0 0 0 blank\n', '        SAXS 0 0 0 "blank"\n', '\n', '    RESULTS::\n', '\n', "        ['FlyScan', '0', '0', '0', 'blank']\n", "        ['FlyScan', '5', '2', '0', 'empty container']\n", "        ['FlyScan', '5', '12', '0', 'even longer name']\n", "        ['SAXS', '0', '0', '0', 'blank']\n", "        ['SAXS', '0', '0', '0', 'blank']\n", '\n', '    """\n', '    parts = []\n', '\n', '    # look for open and close quoted parts and combine them\n', '    quoted = False\n', '    multi = None\n', '    for p in line.split():\n', '        if not quoted and p.startswith(\'"\'):   # begin quoted text\n', '            quoted = True\n', '            multi = ""\n', '\n', '        if quoted:\n', '            if len(multi) > 0:\n', '                multi += " "\n', '            multi += p\n', '            if p.endswith(\'"\'):     # end quoted text\n', '                quoted = False\n', '\n', '        if not quoted:\n', '            if multi is not None:\n', '                parts.append(multi[1:-1])   # remove enclosing quotes\n', '                multi = None\n', '            else:\n', '                parts.append(p)\n', '\n', '    return parts\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def parse_Excel_command_file(filename):\n', '    """\n', '    parse an Excel spreadsheet with commands, return as command list\n', '\n', '    TEXT view of spreadsheet (Excel file line numbers shown)::\n', '\n', '        [1] List of sample scans to be run\n', '        [2]\n', '        [3]\n', '        [4] scan    sx  sy  thickness   sample name\n', '        [5] FlyScan 0   0   0   blank\n', '        [6] FlyScan 5   2   0   blank\n', '\n', '    PARAMETERS\n', '\n', '    filename : str\n', '        Name of input Excel spreadsheet file.  Can be relative or absolute path,\n', '        such as "actions.xslx", "../sample.xslx", or\n', '        "/path/to/overnight.xslx".\n', '\n', '    RETURNS\n', '\n', '    list of commands : list[command]\n', '        List of command tuples for use in ``execute_command_list()``\n', '\n', '    RAISES\n', '\n', '    FileNotFoundError\n', '        if file cannot be found\n', '\n', '    """\n', '    full_filename = os.path.abspath(filename)\n', '    assert os.path.exists(full_filename)\n', '    xl = APS_utils.ExcelDatabaseFileGeneric(full_filename)\n', '\n', '    commands = []\n', '\n', '    if len(xl.db) > 0:\n', '        for i, row in enumerate(xl.db.values()):\n', '            action, *values = list(row.values())\n', '\n', '            # trim off any None values from end\n', '            while len(values) > 0:\n', '                if values[-1] is not None:\n', '                    break\n', '                values = values[:-1]\n', '\n', '            commands.append((action, values, i+1, list(row.values())))\n', '\n', '    return commands\n', '\n', '\n', 'def parse_text_command_file(filename):\n', '    """\n', '    parse a text file with commands, return as command list\n', '\n', '    * The text file is interpreted line-by-line.\n', '    * Blank lines are ignored.\n', '    * A pound sign (#) marks the rest of that line as a comment.\n', '    * All remaining lines are interpreted as commands with arguments.\n', '\n', '    Example of text file (no line numbers shown)::\n', '\n', '        #List of sample scans to be run\n', '        # pound sign starts a comment (through end of line)\n', '\n', '        # action  value\n', '        mono_shutter open\n', '\n', '        # action  x y width height\n', '        uslits 0 0 0.4 1.2\n', '\n', '        # action  sx  sy  thickness   sample name\n', '        FlyScan 0   0   0   blank\n', '        FlyScan 5   2   0   "empty container"\n', '\n', '        # action  sx  sy  thickness   sample name\n', '        SAXS 0 0 0 blank\n', '\n', '        # action  value\n', '        mono_shutter close\n', '\n', '    PARAMETERS\n', '\n', '    filename : str\n', '        Name of input text file.  Can be relative or absolute path,\n', '        such as "actions.txt", "../sample.txt", or\n', '        "/path/to/overnight.txt".\n', '\n', '    RETURNS\n', '\n', '    list of commands : list[command]\n', '        List of command tuples for use in ``execute_command_list()``\n', '\n', '    RAISES\n', '\n', '    FileNotFoundError\n', '        if file cannot be found\n', '    """\n', '    full_filename = os.path.abspath(filename)\n', '    assert os.path.exists(full_filename)\n', '    with open(full_filename, "r") as fp:\n', '        buf = fp.readlines()\n', '\n', '    commands = []\n', '    for i, raw_command in enumerate(buf):\n', '        row = raw_command.strip()\n', '        if row == "" or row.startswith("#"):\n', '            continue                    # comment or blank\n', '\n', '        else:                           # command line\n', '            action, *values = split_quoted_line(row)\n', '            commands.append((action, values, i+1, raw_command.rstrip()))\n', '\n', '    return commands\n', '\n', '\n', 'def command_list_as_table(commands):\n', '    """\n', '    format a command list as a pyRestTable.Table object\n', '    """\n', '    tbl = pyRestTable.Table()\n', '    tbl.addLabel("line #")\n', '    tbl.addLabel("action")\n', '    tbl.addLabel("parameters")\n', '    for command in commands:\n', '        action, args, line_number, raw_command = command\n', '        row = [line_number, action, ", ".join(map(str, args))]\n', '        tbl.addRow(row)\n', '    return tbl\n', '\n', '\n', 'def get_command_list(filename):\n', '    """\n', '    return command list from either text or Excel file\n', '    """\n', '    full_filename = os.path.abspath(filename)\n', '    assert os.path.exists(full_filename)\n', '    try:\n', '        commands = parse_Excel_command_file(filename)\n', '    except Exception:          # TODO: XLRDError\n', '        commands = parse_text_command_file(filename)\n', '    return commands\n', '\n', '\n', 'def summarize_command_file(filename):\n', '    """\n', '    print the command list from a text or Excel file\n', '    """\n', '    commands = get_command_list(filename)\n', '    print(f"Command file: {filename}")\n', '    print(command_list_as_table(commands))\n', '\n', '\n', 'def run_command_file(filename, md={}):\n', '    """\n', '    plan: execute a list of commands from a text or Excel file\n', '\n', '    * Parse the file into a command list\n', '    * yield the command list to the RunEngine (or other)\n', '    """\n', '    commands = get_command_list(filename)\n', '    yield from execute_command_list(filename, commands)\n', '\n', '\n', 'def execute_command_list(filename, commands, md={}):\n', '    """\n', '    plan: execute the command list\n', '\n', '    The command list is a tuple described below.\n', '\n', '    * Only recognized commands will be executed.\n', '    * Unrecognized commands will be reported as comments.\n', '\n', '    PARAMETERS\n', '\n', '    filename : str\n', '        Name of input text file.  Can be relative or absolute path,\n', '        such as "actions.txt", "../sample.txt", or\n', '        "/path/to/overnight.txt".\n', '    commands : list[command]\n', '        List of command tuples for use in ``execute_command_list()``\n', '\n', '    where\n', '\n', '    command : tuple\n', '        (action, OrderedDict, line_number, raw_command)\n', '    action: str\n', '        names a known action to be handled\n', '    parameters: list\n', '        List of parameters for the action.\n', '        The list is empty of there are no values\n', '    line_number: int\n', '        line number (1-based) from the input text file\n', '    raw_command: obj (str or list(str)\n', '        contents from input file, such as:\n', '        ``SAXS 0 0 0 blank``\n', '    """\n', '    full_filename = os.path.abspath(filename)\n', '\n', '    if len(commands) == 0:\n', '        yield from bps.null()\n', '        return\n', '\n', '    text = f"Command file: {filename}\\n"\n', '    text += str(command_list_as_table(commands))\n', '    print(text)\n', '    archive = instrument_archive(text)\n', '\n', '    yield from before_command_list(md=md, commands=commands)\n', '    for command in commands:\n', '        action, args, i, raw_command = command\n', '        print(f"file line {i}: {raw_command}")\n', '\n', '        _md = {}\n', '        _md["full_filename"] = full_filename\n', '        _md["filename"] = filename\n', '        _md["line_number"] = i\n', '        _md["action"] = action\n', '        _md["parameters"] = args    # args is shorter than parameters, means the same thing here\n', '        _md["archive"] = archive\n', '\n', '        _md.update(md or {})      # overlay with user-supplied metadata\n', '\n', '        action = action.lower()\n', '        if action == "preusaxstune":\n', '            yield from preUSAXStune(md=_md)\n', '\n', '        elif action in ("flyscan", "usaxsscan"):\n', '            # TODO: watch for PV which indicates, if USAXS should run step scan or flyscan\n', '            sx = float(args[0])\n', '            sy = float(args[1])\n', '            sth = float(args[2])\n', '            snm = args[3]\n', '            yield from Flyscan(sx, sy, sth, snm, md=_md)\n', '\n', '        elif action in ("saxs", "saxsexp"):\n', '            sx = float(args[0])\n', '            sy = float(args[1])\n', '            sth = float(args[2])\n', '            snm = args[3]\n', '            yield from SAXS(sx, sy, sth, snm, md=_md)\n', '\n', '        elif action in ("waxs", "waxsexp"):\n', '            sx = float(args[0])\n', '            sy = float(args[1])\n', '            sth = float(args[2])\n', '            snm = args[3]\n', '            yield from WAXS(sx, sy, sth, snm, md=_md)\n', '\n', '        else:\n', '            print(f"no handling for line {i}: {raw_command}")\n', '\n', '    yield from after_command_list(md=md)\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def uascan():\n', '    """\n', '    USAXS step scan\n', '\n', '    https://github.com/APS-USAXS/ipython-usaxs/issues/8\n', '    """\n', "    # TODO: needs proper args & kwargs matching SPEC's signature\n", '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def preUSAXStune(md={}):\n', '    """\n', '    tune the USAXS optics *only* if in USAXS mode\n', '\n', '    USAGE:  ``RE(preUSAXStune())``\n', '    """\n', '    yield from bps.mv(\n', '        monochromator.feedback.on, MONO_FEEDBACK_ON,\n', '        mono_shutter, "open",\n', '        ccd_shutter, "close",\n', '    )\n', '    yield from IfRequestedStopBeforeNextScan()         # stop if user chose to do so.\n', '\n', '    yield from mode_USAXS()\n', '\n', '    if terms.preUSAXStune.use_specific_location.value in (1, "yes"):\n', '        yield from bps.mv(\n', '            s_stage.x, terms.preUSAXStune.sx.value,\n', '            s_stage.y, terms.preUSAXStune.sy.value,\n', '            )\n', '\n', '    yield from bps.mv(\n', '        # ensure diode in place (Radiography puts it elsewhere)\n', '        d_stage.x, terms.USAXS.diode.dx.value,\n', '        d_stage.y, terms.USAXS.diode.dy.value,\n', '\n', '        user_data.time_stamp, str(datetime.datetime.now()),\n', '        user_data.state, "pre-USAXS optics tune",\n', '\n', '        # Is this covered by user_mode, "USAXS"?\n', '        usaxs_slit.v_size,  terms.SAXS.usaxs_v_size.value,\n', '        usaxs_slit.h_size,  terms.SAXS.usaxs_h_size.value,\n', '        guard_slit.v_size,  terms.SAXS.usaxs_guard_v_size.value,\n', '        guard_slit.h_size,  terms.SAXS.usaxs_guard_h_size.value,\n', '\n', '        scaler0.preset_time,  0.1,\n', '    )\n', '    # when all that is complete, then ...\n', '    yield from bps.mv(ti_filter_shutter, "open")\n', '\n', '    # TODO: install suspender using usaxs_CheckBeamStandard.value\n', '\n', '    tuners = OrderedDict()                 # list the axes to tune\n', '    tuners[m_stage.r] = tune_mr            # tune M stage to monochromator\n', '    tuners[m_stage.r2p] = tune_m2rp        # make M stage crystals parallel\n', '    if terms.USAXS.useMSstage.value:\n', '        tuners[ms_stage.rp] = tune_msrp    # align MSR stage with M stage\n', '    if terms.USAXS.useSBUSAXS.value:\n', '        tuners[as_stage.rp] = tune_asrp    # align ASR stage with MSR stage and set ASRP0 value\n', '    tuners[a_stage.r] = tune_ar            # tune A stage to M stage\n', '    tuners[a_stage.r2p] = tune_a2rp        # make A stage crystals parallel\n', '\n', '    # now, tune the desired axes, bail out if a tune fails\n', '    yield from bps.install_suspender(suspend_BeamInHutch)\n', '    for axis, tune in tuners.items():\n', '        yield from bps.mv(ti_filter_shutter, "open")\n', '        yield from tune(md=md)\n', '        if axis.tuner.tune_ok:\n', "            # If we don't wait, the next tune often fails\n", '            # intensity stays flat, statistically\n', '            # We need to wait a short bit to allow EPICS database\n', '            # to complete processing and report back to us.\n', '            yield from bps.sleep(1)\n', '        else:\n', '            print("!!! tune failed for axis {axis.name} !!!")\n', '            break\n', '    yield from bps.remove_suspender(suspend_BeamInHutch)\n', '\n', '    print("USAXS count time: {terms.USAXS.usaxs_time.value} second(s)")\n', '    yield from bps.mv(\n', '        scaler0.preset_time,        terms.USAXS.usaxs_time.value,\n', '        user_data.time_stamp,       str(datetime.datetime.now()),\n', '        user_data.state,            "pre-USAXS optics tuning done",\n', '\n', '        terms.preUSAXStune.num_scans_last_tune, 0,\n', '        terms.preUSAXStune.run_tune_next,       0,\n', '        terms.preUSAXStune.epoch_last_tune,     time.time(),\n', '    )\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def Flyscan(pos_X, pos_Y, thickness, scan_title, md={}):\n', '    """\n', '    do one USAXS Fly Scan\n', '    """\n', '    bluesky_runengine_running = RE.state != "idle"\n', '\n', '    yield from IfRequestedStopBeforeNextScan()\n', '\n', '    yield from mode_USAXS()\n', '\n', '    yield from bps.mv(\n', '        usaxs_slit.v_size, terms.SAXS.usaxs_v_size.value,\n', '        usaxs_slit.h_size, terms.SAXS.usaxs_h_size.value,\n', '        guard_slit.v_size, terms.SAXS.usaxs_guard_v_size.value,\n', '        guard_slit.h_size, terms.SAXS.usaxs_guard_h_size.value,\n', '    )\n', '    yield from before_plan()\n', '\n', '    yield from bps.mv(\n', '        s_stage.x, pos_X,\n', '        s_stage.y, pos_Y,\n', '    )\n', '\n', '    scan_title_clean = cleanupText(scan_title)\n', '\n', '    # SPEC-compatibility symbols\n', '    SCAN_N = RE.md["scan_id"]+1     # the next scan number (user-controllable)\n', '    # use our specwriter to get a pseudo-SPEC file name\n', '    DATAFILE = os.path.split(specwriter.spec_filename)[-1]\n', '\n', '    # directory is pwd + DATAFILE + "_usaxs"\n', '    flyscan_path = os.path.join(os.getcwd(), os.path.splitext(DATAFILE)[0] + "_usaxs")\n', '    if not os.path.exists(flyscan_path) and bluesky_runengine_running:\n', '        # must create this directory if not exists\n', '        os.mkdir(flyscan_path)\n', '    flyscan_file_name = "%s_%04d.h5" % (scan_title_clean, terms.FlyScan.order_number.value)\n', '\n', '    usaxs_flyscan.saveFlyData_HDF5_dir = flyscan_path\n', '    usaxs_flyscan.saveFlyData_HDF5_file = flyscan_file_name\n', '\n', '    ts = str(datetime.datetime.now())\n', '    yield from bps.mv(\n', '        user_data.sample_title, scan_title,\n', '        user_data.state, "starting USAXS Flyscan",\n', '        user_data.sample_thickness, thickness,\n', '        user_data.user_name, USERNAME,\n', '        user_data.spec_scan, str(SCAN_N),\n', '        # or terms.FlyScan.order_number.value\n', '        user_data.time_stamp, ts,\n', '        user_data.scan_macro, "FlyScan",    # note camel-case\n', '    )\n', '    yield from bps.mv(\n', '        user_data.user_dir, os.getcwd(),\n', '        user_data.spec_file, os.path.split(specwriter.spec_filename)[-1],\n', '    )\n', '\n', '    # offset the calc from exact zero so can plot log(|Q|)\n', '    # q_offset = terms.USAXS.start_offset.value\n', '    # angle_offset = q2angle(q_offset, monochromator.dcm.wavelength.value)\n', '    # ar0_calc_offset = terms.USAXS.ar_val_center.value + angle_offset\n', '\n', '    yield from bps.mv(\n', '        a_stage.r, terms.USAXS.ar_val_center.value,\n', '        # these two were moved by mode_USAXS(), belt & suspenders here\n', '        d_stage.y, terms.USAXS.diode.dy.value,\n', '        a_stage.y, terms.USAXS.AY0.value,\n', '    )\n', '    yield from user_data.set_state_plan("Moving to Q=0")\n', '    yield from bps.mv(\n', '        usaxs_q_calc.channels.B.value, terms.USAXS.ar_val_center.value,\n', '    )\n', '\n', '    # TODO: what to do with USAXSScanUp?\n', '    # 2019-01-25, prj+jil: this is probably not used now, only known to SPEC\n', "    # it's used to cal Finish_in_Angle and START\n", '    # both of which get passed to EPICS\n', '    # That happens outside of this code.  completely.\n', '\n', '    # measure transmission values using pin diode if desired\n', '    usaxs_flyscan.saveFlyData_HDF5_dir = flyscan_path\n', '    usaxs_flyscan.saveFlyData_HDF5_file = flyscan_file_name\n', '    yield from bps.install_suspender(suspend_BeamInHutch)\n', '    yield from measure_USAXS_Transmission(md=md)\n', '\n', '    yield from bps.mv(\n', '        monochromator.feedback.on, MONO_FEEDBACK_OFF,\n', '        )\n', '\n', '    # enable asrp link to ar for 2D USAXS\n', '    if terms.USAXS.is2DUSAXSscan.value:\n', '        RECORD_SCAN_INDEX_10x_per_second = 9\n', '        yield from bps.mv(terms.FlyScan.asrp_calc_SCAN, RECORD_SCAN_INDEX_10x_per_second)\n', '\n', "    # we'll reset these after the scan is done\n", '    old_femto_change_gain_up = upd_controls.auto.gainU.value\n', '    old_femto_change_gain_down = upd_controls.auto.gainD.value\n', '\n', '    yield from bps.mv(\n', '        upd_controls.auto.gainU, terms.FlyScan.setpoint_up.value,\n', '        upd_controls.auto.gainD, terms.FlyScan.setpoint_down.value,\n', '        ti_filter_shutter, "open",\n', '    )\n', '    yield from autoscale_amplifiers([upd_controls, I0_controls, I00_controls])\n', '\n', '\n', '    FlyScanAutoscaleTime = 0.025\n', '    yield from bps.mv(\n', '        scaler0.update_rate, 0,\n', '        scaler0.auto_count_update_rate, 0,\n', '        upd_controls.auto.mode, "auto+background",\n', '        scaler0.preset_time, FlyScanAutoscaleTime,\n', '        scaler0.auto_count_time, FlyScanAutoscaleTime,\n', '        scaler0.auto_count_delay, FlyScanAutoscaleTime,\n', '        scaler0.delay, 0,\n', '        scaler0.count_mode, SCALER_AUTOCOUNT_MODE,\n', '        )\n', '\n', '   # Pause autosave on LAX to prevent delays in PVs processing.\n', '    yield from bps.mv(\n', '        lax_autosave.disable, 1,\n', '        # autosave will restart after this interval (s)\n', '        lax_autosave.max_time, usaxs_flyscan.scan_time.value+9,\n', '        )\n', '\n', '    yield from user_data.set_state_plan("Running Flyscan")\n', '\n', '    ### move the stages to flyscan starting values from EPICS PVs\n', '    yield from bps.mv(\n', '        a_stage.r, flyscan_trajectories.ar.value[0],\n', '        a_stage.y, flyscan_trajectories.ay.value[0],\n', '        d_stage.y, flyscan_trajectories.dy.value[0],\n', '        ar_start, flyscan_trajectories.ar.value[0],\n', '        # ay_start, flyscan_trajectories.ay.value[0],\n', '        # dy_start, flyscan_trajectories.dy.value[0],\n', '    )\n', '\n', '    SCAN_N = RE.md["scan_id"]+1     # update with next number\n', '    yield from bps.mv(\n', '        terms.FlyScan.order_number, terms.FlyScan.order_number.value + 1,  # increment it\n', '        user_data.scanning, "scanning",          # we are scanning now (or will be very soon)\n', '        user_data.spec_scan, str(SCAN_N),\n', '    )\n', '\n', '    _md = {}\n', '    _md.update(md)\n', '    _md[\'plan_name\'] = "Flyscan"\n', "    _md['plan_args'] = dict(\n", '        pos_X = pos_X,\n', '        pos_Y = pos_Y,\n', '        thickness = thickness,\n', '        scan_title = scan_title,\n', '        )\n', "    _md['fly_scan_time'] = usaxs_flyscan.scan_time.value\n", "        #'detectors': [det.name for det in detectors],\n", "        #'num_points': num,\n", "        #'num_intervals': num_intervals,\n", "        #'hints': {}\n", '\n', '    yield from usaxs_flyscan.plan(md=_md)        # DO THE FLY SCAN\n', '\n', '    yield from bps.mv(\n', '        user_data.scanning, "no",          # for sure, we are not scanning now\n', '        terms.FlyScan.elapsed_time, 0,  # show the users there is no more time\n', '    )\n', '    yield from bps.remove_suspender(suspend_BeamInHutch)\n', '\n', '    # Check if we had bad number of PSO pulses\n', '    diff = flyscan_trajectories.num_pulse_positions.value - struck.current_channel.get()\n', '    if diff > 5 and bluesky_runengine_running:\n', '        msg = "WARNING: Flyscan finished with %g less points" % diff\n', '        logger.warning(msg)\n', '        if NOTIFY_ON_BAD_FLY_SCAN:\n', '            subject = "!!! bad number of PSO pulses !!!"\n', '            email_notices.send(subject, msg)\n', '\n', '    yield from user_data.set_state_plan("Flyscan finished")\n', '\n', '    yield from bps.mv(\n', '        lax_autosave.disable, 0,    # enable\n', '        lax_autosave.max_time, 0,   # start right away\n', '\n', '        ti_filter_shutter, "close",\n', '        monochromator.feedback.on, MONO_FEEDBACK_ON,\n', '\n', '        scaler0.update_rate, 5,\n', '        scaler0.auto_count_delay, 0.25,\n', '        scaler0.delay, 0.05,\n', '        scaler0.preset_time, 1,\n', '        scaler0.auto_count_time, 1,\n', '\n', '        upd_controls.auto.gainU, old_femto_change_gain_up,\n', '        upd_controls.auto.gainD, old_femto_change_gain_down,\n', '        )\n', '\n', '    yield from user_data.set_state_plan("Moving USAXS back and saving data")\n', '    yield from bps.mv(\n', '        a_stage.r, terms.USAXS.ar_val_center.value,\n', '        a_stage.y, terms.USAXS.AY0.value,\n', '        d_stage.y, terms.USAXS.DY0.value,\n', '        )\n', '\n', '    # TODO: make this link for side-bounce\n', '    # disable asrp link to ar for 2D USAXS\n', '    # FS_disableASRP\n', '\n', '    # measure_USAXS_PD_dark_currents    # used to be here, not now\n', '    yield from after_plan(weight=3)\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', 'def SAXS(pos_X, pos_Y, thickness, scan_title, md={}):\n', '    """\n', '    collect SAXS data\n', '     """\n', '    yield from IfRequestedStopBeforeNextScan()\n', '\n', '    yield from mode_SAXS()\n', '\n', '    pinz_target = terms.SAXS.z_in.value + constants["SAXS_PINZ_OFFSET"]\n', '    yield from bps.mv(\n', '        usaxs_slit.v_size, terms.SAXS.v_size.value,\n', '        usaxs_slit.h_size, terms.SAXS.h_size.value,\n', '        guard_slit.v_size, terms.SAXS.guard_v_size.value,\n', '        guard_slit.h_size, terms.SAXS.guard_h_size.value,\n', '        saxs_stage.z, pinz_target,      # MUST move before sample stage moves!\n', '        user_data.sample_thickness, thickness,\n', '    )\n', '    yield from before_plan()\n', '\n', '    yield from bps.mv(\n', '        s_stage.x, pos_X,\n', '        s_stage.y, pos_Y,\n', '    )\n', '\n', '    scan_title_clean = cleanupText(scan_title)\n', '\n', '    # SPEC-compatibility symbols\n', '    SCAN_N = RE.md["scan_id"]+1     # the next scan number (user-controllable)\n', '    # use our specwriter to get a pseudo-SPEC file name\n', '    DATAFILE = os.path.split(specwriter.spec_filename)[-1]\n', '\n', '    # these two templates match each other, sort of\n', '    ad_file_template = "%s%s_%4.4d.hdf"\n', '    local_file_template = "%s_%04d.hdf"\n', '\n', '    # directory is pwd + DATAFILE + "_usaxs"\n', '    # path on local file system\n', '    SAXSscan_path = os.path.join(os.getcwd(), os.path.splitext(DATAFILE)[0] + "_saxs")\n', '    SAXS_file_name = local_file_template % (scan_title_clean, saxs_det.hdf1.file_number.value)\n', '    # NFS-mounted path as the Pilatus detector sees it\n', '    pilatus_path = os.path.join("/mnt/usaxscontrol", *SAXSscan_path.split(os.path.sep)[2:])\n', '    # area detector will create this path if needed ("Create dir. depth" setting)\n', '    if not pilatus_path.endswith("/"):\n', '        pilatus_path += "/"        # area detector needs this\n', '    local_name = os.path.join(SAXSscan_path, SAXS_file_name)\n', '    print(f"Area Detector HDF5 file: {local_name}")\n', '    pilatus_name = os.path.join(pilatus_path, SAXS_file_name)\n', '    print(f"Pilatus computer Area Detector HDF5 file: {pilatus_name}")\n', '\n', '    yield from bps.mv(\n', '        saxs_det.hdf1.file_name, scan_title_clean,\n', '        saxs_det.hdf1.file_path, pilatus_path,\n', '        saxs_det.hdf1.file_template, ad_file_template,\n', '    )\n', '\n', '    ts = str(datetime.datetime.now())\n', '    yield from bps.mv(\n', '        user_data.sample_title, scan_title,\n', '        user_data.state, "starting SAXS collection",\n', '        user_data.sample_thickness, thickness,\n', '        user_data.user_name, USERNAME,\n', '        user_data.spec_scan, str(SCAN_N),\n', '        user_data.time_stamp, ts,\n', '        user_data.scan_macro, "SAXS",       # match the value in the scan logs\n', '    )\n', '    yield from bps.mv(\n', '        user_data.user_dir, os.getcwd(),        # TODO: watch out for string too long for EPICS! (make it an EPICS waveform string)\n', '        user_data.spec_file, os.path.split(specwriter.spec_filename)[-1],\n', '   )\n', '\n', '    yield from bps.install_suspender(suspend_BeamInHutch)\n', '    yield from measure_SAXS_Transmission()\n', '    yield from insertSaxsFilters()\n', '\n', '    yield from bps.mv(\n', '        mono_shutter, "open",\n', '        monochromator.feedback.on, MONO_FEEDBACK_OFF,\n', '        ti_filter_shutter, "open",\n', '        saxs_det.cam.num_images, terms.SAXS.num_images.value,\n', '        saxs_det.cam.acquire_time, terms.SAXS.acquire_time.value,\n', '        saxs_det.cam.acquire_period, terms.SAXS.acquire_time.value + 0.004,\n', '    )\n', '    old_det_stage_sigs = OrderedDict()\n', '    for k, v in saxs_det.hdf1.stage_sigs.items():\n', '        old_det_stage_sigs[k] = v\n', '    if saxs_det.hdf1.stage_sigs.get("capture") is not None:\n', '        del saxs_det.hdf1.stage_sigs["capture"]\n', '    saxs_det.hdf1.stage_sigs["file_template"] = ad_file_template\n', '    saxs_det.hdf1.stage_sigs["file_write_mode"] = "Single"\n', '    saxs_det.hdf1.stage_sigs["blocking_callbacks"] = "No"\n', '\n', '    yield from bps.sleep(0.2)\n', '    yield from autoscale_amplifiers([I0_controls])\n', '\n', '    yield from bps.mv(\n', '        ti_filter_shutter, "close",\n', '    )\n', '\n', '    SCAN_N = RE.md["scan_id"]+1     # update with next number\n', '    old_delay = scaler0.delay.value\n', '    yield from bps.mv(\n', '        scaler1.preset_time, terms.SAXS.acquire_time.value + 1,\n', '        scaler0.preset_time, 1.2*terms.SAXS.acquire_time.value + 1,\n', '        scaler0.count_mode, "OneShot",\n', '        scaler1.count_mode, "OneShot",\n', '\n', '        # update as fast as hardware will allow\n', '        # this is needed to make sure we get as up to date I0 number as possible for AD software.\n', '        scaler0.display_rate, 60,\n', '        scaler1.display_rate, 60,\n', '\n', '        scaler0.delay, 0,\n', '        terms.SAXS_WAXS.start_exposure_time, ts,\n', '        user_data.state, f"SAXS collection for {terms.SAXS.acquire_time.value} s",\n', '        user_data.spec_scan, str(SCAN_N),\n', '    )\n', '\n', '    yield from bps.mv(\n', '        scaler0.count, 1,\n', '        scaler1.count, 1,\n', '    )\n', '\n', '    _md = OrderedDict()\n', '    _md.update(md or {})\n', '    _md[\'plan_name\'] = "SAXS"\n', '    _md["hdf5_file"] = SAXS_file_name\n', '    _md["hdf5_path"] = SAXSscan_path\n', '\n', '    yield from areaDetectorAcquire(saxs_det, _md)\n', '    ts = str(datetime.datetime.now())\n', '    yield from bps.remove_suspender(suspend_BeamInHutch)\n', '\n', '    saxs_det.hdf1.stage_sigs = old_det_stage_sigs    # TODO: needed? not even useful?\n', '\n', '    yield from bps.mv(\n', '        scaler0.count, 0,\n', '        scaler1.count, 0,\n', '        terms.SAXS_WAXS.I0, scaler1.channels.chan02.s.value,\n', '        scaler0.display_rate, 5,\n', '        scaler1.display_rate, 5,\n', '        terms.SAXS_WAXS.end_exposure_time, ts,\n', '        scaler0.delay, old_delay,\n', '\n', '        user_data.state, "Done SAXS",\n', '        user_data.time_stamp, ts,\n', '    )\n', '    logger.info(f"I0 value: {terms.SAXS_WAXS.I0.value}")\n', '    yield from after_plan()\n', '\n', '\n', '# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n', '\n', '\n', 'def WAXS(pos_X, pos_Y, thickness, scan_title, md={}):\n', '    """\n', '    collect WAXS data\n', '     """\n', '    yield from IfRequestedStopBeforeNextScan()\n', '\n', '    yield from mode_WAXS()\n', '\n', '    yield from bps.mv(\n', '        usaxs_slit.v_size, terms.SAXS.v_size.value,\n', '        usaxs_slit.h_size, terms.SAXS.h_size.value,\n', '        guard_slit.v_size, terms.SAXS.guard_v_size.value,\n', '        guard_slit.h_size, terms.SAXS.guard_h_size.value,\n', '        user_data.sample_thickness, thickness,\n', '    )\n', '    yield from before_plan()\n', '\n', '    yield from bps.mv(\n', '        s_stage.x, pos_X,\n', '        s_stage.y, pos_Y,\n', '    )\n', '\n', '    scan_title_clean = cleanupText(scan_title)\n', '\n', '    # SPEC-compatibility symbols\n', '    SCAN_N = RE.md["scan_id"]+1     # the next scan number (user-controllable)\n', '    # use our specwriter to get a pseudo-SPEC file name\n', '    DATAFILE = os.path.split(specwriter.spec_filename)[-1]\n', '\n', '    # these two templates match each other, sort of\n', '    ad_file_template = "%s%s_%4.4d.hdf"\n', '    local_file_template = "%s_%04d.hdf"\n', '\n', '    # directory is pwd + DATAFILE + "_usaxs"\n', '    # path on local file system\n', '    WAXSscan_path = os.path.join(os.getcwd(), os.path.splitext(DATAFILE)[0] + "_waxs")\n', '    WAXS_file_name = local_file_template % (scan_title_clean, waxs_det.hdf1.file_number.value)\n', '    # NFS-mounted path as the Pilatus detector sees it\n', '    pilatus_path = os.path.join("/mnt/usaxscontrol", *WAXSscan_path.split(os.path.sep)[2:])\n', '    # area detector will create this path if needed ("Create dir. depth" setting)\n', '    if not pilatus_path.endswith("/"):\n', '        pilatus_path += "/"        # area detector needs this\n', '    local_name = os.path.join(WAXSscan_path, WAXS_file_name)\n', '    print(f"Area Detector HDF5 file: {local_name}")\n', '    pilatus_name = os.path.join(pilatus_path, WAXS_file_name)\n', '    print(f"Pilatus computer Area Detector HDF5 file: {pilatus_name}")\n', '\n', '    yield from bps.mv(\n', '        waxs_det.hdf1.file_name, scan_title_clean,\n', '        waxs_det.hdf1.file_path, pilatus_path,\n', '        waxs_det.hdf1.file_template, ad_file_template,\n', '    )\n', '\n', '    ts = str(datetime.datetime.now())\n', '    yield from bps.mv(\n', '        user_data.sample_title, scan_title,\n', '        user_data.state, "starting WAXS collection",\n', '        user_data.sample_thickness, thickness,\n', '        user_data.user_name, USERNAME,\n', '        user_data.spec_scan, str(SCAN_N),\n', '        user_data.time_stamp, ts,\n', '        user_data.scan_macro, "WAXS",       # match the value in the scan logs\n', '    )\n', '    yield from bps.mv(\n', '        user_data.user_dir, os.getcwd(),        # TODO: watch out for string too long for EPICS! (make it an EPICS waveform string)\n', '        user_data.spec_file, os.path.split(specwriter.spec_filename)[-1],\n', '   )\n', '\n', '    #yield from measure_SAXS_Transmission()\n', '    yield from insertWaxsFilters()\n', '\n', '    yield from bps.mv(\n', '        mono_shutter, "open",\n', '        monochromator.feedback.on, MONO_FEEDBACK_OFF,\n', '        ti_filter_shutter, "open",\n', '        waxs_det.cam.num_images, terms.WAXS.num_images.value,\n', '        waxs_det.cam.acquire_time, terms.WAXS.acquire_time.value,\n', '        waxs_det.cam.acquire_period, terms.WAXS.acquire_time.value + 0.004,\n', '    )\n', '    yield from bps.install_suspender(suspend_BeamInHutch)\n', '    old_det_stage_sigs = OrderedDict()\n', '    for k, v in waxs_det.hdf1.stage_sigs.items():\n', '        old_det_stage_sigs[k] = v\n', '    if waxs_det.hdf1.stage_sigs.get("capture") is not None:\n', '        del waxs_det.hdf1.stage_sigs["capture"]\n', '    waxs_det.hdf1.stage_sigs["file_template"] = ad_file_template\n', '    waxs_det.hdf1.stage_sigs["file_write_mode"] = "Single"\n', '    waxs_det.hdf1.stage_sigs["blocking_callbacks"] = "No"\n', '\n', '    yield from bps.sleep(0.2)\n', '    yield from autoscale_amplifiers([I0_controls, trd_controls])\n', '\n', '    yield from bps.mv(\n', '        ti_filter_shutter, "close",\n', '    )\n', '\n', '    old_delay = scaler0.delay.value\n', '    yield from bps.mv(\n', '        scaler1.preset_time, terms.WAXS.acquire_time.value + 1,\n', '        scaler0.preset_time, 1.2*terms.WAXS.acquire_time.value + 1,\n', '        scaler0.count_mode, "OneShot",\n', '        scaler1.count_mode, "OneShot",\n', '\n', '        # update as fast as hardware will allow\n', '        # this is needed to make sure we get as up to date I0 number as possible for AD software.\n', '        scaler0.display_rate, 60,\n', '        scaler1.display_rate, 60,\n', '\n', '        scaler0.delay, 0,\n', '        terms.SAXS_WAXS.start_exposure_time, ts,\n', '        user_data.state, f"WAXS collection for {terms.SAXS.acquire_time.value} s",\n', '    )\n', '\n', '    yield from bps.mv(\n', '        scaler0.count, 1,\n', '        scaler1.count, 1,\n', '    )\n', '\n', '    _md = OrderedDict()\n', '    _md.update(md or {})\n', '    _md[\'plan_name\'] = "WAXS"\n', '    _md["hdf5_file"] = WAXS_file_name\n', '    _md["hdf5_path"] = WAXSscan_path\n', '\n', '    yield from areaDetectorAcquire(waxs_det, md=_md)\n', '    ts = str(datetime.datetime.now())\n', '\n', '    waxs_det.hdf1.stage_sigs = old_det_stage_sigs    # TODO: needed? not even useful?\n', '\n', '    yield from bps.mv(\n', '        scaler0.count, 0,\n', '        scaler1.count, 0,\n', '        # WAXS uses same PVs for normalization and transmission as SAXS, should we aliased it same to terms.WAXS???\n', '        terms.SAXS_WAXS.I0, scaler1.channels.chan02.s.value,\n', '        terms.SAXS_WAXS.diode_transmission, scaler0.channels.chan04.s.value,\n', '        terms.SAXS_WAXS.diode_gain, trd_controls.femto.gain.value,\n', '        terms.SAXS_WAXS.I0_transmission, scaler0.channels.chan02.s.value,\n', '        terms.SAXS_WAXS.I0_gain, I0_controls.femto.gain.value,\n', '        scaler0.display_rate, 5,\n', '        scaler1.display_rate, 5,\n', '        terms.SAXS_WAXS.end_exposure_time, ts,\n', '        scaler0.delay, old_delay,\n', '\n', '        user_data.state, "Done WAXS",\n', '        user_data.time_stamp, ts,\n', '    )\n', '    yield from bps.remove_suspender(suspend_BeamInHutch)\n', '    logger.info(f"I0 value: {terms.SAXS_WAXS.I0.value}")\n', '    yield from after_plan()\n']), ('archive_file', '/share1/log/macros/20190619-150413-usaxs.txt'), ('archive_timestamp', '20190619-150413')])
#MD beamline_id = APS USAXS 9-ID-C
#MD datetime = 2019-06-19 15:05:14.746430
#MD filename = usaxs.mac
#MD fly_scan_time = 90.0
#MD full_filename = /share1/USAXS_data/2019-06/usaxs.mac
#MD hdf5_file = Peter_TEB_0105.h5
#MD hdf5_path = /share1/USAXS_data/2019-06/06_19_Peter_Capillaries_usaxs
APS-USAXS / livedata

test livedata SAXS & WAXS data reduction from Bluesky #14
