JSON parse fail with perf stat wrapper

[tcherrou]

Parsing perf stat JSON output throws an error when using the perf stat command Wrapper (PerfStatWrap) Error thrown:

Traceback (most recent call last):
  File "/home/taoufik/Documents/PAAE/benchkit/examples/par-dpll/minimal_bench.py", line 152, in <module>
    test_campaign.run()
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/campaign.py", line 197, in run
    self.campaign_run(other_campaigns_seconds=0, barrier=None)
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/campaign.py", line 187, in campaign_run
    self._benchmark.run(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 538, in run
    expected_total_seconds = self.expected_total_duration_seconds()
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 428, in expected_total_duration_seconds
    result = self.total_nb_runs() * bds
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 396, in total_nb_runs
    build_variables, run_variables, _, _ = self._group_record_parameters(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 898, in _group_record_parameters
    build_variables = {
TypeError: 'NoneType' object is not iterable

Minimal reproducible example:

import pathlib
from typing import Any, Dict, Iterable, List

from benchkit.benchmark import (
    Benchmark,
    CommandAttachment,
    CommandWrapper,
    PostRunHook,
    PreRunHook,
    RecordResult,
    SharedLib,
    WriteRecordFileFunction,
)
from benchkit.dependencies.packages import PackageDependency
from benchkit.platforms import Platform, get_current_platform
from benchkit.utils.misc import TimeMeasure
from benchkit.utils.types import PathType
from benchkit.campaign import CampaignCartesianProduct
from benchkit.commandwrappers.perf import PerfStatWrap

class Test(Benchmark):

    def __init__(
        self,
        src_dir: PathType,
        command_wrappers: Iterable[CommandWrapper] = (),
        command_attachments: Iterable[CommandAttachment] = (),
        shared_libs: Iterable[SharedLib] = (),
        pre_run_hooks: Iterable[PreRunHook] = (),
        post_run_hooks: Iterable[PostRunHook] = (),
        platform: Platform | None = None,
    ) -> None:
        super().__init__(
            command_wrappers=command_wrappers,
            command_attachments=command_attachments,
            shared_libs=shared_libs,
            pre_run_hooks=pre_run_hooks,
            post_run_hooks=post_run_hooks,
        )
        if platform is not None:
            self.platform = platform  # TODO Warning! overriding upper class platform
        bench_src_path = pathlib.Path(src_dir)
        if not self.platform.comm.isdir(bench_src_path):
            raise ValueError(
                f"Invalid source path: {bench_src_path}\n"
                "src_dir argument can be defined manually."
            )

        self.platform = get_current_platform()
        self._bench_src_path = bench_src_path
        self._build_dir = bench_src_path

    @property
    def bench_src_path(self) -> pathlib.Path:
        return self._bench_src_path

    @staticmethod
    def get_build_var_names() -> List[str]:
        pass
        # return ["implementation"]

    @staticmethod
    def get_run_var_names() -> List[str]:
        # return ["instance","implementation"]
        pass

    @staticmethod
    def get_tilt_var_names() -> List[str]:
        return []

    def dependencies(self) -> List[PackageDependency]:
        return super().dependencies() + []

    def build_tilt(self, **kwargs) -> None:
        pass

    def prebuild_bench(
        self,
        **_kwargs,
    ) -> None:
        pass

    def build_bench(
        self,
        implementation: str,
        **kwargs,
    ) -> None:
        pass

    def clean_bench(self) -> None:
        pass

    def single_run(
        self,
        benchmark_duration_seconds: int,
        **kwargs,
    ) -> str:
        environment = {}
        run_command =["sleep","1"]
        wrap_run_command, wrapped_environment = self._wrap_command(
            run_command=run_command,
            environment=environment,
            **kwargs
        )

        output = self.run_bench_command(
            run_command=run_command,
            wrapped_run_command=wrap_run_command,
            current_dir=self._build_dir,
            environment=environment,
            wrapped_environment=wrapped_environment,
            print_output=False
        )
        return output

    def parse_output_to_results(  # pylint: disable=arguments-differ
        self,
        command_output: str,
        run_variables: Dict[str, Any],
        **_kwargs,
    ) -> Dict[str, Any]:
        result_dict = {}
        # Parsing summary
        return result_dict

test_campaign = CampaignCartesianProduct(
    name="test",
    nb_runs=1,
    benchmark=Test(
        src_dir="./",
        command_wrappers= [PerfStatWrap()]),
    variables={},
    constants={},
    debug=False,
    gdb=False,
    enable_data_dir=True,
    continuing=False,
    benchmark_duration_seconds=10,
)

test_campaign.run()

[apaolillo]

this is because get_build_var_names and co methods do not return the expected List of string. If you violate the contract of these functions, the benchkit behaviour is undefined.

[tcherrou]

my apologies, it was a bad example. I updated the minimal example and the output trace:

import pathlib
from typing import Any, Dict, Iterable, List

from benchkit.benchmark import (
    Benchmark,
    CommandAttachment,
    CommandWrapper,
    PostRunHook,
    PreRunHook,
    SharedLib,
)
from benchkit.dependencies.packages import PackageDependency
from benchkit.platforms import Platform, get_current_platform
from benchkit.utils.misc import TimeMeasure
from benchkit.utils.types import PathType
from benchkit.campaign import CampaignCartesianProduct
from benchkit.commandwrappers.perf import PerfStatWrap

class Test(Benchmark):

    def __init__(
        self,
        src_dir: PathType,
        command_wrappers: Iterable[CommandWrapper] = (),
        command_attachments: Iterable[CommandAttachment] = (),
        shared_libs: Iterable[SharedLib] = (),
        pre_run_hooks: Iterable[PreRunHook] = (),
        post_run_hooks: Iterable[PostRunHook] = (),
        platform: Platform | None = None,
    ) -> None:
        super().__init__(
            command_wrappers=command_wrappers,
            command_attachments=command_attachments,
            shared_libs=shared_libs,
            pre_run_hooks=pre_run_hooks,
            post_run_hooks=post_run_hooks,
        )
        if platform is not None:
            self.platform = platform  # TODO Warning! overriding upper class platform
        bench_src_path = pathlib.Path(src_dir)
        if not self.platform.comm.isdir(bench_src_path):
            raise ValueError(
                f"Invalid source path: {bench_src_path}\n"
                "src_dir argument can be defined manually."
            )

        self.platform = get_current_platform()
        self._bench_src_path = bench_src_path
        self._build_dir = bench_src_path

    @property
    def bench_src_path(self) -> pathlib.Path:
        return self._bench_src_path

    @staticmethod
    def get_build_var_names() -> List[str]:
        return []

    @staticmethod
    def get_run_var_names() -> List[str]:
        return []

    @staticmethod
    def get_tilt_var_names() -> List[str]:
        return []

    def dependencies(self) -> List[PackageDependency]:
        return super().dependencies() + []

    def build_tilt(self, **kwargs) -> None:
        pass

    def prebuild_bench(
        self,
        **_kwargs,
    ) -> None:
        pass

    def build_bench(
        self,
        **kwargs,
    ) -> None:
        pass
        # self.platform.comm.shell(
        #     command="make",
        #     current_dir=implementation,
        #     output_is_log=True,
        # )

    def clean_bench(self) -> None:
        pass

    def single_run(
        self,
        benchmark_duration_seconds: int,
        **kwargs,
    ) -> str:
        environment = {}
        run_command =["sleep","1"]
        wrap_run_command, wrapped_environment = self._wrap_command(
            run_command=run_command,
            environment=environment,
            **kwargs
        )

        output = self.run_bench_command(
            run_command=run_command,
            wrapped_run_command=wrap_run_command,
            current_dir=self._build_dir,
            environment=environment,
            wrapped_environment=wrapped_environment,
            print_output=False
        )
        return output

    def parse_output_to_results(  # pylint: disable=arguments-differ
        self,
        command_output: str,
        run_variables: Dict[str, Any],
        **_kwargs,
    ) -> Dict[str, Any]:
        result_dict = {}
        # Parsing summary
        return result_dict
wrapper = PerfStatWrap()
test_campaign = CampaignCartesianProduct(
    name="test",
    nb_runs=1,
    benchmark=Test(
        src_dir="./",
        command_wrappers= [wrapper],
        post_run_hooks=[wrapper.post_run_hook_update_results]),
    variables={},
    constants={},
    debug=False,
    gdb=False,
    enable_data_dir=True,
    continuing=False,
    benchmark_duration_seconds=10,
)

test_campaign.run()

[INFO] Total number of runs: 1
[INFO] Expected duration: 10
[RUN] uname -a
[OUT]
Linux fedora 6.2.15-100.fc36.x86_64 #1 SMP PREEMPT_DYNAMIC Thu May 11 16:51:53 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
[INFO] Run  1/1, current campaign expected remaining time: ~10 seconds, i.e. ~0:00:10
[INFO] Full campaign suite estimated remaining time: ~10 seconds, i.e. ~0:00:10
[CD] .
[ENV:LC_NUMERIC] en_US.UTF-8
[RUN] /usr/bin/perf stat --json --output /home/taoufik/Documents/PAAE/benchkit/examples/par-dpll/results/benchmark_fedora_test_20240901_210034_862294/run-1/perf-stat.txt sleep 1
Traceback (most recent call last):
  File "/home/taoufik/Documents/PAAE/benchkit/examples/par-dpll/minimal_bench.py", line 150, in <module>
    test_campaign.run()
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/campaign.py", line 197, in run
    self.campaign_run(other_campaigns_seconds=0, barrier=None)
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/campaign.py", line 187, in campaign_run
    self._benchmark.run(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 590, in run
    self._run_single_run(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/benchmark.py", line 1095, in _run_single_run
    hook_dict = post_run_hook(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/commandwrappers/perf.py", line 455, in post_run_hook_update_results
    output_dict |= self._results_global(perf_stat_pathname=perf_stat_pathname)
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/commandwrappers/perf.py", line 578, in _results_global
    counter_rows = self._parse(
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/commandwrappers/perf.py", line 500, in _parse
    return self._parse_json(perf_stat_pathname=perf_stat_pathname, field_names=field_names)
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/commandwrappers/perf.py", line 511, in _parse_json
    json_lines = [json.loads(line) for line in lines]
  File "/home/taoufik/Documents/PAAE/benchkit/benchkit/commandwrappers/perf.py", line 511, in <listcomp>
    json_lines = [json.loads(line) for line in lines]
  File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

[apaolillo]

mmh.. i have a different error with this one. Could you paste here the content of perf-stat.txt?

[tcherrou]

This is the output that I get:

# started on Sun Sep  1 23:12:33 2024

{"counter-value" : "1.397967", "unit" : "msec", "event" : "task-clock", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 0.001393, "metric-unit" : "CPUs utilized"}
{"counter-value" : "1.000000", "unit" : "", "event" : "context-switches", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 715.324468, "metric-unit" : "/sec"}
{"counter-value" : "0.000000", "unit" : "", "event" : "cpu-migrations", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 0.000000, "metric-unit" : "/sec"}
{"counter-value" : "66.000000", "unit" : "", "event" : "page-faults", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 47.211415, "metric-unit" : "K/sec"}
{"counter-value" : "1419380.000000", "unit" : "", "event" : "cycles", "event-runtime" : 139903, "pcnt-running" : 10.00, "metric-value" : 1.015317, "metric-unit" : "GHz"}
{"counter-value" : "141415.000000", "unit" : "", "event" : "stalled-cycles-frontend", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 9.963153, "metric-unit" : "frontend cycles idle"}
{"counter-value" : "259420.000000", "unit" : "", "event" : "stalled-cycles-backend", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 18.276994, "metric-unit" : "backend cycles idle"}
{"counter-value" : "1397337.000000", "unit" : "", "event" : "instructions", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 0.984470, "metric-unit" : "insn per cycle"}
{"metric-value" : 0.185653, "metric-unit" : "stalled cycles per insn"}
{"counter-value" : "332229.000000", "unit" : "", "event" : "branches", "event-runtime" : 1397967, "pcnt-running" : 100.00, "metric-value" : 237.651533, "metric-unit" : "M/sec"}
{"counter-value" : "11069.000000", "unit" : "", "event" : "branch-misses", "event-runtime" : 1258064, "pcnt-running" : 89.00, "metric-value" : 3.331738, "metric-unit" : "of all branches"}

I believe it might fail on the first line since it is not JSON. Also some JSON lines do not have all the keys such as "event" (However, I believe this is only a problem when no events are passed as options)

EDIT: "event" instead of "event_name"

[apaolillo]

On what platform are you running this?

[tcherrou]

platform: Linux fedora 6.2.15-100.fc36.x86_64 perf version 6.2.6