BUG: Onnx Export

[andife]

tracer = <torch._dynamo.output_graph.OutputGraph object at 0x78702014cf50>, node = cat_1, args = ((FakeTensor(..., size=(2, 3, 1, 1)), FakeTensor(..., size=(2, 3, 1, u0))), -1), kwargs = {}, nnmodule = None

def run_node(tracer, node, args, kwargs, nnmodule):
    """
    Runs a given node, with the given args and kwargs.

    Behavior is dictated by a node's op.

    run_node is useful for extracting real values out of nodes.
    See get_real_value for more info on common usage.

    Note: The tracer arg is only used for 'get_attr' ops
    Note: The nnmodule arg is only used for 'call_module' ops

    Nodes that are not call_function, call_method, call_module, or get_attr will
    raise an AssertionError.
    """
    op = node.op

    with set_current_node(node):

        def make_error_message(e):
            return f"Failed running {op} {node.target}(*{args}, **{kwargs}):\n" + str(e)

        try:
            if op == "call_function":

              return node.target(*args, **kwargs)

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1904:

---

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/utils/_stats.py:20: in wrapper return fn(*args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1060: in __torch_dispatch__ return self.dispatch(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1449: in dispatch return self._cached_dispatch_impl(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1152: in _cached_dispatch_impl output = self._dispatch_impl(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1689: in _dispatch_impl return decomposition_table[func](*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/wrappers.py:265: in _fn result = fn(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/wrappers.py:137: in _fn result = fn(*bound.arguments) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_refs/init.py:2772: in cat memory_format = cat_compute_output_memory_format(tensors) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_refs/init.py:2698: in cat_compute_output_memory_format f = utils.suggest_memory_format(t) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py:1849: in suggest_memory_format if are_strides_like_channels_last(x.shape, x.stride()): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py:1840: in are_strides_like_channels_last if strides[d] > 1: ../miniforge3/envs/af20240530poetry/lib/python3.11/site-packages/torch/init.py:429: in bool return self.node.bool() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/symnode.py:458: in bool return self.guard_bool("", 0) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/sym_node.py:396: in guard_bool r = self.shape_env.evaluate_expr(self.expr, self.hint, fx_node=self.fx_node) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/recording.py:244: in wrapper return fn(args, **kwargs)

---

self = <torch.fx.experimental.symbolic_shapes.ShapeEnv object at 0x787020108210>, orig_expr = Max(1, u0 + 1) > 1, hint = None, fx_node = None, expect_rational = True, size_oblivious = False

@lru_cache(256)
@record_shapeenv_event(save_tracked_fakes=True)
def evaluate_expr(self, orig_expr: "sympy.Expr", hint=None, fx_node=None,
                  expect_rational=True, size_oblivious: bool = False, *, forcing_spec: bool = False):
    """
    Given an expression, evaluates it, adding guards if necessary
    """

    # TODO: split conjunctions and evaluate them separately

    # Don't track this one
    @functools.lru_cache(None)
    def compute_concrete_val():
        if hint is None:
            return self.size_hint(orig_expr)
        else:
            return sympy.sympify(hint)

    # Check if:
    #   1. 'translation_validation' is set
    #   2. the corresponding 'fx_node' is not 'None'
    #   3. the guard should not be suppressed
    #
    # If all of the above check, we create an FX node representing the
    # actual expression to be guarded.
    node = None
    fresh = False
    if (
            self._translation_validation_enabled
            and fx_node is not None
            and not self._suppress_guards_tls()
            and not size_oblivious
    ):
        concrete_val = compute_concrete_val()
        if concrete_val is sympy.true:
            node, fresh = self._create_fx_call_function(torch._assert, (fx_node,))
        elif concrete_val is sympy.false:
            neg, _ = self._create_fx_call_function(operator.not_, (fx_node,))
            node, fresh = self._create_fx_call_function(torch._assert, (neg,))
        else:
            eql, _ = self._create_fx_call_function(operator.eq, (fx_node, concrete_val))
            node, fresh = self._create_fx_call_function(torch._assert, (eql,))

        assert node is not None
        # If this is a fresh node, we have to remember the event index that
        # corresponds to this assertion node.
        # Reason: so that, given an assertion node, we can replay the ShapeEnv
        # events until the point where this assertion node was freshly created.
        if fresh:
            self._add_fx_node_metadata(node)

    # After creating the FX node corresponding to orig_expr, we must make sure that
    # no error will be raised until the end of this function.
    #
    # Reason: the translation validation may become invalid otherwise.
    #
    # If an error is raised before the end of this function, we remove the FX node
    # inserted, and re-raise the error.
    guard = None
    tb = None

    try:
        if orig_expr.is_number:
            self.log.debug("eval %s [trivial]", orig_expr)
            if hint is not None:
                assert orig_expr == hint, f"{orig_expr} != {hint}"
            return orig_expr

        expr = orig_expr

        static_expr = self._maybe_evaluate_static(expr,
                                                  expect_rational=expect_rational,
                                                  size_oblivious=size_oblivious)
        if static_expr is not None:
            self.log.debug("eval %s == %s [statically known]", orig_expr, static_expr)
            if hint is not None:
                assert static_expr == hint, f"{static_expr} != {hint}"
            return static_expr

        transmute_into_runtime_assert = False

        concrete_val = None
        if not (expr.free_symbols <= self.var_to_val.keys()):
            # TODO: dedupe this with _maybe_evaluate_static
            # Attempt to eliminate the unbacked SymInt
            new_expr = self._maybe_evaluate_static(expr, unbacked_only=True)
            if not (new_expr.free_symbols <= self.var_to_val.keys()):
                size_oblivious_result = None
                if not size_oblivious:
                    size_oblivious_result = self._maybe_evaluate_static(
                        expr,
                        expect_rational=expect_rational,
                        size_oblivious=True
                    )

                # Last ditch
                if (
                    self.unbacked_var_to_val and
                    not (unsound_result := orig_expr.xreplace(self.unbacked_var_to_val)).free_symbols
                ):
                    log.warning("propagate_real_tensors evaluate_expr(%s) -> %s", orig_expr, unsound_result)
                    trace_structured(
                        "propagate_real_tensors",
                        metadata_fn=lambda: {
                            "expr": repr(orig_expr),
                            "result": repr(unsound_result),
                            "stack": structured.from_traceback(CapturedTraceback.extract(skip=1).summary()),
                        },
                    )
                    transmute_into_runtime_assert = True
                    concrete_val = unsound_result
                else:

                  raise self._make_data_dependent_error(

expr.xreplace(self.var_to_val), expr, size_oblivious_result=size_oblivious_result ) E torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode: Could not guard on data-dependent expression Max(1, u0 + 1) > 1 (unhinted: Max(1, u0 + 1) > 1). (Size-like symbols: u0) E E ATTENTION: guard_size_oblivious would fix the error, evaluating expression to True. E Maybe you need to add guard_size_oblivious to framework code, see doc below for more guidance. E E Potential framework code culprit (scroll up for full backtrace): E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py", line 1840, in are_strides_like_channels_last E if strides[d] > 1: E E For more information, run with TORCH_LOGS="dynamic" E For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0" E If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1 E For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing E E User Stack (most recent call last): E (snipped, see stack below for prefix) E File "/local_data/andife/Spherinator/tests/test_onnx_export.py", line 37, in forward E return self.spherical.rsample(x.shape) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transformed_distribution.py", line 152, in rsample E x = transform(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transforms.py", line 156, in call E return self._call(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/power_spherical/distributions.py", line 19, in _call E return torch.cat((t, v * torch.sqrt(torch.clamp(1 - t ** 2, _EPS))), -1) E E For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/symbolic_shapes.py:5138: GuardOnDataDependentSymNode

The above exception was the direct cause of the following exception:

node = cat_1, tx = <torch._dynamo.symbolic_convert.InliningInstructionTranslator object at 0x78701deeead0>, allow_non_graph_fake = True

def get_fake_value(node, tx, allow_non_graph_fake=False):
    """
    Run the computation represented by `node` using fake tensors and return the result.

    allow_non_graph_fake: whether to allow the return result to be:
        1. non-fake or 2. fake that is not created by this instance of Dynamo.
        If `True`, you must be prepared to deal with such return values, ideally
        by further wrapping them as this graph's fakes.
    """
    from torch.utils._sympy.value_ranges import ValueRangeError
    from .exc import (
        TorchRuntimeError,
        unimplemented,
        Unsupported,
        UserError,
        UserErrorType,
    )

    op = node.op

    # FX Node should always return the same fake value
    if "example_value" in node.meta and is_fake(node.meta["example_value"]):
        return node.meta["example_value"]

    args, kwargs = get_fake_values_from_nodes(
        tx, (node.args, node.kwargs), allow_non_graph_fake
    )

    nnmodule = None
    if op == "call_method" and len(args) > 0 and isinstance(args[0], torch.nn.Module):
        # If the first argument is nn.Module, should copy to fake mode.
        args = (deepcopy_to_fake_tensor(args[0], tx.fake_mode),) + tuple(args[1:])

    if op == "call_module":
        nnmodule = tx.output.nn_modules[node.target]

        if is_lazy_module(nnmodule) and hasattr(nnmodule, "_initialize_hook"):
            # In the case of a lazy module, we want to run
            # the pre-hooks which initialize it.
            # Afterwards, lazy module deletes its pre-hooks
            # to avoid treating it as lazy on subsequent recompile.
            nnmodule._infer_parameters(nnmodule, args)

        # no matter it's lazy module or not, we should copy to fake mode.
        nnmodule = deepcopy_to_fake_tensor(nnmodule, tx.fake_mode)

    try:
        with tx.fake_mode, enable_python_dispatcher():

          ret_val = wrap_fake_exception(

lambda: run_node(tx.output, node, args, kwargs, nnmodule) )

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1786:

---

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1301: in wrap_fake_exception return fn() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1787: in lambda: run_node(tx.output, node, args, kwargs, nnmodule) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1922: in run_node raise RuntimeError(make_error_message(e)).with_traceback( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1904: in run_node return node.target(*args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/utils/_stats.py:20: in wrapper return fn(*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1060: in __torch_dispatch__ return self.dispatch(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1449: in dispatch return self._cached_dispatch_impl(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1152: in _cached_dispatch_impl output = self._dispatch_impl(func, types, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py:1689: in _dispatch_impl return decomposition_table[func](args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/wrappers.py:265: in _fn result = fn(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/wrappers.py:137: in _fn result = fn(bound.arguments) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_refs/init.py:2772: in cat memory_format = cat_compute_output_memory_format(tensors) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_refs/init.py:2698: in cat_compute_output_memory_format f = utils.suggest_memory_format(t) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py:1849: in suggest_memory_format if are_strides_like_channels_last(x.shape, x.stride()): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py:1840: in are_strides_like_channels_last if strides[d] > 1: ../miniforge3/envs/af20240530poetry/lib/python3.11/site-packages/torch/init.py:429: in bool return self.node.bool() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/symnode.py:458: in bool return self.guard_bool("", 0) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/sym_node.py:396: in guard_bool r = self.shape_env.evaluate_expr(self.expr, self.hint, fx_node=self.fx_node) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/recording.py:244: in wrapper return fn(args, **kwargs)

---

self = <torch.fx.experimental.symbolic_shapes.ShapeEnv object at 0x787020108210>, orig_expr = Max(1, u0 + 1) > 1, hint = None, fx_node = None, expect_rational = True, size_oblivious = False

@lru_cache(256)
@record_shapeenv_event(save_tracked_fakes=True)
def evaluate_expr(self, orig_expr: "sympy.Expr", hint=None, fx_node=None,
                  expect_rational=True, size_oblivious: bool = False, *, forcing_spec: bool = False):
    """
    Given an expression, evaluates it, adding guards if necessary
    """

    # TODO: split conjunctions and evaluate them separately

    # Don't track this one
    @functools.lru_cache(None)
    def compute_concrete_val():
        if hint is None:
            return self.size_hint(orig_expr)
        else:
            return sympy.sympify(hint)

    # Check if:
    #   1. 'translation_validation' is set
    #   2. the corresponding 'fx_node' is not 'None'
    #   3. the guard should not be suppressed
    #
    # If all of the above check, we create an FX node representing the
    # actual expression to be guarded.
    node = None
    fresh = False
    if (
            self._translation_validation_enabled
            and fx_node is not None
            and not self._suppress_guards_tls()
            and not size_oblivious
    ):
        concrete_val = compute_concrete_val()
        if concrete_val is sympy.true:
            node, fresh = self._create_fx_call_function(torch._assert, (fx_node,))
        elif concrete_val is sympy.false:
            neg, _ = self._create_fx_call_function(operator.not_, (fx_node,))
            node, fresh = self._create_fx_call_function(torch._assert, (neg,))
        else:
            eql, _ = self._create_fx_call_function(operator.eq, (fx_node, concrete_val))
            node, fresh = self._create_fx_call_function(torch._assert, (eql,))

        assert node is not None
        # If this is a fresh node, we have to remember the event index that
        # corresponds to this assertion node.
        # Reason: so that, given an assertion node, we can replay the ShapeEnv
        # events until the point where this assertion node was freshly created.
        if fresh:
            self._add_fx_node_metadata(node)

    # After creating the FX node corresponding to orig_expr, we must make sure that
    # no error will be raised until the end of this function.
    #
    # Reason: the translation validation may become invalid otherwise.
    #
    # If an error is raised before the end of this function, we remove the FX node
    # inserted, and re-raise the error.
    guard = None
    tb = None

    try:
        if orig_expr.is_number:
            self.log.debug("eval %s [trivial]", orig_expr)
            if hint is not None:
                assert orig_expr == hint, f"{orig_expr} != {hint}"
            return orig_expr

        expr = orig_expr

        static_expr = self._maybe_evaluate_static(expr,
                                                  expect_rational=expect_rational,
                                                  size_oblivious=size_oblivious)
        if static_expr is not None:
            self.log.debug("eval %s == %s [statically known]", orig_expr, static_expr)
            if hint is not None:
                assert static_expr == hint, f"{static_expr} != {hint}"
            return static_expr

        transmute_into_runtime_assert = False

        concrete_val = None
        if not (expr.free_symbols <= self.var_to_val.keys()):
            # TODO: dedupe this with _maybe_evaluate_static
            # Attempt to eliminate the unbacked SymInt
            new_expr = self._maybe_evaluate_static(expr, unbacked_only=True)
            if not (new_expr.free_symbols <= self.var_to_val.keys()):
                size_oblivious_result = None
                if not size_oblivious:
                    size_oblivious_result = self._maybe_evaluate_static(
                        expr,
                        expect_rational=expect_rational,
                        size_oblivious=True
                    )

                # Last ditch
                if (
                    self.unbacked_var_to_val and
                    not (unsound_result := orig_expr.xreplace(self.unbacked_var_to_val)).free_symbols
                ):
                    log.warning("propagate_real_tensors evaluate_expr(%s) -> %s", orig_expr, unsound_result)
                    trace_structured(
                        "propagate_real_tensors",
                        metadata_fn=lambda: {
                            "expr": repr(orig_expr),
                            "result": repr(unsound_result),
                            "stack": structured.from_traceback(CapturedTraceback.extract(skip=1).summary()),
                        },
                    )
                    transmute_into_runtime_assert = True
                    concrete_val = unsound_result
                else:

                  raise self._make_data_dependent_error(

expr.xreplace(self.var_to_val), expr, size_oblivious_result=size_oblivious_result ) E RuntimeError: Failed running call_function <built-in method cat of type object at 0x786fe141c040>(*((FakeTensor(..., size=(2, 3, 1, 1)), FakeTensor(..., size=(2, 3, 1, u0))), -1), *{}): E Could not guard on data-dependent expression Max(1, u0 + 1) > 1 (unhinted: Max(1, u0 + 1) > 1). (Size-like symbols: u0) E E ATTENTION: guard_size_oblivious would fix the error, evaluating expression to True. E Maybe you need to add guard_size_oblivious to framework code, see doc below for more guidance. E E Potential framework code culprit (scroll up for full backtrace): E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py", line 1840, in are_strides_like_channels_last E if strides[d] > 1: E E For more information, run with TORCH_LOGS="dynamic" E For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0" E If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1 E For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing E E User Stack (most recent call last): E (snipped, see stack below for prefix) E File "/local_data/andife/Spherinator/tests/test_onnx_export.py", line 37, in forward E return self.spherical.rsample(x.shape) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transformed_distribution.py", line 152, in rsample E x = transform(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transforms.py", line 156, in call E return self._call(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/power_spherical/distributions.py", line 19, in _call E return torch.cat((t, v torch.sqrt(torch.clamp(1 - t ** 2, _EPS))), -1) E E For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/fx/experimental/symbolic_shapes.py:5138: RuntimeError

During handling of the above exception, another exception occurred:

def test_dynamo_export_spherical():
    class Model(torch.nn.Module):
        def __init__(self):
            self.spherical = PowerSpherical(
                torch.Tensor([0.0, 1.0]), torch.Tensor([1.0])
            )
            super().__init__()

        def forward(self, x):
            return self.spherical.rsample(x.shape)

    x = torch.randn(2, 3)

  exported_program = torch.export.export(Model(), args=(x,))

tests/test_onnx_export.py:40:

---

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/init.py:174: in export return _export( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/_trace.py:946: in wrapper raise e ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/_trace.py:929: in wrapper ep = fn(*args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/exported_program.py:88: in wrapper return fn(*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/_trace.py:1455: in _export aten_export_artifact = export_func( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/_trace.py:1060: in _strict_export gm_torch_level = _export_to_torch_ir( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/export/_trace.py:512: in _export_to_torch_ir gm_torchlevel, = torch._dynamo.export( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py:1350: in inner result_traced = opt_f(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/nn/modules/module.py:1552: in _wrapped_call_impl return self._call_impl(*args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/nn/modules/module.py:1561: in _call_impl return forward_call(*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py:421: in _fn return fn(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/nn/modules/module.py:1552: in _wrapped_call_impl return self._call_impl(*args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/nn/modules/module.py:1561: in _call_impl return forward_call(*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:1078: in catch_errors return callback(frame, cache_entry, hooks, frame_state, skip=1) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:456: in _convert_frame_assert return _compile( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_utils_internal.py:83: in wrapper_function return StrobelightCompileTimeProfiler.profile_compile_time( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_strobelight/compile_time_profiler.py:129: in profile_compile_time return func(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/contextlib.py:81: in inner return func(*args, kwds) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:799: in _compile guarded_code = compile_inner(code, one_graph, hooks, transform) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:232: in time_wrapper r = func(*args, *kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:618: in compile_inner out_code = transform_code_object(code, transform) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/bytecode_transformation.py:1184: in transform_code_object transformations(instructions, code_options) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:177: in _fn return fn(args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py:564: in transform tracer.run() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2450: in run super().run() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:892: in run while self.step(): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:804: in step self.dispatch_table[inst.opcode](self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:498: in wrapper return inner_fn(self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2058: in CALL self.call_function(fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:742: in call_function self.push(fn.call_function(self, args, kwargs)) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:342: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:294: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:91: in call_function return tx.inline_user_function_return(self, [self.self_args(), args], kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:748: in inline_user_function_return return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2665: in inline_call return cls.inlinecall(parent, func, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2781: in inlinecall tracer.run() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:892: in run while self.step(): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:804: in step self.dispatch_table[inst.opcode](self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:498: in wrapper return inner_fn(self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2058: in CALL self.call_function(fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:742: in call_function self.push(fn.call_function(self, args, kwargs)) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/user_defined.py:789: in call_function return self.call_method(tx, "call", args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/user_defined.py:641: in call_method return UserMethodVariable(method, self, source=source).call_function( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:342: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:294: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:91: in call_function return tx.inline_user_function_return(self, [self.self_args(), args], kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:748: in inline_user_function_return return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2665: in inline_call return cls.inlinecall(parent, func, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2781: in inlinecall tracer.run() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:892: in run while self.step(): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:804: in step self.dispatch_table[inst.opcode](self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:498: in wrapper return inner_fn(self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2058: in CALL self.call_function(fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:742: in call_function self.push(fn.call_function(self, args, kwargs)) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:342: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:294: in call_function return super().call_function(tx, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:91: in call_function return tx.inline_user_function_return(self, [self.self_args(), args], kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:748: in inline_user_function_return return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2665: in inline_call return cls.inlinecall(parent, func, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2781: in inlinecall tracer.run() ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:892: in run while self.step(): ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:804: in step self.dispatch_table[inst.opcode](self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:498: in wrapper return inner_fn(self, inst) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:2058: in CALL self.call_function(fn, args, kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py:742: in call_function self.push(fn.call_function(self, args, kwargs)) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/torch.py:756: in call_function tensor_variable = wrap_fx_proxy( ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/builder.py:1713: in wrap_fx_proxy return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs) ../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/variables/builder.py:1798: in wrap_fx_proxy_cls example_value = get_fake_value(proxy.node, tx, allow_non_graph_fake=True)

---

node = cat_1, tx = <torch._dynamo.symbolic_convert.InliningInstructionTranslator object at 0x78701deeead0>, allow_non_graph_fake = True

def get_fake_value(node, tx, allow_non_graph_fake=False):
    """
    Run the computation represented by `node` using fake tensors and return the result.

    allow_non_graph_fake: whether to allow the return result to be:
        1. non-fake or 2. fake that is not created by this instance of Dynamo.
        If `True`, you must be prepared to deal with such return values, ideally
        by further wrapping them as this graph's fakes.
    """
    from torch.utils._sympy.value_ranges import ValueRangeError
    from .exc import (
        TorchRuntimeError,
        unimplemented,
        Unsupported,
        UserError,
        UserErrorType,
    )

    op = node.op

    # FX Node should always return the same fake value
    if "example_value" in node.meta and is_fake(node.meta["example_value"]):
        return node.meta["example_value"]

    args, kwargs = get_fake_values_from_nodes(
        tx, (node.args, node.kwargs), allow_non_graph_fake
    )

    nnmodule = None
    if op == "call_method" and len(args) > 0 and isinstance(args[0], torch.nn.Module):
        # If the first argument is nn.Module, should copy to fake mode.
        args = (deepcopy_to_fake_tensor(args[0], tx.fake_mode),) + tuple(args[1:])

    if op == "call_module":
        nnmodule = tx.output.nn_modules[node.target]

        if is_lazy_module(nnmodule) and hasattr(nnmodule, "_initialize_hook"):
            # In the case of a lazy module, we want to run
            # the pre-hooks which initialize it.
            # Afterwards, lazy module deletes its pre-hooks
            # to avoid treating it as lazy on subsequent recompile.
            nnmodule._infer_parameters(nnmodule, args)

        # no matter it's lazy module or not, we should copy to fake mode.
        nnmodule = deepcopy_to_fake_tensor(nnmodule, tx.fake_mode)

    try:
        with tx.fake_mode, enable_python_dispatcher():
            ret_val = wrap_fake_exception(
                lambda: run_node(tx.output, node, args, kwargs, nnmodule)
            )
    except Unsupported:
        raise
    except RuntimeError as e:
        cause: BaseException = e
        if e.__cause__ is not None:
            cause = e.__cause__

        if isinstance(
            cause, torch._subclasses.fake_tensor.DataDependentOutputException
        ):
            unimplemented(
                f"data dependent operator: {cause.func}; "
                "to enable, set torch._dynamo.config.capture_scalar_outputs = True"
            )
        elif isinstance(
            cause, torch._subclasses.fake_tensor.DynamicOutputShapeException
        ):
            if not torch._dynamo.config.capture_dynamic_output_shape_ops:
                unimplemented(
                    f"dynamic shape operator: {cause.func}; "
                    "to enable, set torch._dynamo.config.capture_dynamic_output_shape_ops = True"
                )
            else:
                unimplemented(
                    f"dynamic shape operator: {cause.func}; "
                    "Operator does not have a meta kernel that supports dynamic output shapes, "
                    "please report an issue to PyTorch"
                )
        elif isinstance(
            cause, torch._subclasses.fake_tensor.UnsupportedOperatorException
        ):
            op = cause.func
            import_suggestion = ""
            if isinstance(op, torch._ops.OpOverload):
                maybe_pystub = torch._C._dispatch_pystub(
                    op._schema.name, op._schema.overload_name
                )
                if maybe_pystub is not None:
                    module, ctx = maybe_pystub
                    import_suggestion = (
                        f"It's possible that the support was implemented in "
                        f"module `{module}` and you may need to `import {module}`"
                        f"({ctx}), otherwise "
                    )
            unimplemented(
                f"unsupported operator: {cause.func} ({import_suggestion}see "
                "https://docs.google.com/document/d/1GgvOe7C8_NVOMLOCwDaYV1mXXyHMXY7ExoewHqooxrs/edit#heading=h.64r4npvq0w0"
                " for how to fix)"
            )
        elif isinstance(
            cause, torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode
        ):

          raise UserError(  # noqa: B904

UserErrorType.CONSTRAINT_VIOLATION, "Tried to use data-dependent value in the subsequent computation. " "This can happen when we encounter unbounded dynamic value that is unknown during tracing time. " "You will need to explicitly give hint to the compiler. Please take a look at " f"torch._check OR torch._check_is_size APIs. {cause}", case_name="constrain_as_size_example", E torch._dynamo.exc.UserError: Tried to use data-dependent value in the subsequent computation. This can happen when we encounter unbounded dynamic value that is unknown during tracing time. You will need to explicitly give hint to the compiler. Please take a look at torch._check OR torch._check_is_size APIs. Could not guard on data-dependent expression Max(1, u0 + 1) > 1 (unhinted: Max(1, u0 + 1) > 1). (Size-like symbols: u0) E E ATTENTION: guard_size_oblivious would fix the error, evaluating expression to True. E Maybe you need to add guard_size_oblivious to framework code, see doc below for more guidance. E E Potential framework code culprit (scroll up for full backtrace): E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_prims_common/init.py", line 1840, in are_strides_like_channels_last E if strides[d] > 1: E E For more information, run with TORCH_LOGS="dynamic" E For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0" E If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1 E For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing E E User Stack (most recent call last): E (snipped, see stack below for prefix) E File "/local_data/andife/Spherinator/tests/test_onnx_export.py", line 37, in forward E return self.spherical.rsample(x.shape) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transformed_distribution.py", line 152, in rsample E x = transform(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transforms.py", line 156, in call E return self._call(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/power_spherical/distributions.py", line 19, in _call E return torch.cat((t, v * torch.sqrt(torch.clamp(1 - t * 2, _EPS))), -1) E E For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1 E For more information about this error, see: https://pytorch.org/docs/main/generated/exportdb/index.html#constrain-as-size-example E E from user code: E File "/local_data/andife/Spherinator/tests/test_onnx_export.py", line 37, in forward E return self.spherical.rsample(x.shape) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transformed_distribution.py", line 152, in rsample E x = transform(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/distributions/transforms.py", line 156, in call E return self._call(x) E File "/local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/power_spherical/distributions.py", line 19, in _call E return torch.cat((t, v torch.sqrt(torch.clamp(1 - t ** 2, _EPS))), -1) E E Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information

../miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/_dynamo/utils.py:1841: UserError ------------------------------------------------------------------------------------------------------------------------------------ Captured stderr call ------------------------------------------------------------------------------------------------------------------------------------ E0606 07:20:09.492000 132424664700736 torch/fx/experimental/recording.py:280] [1/0] failed while running evaluate_expr(*(Max(1, u0 + 1) > 1, None), **{'fx_node': None}) ====================================================================================================================================== warnings summary ====================================================================================================================================== tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[2-exception0] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[5-exception1] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[samples2-exception2] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[samples3-exception3] /local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/lightning/pytorch/trainer/setup.py:187: GPU available but not used. You can set it by doing Trainer(accelerator='gpu').

tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[2-exception0] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[5-exception1] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[samples2-exception2] tests/test_log_reconstruction_callback.py::test_on_train_epoch_end[samples3-exception3] /local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:268: You requested to overfit but enabled train dataloader shuffling. We are turning off the train dataloader shuffling for you.

tests/test_onnx_export.py::test_dynamo_export_normal /local_data/andife/miniforge3/envs/af20240530_poetry/lib/python3.11/site-packages/torch/onnx/_internal/exporter.py:136: UserWarning: torch.onnx.dynamo_export only implements opset version 18 for now. If you need to use a different opset version, please register them with register_custom_op. warnings.warn(

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html ================================================================================================================================== short test summary info =================================================================================================================================== FAILED tests/test_onnx_export.py::test_dynamo_export_spherical - torch._dynamo.exc.UserError: Tried to use data-dependent value in the subsequent computation. This can happen when we encounter unbounded dynamic value that is unknown during tracing time. You will need to explicitly give hint to the compiler. Please take a look at torch._check O...

[andife]

Currently aten._sample_dirichlet.default is not supported. See https://github.com/pytorch/pytorch/issues/116336