Last active
October 31, 2023 21:10
-
-
Save davidberard98/c8522c51d11ded74852631d18df8623b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Traceback (most recent call last): | |
File "/data/users/dberard/scripts/oncall/112489.py", line 8, in <module> | |
fn_opt(*inputs) | |
File "/data/users/dberard/pytorch/torch/_dynamo/eval_frame.py", line 411, in _fn | |
return fn(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/_dynamo/eval_frame.py", line 559, in catch_errors | |
return callback(frame, cache_entry, hooks, frame_state) | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 687, in _convert_frame | |
result = inner_convert(frame, cache_entry, hooks, frame_state) | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 148, in _fn | |
return fn(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 406, in _convert_frame_assert | |
return _compile( | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 614, in _compile | |
guarded_code = compile_inner(code, one_graph, hooks, transform) | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 221, in time_wrapper | |
r = func(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 531, in compile_inner | |
out_code = transform_code_object(code, transform) | |
File "/data/users/dberard/pytorch/torch/_dynamo/bytecode_transformation.py", line 1028, in transform_code_object | |
transformations(instructions, code_options) | |
File "/data/users/dberard/pytorch/torch/_dynamo/convert_frame.py", line 501, in transform | |
tracer.run() | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 2119, in run | |
super().run() | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 752, in run | |
and self.step() | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 715, in step | |
getattr(self, inst.opname)(inst) | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 405, in wrapper | |
return inner_fn(self, inst) | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 1200, in CALL_FUNCTION_KW | |
self.call_function(fn, args, kwargs) | |
File "/data/users/dberard/pytorch/torch/_dynamo/symbolic_convert.py", line 586, in call_function | |
self.push(fn.call_function(self, args, kwargs)) | |
File "/data/users/dberard/pytorch/torch/_dynamo/variables/torch.py", line 722, in call_function | |
tensor_variable = wrap_fx_proxy( | |
File "/data/users/dberard/pytorch/torch/_dynamo/variables/builder.py", line 1341, in wrap_fx_proxy | |
return wrap_fx_proxy_cls( | |
File "/data/users/dberard/pytorch/torch/_dynamo/variables/builder.py", line 1431, in wrap_fx_proxy_cls | |
example_value = get_fake_value(proxy.node, tx, allow_non_graph_fake=True) | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 1483, in get_fake_value | |
raise TorchRuntimeError(str(e)).with_traceback(e.__traceback__) from None | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 1444, in get_fake_value | |
ret_val = wrap_fake_exception( | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 993, in wrap_fake_exception | |
return fn() | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 1445, in <lambda> | |
lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 1548, in run_node | |
raise RuntimeError(fn_str + str(e)).with_traceback(e.__traceback__) from e | |
File "/data/users/dberard/pytorch/torch/_dynamo/utils.py", line 1527, in run_node | |
return node.target(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/utils/_stats.py", line 20, in wrapper | |
return fn(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/_subclasses/fake_tensor.py", line 1378, in __torch_dispatch__ | |
return self.dispatch(func, types, args, kwargs) | |
File "/data/users/dberard/pytorch/torch/_subclasses/fake_tensor.py", line 1676, in dispatch | |
r = func(*args, **kwargs) | |
File "/data/users/dberard/pytorch/torch/_ops.py", line 513, in __call__ | |
return self._op(*args, **kwargs or {}) | |
torch._dynamo.exc.TorchRuntimeError: Failed running call_function <built-in method mm of type object at 0x7f45f0feaf40>(*(FakeTensor(..., size=(s0, s0)), FakeTensor(..., size=(s0, s0))), **{'out': FakeTensor(..., size=(s0, s0))}): | |
Cannot call sizes() on tensor with symbolic sizes/strides | |
Exception raised from throw_cannot_call_with_symbolic at ../c10/core/TensorImpl.cpp:593 (most recent call first): | |
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x95 (0x7f45f1909475 in /data/users/dberard/pytorch/torch/lib/libc10.so) | |
frame #1: c10::TensorImpl::throw_cannot_call_with_symbolic(char const*) const + 0x8d (0x7f45f18bced9 in /data/users/dberard/pytorch/torch/lib/libc10.so) | |
frame #2: c10::TensorImpl::sizes_custom() const + 0x9f (0x7f45f18e2cbf in /data/users/dberard/pytorch/torch/lib/libc10.so) | |
frame #3: at::meta::structured_mm::meta(at::Tensor const&, at::Tensor const&) + 0x31e (0x7f45e679e12e in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #4: <unknown function> + 0x29dbf2e (0x7f45e7bdbf2e in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #5: <unknown function> + 0x2a087de (0x7f45e7c087de in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #6: <unknown function> + 0x69f89b (0x7f45f069f89b in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #7: <unknown function> + 0x69c995 (0x7f45f069c995 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #8: <unknown function> + 0x49c45d4 (0x7f45e9bc45d4 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #9: torch::jit::invokeOperatorFromPython(std::vector<std::shared_ptr<torch::jit::Operator>, std::allocator<std::shared_ptr<torch::jit::Operator> > > const&, pybind11::args, pybind11::kwargs const&, c10::optional<c10::DispatchKey>) + 0x334 (0x7f45f08a4de4 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #10: torch::jit::_get_operation_for_overload_or_packet(std::vector<std::shared_ptr<torch::jit::Operator>, std::allocator<std::shared_ptr<torch::jit::Operator> > > const&, c10::Symbol, pybind11::args, pybind11::kwargs const&, bool, c10::optional<c10::DispatchKey>) + 0x69f (0x7f45f08a562f in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #11: <unknown function> + 0x78f9e9 (0x7f45f078f9e9 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #12: <unknown function> + 0x3b8f57 (0x7f45f03b8f57 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #13: python() [0x507507] | |
<omitting python frames> | |
frame #16: python() [0x4e67ea] | |
frame #20: python() [0x5cb013] | |
frame #23: python() [0x4e67ea] | |
frame #26: python() [0x4e67ea] | |
frame #29: python() [0x4e67ea] | |
frame #31: python() [0x5052b1] | |
frame #32: python() [0x595396] | |
frame #34: torch::handle_torch_function_no_python_arg_parser(c10::ArrayRef<_object*>, _object*, _object*, char const*, _object*, char const*, torch::TorchFunctionName) + 0x423 (0x7f45f0addb13 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #35: <unknown function> + 0x69aea3 (0x7f45f069aea3 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #36: <unknown function> + 0x134c8c9 (0x7f45e654c8c9 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #37: <unknown function> + 0x69f89b (0x7f45f069f89b in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #38: <unknown function> + 0x69c995 (0x7f45f069c995 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #39: <unknown function> + 0x18b6fa6 (0x7f45e6ab6fa6 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #40: <unknown function> + 0x3fe4711 (0x7f45e91e4711 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #41: <unknown function> + 0x69f89b (0x7f45f069f89b in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #42: <unknown function> + 0x69c995 (0x7f45f069c995 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #43: <unknown function> + 0x18b6fa6 (0x7f45e6ab6fa6 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #44: <unknown function> + 0x38e36ae (0x7f45e8ae36ae in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #45: <unknown function> + 0x38ec671 (0x7f45e8aec671 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #46: <unknown function> + 0x69f89b (0x7f45f069f89b in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #47: <unknown function> + 0x69c995 (0x7f45f069c995 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #48: <unknown function> + 0x134c77b (0x7f45e654c77b in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #49: <unknown function> + 0x69f89b (0x7f45f069f89b in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #50: <unknown function> + 0x69c995 (0x7f45f069c995 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #51: <unknown function> + 0x18b6fa6 (0x7f45e6ab6fa6 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #52: at::_ops::mm_out::call(at::Tensor const&, at::Tensor const&, at::Tensor&) + 0x307 (0x7f45e7336d27 in /data/users/dberard/pytorch/torch/lib/libtorch_cpu.so) | |
frame #53: <unknown function> + 0x51b6f3 (0x7f45f051b6f3 in /data/users/dberard/pytorch/torch/lib/libtorch_python.so) | |
frame #54: python() [0x507507] | |
frame #57: python() [0x4f7ec3] | |
frame #59: python() [0x4e67ea] | |
frame #62: python() [0x4f7ec3] | |
from user code: | |
File "/data/users/dberard/scripts/oncall/112489.py", line 4, in fn | |
return torch.mm(x, y, out=z) | |
Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
You can suppress this exception and fall back to eager by setting: | |
import torch._dynamo | |
torch._dynamo.config.suppress_errors = True | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment