2025-01-02 14:09:46,463 ERROR [train.py:286] Node[0] train failed! '_update_res'
Traceback (most recent call last):
File "tools/train.py", line 287, in
raise e
File "tools/train.py", line 273, in
train(
File "tools/train.py", line 254, in train
launch(
File "/usr/local/lib/python3.8/dist-packages/hat/engine/calibrator.py", line 136, in launch
main_func(current_device, *args)
File "tools/train.py", line 185, in train_entrance
trainer = build_from_registry(trainer)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 417, in build_from_registry
return _impl(x)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 400, in _impl
obj = build_from_cfg(OBJECT_REGISTRY, x)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 253, in build_from_cfg
instance = obj_cls(**cfg)
File "/usr/local/lib/python3.8/dist-packages/hat/engine/calibrator.py", line 61, in init
super(Calibrator, self).init(
File "/usr/local/lib/python3.8/dist-packages/hat/engine/loop_base.py", line 258, in init
self.model = model_convert_pipeline(self.model)
File "/usr/local/lib/python3.8/dist-packages/hat/models/model_convert/pipelines.py", line 52, in call
model = converter(model)
File "/usr/local/lib/python3.8/dist-packages/hat/models/model_convert/converters.py", line 268, in call
model = horizon.quantization.prepare_qat_fx(
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/utils/typeguard.py", line 1076, in wrapper
retval = func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/quantization/quantize_fx.py", line 492, in prepare_qat_fx
return _prepare_fx(
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/quantization/quantize_fx.py", line 252, in _prepare_fx
graph = tracer.trace(model)
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/utils/fx_helper.py", line 290, in trace
return super(CustomTracer, self).trace(root, *args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/torch/fx/_symbolic_trace.py", line 730, in trace
_patch_wrapped_functions(patcher)
File "/usr/local/lib/python3.8/dist-packages/torch/fx/_symbolic_trace.py", line 932, in _patch_wrapped_functions
orig_fn = frame_dict[name]
KeyError: '_update_res'
Traceback (most recent call last):
File "tools/train.py", line 287, in
raise e
File "tools/train.py", line 273, in
train(
File "tools/train.py", line 254, in train
launch(
File "/usr/local/lib/python3.8/dist-packages/hat/engine/calibrator.py", line 136, in launch
main_func(current_device, *args)
File "tools/train.py", line 185, in train_entrance
trainer = build_from_registry(trainer)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 417, in build_from_registry
return _impl(x)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 400, in _impl
obj = build_from_cfg(OBJECT_REGISTRY, x)
File "/usr/local/lib/python3.8/dist-packages/hat/registry.py", line 253, in build_from_cfg
instance = obj_cls(**cfg)
File "/usr/local/lib/python3.8/dist-packages/hat/engine/calibrator.py", line 61, in init
super(Calibrator, self).init(
File "/usr/local/lib/python3.8/dist-packages/hat/engine/loop_base.py", line 258, in init
self.model = model_convert_pipeline(self.model)
File "/usr/local/lib/python3.8/dist-packages/hat/models/model_convert/pipelines.py", line 52, in call
model = converter(model)
File "/usr/local/lib/python3.8/dist-packages/hat/models/model_convert/converters.py", line 268, in call
model = horizon.quantization.prepare_qat_fx(
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/utils/typeguard.py", line 1076, in wrapper
retval = func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/quantization/quantize_fx.py", line 492, in prepare_qat_fx
return _prepare_fx(
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/quantization/quantize_fx.py", line 252, in _prepare_fx
graph = tracer.trace(model)
File "/usr/local/lib/python3.8/dist-packages/horizon_plugin_pytorch/utils/fx_helper.py", line 290, in trace
return super(CustomTracer, self).trace(root, *args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/torch/fx/_symbolic_trace.py", line 730, in trace
_patch_wrapped_functions(patcher)
File "/usr/local/lib/python3.8/dist-packages/torch/fx/_symbolic_trace.py", line 932, in _patch_wrapped_functions
orig_fn = frame_dict[name]
KeyError: '_update_res'


