SystemError: _PyEval_EvalFrameDefault returned NULL without setting an exception #515
Description
Python Debug Console in VS Code report:
Epoch 0: | | 556/? [02:27<00:00, 3.77it/s]Traceback (most recent call last):
File "/home/jane/anaconda3/envs/3s/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/jane/anaconda3/envs/3s/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/jane/.vscode/extensions/ms-python.python-2022.4.0/pythonFiles/lib/python/debugpy/__main__.py", line 45, in <module>
cli.main()
File "/home/jane/.vscode/extensions/ms-python.python-2022.4.0/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 444, in main
run()
File "/home/jane/.vscode/extensions/ms-python.python-2022.4.0/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 285, in run_file
runpy.run_path(target_as_str, run_name=compat.force_str("__main__"))
File "/home/jane/anaconda3/envs/3s/lib/python3.10/runpy.py", line 289, in run_path
return _run_module_code(code, init_globals, run_name,
File "/home/jane/anaconda3/envs/3s/lib/python3.10/runpy.py", line 96, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "/home/jane/anaconda3/envs/3s/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/jane/Desktop/threestudio/launch.py", line 311, in <module>
main(args, extras)
File "/home/jane/Desktop/threestudio/launch.py", line 254, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 538, in fit
call._call_and_handle_interrupt(
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 574, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 981, in _run
results = self._run_stage()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1025, in _run_stage
self.fit_loop.run()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 205, in run
self.advance()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 363, in advance
self.epoch_loop.run(self._data_fetcher)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 140, in run
self.advance(data_fetcher)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 250, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 190, in run
self._optimizer_step(batch_idx, closure)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 268, in _optimizer_step
call._call_lightning_module_hook(
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 167, in _call_lightning_module_hook
output = fn(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1306, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 153, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 238, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision.py", line 122, in optimizer_step
return optimizer.step(closure=closure, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/optim/optimizer.py", line 484, in wrapper
out = func(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/optim/optimizer.py", line 89, in _use_grad
ret = func(self, *args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/optim/adam.py", line 205, in step
loss = closure()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision.py", line 108, in _wrap_closure
closure_result = closure()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 144, in __call__
self._result = self.closure(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 129, in closure
step_output = self._step_fn()
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 317, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 319, in _call_strategy_hook
output = fn(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 390, in training_step
return self.lightning_module.training_step(*args, **kwargs)
File "/home/jane/Desktop/threestudio/threestudio/systems/magic123.py", line 52, in training_step
guidance_3d_out = self.guidance_3d(
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
File "/home/jane/Desktop/threestudio/threestudio/models/guidance/zero123_unified_guidance.py", line 564, in forward
eps_pretrain = self.get_eps_pretrain(
File "/home/jane/Desktop/threestudio/threestudio/models/guidance/zero123_unified_guidance.py", line 370, in get_eps_pretrain
noise_pred = self.forward_unet(
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 43, in decorate_autocast
return func(*args, **kwargs)
File "/home/jane/Desktop/threestudio/threestudio/models/guidance/zero123_unified_guidance.py", line 304, in forward_unet
encoder_hidden_states=encoder_hidden_states.to(unet.dtype),
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/diffusers/models/modeling_utils.py", line 848, in dtype
return get_parameter_dtype(self)
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/diffusers/models/modeling_utils.py", line 81, in get_parameter_dtype
params = tuple(parameter.parameters())
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2255, in parameters
for name, param in self.named_parameters(recurse=recurse):
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2288, in named_parameters
yield from gen
File "/home/jane/anaconda3/envs/3s/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2223, in _named_members
for module_prefix, module in modules:
SystemError: _PyEval_EvalFrameDefault returned NULL without setting an exception
Epoch 0: | | 556/? [02:29<00:00, 3.71it/s]
It occurs sometimes and i can not solve it, help plz orz