RuntimeError: Placeholder storage has not been allocated on MPS device!

[kkckk1110]

---

RuntimeError Traceback (most recent call last) Cell In[22], line 1 ----> 1 zeroshot_trainer.evaluate(test_dataset)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/transformers/trainer.py:3641, in Trainer.evaluate(self, eval_dataset, ignore_keys, metric_key_prefix) 3638 start_time = time.time() 3640 eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop -> 3641 output = eval_loop( 3642 eval_dataloader, 3643 description="Evaluation", 3644 # No point gathering the predictions if there are no metrics, otherwise we defer to 3645 # self.args.prediction_loss_only 3646 prediction_loss_only=True if self.compute_metrics is None else None, 3647 ignore_keys=ignore_keys, 3648 metric_key_prefix=metric_key_prefix, 3649 ) 3651 total_batch_size = self.args.eval_batch_size * self.args.world_size 3652 if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/transformers/trainer.py:3826, in Trainer.evaluation_loop(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix) 3823 batch_size = observed_batch_size 3825 # Prediction step -> 3826 losses, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys) 3827 main_input_name = getattr(self.model, "main_input_name", "input_ids") 3828 inputs_decode = self._prepare_input(inputs[main_input_name]) if args.include_inputs_for_metrics else None

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/transformers/trainer.py:4040, in Trainer.prediction_step(self, model, inputs, prediction_loss_only, ignore_keys) 4038 if has_labels or loss_without_labels: 4039 with self.compute_loss_context_manager(): -> 4040 loss, outputs = self.compute_loss(model, inputs, return_outputs=True) 4041 loss = loss.mean().detach() 4043 if isinstance(outputs, dict):

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/transformers/trainer.py:3338, in Trainer.compute_loss(self, model, inputs, return_outputs) 3336 else: 3337 labels = None -> 3338 outputs = model(**inputs) 3339 # Save past state if it exists 3340 # TODO: this needs to be fixed and made cleaner later. 3341 if self.args.past_index >= 0:

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, kwargs) 1530 return self._compiled_call_impl(*args, *kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(args, kwargs)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, *kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(args, **kwargs) 1543 try: 1544 result = None

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/tsfm_public/models/tinytimemixer/modeling_tinytimemixer.py:1473, in TinyTimeMixerForPrediction.forward(self, past_values, future_values, observed_mask, output_hidden_states, return_loss, return_dict, freq_token) 1470 return_dict = return_dict if return_dict is not None else self.use_return_dict 1472 # past_values: tensor [batch_size x context_length x num_input_channels] -> 1473 model_output = self.backbone( 1474 past_values, 1475 observed_mask=observed_mask, 1476 output_hidden_states=output_hidden_states, 1477 return_dict=return_dict, 1478 freq_token=freq_token, 1479 ) # model_output: [batch_size x nvars x num_patch x d_model] 1481 if isinstance(model_output, tuple): 1482 model_output = TinyTimeMixerModelOutput(*model_output)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, kwargs) 1530 return self._compiled_call_impl(*args, *kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(args, kwargs)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, *kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(args, **kwargs) 1543 try: 1544 result = None

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/tsfm_public/models/tinytimemixer/modeling_tinytimemixer.py:1279, in TinyTimeMixerModel.forward(self, past_values, observed_mask, output_hidden_states, return_dict, freq_token) 1275 patched_x = self.patching(scaled_past_values) # [batch_size x num_input_channels x num_patch x patch_length 1277 enc_input = patched_x -> 1279 encoder_output = self.encoder( 1280 enc_input, 1281 output_hidden_states=output_hidden_states, 1282 return_dict=return_dict, 1283 freq_token=freq_token, 1284 ) 1286 if isinstance(encoder_output, tuple): 1287 encoder_output = TinyTimeMixerEncoderOutput(*encoder_output)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, kwargs) 1530 return self._compiled_call_impl(*args, *kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(args, kwargs)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, *kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(args, **kwargs) 1543 try: 1544 result = None

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/tsfm_public/models/tinytimemixer/modeling_tinytimemixer.py:1158, in TinyTimeMixerEncoder.forward(self, past_values, output_hidden_states, return_dict, freq_token) 1155 return_dict = return_dict if return_dict is not None else self.use_return_dict 1157 # flatten [bs x num_patch x d_model]. common_channel/mix_channel: [bs x n_vars x num_patch x d_model] -> 1158 patches = self.patcher(past_values) 1160 if self.resolution_prefix_tuning: 1161 if freq_token is not None:

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, kwargs) 1530 return self._compiled_call_impl(*args, *kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(args, kwargs)

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, *kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(args, **kwargs) 1543 try: 1544 result = None

File ~/miniforge3/envs/TTM/lib/python3.9/site-packages/torch/nn/modules/linear.py:116, in Linear.forward(self, input) 115 def forward(self, input: Tensor) -> Tensor: --> 116 return F.linear(input, self.weight, self.bias)

RuntimeError: Placeholder storage has not been allocated on MPS device!

[wgifford]

@kkckk1110 Thanks for the stack trace! Can you provide a little more detail around what notebook / python code you were trying to run?

[kkckk1110]

Of course! The code is zeroshot_trainer.evaluate(test_dataset) which is in the ttm_tutorial.ipynb

[wgifford]

Occasionally, I see similar issues on my macbook pro M2. This usually stems from the torch dataset test_dataset having the tensors on the cpu device, while the model seems to be placed on the MPS device.

Easiest way around it is to move the model to the CPU device (i.e., zeroshot_model.to("cpu")). In the future, we may add an option during torch dataset creation to specify the device.