AttributeError: 'NoneType' object has no attribute 'device'

[keeper-jie]

I got the error as blow and hope someone can solve it. I have change the device_map(such as "balanced", "balanced_low_0", "sequential") in

model = LLaMAForCausalLM.from_pretrained(
    "decapoda-research/llama-7b-hf",
    load_in_8bit=True,
    device_map="auto",
)

but not working.

Error

evaluate(input("Instruction: ")) # how to learn english

    Instruction: how to learn english

    ---------------------------------------------------------------------------

    AttributeError                            Traceback (most recent call last)

    Cell In[16], line 1
    ----> 1 evaluate(input("Instruction: "))

    Cell In[15], line 11, in evaluate(instruction, input)
          9 inputs = tokenizer(prompt, return_tensors="pt")
         10 input_ids = inputs["input_ids"].cuda()
    ---> 11 generation_output = model.generate(
         12     input_ids=input_ids,
         13     generation_config=generation_config,
         14     return_dict_in_generate=True,
         15     output_scores=True,
         16     max_new_tokens=256
         17 )
         18 for s in generation_output.sequences:
         19     output = tokenizer.decode(s)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/peft/peft_model.py:581, in PeftModelForCausalLM.generate(self, **kwargs)
        579 try:
        580     if not isinstance(self.peft_config, PromptLearningConfig):
    --> 581         outputs = self.base_model.generate(**kwargs)
        582     else:
        583         if "input_ids" not in kwargs:

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
        112 @functools.wraps(func)
        113 def decorate_context(*args, **kwargs):
        114     with ctx_factory():
    --> 115         return func(*args, **kwargs)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/generation/utils.py:1490, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, **kwargs)
       1483     input_ids, model_kwargs = self._expand_inputs_for_generation(
       1484         input_ids=input_ids,
       1485         expand_size=generation_config.num_beams,
       1486         is_encoder_decoder=self.config.is_encoder_decoder,
       1487         **model_kwargs,
       1488     )
       1489     # 13. run beam search
    -> 1490     return self.beam_search(
       1491         input_ids,
       1492         beam_scorer,
       1493         logits_processor=logits_processor,
       1494         stopping_criteria=stopping_criteria,
       1495         pad_token_id=generation_config.pad_token_id,
       1496         eos_token_id=generation_config.eos_token_id,
       1497         output_scores=generation_config.output_scores,
       1498         return_dict_in_generate=generation_config.return_dict_in_generate,
       1499         synced_gpus=synced_gpus,
       1500         **model_kwargs,
       1501     )
       1503 elif is_beam_sample_gen_mode:
       1504     # 11. prepare logits warper
       1505     logits_warper = self._get_logits_warper(generation_config)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/generation/utils.py:2749, in GenerationMixin.beam_search(self, input_ids, beam_scorer, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, **model_kwargs)
       2745         break
       2747 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
    -> 2749 outputs = self(
       2750     **model_inputs,
       2751     return_dict=True,
       2752     output_attentions=output_attentions,
       2753     output_hidden_states=output_hidden_states,
       2754 )
       2756 if synced_gpus and this_peer_finished:
       2757     cur_len = cur_len + 1

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
       1496 # If we don't have any hooks, we want to skip the rest of the logic in
       1497 # this function, and just call forward.
       1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
       1499         or _global_backward_pre_hooks or _global_backward_hooks
       1500         or _global_forward_hooks or _global_forward_pre_hooks):
    -> 1501     return forward_call(*args, **kwargs)
       1502 # Do not call functions when jit is used
       1503 full_backward_hooks, non_full_backward_hooks = [], []

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
        163         output = old_forward(*args, **kwargs)
        164 else:
    --> 165     output = old_forward(*args, **kwargs)
        166 return module._hf_hook.post_forward(module, output)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py:770, in LLaMAForCausalLM.forward(self, input_ids, attention_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)
        767 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        769 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    --> 770 outputs = self.model(
        771     input_ids=input_ids,
        772     attention_mask=attention_mask,
        773     past_key_values=past_key_values,
        774     inputs_embeds=inputs_embeds,
        775     use_cache=use_cache,
        776     output_attentions=output_attentions,
        777     output_hidden_states=output_hidden_states,
        778     return_dict=return_dict,
        779 )
        781 hidden_states = outputs[0]
        782 logits = self.lm_head(hidden_states)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
       1496 # If we don't have any hooks, we want to skip the rest of the logic in
       1497 # this function, and just call forward.
       1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
       1499         or _global_backward_pre_hooks or _global_backward_hooks
       1500         or _global_forward_hooks or _global_forward_pre_hooks):
    -> 1501     return forward_call(*args, **kwargs)
       1502 # Do not call functions when jit is used
       1503 full_backward_hooks, non_full_backward_hooks = [], []

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py:619, in LLaMAModel.forward(self, input_ids, attention_mask, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)
        612     layer_outputs = torch.utils.checkpoint.checkpoint(
        613         create_custom_forward(decoder_layer),
        614         hidden_states,
        615         attention_mask,
        616         None,
        617     )
        618 else:
    --> 619     layer_outputs = decoder_layer(
        620         hidden_states,
        621         attention_mask=attention_mask,
        622         past_key_value=past_key_value,
        623         output_attentions=output_attentions,
        624         use_cache=use_cache,
        625     )
        627 hidden_states = layer_outputs[0]
        629 if use_cache:

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
       1496 # If we don't have any hooks, we want to skip the rest of the logic in
       1497 # this function, and just call forward.
       1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
       1499         or _global_backward_pre_hooks or _global_backward_hooks
       1500         or _global_forward_hooks or _global_forward_pre_hooks):
    -> 1501     return forward_call(*args, **kwargs)
       1502 # Do not call functions when jit is used
       1503 full_backward_hooks, non_full_backward_hooks = [], []

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
        163         output = old_forward(*args, **kwargs)
        164 else:
    --> 165     output = old_forward(*args, **kwargs)
        166 return module._hf_hook.post_forward(module, output)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py:316, in LLaMADecoderLayer.forward(self, hidden_states, attention_mask, output_attentions, use_cache, past_key_value)
        313 hidden_states = self.input_layernorm(hidden_states)
        315 # Self Attention
    --> 316 hidden_states, self_attn_weights, present_key_value = self.self_attn(
        317     hidden_states=hidden_states,
        318     past_key_value=past_key_value,
        319     attention_mask=attention_mask,
        320     output_attentions=output_attentions,
        321 )
        322 hidden_states = residual + hidden_states
        324 # Fully Connected

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
       1496 # If we don't have any hooks, we want to skip the rest of the logic in
       1497 # this function, and just call forward.
       1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
       1499         or _global_backward_pre_hooks or _global_backward_hooks
       1500         or _global_forward_hooks or _global_forward_pre_hooks):
    -> 1501     return forward_call(*args, **kwargs)
       1502 # Do not call functions when jit is used
       1503 full_backward_hooks, non_full_backward_hooks = [], []

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
        163         output = old_forward(*args, **kwargs)
        164 else:
    --> 165     output = old_forward(*args, **kwargs)
        166 return module._hf_hook.post_forward(module, output)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py:216, in LLaMAAttention.forward(self, hidden_states, past_key_value, attention_mask, output_attentions)
        212 """Input shape: Batch x Time x Channel"""
        214 bsz, q_len, _ = hidden_states.size()
    --> 216 query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
        217 key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
        218 value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
       1496 # If we don't have any hooks, we want to skip the rest of the logic in
       1497 # this function, and just call forward.
       1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
       1499         or _global_backward_pre_hooks or _global_backward_hooks
       1500         or _global_forward_hooks or _global_forward_pre_hooks):
    -> 1501     return forward_call(*args, **kwargs)
       1502 # Do not call functions when jit is used
       1503 full_backward_hooks, non_full_backward_hooks = [], []

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
        163         output = old_forward(*args, **kwargs)
        164 else:
    --> 165     output = old_forward(*args, **kwargs)
        166 return module._hf_hook.post_forward(module, output)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/peft/tuners/lora.py:522, in Linear8bitLt.forward(self, x)
        521 def forward(self, x: torch.Tensor):
    --> 522     result = super().forward(x)
        524     if self.disable_adapters:
        525         return result

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/bitsandbytes/nn/modules.py:242, in Linear8bitLt.forward(self, x)
        239 if self.bias is not None and self.bias.dtype != x.dtype:
        240     self.bias.data = self.bias.data.to(x.dtype)
    --> 242 out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)
        243 if not self.state.has_fp16_weights:
        244     if self.state.CB is not None and self.state.CxB is not None:
        245         # we converted 8-bit row major to turing/ampere format in the first inference pass
        246         # we no longer need the row-major weight

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/bitsandbytes/autograd/_functions.py:488, in matmul(A, B, out, state, threshold, bias)
        486 if threshold > 0.0:
        487     state.threshold = threshold
    --> 488 return MatMul8bitLt.apply(A, B, out, bias, state)

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/torch/autograd/function.py:506, in Function.apply(cls, *args, **kwargs)
        503 if not torch._C._are_functorch_transforms_active():
        504     # See NOTE: [functorch vjp and autograd interaction]
        505     args = _functorch.utils.unwrap_dead_wrappers(args)
    --> 506     return super().apply(*args, **kwargs)  # type: ignore[misc]
        508 if cls.setup_context == _SingleLevelFunction.setup_context:
        509     raise RuntimeError(
        510         'In order to use an autograd.Function with functorch transforms '
        511         '(vmap, grad, jvp, jacrev, ...), it must override the setup_context '
        512         'staticmethod. For more details, please see '
        513         'https://pytorch.org/docs/master/notes/extending.func.html')

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/bitsandbytes/autograd/_functions.py:317, in MatMul8bitLt.forward(ctx, A, B, out, bias, state)
        313     else:
        314         if state.CxB is None and using_igemmlt:
        315             # B in in 8-bit row-major, we can transform it back to 16-bit to extract outlier dimensions
        316             # we also need to convert it to the turing/ampere format
    --> 317             state.CxB, state.SB = F.transform(state.CB, to_order=formatB)
        318 else:
        319     if not state.has_fp16_weights and state.CxB is None and using_igemmlt:

    File ~/miniconda3/envs/LiuJieTest/lib/python3.9/site-packages/bitsandbytes/functional.py:1698, in transform(A, to_order, from_order, out, transpose, state, ld)
       1697 def transform(A, to_order, from_order='row', out=None, transpose=False, state=None, ld=None):
    -> 1698     prev_device = pre_call(A.device)
       1699     if state is None: state = (A.shape, from_order)
       1700     else: from_order = state[1]

    AttributeError: 'NoneType' object has no attribute 'device'

Enviroment:

1. !nvidia-smi

Thu Mar 16 16:37:03 2023
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 510.39.01    Driver Version: 510.39.01    CUDA Version: 11.6     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  Off  | 00000000:1A:00.0 Off |                  N/A |
| 31%   32C    P2    50W / 250W |   8012MiB / 11264MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
|   1  NVIDIA GeForce ...  Off  | 00000000:3D:00.0 Off |                  N/A |
| 29%   31C    P2    50W / 250W |   4350MiB / 11264MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      3903      C   ...nvs/LiuJieTest/bin/python     8009MiB |
|    1   N/A  N/A      3903      C   ...nvs/LiuJieTest/bin/python     4347MiB |
+-----------------------------------------------------------------------------+

2. conda list

   # packages in environment at /home/zhuji/miniconda3/envs/LiuJieTest:
   #
   # Name                    Version                   Build  Channel
   _libgcc_mutex             0.1                        main  
   _openmp_mutex             5.1                       1_gnu  
   accelerate                0.17.1                   pypi_0    pypi
   aiohttp                   3.8.4                    pypi_0    pypi
   aiosignal                 1.3.1                    pypi_0    pypi
   anyio                     3.6.2                    pypi_0    pypi
   argon2-cffi               21.3.0                   pypi_0    pypi
   argon2-cffi-bindings      21.2.0                   pypi_0    pypi
   arrow                     1.2.3                    pypi_0    pypi
   asgiref                   3.6.0                    pypi_0    pypi
   asttokens                 2.2.1                    pypi_0    pypi
   async-timeout             4.0.2                    pypi_0    pypi
   attrs                     22.2.0                   pypi_0    pypi
   backcall                  0.2.0                    pypi_0    pypi
   beautifulsoup4            4.11.2                   pypi_0    pypi
   bitsandbytes              0.37.1                   pypi_0    pypi
   bleach                    6.0.0                    pypi_0    pypi
   ca-certificates           2023.01.10           h06a4308_0  
   certifi                   2022.12.7        py39h06a4308_0  
   cffi                      1.15.1                   pypi_0    pypi
   charset-normalizer        3.1.0                    pypi_0    pypi
   cmake                     3.26.0                   pypi_0    pypi
   comm                      0.1.2                    pypi_0    pypi
   datasets                  2.10.1                   pypi_0    pypi
   debugpy                   1.6.6                    pypi_0    pypi
   decorator                 5.1.1                    pypi_0    pypi
   defusedxml                0.7.1                    pypi_0    pypi
   dill                      0.3.6                    pypi_0    pypi
   django                    4.1.7                    pypi_0    pypi
   executing                 1.2.0                    pypi_0    pypi
   fastjsonschema            2.16.3                   pypi_0    pypi
   filelock                  3.10.0                   pypi_0    pypi
   fqdn                      1.5.1                    pypi_0    pypi
   frozenlist                1.3.3                    pypi_0    pypi
   fsspec                    2023.3.0                 pypi_0    pypi
   huggingface-hub           0.13.2                   pypi_0    pypi
   idna                      3.4                      pypi_0    pypi
   importlib-metadata        6.0.0                    pypi_0    pypi
   ipykernel                 6.21.3                   pypi_0    pypi
   ipython                   8.11.0                   pypi_0    pypi
   ipython-genutils          0.2.0                    pypi_0    pypi
   ipywidgets                8.0.4                    pypi_0    pypi
   isoduration               20.11.0                  pypi_0    pypi
   jedi                      0.18.2                   pypi_0    pypi
   jinja2                    3.1.2                    pypi_0    pypi
   jsonpointer               2.3                      pypi_0    pypi
   jsonschema                4.17.3                   pypi_0    pypi
   jupyter                   1.0.0                    pypi_0    pypi
   jupyter-client            8.0.3                    pypi_0    pypi
   jupyter-console           6.6.3                    pypi_0    pypi
   jupyter-core              5.2.0                    pypi_0    pypi
   jupyter-events            0.6.3                    pypi_0    pypi
   jupyter-server            2.4.0                    pypi_0    pypi
   jupyter-server-terminals  0.4.4                    pypi_0    pypi
   jupyterlab-pygments       0.2.2                    pypi_0    pypi
   jupyterlab-widgets        3.0.5                    pypi_0    pypi
   ld_impl_linux-64          2.38                 h1181459_1  
   libffi                    3.4.2                h6a678d5_6  
   libgcc-ng                 11.2.0               h1234567_1  
   libgomp                   11.2.0               h1234567_1  
   libstdcxx-ng              11.2.0               h1234567_1  
   lit                       15.0.7                   pypi_0    pypi
   loralib                   0.1.1                    pypi_0    pypi
   markupsafe                2.1.2                    pypi_0    pypi
   matplotlib-inline         0.1.6                    pypi_0    pypi
   mistune                   2.0.5                    pypi_0    pypi
   mpmath                    1.3.0                    pypi_0    pypi
   multidict                 6.0.4                    pypi_0    pypi
   multiprocess              0.70.14                  pypi_0    pypi
   nbclassic                 0.5.3                    pypi_0    pypi
   nbclient                  0.7.2                    pypi_0    pypi
   nbconvert                 7.2.10                   pypi_0    pypi
   nbformat                  5.7.3                    pypi_0    pypi
   ncurses                   6.4                  h6a678d5_0  
   nest-asyncio              1.5.6                    pypi_0    pypi
   networkx                  3.0                      pypi_0    pypi
   notebook                  6.5.3                    pypi_0    pypi
   notebook-shim             0.2.2                    pypi_0    pypi
   numpy                     1.24.2                   pypi_0    pypi
   nvidia-cublas-cu11        11.10.3.66               pypi_0    pypi
   nvidia-cuda-cupti-cu11    11.7.101                 pypi_0    pypi
   nvidia-cuda-nvrtc-cu11    11.7.99                  pypi_0    pypi
   nvidia-cuda-runtime-cu11  11.7.99                  pypi_0    pypi
   nvidia-cudnn-cu11         8.5.0.96                 pypi_0    pypi
   nvidia-cufft-cu11         10.9.0.58                pypi_0    pypi
   nvidia-curand-cu11        10.2.10.91               pypi_0    pypi
   nvidia-cusolver-cu11      11.4.0.1                 pypi_0    pypi
   nvidia-cusparse-cu11      11.7.4.91                pypi_0    pypi
   nvidia-nccl-cu11          2.14.3                   pypi_0    pypi
   nvidia-nvtx-cu11          11.7.91                  pypi_0    pypi
   openssl                   1.1.1t               h7f8727e_0  
   packaging                 23.0                     pypi_0    pypi
   pandas                    1.5.3                    pypi_0    pypi
   pandocfilters             1.5.0                    pypi_0    pypi
   parso                     0.8.3                    pypi_0    pypi
   peft                      0.3.0.dev0               pypi_0    pypi
   pexpect                   4.8.0                    pypi_0    pypi
   pickleshare               0.7.5                    pypi_0    pypi
   pip                       23.0.1           py39h06a4308_0  
   platformdirs              3.1.1                    pypi_0    pypi
   prometheus-client         0.16.0                   pypi_0    pypi
   prompt-toolkit            3.0.38                   pypi_0    pypi
   psutil                    5.9.4                    pypi_0    pypi
   ptyprocess                0.7.0                    pypi_0    pypi
   pure-eval                 0.2.2                    pypi_0    pypi
   pyarrow                   11.0.0                   pypi_0    pypi
   pycparser                 2.21                     pypi_0    pypi
   pygments                  2.14.0                   pypi_0    pypi
   pyrsistent                0.19.3                   pypi_0    pypi
   python                    3.9.16               h7a1cb2a_2  
   python-dateutil           2.8.2                    pypi_0    pypi
   python-json-logger        2.0.7                    pypi_0    pypi
   pytz                      2022.7.1                 pypi_0    pypi
   pyyaml                    6.0                      pypi_0    pypi
   pyzmq                     25.0.1                   pypi_0    pypi
   qtconsole                 5.4.1                    pypi_0    pypi
   qtpy                      2.3.0                    pypi_0    pypi
   readline                  8.2                  h5eee18b_0  
   regex                     2022.10.31               pypi_0    pypi
   requests                  2.28.2                   pypi_0    pypi
   responses                 0.18.0                   pypi_0    pypi
   rfc3339-validator         0.1.4                    pypi_0    pypi
   rfc3986-validator         0.1.1                    pypi_0    pypi
   send2trash                1.8.0                    pypi_0    pypi
   sentencepiece             0.1.97                   pypi_0    pypi
   setuptools                65.6.3           py39h06a4308_0  
   six                       1.16.0                   pypi_0    pypi
   sniffio                   1.3.0                    pypi_0    pypi
   soupsieve                 2.4                      pypi_0    pypi
   sqlite                    3.41.1               h5eee18b_0  
   sqlparse                  0.4.3                    pypi_0    pypi
   stack-data                0.6.2                    pypi_0    pypi
   sympy                     1.11.1                   pypi_0    pypi
   terminado                 0.17.1                   pypi_0    pypi
   tinycss2                  1.2.1                    pypi_0    pypi
   tk                        8.6.12               h1ccaba5_0  
   tokenizers                0.13.2                   pypi_0    pypi
   torch                     2.0.0                    pypi_0    pypi
   tornado                   6.2                      pypi_0    pypi
   tqdm                      4.65.0                   pypi_0    pypi
   traitlets                 5.9.0                    pypi_0    pypi
   transformers              4.27.0.dev0              pypi_0    pypi
   triton                    2.0.0                    pypi_0    pypi
   typing-extensions         4.5.0                    pypi_0    pypi
   tzdata                    2022g                h04d1e81_0  
   uri-template              1.2.0                    pypi_0    pypi
   urllib3                   1.26.15                  pypi_0    pypi
   wcwidth                   0.2.6                    pypi_0    pypi
   webcolors                 1.12                     pypi_0    pypi
   webencodings              0.5.1                    pypi_0    pypi
   websocket-client          1.5.1                    pypi_0    pypi
   wheel                     0.38.4           py39h06a4308_0  
   widgetsnbextension        4.0.5                    pypi_0    pypi
   xxhash                    3.2.0                    pypi_0    pypi
   xz                        5.2.10               h5eee18b_1  
   yarl                      1.8.2                    pypi_0    pypi
   zipp                      3.15.0                   pypi_0    pypi
   zlib                      1.2.13               h5eee18b_0  

   3.pip list

   Package                  Version
   ------------------------ -----------
   accelerate               0.17.1
   aiohttp                  3.8.4
   aiosignal                1.3.1
   anyio                    3.6.2
   argon2-cffi              21.3.0
   argon2-cffi-bindings     21.2.0
   arrow                    1.2.3
   asgiref                  3.6.0
   asttokens                2.2.1
   async-timeout            4.0.2
   attrs                    22.2.0
   backcall                 0.2.0
   beautifulsoup4           4.11.2
   bitsandbytes             0.37.1
   bleach                   6.0.0
   certifi                  2022.12.7
   cffi                     1.15.1
   charset-normalizer       3.1.0
   cmake                    3.26.0
   comm                     0.1.2
   datasets                 2.10.1
   debugpy                  1.6.6
   decorator                5.1.1
   defusedxml               0.7.1
   dill                     0.3.6
   Django                   4.1.7
   executing                1.2.0
   fastjsonschema           2.16.3
   filelock                 3.10.0
   fqdn                     1.5.1
   frozenlist               1.3.3
   fsspec                   2023.3.0
   huggingface-hub          0.13.2
   idna                     3.4
   importlib-metadata       6.0.0
   ipykernel                6.21.3
   ipython                  8.11.0
   ipython-genutils         0.2.0
   ipywidgets               8.0.4
   isoduration              20.11.0
   jedi                     0.18.2
   Jinja2                   3.1.2
   jsonpointer              2.3
   jsonschema               4.17.3
   jupyter                  1.0.0
   jupyter_client           8.0.3
   jupyter-console          6.6.3
   jupyter_core             5.2.0
   jupyter-events           0.6.3
   jupyter_server           2.4.0
   jupyter_server_terminals 0.4.4
   jupyterlab-pygments      0.2.2
   jupyterlab-widgets       3.0.5
   lit                      15.0.7
   loralib                  0.1.1
   MarkupSafe               2.1.2
   matplotlib-inline        0.1.6
   mistune                  2.0.5
   mpmath                   1.3.0
   multidict                6.0.4
   multiprocess             0.70.14
   nbclassic                0.5.3
   nbclient                 0.7.2
   nbconvert                7.2.10
   nbformat                 5.7.3
   nest-asyncio             1.5.6
   networkx                 3.0
   notebook                 6.5.3
   notebook_shim            0.2.2
   numpy                    1.24.2
   nvidia-cublas-cu11       11.10.3.66
   nvidia-cuda-cupti-cu11   11.7.101
   nvidia-cuda-nvrtc-cu11   11.7.99
   nvidia-cuda-runtime-cu11 11.7.99
   nvidia-cudnn-cu11        8.5.0.96
   nvidia-cufft-cu11        10.9.0.58
   nvidia-curand-cu11       10.2.10.91
   nvidia-cusolver-cu11     11.4.0.1
   nvidia-cusparse-cu11     11.7.4.91
   nvidia-nccl-cu11         2.14.3
   nvidia-nvtx-cu11         11.7.91
   packaging                23.0
   pandas                   1.5.3
   pandocfilters            1.5.0
   parso                    0.8.3
   peft                     0.3.0.dev0
   pexpect                  4.8.0
   pickleshare              0.7.5
   pip                      23.0.1
   platformdirs             3.1.1
   prometheus-client        0.16.0
   prompt-toolkit           3.0.38
   psutil                   5.9.4
   ptyprocess               0.7.0
   pure-eval                0.2.2
   pyarrow                  11.0.0
   pycparser                2.21
   Pygments                 2.14.0
   pyrsistent               0.19.3
   python-dateutil          2.8.2
   python-json-logger       2.0.7
   pytz                     2022.7.1
   PyYAML                   6.0
   pyzmq                    25.0.1
   qtconsole                5.4.1
   QtPy                     2.3.0
   regex                    2022.10.31
   requests                 2.28.2
   responses                0.18.0
   rfc3339-validator        0.1.4
   rfc3986-validator        0.1.1
   Send2Trash               1.8.0
   sentencepiece            0.1.97
   setuptools               65.6.3
   six                      1.16.0
   sniffio                  1.3.0
   soupsieve                2.4
   sqlparse                 0.4.3
   stack-data               0.6.2
   sympy                    1.11.1
   terminado                0.17.1
   tinycss2                 1.2.1
   tokenizers               0.13.2
   torch                    2.0.0
   tornado                  6.2
   tqdm                     4.65.0
   traitlets                5.9.0
   transformers             4.27.0.dev0
   triton                   2.0.0
   typing_extensions        4.5.0
   uri-template             1.2.0
   urllib3                  1.26.15
   wcwidth                  0.2.6
   webcolors                1.12
   webencodings             0.5.1
   websocket-client         1.5.1
   wheel                    0.38.4
   widgetsnbextension       4.0.5
   xxhash                   3.2.0
   yarl                     1.8.2
   zipp                     3.15.0

[T-Atlas]

Try this ： https://github.com/tloen/alpaca-lora/issues/14#issuecomment-1471263165

[keeper-jie]

Try this ： #14 (comment)

Thank you very much for your help, I modified the following code to finish running the program.

model = PeftModel.from_pretrained( model, "tloen/alpaca-lora-7b", device_map={'':0})

[keeper-jie]

Try this ： #14 (comment)

I find a more appropriate approach to deploy the model.

tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf", device_map={'':0})

model = LlamaForCausalLM.from_pretrained(
    "decapoda-research/llama-7b-hf",
    load_in_8bit=True,
    torch_dtype=torch.float16,
    device_map={'':0}
)

# add device_map={'':0} in PeftModel.from_pretrained to confirm 2 2080Ti can work
model = PeftModel.from_pretrained(
    model, "tloen/alpaca-lora-7b", torch_dtype=torch.float16, device_map={'':0}
)

When I download the colab code and run it in my GPU server, which is different with git clone the repository to run. I modified the code and tested by my 2 2080Ti GPU server and pulled my code.

[bannsec]

generate used to work for me but i rebuilt it today and now i'm getting the same error... Note that I only have a single GPU, so this doesn't have anything to do with having multiple.

[TangJiakai]

generate used to work for me but i rebuilt it today and now i'm getting the same error... Note that I only have a single GPU, so this doesn't have anything to do with having multiple.

I have the same problem, help!!!