RonanKMcGovern
commented
5 days ago Note that I have tried:
!pip install outlines -qUthen:
import outlines
model = outlines.models.transformers("/kaggle/input/llama-3.1/transformers/8b-instruct/2",
device="cuda")
prompt = "What is the IP address of the Google DNS servers? "
generator = outlines.generate.text(model)
unstructured = generator(prompt, max_tokens=30)
generator = outlines.generate.regex(
model,
r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)",
)
structured = generator(prompt, max_tokens=30)
print(unstructured)which gives:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[13], line 3
1 import outlines
----> 3 model = outlines.models.transformers("/kaggle/input/llama-3.1/transformers/8b-instruct/2",
4 device="cuda")
6 prompt = "What is the IP address of the Google DNS servers? "
8 generator = outlines.generate.text(model)
File /opt/conda/lib/python3.10/site-packages/outlines/models/transformers.py:231, in transformers(model_name, device, model_kwargs, tokenizer_kwargs)
228 if device is not None:
229 model_kwargs["device_map"] = device
--> 231 model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs)
233 tokenizer_kwargs.setdefault("padding_side", "left")
234 tokenizer = AutoTokenizer.from_pretrained(model_name, **tokenizer_kwargs)
File /opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:523, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
520 if kwargs.get("quantization_config", None) is not None:
521 _ = kwargs.pop("quantization_config")
--> 523 config, kwargs = AutoConfig.from_pretrained(
524 pretrained_model_name_or_path,
525 return_unused_kwargs=True,
526 trust_remote_code=trust_remote_code,
527 code_revision=code_revision,
528 _commit_hash=commit_hash,
529 **hub_kwargs,
530 **kwargs,
531 )
533 # if torch_dtype=auto was passed here, ensure to pass it on
534 if kwargs_orig.get("torch_dtype", None) == "auto":
File /opt/conda/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py:958, in AutoConfig.from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
952 except KeyError:
953 raise ValueError(
954 f"The checkpoint you are trying to load has model type `{config_dict['model_type']}` "
955 "but Transformers does not recognize this architecture. This could be because of an "
956 "issue with the checkpoint, or because your version of Transformers is out of date."
957 )
--> 958 return config_class.from_dict(config_dict, **unused_kwargs)
959 else:
960 # Fallback: use pattern matching on the string.
961 # We go from longer names to shorter names to catch roberta before bert (for instance)
962 for pattern in sorted(CONFIG_MAPPING.keys(), key=len, reverse=True):
File /opt/conda/lib/python3.10/site-packages/transformers/configuration_utils.py:768, in PretrainedConfig.from_dict(cls, config_dict, **kwargs)
765 # We remove it from kwargs so that it does not appear in `return_unused_kwargs`.
766 config_dict["attn_implementation"] = kwargs.pop("attn_implementation", None)
--> 768 config = cls(**config_dict)
770 if hasattr(config, "pruned_heads"):
771 config.pruned_heads = {int(key): value for key, value in config.pruned_heads.items()}
File /opt/conda/lib/python3.10/site-packages/transformers/models/llama/configuration_llama.py:161, in LlamaConfig.__init__(self, vocab_size, hidden_size, intermediate_size, num_hidden_layers, num_attention_heads, num_key_value_heads, hidden_act, max_position_embeddings, initializer_range, rms_norm_eps, use_cache, pad_token_id, bos_token_id, eos_token_id, pretraining_tp, tie_word_embeddings, rope_theta, rope_scaling, attention_bias, attention_dropout, mlp_bias, **kwargs)
159 self.rope_theta = rope_theta
160 self.rope_scaling = rope_scaling
--> 161 self._rope_scaling_validation()
162 self.attention_bias = attention_bias
163 self.attention_dropout = attention_dropout
File /opt/conda/lib/python3.10/site-packages/transformers/models/llama/configuration_llama.py:182, in LlamaConfig._rope_scaling_validation(self)
179 return
181 if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
--> 182 raise ValueError(
183 "`rope_scaling` must be a dictionary with two fields, `type` and `factor`, " f"got {self.rope_scaling}"
184 )
185 rope_scaling_type = self.rope_scaling.get("type", None)
186 rope_scaling_factor = self.rope_scaling.get("factor", None)
ValueError: `rope_scaling` must be a dictionary with two fields, `type` and `factor`, got {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}I suspect, if this worked, it wouldn't necessarily dispatch to both GPUs, but I'm uncertain.
I'm also unclear on where I would need to pass in device_map="auto".
I see there is a separate way to load as follows:
import outlines
from transformers import AutoModelForCausalLM, AutoTokenizer
from outlines import models
model = AutoModelForCausalLM.from_pretrained("/kaggle/input/llama-3.1/transformers/8b-instruct/2", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained("/kaggle/input/llama-3.1/transformers/8b-instruct/2")
model = models.Transformers(llm, tokenizer)
# model = outlines.models.transformers("/kaggle/input/llama-3.1/transformers/8b-instruct/2",
# device_map="auto")
prompt = "What is the IP address of the Google DNS servers? "
generator = outlines.generate.text(model)
unstructured = generator(prompt, max_tokens=30)
generator = outlines.generate.regex(
model,
r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)",
)
structured = generator(prompt, max_tokens=30)
print(unstructured)but this gives the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[10], line 5
2 from transformers import AutoModelForCausalLM, AutoTokenizer
3 from outlines import models
----> 5 model = AutoModelForCausalLM.from_pretrained("/kaggle/input/llama-3.1/transformers/8b-instruct/2", device_map="auto")
7 tokenizer = AutoTokenizer.from_pretrained("/kaggle/input/llama-3.1/transformers/8b-instruct/2")
9 model = models.Transformers(llm, tokenizer)
File /opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:523, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
520 if kwargs.get("quantization_config", None) is not None:
521 _ = kwargs.pop("quantization_config")
--> 523 config, kwargs = AutoConfig.from_pretrained(
524 pretrained_model_name_or_path,
525 return_unused_kwargs=True,
526 trust_remote_code=trust_remote_code,
527 code_revision=code_revision,
528 _commit_hash=commit_hash,
529 **hub_kwargs,
530 **kwargs,
531 )
533 # if torch_dtype=auto was passed here, ensure to pass it on
534 if kwargs_orig.get("torch_dtype", None) == "auto":
File /opt/conda/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py:958, in AutoConfig.from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
952 except KeyError:
953 raise ValueError(
954 f"The checkpoint you are trying to load has model type `{config_dict['model_type']}` "
955 "but Transformers does not recognize this architecture. This could be because of an "
956 "issue with the checkpoint, or because your version of Transformers is out of date."
957 )
--> 958 return config_class.from_dict(config_dict, **unused_kwargs)
959 else:
960 # Fallback: use pattern matching on the string.
961 # We go from longer names to shorter names to catch roberta before bert (for instance)
962 for pattern in sorted(CONFIG_MAPPING.keys(), key=len, reverse=True):
File /opt/conda/lib/python3.10/site-packages/transformers/configuration_utils.py:768, in PretrainedConfig.from_dict(cls, config_dict, **kwargs)
765 # We remove it from kwargs so that it does not appear in `return_unused_kwargs`.
766 config_dict["attn_implementation"] = kwargs.pop("attn_implementation", None)
--> 768 config = cls(**config_dict)
770 if hasattr(config, "pruned_heads"):
771 config.pruned_heads = {int(key): value for key, value in config.pruned_heads.items()}
File /opt/conda/lib/python3.10/site-packages/transformers/models/llama/configuration_llama.py:161, in LlamaConfig.__init__(self, vocab_size, hidden_size, intermediate_size, num_hidden_layers, num_attention_heads, num_key_value_heads, hidden_act, max_position_embeddings, initializer_range, rms_norm_eps, use_cache, pad_token_id, bos_token_id, eos_token_id, pretraining_tp, tie_word_embeddings, rope_theta, rope_scaling, attention_bias, attention_dropout, mlp_bias, **kwargs)
159 self.rope_theta = rope_theta
160 self.rope_scaling = rope_scaling
--> 161 self._rope_scaling_validation()
162 self.attention_bias = attention_bias
163 self.attention_dropout = attention_dropout
File /opt/conda/lib/python3.10/site-packages/transformers/models/llama/configuration_llama.py:182, in LlamaConfig._rope_scaling_validation(self)
179 return
181 if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
--> 182 raise ValueError(
183 "`rope_scaling` must be a dictionary with two fields, `type` and `factor`, " f"got {self.rope_scaling}"
184 )
185 rope_scaling_type = self.rope_scaling.get("type", None)
186 rope_scaling_factor = self.rope_scaling.get("factor", None)
ValueError: `rope_scaling` must be a dictionary with two fields, `type` and `factor`, got {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}btw, running !transformers-cli env gives:
- `transformers` version: 4.45.1
- Platform: Linux-5.15.154+-x86_64-with-glibc2.31
- Python version: 3.10.13
- Huggingface_hub version: 0.23.2
- Safetensors version: 0.4.3
- Accelerate version: 0.34.2
- Accelerate config: not found
- PyTorch version (GPU?): 2.1.2 (True)
- Tensorflow version (GPU?): 2.15.0 (True)
- Flax version (CPU?/GPU?/TPU?): 0.8.4 (gpu)
- Jax version: 0.4.26
- JaxLib version: 0.4.26.dev20240504
- Using distributed or parallel set-up in script?: <fill in>
- Using GPU in script?: <fill in>
- GPU type: Tesla T4and I'm in kaggle with 2xT4 (I want to run this llama 3.1 8B model on two gpus).
njhill
joerunde
Your current environment
N/A
How would you like to use vllm
I want to use guided_regex but in offline mode.
I had thought maybe I could pass guided_regex via sampling_params or when calling llm.chat() or llm.generate() but I don't see where that can be done.
I know I can do this via a server, but I'd like to do it locally instead in a notebook, without a server. Thanks
Before submitting a new issue...