commaai
/
tinygrad
mirror of https://github.com/commaai/tinygrad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

faster tinychat (#5993)

This commit is contained in:
wozeparrot 2024-08-08 19:16:26 -07:00 committed by GitHub
parent 1f1eb46af6
commit d269bc95fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
examples/llama3.py
View File

@ -179,10 +179,10 @@ def build_transformer(model_path: Path, model_size="8B", quantize=None, device=N
return model
# default settings
TEMPERATURE = 0.85
TOP_K = 25
TOP_P = 0.9
ALPHA_F = 0.1
TEMPERATURE = 0.95
TOP_K = 0
TOP_P = 0.0
ALPHA_F = 0.0
ALPHA_P = 0.0
last_seen_toks = []

2
extra/models/llama.py
View File

@ -109,6 +109,8 @@ def sample(logits: Tensor, temp: float, k: int, p: float, af: float, ap: float):
# if temperature is very low just use argmax
if temp < 1e-6: return logits.argmax()
logits = logits.to(Device.DEFAULT)
# alpha sampling
if af or ap:
if not hasattr(sample, "alpha_counter"):