commaai
/
tinygrad
mirror of https://github.com/commaai/tinygrad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add contiguous (#1246)

This commit is contained in:
Yixiang Gao 2023-07-15 10:36:34 -05:00 committed by GitHub
parent 872e2198fe
commit a8f2c16f8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/hlb_cifar10.py
View File

@ -131,7 +131,7 @@ def train_cifar(bs=512, eval_bs=500, steps=1000, div_factor=1e16, final_lr_ratio
left_batcher, right_batcher = fetch_batches(X_train, Y_train, BS=BS, seed=seed, is_train=True), fetch_batches(X_train, Y_train, BS=BS, seed=seed+1, is_train=True)
while i <= STEPS:
(Xr, Yr), (Xl, Yl) = next(right_batcher), next(left_batcher)
mixup_prob = Tensor(np.random.beta(MIXUP_ALPHA, MIXUP_ALPHA, (1, )).astype(np.float32)) if MIXUP_ALPHA > 0 else Tensor.ones(Xr.shape[0], 1, 1, 1)
mixup_prob = Tensor(np.random.beta(MIXUP_ALPHA, MIXUP_ALPHA, (1, )).astype(np.float32)).contiguous() if MIXUP_ALPHA > 0 else Tensor.ones(Xr.shape[0], 1, 1, 1)
if i%50 == 0 and i > 1:
# batchnorm is frozen, no need for Tensor.training=False
corrects = []

4
extra/lr_scheduler.py
View File

@ -66,8 +66,8 @@ class CosineAnnealingLR(LR_Scheduler):
class OneCycleLR(LR_Scheduler):
def __init__(self, optimizer: Optimizer, max_lr: float, div_factor: float, final_div_factor: float, total_steps: int, pct_start: float,
anneal_strategy: str = 'linear', cycle_momentum: bool = False):
self.initial_lr = Tensor([max_lr / div_factor])
self.max_lr = Tensor([max_lr])
self.initial_lr = Tensor([max_lr / div_factor]).contiguous()
self.max_lr = Tensor([max_lr]).contiguous()
self.min_lr = self.initial_lr/final_div_factor
super().__init__(optimizer)
self.total_steps = total_steps

2
tinygrad/nn/optim.py
View File

@ -11,7 +11,7 @@ class Optimizer:
self.params: List[Tensor] = dedup([x for x in params if x.requires_grad])
self.buffers: List[Tensor] = dedup([x for x in params if not x.requires_grad]) # buffers are still realized
self.lr = Tensor([lr], requires_grad=False)
self.lr = Tensor([lr], requires_grad=False).contiguous()
def zero_grad(self):
for param in self.params: param.grad = None