# for imagenet download prepare.sh and run it import glob, random, json, math import numpy as np from PIL import Image import functools, pathlib from tinygrad.helpers import diskcache, getenv @functools.lru_cache(None) def get_imagenet_categories(): ci = json.load(open(BASEDIR / "imagenet_class_index.json")) return {v[0]: int(k) for k,v in ci.items()} if getenv("MNISTMOCK"): BASEDIR = pathlib.Path(__file__).parent / "mnist" @functools.lru_cache(None) def get_train_files(): if not BASEDIR.exists(): from extra.datasets.fake_imagenet_from_mnist import create_fake_mnist_imagenet create_fake_mnist_imagenet(BASEDIR) if not (files:=glob.glob(p:=str(BASEDIR / "train/*/*"))): raise FileNotFoundError(f"No training files in {p}") return files else: BASEDIR = pathlib.Path(__file__).parent / "imagenet" @diskcache def get_train_files(): if not (files:=glob.glob(p:=str(BASEDIR / "train/*/*"))): raise FileNotFoundError(f"No training files in {p}") return files @functools.lru_cache(None) def get_val_files(): if not (files:=glob.glob(p:=str(BASEDIR / "val/*/*"))): raise FileNotFoundError(f"No validation files in {p}") return files def image_resize(img, size, interpolation): w, h = img.size w_new = int((w / h) * size) if w > h else size h_new = int((h / w) * size) if h > w else size return img.resize([w_new, h_new], interpolation) def rand_flip(img): if random.random() < 0.5: img = np.flip(img, axis=1).copy() return img def center_crop(img): rescale = min(img.size) / 256 crop_left = (img.width - 224 * rescale) / 2.0 crop_top = (img.height - 224 * rescale) / 2.0 img = img.resize((224, 224), Image.BILINEAR, box=(crop_left, crop_top, crop_left + 224 * rescale, crop_top + 224 * rescale)) return img # we don't use supplied imagenet bounding boxes, so scale min is just min_object_covered # https://github.com/tensorflow/tensorflow/blob/e193d8ea7776ef5c6f5d769b6fb9c070213e737a/tensorflow/core/kernels/image/sample_distorted_bounding_box_op.cc def random_resized_crop(img, size, scale=(0.10, 1.0), ratio=(3/4, 4/3)): w, h = img.size area = w * h # Crop random_solution_found = False for _ in range(100): aspect_ratio = random.uniform(ratio[0], ratio[1]) max_scale = min(min(w * aspect_ratio / h, h / aspect_ratio / w), scale[1]) target_area = area * random.uniform(scale[0], max_scale) w_new = int(round(math.sqrt(target_area * aspect_ratio))) h_new = int(round(math.sqrt(target_area / aspect_ratio))) if 0 < w_new <= w and 0 < h_new <= h: crop_left = random.randint(0, w - w_new) crop_top = random.randint(0, h - h_new) img = img.crop((crop_left, crop_top, crop_left + w_new, crop_top + h_new)) random_solution_found = True break if not random_solution_found: # Center crop img = center_crop(img) else: # Resize img = img.resize([size, size], Image.BILINEAR) return img def preprocess_train(img): img = random_resized_crop(img, 224) img = rand_flip(np.array(img)) return img