From e6c7c3e499e8a0b64458b9c18ffadd1ce43182eb Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 10 Aug 2024 14:41:09 -0400 Subject: [PATCH] update pylint path to check indent/space for all (#6022) also fixed many errors. it was not checking nested dirs. exclude autogen for now. can we use ruff for this? --- .github/workflows/test.yml | 4 ++-- .pylintrc | 2 +- extra/datasets/wikipedia_download.py | 6 +++--- extra/mockgpu/amd/amdgpu.py | 8 ++++---- extra/mockgpu/driver.py | 2 +- extra/mockgpu/mockgpu.py | 2 +- extra/mockgpu/nv/nvdriver.py | 8 ++++---- extra/mockgpu/nv/nvgpu.py | 2 +- extra/models/bert.py | 10 +++++----- extra/models/clip.py | 2 +- extra/nv_gpu_driver/nv_ioctl.py | 2 +- extra/optimization/search.py | 2 +- test/imported/test_indexing.py | 14 +++++++------- test/unit/test_symbolic.py | 2 +- test/unit/test_uop_symbolic.py | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aaa5f17c..f60c0109 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,8 +87,8 @@ jobs: key: linting-packages-${{ hashFiles('**/setup.py') }}-3.8 - name: Install dependencies run: pip install -e '.[linting,testing,docs]' --extra-index-url https://download.pytorch.org/whl/cpu - - name: Lint with pylint - run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string=' ' **/*.py + - name: Lint bad-indentation and trailing-whitespace with pylint + run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string=' ' --recursive=y . - name: Lint with ruff run: | pip3 install --upgrade --force-reinstall ruff diff --git a/.pylintrc b/.pylintrc index b37388ce..18f7e1d5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,7 @@ extension-pkg-whitelist=scipy,cereal.messaging.messaging_pyx,PyQt5,av # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS +ignore=CVS,autogen,msm_kgsl.py # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. diff --git a/extra/datasets/wikipedia_download.py b/extra/datasets/wikipedia_download.py index abb1384b..50e7b3c1 100644 --- a/extra/datasets/wikipedia_download.py +++ b/extra/datasets/wikipedia_download.py @@ -7,7 +7,7 @@ import gdown from tqdm import tqdm from tinygrad.helpers import getenv -def gdrive_download(url:str, path:str): +def gdrive_download(url:str, path:str): if not os.path.exists(path): gdown.download(url, path) def wikipedia_uncompress_and_extract(file:str, path:str, small:bool=False): @@ -47,8 +47,8 @@ def download_wikipedia(path:str): gdrive_download("https://drive.google.com/uc?id=1tmMgLwoBvbEJEHXh77sqrXYw5RpqT8R_", os.path.join(path, "bert_reference_results_text_md5.txt")) gdrive_download("https://drive.google.com/uc?id=14xV2OUGSQDG_yDBrmbSdcDC-QGeqpfs_", os.path.join(path, "results_text.tar.gz")) wikipedia_uncompress_and_extract(os.path.join(path, "results_text.tar.gz"), path) - if getenv("VERIFY_CHECKSUM", 0): + if getenv("VERIFY_CHECKSUM", 0): verify_checksum(os.path.join(path, "results4"), os.path.join(path, "bert_reference_results_text_md5.txt")) -if __name__ == "__main__": +if __name__ == "__main__": download_wikipedia(getenv("BASEDIR", os.path.join(Path(__file__).parent / "wiki"))) \ No newline at end of file diff --git a/extra/mockgpu/amd/amdgpu.py b/extra/mockgpu/amd/amdgpu.py index 86d908d0..31da87e0 100644 --- a/extra/mockgpu/amd/amdgpu.py +++ b/extra/mockgpu/amd/amdgpu.py @@ -62,7 +62,7 @@ class AMDQueue(): self.wptr = to_mv(wptr, 8).cast("Q") class PM4Executor(AMDQueue): - def __init__(self, gpu, base, size, rptr, wptr): + def __init__(self, gpu, base, size, rptr, wptr): self.gpu = gpu super().__init__(base, size, rptr, wptr) @@ -79,7 +79,7 @@ class PM4Executor(AMDQueue): op = (header >> 8) & 0xFF n = (header >> 16) & 0x3FFF assert packet_type == 3, "Can parse only packet3" - if op == amd_gpu.PACKET3_SET_SH_REG: self._exec_set_sh_reg(n) + if op == amd_gpu.PACKET3_SET_SH_REG: self._exec_set_sh_reg(n) elif op == amd_gpu.PACKET3_ACQUIRE_MEM: self._exec_acquire_mem(n) elif op == amd_gpu.PACKET3_RELEASE_MEM: self._exec_release_mem(n) elif op == amd_gpu.PACKET3_WAIT_REG_MEM: cont = self._exec_wait_reg_mem(n) @@ -111,7 +111,7 @@ class PM4Executor(AMDQueue): if mem_data_sel == 1 or mem_data_sel == 2: ptr.cast('Q')[0] = val elif mem_data_sel == 3: if mem_event_type == CACHE_FLUSH_AND_INV_TS_EVENT: ptr.cast('Q')[0] = int(time.perf_counter() * 1e8) - else: raise RuntimeError(f"Unknown {mem_data_sel=} {mem_event_type=}") + else: raise RuntimeError(f"Unknown {mem_data_sel=} {mem_event_type=}") else: raise RuntimeError(f"Unknown {mem_data_sel=}") def _exec_wait_reg_mem(self, n): @@ -180,7 +180,7 @@ class PM4Executor(AMDQueue): _ = self._next_dword() # do not emulate events for now class SDMAExecutor(AMDQueue): - def __init__(self, gpu, base, size, rptr, wptr): + def __init__(self, gpu, base, size, rptr, wptr): self.gpu, self.base = gpu, base super().__init__(base, size, rptr, wptr) diff --git a/extra/mockgpu/driver.py b/extra/mockgpu/driver.py index d35737fd..e8206570 100644 --- a/extra/mockgpu/driver.py +++ b/extra/mockgpu/driver.py @@ -75,7 +75,7 @@ class VirtFile(): @staticmethod def build_fstat(st_dev=0x20, st_ino=0x100000, st_mode=0o100777, st_nlink=1, st_uid=0, st_gid=0, st_rdev=0, st_size=0, - st_blksize=4096, st_blocks=0, st_atime=0, st_mtime=0, st_ctime=0): + st_blksize=4096, st_blocks=0, st_atime=0, st_mtime=0, st_ctime=0): assert (ssz:=struct.calcsize(fmt_string:='QQQIIIQQiQqqq')) == 96, f"{ssz} != 96" return struct.pack(fmt_string, st_dev, st_ino, st_nlink, st_mode, st_uid, st_gid, st_rdev, st_size, st_blksize, st_blocks, st_atime, st_mtime, st_ctime) diff --git a/extra/mockgpu/mockgpu.py b/extra/mockgpu/mockgpu.py index 05b40a25..5374feb7 100644 --- a/extra/mockgpu/mockgpu.py +++ b/extra/mockgpu/mockgpu.py @@ -169,7 +169,7 @@ class TrackedMemoryView: self.mv[index] = value self.wcb(self.mv, index) - def cast(self, new_type, **kwargs): + def cast(self, new_type, **kwargs): self.mv = self.mv.cast(new_type, **kwargs) return self diff --git a/extra/mockgpu/nv/nvdriver.py b/extra/mockgpu/nv/nvdriver.py index 07df59e9..5444752e 100644 --- a/extra/mockgpu/nv/nvdriver.py +++ b/extra/mockgpu/nv/nvdriver.py @@ -43,7 +43,7 @@ class NVDevFileDesc(VirtFileDesc): self._mapping_userland = False def ioctl(self, fd, request, argp): return self.driver.dev_ioctl(self.gpu, request, argp) - def mmap(self, start, sz, prot, flags, fd, offset): + def mmap(self, start, sz, prot, flags, fd, offset): start = libc.mmap(start, sz, prot, flags|mmap.MAP_ANONYMOUS, -1, 0) if self._mapping_userland: self.driver.track_address(start, start+sz, lambda mv,off: None, lambda mv, off: self.driver._gpu_mmio_write(mv, off, self.gpu)) return start @@ -81,7 +81,7 @@ class NVDriver(VirtDriver): self.gpus[gpu_id] = NVGPU(gpu_id) self.tracked_files += [VirtFile(f'/dev/nvidia{gpu_id}', functools.partial(NVDevFileDesc, driver=self, gpu=self.gpus[gpu_id]))] - def open(self, name, flags, mode, virtfile): + def open(self, name, flags, mode, virtfile): cl = virtfile.fdcls(self._alloc_fd()) self.opened_fds[cl.fd] = cl return cl @@ -185,7 +185,7 @@ class NVDriver(VirtDriver): elif nr == nv_gpu.NV_ESC_RM_MAP_MEMORY: st:Any = nv_gpu.nv_ioctl_nvos33_parameters_with_fd.from_address(argp) obj = self.object_by_handle[st.params.hMemory] - if isinstance(obj, NVUserMode): + if isinstance(obj, NVUserMode): file = self.opened_fds[st.fd] assert isinstance(file, NVDevFileDesc) file._mapping_userland = True @@ -208,7 +208,7 @@ class NVDriver(VirtDriver): assert any(all(st.gpu_uuid.uuid[i] == gpu.gpu_uuid()[i] for i in range(16)) for gpu in self.gpus.values()) elif nr == nv_gpu.UVM_REGISTER_GPU_VASPACE: pass elif nr == nv_gpu.UVM_ENABLE_PEER_ACCESS: pass # uvm and shared spaced are setup already, no emulation for now - elif nr == nv_gpu.UVM_CREATE_EXTERNAL_RANGE: + elif nr == nv_gpu.UVM_CREATE_EXTERNAL_RANGE: st = nv_gpu.UVM_CREATE_EXTERNAL_RANGE_PARAMS.from_address(argp) libc.mmap(st.base, st.length, mmap.PROT_READ|mmap.PROT_WRITE, MAP_FIXED|mmap.MAP_SHARED|mmap.MAP_ANONYMOUS, -1, 0) elif nr == nv_gpu.UVM_MAP_EXTERNAL_ALLOCATION: diff --git a/extra/mockgpu/nv/nvgpu.py b/extra/mockgpu/nv/nvgpu.py index 3e81195c..3bb401de 100644 --- a/extra/mockgpu/nv/nvgpu.py +++ b/extra/mockgpu/nv/nvgpu.py @@ -52,7 +52,7 @@ class GPFIFO: def _state(self, reg): return self.state[reg] def _state64(self, reg): return (self.state[reg] << 32) + self.state[reg + 4] def _state64_le(self, reg): return (self.state[reg + 4] << 32) + self.state[reg] - + def _reset_buf_state(self): self.buf, self.buf_ptr = None, 0 def _set_buf_state(self, gpfifo_entry): ptr = ((gpfifo_entry >> 2) & 0xfffffffff) << 2 diff --git a/extra/models/bert.py b/extra/models/bert.py index fdf702c3..4891888b 100644 --- a/extra/models/bert.py +++ b/extra/models/bert.py @@ -44,11 +44,11 @@ class BertForPretraining: """Default is BERT-large""" self.bert = Bert(hidden_size, intermediate_size, max_position_embeddings, num_attention_heads, num_hidden_layers, type_vocab_size, vocab_size, attention_probs_dropout_prob, hidden_dropout_prob) self.cls = BertPreTrainingHeads(hidden_size, vocab_size, self.bert.embeddings.word_embeddings.weight) - + def __call__(self, input_ids:Tensor, attention_mask:Tensor, masked_lm_positions:Tensor, token_type_ids:Tensor): output = self.bert(input_ids, attention_mask, token_type_ids) return self.cls(output, masked_lm_positions) - + def loss(self, prediction_logits:Tensor, seq_relationship_logits:Tensor, masked_lm_ids:Tensor, masked_lm_weights:Tensor, next_sentence_labels:Tensor): # Reference has residual on denominator: https://github.com/mlcommons/training/blob/master/language_model/tensorflow/bert/run_pretraining.py#L315 def sparse_categorical_crossentropy(predictions:Tensor, labels:Tensor, ignore_index=-1): @@ -60,7 +60,7 @@ class BertForPretraining: masked_lm_loss = sparse_categorical_crossentropy(prediction_logits, masked_lm_ids, ignore_index=masked_lm_weights) next_sentence_loss = seq_relationship_logits.binary_crossentropy_logits(next_sentence_labels) return masked_lm_loss + next_sentence_loss - + def accuracy(self, prediction_logits:Tensor, seq_relationship_logits:Tensor, masked_lm_ids:Tensor, masked_lm_weights:Tensor, next_sentence_labels:Tensor): valid = masked_lm_ids != 0 @@ -73,7 +73,7 @@ class BertForPretraining: next_sentence_loss = seq_relationship_logits.binary_crossentropy_logits(next_sentence_labels) return masked_lm_accuracy.sum() / valid.sum(), seq_relationship_accuracy.mean(), masked_lm_loss, next_sentence_loss - + def load_from_pretrained(self, tf_weight_path:str=Path(__file__).parent.parent / "datasets" / "wiki"): os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Mute tf flag info # load from tensorflow @@ -148,7 +148,7 @@ class BertPredictionHeadTransform: self.LayerNorm = LayerNorm(hidden_size, eps=1e-12) def __call__(self, hidden_states:Tensor): - return self.LayerNorm(gelu(self.dense(hidden_states))) + return self.LayerNorm(gelu(self.dense(hidden_states))) class BertPooler: def __init__(self, hidden_size:int): diff --git a/extra/models/clip.py b/extra/models/clip.py index c581e654..dc13857f 100644 --- a/extra/models/clip.py +++ b/extra/models/clip.py @@ -315,7 +315,7 @@ class Open: pooled = x[:, text.argmax(dim=-1)] @ self.text_projection return pooled - + class ClipVisionTransformer: def __init__(self, width:int, layers:int, d_head:int, image_size:int, patch_size:int): grid_size = image_size // patch_size diff --git a/extra/nv_gpu_driver/nv_ioctl.py b/extra/nv_gpu_driver/nv_ioctl.py index c250e65a..00f319d5 100644 --- a/extra/nv_gpu_driver/nv_ioctl.py +++ b/extra/nv_gpu_driver/nv_ioctl.py @@ -202,7 +202,7 @@ def _dump_qmd(address, packets): bits = vv if isinstance(vv, types.FunctionType): bits = vv(0) - + if bits is not None: res = 0 for bt in range(bits[1], bits[0]+1): res |= ((gpfifo[i + 3 + bt // 32] >> (bt % 32)) & 0x1) << (bt - bits[1]) diff --git a/extra/optimization/search.py b/extra/optimization/search.py index 57a29c8d..9b8aec9b 100644 --- a/extra/optimization/search.py +++ b/extra/optimization/search.py @@ -21,7 +21,7 @@ if __name__ == '__main__': ast_strs = [args.ast] elif args.file is not None: with open(args.file, 'r') as file: - ast_strs = file.readlines() + ast_strs = file.readlines() for i, ast_str in enumerate(ast_strs): print(f"optimizing {i}/{len(ast_strs)}\nast={ast_str}") diff --git a/test/imported/test_indexing.py b/test/imported/test_indexing.py index c5c42187..b0bdb6e6 100644 --- a/test/imported/test_indexing.py +++ b/test/imported/test_indexing.py @@ -908,10 +908,10 @@ class TestIndexing(unittest.TestCase): numpy_testing_assert_equal_helper(out, ref) def test_int_indices(self): - v = Tensor.randn(5, 7, 3) - numpy_testing_assert_equal_helper(v[[0, 4, 2]].shape, (3, 7, 3)) - numpy_testing_assert_equal_helper(v[:, [0, 4, 2]].shape, (5, 3, 3)) - numpy_testing_assert_equal_helper(v[:, [[0, 1], [4, 3]]].shape, (5, 2, 2, 3)) + v = Tensor.randn(5, 7, 3) + numpy_testing_assert_equal_helper(v[[0, 4, 2]].shape, (3, 7, 3)) + numpy_testing_assert_equal_helper(v[:, [0, 4, 2]].shape, (5, 3, 3)) + numpy_testing_assert_equal_helper(v[:, [[0, 1], [4, 3]]].shape, (5, 2, 2, 3)) # TODO fancy setitem ''' @@ -983,7 +983,7 @@ class TestIndexing(unittest.TestCase): numpy_testing_assert_equal_helper(x[[1, 2]].shape, (2, 0)) numpy_testing_assert_equal_helper(x[[], []].shape, (0,)) with self.assertRaises(IndexError): - x[:, [0, 1]] + x[:, [0, 1]] def test_empty_slice(self): x = Tensor.randn(2, 3, 4, 5) @@ -1036,8 +1036,8 @@ class TestIndexing(unittest.TestCase): a[...] = neg_ones_expanded * 4 numpy_testing_assert_equal_helper(a, neg_ones * 4) if a.dim() == 0: - with self.assertRaises(IndexError): - a[:] = neg_ones_expanded * 5 + with self.assertRaises(IndexError): + a[:] = neg_ones_expanded * 5 @unittest.skip("bool indexing not supported") def test_index_scalar_with_bool_mask(self): diff --git a/test/unit/test_symbolic.py b/test/unit/test_symbolic.py index 0c6b2113..e0edeb4f 100644 --- a/test/unit/test_symbolic.py +++ b/test/unit/test_symbolic.py @@ -46,7 +46,7 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(expr, 0, 1, "(((idx1*4)+FLOAT4_INDEX)<512)") def test_div_reduction(self): - self.helper_test_variable(Variable("a", 2, 3)//2, 1, 1, "1") + self.helper_test_variable(Variable("a", 2, 3)//2, 1, 1, "1") def test_var_becomes_num(self): assert isinstance(Variable("a", 2, 2), NumNode) diff --git a/test/unit/test_uop_symbolic.py b/test/unit/test_uop_symbolic.py index 172e223f..67c188db 100644 --- a/test/unit/test_uop_symbolic.py +++ b/test/unit/test_uop_symbolic.py @@ -100,7 +100,7 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(expr, 0, 1, "(((idx1*4)+FLOAT4_INDEX)<512)") def test_div_reduction(self): - self.helper_test_variable(Variable("a", 2, 3)//2, 1, 1, "1") + self.helper_test_variable(Variable("a", 2, 3)//2, 1, 1, "1") #def test_var_becomes_num(self): # assert isinstance(Variable("a", 2, 2), NumNode)