diff --git a/README.md b/README.md index 44bfae37..b476d565 100644 --- a/README.md +++ b/README.md @@ -178,4 +178,4 @@ python3 -m pytest test/ # whole test suite #### Process replay tests -[Process replay](https://github.com/tinygrad/tinygrad/blob/master/test/external/process_replay/process_replay.py) compares your PR's generated kernels against master. If your PR is a refactor or speedup without any expected behavior change, It should include [run_process_replay] in the PR title, [example](https://github.com/tinygrad/tinygrad/pull/4995). Note that you should keep your branch up-to-date with master. +[Process replay](https://github.com/tinygrad/tinygrad/blob/master/test/external/process_replay/process_replay.py) compares your PR's generated kernels against master. If your PR is a refactor or speedup without any expected behavior change, It should include [pr] in the pull request title, [example](https://github.com/tinygrad/tinygrad/pull/4995). Note that you should keep your branch up-to-date with master. diff --git a/test/external/process_replay/README.md b/test/external/process_replay/README.md index 67873a14..917b14b0 100644 --- a/test/external/process_replay/README.md +++ b/test/external/process_replay/README.md @@ -2,7 +2,7 @@ Process replay is a tool for creating a diff of generated kernels between two commits. By default, process replay doesn't assert kernel diffs. -Refactor and speedup prs must enable the assert by including `[run_process_replay]` in the pr title. +Refactor and speedup prs must enable the assert by including `[pr]` in the pull request title. Note that process replay [early stops when over 20% of kernels change, for speed.](https://github.com/tinygrad/tinygrad/pull/5480). diff --git a/test/external/process_replay/process_replay.py b/test/external/process_replay/process_replay.py index 85c66c0a..6596cc6a 100755 --- a/test/external/process_replay/process_replay.py +++ b/test/external/process_replay/process_replay.py @@ -18,7 +18,9 @@ early_stop = multiprocessing.Event() logging.basicConfig(level=logging.INFO, format="%(message)s") # user config -ASSERT_DIFF = getenv("ASSERT_PROCESS_REPLAY", int((k:="[run_process_replay]") in os.getenv("COMMIT_MESSAGE", k) or k in os.getenv("PR_TITLE", k))) +ASSERT_FLAGS = {"[pr]", "[run_process_replay]"} +ASSERT_DIFF = int(any(flag in os.getenv("COMMIT_MESSAGE", flag) or flag in os.getenv("PR_TITLE", flag) for flag in ASSERT_FLAGS)) +if not getenv("ASSERT_PROCESS_REPLAY", 1): ASSERT_DIFF = 0 SKIP_PROCESS_REPLAY = (k:="[skip_process_replay]") in os.getenv("COMMIT_MESSAGE", "") or k in os.getenv("PR_TITLE", "") COMPARE_SCHEDULE = getenv("COMPARE_SCHEDULE", 1) if REF == "master": SKIP_PROCESS_REPLAY = True