diff --git a/sunnypilot/selfdrive/controls/lib/speed_limit/speed_limit_assist.py b/sunnypilot/selfdrive/controls/lib/speed_limit/speed_limit_assist.py
index 04abdc642b..ec9d1d64d7 100644
--- a/sunnypilot/selfdrive/controls/lib/speed_limit/speed_limit_assist.py
+++ b/sunnypilot/selfdrive/controls/lib/speed_limit/speed_limit_assist.py
@@ -159,7 +159,7 @@ class SpeedLimitAssist:
       self.state = SpeedLimitAssistState.pending
 
   def _update_non_pcm_long_confirmed_state(self, CS) -> bool:
-    if self.target_set_speed_confirmed:
+    if self.target_set_speed_confirmed and self.state != SpeedLimitAssistState.inactive:
       return True
 
     if self.state == SpeedLimitAssistState.preActive: