diff --git a/components/philips_action_button/action_button.cpp b/components/philips_action_button/action_button.cpp
index b4971df..84a2758 100644
--- a/components/philips_action_button/action_button.cpp
+++ b/components/philips_action_button/action_button.cpp
@@ -18,7 +18,7 @@ namespace esphome
             void ActionButton::loop()
             {
                 // Repeated message sending for long presses
-                if (millis() - press_start_ <= LONG_PRESS_DURATION)
+                if (should_long_press_ && millis() - press_start_ <= LONG_PRESS_DURATION)
                 {
                     if (millis() - last_message_sent_ > LONG_PRESS_REPETITION_DELAY)
                     {
diff --git a/components/philips_action_button/action_button.h b/components/philips_action_button/action_button.h
index 7b497b7..24ba191 100644
--- a/components/philips_action_button/action_button.h
+++ b/components/philips_action_button/action_button.h
@@ -110,7 +110,7 @@ namespace esphome
                 /// @brief true if the component is currently performing a long press
                 bool is_long_pressing_ = false;
                 /// @brief time at which the button press was started
-                long press_start_ = 0;
+                long press_start_ = -(LONG_PRESS_DURATION + 1);
                 /// @brief time at which the last message was sent
                 long last_message_sent_ = 0;
             };