diff --git a/src/core/meshcop/border_agent.cpp b/src/core/meshcop/border_agent.cpp
index 20523602e4c..fb62e1225ab 100644
--- a/src/core/meshcop/border_agent.cpp
+++ b/src/core/meshcop/border_agent.cpp
@@ -56,6 +56,7 @@ BorderAgent::BorderAgent(Instance &aInstance)
 #endif
 #if OPENTHREAD_CONFIG_BORDER_AGENT_EPHEMERAL_KEY_ENABLE
     , mUsingEphemeralKey(false)
+    , mDidConnectWithEphemeralKey(false)
     , mOldUdpPort(0)
     , mEphemeralKeyTimer(aInstance)
     , mEphemeralKeyTask(aInstance)
@@ -246,6 +247,7 @@ void BorderAgent::HandleConnected(Dtls::Session::ConnectEvent aEvent)
 #if OPENTHREAD_CONFIG_BORDER_AGENT_EPHEMERAL_KEY_ENABLE
         if (mUsingEphemeralKey)
         {
+            mDidConnectWithEphemeralKey = true;
             mCounters.mEpskcSecureSessionSuccesses++;
             mEphemeralKeyTask.Post();
         }
@@ -264,7 +266,10 @@ void BorderAgent::HandleConnected(Dtls::Session::ConnectEvent aEvent)
 #if OPENTHREAD_CONFIG_BORDER_AGENT_EPHEMERAL_KEY_ENABLE
         if (mUsingEphemeralKey)
         {
-            RestartAfterRemovingEphemeralKey();
+            if (mDidConnectWithEphemeralKey)
+            {
+                RestartAfterRemovingEphemeralKey();
+            }
 
             if (aEvent == Dtls::Session::kDisconnectedError)
             {
@@ -735,7 +740,8 @@ Error BorderAgent::SetEphemeralKey(const char *aKeyString, uint32_t aTimeout, ui
     // callbacks (like `HandleConnected()`) may be invoked from
     // `Start()` itself.
 
-    mUsingEphemeralKey = true;
+    mUsingEphemeralKey          = true;
+    mDidConnectWithEphemeralKey = false;
 
     error = Start(aUdpPort, reinterpret_cast<const uint8_t *>(aKeyString), static_cast<uint8_t>(length));
 
diff --git a/src/core/meshcop/border_agent.hpp b/src/core/meshcop/border_agent.hpp
index 60f4160d1e5..ba96bc926dd 100644
--- a/src/core/meshcop/border_agent.hpp
+++ b/src/core/meshcop/border_agent.hpp
@@ -337,7 +337,8 @@ class BorderAgent : public InstanceLocator, private NonCopyable
     bool mIdInitialized;
 #endif
 #if OPENTHREAD_CONFIG_BORDER_AGENT_EPHEMERAL_KEY_ENABLE
-    bool                           mUsingEphemeralKey;
+    bool                           mUsingEphemeralKey : 1;
+    bool                           mDidConnectWithEphemeralKey : 1;
     uint16_t                       mOldUdpPort;
     EphemeralKeyTimer              mEphemeralKeyTimer;
     EphemeralKeyTask               mEphemeralKeyTask;