From 919a0aaaf91673ec7d7b9f5d4788d62f2a8b7972 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 3 Jan 2022 03:05:46 +0000
Subject: [PATCH 01/20] Count ICMP Unknown events

---
 src/link_prober/LinkProber.cpp | 10 ++++++++++
 src/link_prober/LinkProber.h   |  3 +++
 2 files changed, 13 insertions(+)

diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index fc526781..9932e839 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -319,10 +319,12 @@ void LinkProber::handleRecv(
                 // echo reply for an echo request generated by this/active ToR
                 mRxSelfSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpSelfEvent());
+                mContinuousIcmpUnknownEventCount = 0;
             } else {
                 // echo reply for an echo request generated by peer ToR
                 mRxPeerSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpPeerEvent());
+                mContinuousIcmpUnknownEventCount = 0;
                 if (ntohl(icmpPayload->command) == static_cast<uint32_t> (Command::COMMAND_SWITCH_ACTIVE)) {
                     boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
                     boost::asio::io_service &ioService = strand.context();
@@ -394,6 +396,14 @@ void LinkProber::handleTimeout(boost::system::error_code errorCode)
     if (mTxSeqNo != mRxSelfSeqNo && mTxSeqNo != mRxPeerSeqNo) {
         // post unknown event
         mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpUnknownEvent());
+        mContinuousIcmpUnknownEventCount++; 
+        mIcmpUnknownEventCount++;
+
+        MUXLOGINFO(boost::format("%s: continusous ICMP Unknown event count: %d, total Unknown event count since initialization: %d") %
+            mMuxPortConfig.getPortName() %
+            mContinuousIcmpUnknownEventCount %
+            mIcmpUnknownEventCount
+        );
     }
 
     // start another cycle of send/recv
diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
index c4b87af2..f6297e59 100644
--- a/src/link_prober/LinkProber.h
+++ b/src/link_prober/LinkProber.h
@@ -373,6 +373,9 @@ class LinkProber
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> mRxBuffer;
 
     bool mSuspendTx = false;
+
+    uint32_t mContinuousIcmpUnknownEventCount = 0;
+    uint32_t mIcmpUnknownEventCount = 0;
 };
 
 } /* namespace link_prober */

From c974691246c9da597f11458f411ddc72f86e8b0e Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 3 Jan 2022 18:54:55 +0000
Subject: [PATCH 02/20] Calculate pck loss rate.

---
 src/link_prober/LinkProber.cpp | 6 ++++--
 src/link_prober/LinkProber.h   | 3 ++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index 9932e839..512664ba 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -399,10 +399,10 @@ void LinkProber::handleTimeout(boost::system::error_code errorCode)
         mContinuousIcmpUnknownEventCount++; 
         mIcmpUnknownEventCount++;
 
-        MUXLOGINFO(boost::format("%s: continusous ICMP Unknown event count: %d, total Unknown event count since initialization: %d") %
+        MUXLOGINFO(boost::format("%s: continusous ICMP Unknown event count: %d, pck loss rate since initialization: %.2f %%") %
             mMuxPortConfig.getPortName() %
             mContinuousIcmpUnknownEventCount %
-            mIcmpUnknownEventCount
+            (static_cast<double_t>(mIcmpUnknownEventCount) / mIcmpPacketCount * 100) 
         );
     }
 
@@ -443,6 +443,8 @@ void LinkProber::startRecv()
 {
     MUXLOGTRACE(mMuxPortConfig.getPortName());
 
+    mIcmpPacketCount++;
+
     mStream.async_read_some(
         boost::asio::buffer(mRxBuffer, MUX_MAX_ICMP_BUFFER_SIZE),
         mStrand.wrap(boost::bind(
diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
index f6297e59..2d55cac4 100644
--- a/src/link_prober/LinkProber.h
+++ b/src/link_prober/LinkProber.h
@@ -375,7 +375,8 @@ class LinkProber
     bool mSuspendTx = false;
 
     uint32_t mContinuousIcmpUnknownEventCount = 0;
-    uint32_t mIcmpUnknownEventCount = 0;
+    uint64_t mIcmpUnknownEventCount = 0;
+    uint64_t mIcmpPacketCount = 0;
 };
 
 } /* namespace link_prober */

From 1fd40a81aa27935063802bc1059ec42bba58b84f Mon Sep 17 00:00:00 2001
From: Longxiang Lyu <lolv@microsoft.com>
Date: Mon, 13 Dec 2021 14:54:58 +0000
Subject: [PATCH 03/20] Improve PR template (#16)

Signed-off-by: Longxiang Lyu <lolv@microsoft.com>
---
 .github/pull_request_template.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e18f6929..cfdbefd2 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -23,8 +23,9 @@ Fixes # (issue)
 -->
 
 - [ ] Bug fix
-- [ ] Test case(new/improvement)
-
+- [ ] New feature
+- [ ] Doc/Design
+- [ ] Unit test
 
 ### Approach
 #### What is the motivation for this PR?
@@ -35,8 +36,6 @@ Fixes # (issue)
 
 #### Any platform specific information?
 
-#### Supported testbed topology if it's a new test case?
-
 ### Documentation 
 <!--
 (If it's a new feature, new test case)

From 442cac794a05845db1a3de36e316286d8012474d Mon Sep 17 00:00:00 2001
From: Ying Xie <ying.xie@microsoft.com>
Date: Tue, 4 Jan 2022 23:32:49 +0000
Subject: [PATCH 04/20] [linkmgrd] update README, set coding style guidance
 (#15)

Signed-off-by: Ying Xie <ying.xie@microsoft.com>
---
 README.md | 81 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 58 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 5cd7cecf..86799b06 100644
--- a/README.md
+++ b/README.md
@@ -1,33 +1,68 @@
 # Project
 
-> This repo has been populated by an initial template to help get you started. Please
-> make sure to update the content to build a great experience for community-building.
 
-As the maintainer of this project, please make a few updates:
+This repo tracks the source code of linkmgrd for dual-ToR topology.
 
-- Improving this README.MD file to provide a great experience
-- Updating SUPPORT.MD with content about this project's support experience
-- Understanding the security reporting process in SECURITY.MD
-- Remove this section from the README
+# Coding style guide
 
-## Contributing
+Please follow the simple rules laid out below (will update as needed):
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+## Spacing
+- Please configure your editor to enable expand white spaces. Do not use tab for spacing in source code.
 
-When you submit a pull request, a CLA bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
+- Please configure your editor to set tab stop at 4 spaces. And use 4 white spaces for each indentation level.
 
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+- Please leave one white space after following keywords:
 
-## Trademarks
+```
+    if (...)
 
-This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
-trademarks or logos is subject to and must follow 
-[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
-Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
-Any use of third-party trademarks or logos are subject to those third-party's policies.
+    while (...)
+
+    for (...)
+```
+
+## Curly braces
+- Function definition: start a new line
+```
+void function(...)
+{
+    // code block
+}
+```
+
+- if/for/while condition when condition is one liner: same line
+```
+if (...) {
+    // code block
+}
+```
+
+- if/for/while condition expands multiple lines: start a new line with ending ')' and '{'
+```
+if (...
+    ...
+) {
+    // code block
+}
+```
+
+## Indentations
+- Please use 4 white spaces for each indentation level.
+
+- Multiple line condition indentation for (if/while/for) and/or function calls: indent one level and the ')' indent at the same level of the keyword or function name
+```
+if (condition line 1 ..
+    condition line 2 (indent to '(' above)
+    condition line 3
+) {
+    // code block
+}
+
+foo(
+    parameter1,
+    parameter2,
+    parameter3
+);
+
+```

From c9a3cccdae0d5fc044330ab1baf9cba015045bf4 Mon Sep 17 00:00:00 2001
From: Longxiang Lyu <lolv@microsoft.com>
Date: Thu, 6 Jan 2022 02:00:42 +0000
Subject: [PATCH 05/20] Add missing intermediate states (#16)

Signed-off-by: Longxiang Lyu <lolv@microsoft.com>
---
 test/LinkManagerStateMachineTest.cpp | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 4a2d106b..59bc8946 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -757,6 +757,8 @@ TEST_F(LinkManagerStateMachineTest, StandbyStateToProberUnknownMuxUnknownLinkUp)
     VALIDATE_STATE(Standby, Wait, Up);
 
     postLinkProberEvent(link_prober::LinkProberState::Unknown, 2);
+    VALIDATE_STATE(Unknown, Wait, Up);
+
     handleProbeMuxState("unknown", 3);
     VALIDATE_STATE(Unknown, Unknown, Up);
 
@@ -776,7 +778,10 @@ TEST_F(LinkManagerStateMachineTest, StandbyStateToProberUnknownMuxUnknownLinkUp)
     EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
 
     // swss notification
-    handleMuxState("unknown", 4);
+    handleMuxState("unknown", 3);
+    VALIDATE_STATE(Wait, Unknown, Up);
+
+    runIoService(2);
     VALIDATE_STATE(Wait, Wait, Up);
 
     postLinkProberEvent(link_prober::LinkProberState::Active, 2);
@@ -801,7 +806,10 @@ TEST_F(LinkManagerStateMachineTest, ProberUnknownMuxUnknownLinkDown)
     handleMuxState("unknown", 4);
     VALIDATE_STATE(Active, Unknown, Down);
 
-    postLinkProberEvent(link_prober::LinkProberState::Unknown, 3);
+    postLinkProberEvent(link_prober::LinkProberState::Unknown, 2);
+    VALIDATE_STATE(Unknown, Unknown, Down);
+
+    runIoService(2);
     VALIDATE_STATE(Unknown, Wait, Down);
 
     handleProbeMuxState("unknown", 4);
@@ -839,7 +847,10 @@ TEST_F(LinkManagerStateMachineTest, ProberWaitMuxUnknownLinkDown)
     EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
 
     // swss notification
-    handleMuxState("unknown", 4);
+    handleMuxState("unknown", 3);
+    VALIDATE_STATE(Wait, Unknown, Up);
+
+    runIoService(2);
     VALIDATE_STATE(Wait, Wait, Up);
 
     handleLinkState("down", 3);

From aa037658ee85c8ef7ca7e93a82177421e99b781d Mon Sep 17 00:00:00 2001
From: Longxiang Lyu <lolv@microsoft.com>
Date: Mon, 13 Dec 2021 08:28:37 +0000
Subject: [PATCH 06/20] Add TLV support to ICMP payload (#11)

Signed-off-by: Longxiang Lyu <lolv@microsoft.com>
---
 src/link_prober/IcmpPayload.cpp |   4 +-
 src/link_prober/IcmpPayload.h   |  32 ++++-
 src/link_prober/LinkProber.cpp  | 201 ++++++++++++++++++++++++++------
 src/link_prober/LinkProber.h    |  64 +++++++++-
 test/LinkProberTest.cpp         | 130 ++++++++++++++++++---
 test/LinkProberTest.h           |   8 ++
 6 files changed, 380 insertions(+), 59 deletions(-)

diff --git a/src/link_prober/IcmpPayload.cpp b/src/link_prober/IcmpPayload.cpp
index 528da60e..2d76acce 100644
--- a/src/link_prober/IcmpPayload.cpp
+++ b/src/link_prober/IcmpPayload.cpp
@@ -46,9 +46,9 @@ uint32_t IcmpPayload::mVersion = 0;
 IcmpPayload::IcmpPayload() :
     cookie(htonl(mCookie)),
     version(htonl(mVersion)),
-    command(htonl(static_cast<uint32_t> (Command::COMMAND_NONE)))
+    seq(0)
 {
-    memcpy(un.uuid.data, mGuid.data, sizeof(un.uuid.data));
+    memcpy(uuid, mGuid.data, sizeof(uuid));
 }
 
 //
diff --git a/src/link_prober/IcmpPayload.h b/src/link_prober/IcmpPayload.h
index bdcb28ee..90a71d10 100644
--- a/src/link_prober/IcmpPayload.h
+++ b/src/link_prober/IcmpPayload.h
@@ -42,13 +42,36 @@ __BEGIN_DECLS
 
 namespace link_prober
 {
+/**
+ *@enum TlvType
+ *
+ *@brief Supported TLV types
+ */
+enum TlvType: uint8_t {
+    TLV_COMMAND = 0x5,
+    TLV_DUMMY = 0xfe,
+    TLV_SENTINEL = 0xff,
+};
+
+struct TlvHead {
+    uint8_t type;
+    uint16_t length;
+} __attribute__((packed));
+
+struct Tlv {
+    TlvHead tlvhead;
+    union {
+        uint8_t command;
+        uint8_t data[1];
+    };
+} __attribute__((packed));
 
 /**
  *@enum Command
  *
  *@brief Command to be sent to peer device
  */
-enum class Command: uint32_t {
+enum class Command: uint8_t {
     COMMAND_NONE,
     COMMAND_SWITCH_ACTIVE,
 
@@ -63,11 +86,8 @@ enum class Command: uint32_t {
 struct IcmpPayload {
     uint32_t cookie;
     uint32_t version;
-    union {
-        uint64_t guid[2];
-        boost::uuids::uuid uuid;
-    } un;
-    uint32_t command;
+    uint8_t uuid[8];
+    uint64_t seq;
 
     /**
     *@method IcmpPayload
diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index 512664ba..5cafa146 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -232,18 +232,23 @@ void LinkProber::handleUpdateEthernetFrame()
 //
 void LinkProber::handleSendSwitchCommand()
 {
-    IcmpPayload *icmpPayload = reinterpret_cast<IcmpPayload *> (
-        mTxBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
-    );
-    icmpPayload->command = htonl(static_cast<uint32_t> (Command::COMMAND_SWITCH_ACTIVE));
+    resetTxBufferTlv();
+    appendTlvCommand(Command::COMMAND_SWITCH_ACTIVE);
+    appendTlvSentinel();
 
+    size_t totalPayloadSize = mTxPacketSize - mPacketHeaderSize;
+    iphdr *ipHeader = reinterpret_cast<iphdr *> (mTxBuffer.data() + sizeof(ether_header));
     icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (mTxBuffer.data() + sizeof(ether_header) + sizeof(iphdr));
-    computeChecksum(icmpHeader, sizeof(icmphdr) + sizeof(*icmpPayload));
+    computeChecksum(icmpHeader, sizeof(icmphdr) + totalPayloadSize);
+    ipHeader->tot_len = htons(sizeof(iphdr) + sizeof(icmphdr) + totalPayloadSize);
 
     sendHeartbeat();
 
-    icmpPayload->command = htonl(static_cast<uint32_t> (Command::COMMAND_NONE));
-    computeChecksum(icmpHeader, sizeof(icmphdr) + sizeof(*icmpPayload));
+    resetTxBufferTlv();
+    appendTlvSentinel();
+    totalPayloadSize = mTxPacketSize - mPacketHeaderSize;
+    computeChecksum(icmpHeader, sizeof(icmphdr) + totalPayloadSize);
+    ipHeader->tot_len = htons(sizeof(iphdr) + sizeof(icmphdr) + totalPayloadSize);
 
     // inform the composite state machine about commend send completion
     boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
@@ -269,12 +274,7 @@ void LinkProber::sendHeartbeat()
     if (!mSuspendTx) {
         updateIcmpSequenceNo();
         boost::system::error_code errorCode;
-        mStream.write_some(
-            boost::asio::buffer(
-                mTxBuffer.data(), sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr) + sizeof(IcmpPayload)
-            ),
-            errorCode
-        );
+        mStream.write_some(boost::asio::buffer(mTxBuffer.data(), mTxPacketSize), errorCode);
 
         if (errorCode) {
             MUXLOGTRACE(mMuxPortConfig.getPortName() + ": Failed to send heartbeat!");
@@ -284,6 +284,30 @@ void LinkProber::sendHeartbeat()
     }
 }
 
+//
+// ---> handleTlvCommandRecv(Tlv *tlvPtr,, bool isPeer);
+//
+// handle packet reception
+//
+void LinkProber::handleTlvCommandRecv(
+    Tlv *tlvPtr,
+    bool isPeer
+)
+{
+    if (isPeer) {
+        if (tlvPtr->command == static_cast<uint8_t> (Command::COMMAND_SWITCH_ACTIVE)) {
+            boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
+            boost::asio::io_service &ioService = strand.context();
+            ioService.post(strand.wrap(boost::bind(
+                static_cast<void (LinkProberStateMachine::*) (SwitchActiveRequestEvent&)>
+                    (&LinkProberStateMachine::processEvent),
+                &mLinkProberStateMachine,
+                LinkProberStateMachine::getSwitchActiveRequestEvent()
+            )));
+        }
+    }
+}
+
 //
 // ---> handleRecv(const boost::system::error_code& errorCode, size_t bytesTransferred);
 //
@@ -302,8 +326,14 @@ void LinkProber::handleRecv(
             mRxBuffer.data() + sizeof(ether_header) + sizeof(iphdr)
         );
 
+        MUXLOGTRACE(boost::format("%s: Got data from: %s, size: %d") %
+            mMuxPortConfig.getPortName() %
+            boost::asio::ip::address_v4(ntohl(ipHeader->saddr)).to_string() %
+            (bytesTransferred - sizeof(iphdr) - sizeof(ether_header))
+        );
+
         IcmpPayload *icmpPayload = reinterpret_cast<IcmpPayload *> (
-            mRxBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
+            mRxBuffer.data() + mPacketHeaderSize
         );
 
         if (ntohl(icmpPayload->cookie) == IcmpPayload::getCookie() &&
@@ -313,17 +343,17 @@ void LinkProber::handleRecv(
                 mMuxPortConfig.getPortName() %
                 mMuxPortConfig.getBladeIpv4Address().to_string()
             );
-            if (memcmp(icmpPayload->un.uuid.data, IcmpPayload::getGuidData(),
-                       IcmpPayload::getGuid().size()) == 0) {
+            bool isMatch = (memcmp(icmpPayload->uuid, IcmpPayload::getGuidData(), sizeof(icmpPayload->uuid)) == 0);
+            if (isMatch) {
                 MUXLOGTRACE(boost::format("%s: Matching Guid") % mMuxPortConfig.getPortName());
                 // echo reply for an echo request generated by this/active ToR
                 mRxSelfSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpSelfEvent());
                 mContinuousIcmpUnknownEventCount = 0;
             } else {
-                // echo reply for an echo request generated by peer ToR
                 mRxPeerSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpPeerEvent());
+<<<<<<< HEAD
                 mContinuousIcmpUnknownEventCount = 0;
                 if (ntohl(icmpPayload->command) == static_cast<uint32_t> (Command::COMMAND_SWITCH_ACTIVE)) {
                     boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
@@ -334,17 +364,44 @@ void LinkProber::handleRecv(
                         &mLinkProberStateMachine,
                         LinkProberStateMachine::getSwitchActiveRequestEvent()
                     )));
+=======
+            }
+
+            size_t nextTlvOffset = mTlvStartOffset;
+            size_t nextTlvSize = 0;
+            bool stopProcessTlv = false;
+            while ((nextTlvSize = findNextTlv(nextTlvOffset, bytesTransferred)) > 0 && !stopProcessTlv) {
+                Tlv *nextTlvPtr = reinterpret_cast<Tlv *> (mRxBuffer.data() + nextTlvOffset);
+                switch (nextTlvPtr->tlvhead.type) {
+                    case TlvType::TLV_COMMAND: {
+                        handleTlvCommandRecv(nextTlvPtr, !isMatch);
+                        break;
+                    }
+                    case TlvType::TLV_SENTINEL: {
+                        // sentinel TLV, stop processing
+                        stopProcessTlv = true;
+                        break;
+                    }
+                    default: {
+                        // try to skip unknown TLV with valid length(>0)
+                        stopProcessTlv = (nextTlvSize == sizeof(Tlv));
+                        break;
+                    }
+>>>>>>> 12b9951... Add TLV support to ICMP payload (#11)
                 }
+                nextTlvOffset += nextTlvSize;
+            }
+
+            if (nextTlvOffset < bytesTransferred) {
+                size_t BytesNotProcessed = bytesTransferred - nextTlvOffset;
+                MUXLOGTRACE(boost::format("%s: %d bytes in RxBuffer not processed") %
+                    mMuxPortConfig.getPortName() %
+                    BytesNotProcessed
+                );
             }
         } else {
             // Unknown ICMP packet, ignore.
         }
-
-        MUXLOGTRACE(boost::format("%s: Got data from: %s, size: %d") %
-            mMuxPortConfig.getPortName() %
-            boost::asio::ip::address_v4(ntohl(ipHeader->saddr)).to_string() %
-            (bytesTransferred - sizeof(iphdr) - sizeof(ether_header))
-        );
         // start another receive to consume as much as possible of backlog packets if any
         startRecv();
     }
@@ -501,11 +558,15 @@ void LinkProber::startTimer()
 uint32_t LinkProber::calculateChecksum(uint16_t *data, size_t size)
 {
     uint32_t sum = 0;
-    size_t offset = 0;
 
-    do {
-        sum += ntohs(data[offset++]);
-    } while (offset < size);
+    while (size > 1) {
+        sum += ntohs(*data++);
+        size -= sizeof(uint16_t);
+    }
+
+    if (size) {
+        sum += ntohs(static_cast<uint16_t> ((*reinterpret_cast<uint8_t *> (data))));
+    }
 
     return sum;
 }
@@ -531,7 +592,7 @@ void LinkProber::computeChecksum(icmphdr *icmpHeader, size_t size)
 {
     icmpHeader->checksum = 0;
     mIcmpChecksum = calculateChecksum(
-        reinterpret_cast<uint16_t *> (icmpHeader), size / 2
+        reinterpret_cast<uint16_t *> (icmpHeader), size
     );
     addChecksumCarryover(&icmpHeader->checksum, mIcmpChecksum);
 }
@@ -545,7 +606,7 @@ void LinkProber::computeChecksum(iphdr *ipHeader, size_t size)
 {
     ipHeader->check = 0;
     mIpChecksum = calculateChecksum(
-        reinterpret_cast<uint16_t *> (ipHeader), size / 2
+        reinterpret_cast<uint16_t *> (ipHeader), size
     );
     addChecksumCarryover(&ipHeader->check, mIpChecksum);
 }
@@ -563,10 +624,17 @@ void LinkProber::initializeSendBuffer()
     ethHeader->ether_type = htons(ETHERTYPE_IP);
 
     iphdr *ipHeader = reinterpret_cast<iphdr *> (mTxBuffer.data() + sizeof(ether_header));
+    icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (mTxBuffer.data() + sizeof(ether_header) + sizeof(iphdr));
+
+    new (mTxBuffer.data() + mPacketHeaderSize) IcmpPayload();
+    resetTxBufferTlv();
+    appendTlvSentinel();
+    size_t totalPayloadSize = mTxPacketSize - mPacketHeaderSize;
+
     ipHeader->ihl = sizeof(iphdr) >> 2;
     ipHeader->version = IPVERSION;
     ipHeader->tos = 0xb8;
-    ipHeader->tot_len = htons(sizeof(iphdr) + sizeof(icmphdr) + sizeof(IcmpPayload));
+    ipHeader->tot_len = htons(sizeof(iphdr) + sizeof(icmphdr) + totalPayloadSize);
     ipHeader->id = static_cast<uint16_t> (rand());
     ipHeader->frag_off = 0;
     ipHeader->ttl = 64;
@@ -576,14 +644,12 @@ void LinkProber::initializeSendBuffer()
     ipHeader->daddr = htonl(mMuxPortConfig.getBladeIpv4Address().to_v4().to_uint());
     computeChecksum(ipHeader, ipHeader->ihl << 2);
 
-    icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (mTxBuffer.data() + sizeof(ether_header) + sizeof(iphdr));
     icmpHeader->type = ICMP_ECHO;
     icmpHeader->code = 0;
     icmpHeader->un.echo.id = htons(mMuxPortConfig.getServerId());
     icmpHeader->un.echo.sequence = htons(mTxSeqNo);
 
-    IcmpPayload *icmpPayload = new (mTxBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)) IcmpPayload();
-    computeChecksum(icmpHeader, sizeof(icmphdr) + sizeof(*icmpPayload));
+    computeChecksum(icmpHeader, sizeof(icmphdr) + totalPayloadSize);
 }
 
 //
@@ -603,4 +669,73 @@ void LinkProber::updateIcmpSequenceNo()
     addChecksumCarryover(&icmpHeader->checksum, mIcmpChecksum);
 }
 
+//
+// ---> findNextTlv
+//
+// Find next TLV to process in rxBuffer
+//
+size_t LinkProber::findNextTlv(size_t readOffset, size_t bytesTransferred)
+{
+    size_t tlvSize = 0;
+    if (readOffset + sizeof(TlvHead) <= bytesTransferred) {
+        Tlv *tlvPtr = reinterpret_cast<Tlv *> (mRxBuffer.data() + readOffset);
+        tlvSize = (sizeof(TlvHead) + ntohs(tlvPtr->tlvhead.length));
+        if (readOffset + tlvSize > bytesTransferred) {
+            tlvSize = 0;
+        }
+    }
+    return tlvSize;
+}
+
+//
+// ---> appendTlvCommand
+//
+// Append TlvCommand to the end of txBuffer
+//
+size_t LinkProber::appendTlvCommand(Command commandType)
+{
+    size_t tlvSize = sizeof(TlvHead) + sizeof(Command);
+    assert(mTxPacketSize + tlvSize <= MUX_MAX_ICMP_BUFFER_SIZE);
+    Tlv *tlvPtr = reinterpret_cast<Tlv *> (mTxBuffer.data() + mTxPacketSize);
+    tlvPtr->tlvhead.type = TlvType::TLV_COMMAND;
+    tlvPtr->tlvhead.length = htons(sizeof(Command));
+    tlvPtr->command = static_cast<uint8_t> (commandType);
+    mTxPacketSize += tlvSize;
+    return tlvSize;
+}
+
+//
+// ---> appendTlvSentinel
+//
+// Append TlvSentinel to the end of txBuffer
+//
+size_t LinkProber::appendTlvSentinel()
+{
+    size_t tlvSize = sizeof(TlvHead);
+    assert(mTxPacketSize + tlvSize <= MUX_MAX_ICMP_BUFFER_SIZE);
+    Tlv *tlvPtr = reinterpret_cast<Tlv *> (mTxBuffer.data() + mTxPacketSize);
+    tlvPtr->tlvhead.type = TlvType::TLV_SENTINEL;
+    tlvPtr->tlvhead.length = 0;
+    mTxPacketSize += tlvSize;
+    return tlvSize;
+}
+
+//
+// ---> appendTlvDummy
+//
+// Append a dummy TLV, test purpose only
+//
+size_t LinkProber::appendTlvDummy(size_t paddingSize, int seqNo)
+{
+    size_t tlvSize = sizeof(TlvHead) + paddingSize + sizeof(uint32_t);
+    assert(mTxPacketSize + tlvSize <= MUX_MAX_ICMP_BUFFER_SIZE);
+    Tlv *tlvPtr = reinterpret_cast<Tlv *> (mTxBuffer.data() + mTxPacketSize);
+    tlvPtr->tlvhead.type = TlvType::TLV_DUMMY;
+    tlvPtr->tlvhead.length = htons(paddingSize + sizeof(uint32_t));
+    memset(tlvPtr->data, 0, paddingSize);
+    *(reinterpret_cast<uint32_t *> (tlvPtr->data + paddingSize)) = htonl(seqNo);
+    mTxPacketSize += tlvSize;
+    return tlvSize;
+}
+
 } /* namespace link_prober */
diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
index 2d55cac4..d83aeaf3 100644
--- a/src/link_prober/LinkProber.h
+++ b/src/link_prober/LinkProber.h
@@ -188,6 +188,21 @@ class LinkProber
     */
     void sendHeartbeat();
 
+    /**
+    *@method handleTlvCommandRecv
+    *
+    *@brief handle TLV command
+    *
+    *@param tlvPtr (in)     Tlv ptr points to the start of TlvCommand in mRxBuffer
+    *@param isPeer (in)     True if the reply received is from the peer ToR
+    *
+    *@return none
+    */
+    void handleTlvCommandRecv(
+        Tlv *tlvPtr,
+        bool isPeer
+    );
+
     /**
     *@method handleRecv
     *
@@ -338,10 +353,53 @@ class LinkProber
     *
     *@brief getter for TxBuffer used for testing
     *
-    *@return CRC checksum
+    *@return tx buffer
     */
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> getTxBuffer() {return mTxBuffer;};
 
+    /**
+    *@method findNextTlv
+    *
+    *@brief Find next TLV in rxBuffer starting at readOffset
+    *
+    *@param readOffset (in)         starting offset to read
+    *@param bytesTransferred (in)   total bytes received in rxBuffer
+    *
+    *@return the next TLV size
+    */
+    size_t findNextTlv(size_t readOffset, size_t bytesTransferred);
+
+    void resetTxBufferTlv() {mTxPacketSize = mTlvStartOffset;};
+
+    /**
+    *@method appendTlvCommand
+    *
+    *@brief append TlvCommand to txBuffer
+    *
+    *@param commandType (in)    command type
+    *
+    *@return the appended TLV size
+    */
+    size_t appendTlvCommand(Command commandType = Command::COMMAND_SWITCH_ACTIVE);
+
+    /**
+    *@method appendTlvSentinel
+    *
+    *@brief append TlvSentinel to txBuffer
+    *
+    *@return the appended TLV size
+    */
+    size_t appendTlvSentinel();
+
+    /**
+    *@method appendTlvDummy
+    *
+    *@brief append dummy TLV, test purpose only
+    *
+    *@return the appended TLV size
+    */
+    size_t appendTlvDummy(size_t paddingSize, int seqNo);
+
     friend class test::LinkProberTest;
 
 private:
@@ -359,6 +417,9 @@ class LinkProber
     uint32_t mIcmpChecksum = 0;
     uint32_t mIpChecksum = 0;
 
+    static const size_t mPacketHeaderSize = sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr);
+    static const size_t mTlvStartOffset = sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr) + sizeof(IcmpPayload);
+
     boost::asio::io_service::strand mStrand;
     boost::asio::deadline_timer mDeadlineTimer;
     boost::asio::deadline_timer mSuspendTimer;
@@ -369,6 +430,7 @@ class LinkProber
 
     int mSocket = 0;
 
+    std::size_t mTxPacketSize;
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> mTxBuffer;
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> mRxBuffer;
 
diff --git a/test/LinkProberTest.cpp b/test/LinkProberTest.cpp
index 31a0d0c0..c2316c31 100644
--- a/test/LinkProberTest.cpp
+++ b/test/LinkProberTest.cpp
@@ -49,6 +49,26 @@ LinkProberTest::LinkProberTest() :
     mMuxConfig.setTimeoutIpv4_msec(1);
 }
 
+size_t LinkProberTest::appendTlvCommand(link_prober::Command commandType)
+{
+    return mLinkProber.appendTlvCommand(commandType);
+}
+
+size_t LinkProberTest::appendTlvSentinel()
+{
+    return mLinkProber.appendTlvSentinel();
+}
+
+size_t LinkProberTest::appendTlvDummy(size_t paddingSize, int seqNo)
+{
+    return mLinkProber.appendTlvDummy(paddingSize, seqNo);
+}
+
+size_t LinkProberTest::findNextTlv(size_t readOffset, size_t bytesTransferred)
+{
+    return mLinkProber.findNextTlv(readOffset, bytesTransferred);
+}
+
 TEST_F(LinkProberTest, InitializeSendBuffer)
 {
     initializeSendBuffer();
@@ -63,34 +83,36 @@ TEST_F(LinkProberTest, InitializeSendBuffer)
     EXPECT_TRUE(ethHeader->ether_type == htons(ETHERTYPE_IP));
 
     iphdr *ipHeader = reinterpret_cast<iphdr *> (txBuffer.data() + sizeof(ether_header));
+    icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (txBuffer.data() + sizeof(ether_header) + sizeof(iphdr));
+    link_prober::IcmpPayload *icmpPayload = reinterpret_cast<link_prober::IcmpPayload *> (txBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr));
+    link_prober::Tlv *tlvPtr = reinterpret_cast<link_prober::Tlv *> (txBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr) + sizeof(*icmpPayload));
+
     EXPECT_TRUE(ipHeader->ihl == sizeof(iphdr) >> 2);
     EXPECT_TRUE(ipHeader->version == IPVERSION);
     EXPECT_TRUE(ipHeader->tos == 0xb8);
-    EXPECT_TRUE(ipHeader->tot_len == htons(sizeof(iphdr) + sizeof(icmphdr) + sizeof(link_prober::IcmpPayload)));
+    EXPECT_TRUE(ipHeader->tot_len == htons(sizeof(iphdr) + sizeof(icmphdr) + sizeof(link_prober::IcmpPayload) + sizeof(link_prober::TlvHead)));
     EXPECT_TRUE(ipHeader->frag_off == 0);
     EXPECT_TRUE(ipHeader->ttl == 64);
     EXPECT_TRUE(ipHeader->protocol == IPPROTO_ICMP);
-    EXPECT_TRUE(ipHeader->check == 62663);
+    EXPECT_TRUE(ipHeader->check == 62919);
     EXPECT_TRUE(ipHeader->saddr == htonl(mFakeMuxPort.getMuxPortConfig().getLoopbackIpv4Address().to_v4().to_uint()));
     EXPECT_TRUE(ipHeader->daddr == htonl(mFakeMuxPort.getMuxPortConfig().getBladeIpv4Address().to_v4().to_uint()));
 
-    icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (txBuffer.data() + sizeof(ether_header) + sizeof(iphdr));
     EXPECT_TRUE(icmpHeader->type == ICMP_ECHO);
     EXPECT_TRUE(icmpHeader->code == 0);
     EXPECT_TRUE(icmpHeader->un.echo.id == htons(mFakeMuxPort.getMuxPortConfig().getServerId()));
     EXPECT_TRUE(icmpHeader->un.echo.sequence == htons(0xffff));
 
-    link_prober::IcmpPayload *icmpPayload = new (
-        txBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
-    ) link_prober::IcmpPayload();
-
     EXPECT_TRUE(icmpPayload->cookie == htonl(link_prober::IcmpPayload::getCookie()));
     EXPECT_TRUE(icmpPayload->version == htonl(link_prober::IcmpPayload::getVersion()));
     EXPECT_TRUE(memcmp(
-        icmpPayload->un.uuid.data,
+        icmpPayload->uuid,
         link_prober::IcmpPayload::getGuidData(),
-        link_prober::IcmpPayload::getGuid().size()
+        sizeof(icmpPayload->uuid)
     ) == 0);
+
+    EXPECT_TRUE(tlvPtr->tlvhead.type == link_prober::TlvType::TLV_SENTINEL);
+    EXPECT_TRUE(tlvPtr->tlvhead.length == 0);
 }
 
 TEST_F(LinkProberTest, CalculateChecksum)
@@ -99,11 +121,11 @@ TEST_F(LinkProberTest, CalculateChecksum)
         getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
     ) link_prober::IcmpPayload();
     boost::uuids::uuid guid = boost::lexical_cast<boost::uuids::uuid> ("44f49d86-c312-414b-b6a1-be82901ac459");
-    memcpy(icmpPayload->un.uuid.data, guid.data, guid.size());
+    memcpy(icmpPayload->uuid, guid.data, sizeof(icmpPayload->uuid));
     initializeSendBuffer();
 
     icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr));
-    EXPECT_TRUE(icmpHeader->checksum == 12355);
+    EXPECT_TRUE(icmpHeader->checksum == 12100);
 }
 
 TEST_F(LinkProberTest, UpdateEthernetFrame)
@@ -112,11 +134,11 @@ TEST_F(LinkProberTest, UpdateEthernetFrame)
         getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
     ) link_prober::IcmpPayload();
     boost::uuids::uuid guid = boost::lexical_cast<boost::uuids::uuid> ("44f49d86-c312-414b-b6a1-be82901ac459");
-    memcpy(icmpPayload->un.uuid.data, guid.data, guid.size());
+    memcpy(icmpPayload->uuid, guid.data, sizeof(icmpPayload->uuid));
     handleUpdateEthernetFrame();
 
     icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr));
-    EXPECT_TRUE(icmpHeader->checksum == 12355);
+    EXPECT_TRUE(icmpHeader->checksum == 12100);
 }
 
 TEST_F(LinkProberTest, UpdateSequenceNo)
@@ -125,7 +147,7 @@ TEST_F(LinkProberTest, UpdateSequenceNo)
         getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
     ) link_prober::IcmpPayload();
     boost::uuids::uuid guid = boost::lexical_cast<boost::uuids::uuid> ("44f49d86-c312-414b-b6a1-be82901ac459");
-    memcpy(icmpPayload->un.uuid.data, guid.data, guid.size());
+    memcpy(icmpPayload->uuid, guid.data, sizeof(icmpPayload->uuid));
 
     handleUpdateEthernetFrame();
 
@@ -134,7 +156,7 @@ TEST_F(LinkProberTest, UpdateSequenceNo)
     handleUpdateSequenceNumber();
 
     icmphdr *icmpHeader = reinterpret_cast<icmphdr *> (getTxBuffer().data() + sizeof(ether_header) + sizeof(iphdr));
-    EXPECT_TRUE(icmpHeader->checksum == 12099);
+    EXPECT_TRUE(icmpHeader->checksum == 11844);
 
     EXPECT_TRUE(getRxSelfSeqNo() + 1 == ntohs(icmpHeader->un.echo.sequence));
     EXPECT_TRUE(getRxPeerSeqNo() + 1 == ntohs(icmpHeader->un.echo.sequence));
@@ -150,9 +172,9 @@ TEST_F(LinkProberTest, GenerateGuid)
         txBuffer.data() + sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr)
     ) link_prober::IcmpPayload();
     EXPECT_TRUE(memcmp(
-        icmpPayload->un.uuid.data,
+        icmpPayload->uuid,
         link_prober::IcmpPayload::getGuidData(),
-        link_prober::IcmpPayload::getGuid().size()
+        sizeof(icmpPayload->uuid)
     ) == 0);
 }
 
@@ -183,6 +205,80 @@ TEST_F(LinkProberTest, UpdateToRMac)
     EXPECT_TRUE(ipHeader->daddr == htonl(ipAddress.to_v4().to_uint()));
 }
 
+TEST_F(LinkProberTest, ReadWriteTlv)
+{
+    initializeSendBuffer();
+    size_t tlvStartOffset = sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr) + sizeof(link_prober::IcmpPayload);
+    // check initial tx buffer packet size
+    EXPECT_TRUE(getTxPacketSize() == tlvStartOffset + sizeof(link_prober::TlvHead));
+
+    // build txBuffer
+    resetTxBufferTlv();
+    size_t tlvCommandSize = appendTlvCommand(link_prober::Command::COMMAND_SWITCH_ACTIVE);
+    EXPECT_TRUE(tlvCommandSize == (sizeof(link_prober::TlvHead) + sizeof(link_prober::Command)));
+    size_t tlvSentinelSize = appendTlvSentinel();
+    EXPECT_TRUE(tlvSentinelSize == sizeof(link_prober::TlvHead));
+    EXPECT_TRUE(getTxPacketSize() == tlvStartOffset + tlvCommandSize + tlvSentinelSize);
+
+    // build rxBuffer
+    size_t bytesTransferred = getTxPacketSize();
+    memcpy(getRxBufferData(), getTxBufferData(), bytesTransferred);
+
+    // start read TLV from rxBuffer
+    size_t rxReadOffset = tlvStartOffset;
+    size_t tlvSize = findNextTlv(rxReadOffset, bytesTransferred);
+    link_prober::Tlv *tlvPtr = reinterpret_cast<link_prober::Tlv *> (getRxBufferData() + rxReadOffset);
+    EXPECT_TRUE(tlvSize == tlvCommandSize);
+    EXPECT_TRUE(tlvPtr->tlvhead.type == link_prober::TlvType::TLV_COMMAND);
+    EXPECT_TRUE(tlvPtr->tlvhead.length == htons(1));
+    EXPECT_TRUE(tlvPtr->command == static_cast<uint8_t> (link_prober::Command::COMMAND_SWITCH_ACTIVE));
+    rxReadOffset += tlvSize;
+
+    tlvSize = findNextTlv(rxReadOffset, bytesTransferred);
+    tlvPtr = reinterpret_cast<link_prober::Tlv *> (getRxBufferData() + rxReadOffset);
+    EXPECT_TRUE(tlvSize == tlvSentinelSize);
+    EXPECT_TRUE(tlvPtr->tlvhead.type == link_prober::TlvType::TLV_SENTINEL);
+    EXPECT_TRUE(tlvPtr->tlvhead.length == 0);
+    rxReadOffset += tlvSize;
+
+    tlvSize = findNextTlv(rxReadOffset, bytesTransferred);
+    EXPECT_TRUE(tlvSize == 0);
+}
+
+TEST_F(LinkProberTest, ReadWriteVariableSizedTlv)
+{
+    initializeSendBuffer();
+    size_t tlvStartOffset = sizeof(ether_header) + sizeof(iphdr) + sizeof(icmphdr) + sizeof(link_prober::IcmpPayload);
+    // check initial tx buffer packet size
+    EXPECT_TRUE(getTxPacketSize() == tlvStartOffset + sizeof(link_prober::TlvHead));
+
+    // build txBuffer
+    resetTxBufferTlv();
+    std::vector<size_t> paddingSizes{0, 1, 2, 3};
+    for (const size_t &paddingSize : paddingSizes) {
+        size_t tlvSize = appendTlvDummy(paddingSize, paddingSize);
+        EXPECT_TRUE(tlvSize == (sizeof(link_prober::TlvHead) + sizeof(uint32_t) + paddingSize));
+    }
+
+    // build rxBuffer
+    size_t bytesTransferred = getTxPacketSize();
+    memcpy(getRxBufferData(), getTxBufferData(), bytesTransferred);
+
+    // start read TLV from rxBuffer
+    size_t rxReadOffset = tlvStartOffset;
+    for (const size_t &paddingSize : paddingSizes) {
+        size_t tlvSize = findNextTlv(rxReadOffset, bytesTransferred);
+        link_prober::Tlv *tlvPtr = reinterpret_cast<link_prober::Tlv *> (getRxBufferData() + rxReadOffset);
+        EXPECT_TRUE(tlvSize == (sizeof(link_prober::TlvHead) + sizeof(uint32_t) + paddingSize));
+
+        uint32_t *seqNoPtr = reinterpret_cast<uint32_t *> (tlvPtr->data + paddingSize);
+        EXPECT_TRUE(ntohl(*seqNoPtr) == paddingSize);
+        rxReadOffset += tlvSize;
+    }
+
+    EXPECT_TRUE(findNextTlv(rxReadOffset, bytesTransferred) == 0);
+}
+
 TEST_F(LinkProberTest, InitializeException)
 {
     EXPECT_THROW(initialize(), common::SocketErrorException);
diff --git a/test/LinkProberTest.h b/test/LinkProberTest.h
index f505020e..137b4343 100644
--- a/test/LinkProberTest.h
+++ b/test/LinkProberTest.h
@@ -42,7 +42,15 @@ class LinkProberTest: public ::testing::Test
     void initializeSendBuffer() {mLinkProber.initializeSendBuffer();};
     void handleUpdateEthernetFrame() {mLinkProber.handleUpdateEthernetFrame();};
     void handleUpdateSequenceNumber() {mLinkProber.updateIcmpSequenceNo();};
+    void resetTxBufferTlv() {mLinkProber.resetTxBufferTlv();};
+    size_t getTxPacketSize() {return mLinkProber.mTxPacketSize;};
+    size_t appendTlvCommand(link_prober::Command commandType);
+    size_t appendTlvSentinel();
+    size_t appendTlvDummy(size_t paddingSize, int seqNo);
+    size_t findNextTlv(size_t readOffset, size_t bytesTransferred);
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> getTxBuffer() {return mLinkProber.getTxBuffer();};
+    uint8_t *getTxBufferData() {return mLinkProber.mTxBuffer.data();};
+    uint8_t *getRxBufferData() {return mLinkProber.mRxBuffer.data();};
 
     uint16_t getRxSelfSeqNo() {return mLinkProber.mRxSelfSeqNo;};
     uint16_t getRxPeerSeqNo() {return mLinkProber.mRxPeerSeqNo;};

From deef0749b17c9f7b86416ddaa973ed4d9611d079 Mon Sep 17 00:00:00 2001
From: Jing Zhang <zhangjing@microsoft.com>
Date: Wed, 19 Jan 2022 12:59:49 -0800
Subject: [PATCH 07/20] Linkmgrd subscribing State DB route event  (#13)

Summary:
Fixes # (issue)

This PR is to make linkmgrd subscribe events from ROUTE_TABLE in State DB, and react accordingly:
- If any of the two default route state appears to be 'na', linkmgrd should switch to standby.
- If both are 'ok', there will be a mux state probing and what happens next depends on linkmgrd state machine and the probing response.

Type of change:
New feature

Motivation for this PR:
To make linkmgrd subscribe state DB route event, and handle the switchovers.

Documentation:
Related PR: https://github.com/Azure/sonic-swss/pull/2009
---
 src/DbInterface.cpp                          | 58 ++++++++++++++++++++
 src/DbInterface.h                            | 22 ++++++++
 src/MuxManager.cpp                           | 28 ++++++++++
 src/MuxManager.h                             | 16 ++++++
 src/MuxPort.cpp                              | 16 ++++++
 src/MuxPort.h                                | 11 ++++
 src/link_manager/LinkManagerStateMachine.cpp | 22 ++++++++
 src/link_manager/LinkManagerStateMachine.h   | 11 ++++
 test/LinkManagerStateMachineTest.cpp         | 40 ++++++++++++++
 test/LinkManagerStateMachineTest.h           |  1 +
 10 files changed, 225 insertions(+)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index 48712db5..a67502fd 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -730,6 +730,59 @@ void DbInterface::handleMuxStateNotifiction(swss::SubscriberStateTable &statedbP
     processMuxStateNotifiction(entries);
 }
 
+//
+// ---> processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries)
+// 
+// process default route state notification from orchagent
+//
+void DbInterface::processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries)
+{
+    for (auto &entry: entries) {
+        std::string key = kfvKey(entry);
+        std::string op = kfvOp(entry);
+        std::vector<swss::FieldValueTuple> fieldValues = kfvFieldsValues(entry);
+
+        std::vector<swss::FieldValueTuple>::const_iterator cit = std::find_if(
+            fieldValues.cbegin(),
+            fieldValues.cend(),
+            [] (const swss::FieldValueTuple &fv) {return fvField(fv) == "state";}
+        );
+
+        if (cit != fieldValues.cend()) {
+            const std::string field = cit->first;
+            const std::string value = cit->second;
+
+            MUXLOGDEBUG(boost::format("key: %s, operation: %s, field: %s, value: %s") %
+                key %
+                op %
+                field %
+                value
+            );
+
+            if (key == "0.0.0.0/0") {
+                mMuxManagerPtr->addOrUpdateDefaultRouteState(true, value);
+            } else if (key == "::/0") {
+                mMuxManagerPtr->addOrUpdateDefaultRouteState(false, value);
+            } else {
+                MUXLOGFATAL(boost::format("Received Invalid IP: %s") % key );
+            }
+        }
+    }
+}
+
+//
+// ---> handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable);
+//
+// handle Default Route State notification from orchagent
+//
+void DbInterface::handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable)
+{
+    std::deque<swss::KeyOpFieldsValuesTuple> entries;
+
+    statedbRouteTable.pops(entries);
+    processDefaultRouteStateNotification(entries);
+}
+
 //
 // ---> handleSwssNotification();
 //
@@ -752,6 +805,8 @@ void DbInterface::handleSwssNotification()
     swss::SubscriberStateTable appDbMuxResponseTable(appDbPtr.get(), APP_MUX_CABLE_RESPONSE_TABLE_NAME);
     // for getting state db MUX state when orchagent updates it
     swss::SubscriberStateTable stateDbPortTable(stateDbPtr.get(), STATE_MUX_CABLE_TABLE_NAME);
+    // for getting state db default route state 
+    swss::SubscriberStateTable stateDbRouteTable(stateDbPtr.get(), STATE_ROUTE_TABLE_NAME);
 
     getTorMacAddress(configDbPtr);
     getLoopback2InterfaceInfo(configDbPtr);
@@ -771,6 +826,7 @@ void DbInterface::handleSwssNotification()
     swssSelect.addSelectable(&appDbPortTable);
     swssSelect.addSelectable(&appDbMuxResponseTable);
     swssSelect.addSelectable(&stateDbPortTable);
+    swssSelect.addSelectable(&stateDbRouteTable);
     swssSelect.addSelectable(&netlinkNeighbor);
 
     while (mPollSwssNotifcation) {
@@ -797,6 +853,8 @@ void DbInterface::handleSwssNotification()
             handleMuxResponseNotifiction(appDbMuxResponseTable);
         } else if (selectable == static_cast<swss::Selectable *> (&stateDbPortTable)) {
             handleMuxStateNotifiction(stateDbPortTable);
+        } else if (selectable == static_cast<swss::Selectable *> (&stateDbRouteTable)) {
+            handleDefaultRouteStateNotification(stateDbRouteTable);
         } else if (selectable == static_cast<swss::Selectable *> (&netlinkNeighbor)) {
             continue;
         } else {
diff --git a/src/DbInterface.h b/src/DbInterface.h
index 8403da83..1497bf6e 100644
--- a/src/DbInterface.h
+++ b/src/DbInterface.h
@@ -462,6 +462,28 @@ class DbInterface
     */
     void handleSwssNotification();
 
+    /**
+     * @method processDefaultRouteStateNotification
+     * 
+     * @brief process default route state notification from orchagent
+     * 
+     * @param entries reference to state db default route state entries
+     * 
+     * @return none
+    */
+    void processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries);
+
+    /**
+     * @method handleDefaultRouteStateNotification
+     * 
+     * @brief handle Default Route State notification from orchagent
+     * 
+     * @param statedbRouteTable reference to state db route table 
+     * 
+     * @return none
+    */
+    void handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable);
+
 private:
     static std::vector<std::string> mMuxState;
     static std::vector<std::string> mMuxLinkmgrState;
diff --git a/src/MuxManager.cpp b/src/MuxManager.cpp
index 5472d19f..a1bab80c 100644
--- a/src/MuxManager.cpp
+++ b/src/MuxManager.cpp
@@ -234,6 +234,34 @@ void MuxManager::processProbeMuxState(const std::string &portName, const std::st
     }
 }
 
+//
+// ---> addOrUpdateDefaultRouteState(boost::asio::ip::address& address, const std::string &routeState);
+//
+// update default route state based on state db notification
+//
+void MuxManager::addOrUpdateDefaultRouteState(bool is_v4, const std::string &routeState) 
+{
+    if (is_v4) {
+        mIpv4DefaultRouteState = routeState;
+    } else {
+        mIpv6DefaultRouteState = routeState;
+    }
+
+    std::string nextState = "na";
+    // For now we only need IPv4 default route state to be "ok". If we switch to IPv6 in the furture, this will cause an issue. 
+    if (mIpv4DefaultRouteState == "ok") {
+        nextState = "ok";
+    }
+
+    MUXLOGINFO(boost::format("Default route state: %s") % nextState);
+
+    PortMapIterator portMapIterator = mPortMap.begin();
+    while (portMapIterator != mPortMap.end()) {
+        portMapIterator->second->handleDefaultRouteState(nextState);
+        portMapIterator ++;
+    }
+}
+
 //
 // ---> getMuxPortPtrOrThrow(const std::string &portName);
 //
diff --git a/src/MuxManager.h b/src/MuxManager.h
index 59c076b0..eb54b3f1 100644
--- a/src/MuxManager.h
+++ b/src/MuxManager.h
@@ -305,6 +305,19 @@ class MuxManager
     */
     void processProbeMuxState(const std::string &portName, const std::string &muxState);
 
+    /**
+     * @method addOrUpdateDefaultRouteState
+     * 
+     * @brief update default route state based on state db notification
+     * 
+     * @param ipAddress
+     * @param routeState
+     *  
+     * @return none
+     * 
+    */
+    void addOrUpdateDefaultRouteState(bool is_v4, const std::string &routeState);
+
 private:
     /**
     *@method getMuxPortPtrOrThrow
@@ -360,6 +373,9 @@ class MuxManager
     std::shared_ptr<mux::DbInterface> mDbInterfacePtr;
 
     PortMap mPortMap;
+
+    std::string mIpv4DefaultRouteState = "na";
+    std::string mIpv6DefaultRouteState = "na";
 };
 
 } /* namespace mux */
diff --git a/src/MuxPort.cpp b/src/MuxPort.cpp
index 3f0d5499..496d6235 100644
--- a/src/MuxPort.cpp
+++ b/src/MuxPort.cpp
@@ -228,4 +228,20 @@ void MuxPort::handleMuxConfig(const std::string &config)
     )));
 }
 
+// 
+// ---> handleDefaultRouteState(const std::string &routeState);
+// 
+// handles default route state notification
+//
+void MuxPort::handleDefaultRouteState(const std::string &routeState)
+{
+    MUXLOGDEBUG(boost::format("port: %s, state db default route state: %s") % mMuxPortConfig.getPortName() % routeState);
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &link_manager::LinkManagerStateMachine::handleDefaultRouteStateNotification,
+        &mLinkManagerStateMachine,
+        routeState
+    )));
+}
 } /* namespace mux */
diff --git a/src/MuxPort.h b/src/MuxPort.h
index 2250d535..3d2981a2 100644
--- a/src/MuxPort.h
+++ b/src/MuxPort.h
@@ -252,6 +252,17 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
     */
     void handleMuxConfig(const std::string &config);
 
+    /**
+     * @method handleDefaultRouteState(const std::string &routeState)
+     * 
+     * @brief handles default route state notification
+     * 
+     * @param routeState 
+     * 
+     * @return none
+    */
+    void handleDefaultRouteState(const std::string &routeState);
+
 protected:
     friend class test::MuxManagerTest;
     friend class test::FakeMuxPort;
diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index 87a7c79f..dd547c99 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -815,6 +815,28 @@ void LinkManagerStateMachine::handleSwitchActiveRequestEvent()
     }
 }
 
+// 
+// ---> handleDefaultRouteStateNotification(const std::string &routeState);
+// 
+// handle default route state notification from routeorch
+//
+void LinkManagerStateMachine::handleDefaultRouteStateNotification(const std::string &routeState)
+{
+    MUXLOGWARNING(boost::format("%s: state db default route state: %s") % mMuxPortConfig.getPortName() % routeState);
+
+    if (mComponentInitState.test(MuxStateComponent)) {
+        if (ms(mCompositeState) != mux_state::MuxState::Label::Standby && routeState == "na") {
+            CompositeState nextState = mCompositeState;
+            enterLinkProberState(nextState, link_prober::LinkProberState::Wait);
+            switchMuxState(nextState, mux_state::MuxState::Label::Standby, true);
+            LOGWARNING_MUX_STATE_TRANSITION(mMuxPortConfig.getPortName(), mCompositeState, nextState);
+            mCompositeState = nextState;
+        } else {
+            enterMuxWaitState(mCompositeState);
+        }
+    } 
+}
+
 //
 // ---> updateMuxLinkmgrState();
 //
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index 0cac9284..6a985f7c 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -476,6 +476,17 @@ class LinkManagerStateMachine: public common::StateMachine,
     */
     void handleSwitchActiveRequestEvent();
 
+    /**
+     * @method handleDefaultRouteStateNotification(const std::string &routeState)
+     * 
+     * @brief handle default route state notification from routeorch
+     * 
+     * @param routeState
+     * 
+     * @return none
+    */
+    void handleDefaultRouteStateNotification(const std::string &routeState);
+
 private:
     /**
     *@method updateMuxLinkmgrState
diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 59bc8946..dcd416ae 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -230,6 +230,12 @@ void LinkManagerStateMachineTest::setMuxStandby()
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+void LinkManagerStateMachineTest::postDefaultRouteEvent(std::string routeState, uint32_t count)
+{
+    mFakeMuxPort.handleDefaultRouteState(routeState);
+    runIoService(count);
+}
+
 TEST_F(LinkManagerStateMachineTest, MuxActiveSwitchOver)
 {
     setMuxActive();
@@ -1013,4 +1019,38 @@ TEST_F(LinkManagerStateMachineTest, MuxStandby2Unknown2Error)
     VALIDATE_STATE(Standby, Error, Down);
 }
 
+TEST_F(LinkManagerStateMachineTest, MuxActivDefaultRouteStateNA) 
+{
+    setMuxActive();
+
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 0);
+    postDefaultRouteEvent("na", 3);
+
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
+
+    postLinkProberEvent(link_prober::LinkProberState::Standby, 3);
+    VALIDATE_STATE(Standby, Wait, Up);
+
+    handleMuxState("standby", 3);
+    VALIDATE_STATE(Standby, Standby, Up);
+}
+
+TEST_F(LinkManagerStateMachineTest, MuxStandbyDefaultRouteStateOK) 
+{
+    setMuxStandby();
+
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 0);
+    postDefaultRouteEvent("ok", 2);
+
+    VALIDATE_STATE(Standby, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 1);
+
+    postLinkProberEvent(link_prober::LinkProberState::Standby, 3);
+    VALIDATE_STATE(Standby, Wait, Up);
+
+    handleMuxState("standby", 3);
+    VALIDATE_STATE(Standby, Standby, Up);
+}
+
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.h b/test/LinkManagerStateMachineTest.h
index 65a3d39b..c752a0e1 100644
--- a/test/LinkManagerStateMachineTest.h
+++ b/test/LinkManagerStateMachineTest.h
@@ -51,6 +51,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void activateStateMachine();
     void setMuxActive();
     void setMuxStandby();
+    void postDefaultRouteEvent(std::string routeState, uint32_t count = 0);
 
 public:
     boost::asio::io_service mIoService;

From e2c4cd12df88ebeeedb434f76310bd1c1e17161c Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Sat, 22 Jan 2022 00:32:11 +0000
Subject: [PATCH 08/20] Add DbInterface related handlers

---
 src/DbInterface.cpp                        | 58 ++++++++++++++++++++++
 src/DbInterface.h                          | 39 +++++++++++++++
 src/MuxPort.h                              | 13 +++++
 src/link_manager/LinkManagerStateMachine.h |  7 +++
 4 files changed, 117 insertions(+)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index a67502fd..7bb1e772 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -46,6 +46,7 @@ constexpr auto DEFAULT_TIMEOUT_MSEC = 1000;
 std::vector<std::string> DbInterface::mMuxState = {"active", "standby", "unknown", "Error"};
 std::vector<std::string> DbInterface::mMuxLinkmgrState = {"uninitialized", "unhealthy", "healthy"};
 std::vector<std::string> DbInterface::mMuxMetrics = {"start", "end"};
+std::vector<std::string> DbInterface::mLinkProbeMetrics = {"pck_loss_start", "pck_loss_end"};
 
 //
 // ---> DbInterface(mux::MuxManager *muxManager);
@@ -161,6 +162,33 @@ void DbInterface::postMetricsEvent(
     )));
 }
 
+// 
+// ---> postLinkProberMetricsEvent(
+//        const std::string &portName, 
+//        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
+//    );
+//
+// post link probe pck loss event to state db 
+void DbInterface::postLinkProberMetricsEvent(
+        const std::string &portName, 
+        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
+)
+{
+    MUXLOGDEBUG(boost::format("%s: posting link prober pck loss event %s") %
+        portName %
+        mLinkProbeMetrics[static_cast<int> (metrics)]
+    );
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &DbInterface::handlePostLinkProberMetrics,
+        this,
+        portName,
+        metrics,
+        boost::posix_time::microsec_clock::universal_time()
+    )));
+}
+
 //
 // ---> initialize();
 //
@@ -184,6 +212,9 @@ void DbInterface::initialize()
         mStateDbMuxMetricsTablePtr = std::make_shared<swss::Table> (
             mStateDbPtr.get(), STATE_MUX_METRICS_TABLE_NAME
         );
+        mStateDbLinkProbeStatsTablePtr = std::make_shared<swss::Table> (
+            mStateDbPtr.get(), LINK_PROBE_STATS_TABLE_NAME
+        );
         mMuxStateTablePtr = std::make_shared<swss::Table> (mStateDbPtr.get(), STATE_MUX_CABLE_TABLE_NAME);
 
         mSwssThreadPtr = std::make_shared<boost::thread> (&DbInterface::handleSwssNotification, this);
@@ -319,6 +350,33 @@ void DbInterface::handlePostMuxMetrics(
     );
 }
 
+// 
+// ---> handlePostLinkProberMetrics(
+//        const std::string portName,
+//        link_manager::LinkManagerStateMachine::LinkProberMetrics,
+//        boost::posix_time::ptime time
+//    );
+//
+// post link prober pck loss event to state db 
+void DbInterface::handlePostLinkProberMetrics(
+    const std::string portName,
+    link_manager::LinkManagerStateMachine::LinkProberMetrics metrics,
+    boost::posix_time::ptime time
+)
+{
+    MUXLOGDEBUG(boost::format("%s: posting link prober pck loss event %s") %
+        portName %
+        mLinkProbeMetrics[static_cast<int> (metrics)]
+    );
+
+    if (metrics == link_manager::LinkManagerStateMachine::LinkProberMetrics::PckLossStart) {
+        mStateDbLinkProbeStatsTablePtr.hdel(portName, mLinkProbeMetrics[0]);
+        mStateDbLinkProbeStatsTablePtr.hdel(portName, mLinkProbeMetrics[1]);
+    }
+
+    mStateDbLinkProbeStatsTablePtr.hset(portName, mLinkProbeMetrics[static_cast<int> (metrics)], boost::posix_time::to_simple_string(time));
+}
+
 //
 // ---> processTorMacAddress(std::string& mac);
 //
diff --git a/src/DbInterface.h b/src/DbInterface.h
index 1497bf6e..1f633a82 100644
--- a/src/DbInterface.h
+++ b/src/DbInterface.h
@@ -44,6 +44,8 @@ class MuxManagerTest;
 
 namespace mux
 {
+#define LINK_PROBE_STATS_TABLE_NAME "LINK_RPOBE_STATS" 
+
 class MuxManager;
 using ServerIpPortMap = std::map<boost::asio::ip::address, std::string>;
 
@@ -170,6 +172,23 @@ class DbInterface
         mux_state::MuxState::Label label
     );
 
+    /**
+     * @method postLinkProberMetricsEvent
+     * 
+     * @brief post link prober pck loss event
+     * 
+     * @param portName (in) port name 
+     * @param metrics (in) pck loss event name 
+     * 
+     * @return none
+     * 
+    */
+    virtual void postLinkProberMetricsEvent(
+        const std::string &portName, 
+        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
+    );
+
+
     /**
     *@method initialize
     *
@@ -277,6 +296,23 @@ class DbInterface
         boost::posix_time::ptime time
     );
 
+    /**
+     * @method handlePostLinkProberMetrics
+     * 
+     * @brief post link prober pck loss event to state db 
+     * 
+     * @param portName (in) port name
+     * @param metrics (in) metrics data
+     * @param time (in) event time stamp
+     * 
+     * @return none 
+    */
+    void handlePostLinkProberMetrics(
+        const std::string portName,
+        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics,
+        boost::posix_time::ptime time
+    );
+
     /**
     *@method processTorMacAddress
     *
@@ -488,6 +524,7 @@ class DbInterface
     static std::vector<std::string> mMuxState;
     static std::vector<std::string> mMuxLinkmgrState;
     static std::vector<std::string> mMuxMetrics;
+    static std::vector<std::string> mLinkProbeMetrics;
 
 private:
     mux::MuxManager *mMuxManagerPtr;
@@ -505,6 +542,8 @@ class DbInterface
     std::shared_ptr<swss::Table> mStateDbMuxLinkmgrTablePtr;
     // for writing mux metrics
     std::shared_ptr<swss::Table> mStateDbMuxMetricsTablePtr;
+    // for writing link probe statistics data
+    std::shared_ptr<swss::Table> mStateDbLinkProbeStatsTablePtr;
 
     std::shared_ptr<boost::thread> mSwssThreadPtr;
 
diff --git a/src/MuxPort.h b/src/MuxPort.h
index 3d2981a2..85bed7bd 100644
--- a/src/MuxPort.h
+++ b/src/MuxPort.h
@@ -164,6 +164,19 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
         mDbInterfacePtr->postMetricsEvent(mMuxPortConfig.getPortName(), metrics, label);
     };
 
+    /**
+     * @method postLinkProberMetricsEvent
+     * 
+     * @brief post link prober pck loss event
+     * 
+     * @param metrics (in) metrics to post
+     * 
+     * @return none
+    */
+    inline void postLinkProberMetricsEvent(link_manager::LinkManagerStateMachine::LinkProberMetrics metrics) {
+        mDbInterfacePtr->postLinkProberMetricsEvent(mMuxPortConfig.getPortName(), metrics);
+    };
+
     /**
     *@method setServerIpv4Address
     *
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index 6a985f7c..dd5f7f5b 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -121,6 +121,13 @@ class LinkManagerStateMachine: public common::StateMachine,
         Count
     };
 
+    enum class LinkProberMetrics {
+        PckLossStart, 
+        PckLossEnd,
+
+        Count
+    };
+
 private:
     /**
      *@enum anonymous

From d12b7fdee5a2695d1e60f309905c5ed79f95c156 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 05:49:20 +0000
Subject: [PATCH 09/20] Add handlers for pck loss updates

---
 src/DbInterface.cpp                          | 55 ++++++++++++++++++--
 src/DbInterface.h                            | 29 +++++++++++
 src/MuxPort.h                                | 13 +++++
 src/link_manager/LinkManagerStateMachine.cpp | 24 ++++++++-
 src/link_manager/LinkManagerStateMachine.h   | 15 +++++-
 src/link_prober/LinkProber.cpp               | 18 ++++---
 src/link_prober/LinkProber.h                 |  1 -
 src/link_prober/LinkProberStateMachine.cpp   | 16 ++++++
 src/link_prober/LinkProberStateMachine.h     | 11 ++++
 9 files changed, 165 insertions(+), 17 deletions(-)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index 7bb1e772..ff0849c8 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -46,7 +46,7 @@ constexpr auto DEFAULT_TIMEOUT_MSEC = 1000;
 std::vector<std::string> DbInterface::mMuxState = {"active", "standby", "unknown", "Error"};
 std::vector<std::string> DbInterface::mMuxLinkmgrState = {"uninitialized", "unhealthy", "healthy"};
 std::vector<std::string> DbInterface::mMuxMetrics = {"start", "end"};
-std::vector<std::string> DbInterface::mLinkProbeMetrics = {"pck_loss_start", "pck_loss_end"};
+std::vector<std::string> DbInterface::mLinkProbeMetrics = {"link_prober_unknown_start", "link_prober_unknown_end"};
 
 //
 // ---> DbInterface(mux::MuxManager *muxManager);
@@ -189,6 +189,31 @@ void DbInterface::postLinkProberMetricsEvent(
     )));
 }
 
+//
+// ---> postPckLossRatio(
+//        const std::string &portName,
+//        const double_t ratio
+//    );
+//  post pck loss ratio update to state db 
+void DbInterface::postPckLossRatio(
+        const std::string &portName,
+        const double_t ratio
+)
+{
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+        portName %
+        ratio
+    );
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &DbInterface::handlePostPckLossRatio,
+        this,
+        portName,
+        ratio
+    ))); 
+}
+
 //
 // ---> initialize();
 //
@@ -369,12 +394,32 @@ void DbInterface::handlePostLinkProberMetrics(
         mLinkProbeMetrics[static_cast<int> (metrics)]
     );
 
-    if (metrics == link_manager::LinkManagerStateMachine::LinkProberMetrics::PckLossStart) {
-        mStateDbLinkProbeStatsTablePtr.hdel(portName, mLinkProbeMetrics[0]);
-        mStateDbLinkProbeStatsTablePtr.hdel(portName, mLinkProbeMetrics[1]);
+    if (metrics == link_manager::LinkManagerStateMachine::LinkProberMetrics::LinkProberUnknownStart) {
+        mStateDbLinkProbeStatsTablePtr->hdel(portName, mLinkProbeMetrics[0]);
+        mStateDbLinkProbeStatsTablePtr->hdel(portName, mLinkProbeMetrics[1]);
     }
 
-    mStateDbLinkProbeStatsTablePtr.hset(portName, mLinkProbeMetrics[static_cast<int> (metrics)], boost::posix_time::to_simple_string(time));
+    mStateDbLinkProbeStatsTablePtr->hset(portName, mLinkProbeMetrics[static_cast<int> (metrics)], boost::posix_time::to_simple_string(time));
+}
+
+// 
+// ---> handlePostPckLossRatio(
+//        const std::string portName,
+//        const double_t ratio
+//    );
+//
+// handle post pck loss ratio 
+void DbInterface::handlePostPckLossRatio(
+        const std::string portName,
+        const double_t ratio
+)
+{
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+        portName %
+        ratio
+    );
+
+    mStateDbLinkProbeStatsTablePtr->hset(portName, "pck_loss_ratio", std::to_string(ratio));
 }
 
 //
diff --git a/src/DbInterface.h b/src/DbInterface.h
index 1f633a82..0d95c4b9 100644
--- a/src/DbInterface.h
+++ b/src/DbInterface.h
@@ -188,6 +188,20 @@ class DbInterface
         link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
     );
 
+    /**
+     * @method postPckLossRatio
+     * 
+     * @brief post pck loss ratio update to state db
+     * 
+     * @param portName (in) port name 
+     * @param ratio (in) pck loss ratio
+     * 
+     * @return none
+    */
+    virtual void postPckLossRatio(
+        const std::string &portName,
+        const double_t ratio
+    );
 
     /**
     *@method initialize
@@ -313,6 +327,21 @@ class DbInterface
         boost::posix_time::ptime time
     );
 
+    /**
+     * @method handlePostPckLossRatio
+     * 
+     * @brief handle post pck loss ratio update 
+     * 
+     * @param portName (in) port name 
+     * @param ratio (in) pck loss ratio
+     * 
+     * @return none
+    */
+    void handlePostPckLossRatio(
+        const std::string portName,
+        const double_t ratio
+    );
+
     /**
     *@method processTorMacAddress
     *
diff --git a/src/MuxPort.h b/src/MuxPort.h
index 85bed7bd..49ece395 100644
--- a/src/MuxPort.h
+++ b/src/MuxPort.h
@@ -177,6 +177,19 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
         mDbInterfacePtr->postLinkProberMetricsEvent(mMuxPortConfig.getPortName(), metrics);
     };
 
+    /**
+     * @method postPckLossRatio
+     * 
+     * @brief post pck loss ratio update to state db 
+     * 
+     * @param ratio (in) pck loss ratio 
+     * 
+     * @return none
+    */
+    inline void postPckLossRatio(const double_t ratio) {
+        mDbInterfacePtr->postPckLossRatio(mMuxPortConfig.getPortName(), ratio);
+    };
+
     /**
     *@method setServerIpv4Address
     *
diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index dd547c99..0259361e 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -458,7 +458,7 @@ void LinkManagerStateMachine::activateStateMachine()
 //
 // ---> handleStateChange(LinkProberEvent &event, link_prober::LinkProberState::Label state);
 //
-// handles LinkProverEvent
+// handles LinkProberEvent
 //
 void LinkManagerStateMachine::handleStateChange(LinkProberEvent &event, link_prober::LinkProberState::Label state)
 {
@@ -468,6 +468,13 @@ void LinkManagerStateMachine::handleStateChange(LinkProberEvent &event, link_pro
             mLinkProberStateName[state]
         );
 
+        // update state db link prober metrics to collect pck loss data
+        if (ps(mCompositeState) == link_prober::LinkProberState::Unknown && state != link_prober::LinkProberState::Unknown) {
+            mMuxPortPtr->postLinkProberMetricsEvent(link_manager::LinkManagerStateMachine::LinkProberMetrics::LinkProberUnknownEnd);
+        } else if (state == link_prober::LinkProberState::Label::Unknown) {
+            mMuxPortPtr->postLinkProberMetricsEvent(link_manager::LinkManagerStateMachine::LinkProberMetrics::LinkProberUnknownStart);
+        }
+
         CompositeState nextState = mCompositeState;
         ps(nextState) = state;
         mStateTransitionHandler[ps(nextState)][ms(nextState)][ls(nextState)](this, nextState);
@@ -837,6 +844,21 @@ void LinkManagerStateMachine::handleDefaultRouteStateNotification(const std::str
     } 
 }
 
+//
+// ---> handlePostPckLossRatioNotification(const double_t ratio);
+// 
+// handle post pck loss ratio 
+//
+void LinkManagerStateMachine::handlePostPckLossRatioNotification(const double_t ratio)
+{
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+        mMuxPortConfig.getPortName() %
+        ratio
+    );
+    
+    mMuxPortPtr->postPckLossRatio(ratio);
+}
+
 //
 // ---> updateMuxLinkmgrState();
 //
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index dd5f7f5b..b4aea4ba 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -122,8 +122,8 @@ class LinkManagerStateMachine: public common::StateMachine,
     };
 
     enum class LinkProberMetrics {
-        PckLossStart, 
-        PckLossEnd,
+        LinkProberUnknownStart, 
+        LinkProberUnknownEnd,
 
         Count
     };
@@ -494,6 +494,17 @@ class LinkManagerStateMachine: public common::StateMachine,
     */
     void handleDefaultRouteStateNotification(const std::string &routeState);
 
+    /**
+     * @method handlePostPckLossRatioNotification
+     * 
+     * @brief handle get post pck loss ratio 
+     * 
+     * @param ratio (in) pck loss ratio 
+     * 
+     * @return none
+    */
+    void handlePostPckLossRatioNotification(const double_t ratio);
+
 private:
     /**
     *@method updateMuxLinkmgrState
diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index 5cafa146..0399f3d6 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -349,7 +349,6 @@ void LinkProber::handleRecv(
                 // echo reply for an echo request generated by this/active ToR
                 mRxSelfSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpSelfEvent());
-                mContinuousIcmpUnknownEventCount = 0;
             } else {
                 mRxPeerSeqNo = mTxSeqNo;
                 mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpPeerEvent());
@@ -453,14 +452,18 @@ void LinkProber::handleTimeout(boost::system::error_code errorCode)
     if (mTxSeqNo != mRxSelfSeqNo && mTxSeqNo != mRxPeerSeqNo) {
         // post unknown event
         mLinkProberStateMachine.postLinkProberStateEvent(LinkProberStateMachine::getIcmpUnknownEvent());
-        mContinuousIcmpUnknownEventCount++; 
         mIcmpUnknownEventCount++;
+    } 
 
-        MUXLOGINFO(boost::format("%s: continusous ICMP Unknown event count: %d, pck loss rate since initialization: %.2f %%") %
-            mMuxPortConfig.getPortName() %
-            mContinuousIcmpUnknownEventCount %
-            (static_cast<double_t>(mIcmpUnknownEventCount) / mIcmpPacketCount * 100) 
-        );
+    mIcmpPacketCount++;
+    if (mIcmpPacketCount % mMuxPortConfig.getNegativeStateChangeRetryCount() == 0) {
+        boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
+        boost::asio::io_service &ioService = strand.context();
+        ioService.post(strand.wrap(boost::bind(
+            &LinkProberStateMachine::handlePckLossRatioUpdate,
+            &mLinkProberStateMachine,
+            (static_cast<double_t>(mIcmpUnknownEventCount) / mIcmpPacketCount)
+        )));
     }
 
     // start another cycle of send/recv
@@ -500,7 +503,6 @@ void LinkProber::startRecv()
 {
     MUXLOGTRACE(mMuxPortConfig.getPortName());
 
-    mIcmpPacketCount++;
 
     mStream.async_read_some(
         boost::asio::buffer(mRxBuffer, MUX_MAX_ICMP_BUFFER_SIZE),
diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
index d83aeaf3..2c8ec9bb 100644
--- a/src/link_prober/LinkProber.h
+++ b/src/link_prober/LinkProber.h
@@ -436,7 +436,6 @@ class LinkProber
 
     bool mSuspendTx = false;
 
-    uint32_t mContinuousIcmpUnknownEventCount = 0;
     uint64_t mIcmpUnknownEventCount = 0;
     uint64_t mIcmpPacketCount = 0;
 };
diff --git a/src/link_prober/LinkProberStateMachine.cpp b/src/link_prober/LinkProberStateMachine.cpp
index fb678268..6d462a08 100644
--- a/src/link_prober/LinkProberStateMachine.cpp
+++ b/src/link_prober/LinkProberStateMachine.cpp
@@ -255,4 +255,20 @@ void LinkProberStateMachine::handleMackAddressUpdate(const std::array<uint8_t, E
     )));
 }
 
+// 
+// ---> handlePckLossRatioUpdate(const double_t ratio);
+//
+// post pck loss ratio update to link manager
+//
+void LinkProberStateMachine::handlePckLossRatioUpdate(const double_t ratio) 
+{
+    boost::asio::io_service::strand &strand = mLinkManagerStateMachine.getStrand();
+    boost::asio::io_service &ioService = strand.context();
+    ioService.post(strand.wrap(boost::bind(
+        &link_manager::LinkManagerStateMachine::handlePostPckLossRatioNotification,
+        &mLinkManagerStateMachine,
+        ratio
+    )));
+}
+
 } /* namespace link_prober */
diff --git a/src/link_prober/LinkProberStateMachine.h b/src/link_prober/LinkProberStateMachine.h
index 18141d94..7a9b4f17 100644
--- a/src/link_prober/LinkProberStateMachine.h
+++ b/src/link_prober/LinkProberStateMachine.h
@@ -319,6 +319,17 @@ class LinkProberStateMachine:  public common::StateMachine
     */
     static SwitchActiveRequestEvent& getSwitchActiveRequestEvent() {return mSwitchActiveRequestEvent;};
 
+    /**
+     * @method handlePckLossRatioUpdate
+     * 
+     * @brief post pck loss ratio update to link manager
+     * 
+     * @param ratio (in) pck loss ratio
+     * 
+     * @return none
+    */
+    void handlePckLossRatioUpdate(const double_t ratio);
+
 private:
     /**
     *@method postLinkManagerEvent

From f3b5966471c8ac8820d6ad8d0e9b35dd0bd3c83f Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 08:18:55 +0000
Subject: [PATCH 10/20] update tests

---
 test/FakeDbInterface.cpp | 16 ++++++++++++++++
 test/FakeDbInterface.h   | 12 ++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/test/FakeDbInterface.cpp b/test/FakeDbInterface.cpp
index 7becc8d5..27a2a13b 100644
--- a/test/FakeDbInterface.cpp
+++ b/test/FakeDbInterface.cpp
@@ -67,4 +67,20 @@ void FakeDbInterface::postMetricsEvent(
     mPostMetricsInvokeCount++;
 }
 
+void FakeDbInterface::postLinkProberMetricsEvent(
+        const std::string &portName, 
+        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
+)
+{
+    mPostLinkProberMetricsInvokeCount++;
+}
+
+void FakeDbInterface::postPckLossRatio(
+        const std::string &portName,
+        const double_t ratio
+)
+{
+    mPostPckLossRatioInvokeCount;
+} 
+
 } /* namespace test */
diff --git a/test/FakeDbInterface.h b/test/FakeDbInterface.h
index 6a9e0e6d..81c71175 100644
--- a/test/FakeDbInterface.h
+++ b/test/FakeDbInterface.h
@@ -48,9 +48,19 @@ class FakeDbInterface: public mux::DbInterface
         link_manager::LinkManagerStateMachine::Metrics metrics,
         mux_state::MuxState::Label label
     ) override;
+    virtual void postLinkProberMetricsEvent(
+        const std::string &portName, 
+        link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
+    ) override;
+    virtual void postPckLossRatio(
+        const std::string &portName,
+        const double_t ratio
+    ) override;
+
 
     void setNextMuxState(mux_state::MuxState::Label label) {mNextMuxState = label;};
 
+
 public:
     mux_state::MuxState::Label mNextMuxState;
 
@@ -61,6 +71,8 @@ class FakeDbInterface: public mux::DbInterface
     uint32_t mProbeMuxStateInvokeCount = 0;
     uint32_t mSetMuxLinkmgrStateInvokeCount = 0;
     uint32_t mPostMetricsInvokeCount = 0;
+    uint32_t mPostLinkProberMetricsInvokeCount = 0;
+    uint32_t mPostPckLossRatioInvokeCount = 0;
 };
 
 } /* namespace test */

From b1b592664f54759dda5259e99dbc7880c8bcd831 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 09:33:08 +0000
Subject: [PATCH 11/20] fix table name typo

---
 src/DbInterface.cpp                          | 4 ++--
 src/DbInterface.h                            | 2 +-
 src/link_manager/LinkManagerStateMachine.cpp | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index ff0849c8..976e23ff 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -200,7 +200,7 @@ void DbInterface::postPckLossRatio(
         const double_t ratio
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
         portName %
         ratio
     );
@@ -414,7 +414,7 @@ void DbInterface::handlePostPckLossRatio(
         const double_t ratio
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
         portName %
         ratio
     );
diff --git a/src/DbInterface.h b/src/DbInterface.h
index 0d95c4b9..a1e6bee0 100644
--- a/src/DbInterface.h
+++ b/src/DbInterface.h
@@ -44,7 +44,7 @@ class MuxManagerTest;
 
 namespace mux
 {
-#define LINK_PROBE_STATS_TABLE_NAME "LINK_RPOBE_STATS" 
+#define LINK_PROBE_STATS_TABLE_NAME "LINK_PROBE_STATS" 
 
 class MuxManager;
 using ServerIpPortMap = std::map<boost::asio::ip::address, std::string>;
diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index 0259361e..f10e629f 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -851,7 +851,7 @@ void LinkManagerStateMachine::handleDefaultRouteStateNotification(const std::str
 //
 void LinkManagerStateMachine::handlePostPckLossRatioNotification(const double_t ratio)
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f %%") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
         mMuxPortConfig.getPortName() %
         ratio
     );

From bd885eefaac5bdef0a721f6efc6a31f8cbb986e8 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 09:42:51 +0000
Subject: [PATCH 12/20] add unit test

---
 test/LinkManagerStateMachineTest.cpp | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index dcd416ae..2e691b4d 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -1053,4 +1053,17 @@ TEST_F(LinkManagerStateMachineTest, MuxStandbyDefaultRouteStateOK)
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent) 
+{
+    setMuxStandby();
+    
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 0);
+    postLinkProberEvent(link_prober::LinkProberState::Unknown);
+
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
+    postLinkProberEvent(link_prober::LinkProberState::Active);
+    
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 2);
+}
+
 } /* namespace test */

From 815839f45e264aee2c22570bdb3ef412d897d909 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 09:42:51 +0000
Subject: [PATCH 13/20] add unit test

---
 test/FakeDbInterface.cpp             |  2 +-
 test/LinkManagerStateMachineTest.cpp | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/test/FakeDbInterface.cpp b/test/FakeDbInterface.cpp
index 27a2a13b..88f69f1e 100644
--- a/test/FakeDbInterface.cpp
+++ b/test/FakeDbInterface.cpp
@@ -80,7 +80,7 @@ void FakeDbInterface::postPckLossRatio(
         const double_t ratio
 )
 {
-    mPostPckLossRatioInvokeCount;
+    mPostPckLossRatioInvokeCount++;
 } 
 
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index dcd416ae..0d8b3ef4 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -1053,4 +1053,17 @@ TEST_F(LinkManagerStateMachineTest, MuxStandbyDefaultRouteStateOK)
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent) 
+{
+    setMuxStandby();
+    
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 0);
+    postLinkProberEvent(link_prober::LinkProberState::Unknown, 3);
+
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
+    postLinkProberEvent(link_prober::LinkProberState::Active, 3);
+    
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 2);
+}
+
 } /* namespace test */

From 4416b0b6ed0bdc9ba7dfe281956666330816a085 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 16:49:58 +0000
Subject: [PATCH 14/20] add flag to hanlde edge case: link probe stays wait
 when icmp_responder is off

---
 src/link_manager/LinkManagerStateMachine.cpp | 4 +++-
 src/link_manager/LinkManagerStateMachine.h   | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index f10e629f..033d822c 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -469,9 +469,11 @@ void LinkManagerStateMachine::handleStateChange(LinkProberEvent &event, link_pro
         );
 
         // update state db link prober metrics to collect pck loss data
-        if (ps(mCompositeState) == link_prober::LinkProberState::Unknown && state != link_prober::LinkProberState::Unknown) {
+        if (mContinuousLinkProberUnknownEvent == true && state != link_prober::LinkProberState::Unknown) {
+            mContinuousLinkProberUnknownEvent = false;
             mMuxPortPtr->postLinkProberMetricsEvent(link_manager::LinkManagerStateMachine::LinkProberMetrics::LinkProberUnknownEnd);
         } else if (state == link_prober::LinkProberState::Label::Unknown) {
+            mContinuousLinkProberUnknownEvent = true;
             mMuxPortPtr->postLinkProberMetricsEvent(link_manager::LinkManagerStateMachine::LinkProberMetrics::LinkProberUnknownStart);
         }
 
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index b4aea4ba..7678e87e 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -951,6 +951,8 @@ class LinkManagerStateMachine: public common::StateMachine,
     bool mPendingMuxModeChange = false;
     common::MuxPortConfig::Mode mTargetMuxMode = common::MuxPortConfig::Mode::Auto;
 
+    bool mContinuousLinkProberUnknownEvent = false;
+
     std::bitset<ComponentCount> mComponentInitState = {0};
     Label mLabel = Label::Uninitialized;
 };

From 1dbc400259c3a0d35defb708a059b08676cfd21a Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 09:42:51 +0000
Subject: [PATCH 15/20] add unit test

---
 test/LinkManagerStateMachineTest.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 0d8b3ef4..76563b70 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -1058,10 +1058,17 @@ TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent)
     setMuxStandby();
     
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 0);
+<<<<<<< HEAD
     postLinkProberEvent(link_prober::LinkProberState::Unknown, 3);
 
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
     postLinkProberEvent(link_prober::LinkProberState::Active, 3);
+=======
+    postLinkProberEvent(link_prober::LinkProberState::Unknown);
+
+    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
+    postLinkProberEvent(link_prober::LinkProberState::Active);
+>>>>>>> bd885ee... add unit test
     
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 2);
 }

From 913cd1f713901bf8bcc18b0701bf1389fe13fc9b Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Mon, 24 Jan 2022 18:19:52 +0000
Subject: [PATCH 16/20] add more unit tests

---
 test/FakeDbInterface.cpp             |  2 +-
 test/FakeDbInterface.h               |  2 +-
 test/LinkManagerStateMachineTest.cpp | 20 +++++++++++++-------
 test/LinkManagerStateMachineTest.h   |  1 +
 4 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/test/FakeDbInterface.cpp b/test/FakeDbInterface.cpp
index 88f69f1e..3082437b 100644
--- a/test/FakeDbInterface.cpp
+++ b/test/FakeDbInterface.cpp
@@ -80,7 +80,7 @@ void FakeDbInterface::postPckLossRatio(
         const double_t ratio
 )
 {
-    mPostPckLossRatioInvokeCount++;
+    mPckLossRatio = ratio;
 } 
 
 } /* namespace test */
diff --git a/test/FakeDbInterface.h b/test/FakeDbInterface.h
index 81c71175..19eb0513 100644
--- a/test/FakeDbInterface.h
+++ b/test/FakeDbInterface.h
@@ -72,7 +72,7 @@ class FakeDbInterface: public mux::DbInterface
     uint32_t mSetMuxLinkmgrStateInvokeCount = 0;
     uint32_t mPostMetricsInvokeCount = 0;
     uint32_t mPostLinkProberMetricsInvokeCount = 0;
-    uint32_t mPostPckLossRatioInvokeCount = 0;
+    double_t mPckLossRatio = 0;
 };
 
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 76563b70..74abc066 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -236,6 +236,12 @@ void LinkManagerStateMachineTest::postDefaultRouteEvent(std::string routeState,
     runIoService(count);
 }
 
+void LinkManagerStateMachineTest::postPckLossRatioUpdateEvent(double_t ratio)
+{
+    mFakeMuxPort.postPckLossRatio(ratio);
+    runIoService();
+}
+
 TEST_F(LinkManagerStateMachineTest, MuxActiveSwitchOver)
 {
     setMuxActive();
@@ -1058,19 +1064,19 @@ TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent)
     setMuxStandby();
     
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 0);
-<<<<<<< HEAD
     postLinkProberEvent(link_prober::LinkProberState::Unknown, 3);
 
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
     postLinkProberEvent(link_prober::LinkProberState::Active, 3);
-=======
-    postLinkProberEvent(link_prober::LinkProberState::Unknown);
-
-    EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 1);
-    postLinkProberEvent(link_prober::LinkProberState::Active);
->>>>>>> bd885ee... add unit test
     
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 2);
 }
 
+TEST_F(LinkManagerStateMachineTest, PostPckLossUpdateEvent)
+{
+    double_t ratio = 1.0;
+    postPckLossRatioUpdateEvent(ratio);
+    EXPECT_EQ(mDbInterfacePtr->mPckLossRatio, ratio);
+}
+
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.h b/test/LinkManagerStateMachineTest.h
index c752a0e1..8d966391 100644
--- a/test/LinkManagerStateMachineTest.h
+++ b/test/LinkManagerStateMachineTest.h
@@ -52,6 +52,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void setMuxActive();
     void setMuxStandby();
     void postDefaultRouteEvent(std::string routeState, uint32_t count = 0);
+    void postPckLossRatioUpdateEvent(double_t ratio);
 
 public:
     boost::asio::io_service mIoService;

From 22b205f71c0e76864c19419794b871eed7bb09c6 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Tue, 25 Jan 2022 19:21:32 +0000
Subject: [PATCH 17/20] Post packet loss counts instead of ratio

---
 src/DbInterface.cpp                          | 28 +++++++++++++-------
 src/DbInterface.h                            | 12 ++++++---
 src/MuxPort.h                                |  7 ++---
 src/link_manager/LinkManagerStateMachine.cpp | 11 ++++----
 src/link_manager/LinkManagerStateMachine.h   |  5 ++--
 src/link_prober/LinkProber.cpp               |  3 ++-
 src/link_prober/LinkProberStateMachine.cpp   |  7 ++---
 src/link_prober/LinkProberStateMachine.h     |  5 ++--
 8 files changed, 48 insertions(+), 30 deletions(-)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index 976e23ff..16f00423 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -192,17 +192,20 @@ void DbInterface::postLinkProberMetricsEvent(
 //
 // ---> postPckLossRatio(
 //        const std::string &portName,
-//        const double_t ratio
+//        const uint64_t unknownEventCount, 
+//        const uint64_t expectedPacketCount
 //    );
 //  post pck loss ratio update to state db 
 void DbInterface::postPckLossRatio(
         const std::string &portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio, pck_loss_count / pck_expected_count : %d / %d") %
         portName %
-        ratio
+        unknownEventCount % 
+        expectedPacketCount
     );
 
     boost::asio::io_service &ioService = mStrand.context();
@@ -210,7 +213,8 @@ void DbInterface::postPckLossRatio(
         &DbInterface::handlePostPckLossRatio,
         this,
         portName,
-        ratio
+        unknownEventCount,
+        expectedPacketCount
     ))); 
 }
 
@@ -405,21 +409,25 @@ void DbInterface::handlePostLinkProberMetrics(
 // 
 // ---> handlePostPckLossRatio(
 //        const std::string portName,
-//        const double_t ratio
+//        const uint64_t unknownEventCount, 
+//        const uint64_t expectedPacketCount
 //    );
 //
 // handle post pck loss ratio 
 void DbInterface::handlePostPckLossRatio(
         const std::string portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio, pck_loss_count / pck_expected_count : %d / %d") %
         portName %
-        ratio
+        unknownEventCount % 
+        expectedPacketCount
     );
 
-    mStateDbLinkProbeStatsTablePtr->hset(portName, "pck_loss_ratio", std::to_string(ratio));
+    mStateDbLinkProbeStatsTablePtr->hset(portName, "pck_loss_count", std::to_string(unknownEventCount));
+    mStateDbLinkProbeStatsTablePtr->hset(portName, "pck_expected_count", std::to_string(expectedPacketCount));
 }
 
 //
diff --git a/src/DbInterface.h b/src/DbInterface.h
index a1e6bee0..5afcc60b 100644
--- a/src/DbInterface.h
+++ b/src/DbInterface.h
@@ -194,13 +194,15 @@ class DbInterface
      * @brief post pck loss ratio update to state db
      * 
      * @param portName (in) port name 
-     * @param ratio (in) pck loss ratio
+     * @param unknownEventCount (in) count of missing icmp packets
+     * @param expectedPacketCount (in) count of expected icmp packets 
      * 
      * @return none
     */
     virtual void postPckLossRatio(
         const std::string &portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
     );
 
     /**
@@ -333,13 +335,15 @@ class DbInterface
      * @brief handle post pck loss ratio update 
      * 
      * @param portName (in) port name 
-     * @param ratio (in) pck loss ratio
+     * @param unknownEventCount (in) count of missing icmp packets
+     * @param expectedPacketCount (in) count of expected icmp packets 
      * 
      * @return none
     */
     void handlePostPckLossRatio(
         const std::string portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
     );
 
     /**
diff --git a/src/MuxPort.h b/src/MuxPort.h
index 49ece395..cc37ca3d 100644
--- a/src/MuxPort.h
+++ b/src/MuxPort.h
@@ -182,12 +182,13 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
      * 
      * @brief post pck loss ratio update to state db 
      * 
-     * @param ratio (in) pck loss ratio 
+     * @param unknownEventCount (in) count of missing icmp packets
+     * @param expectedPacketCount (in) count of expected icmp packets 
      * 
      * @return none
     */
-    inline void postPckLossRatio(const double_t ratio) {
-        mDbInterfacePtr->postPckLossRatio(mMuxPortConfig.getPortName(), ratio);
+    inline void postPckLossRatio(const uint64_t unknownEventCount, const uint64_t expectedPacketCount) {
+        mDbInterfacePtr->postPckLossRatio(mMuxPortConfig.getPortName(), unknownEventCount, expectedPacketCount);
     };
 
     /**
diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index 033d822c..de633f18 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -847,18 +847,19 @@ void LinkManagerStateMachine::handleDefaultRouteStateNotification(const std::str
 }
 
 //
-// ---> handlePostPckLossRatioNotification(const double_t ratio);
+// ---> handlePostPckLossRatioNotification(const uint64_t unknownEventCount, const uint64_t expectedPacketCount);
 // 
 // handle post pck loss ratio 
 //
-void LinkManagerStateMachine::handlePostPckLossRatioNotification(const double_t ratio)
+void LinkManagerStateMachine::handlePostPckLossRatioNotification(const uint64_t unknownEventCount, const uint64_t expectedPacketCount)
 {
-    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio: %.2f ") %
+    MUXLOGDEBUG(boost::format("%s: posting pck loss ratio, pck_loss_count / pck_expected_count : %d / %d") %
         mMuxPortConfig.getPortName() %
-        ratio
+        unknownEventCount % 
+        expectedPacketCount
     );
     
-    mMuxPortPtr->postPckLossRatio(ratio);
+    mMuxPortPtr->postPckLossRatio(unknownEventCount, expectedPacketCount);
 }
 
 //
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index 7678e87e..2efea929 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -499,11 +499,12 @@ class LinkManagerStateMachine: public common::StateMachine,
      * 
      * @brief handle get post pck loss ratio 
      * 
-     * @param ratio (in) pck loss ratio 
+     * @param unknownEventCount (in) count of missing icmp packets
+     * @param expectedPacketCount (in) count of expected icmp packets
      * 
      * @return none
     */
-    void handlePostPckLossRatioNotification(const double_t ratio);
+    void handlePostPckLossRatioNotification(const uint64_t unknownEventCount, const uint64_t expectedPacketCount);
 
 private:
     /**
diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index 258d755d..71eadee7 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -449,7 +449,8 @@ void LinkProber::handleTimeout(boost::system::error_code errorCode)
         ioService.post(strand.wrap(boost::bind(
             &LinkProberStateMachine::handlePckLossRatioUpdate,
             &mLinkProberStateMachine,
-            (static_cast<double_t>(mIcmpUnknownEventCount) / mIcmpPacketCount)
+            mIcmpUnknownEventCount,
+            mIcmpPacketCount
         )));
     }
 
diff --git a/src/link_prober/LinkProberStateMachine.cpp b/src/link_prober/LinkProberStateMachine.cpp
index 6d462a08..15180461 100644
--- a/src/link_prober/LinkProberStateMachine.cpp
+++ b/src/link_prober/LinkProberStateMachine.cpp
@@ -256,18 +256,19 @@ void LinkProberStateMachine::handleMackAddressUpdate(const std::array<uint8_t, E
 }
 
 // 
-// ---> handlePckLossRatioUpdate(const double_t ratio);
+// ---> handlePckLossRatioUpdate(const uint64_t unknownEventCount, const uint64_t expectedPacketCount);
 //
 // post pck loss ratio update to link manager
 //
-void LinkProberStateMachine::handlePckLossRatioUpdate(const double_t ratio) 
+void LinkProberStateMachine::handlePckLossRatioUpdate(const uint64_t unknownEventCount, const uint64_t expectedPacketCount) 
 {
     boost::asio::io_service::strand &strand = mLinkManagerStateMachine.getStrand();
     boost::asio::io_service &ioService = strand.context();
     ioService.post(strand.wrap(boost::bind(
         &link_manager::LinkManagerStateMachine::handlePostPckLossRatioNotification,
         &mLinkManagerStateMachine,
-        ratio
+        unknownEventCount,
+        expectedPacketCount
     )));
 }
 
diff --git a/src/link_prober/LinkProberStateMachine.h b/src/link_prober/LinkProberStateMachine.h
index 7a9b4f17..4ffa96b3 100644
--- a/src/link_prober/LinkProberStateMachine.h
+++ b/src/link_prober/LinkProberStateMachine.h
@@ -324,11 +324,12 @@ class LinkProberStateMachine:  public common::StateMachine
      * 
      * @brief post pck loss ratio update to link manager
      * 
-     * @param ratio (in) pck loss ratio
+     * @param unknownEventCount (in) count of missing icmp packets
+     * @param expectedPacketCount (in) count of expected icmp packets
      * 
      * @return none
     */
-    void handlePckLossRatioUpdate(const double_t ratio);
+    void handlePckLossRatioUpdate(const uint64_t unknownEventCount, const uint64_t expectedPacketCount);
 
 private:
     /**

From 17c989bb7a76fe4963f77867c3eca570e1c01c15 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Tue, 25 Jan 2022 19:22:51 +0000
Subject: [PATCH 18/20] Update log levels

---
 src/DbInterface.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index 16f00423..2c4d7357 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -174,7 +174,7 @@ void DbInterface::postLinkProberMetricsEvent(
         link_manager::LinkManagerStateMachine::LinkProberMetrics metrics
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting link prober pck loss event %s") %
+    MUXLOGWARNING(boost::format("%s: posting link prober pck loss event %s") %
         portName %
         mLinkProbeMetrics[static_cast<int> (metrics)]
     );
@@ -393,7 +393,7 @@ void DbInterface::handlePostLinkProberMetrics(
     boost::posix_time::ptime time
 )
 {
-    MUXLOGDEBUG(boost::format("%s: posting link prober pck loss event %s") %
+    MUXLOGWARNING(boost::format("%s: posting link prober pck loss event %s") %
         portName %
         mLinkProbeMetrics[static_cast<int> (metrics)]
     );

From 14aea6ac9357786fdc7886ae9cbee4ba11eb7d13 Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Wed, 26 Jan 2022 03:27:45 +0000
Subject: [PATCH 19/20] Add handlers for packet loss count resetting

---
 src/DbInterface.cpp                          | 20 ++++++++++++++++++++
 src/MuxManager.cpp                           | 15 +++++++++++++++
 src/MuxManager.h                             | 11 +++++++++++
 src/MuxPort.cpp                              | 17 +++++++++++++++++
 src/MuxPort.h                                |  9 +++++++++
 src/link_manager/LinkManagerStateMachine.cpp | 14 ++++++++++++++
 src/link_manager/LinkManagerStateMachine.h   | 10 ++++++++++
 src/link_prober/LinkProber.cpp               | 20 ++++++++++++++++++++
 src/link_prober/LinkProber.h                 |  9 +++++++++
 test/FakeDbInterface.cpp                     |  6 ++++--
 test/FakeDbInterface.h                       |  6 ++++--
 test/LinkManagerStateMachineTest.cpp         | 10 +++++-----
 test/LinkManagerStateMachineTest.h           |  2 +-
 13 files changed, 139 insertions(+), 10 deletions(-)

diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
index 2c4d7357..dbabc895 100644
--- a/src/DbInterface.cpp
+++ b/src/DbInterface.cpp
@@ -611,8 +611,28 @@ void DbInterface::processMuxPortConfigNotifiction(std::deque<swss::KeyOpFieldsVa
                 f %
                 v
             );
+            
             mMuxManagerPtr->updateMuxPortConfig(port, v);
         }
+
+        std::vector<swss::FieldValueTuple>::const_iterator c_it = std::find_if(
+            fieldValues.cbegin(),
+            fieldValues.cend(),
+            [] (const swss::FieldValueTuple &fv) {return fvField(fv) == "pck_loss_data_reset";}
+        );
+        if (c_it != fieldValues.cend()) {
+            const std::string f = c_it->first;
+            const std::string v = c_it->second;
+
+            MUXLOGDEBUG(boost::format("key: %s, Operation: %s, f: %s, v: %s") %
+                port %
+                operation %
+                f %
+                v
+            );
+            
+            mMuxManagerPtr->resetPckLossCount(port);
+        }
     }
 }
 
diff --git a/src/MuxManager.cpp b/src/MuxManager.cpp
index a1bab80c..a01f4dd5 100644
--- a/src/MuxManager.cpp
+++ b/src/MuxManager.cpp
@@ -160,6 +160,21 @@ void MuxManager::updateMuxPortConfig(const std::string &portName, const std::str
     }
 }
 
+// 
+// ---> resetPckLossCount(const std::string &portName);
+//
+// reset ICMP packet loss count 
+// 
+void MuxManager::resetPckLossCount(const std::string &portName)
+{
+    MUXLOGWARNING(boost::format("%s: reset ICMP packet loss count ") % portName);
+
+    PortMapIterator portMapIterator = mPortMap.find(portName);
+    if(portMapIterator != mPortMap.end()) {
+        portMapIterator->second->resetPckLossCount();
+    }
+}
+
 //
 // ---> addOrUpdateMuxPortLinkState(const std::string &portName, const std::string &linkState);
 //
diff --git a/src/MuxManager.h b/src/MuxManager.h
index eb54b3f1..d7c0c97c 100644
--- a/src/MuxManager.h
+++ b/src/MuxManager.h
@@ -245,6 +245,17 @@ class MuxManager
     */
     void updateMuxPortConfig(const std::string &portName, const std::string &linkState);
 
+    /**
+     * @method resetPckLossCount
+     * 
+     * @brief reset ICMP packet loss count. 
+     * 
+     * @param portName (in) Mux port name 
+     * 
+     * @return none
+    */
+    void resetPckLossCount(const std::string &portName);
+
     /**
     *@method addOrUpdateMuxPortLinkState
     *
diff --git a/src/MuxPort.cpp b/src/MuxPort.cpp
index 496d6235..912901ed 100644
--- a/src/MuxPort.cpp
+++ b/src/MuxPort.cpp
@@ -244,4 +244,21 @@ void MuxPort::handleDefaultRouteState(const std::string &routeState)
         routeState
     )));
 }
+
+// 
+// ---> resetPckLossCount();
+// 
+// reset ICMP packet loss count 
+//
+void MuxPort::resetPckLossCount()
+{
+    MUXLOGDEBUG(boost::format("port: %s, reset ICMP packet loss counts ") % mMuxPortConfig.getPortName());
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &link_manager::LinkManagerStateMachine::handleResetLinkProberPckLossCount,
+        &mLinkManagerStateMachine
+    )));
+}
+
 } /* namespace mux */
diff --git a/src/MuxPort.h b/src/MuxPort.h
index cc37ca3d..34e936df 100644
--- a/src/MuxPort.h
+++ b/src/MuxPort.h
@@ -290,6 +290,15 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
     */
     void handleDefaultRouteState(const std::string &routeState);
 
+    /**
+     * @method resetPckLossCount
+     * 
+     * @brief reset ICMP packet loss count 
+     * 
+     * @return none
+    */
+    void resetPckLossCount();
+
 protected:
     friend class test::MuxManagerTest;
     friend class test::FakeMuxPort;
diff --git a/src/link_manager/LinkManagerStateMachine.cpp b/src/link_manager/LinkManagerStateMachine.cpp
index de633f18..9186a115 100644
--- a/src/link_manager/LinkManagerStateMachine.cpp
+++ b/src/link_manager/LinkManagerStateMachine.cpp
@@ -403,6 +403,9 @@ void LinkManagerStateMachine::handleSwssBladeIpv4AddressUpdate(boost::asio::ip::
             mSendPeerSwitchCommandFnPtr = boost::bind(
                 &link_prober::LinkProber::sendPeerSwitchCommand, mLinkProberPtr.get()
             );
+            mResetIcmpPacketCountsFnPtr = boost::bind(
+                &link_prober::LinkProber::resetIcmpPacketCounts, mLinkProberPtr.get()
+            );
             mComponentInitState.set(LinkProberComponent);
 
             activateStateMachine();
@@ -862,6 +865,17 @@ void LinkManagerStateMachine::handlePostPckLossRatioNotification(const uint64_t
     mMuxPortPtr->postPckLossRatio(unknownEventCount, expectedPacketCount);
 }
 
+// ---> handleResetLinkProberPckLossCount();
+// 
+// reset link prober heartbeat packet loss count 
+// 
+void LinkManagerStateMachine::handleResetLinkProberPckLossCount()
+{
+    MUXLOGDEBUG(boost::format("%s: reset link prober packet loss counts ") % mMuxPortConfig.getPortName());
+
+    mResetIcmpPacketCountsFnPtr();
+}
+
 //
 // ---> updateMuxLinkmgrState();
 //
diff --git a/src/link_manager/LinkManagerStateMachine.h b/src/link_manager/LinkManagerStateMachine.h
index 2efea929..bc9700d2 100644
--- a/src/link_manager/LinkManagerStateMachine.h
+++ b/src/link_manager/LinkManagerStateMachine.h
@@ -506,6 +506,15 @@ class LinkManagerStateMachine: public common::StateMachine,
     */
     void handlePostPckLossRatioNotification(const uint64_t unknownEventCount, const uint64_t expectedPacketCount);
 
+    /**
+     * @method handleResetLinkProberPckLossCount
+     * 
+     * @brief reset link prober heartbeat packet loss count 
+     * 
+     * @return none
+    */
+    void handleResetLinkProberPckLossCount();
+
 private:
     /**
     *@method updateMuxLinkmgrState
@@ -945,6 +954,7 @@ class LinkManagerStateMachine: public common::StateMachine,
     boost::function<void (uint32_t suspendTime_msec)> mSuspendTxFnPtr;
     boost::function<void ()> mResumeTxFnPtr;
     boost::function<void ()> mSendPeerSwitchCommandFnPtr;
+    boost::function<void ()> mResetIcmpPacketCountsFnPtr;
 
     uint32_t mWaitActiveUpCount = 0;
     uint32_t mMuxUnknownBackoffFactor = 1;
diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
index 71eadee7..21080833 100644
--- a/src/link_prober/LinkProber.cpp
+++ b/src/link_prober/LinkProber.cpp
@@ -728,4 +728,24 @@ size_t LinkProber::appendTlvDummy(size_t paddingSize, int seqNo)
     return tlvSize;
 }
 
+// 
+// ---> resetIcmpPacketCounts
+//
+// reset Icmp packet counts, post a pck loss ratio update immediately 
+//
+void LinkProber::resetIcmpPacketCounts()
+{
+    mIcmpUnknownEventCount = 0;
+    mIcmpPacketCount = 0;
+
+    boost::asio::io_service::strand &strand = mLinkProberStateMachine.getStrand();
+    boost::asio::io_service &ioService = strand.context();
+    ioService.post(strand.wrap(boost::bind(
+        &LinkProberStateMachine::handlePckLossRatioUpdate,
+        &mLinkProberStateMachine,
+        mIcmpUnknownEventCount,
+        mIcmpPacketCount
+    )));
+}
+
 } /* namespace link_prober */
diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
index 2c8ec9bb..a9961575 100644
--- a/src/link_prober/LinkProber.h
+++ b/src/link_prober/LinkProber.h
@@ -160,6 +160,15 @@ class LinkProber
     */
     void sendPeerSwitchCommand();
 
+    /**
+     * @method resetIcmpPacketCounts()
+     * 
+     * @brief reset Icmp packet counts, post a pck loss ratio update immediately 
+     * 
+     * @return none
+    */
+    void resetIcmpPacketCounts();
+
 private:
     /**
     *@method handleUpdateEthernetFrame
diff --git a/test/FakeDbInterface.cpp b/test/FakeDbInterface.cpp
index 3082437b..5e0dfba2 100644
--- a/test/FakeDbInterface.cpp
+++ b/test/FakeDbInterface.cpp
@@ -77,10 +77,12 @@ void FakeDbInterface::postLinkProberMetricsEvent(
 
 void FakeDbInterface::postPckLossRatio(
         const std::string &portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
 )
 {
-    mPckLossRatio = ratio;
+    mUnknownEventCount = unknownEventCount;
+    mExpectedPacketCount = expectedPacketCount;
 } 
 
 } /* namespace test */
diff --git a/test/FakeDbInterface.h b/test/FakeDbInterface.h
index 19eb0513..c927012a 100644
--- a/test/FakeDbInterface.h
+++ b/test/FakeDbInterface.h
@@ -54,7 +54,8 @@ class FakeDbInterface: public mux::DbInterface
     ) override;
     virtual void postPckLossRatio(
         const std::string &portName,
-        const double_t ratio
+        const uint64_t unknownEventCount, 
+        const uint64_t expectedPacketCount
     ) override;
 
 
@@ -72,7 +73,8 @@ class FakeDbInterface: public mux::DbInterface
     uint32_t mSetMuxLinkmgrStateInvokeCount = 0;
     uint32_t mPostMetricsInvokeCount = 0;
     uint32_t mPostLinkProberMetricsInvokeCount = 0;
-    double_t mPckLossRatio = 0;
+    uint64_t mUnknownEventCount = 0;
+    uint64_t mExpectedPacketCount = 0;
 };
 
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 74abc066..66f87809 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -236,9 +236,9 @@ void LinkManagerStateMachineTest::postDefaultRouteEvent(std::string routeState,
     runIoService(count);
 }
 
-void LinkManagerStateMachineTest::postPckLossRatioUpdateEvent(double_t ratio)
+void LinkManagerStateMachineTest::postPckLossRatioUpdateEvent(uint64_t unknownCount, uint64_t totalCount)
 {
-    mFakeMuxPort.postPckLossRatio(ratio);
+    mFakeMuxPort.postPckLossRatio(unknownCount, totalCount);
     runIoService();
 }
 
@@ -1074,9 +1074,9 @@ TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent)
 
 TEST_F(LinkManagerStateMachineTest, PostPckLossUpdateEvent)
 {
-    double_t ratio = 1.0;
-    postPckLossRatioUpdateEvent(ratio);
-    EXPECT_EQ(mDbInterfacePtr->mPckLossRatio, ratio);
+    postPckLossRatioUpdateEvent(3,4);
+    EXPECT_EQ(mDbInterfacePtr->mUnknownEventCount, 3);
+    EXPECT_EQ(mDbInterfacePtr->mExpectedPacketCount, 4);
 }
 
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.h b/test/LinkManagerStateMachineTest.h
index 8d966391..92cce4b0 100644
--- a/test/LinkManagerStateMachineTest.h
+++ b/test/LinkManagerStateMachineTest.h
@@ -52,7 +52,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void setMuxActive();
     void setMuxStandby();
     void postDefaultRouteEvent(std::string routeState, uint32_t count = 0);
-    void postPckLossRatioUpdateEvent(double_t ratio);
+    void postPckLossRatioUpdateEvent(uint64_t unknownCount, uint64_t totalCount);
 
 public:
     boost::asio::io_service mIoService;

From 07330344c7a15d75adaa21ed45414498a88bffad Mon Sep 17 00:00:00 2001
From: zjswhhh <zhangjing@microsoft.com>
Date: Wed, 26 Jan 2022 19:24:47 +0000
Subject: [PATCH 20/20] Add unit tests for resetting

---
 test/FakeLinkProber.cpp              | 16 +++++++++++++++
 test/FakeLinkProber.h                |  4 ++++
 test/LinkManagerStateMachineTest.cpp | 29 ++++++++++++++++++++++++----
 test/LinkManagerStateMachineTest.h   |  1 +
 4 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/test/FakeLinkProber.cpp b/test/FakeLinkProber.cpp
index 9b7ae3a6..256d3f1e 100644
--- a/test/FakeLinkProber.cpp
+++ b/test/FakeLinkProber.cpp
@@ -121,4 +121,20 @@ void FakeLinkProber::sendPeerSwitchCommand()
     mSendPeerSwitchCommand++;
 }
 
+void FakeLinkProber::resetIcmpPacketCounts()
+{
+    MUXLOGINFO("");
+
+    mIcmpUnknownEventCount = 0;
+    mIcmpPacketCount = 0;
+
+    boost::asio::io_service::strand &strand = mLinkProberStateMachine->getStrand();
+    boost::asio::io_service &ioService = strand.context();
+    ioService.post(strand.wrap(boost::bind(
+        &link_prober::LinkProberStateMachine::handlePckLossRatioUpdate,
+        mLinkProberStateMachine,
+        mIcmpUnknownEventCount,
+        mIcmpPacketCount
+    )));
+}
 } /* namespace test */
diff --git a/test/FakeLinkProber.h b/test/FakeLinkProber.h
index a920dbe2..fa35ae3a 100644
--- a/test/FakeLinkProber.h
+++ b/test/FakeLinkProber.h
@@ -46,6 +46,7 @@ class FakeLinkProber
     void suspendTxProbes(uint32_t suspendTime_msec);
     void resumeTxProbes();
     void sendPeerSwitchCommand();
+    void resetIcmpPacketCounts();
 
 public:
     uint32_t mInitializeCallCount = 0;
@@ -56,6 +57,9 @@ class FakeLinkProber
     uint32_t mResumeTxProbeCallCount = 0;
     uint32_t mSendPeerSwitchCommand = 0;
 
+    uint64_t mIcmpUnknownEventCount = 0;
+    uint64_t mIcmpPacketCount = 0;
+
 private:
     link_prober::LinkProberStateMachine *mLinkProberStateMachine;
 };
diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
index 66f87809..339fa9e3 100644
--- a/test/LinkManagerStateMachineTest.cpp
+++ b/test/LinkManagerStateMachineTest.cpp
@@ -239,9 +239,19 @@ void LinkManagerStateMachineTest::postDefaultRouteEvent(std::string routeState,
 void LinkManagerStateMachineTest::postPckLossRatioUpdateEvent(uint64_t unknownCount, uint64_t totalCount)
 {
     mFakeMuxPort.postPckLossRatio(unknownCount, totalCount);
+    mFakeMuxPort.mFakeLinkProber->mIcmpUnknownEventCount = unknownCount;
+    mFakeMuxPort.mFakeLinkProber->mIcmpPacketCount = totalCount;
+
     runIoService();
 }
 
+void LinkManagerStateMachineTest::postPckLossCountsResetEvent()
+{
+    mFakeMuxPort.mFakeLinkProber->resetIcmpPacketCounts();
+
+    runIoService(2);
+}
+
 TEST_F(LinkManagerStateMachineTest, MuxActiveSwitchOver)
 {
     setMuxActive();
@@ -1072,11 +1082,22 @@ TEST_F(LinkManagerStateMachineTest, PostPckLossMetricsEvent)
     EXPECT_EQ(mDbInterfacePtr->mPostLinkProberMetricsInvokeCount, 2);
 }
 
-TEST_F(LinkManagerStateMachineTest, PostPckLossUpdateEvent)
+TEST_F(LinkManagerStateMachineTest, PostPckLossUpdateAndResetEvent)
 {
-    postPckLossRatioUpdateEvent(3,4);
-    EXPECT_EQ(mDbInterfacePtr->mUnknownEventCount, 3);
-    EXPECT_EQ(mDbInterfacePtr->mExpectedPacketCount, 4);
+    uint64_t unknownCount = 999;
+    uint64_t totalCount = 10000;
+
+    postPckLossRatioUpdateEvent(unknownCount,totalCount);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mIcmpUnknownEventCount, unknownCount);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mIcmpPacketCount, totalCount);
+    EXPECT_EQ(mDbInterfacePtr->mUnknownEventCount, unknownCount);
+    EXPECT_EQ(mDbInterfacePtr->mExpectedPacketCount, totalCount);
+
+    postPckLossCountsResetEvent();
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mIcmpUnknownEventCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mIcmpPacketCount, 0);
+    EXPECT_EQ(mDbInterfacePtr->mUnknownEventCount, 0);
+    EXPECT_EQ(mDbInterfacePtr->mExpectedPacketCount, 0);    
 }
 
 } /* namespace test */
diff --git a/test/LinkManagerStateMachineTest.h b/test/LinkManagerStateMachineTest.h
index 92cce4b0..f687ebcc 100644
--- a/test/LinkManagerStateMachineTest.h
+++ b/test/LinkManagerStateMachineTest.h
@@ -53,6 +53,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void setMuxStandby();
     void postDefaultRouteEvent(std::string routeState, uint32_t count = 0);
     void postPckLossRatioUpdateEvent(uint64_t unknownCount, uint64_t totalCount);
+    void postPckLossCountsResetEvent();
 
 public:
     boost::asio::io_service mIoService;