Fix #2095, Automatic suppression of flooding events

cmake/sample_defs/cpu1_platform_cfg.h

@@ -1337,6 +1337,29 @@
 */
 #define CFE_PLATFORM_EVS_MAX_EVENT_FILTERS 8
 
+/**
+**  \cfeevscfg Maximum number of event before squelching
+**
+**  \par Description:
+**       Maximum number of events that may be emitted per app per second.
+**       Setting this to 0 will cause events to be unrestricted.
+**
+**  \par Limits
+**       This number must be less than or equal to INT_MAX/1000
+*/
+#define CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST 16
+
+/**
+**  \cfeevscfg Sustained number of event messages per second per app before squelching
+**
+**  \par Description:
+**       Sustained number of events that may be emitted per app per second.
+**
+**  \par Limits
+**       This number must be less than or equal to #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST
+*/
+#define CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC 8
+
 /**
 **  \cfeevscfg Default Event Log Filename
 **

docs/cFE_FunctionalRequirements.csv

@@ -299,6 +299,12 @@ EVS: Initialize Logging Mode On Power On Reset,cEVS3203,"Upon a Power-on Reset,
 EVS: Preserve Event Log Reset Mode On Processor Reset,cEVS3207,"Upon a Processor Reset, the cFE shall preserve the Event Logging Mode configuration parameter.",Try to retain mode across a processor reset. The contents of the Local Event Log will be preserved if the Event Logging Mode is configured to Discard. The contents of the Local Event Log may be overwritten (depending on the size and contents of the log prior to the reset) if the Event Logging Mode is configured to Overwrite.
 EVS: Preserve Log Full State On Processor Reset,cEVS3208,"Upon a Processor Reset, the cFE shall preserve the Local Event Log Full state.",Retain the cFE state across Processor Resets.
 EVS: Preserve Log Overflow Counter On Processor Reset,cEVS3210,"Upon a Processor Reset, the cFE shall preserve the Local Event Log Overflow Counter.",Retain the cFE state across Processor Resets.
+EVS: Suppress Flooding Event Messages,cEVS3220,The cFE shall suppress event messages from an application that exceed a pre-defined rate.,Event message flooding may exhaust critical resources required on the spacecraft
+EVS: Suppress Flooding Event Messages - Allow short bursts,cEVS3220.1,The cFE shall allow limited bursts of event messages from an application exceeding the suppress threshold rate. ,"Startup or other events may cause a brief burst of events that exceed the nominal rate, but should still be captured."
+EVS: Suppress Flooding Event Messages - Resume events when not flooding,cEVS3220.2,The cFE shall automatically resume processing event messages from an application which is no longer flooding,"Flooding may be a transient condition, and resuming nominal operations when this condition goes away would allow continued delivery of useful diagnostic data. This also eases implementation. "
+EVS: Suppress Flooding Event Messages - Increment counter,cEVS3220.3,The cFE shall increment the squelched counter for a flooding app when events for that app have been suppressed.,A counter provides a means for keeping track of how many events have been suppressed
+EVS: Suppress Flooding Event Messages - Retain Maximum Value,cEVS3220.4,If the squelched event counters have reached their maximum value of (2^8)-1 the cFE shall retain the maximum value (i.e. do not rollover to zero).,Preventing a counter rollover to zero eliminates the case when a user may think no events have occurred when in fact many events have occurred.
+EVS: Suppress Flooding Event Messages - Retain Maximum Value,cEVS3220.5,"The cFE shall issue an event message when an application transitions to the squelched state, containing the name of the offending application.","An event message provides an obvious useful diagnostic when event suppression occurs, which is an anomaly which should be dealt with."
 SB: NOOP Event,cSB4000,"Upon receipt of a Command, the cFE shall generate a NO-OP event message.",This command is useful as a general sub-system aliveness test.
 SB: Zero Counters,cSB4001,"Upon receipt of Command the cFE shall set to zero the following counters in housekeeping telemetry:
 

docs/src/cfe_evs.dox

@@ -23,6 +23,7 @@
     <LI> \subpage cfeevsugcounters <BR>
     <LI> \subpage cfeevsugresetctrs <BR>
     <LI> \subpage cfeevsugprocreset <BR>
+    <LI> \subpage cfeevsugsquelch <BR>
     <LI> \subpage cfeevsugfaq <BR>
   </UL>
 
@@ -428,11 +429,38 @@
   transfer it to the ground in order to help debug a reset.
 
 
-  Next: \ref cfeevsugfaq <BR>
+  Next: \ref cfeevsugsquelch <BR>
   Prev: \ref cfeevsugresetctrs <BR>
   Up To: \ref cfeevsovr
 **/
 
+/**
+  \page cfeevsugsquelch EVS squelching of misbehaving apps
+
+  Event squelching is an optional feature for suppressing excessive events from misbehaving apps. It
+  is enabled by setting #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST to a nonzero positive value, and 
+  #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC equal to or less than that value. 
+
+  #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST controls the maximum events that can be sent at a given moment, 
+  and #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC is the sustained event throughput per second. 
+
+  The suppression mechanism initializes with #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000 credits. 
+  Each event costs 1000 credits. Credits are restored at a rate of 
+  #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC * 1000 up to a maximum balance of #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST*1000, and the
+  maximum "debt" is -#CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000. When the credit count crosses from positive to negative, 
+  a squelched event message is emitted and events are supppressed, until the credit count becomes positive again.
+
+  Figure EVS-1 is a notional state diagram of the event squelching mechanism.
+
+  \image html evs_squelch_states.png "Figure EVS-1: EVS Squelching State Diagram"
+  \image latex evs_squelch_states.png "Figure EVS-1: EVS Squelching State Diagram"
+
+
+  Next: \ref cfeevsugfaq <BR>
+  Prev: \ref cfeevsugprocreset <BR>
+  Up To: \ref cfeevsovr
+**/
+
 /**
   \page cfeevsugfaq Frequently Asked Questions about Event Services
 

docs/src/mnem_maps/cfe_evs_tlm_mnem_map

@@ -33,7 +33,7 @@ EVS_APP=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS] \
 EVS_APPID=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPID \
 EVS_APPMSGSENTC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPMSGSENTC \
 EVS_APPENASTAT=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPENASTAT \
-EVS_SPARE2ALIGN3=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SPARE2ALIGN3 \
+EVS_SQUELCHEDC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SQUELCHEDC \
 EVS_APPNAME=$sc_$cpu_EVS_APPNAME[OS_MAX_API_NAME] \
 EVS_EVENTID=$sc_$cpu_EVS_EVENTID \
 EVS_EVENTTYPE=$sc_$cpu_EVS_EVENTTYPE \

modules/core_api/fsw/inc/cfe_error.h

@@ -256,6 +256,14 @@ typedef int32 CFE_Status_t;
  */
 #define CFE_EVS_INVALID_PARAMETER ((CFE_Status_t)0xc2000008)
 
+/**
+ * @brief Event squelched
+ *
+ *  Event squelched due to being sent at too high a rate
+ *
+ */
+#define CFE_EVS_APP_SQUELCHED ((CFE_Status_t)0xc2000009)
+
 /**
  * @brief Not Implemented
  *

modules/evs/eds/cfe_evs.xml

@@ -179,6 +179,11 @@
               \cfetlmmnemonic  \EVS_APPENASTAT
             </LongDescription>
           </Entry>
+          <Entry name="AppMessageSquelchedCounter" type="BASE_TYPES/uint8" shortDescription="Application event squelch count">
+            <LongDescription>
+              \cfetlmmnemonic  \EVS_SQUELCHEDC
+            </LongDescription>
+          </Entry>
         </EntryList>
       </ContainerDataType>
 

modules/evs/fsw/inc/cfe_evs_events.h

@@ -466,6 +466,19 @@
  *  Invalid length for the command code in message ID #CFE_EVS_CMD_MID received on the EVS message pipe.
  */
 #define CFE_EVS_LEN_ERR_EID 43
+
+/**
+ * \brief EVS Events Squelched Error Event ID
+ *
+ *  \par Type: ERROR
+ *
+ *  \par Cause:
+ *
+ *  Events generated in app at a rate in excess of
+ *  #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST in one moment or
+ *  #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC sustained
+ */
+#define CFE_EVS_SQUELCHED_ERR_EID 44
 /**\}*/
 
 #endif /* CFE_EVS_EVENTS_H */

modules/evs/fsw/inc/cfe_evs_msg.h

@@ -1157,14 +1157,14 @@ typedef CFE_EVS_AppNameEventIDMaskCmd_t CFE_EVS_SetFilterCmd_t;
 /**********************************/
 typedef struct CFE_EVS_AppTlmData
 {
-    CFE_ES_AppId_t AppID;         /**< \cfetlmmnemonic \EVS_APPID
-                                       \brief Numerical application identifier */
-    uint16 AppMessageSentCounter; /**< \cfetlmmnemonic \EVS_APPMSGSENTC
-                                       \brief Application message sent counter */
-    uint8 AppEnableStatus;        /**< \cfetlmmnemonic \EVS_APPENASTAT
-                                       \brief Application event service enable status */
-    uint8 Padding;                /**< \cfetlmmnemonic \EVS_SPARE2ALIGN3
-                                       \brief Padding for 32 bit boundary */
+    CFE_ES_AppId_t AppID;             /**< \cfetlmmnemonic \EVS_APPID
+                                           \brief Numerical application identifier */
+    uint16 AppMessageSentCounter;     /**< \cfetlmmnemonic \EVS_APPMSGSENTC
+                                           \brief Application message sent counter */
+    uint8 AppEnableStatus;            /**< \cfetlmmnemonic \EVS_APPENASTAT
+                                           \brief Application event service enable status */
+    uint8 AppMessageSquelchedCounter; /**< \cfetlmmnemonic \EVS_SQUELCHEDC
+                                       \brief Number of events squelched */
 
 } CFE_EVS_AppTlmData_t;
 

modules/evs/fsw/src/cfe_evs.c

@@ -71,6 +71,7 @@ CFE_Status_t CFE_EVS_Register(const void *Filters, uint16 NumEventFilters, uint1
             /* Initialize application event data */
             AppDataPtr->ActiveFlag           = true;
             AppDataPtr->EventTypesActiveFlag = CFE_PLATFORM_EVS_DEFAULT_TYPE_FLAG;
+            AppDataPtr->SquelchTokens        = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000;
 
             /* Set limit for number of provided filters */
             if (NumEventFilters < CFE_PLATFORM_EVS_MAX_EVENT_FILTERS)
@@ -143,13 +144,20 @@ CFE_Status_t CFE_EVS_SendEvent(uint16 EventID, uint16 EventType, const char *Spe
         }
         else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
         {
-            /* Get current spacecraft time */
-            Time = CFE_TIME_GetTime();
+            if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
+            {
+                /* Get current spacecraft time */
+                Time = CFE_TIME_GetTime();
 
-            /* Send the event packets */
-            va_start(Ptr, Spec);
-            EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
-            va_end(Ptr);
+                /* Send the event packets */
+                va_start(Ptr, Spec);
+                EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
+                va_end(Ptr);
+            }
+            else
+            {
+                Status = CFE_EVS_APP_SQUELCHED;
+            }
         }
     }
 
@@ -188,13 +196,20 @@ CFE_Status_t CFE_EVS_SendEventWithAppID(uint16 EventID, uint16 EventType, CFE_ES
     }
     else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
     {
-        /* Get current spacecraft time */
-        Time = CFE_TIME_GetTime();
+        if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
+        {
+            /* Get current spacecraft time */
+            Time = CFE_TIME_GetTime();
 
-        /* Send the event packets */
-        va_start(Ptr, Spec);
-        EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
-        va_end(Ptr);
+            /* Send the event packets */
+            va_start(Ptr, Spec);
+            EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
+            va_end(Ptr);
+        }
+        else
+        {
+            Status = CFE_EVS_APP_SQUELCHED;
+        }
     }
 
     return Status;
@@ -231,10 +246,17 @@ CFE_Status_t CFE_EVS_SendTimedEvent(CFE_TIME_SysTime_t Time, uint16 EventID, uin
         }
         else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
         {
-            /* Send the event packets */
-            va_start(Ptr, Spec);
-            EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
-            va_end(Ptr);
+            if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
+            {
+                /* Send the event packets */
+                va_start(Ptr, Spec);
+                EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr);
+                va_end(Ptr);
+            }
+            else
+            {
+                Status = CFE_EVS_APP_SQUELCHED;
+            }
         }
     }
 

modules/evs/fsw/src/cfe_evs_task.c

@@ -78,6 +78,8 @@ int32 CFE_EVS_EarlyInit(void)
     CFE_EVS_Global.EVS_TlmPkt.Payload.OutputPort        = CFE_PLATFORM_EVS_PORT_DEFAULT;
     CFE_EVS_Global.EVS_TlmPkt.Payload.LogMode           = CFE_PLATFORM_EVS_DEFAULT_LOG_MODE;
 
+    CFE_EVS_Global.EVS_EventBurstMax = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST;
+
     /* Get a pointer to the CFE reset area from the BSP */
     PspStatus = CFE_PSP_GetResetArea(&resetAreaAddr, &resetAreaSize);
 
@@ -653,9 +655,11 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data)
     {
         if (EVS_AppDataIsUsed(AppDataPtr))
         {
-            AppTlmDataPtr->AppID                 = EVS_AppDataGetID(AppDataPtr);
-            AppTlmDataPtr->AppEnableStatus       = AppDataPtr->ActiveFlag;
-            AppTlmDataPtr->AppMessageSentCounter = AppDataPtr->EventCount;
+            AppTlmDataPtr->AppID                      = EVS_AppDataGetID(AppDataPtr);
+            AppTlmDataPtr->AppEnableStatus            = AppDataPtr->ActiveFlag;
+            AppTlmDataPtr->AppMessageSentCounter      = AppDataPtr->EventCount;
+            AppTlmDataPtr->AppMessageSquelchedCounter = AppDataPtr->SquelchedCount;
+
             ++j;
             ++AppTlmDataPtr;
         }
@@ -665,9 +669,10 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data)
     /* Clear unused portion of event state data in telemetry packet */
     for (i = j; i < CFE_MISSION_ES_MAX_APPLICATIONS; i++)
     {
-        AppTlmDataPtr->AppID                 = CFE_ES_APPID_UNDEFINED;
-        AppTlmDataPtr->AppEnableStatus       = false;
-        AppTlmDataPtr->AppMessageSentCounter = 0;
+        AppTlmDataPtr->AppID                      = CFE_ES_APPID_UNDEFINED;
+        AppTlmDataPtr->AppEnableStatus            = false;
+        AppTlmDataPtr->AppMessageSentCounter      = 0;
+        AppTlmDataPtr->AppMessageSquelchedCounter = 0;
     }
 
     CFE_SB_TimeStampMsg(CFE_MSG_PTR(CFE_EVS_Global.EVS_TlmPkt.TelemetryHeader));
@@ -1245,7 +1250,8 @@ int32 CFE_EVS_ResetAppCounterCmd(const CFE_EVS_ResetAppCounterCmd_t *data)
 
     if (Status == CFE_SUCCESS)
     {
-        AppDataPtr->EventCount = 0;
+        AppDataPtr->EventCount     = 0;
+        AppDataPtr->SquelchedCount = 0;
 
         EVS_SendEvent(CFE_EVS_RSTEVTCNT_EID, CFE_EVS_EventType_DEBUG,
                       "Reset Event Counter Command Received with AppName = %s", LocalName);
@@ -1614,6 +1620,7 @@ int32 CFE_EVS_WriteAppDataFileCmd(const CFE_EVS_WriteAppDataFileCmd_t *data)
                     AppDataFile.ActiveFlag           = AppDataPtr->ActiveFlag;
                     AppDataFile.EventCount           = AppDataPtr->EventCount;
                     AppDataFile.EventTypesActiveFlag = AppDataPtr->EventTypesActiveFlag;
+                    AppDataFile.SquelchedCount       = AppDataPtr->SquelchedCount;
 
                     /* Copy application filter data to application file data record */
                     memcpy(AppDataFile.Filters, AppDataPtr->BinFilters,

modules/evs/fsw/src/cfe_evs_task.h

@@ -44,6 +44,7 @@
 #include "cfe_platform_cfg.h"
 #include "cfe_mission_cfg.h"
 #include "osconfig.h"
+#include "cfe_time.h"
 #include "cfe_evs_api_typedefs.h"
 #include "cfe_evs_log_typedef.h"
 #include "cfe_sb_api_typedefs.h"
@@ -57,6 +58,7 @@
 #define CFE_EVS_PIPE_DEPTH           32
 #define CFE_EVS_MAX_EVENT_SEND_COUNT 65535
 #define CFE_EVS_MAX_FILTER_COUNT     65535
+#define CFE_EVS_MAX_SQUELCH_COUNT    255
 #define CFE_EVS_PIPE_NAME            "EVS_CMD_PIPE"
 #define CFE_EVS_MAX_PORT_MSG_LENGTH  (CFE_MISSION_EVS_MAX_MESSAGE_LENGTH + OS_MAX_API_NAME + 30)
 
@@ -86,18 +88,23 @@ typedef struct
 
     EVS_BinFilter_t BinFilters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Array of binary filters */
 
-    uint8  ActiveFlag;           /* Application event service active flag */
-    uint8  EventTypesActiveFlag; /* Application event types active flag */
-    uint16 EventCount;           /* Application event counter */
+    uint8     ActiveFlag;                /* Application event service active flag */
+    uint8     EventTypesActiveFlag;      /* Application event types active flag */
+    uint16    EventCount;                /* Application event counter */
+    OS_time_t LastSquelchCreditableTime; /* Time of last squelch token return */
+    int32     SquelchTokens;             /* Application event squelch token counter */
+    uint8     SquelchedCount;            /* Application events squelched counter */
 
 } EVS_AppData_t;
 
 typedef struct
 {
-    char            AppName[OS_MAX_API_NAME];                    /* Application name */
-    uint8           ActiveFlag;                                  /* Application event service active flag */
-    uint8           EventTypesActiveFlag;                        /* Application event types active flag */
-    uint16          EventCount;                                  /* Application event counter */
+    char            AppName[OS_MAX_API_NAME]; /* Application name */
+    uint8           ActiveFlag;               /* Application event service active flag */
+    uint8           EventTypesActiveFlag;     /* Application event types active flag */
+    uint16          EventCount;               /* Application event counter */
+    uint8           SquelchedCount;           /* Application events squelched counter */
+    uint8           Spare[3];
     EVS_BinFilter_t Filters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Application event filters */
 
 } CFE_EVS_AppDataFile_t;
@@ -117,7 +124,7 @@ typedef struct
     CFE_SB_PipeId_t           EVS_CommandPipe;
     osal_id_t                 EVS_SharedDataMutexID;
     CFE_ES_AppId_t            EVS_AppID;
-
+    uint32                    EVS_EventBurstMax;
 } CFE_EVS_Global_t;
 
 /*