Fault injection macros and functionality (plus example)

CMakeLists.txt

@@ -51,6 +51,7 @@ set(rcutils_sources
   src/strerror.c
   src/string_array.c
   src/string_map.c
+  src/testing/fault_injection.c
   src/time.c
   ${time_impl_c}
   src/uint8_array.c

Doxyfile

@@ -24,6 +24,7 @@ EXPAND_ONLY_PREDEF     = YES
 PREDEFINED             += RCUTILS_PUBLIC=
 PREDEFINED             += RCUTILS_PUBLIC_TYPE=
 PREDEFINED             += RCUTILS_WARN_UNUSED=
+PREDEFINED             += RCUTILS_ENABLE_FAULT_INJECTION=
 
 # Tag files that do not exist will produce a warning and cross-project linking will not work.
 TAGFILES += "../../../doxygen_tag_files/cppreference-doxygen-web.tag.xml=http://en.cppreference.com/w/"

include/rcutils/error_handling.h

@@ -36,6 +36,7 @@ extern "C"
 #include "rcutils/allocator.h"
 #include "rcutils/macros.h"
 #include "rcutils/snprintf.h"
+#include "rcutils/testing/fault_injection.h"
 #include "rcutils/types/rcutils_ret.h"
 #include "rcutils/visibility_control.h"
 

include/rcutils/macros.h

@@ -131,6 +131,55 @@ extern "C"
 # define RCUTILS_UNLIKELY(x) (x)
 #endif  // _WIN32
 
+#if defined RCUTILS_ENABLE_FAULT_INJECTION
+#include "rcutils/testing/fault_injection.h"
+
+/**
+ * \def RCUTILS_CAN_RETURN_WITH_ERROR_OF
+ * Indicating macro that the function intends to return possible error value.
+ *
+ * Put this macro as the first line in the function. For example:
+ *
+ * int rcutils_function_that_can_fail() {
+ *   RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+ *   ...  // rest of function
+ * }
+ *
+ * For now, this macro just simply calls `RCUTILS_MAYBE_RETURN_ERROR` if fault injection is
+ * enabled. However, for source code, the macro annotation `RCUTILS_CAN_RETURN_WITH_ERROR_OF` helps
+ * clarify that a function may return a value signifying an error and what those are.
+ *
+ * In general, you should only include a return value that originates in the function you're
+ * annotating instead of one that is merely passed on from a called function already annotated with
+ *`RCUTILS_CAN_RETURN_WITH_ERROR_OF`. If you are passing on return values from a called function,
+ * but that function is not annotated with `RCUTILS_CAN_RETURN_WITH_ERROR_OF`, then you might
+ * consider annotating that function first. If for some reason that is not desired or possible,
+ * then annotate your function as if the return values you are passing on originated from your
+ * function.
+ *
+ * If the function can return multiple return values indicating separate failure types, each one
+ * should go on a separate line.
+ *
+ * If in your function, there are expected effects on output parameters that occur during
+ * the failure case, then it will introduce a discrepency between fault injection testing and
+ * production operation. This is because the fault injection will cause the function to return
+ * where this macro is used, not at the location the error values are typically returned. To help
+ * protect against this scenario you may consider adding unit tests that checks your function does
+ * not modify output parameters when it actually returns a failing error code.
+ *
+ * If your function is void, this macro can be used without parameters. However, for the above
+ * reasoning, there should be no side effects on output parameters for all possible early returns.
+ *
+ * \param error_return_value the value returned as a result of an error. It does not need to be
+ * a rcutils_ret_t type. It could also be NULL, -1, a string error message, etc
+ */
+# define RCUTILS_CAN_RETURN_WITH_ERROR_OF(error_return_value) \
+  RCUTILS_MAYBE_RETURN_ERROR(error_return_value);
+
+#else
+# define RCUTILS_CAN_RETURN_WITH_ERROR_OF(error_return_value)
+#endif  // defined RCUTILS_ENABLE_FAULT_INJECTION
+
 #ifdef __cplusplus
 }
 #endif

include/rcutils/testing/fault_injection.h

@@ -0,0 +1,122 @@
+// Copyright 2020 Open Source Robotics Foundation, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef RCUTILS__TESTING__FAULT_INJECTION_H_
+#define RCUTILS__TESTING__FAULT_INJECTION_H_
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdint.h>
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#define RCUTILS_FAULT_INJECTION_NEVER_FAIL -1
+
+#define RCUTILS_FAULT_INJECTION_FAIL_NOW 0
+
+bool rcutils_fault_injection_is_test_complete();
+
+void _rcutils_set_fault_injection_count(int count);
+
+int_least64_t _rcutils_get_fault_injection_count();
+
+int _rcutils_maybe_fail();
+
+#if defined RCUTILS_ENABLE_FAULT_INJECTION
+
+/**
+ * \def RCUTILS_MAYBE_RETURN_ERROR
+ * \brief This macro checks and decrements a static global variable atomic counter and returns
+ * `return_value_on_error` if 0.
+ *
+ * This macro is not a function itself, so when this macro returns it will cause the calling
+ * function to return with the return value.
+ *
+ * Set the counter with `RCUTILS_SET_FAULT_INJECTION_COUNT`. If the count is less than 0, then
+ * `RCUTILS_MAYBE_RETURN_ERROR` will not cause an early return.
+ *
+ * This macro is thread-safe, and ensures that at most one invocation results in a failure for each
+ * time the fault injection counter is set with `RCUTILS_SET_FAULT_INJECTION_COUNT`
+ *
+ * \param return_value_on_error the value to return in the case of fault injected failure.
+ */
+#define RCUTILS_MAYBE_RETURN_ERROR(return_value_on_error) \
+  if (RCUTILS_FAULT_INJECTION_FAIL_NOW == _rcutils_maybe_fail()) { \
+    printf( \
+      "%s:%d Injecting fault and returning " #return_value_on_error "\n", __FILE__, __LINE__); \
+    return return_value_on_error; \
+  }
+
+/**
+ * \def RCUTILS_SET_FAULT_INJECTION_COUNT
+ * \brief Atomically set the fault injection counter.
+ *
+ * There will be at most one fault injected failure per call to RCUTILS_SET_FAULT_INJECTION_COUNT.
+ * To test all reachable fault injection locations, call this macro inside a loop and set the count
+ * to an incrementing count variable.
+ *
+ *  for (int i = 0; i < SUFFICIENTLY_LARGE_ITERATION_COUNT; ++i) {
+ *    RCUTILS_SET_FAULT_INJECTION_COUNT(i);
+ *    ... // Call function under test
+ *  }
+ * ASSERT_LT(RCUTILS_FAULT_INJECTION_NEVER_FAIL, RCUTILS_GET_FAULT_INJECTION_COUNT());
+ *
+ * Where SUFFICIENTLY_LARGE_ITERATION_COUNT is a value larger than the maximum expected calls to
+ * `RCUTILS_MAYBE_RETURN_ERROR`. This last assertion just insures that your choice for
+ * SUFFICIENTLY_LARGE_ITERATION_COUNT was large enough. To avoid having to choose this count
+ * yourself, you can use a do-while loop.
+ *
+ * int i = 0;
+ * do {
+ *   RCUTILS_SET_FAULT_INJECTION_COUNT(i++);
+ *    ... // Call function under test
+ * } while (RCUTILS_GET_FAULT_INJECTION_COUNT() <= RCUTILS_FAULT_INJECTION_NEVER_FAIL);
+ *
+ * \param count The count to set the fault injection counter to. If count is negative, then fault
+ * injection errors will be disabled. The counter is globally initialized to
+ * RCUTILS_FAULT_INJECTION_NEVER_FAIL.
+ */
+#define RCUTILS_SET_FAULT_INJECTION_COUNT(count) \
+  _rcutils_set_fault_injection_count(count);
+
+/**
+ * \def RCUTILS_GET_FAULT_INJECTION_COUNT
+ * \brief Atomically get the fault injection counter value
+ *
+ * Use this macro after running the code under test to check whether the counter reached a negative
+ * value. This is helpful so you can verify that you ran the fault injection test in a loop a
+ * sufficient number of times. Likewise, if the code under test returned with an error, but the
+ * count value was greater or equal to 0, then the failure was not caused by the fault injection
+ * counter.
+ */
+#define RCUTILS_GET_FAULT_INJECTION_COUNT() \
+  _rcutils_get_fault_injection_count();
+
+#else  // RCUTILS_ENABLE_FAULT_INJECTION
+
+#define RCUTILS_SET_FAULT_INJECTION_COUNT(count)
+
+// This needs to be set to an int for compatibility
+#define RCUTILS_GET_FAULT_INJECTION_COUNT() RCUTILS_FAULT_INJECTION_NEVER_FAIL
+
+#define RCUTILS_MAYBE_RETURN_ERROR(msg, error_statement)
+
+#endif  // defined RCUTILS_ENABLE_FAULT_INJECTION
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // RCUTILS__TESTING__FAULT_INJECTION_H_

src/shared_library.c

@@ -27,6 +27,7 @@ C_ASSERT(sizeof(void *) == sizeof(HINSTANCE));
 #endif  // _WIN32
 
 #include "rcutils/error_handling.h"
+#include "rcutils/macros.h"
 #include "rcutils/shared_library.h"
 #include "rcutils/strdup.h"
 
@@ -46,6 +47,10 @@ rcutils_load_shared_library(
   const char * library_path,
   rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_BAD_ALLOC);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_ERROR);
+
   RCUTILS_CHECK_ARGUMENT_FOR_NULL(lib, RCUTILS_RET_INVALID_ARGUMENT);
   RCUTILS_CHECK_ARGUMENT_FOR_NULL(library_path, RCUTILS_RET_INVALID_ARGUMENT);
   RCUTILS_CHECK_ALLOCATOR(&allocator, return RCUTILS_RET_INVALID_ARGUMENT);

src/snprintf.c

@@ -39,6 +39,8 @@ rcutils_snprintf(char * buffer, size_t buffer_size, const char * format, ...)
 int
 rcutils_vsnprintf(char * buffer, size_t buffer_size, const char * format, va_list args)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(-1);
+
   if (NULL == format) {
     errno = EINVAL;
     return -1;

src/strdup.c

@@ -17,16 +17,19 @@ extern "C"
 {
 #endif
 
-#include "rcutils/strdup.h"
-
 #include <stddef.h>
 #include <string.h>
 
 #include "./common.h"
+#include "rcutils/macros.h"
+#include "rcutils/strdup.h"
+
 
 char *
 rcutils_strdup(const char * str, rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(NULL);
+
   if (NULL == str) {
     return NULL;
   }
@@ -36,6 +39,8 @@ rcutils_strdup(const char * str, rcutils_allocator_t allocator)
 char *
 rcutils_strndup(const char * str, size_t string_length, rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(NULL);
+
   if (NULL == str) {
     return NULL;
   }

src/string_array.c

@@ -42,6 +42,9 @@ rcutils_string_array_init(
   size_t size,
   const rcutils_allocator_t * allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_BAD_ALLOC);
+
   if (NULL == allocator) {
     RCUTILS_SET_ERROR_MSG("allocator is null");
     return RCUTILS_RET_INVALID_ARGUMENT;
@@ -63,6 +66,8 @@ rcutils_string_array_init(
 rcutils_ret_t
 rcutils_string_array_fini(rcutils_string_array_t * string_array)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+
   if (NULL == string_array) {
     RCUTILS_SET_ERROR_MSG("string_array is null");
     return RCUTILS_RET_INVALID_ARGUMENT;

src/testing/fault_injection.c

@@ -0,0 +1,60 @@
+// Copyright 2020 Open Source Robotics Foundation, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "rcutils/testing/fault_injection.h"
+
+#include "rcutils/stdatomic_helper.h"
+
+static atomic_int_least64_t g_rcutils_fault_injection_count = -1;
+
+bool rcutils_fault_injection_is_test_complete()
+{
+#ifndef RCUTILS_ENABLE_FAULT_INJECTION
+  return true;
+#endif  // RCUTILS_ENABLE_FAULT_INJECTION
+
+  return _rcutils_get_fault_injection_count() > RCUTILS_FAULT_INJECTION_NEVER_FAIL;
+}
+
+int _rcutils_maybe_fail()
+{
+  bool set_atomic_success = false;
+  int_least64_t current_count = RCUTILS_FAULT_INJECTION_NEVER_FAIL;
+  rcutils_atomic_load(&g_rcutils_fault_injection_count, current_count);
+  do {
+    // A fault_injection_count less than 0 means that maybe_fail doesn't fail, so just return.
+    if (current_count <= RCUTILS_FAULT_INJECTION_NEVER_FAIL) {
+      return current_count;
+    }
+
+    // Otherwise decrement by one, but do so in a thread-safe manner so that exactly one calling
+    // thread gets the 0 case.
+    int_least64_t desired_count = current_count - 1;
+    rcutils_atomic_compare_exchange_strong(
+      &g_rcutils_fault_injection_count, set_atomic_success, &current_count, desired_count);
+  } while (!set_atomic_success);
+  return current_count;
+}
+
+void _rcutils_set_fault_injection_count(int count)
+{
+  rcutils_atomic_store(&g_rcutils_fault_injection_count, count);
+}
+
+int_least64_t _rcutils_get_fault_injection_count()
+{
+  int_least64_t count = 0;
+  rcutils_atomic_load(&g_rcutils_fault_injection_count, count);
+  return count;
+}