Updated SDK commit to use the latest SDK code (#416)

* Updated SDK commit to use the latest SDK code * Addressed reviewers comments and updated unit tests to support the change * Resolved indentation errors * Resolved indentation errors * Resolved indentation errors * Resolved indentation errors * Resolved indentation errors * update device mqtt connection logic * removing DC initialization code to create multiple lock files * addressed reviewers comments to resolve test failures * resolved unit test failure * remove MemoryTrace unit test * resolved Lint errors * explicitly calling the code in SharedCrtManager to initialize allocator and api handle. This will resolve the Sensor pullish unit tests which manually creates and initializes allocator and eventloop. * addressed lint errors * addressed lint errors * addressed reviewers comment and added the config serialization test back * Update GTestMain.cpp to not cleanup resources after testing is completed This change will be reverted later. Added this to investigate DD integ test failure * Update e2e-tests-ubutu-x86 test to skip secure tunneling test * Compile device client binaries as Release build type (#422) * update sdk commit again * Updated code to shutdown on tunnel close callback and updated zlib dependency version(#426) * updated code to shutdown on tunnel close callback * addressed lint errors * addressed lint errors * addressed lint errors * updated zlib version * updated zlib version * updated comment * Updated Secure Tunneling close logic for Secure Tunneling Component (#427) * Updated tunnel shutdown logic for Secure Tunneling component * removed extra log statement * Added null check before closing connection (#429) * fixed bug to read SIGTERM signal when sent form kernal (#430) * Fix docker-build.sh (#423) There is an additional level of indirection required for the script to be able to build the container. Co-authored-by: Miguel Cano <macano953@users.noreply.github.com> * Updated google-test lib version to v1.12.0 (#431) * disabled jobs and secure tunneling if they are not compiled into binary (#436) * Update Dockerfile OpenSSL version (#435) * Update Dockerfile * Update README.md * Removed number of slash validation for MQTT topics (#438) * conflict fix * add http proxy support for secure tunneling * cleanup http proxy changes * fix format * fix format --------- Co-authored-by: marco morais <93558171+marcoemorais-aws@users.noreply.github.com> Co-authored-by: RogerZhongAWS <rgz@amazon.com> Co-authored-by: RogerZhongAWS <100961047+RogerZhongAWS@users.noreply.github.com> Co-authored-by: Miguel Cano <macano953@users.noreply.github.com>
awslabs · Dec 20, 2023 · f4da551 · f4da551
1 parent cb81641
commit f4da551
Show file tree

Hide file tree

Showing 27 changed files with 347 additions and 120 deletions.
diff --git a/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile b/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile
@@ -89,7 +89,7 @@ RUN mkdir sdk-cpp-workspace \
     && cd sdk-cpp-workspace \
     && git clone https://github.com/aws/aws-iot-device-sdk-cpp-v2.git \
     && cd aws-iot-device-sdk-cpp-v2 \
-    && git checkout ac3ba3774b031dde1b988e698880d6064d53b9d9 \
+    && git checkout 74c8b683ebe5b1cbf484f6acaa281f56aaa63948 \
     && git submodule update --init --recursive \
     && cd .. \
     && mkdir aws-iot-device-sdk-cpp-v2-build \

diff --git a/.github/docker-images/base-images/device-client/ubi8/Dockerfile b/.github/docker-images/base-images/device-client/ubi8/Dockerfile
@@ -102,7 +102,7 @@ RUN mkdir sdk-cpp-workspace \
     && cd sdk-cpp-workspace \
     && git clone https://github.com/aws/aws-iot-device-sdk-cpp-v2.git \
     && cd aws-iot-device-sdk-cpp-v2 \
-    && git checkout ac3ba3774b031dde1b988e698880d6064d53b9d9 \
+    && git checkout 74c8b683ebe5b1cbf484f6acaa281f56aaa63948 \
     && git submodule update --init --recursive \
     && cd .. \
     && mkdir aws-iot-device-sdk-cpp-v2-build \

diff --git a/.github/docker-images/base-images/device-client/ubuntu/Dockerfile b/.github/docker-images/base-images/device-client/ubuntu/Dockerfile
@@ -86,7 +86,7 @@ RUN mkdir sdk-cpp-workspace \
     && cd sdk-cpp-workspace \
     && git clone https://github.com/aws/aws-iot-device-sdk-cpp-v2.git \
     && cd aws-iot-device-sdk-cpp-v2 \
-    && git checkout ac3ba3774b031dde1b988e698880d6064d53b9d9 \
+    && git checkout 74c8b683ebe5b1cbf484f6acaa281f56aaa63948 \
     && git submodule update --init --recursive \
     && cd .. \
     && mkdir aws-iot-device-sdk-cpp-v2-build \

diff --git a/.github/workflows/e2e-ci.yml b/.github/workflows/e2e-ci.yml
@@ -370,7 +370,7 @@ jobs:
           DEVICE_KEY_SECRET: ${{ secrets.FP_DEVICE_KEY_SECRET }}
           AMAZON_ROOT_CA: ${{ secrets.AMAZON_ROOT_CA }}
         run: |
-          docker run -e AWS_ACCESS_KEY_ID="$(echo ${{ secrets.INTEG_USER_KEY_ID }})" -e AWS_SECRET_ACCESS_KEY="$(echo ${{ secrets.INTEG_USER_KEY_SECRET }})" -e IOT_ENDPOINT="$(echo $IOT_ENDPOINT)" -e CERTIFICATE="$(echo $CERTIFICATE)" -e DEVICE_KEY_SECRET="$(echo $DEVICE_KEY_SECRET)" -e AMAZON_ROOT_CA="$(echo $AMAZON_ROOT_CA)" -e THING_NAME=fleetprovisioning ${{ steps.build-test-runner.outputs.imageid }} --clean-up
+          docker run -e AWS_ACCESS_KEY_ID="$(echo ${{ secrets.INTEG_USER_KEY_ID }})" -e AWS_SECRET_ACCESS_KEY="$(echo ${{ secrets.INTEG_USER_KEY_SECRET }})" -e IOT_ENDPOINT="$(echo $IOT_ENDPOINT)" -e CERTIFICATE="$(echo $CERTIFICATE)" -e DEVICE_KEY_SECRET="$(echo $DEVICE_KEY_SECRET)" -e AMAZON_ROOT_CA="$(echo $AMAZON_ROOT_CA)" -e THING_NAME=fleetprovisioning ${{ steps.build-test-runner.outputs.imageid }} --skip-st --clean-up
   e2e-tests-ubuntu-aarch64:
     runs-on: ubuntu-latest
     if: ${{ false }} # Disabled for now. aarch64 local proxy build takes too long

diff --git a/CMakeLists.txt.awssdk b/CMakeLists.txt.awssdk
@@ -5,7 +5,7 @@ project(aws-iot-device-sdk-cpp-v2-download NONE)
 include(ExternalProject)
 ExternalProject_Add(aws-iot-device-sdk-cpp-v2
         GIT_REPOSITORY          https://github.com/aws/aws-iot-device-sdk-cpp-v2.git
-        GIT_TAG                 ac3ba3774b031dde1b988e698880d6064d53b9d9
+        GIT_TAG                 74c8b683ebe5b1cbf484f6acaa281f56aaa63948
         SOURCE_DIR              "${CMAKE_BINARY_DIR}/aws-iot-device-sdk-cpp-v2-src"
         BINARY_DIR              "${CMAKE_BINARY_DIR}/aws-iot-device-sdk-cpp-v2-build"
         CONFIGURE_COMMAND       ""

diff --git a/integration-tests/source/GTestMain.cpp b/integration-tests/source/GTestMain.cpp
@@ -104,7 +104,7 @@ bool parseCliArgs(int argc, char **argv)
         }
         else if (currentArg == CLI_CLEAN_UP)
         {
-            CLEAN_UP = true;
+            CLEAN_UP = false;
         }
         else
         {
@@ -185,4 +185,4 @@ int main(int argc, char **argv)
     printf("Tests Complete!\n");
     Aws::ShutdownAPI(options);
     return rc;
-}
+}
diff --git a/source/SharedCrtResourceManager.cpp b/source/SharedCrtResourceManager.cpp
@@ -39,11 +39,31 @@ bool SharedCrtResourceManager::initialize(
     std::shared_ptr<Util::FeatureRegistry> featureRegistry)
 {
     features = featureRegistry;
-    initializeAllocator(config);
     initialized = buildClient(config) == SharedCrtResourceManager::SUCCESS;
     return initialized;
 }
 
+void SharedCrtResourceManager::loadMemTraceLevelFromEnvironment()
+{
+    const char *memTraceLevelStr = std::getenv("AWS_CRT_MEMORY_TRACING");
+    if (memTraceLevelStr)
+    {
+        switch (atoi(memTraceLevelStr))
+        {
+            case AWS_MEMTRACE_BYTES:
+                LOG_DEBUG(Config::TAG, "Set AWS_CRT_MEMORY_TRACING=AWS_MEMTRACE_BYTES");
+                memTraceLevel = AWS_MEMTRACE_BYTES;
+                break;
+            case AWS_MEMTRACE_STACKS:
+                LOG_DEBUG(Config::TAG, "Set AWS_CRT_MEMORY_TRACING=AWS_MEMTRACE_STACKS");
+                memTraceLevel = AWS_MEMTRACE_STACKS;
+                break;
+            default:
+                break;
+        }
+    }
+}
+
 bool SharedCrtResourceManager::locateCredentials(const PlainConfig &config) const
 {
     struct stat fileInfo;
@@ -174,22 +194,24 @@ bool SharedCrtResourceManager::setupLogging(const PlainConfig &config) const
     return true;
 }
 
-void SharedCrtResourceManager::initializeAllocator(const PlainConfig &config)
+void SharedCrtResourceManager::initializeAllocator()
 {
+    loadMemTraceLevelFromEnvironment();
     allocator = aws_default_allocator();
-    memTraceLevel = config.memTraceLevel;
+
     if (memTraceLevel != AWS_MEMTRACE_NONE)
     {
         // If memTraceLevel == AWS_MEMTRACE_STACKS(2), then by default 8 frames per stack are used.
         allocator = aws_mem_tracer_new(allocator, nullptr, memTraceLevel, 0);
     }
-}
 
-int SharedCrtResourceManager::buildClient(const PlainConfig &config)
-{
     // We MUST declare an instance of the ApiHandle to perform global initialization
     // of the SDK libraries
     apiHandle = unique_ptr<ApiHandle>(new ApiHandle());
+}
+
+int SharedCrtResourceManager::buildClient(const PlainConfig &config)
+{
     if (config.logConfig.sdkLoggingEnabled)
     {
         if (!setupLogging(config))
@@ -318,12 +340,13 @@ int SharedCrtResourceManager::establishConnection(const PlainConfig &config)
     {
         proxyOptions.HostName = proxyConfig.proxyHost->c_str();
         proxyOptions.Port = proxyConfig.proxyPort.value();
+        proxyOptions.ProxyConnectionType = Aws::Crt::Http::AwsHttpProxyConnectionType::Tunneling;
 
         LOGM_INFO(
             TAG,
             "Attempting to establish MQTT connection with proxy: %s:%u",
-            proxyConfig.proxyHost->c_str(),
-            proxyConfig.proxyPort.value());
+            proxyOptions.HostName.c_str(),
+            proxyOptions.Port);
 
         if (proxyConfig.httpProxyAuthEnabled)
         {
@@ -520,6 +543,7 @@ void SharedCrtResourceManager::disconnect()
     {
         return;
     }
+
     if (connection->Disconnect())
     {
         if (connectionClosedPromise.get_future().wait_for(std::chrono::seconds(DEFAULT_WAIT_TIME_SECONDS)) ==

diff --git a/source/SharedCrtResourceManager.h b/source/SharedCrtResourceManager.h
@@ -49,7 +49,7 @@ namespace Aws
 
                 int buildClient(const PlainConfig &config);
 
-                void initializeAllocator(const PlainConfig &config);
+                void loadMemTraceLevelFromEnvironment();
 
               protected:
                 /**
@@ -58,7 +58,7 @@ namespace Aws
                 bool locateCredentials(const PlainConfig &config) const;
 
               public:
-                SharedCrtResourceManager() = default;
+                SharedCrtResourceManager() {}
 
                 virtual ~SharedCrtResourceManager();
 
@@ -80,6 +80,8 @@ namespace Aws
 
                 bool initialize(const PlainConfig &config, std::shared_ptr<Util::FeatureRegistry> featureRegistry);
 
+                void initializeAllocator();
+
                 void initializeAWSHttpLib();
 
                 int establishConnection(const PlainConfig &config);

diff --git a/source/config/Config.cpp b/source/config/Config.cpp
@@ -290,24 +290,6 @@ bool PlainConfig::LoadFromCliArgs(const CliArgs &cliArgs)
 
 bool PlainConfig::LoadFromEnvironment()
 {
-    const char *memTraceLevelStr = std::getenv("AWS_CRT_MEMORY_TRACING");
-    if (memTraceLevelStr)
-    {
-        switch (atoi(memTraceLevelStr))
-        {
-            case AWS_MEMTRACE_BYTES:
-                LOG_DEBUG(Config::TAG, "Set AWS_CRT_MEMORY_TRACING=AWS_MEMTRACE_BYTES");
-                memTraceLevel = AWS_MEMTRACE_BYTES;
-                break;
-            case AWS_MEMTRACE_STACKS:
-                LOG_DEBUG(Config::TAG, "Set AWS_CRT_MEMORY_TRACING=AWS_MEMTRACE_STACKS");
-                memTraceLevel = AWS_MEMTRACE_STACKS;
-                break;
-            default:
-                break;
-        }
-    }
-
     const char *lockFilePathIn = std::getenv("LOCK_FILE_PATH");
     if (lockFilePathIn)
     {

diff --git a/source/config/Config.h b/source/config/Config.h
@@ -108,7 +108,6 @@ namespace Aws
                 Aws::Crt::Optional<std::string> rootCa;
                 Aws::Crt::Optional<std::string> thingName;
 
-                aws_mem_trace_level memTraceLevel{AWS_MEMTRACE_NONE};
                 std::string lockFilePath{DEFAULT_LOCK_FILE_PATH};
 
                 struct LogConfig : public LoadableFromJsonAndCliAndEnvironment

diff --git a/source/main.cpp b/source/main.cpp
@@ -293,12 +293,17 @@ namespace Aws
 
 int main(int argc, char *argv[])
 {
-    CliArgs cliArgs;
+
     if (Config::CheckTerminalArgs(argc, argv))
     {
         LoggerFactory::getLoggerInstance()->shutdown();
         return 0;
     }
+
+    resourceManager = std::make_shared<SharedCrtResourceManager>();
+    resourceManager->initializeAllocator();
+
+    CliArgs cliArgs;
     if (!Config::ParseCliArgs(argc, argv, cliArgs) || !config.init(cliArgs))
     {
         LOGM_ERROR(
@@ -352,8 +357,6 @@ int main(int argc, char *argv[])
     sigprocmask(SIG_BLOCK, &sigset, nullptr);
 
     auto listener = std::make_shared<DefaultClientBaseNotifier>();
-    resourceManager = std::make_shared<SharedCrtResourceManager>();
-
     if (!resourceManager.get()->initialize(config.config, features))
     {
         LOGM_ERROR(TAG, "*** %s: Failed to initialize AWS CRT SDK.", DC_FATAL_ERROR);

diff --git a/source/tunneling/SecureTunnelWrapper.cpp b/source/tunneling/SecureTunnelWrapper.cpp
@@ -40,6 +40,43 @@ SecureTunnelWrapper::SecureTunnelWrapper(
           onSessionReset))
 {
 }
+
+SecureTunnelWrapper::SecureTunnelWrapper(
+    Aws::Crt::Allocator *allocator,
+    Aws::Crt::Io::ClientBootstrap *bootstrap,
+    const Aws::Crt::Io::SocketOptions &socketOptions,
+    const Aws::Crt::Http::HttpClientConnectionProxyOptions &proxyOptions,
+    const std::string &accessToken,
+    aws_secure_tunneling_local_proxy_mode localProxyMode,
+    const std::string &endpoint,
+    const std::string &rootCa,
+    const Aws::Iotsecuretunneling::OnConnectionComplete &onConnectionComplete,
+    const Aws::Iotsecuretunneling::OnConnectionShutdown &onConnectionShutdown,
+    const Aws::Iotsecuretunneling::OnSendDataComplete &onSendDataComplete,
+    const Aws::Iotsecuretunneling::OnDataReceive &onDataReceive,
+    const Aws::Iotsecuretunneling::OnStreamStart &onStreamStart,
+    const Aws::Iotsecuretunneling::OnStreamReset &onStreamReset,
+    const Aws::Iotsecuretunneling::OnSessionReset &onSessionReset)
+    : secureTunnel((Aws::Iotsecuretunneling::SecureTunnelBuilder(
+                        allocator,
+                        *bootstrap,
+                        socketOptions,
+                        accessToken,
+                        localProxyMode,
+                        endpoint))
+                       .WithHttpClientConnectionProxyOptions(proxyOptions)
+                       .WithRootCa(rootCa)
+                       .WithOnConnectionComplete(onConnectionComplete)
+                       .WithOnConnectionShutdown(onConnectionShutdown)
+                       .WithOnSendDataComplete(onSendDataComplete)
+                       .WithOnDataReceive(onDataReceive)
+                       .WithOnStreamStart(onStreamStart)
+                       .WithOnStreamReset(onSessionReset)
+                       .WithOnSessionReset(onSessionReset)
+                       .Build())
+{
+}
+
 int SecureTunnelWrapper::Connect()
 {
     return secureTunnel->Connect();

diff --git a/source/tunneling/SecureTunnelWrapper.h b/source/tunneling/SecureTunnelWrapper.h
@@ -19,10 +19,32 @@ namespace Aws
                   public:
                     SecureTunnelWrapper() = default;
                     virtual ~SecureTunnelWrapper() = default;
+
+                    // Without HTTP Proxy
+                    SecureTunnelWrapper(
+                        Aws::Crt::Allocator *allocator,
+                        Aws::Crt::Io::ClientBootstrap *clientBootstrap,
+                        const Aws::Crt::Io::SocketOptions &socketOptions,
+
+                        const std::string &accessToken,
+                        aws_secure_tunneling_local_proxy_mode localProxyMode,
+                        const std::string &endpointHost,
+                        const std::string &rootCa,
+
+                        const Aws::Iotsecuretunneling::OnConnectionComplete &onConnectionComplete,
+                        const Aws::Iotsecuretunneling::OnConnectionShutdown &onConnectionShutdown,
+                        const Aws::Iotsecuretunneling::OnSendDataComplete &onSendDataComplete,
+                        const Aws::Iotsecuretunneling::OnDataReceive &onDataReceive,
+                        const Aws::Iotsecuretunneling::OnStreamStart &onStreamStart,
+                        const Aws::Iotsecuretunneling::OnStreamReset &onStreamReset,
+                        const Aws::Iotsecuretunneling::OnSessionReset &onSessionReset);
+
+                    // With HTTP Proxy
                     SecureTunnelWrapper(
                         Aws::Crt::Allocator *allocator,
                         Aws::Crt::Io::ClientBootstrap *clientBootstrap,
                         const Aws::Crt::Io::SocketOptions &socketOptions,
+                        const Aws::Crt::Http::HttpClientConnectionProxyOptions &proxyOptions,
 
                         const std::string &accessToken,
                         aws_secure_tunneling_local_proxy_mode localProxyMode,
@@ -47,7 +69,7 @@ namespace Aws
 
                     virtual bool IsValid();
 
-                    std::unique_ptr<Aws::Iotsecuretunneling::SecureTunnel> secureTunnel;
+                    std::shared_ptr<Aws::Iotsecuretunneling::SecureTunnel> secureTunnel;
 
                   private:
                     /**