ENG-2726: Managed DNS to get the endpoints for a YB EE universe.

Summary: https://yugabyte.atlassian.net/browse/ENG-2726 Test Plan: - Configure permissions as described in ENG-2726. - `export YB_LOCAL_MASTER_EXTRA_PARAMS="--tservers_domain_name <DOMAIN_NAME> --tservers_route53_hosted_zone_id <HOSTED_ZONE_ID>"` - `local_cluster.ctl start` - Make sure tserver addresses are appeared in Route53. - Send STOP signal to one of tservers, it should disappear from Route53 once it no longer considered as live (one minute timeout). - Send CONT signal to stopped tserver, it should appear again in Route53 in 5-10 seconds. - Also check with `nslookup <DOMAIN_NAME>` Reviewers: mikhail, bogdan, sergei Reviewed By: sergei Subscribers: ybase, bharat Differential Revision: https://phabricator.dev.yugabyte.com/D4250
yugabyte · Mar 6, 2018 · a44baa1 · a44baa1
1 parent b36c90e
commit a44baa1
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 23 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1426,6 +1426,9 @@ endif()
 ## aws-cpp-sdk
 find_package(AwsCppSdk REQUIRED)
 include_directories(SYSTEM ${AWS_CPP_SDK_INCLUDE_DIRS})
+ADD_THIRDPARTY_LIB(aws-cpp-sdk-core
+  STATIC_LIB "${AWS_CPP_SDK_CORE_STATIC_LIB}"
+  SHARED_LIB "${AWS_CPP_SDK_CORE_SHARED_LIB}")
 ADD_THIRDPARTY_LIB(aws-cpp-sdk-route53
   STATIC_LIB "${AWS_CPP_SDK_ROUTE53_STATIC_LIB}"
   SHARED_LIB "${AWS_CPP_SDK_ROUTE53_SHARED_LIB}")

diff --git a/bin/local_cluster_ctl.sh b/bin/local_cluster_ctl.sh
@@ -550,12 +550,12 @@ cql_rpc_port=9042
 
 common_params=" --version_file_json_path $build_root --callhome_enabled=false"
 
-master_optional_params=""
+master_optional_params=" ${YB_LOCAL_MASTER_EXTRA_PARAMS:-}"
 if [[ -n "$replication_factor" ]]; then
   master_optional_params+=" --replication_factor $replication_factor"
 fi
 
-tserver_optional_params=""
+tserver_optional_params=" ${YB_LOCAL_TSERVER_EXTRA_PARAMS:-}"
 if [[ -n "$yb_num_shards_per_tserver" ]]; then
   tserver_optional_params+=" --yb_num_shards_per_tserver $yb_num_shards_per_tserver"
 fi

diff --git a/cmake_modules/FindAwsCppSdk.cmake b/cmake_modules/FindAwsCppSdk.cmake
@@ -1,6 +1,12 @@
 find_path(AWS_CPP_SDK_INCLUDE_DIR aws
   NO_CMAKE_SYSTEM_PATH
   NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(AWS_CPP_SDK_CORE_SHARED_LIB aws-cpp-sdk-core
+  NO_CMAKE_SYSTEM_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(AWS_CPP_SDK_CORE_STATIC_LIB libaws-cpp-sdk-core.a
+  NO_CMAKE_SYSTEM_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH)
 find_library(AWS_CPP_SDK_ROUTE53_SHARED_LIB aws-cpp-sdk-route53
   NO_CMAKE_SYSTEM_PATH
   NO_SYSTEM_ENVIRONMENT_PATH)

diff --git a/src/yb/master/CMakeLists.txt b/src/yb/master/CMakeLists.txt
@@ -107,7 +107,8 @@ target_link_libraries(master
   ${MASTER_PROTO_LIBS}
   rpc_header_proto
   master_util
-  version_info_proto)
+  version_info_proto
+  ${MASTER_LIBS_EXTENSIONS})
 
 set(MASTER_RPC_SRCS
   master_rpc.cc)

diff --git a/src/yb/master/master.h b/src/yb/master/master.h
@@ -63,6 +63,7 @@ using yb::consensus::RaftConfigPB;
 namespace master {
 
 class CatalogManager;
+class TSDescriptor;
 class TSManager;
 class MasterPathHandlers;
 
@@ -141,6 +142,9 @@ class Master : public server::RpcAndWebServerBase {
   // Returns the number of system tables (used only for testing currently).
   size_t NumSystemTables() const;
 
+  // Called by MasterService on every TS heartbeat.
+  virtual void OnTSHeartbeat(const std::vector<std::shared_ptr<TSDescriptor>>& live_tservers) {}
+
  protected:
   virtual CHECKED_STATUS RegisterServices();
 

diff --git a/src/yb/master/master_service.cc b/src/yb/master/master_service.cc
@@ -187,6 +187,7 @@ void MasterServiceImpl::TSHeartbeat(const TSHeartbeatRequestPB* req,
   for (const auto& desc : descs) {
     desc->GetTSInformationPB(resp->add_tservers());
   }
+  server_->OnTSHeartbeat(descs);
 
   rpc.RespondSuccess();
 }

diff --git a/src/yb/master/master_service.h b/src/yb/master/master_service.h
@@ -46,6 +46,8 @@ class MasterServiceImpl : public MasterServiceIf,
                           public MasterServiceBase {
  public:
   explicit MasterServiceImpl(Master* server);
+  MasterServiceImpl(const MasterServiceImpl&) = delete;
+  void operator=(const MasterServiceImpl&) = delete;
 
   virtual void TSHeartbeat(const TSHeartbeatRequestPB* req,
                            TSHeartbeatResponsePB* resp,
@@ -176,7 +178,6 @@ class MasterServiceImpl : public MasterServiceIf,
       rpc::RpcContext rpc) override;
 
  private:
-  DISALLOW_COPY_AND_ASSIGN(MasterServiceImpl);
 };
 
 } // namespace master

diff --git a/src/yb/util/net/net_util.cc b/src/yb/util/net/net_util.cc
@@ -178,41 +178,52 @@ const string getaddrinfo_rc_to_string(int rc) {
   }
   return Substitute("$0 ($1)", rc, s);
 }
-}  // namespace
 
-Status HostPort::ResolveAddresses(std::vector<Endpoint>* addresses) const {
-  TRACE_EVENT1("net", "HostPort::ResolveAddresses",
-               "host", host_);
+Result<std::unique_ptr<addrinfo, AddrinfoDeleter>> HostToInetAddrInfo(const std::string& host) {
   struct addrinfo hints;
   memset(&hints, 0, sizeof(hints));
   hints.ai_family = AF_INET;
   hints.ai_socktype = SOCK_STREAM;
   struct addrinfo* res = nullptr;
   int rc = 0;
   LOG_SLOW_EXECUTION(WARNING, 200,
-                     Substitute("resolving address for $0", host_)) {
-    rc = getaddrinfo(host_.c_str(), nullptr, &hints, &res);
+                     Substitute("resolving address for $0", host)) {
+    rc = getaddrinfo(host.c_str(), nullptr, &hints, &res);
   }
   if (rc != 0) {
-    return STATUS(NetworkError,
-      StringPrintf("Unable to resolve address '%s', getaddrinfo returned %s",
-                   host_.c_str(),
-                   getaddrinfo_rc_to_string(rc).c_str()),
-      gai_strerror(rc));
+    return STATUS_FORMAT(NetworkError, "Unable to resolve address $0, getaddrinfo returned $1: $2",
+                         host.c_str(), getaddrinfo_rc_to_string(rc).c_str(), gai_strerror(rc));
+  } else {
+    return std::unique_ptr<addrinfo, AddrinfoDeleter>(res);
+  }
+}
+
+template <typename F>
+CHECKED_STATUS ResolveInetAddresses(const std::string& host, F func) {
+  auto addrinfo_holder = CHECK_RESULT(HostToInetAddrInfo(host));
+  struct addrinfo* addrinfo = addrinfo_holder.get();
+  for (; addrinfo != nullptr; addrinfo = addrinfo->ai_next) {
+    CHECK_EQ(addrinfo->ai_family, AF_INET);
+    struct sockaddr_in* addr = reinterpret_cast<struct sockaddr_in*>(addrinfo->ai_addr);
+    func(addr);
   }
-  gscoped_ptr<addrinfo, AddrinfoDeleter> scoped_res(res);
-  for (; res != nullptr; res = res->ai_next) {
-    CHECK_EQ(res->ai_family, AF_INET);
-    struct sockaddr_in* addr = reinterpret_cast<struct sockaddr_in*>(res->ai_addr);
+  return Status::OK();
+}
+
+}  // namespace
+
+Status HostPort::ResolveAddresses(std::vector<Endpoint>* addresses) const {
+  TRACE_EVENT1("net", "HostPort::ResolveAddresses",
+               "host", host_);
+  return ResolveInetAddresses(host_, [this, addresses](struct sockaddr_in* addr) {
     addr->sin_port = htons(port_);
     Endpoint sockaddr;
     memcpy(sockaddr.data(), addr, sizeof(*addr));
     if (addresses) {
       addresses->push_back(sockaddr);
     }
     VLOG(2) << "Resolved address " << sockaddr << " for host/port " << ToString();
-  }
-  return Status::OK();
+  });
 }
 
 Status HostPort::ParseStrings(const string& comma_sep_addrs,
@@ -494,4 +505,29 @@ std::string HostPortToString(const std::string& host, int port) {
   return Format("$0:$1", host, port);
 }
 
+Status HostToAddresses(
+    const std::string& host,
+    boost::container::small_vector<IpAddress, 1>* addresses) {
+  return ResolveInetAddresses(host, [&addresses](struct sockaddr_in* addr) {
+    Endpoint endpoint;
+    memcpy(endpoint.data(), addr, sizeof(*addr));
+    addresses->push_back(endpoint.address());
+  });
+}
+
+Result<IpAddress> HostToAddress(const std::string& host) {
+  boost::container::small_vector<IpAddress, 1> addrs;
+  RETURN_NOT_OK(HostToAddresses(host, &addrs));
+  if (addrs.empty()) {
+    return STATUS(NetworkError, "Unable to resolve address", host);
+  }
+  auto addr = addrs.front();
+  if (addrs.size() > 1) {
+    VLOG(1) << "Hostname " << host << " resolved to more than one address. "
+            << "Using address: " << addr;
+  }
+  return addr;
+}
+
+
 } // namespace yb
diff --git a/src/yb/util/net/net_util.h b/src/yb/util/net/net_util.h
@@ -36,6 +36,8 @@
 #include <vector>
 #include <memory>
 
+#include <boost/container/small_vector.hpp>
+
 #include "yb/util/env.h"
 #include "yb/util/status.h"
 #include "yb/util/net/net_fwd.h"
@@ -162,5 +164,11 @@ Status GetLocalAddresses(std::vector<IpAddress>* result, AddressFilter filter);
 // Convert the given host/port pair to a string of the host:port format.
 std::string HostPortToString(const std::string& host, int port);
 
+CHECKED_STATUS HostToAddresses(
+    const std::string& host,
+    boost::container::small_vector<IpAddress, 1>* addresses);
+
+Result<IpAddress> HostToAddress(const std::string& host);
+
 } // namespace yb
 #endif  // YB_UTIL_NET_NET_UTIL_H
diff --git a/thirdparty/build_definitions/aws_sdk_cpp.py b/thirdparty/build_definitions/aws_sdk_cpp.py
@@ -39,6 +39,7 @@ def build(self, builder):
                                        ['-DCMAKE_BUILD_TYPE=Release',
                                         '-DBUILD_SHARED_LIBS={}'.format(shared),
                                         '-DBUILD_ONLY=route53',
+                                        '-DCUSTOM_MEMORY_MANAGEMENT=0',
                                         '-DENABLE_TESTING=Off',
                                         '-DENABLE_UNITY_BUILD=On',
                                         '-DCMAKE_INSTALL_PREFIX={}'.format(out_dir),

diff --git a/thirdparty/build_definitions/curl.py b/thirdparty/build_definitions/curl.py
@@ -30,4 +30,6 @@ def build(self, builder):
         disabled_features = ['ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet', 'tftp',
                              'pop3', 'imap', 'smtp', 'gopher', 'manual', 'librtmp', 'ipv6']
         extra_args = ['--disable-' + feature for feature in disabled_features]
+        if is_mac():
+            extra_args.append('--with-ssl=/usr/local/opt/openssl')
         builder.build_with_configure(builder.log_prefix(self), extra_args)
diff --git a/thirdparty/version_for_jenkins_linux.txt b/thirdparty/version_for_jenkins_linux.txt
@@ -1 +1 @@
-2018-02-19T10_27_17
+2018-03-01T10_46_26
diff --git a/thirdparty/version_for_jenkins_mac.txt b/thirdparty/version_for_jenkins_mac.txt
@@ -1 +1 @@
-2018-02-19T02_29_59
+2018-03-01T02_58_33