64 bit fix and initial Solaris support

This commit was SVN r16967.

Author: Brad Penoff

ompi/mca/btl/sctp/btl_sctp_component.c

@@ -237,9 +237,11 @@ int mca_btl_sctp_component_open(void)
         !mca_btl_sctp_param_register_int ("use_nagle", 0);
     /* port_min */
     /* port_range */
-    /* use a single one-to-many socket by default */
+    /* use a single one-to-many socket by default except in Solaris (see
+     *  the configure.m4 file)
+     */
     mca_btl_sctp_component.sctp_if_11 =
-        mca_btl_sctp_param_register_int ("if_11", 0);
+        mca_btl_sctp_param_register_int ("if_11", OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET);
 
     /* have lower exclusivity than tcp */
     mca_btl_sctp_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW;

ompi/mca/btl/sctp/btl_sctp_proc.c

@@ -362,7 +362,7 @@ bool mca_btl_sctp_proc_accept(mca_btl_sctp_proc_t* btl_proc, struct sockaddr_in*
  *
  *  TODO - change this to use a hash for constant time performance
  */
-int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *table) {
+int mca_btl_sctp_proc_check(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table) {
 #if MCA_BTL_SCTP_DONT_USE_HASH
     int i;
     for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++) {
@@ -376,6 +376,7 @@ int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *ta
     return INVALID_ENTRY;
 #else
     mca_btl_sctp_proc_t *val;
+    /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */
     int rc = opal_hash_table_get_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, &val);
     if(OPAL_SUCCESS == rc) {
         return VALID_ENTRY;
@@ -394,7 +395,7 @@ int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *ta
  *  TODO change this to a hash table that can expand to eliminate
  *    MCA_BTL_SCTP_PROC_TABLE_SIZE limitation
  */
-void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table) {
+void mca_btl_sctp_proc_add(sctp_assoc_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table) {
 #if MCA_BTL_SCTP_DONT_USE_HASH
     int i;
     for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++) {
@@ -406,6 +407,7 @@ void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct
         }
     }
 #else
+    /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */
     int rc = opal_hash_table_set_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, proc);
     /* TODO handle return code */
 #endif    
@@ -417,7 +419,7 @@ void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct
  * ------------------------------------------
  *  Returns pointer to a proc that is indexed by the association id.
  */
-mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc_table_node *table) {
+mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table) {
 #if MCA_BTL_SCTP_DONT_USE_HASH
     int i;
     for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++){
@@ -428,6 +430,7 @@ mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc
     return NULL;
 #else
     mca_btl_sctp_proc_t *val;
+    /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */
     int rc = opal_hash_table_get_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, &val);
     if(OPAL_SUCCESS == rc) {
         return val;

ompi/mca/btl/sctp/btl_sctp_proc.h

@@ -25,6 +25,7 @@
 #include "btl_sctp.h"
 #include "btl_sctp_addr.h"
 #include "btl_sctp_endpoint.h"
+#include <netinet/sctp.h>
 
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
@@ -94,17 +95,17 @@ enum {
 #define MCA_BTL_SCTP_PROC_TABLE_SIZE 256
 struct mca_btl_sctp_proc_table_node {
     int valid;
-    uint32_t sctp_assoc_id;
+    sctp_assoc_t sctp_assoc_id;
     struct mca_btl_sctp_proc_t *proc;
 };
 typedef struct mca_btl_sctp_proc_table_node mca_btl_sctp_proc_table_node;
 
 extern struct mca_btl_sctp_proc_table_node *recvr_proc_table;
 extern struct mca_btl_sctp_proc_table_node *sender_proc_table;
 
-int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *table);
-void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table);
-mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc_table_node *table);
+int mca_btl_sctp_proc_check(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table);
+void mca_btl_sctp_proc_add(sctp_assoc_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table);
+mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

ompi/mca/btl/sctp/btl_sctp_recv_handler.c

@@ -141,7 +141,7 @@ void mca_btl_sctp_recv_handler(int sd, short flags, void *user) {
 
     /* Check if sender is known to us. */
 
-    if((mca_btl_sctp_proc_check(((uint32_t)(sri.sinfo_assoc_id)), recvr_proc_table)) == VALID_ENTRY) {
+    if((mca_btl_sctp_proc_check(sri.sinfo_assoc_id, recvr_proc_table)) == VALID_ENTRY) {
 
         mca_btl_base_endpoint_t *btl_endpoint;
         mca_btl_sctp_frag_t* frag;
@@ -233,7 +233,7 @@ void mca_btl_sctp_recv_handler(int sd, short flags, void *user) {
             CLOSE_THE_SOCKET(sd);
             return;
         }
-        mca_btl_sctp_proc_add((uint32_t)sri.sinfo_assoc_id, btl_proc, recvr_proc_table);
+        mca_btl_sctp_proc_add(sri.sinfo_assoc_id, btl_proc, recvr_proc_table);
 
 
         /* are there any existing peer instances will to accept this connection */

ompi/mca/btl/sctp/btl_sctp_utils.c

@@ -63,6 +63,39 @@ struct sockaddr_in mca_btl_sctp_utils_sockaddr_from_endpoint(struct mca_btl_base
 int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, 
         struct sockaddr *to_addr, socklen_t to_len, uint16_t stream_no) {
 
+#if OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS
+    /* required for Solaris since struct msghdr has no msg_control field */
+
+    int current_cnt=0, total_offset=0, total=0, byte_sent;
+    char *send_buf;
+
+    /* count the total and allocate space to copy to */
+    while(current_cnt < len) {
+        total += vec[current_cnt].iov_len;
+        current_cnt++;
+    }
+    if(NULL == (send_buf = (char *) malloc(total))) {
+        BTL_ERROR(("Ran out of memory."));
+        return 0;
+    }
+
+    /* combine all iovcnt's into one message.  write all or nothing */
+    current_cnt=0;
+    while(current_cnt < len) {
+        memcpy(send_buf+total_offset, vec[current_cnt].iov_base, vec[current_cnt].iov_len);
+        total_offset += vec[current_cnt].iov_len;
+        current_cnt++;
+    }
+
+    byte_sent = sctp_sendmsg(sd, send_buf, total, (struct sockaddr *) to_addr, 
+                    sizeof(*to_addr), 0, 0, stream_no, 0, 0);
+    free(send_buf);
+
+    return byte_sent;
+
+#else
+    /* uses iovec directly */
+
     char outcmesg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))];
     struct cmsghdr *cmesg;
     struct msghdr outmesg;
@@ -97,4 +130,5 @@ int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len,
     srinfo->sinfo_context = cxt;
 
     return sendmsg(sd, &outmesg, 0);
+#endif
 }

ompi/mca/btl/sctp/configure.m4

@@ -37,12 +37,20 @@ AC_DEFUN([OMPI_CHECK_SCTP],[
     case "$host" in
     *linux*)
 	ompi_sctp_try_to_build="yes"
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0,
+               [Default value for socket style to use with SCTP BTL])
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0,
+               [False if you can use iovec's directly with SCTP BTL])
 	;;
     *bsd*)
         # only add -DFREEBSD once to get extra sin_len field
         btl_sctp_CFLAGS="`echo $btl_sctp_CFLAGS | sed 's/-DFREEBSD//g'`"
         btl_sctp_CFLAGS="$btl_sctp_CFLAGS -DFREEBSD"
 	ompi_sctp_try_to_build="yes"
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0,
+               [Default value for socket style to use with SCTP BTL])
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0,
+               [False if you can use iovec's directly with SCTP BTL])
 	AC_MSG_WARN([Adding -DFREEBSD to set extra sin_len field in sockaddr.])
 	;;
 # Mac OS X support for SCTP NKE. Adjustments should look like *bsd*...
@@ -51,10 +59,23 @@ AC_DEFUN([OMPI_CHECK_SCTP],[
         btl_sctp_CFLAGS="`echo $btl_sctp_CFLAGS | sed 's/-DFREEBSD//g'`"
         btl_sctp_CFLAGS="$btl_sctp_CFLAGS -DFREEBSD"
 	ompi_sctp_try_to_build="yes"
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0,
+               [Default value for socket style to use with SCTP BTL])
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0,
+               [False if you can use iovec's directly with SCTP BTL])
 	AC_MSG_WARN([Adding -DFREEBSD to set extra sin_len field in sockaddr.])
 	;;
+    *solaris*)
+        # Solaris SCTP stack makes different assumptions about one-to-many
+        #   sockets so change the default to use one-to-one sockets
+	ompi_sctp_try_to_build="yes"
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 1,
+               [Default value for socket style to use with SCTP BTL])
+        AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 1,
+               [False if you can use iovec's directly with SCTP BTL])
+	;;
     *)
-	AC_MSG_WARN([Only build sctp BTL on Linux, Mac OS X, and BSD variants])
+	AC_MSG_WARN([Only build sctp BTL on Solaris, Linux, Mac OS X, and BSD variants])
 	;;
     esac