libnbc: retain datatype and schedule them for release

This is a proof of concept, and only ibcast is implemented

Thanks Thomas Ponweiser for reporting this

ompi/mca/coll/libnbc/nbc.c

@@ -10,14 +10,15 @@
  *                         rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  *
  */
+
 #include "nbc_internal.h"
 #include "ompi/mca/coll/base/coll_tags.h"
 #include "ompi/op/op.h"
@@ -241,6 +242,23 @@ int NBC_Sched_unpack (void *inbuf, char tmpinbuf, int count, MPI_Datatype dataty
   return OMPI_SUCCESS;
 }
 
+/* this function schedule the release of one datatype */
+int NBC_Sched_datatype (MPI_Datatype datatype, NBC_Schedule *schedule) {
+  int ret;
+  NBC_Args_datatype datatype_args;
+  datatype_args.type = DATATYPE;
+  datatype_args.datatype = datatype;
+  ret = nbc_schedule_round_append (schedule, &datatype_args, sizeof (datatype_args), false);
+  if (OMPI_SUCCESS != ret) {
+    return ret;
+  }
+  OBJ_RETAIN(datatype);
+
+  NBC_DEBUG(10, "added datatype - ends at byte %i\n", nbc_schedule_get_size (schedule));
+
+  return OMPI_SUCCESS;
+}
+
 /* this function ends a round of a schedule */
 int NBC_Sched_barrier (NBC_Schedule *schedule) {
   return nbc_schedule_round_append (schedule, NULL, 0, true);
@@ -373,6 +391,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
   NBC_Args_op         opargs;
   NBC_Args_copy     copyargs;
   NBC_Args_unpack unpackargs;
+  NBC_Args_datatype datatypeargs;
   void *buf1,  *buf2, *buf3;
 
   /* get round-schedule address */
@@ -524,6 +543,15 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
         }
 
         break;
+
+      case DATATYPE:
+        NBC_DEBUG(5, "  DATATYPE(offset %li) ", offset);
+        NBC_GET_BYTES(ptr,datatypeargs);
+        OBJ_RELEASE(datatypeargs.datatype);
+        res = OMPI_SUCCESS;
+
+        break;
+
       default:
         NBC_Error ("NBC_Start_round: bad type %li at offset %li", (long)type, offset);
         return OMPI_ERROR;

ompi/mca/coll/libnbc/nbc_ibcast.c

@@ -5,7 +5,7 @@
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2006      The Technical University of Chemnitz. All
  *                         rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
+ * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -108,6 +108,12 @@ int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int
       return res;
     }
 
+    res = NBC_Sched_datatype (datatype, schedule);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+      OBJ_RELEASE(schedule);
+      return res;
+    }
+
     res = NBC_Sched_commit (schedule);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);

ompi/mca/coll/libnbc/nbc_internal.h

@@ -10,7 +10,7 @@
  *
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -81,7 +81,8 @@ typedef enum {
   RECV,
   OP,
   COPY,
-  UNPACK
+  UNPACK,
+  DATATYPE
 } NBC_Fn_type;
 
 /* the send argument struct */
@@ -142,6 +143,12 @@ typedef struct {
   char tmpoutbuf;
 } NBC_Args_unpack;
 
+/* release datatype */
+typedef struct {
+  NBC_Fn_type type;
+  MPI_Datatype datatype;
+} NBC_Args_datatype;
+
 /* internal function prototypes */
 int NBC_Sched_send (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier);
 int NBC_Sched_recv (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier);
@@ -151,6 +158,7 @@ int NBC_Sched_copy (void *src, char tmpsrc, int srccount, MPI_Datatype srctype,
                     MPI_Datatype tgttype, NBC_Schedule *schedule, bool barrier);
 int NBC_Sched_unpack (void *inbuf, char tmpinbuf, int count, MPI_Datatype datatype, void *outbuf, char tmpoutbuf,
                       NBC_Schedule *schedule, bool barrier);
+int NBC_Sched_datatype (MPI_Datatype datatype, NBC_Schedule *schedule);
 
 int NBC_Sched_barrier (NBC_Schedule *schedule);
 int NBC_Sched_commit (NBC_Schedule *schedule);
@@ -328,6 +336,10 @@ static inline void nbc_get_round_size (char *p, unsigned long *size) {
       /*printf("found a UNPACK at offset %li\n", (long)p-(long)schedule); */
       offset += sizeof(NBC_Args_unpack);
       break;
+    case DATATYPE:
+      /*printf("found a DATATYPE at offset%li\n", (long)p-(long)schedule); */
+      offset += sizeof(NBC_Args_datatype);
+      break;
     default:
       NBC_Error("NBC_GET_ROUND_SIZE: bad type %i at offset %li", type, offset);
       return;