return attribute nan values as null when ignore_nan param is specified

hsds/attr_sn.py

@@ -48,8 +48,12 @@ async def GET_Attributes(request):
         raise HTTPBadRequest(reason=msg)
 
     include_data = False
+    ignore_nan = False
     if "IncludeData" in params and params["IncludeData"]:
         include_data = True
+        if "ignore_nan" in params and params["ignore_nan"]:
+            ignore_nan = True
+
     limit = None
     if "Limit" in params:
         try:
@@ -112,7 +116,7 @@ async def GET_Attributes(request):
     hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
     resp_json["hrefs"] = hrefs
 
-    resp = await jsonResponse(request, resp_json)
+    resp = await jsonResponse(request, resp_json, ignore_nan=ignore_nan)
     log.response(request, resp=resp)
     return resp
 
@@ -149,6 +153,11 @@ async def GET_Attribute(request):
 
     # TBD - verify that the obj_id belongs to the given domain
     await validateAction(app, domain, obj_id, username, "read")
+    params = request.rel_url.query
+    if "ignore_nan" in params and params["ignore_nan"]:
+        ignore_nan = True
+    else:
+        ignore_nan = False
 
     req = getDataNodeUrl(app, obj_id)
     req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
@@ -159,7 +168,6 @@ async def GET_Attribute(request):
     dn_json = await http_get(app, req, params=params)
     log.debug("got attributes json from dn for obj_id: " + str(obj_id))
 
-
     resp_json = {}
     resp_json["name"] = attr_name
     resp_json["type"] = dn_json["type"]
@@ -177,8 +185,7 @@ async def GET_Attribute(request):
     hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
     hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
     resp_json["hrefs"] = hrefs
-
-    resp = await jsonResponse(request, resp_json)
+    resp = await jsonResponse(request, resp_json, ignore_nan=ignore_nan)
     log.response(request, resp=resp)
     return resp
 
@@ -323,7 +330,6 @@ async def PUT_Attribute(request):
         shape_json["class"] = "H5S_SCALAR"
         dims = [1,]
 
-
     if "value" in body:
         if dims is None:
             msg = "Bad Request: data can not be included with H5S_NULL space"
@@ -463,6 +469,12 @@ async def GET_AttributeValue(request):
     # TBD - verify that the obj_id belongs to the given domain
     await validateAction(app, domain, obj_id, username, "read")
 
+    params = request.rel_url.query
+    if "ignore_nan" in params and params["ignore_nan"]:
+        ignore_nan = True
+    else:
+        ignore_nan = False
+
     req = getDataNodeUrl(app, obj_id)
     req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
     log.debug("get Attribute: " + req)
@@ -532,7 +544,7 @@ async def GET_AttributeValue(request):
         hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
         hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
         resp_json["hrefs"] = hrefs
-        resp = await jsonResponse(request, resp_json)
+        resp = await jsonResponse(request, resp_json, ignore_nan=ignore_nan)
         log.response(request, resp=resp)
     return resp
 

tests/integ/attr_test.py

@@ -12,6 +12,7 @@
 import unittest
 import requests
 import json
+import numpy as np
 import helper
 
 
@@ -88,8 +89,8 @@ def testListAttr(self):
             self.assertTrue("value" not in attrJson)
 
         # get all attributes including data
-        req = self.endpoint + "/groups/" + root_uuid + "/attributes?IncludeData=True"
-        rsp = requests.get(req, headers=headers)
+        params = {"IncludeData": 1}
+        rsp = requests.get(req, headers=headers, params=params)
         self.assertEqual(rsp.status_code, 200)
         rspJson = json.loads(rsp.text)
 
@@ -153,7 +154,6 @@ def testObjAttr(self):
         rspJson = json.loads(rsp.text)
         root_id = rspJson["root"]
 
-
         for col_name in ("groups", "datatypes", "datasets"):
             # create a new obj
             req = self.endpoint + '/' + col_name
@@ -367,8 +367,6 @@ def testNoShapeAttr(self):
         self.assertTrue("value" in rspJson)
         self.assertEqual(rspJson["value"], None)
 
-
-
     def testPutFixedString(self):
         # Test PUT value for 1d attribute with fixed length string types
         print("testPutFixedString", self.base_domain)
@@ -736,8 +734,6 @@ def testPutCompound(self):
         self.assertTrue("value" in rspJson)
         self.assertEqual(rspJson["value"], [42, 0.42])
 
-
-
     def testPutObjReference(self):
         print("testPutObjReference", self.base_domain)
         headers = helper.getRequestHeaders(domain=self.base_domain)
@@ -843,7 +839,6 @@ def testPutVlenObjReference(self):
         g1_3_id = rspJson["id"]
         self.assertTrue(helper.validateId(g1_3_id))
 
-
         # create attr of g1 that is a vlen list of obj ref's
         ref_type = {"class": "H5T_REFERENCE",
                     "base": "H5T_STD_REF_OBJ"}
@@ -1204,6 +1199,106 @@ def testPutAttributeBinaryValue(self):
         self.assertFalse("shape" in rspJson)
         self.assertEqual(rspJson["value"], value)
 
+    def testPutIntegerArray(self):
+        # Test PUT value for 1d attribute with list of integers
+        print("testPutIntegerArray", self.base_domain)
+
+        headers = helper.getRequestHeaders(domain=self.base_domain)
+        req = self.endpoint + '/'
+
+        # Get root uuid
+        rsp = requests.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        root_uuid = rspJson["root"]
+        helper.validateId(root_uuid)
+
+        # create attr
+        value = [2,3,5,7,11,13]
+        data = { "type": 'H5T_STD_I32LE', "shape": 6, "value": value}
+        attr_name = "int_arr_attr"
+        req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name
+        rsp = requests.put(req, data=json.dumps(data), headers=headers)
+        self.assertEqual(rsp.status_code, 201)
+
+        # read attr
+        rsp = requests.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        self.assertTrue("hrefs" in rspJson)
+        self.assertTrue("value" in rspJson)
+        self.assertEqual(rspJson["value"], value)
+        self.assertTrue("type" in rspJson)
+        type_json = rspJson["type"]
+        self.assertTrue("class" in type_json)
+        self.assertEqual(type_json["base"], "H5T_STD_I32LE")
+        self.assertTrue("shape" in rspJson)
+        shape_json = rspJson["shape"]
+        self.assertTrue("class" in shape_json)
+        self.assertTrue(shape_json["class"], 'H5S_SIMPLE')
+        self.assertTrue("dims" in shape_json)
+        self.assertTrue(shape_json["dims"], [6])
+
+        # try creating an array where the shape doesn't match data values
+        data = { "type": 'H5T_STD_I32LE', "shape": 5, "value": value}
+        attr_name = "badarg_arr_attr"
+        req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name
+        rsp = requests.put(req, data=json.dumps(data), headers=headers)
+        self.assertEqual(rsp.status_code, 400)  # Bad request
+
+    def testNaNAttributeValue(self):
+        # Test GET Attribute value with JSON response that contains NaN data
+        print("testNaNAttributeValue", self.base_domain)
+
+        headers = helper.getRequestHeaders(domain=self.base_domain)
+        req = self.endpoint + '/'
+
+        # Get root uuid
+        rsp = requests.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        root_uuid = rspJson["root"]
+        helper.validateId(root_uuid)
+
+        # create attr
+        value = [np.NaN,] * 6
+        data = { "type": 'H5T_IEEE_F32LE', "shape": 6, "value": value}
+        attr_name = "nan_arr_attr"
+        req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name
+        rsp = requests.put(req, data=json.dumps(data), headers=headers)
+        self.assertEqual(rsp.status_code, 201)
+
+        # get all attributes, then by name, and then by value
+        for req_suffix in ("", f"/{attr_name}", f"/{attr_name}/value"):
+            for ignore_nan in (False, True):
+                req = self.endpoint + "/groups/" + root_uuid + "/attributes" + req_suffix
+                params = {}
+                if not req_suffix:
+                    # fetch data when getting all attribute
+                    params["IncludeData"] = 1
+                if ignore_nan:
+                    params["ignore_nan"] = 1
+                rsp = requests.get(req, headers=headers, params=params)
+                self.assertEqual(rsp.status_code, 200)
+                rspJson = json.loads(rsp.text)
+                self.assertTrue("hrefs" in rspJson)
+                if "attributes" in rspJson:
+                    # this is returned for the fetch all attribute req
+                    attrs = rspJson["attributes"]
+                    self.assertEqual(len(attrs), 1)
+                    self.assertTrue("value" in attrs[0])
+                    rspValue = attrs[0]["value"]
+                else:
+                    self.assertTrue("value" in rspJson)
+                    rspValue = rspJson["value"]
+                self.assertEqual(len(rspValue), 6)
+                for i in range(6):
+                    if ignore_nan:
+                        self.assertTrue(rspValue[i] is None)
+                    else:
+                        self.assertTrue(np.isnan(rspValue[i]))
+
+
 
 if __name__ == '__main__':
     #setup test files