Additional DAP4 fixes

This change-set modifies PR #2555
to add the changes listed below. Most of these changes are required
by changes to the Java remotetest.unidata.ucar.edu server.

## DAP4 Related Changes
* Add tests *dap4_test/test_constraints.sh* and *dap4_test/test_hyrax.sh*.
* Provide explicit list of remotetest files to test.
* Cleanup local checksum computing and verification.
* Define a temporary Hyrax hack flag to deal with the way Hyrax handles checksums and add "#hyrax" fragment flag for it.
* Add a hack to get past an LGTM problem with using "http:".
* Improve debug support.

## Other Changes
* Cleanup the recipe in *docs/nczarr.md* for building *aws-sdk-cpp* library.

CMakeLists.txt

@@ -1062,7 +1062,7 @@ ENDIF()
 IF(ENABLE_DAP)
   SET(USE_DAP ON CACHE BOOL "")
   SET(ENABLE_DAP2 ON CACHE BOOL "")
- 
+
   IF(ENABLE_HDF5)
     MESSAGE(STATUS "Enabling DAP4")
     SET(ENABLE_DAP4 ON CACHE BOOL "")

dap4_test/CMakeLists.txt

@@ -52,14 +52,17 @@ ENDIF()
 
   IF(ENABLE_DAP_REMOTE_TESTS)
       add_sh_test(dap4_test test_remote)
-#      add_sh_test(dap4_test test_hyrax)
+      add_sh_test(dap4_test test_hyrax)
 #      add_sh_test(dap4_test test_thredds)
   ENDIF(ENABLE_DAP_REMOTE_TESTS)
 
 ENDIF(ENABLE_TESTS)
 
-#FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 #FILE(COPY ./baseline DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./baselineremote DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./baselinethredds DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./baselinehyrax DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 #FILE(COPY ./cdltestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 #FILE(COPY ./rawtestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -69,6 +72,9 @@ ${CMAKE_CURRENT_SOURCE_DIR}/cdltestfiles
 ${CMAKE_CURRENT_SOURCE_DIR}/rawtestfiles
 ${CMAKE_CURRENT_SOURCE_DIR}/baseline
 ${CMAKE_CURRENT_SOURCE_DIR}/baselineraw
+${CMAKE_CURRENT_SOURCE_DIR}/baselineremote
+#${CMAKE_CURRENT_SOURCE_DIR}/baselinerthredds
+${CMAKE_CURRENT_SOURCE_DIR}/baselinehyrax
 )
 SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
 ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

dap4_test/Makefile.am

@@ -36,23 +36,26 @@ TESTS += test_parse.sh test_meta.sh test_data.sh test_raw.sh
 
 # Note: This program name was changed to findtestserver4
 # to avoid cmake complaint about duplicate targets.
-noinst_PROGRAMS += findtestserver4 pingurl4
+noinst_PROGRAMS += findtestserver4 pingurl4 dump
 findtestserver4_SOURCES = findtestserver4.c
 pingurl4_SOURCES = pingurl4.c
+dump_SOURCES = dump.c
 
 # Disable Dap4 Remote Tests until the test server is working
 if BUILD_UTILITIES
 if ENABLE_DAP_REMOTE_TESTS
 TESTS += test_remote.sh
-# Test require help at the server end
-#  TESTS += test_hyrax.sh test_thredds.sh
+TESTS += test_constraints.sh
+TESTS += test_hyrax.sh
+#TESTS += test_thredds.sh
 endif
 endif
 
 EXTRA_DIST = CMakeLists.txt test_common.h build.sh \
-	d4test_common.sh \
+	d4manifest.sh d4test_common.sh \
 	test_curlopt.sh test_data.sh test_hyrax.sh test_meta.sh \
-	test_parse.sh test_raw.sh test_remote.sh test_thredds.sh \
+	test_parse.sh test_raw.sh \
+        test_remote.sh test_constraints.sh test_thredds.sh \
 	cdltestfiles rawtestfiles \
 	baseline baselineraw baselineremote baselinehyrax baselinethredds
 
@@ -84,3 +87,4 @@ clean-local-check:
 # in the GenerateRaw.java program in the thredds tree.
 maketests::
 	bash ./maketests.sh
+

dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax

@@ -0,0 +1,14 @@
+netcdf amsre_20060131v5 {
+dimensions:
+	_Anonymous3 = 3 ;
+	_Anonymous6 = 6 ;
+variables:
+	byte time_a(_Anonymous3, _Anonymous6) ;
+		string time_a:Equator_Crossing_Time = "1:30 PM" ;
+data:
+
+ time_a =
+  -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1 ;
+}

dap4_test/baselinehyrax/nc4_nc_classic_comp.nc.hyrax

@@ -0,0 +1,104 @@
+netcdf nc4_nc_classic_comp {
+dimensions:
+	lat = 6 ;
+	lon = 5 ;
+	time = 2 ;
+variables:
+	int lat(lat) ;
+		string lat:units = "degrees_north" ;
+	int lon(lon) ;
+		string lon:units = "degrees_east" ;
+	int time(time) ;
+		string time:units = "seconds" ;
+	float z(time, lat, lon) ;
+		string z:units = "meters" ;
+		z:valid_range = 0., 5000. ;
+		float z:_FillValue = 1.f ;
+		string z:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	float t(time, lat, lon) ;
+		string t:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	double p(time, lat, lon) ;
+		double p:_FillValue = -9999. ;
+		string p:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	short rh(time, lat, lon) ;
+		short rh:_FillValue = -1s ;
+		string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	short pixel(lat, lon) ;
+		string pixel:_edu.ucar.maps = "/lat", "/lon" ;
+	string source(lat) ;
+		source:string_length = 5 ;
+		string source:_edu.ucar.maps = "/lat" ;
+data:
+
+ lat = 0, 10, 20, 30, 40, 50 ;
+
+ lon = -140, -118, -96, -84, -52 ;
+
+ time = 1, 2 ;
+
+ z =
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10 ;
+
+ t =
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1 ;
+
+ p =
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1 ;
+
+ rh =
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2 ;
+
+ pixel =
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7 ;
+
+ source = "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa" ;
+}

dap4_test/baselinehyrax/nc4_nc_classic_no_comp.nc.hyrax

@@ -0,0 +1,104 @@
+netcdf nc4_nc_classic_no_comp {
+dimensions:
+	lat = 6 ;
+	lon = 5 ;
+	time = 2 ;
+variables:
+	int lat(lat) ;
+		string lat:units = "degrees_north" ;
+	int lon(lon) ;
+		string lon:units = "degrees_east" ;
+	int time(time) ;
+		string time:units = "seconds" ;
+	float z(time, lat, lon) ;
+		string z:units = "meters" ;
+		z:valid_range = 0., 5000. ;
+		float z:_FillValue = 1.f ;
+		string z:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	float t(time, lat, lon) ;
+		string t:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	double p(time, lat, lon) ;
+		double p:_FillValue = -9999. ;
+		string p:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	short rh(time, lat, lon) ;
+		short rh:_FillValue = -1s ;
+		string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
+	short pixel(lat, lon) ;
+		string pixel:_edu.ucar.maps = "/lat", "/lon" ;
+	string source(lat) ;
+		source:string_length = 5 ;
+		string source:_edu.ucar.maps = "/lat" ;
+data:
+
+ lat = 0, 10, 20, 30, 40, 50 ;
+
+ lon = -140, -118, -96, -84, -52 ;
+
+ time = 1, 2 ;
+
+ z =
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10,
+  10, 10, 10, 10, 10 ;
+
+ t =
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1 ;
+
+ p =
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1,
+  0.1, 0.1, 0.1, 0.1, 0.1 ;
+
+ rh =
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2 ;
+
+ pixel =
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7,
+  7, 7, 7, 7, 7 ;
+
+ source = "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa" ;
+}

dap4_test/baselinehyrax/nc4_strings.nc.hyrax

@@ -0,0 +1,32 @@
+netcdf nc4_strings {
+dimensions:
+	lat = 6 ;
+	lon = 5 ;
+variables:
+	int lat(lat) ;
+	int lon(lon) ;
+	string station(lat, lon) ;
+		string station:who = "james" ;
+		string station:names = "site_1", "site_2", "site_3" ;
+		string station:_edu.ucar.maps = "/lat", "/lon" ;
+	string scan_line(lon) ;
+		string scan_line:_edu.ucar.maps = "/lon" ;
+	string codec_name ;
+data:
+
+ lat = 0, 10, 20, 30, 40, 50 ;
+
+ lon = -140, -118, -96, -84, -52 ;
+
+ station =
+  "one", "two", "three", "four", "five",
+  "one_b", "two_b", "three_b", "four_b", "five_b",
+  "one_c", "two_c", "three_c", "four_c", "five_c",
+  "one", "two", "three", "four", "five",
+  "one", "two", "three", "four", "five",
+  "one_f", "two_f", "three_f", "four_f", "five_f" ;
+
+ scan_line = "r", "r1", "r2", "r3", "r4" ;
+
+ codec_name = "mp3" ;
+}

dap4_test/baselinehyrax/nc4_strings_comp.nc.hyrax

@@ -0,0 +1,32 @@
+netcdf nc4_strings_comp {
+dimensions:
+	lat = 6 ;
+	lon = 5 ;
+variables:
+	int lat(lat) ;
+	int lon(lon) ;
+	string station(lat, lon) ;
+		string station:who = "james" ;
+		string station:names = "site_1", "site_2", "site_3" ;
+		string station:_edu.ucar.maps = "/lat", "/lon" ;
+	string scan_line(lon) ;
+		string scan_line:_edu.ucar.maps = "/lon" ;
+	string codec_name ;
+data:
+
+ lat = 0, 10, 20, 30, 40, 50 ;
+
+ lon = -140, -118, -96, -84, -52 ;
+
+ station =
+  "one", "two", "three", "four", "five",
+  "one_b", "two_b", "three_b", "four_b", "five_b",
+  "one_c", "two_c", "three_c", "four_c", "five_c",
+  "one", "two", "three", "four", "five",
+  "one", "two", "three", "four", "five",
+  "one_f", "two_f", "three_f", "four_f", "five_f" ;
+
+ scan_line = "r", "r1", "r2", "r3", "r4" ;
+
+ codec_name = "mp3" ;
+}