Merge branch 'master' into daos-16639

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.7.100-6) unstable; urgency=medium
+  [ Kris Jacque ]
+  * Bump minimum golang-go version to 1.21
+
+ -- Kris Jacque <kris.jacque@intel.com>  Mon, 23 Sep 2024 11:06:00 -0700
+
 daos (2.7.100-5) unstable; urgency=medium
   [ Michael MacDonald ]
   * Add libdaos_self_test.so to client package

debian/control

@@ -25,7 +25,7 @@ Build-Depends: debhelper (>= 10),
                dpdk-dev (>= 21.11.2),
                libisal-crypto-dev,
                libcunit1-dev,
-               golang-go (>= 1.18),
+               golang-go (>= 2:1.21),
                libboost-dev,
                libspdk-dev (>= 22.01.2),
                libipmctl-dev,
@@ -117,7 +117,7 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml,
          ${shlibs:Depends}, ${misc:Depends},
          daos-client (= ${binary:Version}),
          daos-admin (= ${binary:Version}),
-         golang-go (>=1.18),
+         golang-go (>= 2:1.21),
          libcapstone-dev
 Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source
  software-defined object store designed from the ground up for

site_scons/site_tools/go_builder.py

@@ -5,10 +5,9 @@
 import re
 import subprocess  # nosec B404
 
-from SCons.Script import Configure, Exit, File, GetOption, Glob, Scanner
+from SCons.Script import Configure, Dir, Exit, File, GetOption, Glob, Scanner
 
 GO_COMPILER = 'go'
-MIN_GO_VERSION = '1.18.0'
 include_re = re.compile(r'\#include [<"](\S+[>"])', re.M)
 
 
@@ -49,6 +48,17 @@ def _scan_go_file(node, env, _path):
     return includes
 
 
+def get_min_go_version():
+    """Get go minimum version from go.mod"""
+    go_mod_path = os.path.join(Dir('#').abspath, "src", "control", "go.mod")
+    with open(go_mod_path, 'r') as f:
+        for line in f:
+            if line.startswith('go '):  # e.g. "go 1.21"
+                parts = line.split()
+                return get_go_version("go" + parts[1])
+    return None
+
+
 def get_go_version(output):
     """Capture only the version after 'go'"""
     ver_re = re.compile(r'go([0-9\.]+)')
@@ -81,6 +91,13 @@ def _check_go_version(context):
             context.Result(0)
             return 0
 
+        context.Display('Getting minimum go version... ')
+        min_go_version = get_min_go_version()
+        if min_go_version is None:
+            context.Result('no minimum go version found in go.mod')
+            return 0
+        context.Display(min_go_version + '\n')
+
         context.Display(f'Checking {env.d_go_bin} version... ')
         cmd_rc = subprocess.run([env.d_go_bin, 'version'], check=True, stdout=subprocess.PIPE)
         out = cmd_rc.stdout.decode('utf-8').strip()
@@ -93,11 +110,11 @@ def _check_go_version(context):
         if go_version is None:
             context.Result(f'failed to get version from "{out}"')
             return 0
-        if len([x for x, y in
-                zip(go_version.split('.'), MIN_GO_VERSION.split('.'))
+        if len([x for x, y in zip(go_version.split('.'), min_go_version.split('.'))
                 if int(x) < int(y)]) > 0:
-            context.Result(f'{out} is too old (min supported: {MIN_GO_VERSION}) ')
+            context.Result(f'{out} is too old (min supported: {min_go_version}) ')
             return 0
+
         context.Result(go_version)
         return 1
 

src/client/dfuse/il/int_posix.c

@@ -359,8 +359,10 @@ ioil_init(void)
 static void
 ioil_show_summary()
 {
-	D_INFO("Performed %"PRIu64" reads and %"PRIu64" writes from %"PRIu64" files\n",
-	       ioil_iog.iog_read_count, ioil_iog.iog_write_count, ioil_iog.iog_file_count);
+	D_INFO("Performed %" PRIu64 " reads, %" PRIu64 " writes and %" PRIu64
+	       " fstats from %" PRIu64 " files\n",
+	       ioil_iog.iog_read_count, ioil_iog.iog_write_count, ioil_iog.iog_fstat_count,
+	       ioil_iog.iog_file_count);
 
 	if (ioil_iog.iog_file_count == 0 || !ioil_iog.iog_show_summary)
 		return;
@@ -832,6 +834,7 @@ child_hdlr(void)
 		else
 			ioil_iog.iog_main_eqh = ioil_eqh;
 	}
+	ioil_iog.iog_eq_count = 0;
 }
 
 /* Returns true on success */

src/client/dfuse/pil4dfs/int_dfs.c

@@ -961,6 +961,7 @@ child_hdlr(void)
 		DL_WARN(rc, "daos_reinit() failed in child process");
 	td_eqh = main_eqh = DAOS_HDL_INVAL;
 	context_reset = true;
+	d_eq_count    = 0;
 }
 
 /* only free the reserved low fds when application exits or encounters error */

src/control/cmd/daos/pool.go

@@ -224,9 +224,8 @@ type poolCmd struct {
 
 type poolQueryCmd struct {
 	poolBaseCmd
-	ShowEnabledRanks  bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
-	ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"`
-	HealthOnly        bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
+	ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
+	HealthOnly       bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
 }
 
 func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats {
@@ -340,15 +339,10 @@ func (cmd *poolQueryCmd) Execute(_ []string) error {
 	if cmd.HealthOnly {
 		queryMask = daos.HealthOnlyPoolQueryMask
 	}
-	if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks {
-		return errors.New("show-enabled and show-disabled can't be used at the same time.")
-	}
 	if cmd.ShowEnabledRanks {
 		queryMask.SetOptions(daos.PoolQueryOptionEnabledEngines)
 	}
-	if cmd.ShowDisabledRanks {
-		queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
-	}
+	queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
 
 	cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil)
 	if err != nil {

src/control/cmd/daos/pretty/pool.go

@@ -50,7 +50,7 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error {
 	if pi.EnabledRanks != nil && pi.EnabledRanks.Count() > 0 {
 		fmt.Fprintf(w, "- Enabled ranks: %s\n", pi.EnabledRanks)
 	}
-	if pi.DisabledRanks != nil && pi.DisabledRanks.Count() > 0 {
+	if pi.DisabledRanks.Count() > 0 {
 		fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks)
 	}
 	if pi.Rebuild != nil {

src/control/cmd/dmg/pool.go

@@ -590,9 +590,8 @@ func (cmd *PoolReintegrateCmd) Execute(args []string) error {
 // PoolQueryCmd is the struct representing the command to query a DAOS pool.
 type PoolQueryCmd struct {
 	poolCmd
-	ShowEnabledRanks  bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
-	ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"`
-	HealthOnly        bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
+	ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
+	HealthOnly       bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
 }
 
 // Execute is run when PoolQueryCmd subcommand is activated
@@ -608,9 +607,7 @@ func (cmd *PoolQueryCmd) Execute(args []string) error {
 	if cmd.ShowEnabledRanks {
 		req.QueryMask.SetOptions(daos.PoolQueryOptionEnabledEngines)
 	}
-	if cmd.ShowDisabledRanks {
-		req.QueryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
-	}
+	req.QueryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
 
 	resp, err := control.PoolQuery(cmd.MustLogCtx(), cmd.ctlInvoker, req)
 	if cmd.JSONOutputEnabled() {

src/control/cmd/dmg/pool_test.go

@@ -994,54 +994,6 @@ func TestPoolCommands(t *testing.T) {
 			}, " "),
 			nil,
 		},
-		{
-			"Query pool with UUID and disabled ranks",
-			"pool query --show-disabled 12345678-1234-1234-1234-1234567890ab",
-			strings.Join([]string{
-				printRequest(t, &control.PoolQueryReq{
-					ID:        "12345678-1234-1234-1234-1234567890ab",
-					QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }),
-				}),
-			}, " "),
-			nil,
-		},
-		{
-			"Query pool with UUID and disabled ranks",
-			"pool query -b 12345678-1234-1234-1234-1234567890ab",
-			strings.Join([]string{
-				printRequest(t, &control.PoolQueryReq{
-					ID:        "12345678-1234-1234-1234-1234567890ab",
-					QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }),
-				}),
-			}, " "),
-			nil,
-		},
-		{
-			"Query pool with UUID, enabled ranks and disabled ranks",
-			"pool query --show-disabled --show-enabled 12345678-1234-1234-1234-1234567890ab",
-			strings.Join([]string{
-				printRequest(t, &control.PoolQueryReq{
-					ID: "12345678-1234-1234-1234-1234567890ab",
-					QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
-						qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines)
-					}),
-				}),
-			}, " "),
-			nil,
-		},
-		{
-			"Query pool with UUID, enabled ranks and disabled ranks",
-			"pool query -b -e 12345678-1234-1234-1234-1234567890ab",
-			strings.Join([]string{
-				printRequest(t, &control.PoolQueryReq{
-					ID: "12345678-1234-1234-1234-1234567890ab",
-					QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
-						qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines)
-					}),
-				}),
-			}, " "),
-			nil,
-		},
 		{
 			"Query pool for health only",
 			"pool query --health-only 12345678-1234-1234-1234-1234567890ab",

src/control/go.mod

@@ -1,9 +1,11 @@
 module github.com/daos-stack/daos/src/control
 
+// NB: When updating minimum Go build version, don't forget to update:
+// - rpm packaging version checks: utils/rpms/daos.spec
+// - debian packaging version checks: debian/control
+// Scons uses this file to extract the minimum version.
 go 1.21
 
-toolchain go1.22.3
-
 require (
 	github.com/Jille/raft-grpc-transport v1.2.0
 	github.com/desertbit/grumble v1.1.3

src/control/lib/control/pool_test.go

@@ -840,7 +840,7 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) {
 					UpgradeLayoutVer: 8,
 				},
 			},
-			exp: `{"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`,
+			exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`,
 		},
 		"valid rankset": {
 			pqr: &PoolQueryResp{
@@ -862,7 +862,7 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) {
 					UpgradeLayoutVer: 8,
 				},
 			},
-			exp: `{"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`,
+			exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`,
 		},
 	} {
 		t.Run(name, func(t *testing.T) {

src/control/lib/daos/pool.go

@@ -104,7 +104,7 @@ type (
 
 const (
 	// DefaultPoolQueryMask defines the default pool query mask.
-	DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_DISABLED))
+	DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ C.DPI_ENGINES_ENABLED)
 	// HealthOnlyPoolQueryMask defines the mask for health-only queries.
 	HealthOnlyPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_SPACE))
 

src/control/lib/daos/pool_test.go

@@ -130,7 +130,7 @@ func TestDaos_PoolQueryMask(t *testing.T) {
 			testMask: genTestMask(func(pqm *PoolQueryMask) {
 				*pqm = DefaultPoolQueryMask
 			}),
-			expString: genOptsStr(PoolQueryOptionRebuild, PoolQueryOptionSpace),
+			expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace),
 		},
 		"health-only query mask": {
 			testMask: genTestMask(func(pqm *PoolQueryMask) {

src/tests/ftest/control/dmg_pool_query_ranks.py

@@ -40,28 +40,6 @@ def test_pool_query_ranks_basic(self):
 
         self.log.debug("Checking without ranks state information")
         data = self.dmg.pool_query(self.pool.identifier)
-        self.assertIsNone(
-            data['response'].get('enabled_ranks'),
-            "Invalid enabled_ranks field: want=None, got={}".format(
-                data['response'].get('enabled_ranks')))
-        self.assertIsNone(
-            data['response'].get('disabled_ranks'),
-            "Invalid disabled_ranks field: want=None, got={}".format(
-                data['response'].get('disabled_ranks')))
-
-        self.log.debug("Checking enabled ranks state information")
-        data = self.dmg.pool_query(self.pool.identifier, show_enabled=True)
-        self.assertListEqual(
-            data['response'].get('enabled_ranks'), [0, 1, 2],
-            "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format(
-                data['response'].get('enabled_ranks')))
-        self.assertIsNone(
-            data['response'].get('disabled_ranks'),
-            "Invalid disabled_ranks field: want=None, got={}".format(
-                data['response'].get('disabled_ranks')))
-
-        self.log.debug("Checking disabled ranks state information")
-        data = self.dmg.pool_query(self.pool.identifier, show_disabled=True)
         self.assertIsNone(
             data['response'].get('enabled_ranks'),
             "Invalid enabled_ranks field: want=None, got={}".format(
@@ -71,8 +49,8 @@ def test_pool_query_ranks_basic(self):
             "Invalid disabled_ranks field: want=[], got={}".format(
                 data['response'].get('disabled_ranks')))
 
-        self.log.debug("Checking enabled and disabled ranks state information")
-        data = self.dmg.pool_query(self.pool.identifier, show_enabled=True, show_disabled=True)
+        self.log.debug("Checking enabled ranks state information")
+        data = self.dmg.pool_query(self.pool.identifier, show_enabled=True)
         self.assertListEqual(
             data['response'].get('enabled_ranks'), [0, 1, 2],
             "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format(
@@ -110,8 +88,7 @@ def test_pool_query_ranks_mgmt(self):
             disabled_ranks = sorted(disabled_ranks + [rank])
 
             self.log.debug("Checking enabled ranks state information")
-            data = self.dmg.pool_query(
-                self.pool.identifier, show_enabled=True, show_disabled=True)
+            data = self.dmg.pool_query(self.pool.identifier, show_enabled=True)
             self.assertListEqual(
                 data['response'].get('enabled_ranks'), enabled_ranks,
                 "Invalid enabled_ranks field: want={}, got={}".format(
@@ -145,7 +122,7 @@ def test_pool_query_ranks_mgmt(self):
             disabled_ranks.remove(rank)
 
             self.log.debug("Checking enabled ranks state information")
-            data = self.dmg.pool_query(self.pool.identifier, show_enabled=True, show_disabled=True)
+            data = self.dmg.pool_query(self.pool.identifier, show_enabled=True)
             self.assertListEqual(
                 data['response'].get('enabled_ranks'), enabled_ranks,
                 "Invalid enabled_ranks field: want={}, got={}".format(

src/tests/ftest/control/dmg_pool_query_test.py

@@ -62,6 +62,7 @@ def test_pool_query_basic(self):
             "uuid": self.pool.uuid.lower(),
             "total_targets": self.params.get("total_targets", path="/run/exp_vals/*"),
             "active_targets": self.params.get("active_targets", path="/run/exp_vals/*"),
+            "disabled_ranks": self.params.get("disabled_ranks", path="/run/exp_vals/*"),
             "total_engines": self.params.get("total_engines", path="/run/exp_vals/*"),
             "disabled_targets": self.params.get("disabled_targets", path="/run/exp_vals/*"),
             "version": self.params.get("version", path="/run/exp_vals/*"),

src/tests/ftest/control/dmg_pool_query_test.yaml

@@ -27,12 +27,13 @@ exp_vals:
   pool_state: "Ready"
   total_targets: 4
   active_targets: 4
+  disabled_ranks: []
   total_engines: 1
   disabled_targets: 0
   version: 1
   leader: 0
   replicas: [0]
-  query_mask: "rebuild,space"
+  query_mask: "disabled_engines,rebuild,space"
   scm:
     total: 16000008192
   nvme:

src/tests/ftest/deployment/disk_failure.py

@@ -113,7 +113,6 @@ def test_disk_failure_w_rf(self):
         Test disk failures during the IO operation.
 
         :avocado: tags=all,manual
-        :avocado: tags=hw,medium
         :avocado: tags=deployment,disk_failure
         :avocado: tags=DiskFailureTest,test_disk_failure_w_rf
         """
@@ -125,7 +124,6 @@ def test_disk_fault_to_normal(self):
         Test a disk inducing faults and resetting is back to normal state.
 
         :avocado: tags=all,manual
-        :avocado: tags=hw,medium
         :avocado: tags=deployment,disk_failure
         :avocado: tags=DiskFailureTest,test_disk_fault_to_normal
         """

src/tests/ftest/deployment/network_failure.py

@@ -227,7 +227,7 @@ def verify_network_failure(self, ior_namespace, container_namespace):
 
         # 6. Call dmg pool query -b to find the disabled ranks.
         self.log_step("Find the disabled ranks.")
-        output = dmg_cmd.pool_query(pool=self.pool.identifier, show_disabled=True)
+        output = dmg_cmd.pool_query(pool=self.pool.identifier)
         disabled_ranks = output["response"].get("disabled_ranks")
         self.log.info("Disabled ranks = %s", disabled_ranks)
 

src/tests/ftest/deployment/server_rank_failure.py

@@ -183,7 +183,7 @@ def verify_rank_failure(self, ior_namespace):
                 errors.append("Server rank {} state isn't joined!".format(member["rank"]))
 
         # 9. Call dmg pool query -b to find the disabled ranks.
-        output = self.get_dmg_command().pool_query(pool=self.pool.identifier, show_disabled=True)
+        output = self.get_dmg_command().pool_query(pool=self.pool.identifier)
         disabled_ranks = output["response"].get("disabled_ranks")
         self.log.info("Disabled ranks = %s", disabled_ranks)