mulmat-tune-tool: add --n_pass; document; fix tailing spaces

examples/mulmat-tune/bench-out/13b.q4_0.txt

@@ -87,4 +87,3 @@
  128       10      163 0 0       94 0
  256       17      759 0 0      171 0
  512       39     2837 0 0      321 0
-

examples/mulmat-tune/bench-out/13b.q5_0.txt

@@ -87,4 +87,3 @@
  128       10      167 0 0       60 0
  256       20      733 0 0      129 0
  512       43     3462 0 0      262 0
-

examples/mulmat-tune/bench-out/7b.f16.txt

@@ -83,4 +83,3 @@
  128       10      167 0 0       79 0
  256       19      835 0 0      136 0
  512       39     2856 0 0      283 0
-

examples/mulmat-tune/bench-out/7b.f32.txt

@@ -83,4 +83,3 @@
  128        9      198 0 0       64 0
  256       20      766 0 0      166 0
  512       40     3464 0 0      276 0
-

examples/mulmat-tune/bench-out/7b.q4_0.openblas.txt

@@ -87,4 +87,3 @@
  128        9      201 0 0      128 0
  256       19      886 0 0      172 0
  512       39     3227 0 0      405 0
-

examples/mulmat-tune/bench-out/7b.q4_0.txt

@@ -87,4 +87,3 @@
  128       10      203 0 0      111 0
  256       19      705 0 0      165 0
  512       33     2832 0 0      313 0
-

examples/mulmat-tune/bench-out/7b.q4_1.txt

@@ -87,4 +87,3 @@
  128       10      210 0 0       70 0
  256       20      856 0 0      128 0
  512       40     2949 0 0      334 0
-

examples/mulmat-tune/bench-out/7b.q5_1.txt

@@ -87,4 +87,3 @@
  128       10      202 0 0       76 0
  256       20      850 0 0      123 0
  512       39     2944 0 0      301 0
-

examples/mulmat-tune/bench-out/7b.q8_0.txt

@@ -87,4 +87,3 @@
  128        9      206 0 0       79 0
  256       19      784 0 0      132 0
  512       38     2780 0 0      310 0
-

examples/mulmat-tune/mulmat-tune-tool.c

@@ -24,7 +24,7 @@ static void print_blas_build_tips(void);
 static void progress(int i, int max);
 static bool prompt_yes_no(const char *prompt);
 
-static void cmd_tune(struct ggml_mulmat_tune *b, bool verbose);
+static void cmd_tune(struct ggml_mulmat_tune *b, int n_pass, bool verbose);
 static void cmd_analyze(struct ggml_mulmat_tune *b);
 
 static void usage(char *prog) {
@@ -41,6 +41,9 @@ static void usage(char *prog) {
         "                   default 10\n",
         "--backend BACKEND  blas backend: CUDA | CL | CBLAS\n",
         "                   default: auto detect\n",
+        "--n_pass           number of passes to run\n",
+        "                   default 3\n",
+        "                   requires: in range [1, 5]\n",
         "--file    FILE     data file to write\n",
         "                   default stdout\n",
         "-y                 always answer \"yes\" to all prompts\n",
@@ -94,6 +97,7 @@ int main(int argc, char **argv) {
         const char *arg_model = NULL;
         const char *arg_type = NULL;
         const char *arg_m_num = NULL;
+        const char *arg_n_pass = NULL;
         const char *arg_backend = NULL;
         const char *arg_file = NULL;
         bool always_yes = false;
@@ -114,6 +118,11 @@ int main(int argc, char **argv) {
                     arg_m_num = argv[i + 1];
                     ++i;
                 }
+            } else if (strcmp(argv[i], "--n_pass") == 0) {
+                if (i + 1 < argc) {
+                    arg_n_pass = argv[i + 1];
+                    ++i;
+                }
             } else if (strcmp(argv[i], "--backend") == 0) {
                 if (i + 1 < argc) {
                     arg_backend = argv[i + 1];
@@ -203,6 +212,20 @@ int main(int argc, char **argv) {
             }
         }
 
+        int n_pass = 3;
+        {
+            if (arg_n_pass != NULL) {
+                int v = atoi(arg_n_pass);
+                n_pass = v;
+            }
+            if (n_pass < 1 || n_pass > MAX_NUM_PASS) {
+                fprintf(stderr, "invalid n_pass: %d, expect in range [1, 5]\n",
+                        n_pass);
+                usage(argv[0]);
+                exit(1);
+            }
+        }
+
         {
             enum ggml_backend backend = GGML_BACKEND_UNKNOWN;
             if (arg_backend == NULL) {
@@ -265,7 +288,7 @@ int main(int argc, char **argv) {
                tune.model, tune.type_name, ggml_get_backend_name(tune.backend),
                tune.blas_vendor);
 
-        cmd_tune(&tune, true /* verbose */);
+        cmd_tune(&tune, n_pass, true /* verbose */);
 
         FILE *fp = NULL;
         if (arg_file != NULL) {
@@ -334,7 +357,7 @@ int main(int argc, char **argv) {
     return 0;
 }
 
-void cmd_tune(struct ggml_mulmat_tune *tune, bool verbose) {
+void cmd_tune(struct ggml_mulmat_tune *tune, int n_pass, bool verbose) {
     size_t wsize = 0;
     void *q_buf = NULL;
     void *wdata = NULL;
@@ -477,21 +500,21 @@ void cmd_tune(struct ggml_mulmat_tune *tune, bool verbose) {
                     // without memset, the first run may be significant slow.
                     memset(wdata, 0, wsize);
 
-                    int stage_time[NUM_BENCH];
-                    for (int i_bench = 0; i_bench < NUM_BENCH; i_bench++) {
+                    int stage_time[MAX_NUM_PASS];
+                    for (int i_bench = 0; i_bench < n_pass; i_bench++) {
                         int t0 = (int)ggml_time_us();
 
                         ggml_internal_compute_forward_mul_mat(
                             profile, stage, wsize, wdata, src0, src1, dst);
 
                         stage_time[i_bench] = (int)ggml_time_us() - t0;
                         if (verbose) {
-                            progress(i_bench, NUM_BENCH);
+                            progress(i_bench, n_pass);
                         }
                     }
 
                     item->stages_time[stage] =
-                        tune_time_min(stage_time, NUM_BENCH);
+                        tune_time_min(stage_time, n_pass);
 
                     if (verbose) {
                         line_len++;

examples/mulmat-tune/mulmat-tune.h

@@ -8,7 +8,7 @@
 extern "C" {
 #endif
 
-#define NUM_BENCH 4
+#define MAX_NUM_PASS 5
 
 #define GGML_MULMAT_N_SHAPES 6
 #define GGML_MULMAT_MAX_PROFILES 8

ggml.c

@@ -14396,7 +14396,7 @@ void ggml_graph_compute_mul_mat_set_task_profile(struct ggml_cgraph *cgraph) {
                 if (shape != NULL) {
                     memset(profile_time, 0, sizeof(profile_time));
                     ggml_mulmat_tune_shape_estimate_time(shape, M, cgraph->n_threads, profile_time);
- 
+
                     int min = INT32_MAX;
                     for (int j = 0; j < shape->n_profiles; j++) {
                         int total = profile_time[j].total_time;
@@ -14435,7 +14435,7 @@ void ggml_graph_compute_mul_mat_set_task_profile(struct ggml_cgraph *cgraph) {
                     }
                 }
             }
-        
+
             if (profile == NULL) {
                 for (int j = 0; j < n_profiles; j++) {
                     if (profiles[j].stages[0].backend == GGML_BACKEND_CPU &&

tests/test-mulmat-tune.c

@@ -35,7 +35,7 @@ void test_ggml_mulmat_tune_estimate_time_non_zero_NK(void) {
     const int m_start = m_step;
     const int m_num = 2;
 
-    ggml_mulmat_tune_setup_model(&tune, "7B", m_start, m_step, m_num);
+    ggml_mulmat_tune_setup_model(&tune, "7B", m_num);
 
     struct ggml_mulmat_tune_shape *shape = NULL;
     for (int i = 0; i < tune.n_shapes; i++) {
@@ -149,7 +149,7 @@ void test_ggml_mulmat_tune_estimate_time_non_zero_NK(void) {
             ggml_mulmat_tune_get_shape(&tune, shape->N, shape->K,
                                        shape->src0_type, shape->src1_type);
         GGML_ASSERT(matched_shape);
-        
+
         ggml_mulmat_tune_shape_estimate_time(matched_shape, e->M, e->nth,
                                              profile_time);
 
@@ -193,7 +193,7 @@ void test_ggml_mulmat_tune_estimate_time_zero_NK(void) {
     const int m_start = m_step;
     const int m_num = 2;
 
-    ggml_mulmat_tune_setup_model(&tune, "7B", m_start, m_step, m_num);
+    ggml_mulmat_tune_setup_model(&tune, "7B", m_num);
 
     struct ggml_mulmat_tune_shape *shape = NULL;
     for (int i = 0; i < tune.n_shapes; i++) {