De-duplicate configure options

Signed-off-by: Joshua Hursey <jhursey@us.ibm.com>
open-mpi · Mar 4, 2021 · 0c50fb3 · 0c50fb3
1 parent 5f17e5e
commit 0c50fb3
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 49 deletions.
diff --git a/.gitignore b/.gitignore
@@ -116,6 +116,7 @@ config/ext_no_configure_components.m4
 config/ext_m4_config_include.m4
 config/auto-extracted-pmix-configure-args.m4
 config/auto-extracted-prrte-configure-args.m4
+config/auto-generated-ompi-exclude.ini
 
 contrib/build-mca-comps-outside-of-tree/btl_tcp2_config.h
 contrib/build-mca-comps-outside-of-tree/btl_tcp2_config.h.in

diff --git a/autogen.pl b/autogen.pl
@@ -1562,6 +1562,11 @@ sub replace_config_sub_guess {
 
 dnl 3rd-party package information\n";
 
+# Extract the OMPI options to exclude them when processing PMIx and PRRTE
+if ( ! ("pmix" ~~ @disabled_3rdparty_packages && "prrte" ~~ @disabled_3rdparty_packages) ) {
+    safe_system("./config/extract-3rd-party-configure.pl -p . -n \"OMPI\" -l > config/auto-generated-ompi-exclude.ini");
+}
+
 # these are fairly one-off, so we did not try to do anything
 # generic. Sorry :).
 
@@ -1607,7 +1612,9 @@ sub replace_config_sub_guess {
     $m4 .= "m4_define([package_pmix], [1])\n";
 
     # Grab the unique configure options from each of the 3rd party packages
-    safe_system("./config/extract-3rd-party-configure.pl -p \"3rd-party/openpmix/\" -n \"PMIx\" > config/auto-extracted-pmix-configure-args.m4");
+    safe_system("./config/extract-3rd-party-configure.pl -p \"3rd-party/openpmix/\" -n \"PMIx\" -e config/auto-generated-ompi-exclude.ini > config/auto-extracted-pmix-configure-args.m4");
+    # Add the additional configure options from PMIx
+    safe_system("./config/extract-3rd-party-configure.pl -p \"3rd-party/openpmix/\" -n \"PMIx\" -l >> config/auto-generated-ompi-exclude.ini");
 
     verbose "--- PMIx enabled\n";
 }
@@ -1624,7 +1631,7 @@ sub replace_config_sub_guess {
     $m4 .= "m4_define([package_prrte], [1])\n";
 
     # Grab the unique configure options from each of the 3rd party packages
-    safe_system("./config/extract-3rd-party-configure.pl -p \"3rd-party/prrte/\" -n \"PRRTE\" > config/auto-extracted-prrte-configure-args.m4");
+    safe_system("./config/extract-3rd-party-configure.pl -p \"3rd-party/prrte/\" -n \"PRRTE\" -e config/auto-generated-ompi-exclude.ini > config/auto-extracted-prrte-configure-args.m4");
 
     verbose "--- PRRTE enabled\n";
 }

diff --git a/config/extract-3rd-party-configure.pl b/config/extract-3rd-party-configure.pl
@@ -24,8 +24,10 @@
 my $quiet_arg = 0;
 my $debug_arg = 0;
 my $help_arg = 0;
+my $list_only = 0;
 my $package_dir;
 my $only_filename; # for debugging
+my $ompi_exclude;
 my $project_name;
 
 my $package_configure;
@@ -34,24 +36,30 @@
 my %skip_files = ();
 my %skip_options = ();
 
+my @exclude_config_files = ("3rd-party/exclude-config.ini");
+
 ##############################################################################
 # Command line parameters
 my $OPTION_ALL_MARKER = "--";
 
 my $ok = Getopt::Long::GetOptions("quiet|q" => \$quiet_arg,
                                   "debug|d" => \$debug_arg,
                                   "help|h" => \$help_arg,
+                                  "list|l" => \$list_only,
                                   "package|p=s" => \$package_dir,
                                   "project|n=s" => \$project_name,
                                   "file|f=s" => \$only_filename,
+                                  "exclude|e=s" => \$ompi_exclude,
     );
 
 if (!$ok || $help_arg) {
     print "Invalid command line argument.\n\n"
         if (!$ok);
     print "Options:
   --project | -n                Project name
+  --exclude | -e                Additional exclude file
   --package | -p                Full path to the 3rd party library
+  --list | -l                   List options without m4 syntax in exclude file syntax
   --quiet | -q                  Do not display normal verbose output
   --debug | -d                  Output lots of debug information
   --help | -h                   This help list
@@ -83,6 +91,10 @@
 my $project_name_uc = uc($project_name);
 my $project_name_lc = lc($project_name);
 
+if (defined $ompi_exclude) {
+    push(@exclude_config_files, $ompi_exclude);
+}
+
 #---------------------------------------------------------------------------
 sub process_m4($)
 {
@@ -91,8 +103,7 @@ ($)
     my $char;
     my $kw;
     my $option_name = "";
-    my $mod_option_name = "";
-    my $skip_string = 0;
+    my $option_name_help_str = "";
 
     # Read all config files looking for the keywords
     # Note that we only want the first two arguments
@@ -104,10 +115,12 @@ ($)
     my @the_stack = ();
     # push(@the_stack) / pop(@the_stack) / scalar(@the_stack)
     my $arg_num = 0;
+    my $line_num = 0;
 
     open(FH, '<', $filename) or die $!;
     while($line = <FH>) {
         chomp($line);
+        $line_num += 1;
         my $process_this_line = 0;
 
         # If the stack is empty, then we are looking for a keyword
@@ -116,29 +129,36 @@ ($)
                 # If this is starting a new configure option, then grab it's option name
                 if( $line =~ /$kw\(\s*\[?\s*([-\w\d\$]*)\s*\]?/ ) {
                     $option_name = $1;
-                    $mod_option_name = $project_name_lc . "-" . $option_name;
 
                     # Skip anything excluded
                     if( exists($skip_options{$option_name}) && 
                         ($skip_options{$option_name} eq $OPTION_ALL_MARKER || lc($skip_options{$option_name}) eq $project_name_lc) ) {
-                        print "# Warning: Excluded: $line\n";
-                        print "\n# " . "-"x40 . " Above from $filename\n\n";
+                        if( !$list_only ) {
+                            print "# Warning: Excluded: $option_name\n";
+                            print "# " . "-"x40 . " Above from $filename:$line_num\n\n";
+                        }
                         $process_this_line = 0;
                         last;
                     }
 
                     # Skip any option that contains a variable substition
                     if ( $option_name =~ /\$/ ) {
-                        print "# Warning: Skipped (embedded variable): $line\n";
-                        print "\n# " . "-"x40 . " Above from $filename\n\n";
+                        if( !$list_only ) {
+                            print "# Warning: Skipped (embedded variable): $option_name\n";
+                            print "# " . "-"x40 . " Above from $filename:$line_num\n\n";
+                        }
                         $process_this_line = 0;
                         last;
                     }
 
                     $process_this_line = 1;
 
+                    if( $list_only ) {
+                        print "# $project_name : $kw : From $filename:$line_num\n";
+                        print "OPTION: $option_name\n";
+                    }
+
                     $arg_num = 0;
-                    last;
                 }
             }
         }
@@ -147,6 +167,11 @@ ($)
             $process_this_line = 1;
         }
 
+        # If just listing then there is no need to further process the _ARG_
+        if( $list_only ) {
+            $process_this_line = 0;
+        }
+
         # Process the line character-by-character to extract only the
         # first two arguments. Note that the '(' ')' mark the boundary
         # to the function. The '[' ']' are optional bounds on the argument
@@ -217,7 +242,7 @@ ($)
             # which file we extracted it from, just in case we need
             # to track it down later.
             if( scalar(@the_stack) <= 0 ) {
-                print "\n# " . "-"x40 . " Above from $filename\n\n";
+                print "# " . "-"x40 . " Above from $filename:$line_num\n\n";
             }
         }
     }
@@ -235,50 +260,55 @@ ($)
 #---------------------------------------------------------------------------
 # Look for excluded options
 
-if (-f "3rd-party/exclude-config.ini") {
-    my $line;
+foreach $ompi_exclude (@exclude_config_files) {
+    if (-f "$ompi_exclude") {
+        my $line;
 
-    open(FH, '<', "3rd-party/exclude-config.ini") or die $!;
-    while($line = <FH>) {
-        if( $line =~ /^\s*#/ || $line =~ /^\s*$/) {
-            next;
+        if( $debug_arg ) {
+            print "Processing Exclude File: $ompi_exclude\n";
         }
-        chomp($line);
+        open(FH, '<', "$ompi_exclude") or die $!;
+        while($line = <FH>) {
+            if( $line =~ /^\s*#/ || $line =~ /^\s*$/) {
+                next;
+            }
+            chomp($line);
 
-        # Files to exclude
-        if( $line =~ /^\s*FILE\s*:\s*/ ) {
-            $line =~ s/^\s*FILE\s*:\s*//;
-            $line =~ s/^\s*|\s*$//g;
-            $skip_files{$line} = 1;
-        }
-        # Options to exclude regardless of project
-        elsif( $line =~ /^\s*OPTION\s*:\s*/ ) {
-            $line =~ s/^\s*OPTION\s*:\s*//;
-            $line =~ s/^\s*|\s*$//g;
-            $skip_options{$line} = $OPTION_ALL_MARKER;
-        }
-        # Options to exclude for a specific project
-        elsif( $line =~ /^\s*OPTION\s*\((\w+)\):\s*/ ) {
-            my $proj = $1;
-            $line =~ s/^\s*OPTION\s*\(\w+\):\s*//;
-            $line =~ s/^\s*|\s*$//g;
-            $skip_options{$line} = $proj;
+            # Files to exclude
+            if( $line =~ /^\s*FILE\s*:\s*/ ) {
+                $line =~ s/^\s*FILE\s*:\s*//;
+                $line =~ s/^\s*|\s*$//g;
+                $skip_files{$line} = 1;
+            }
+            # Options to exclude regardless of project
+            elsif( $line =~ /^\s*OPTION\s*:\s*/ ) {
+                $line =~ s/^\s*OPTION\s*:\s*//;
+                $line =~ s/^\s*|\s*$//g;
+                $skip_options{$line} = $OPTION_ALL_MARKER;
+            }
+            # Options to exclude for a specific project
+            elsif( $line =~ /^\s*OPTION\s*\((\w+)\):\s*/ ) {
+                my $proj = $1;
+                $line =~ s/^\s*OPTION\s*\(\w+\):\s*//;
+                $line =~ s/^\s*|\s*$//g;
+                $skip_options{$line} = $proj;
+            }
         }
-    }
-    close(FH);
+        close(FH);
 
-    if( $debug_arg ) {
-        print "---------------- Excluded files\n";
-        foreach my $key (keys(%skip_files)) {
-            print "FILE: $key\n";
-        }
+        if( $debug_arg ) {
+            print "---------------- Excluded files\n";
+            foreach my $key (keys(%skip_files)) {
+                print "FILE: $key\n";
+            }
 
-        print "---------------- Excluded options\n";
-        foreach my $key (keys(%skip_options)) {
-            print "OPTION (".$skip_options{$key}."): $key\n";
-        }
+            print "---------------- Excluded options\n";
+            foreach my $key (keys(%skip_options)) {
+                print "OPTION (".$skip_options{$key}."): $key\n";
+            }
 
-        print "----------------\n\n";
+            print "----------------\n\n";
+        }
     }
 }
 
@@ -321,6 +351,14 @@ ($)
 #---------------------------------------------------------------------------
 # Actually process the files and generate the result to stdout
 
+if( $list_only ) {
+print <<EOL;
+# -----
+# This file is auto-generated during autogen.pl
+# -----
+EOL
+;
+} else {
 print <<EOL;
 # -----
 # This file is auto-generated during autogen.pl
@@ -330,15 +368,25 @@ ($)
 
 EOL
 ;
+}
 print "\n# " . "-"x40 . "\n\n";
 
 foreach $filename (@all_files) {
-    process_m4($filename);
+    if( $filename =~ /auto-extracted-\w+-configure-args.m4$/ ) {
+        if( $debug_arg ) {
+            print "SKIP: $filename\n";
+        }
+    }
+    else {
+        process_m4($filename);
+    }
 }
 
+if( !$list_only ) {
 print <<EOL;
 
 ])
 EOL
+}
 #---------------------------------------------------------------------------
 exit(0);