Merge branch 'v2'

evolarjun · Feb 18, 2020 · d230cf1 · d230cf1
2 parents d4abf02 + 8f73ab4
commit d230cf1
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 60 deletions.
diff --git a/.github/workflows/binary.yml b/.github/workflows/binary.yml
@@ -0,0 +1,29 @@
+name: binary tarball
+
+on: [release]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v1
+        - name: setup
+          run: sudo apt-get install -y hmmer ncbi-blast+ curl build-essential
+        - name: download
+          run: bash -x .github/workflows/get_binary_release.sh
+        - name: where are we
+          run: |
+              pwd
+              ls -al
+        - name: test protein
+          run: |
+              ./amrfinder --plus -p test_prot.fa -g test_prot.gff -O Escherichia > test_prot.got
+              diff test_prot.expected test_prot.got
+        - name: test dna
+          run: |
+              ./amrfinder --plus -n test_dna.fa -O Escherichia --mutation_all test_dna_mut_all.got > test_dna.got
+              diff test_dna.expected test_dna.got
+        - name: test combined
+          run: |
+              ./amrfinder --plus -n test_dna.fa -p test_prot.fa -g test_prot.gff -O Escherichia > test_both.got
+              diff test_both.expected test_both.got
diff --git a/.github/workflows/get_binary_release.sh b/.github/workflows/get_binary_release.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# found and lightly modified these functions
+get_latest_release() {
+    curl --silent "https://api.github.com/repos/$1/releases/latest" |
+    grep '"tag_name":' |
+    cut -d '"' -f 4
+}
+
+get_tarball_url() {
+    curl --silent "https://api.github.com/repos/$1/releases/latest" |
+        fgrep '"browser_download_url":' |
+        cut -d '"' -f 4
+}
+
+
+release=$(get_latest_release ncbi/amr)
+URL=$(get_tarball_url ncbi/amr)
+
+# download and unpack AMRFinder binaries
+    curl --silent -L -O $URL
+    tarball_name=$(echo $URL | perl -pe 's#^.*/(.*)#\1#')
+    tar xfz $tarball_name
+    rm $tarball_name
+
+# download and unpack test
+    curl --silent \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_dna.fa \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.fa \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.gff \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_both.expected \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_dna.expected \
+         -O https://raw.githubusercontent.com/ncbi/amr/master/test_prot.expected
+
+# download database
+    ./amrfinder --update
diff --git a/amrfinder.cpp b/amrfinder.cpp
@@ -33,7 +33,10 @@
 *               cat, cp, cut, grep, head, mkdir, mv, nproc, sort, tail, which
 *
 * Release changes:
-*   3.6.10 02/06/2020 PD-3357, issue#21  --mutation_all bug
+*   3.6.13 02/13/2020 PD-3359,issue#23   ln -s <db>: uses path2canonical()
+*   3.6.12 02/13/2020 PD-3359,issue#23   AMRFinder database directory may contain spaces
+*   3.6.11 02/13/2020 PD-3359,issue#23   AMRFinder code directory may contain spaces
+*   3.6.10 02/06/2020 PD-3357,issue#21  --mutation_all bug
 *          01/24/2020 PD-3345   Improved error message for "GFF file mismatch"
 *   3.6.9  01/13/2020           "Database directory" is printed to stederr
 *          01/10/2020 PD-3329   ln -s .../amrfinder abc: abc calls the right executables
@@ -199,18 +202,18 @@ struct ThisApplication : ShellApplication
 
   string file2link (const string &fName) const
   {
-    const string s (realpath (fName. c_str (), nullptr));
+    const string s (path2canonical (fName));
     if (s == fName)
       return string ();
     return s;
   }
 
 
 
-  StringVector db2organisms (const string &db) const
+  StringVector db2organisms () const
   {
-    exec ("tail -n +2 " + db + "/AMRProt-mutation.tab | cut -f 1 > " + tmp + ".prot_org");
-    exec ("tail -n +2 " + db + "/taxgroup.tab         | cut -f 1 > " + tmp + ".tax_org");
+    exec ("tail -n +2 " + tmp + ".db/AMRProt-mutation.tab" + " | cut -f 1 > " + tmp + ".prot_org");
+    exec ("tail -n +2 " + tmp + ".db/taxgroup.tab" + "         | cut -f 1 > " + tmp + ".tax_org");
     exec ("cat " + tmp + ".prot_org " + tmp + ".tax_org | sort -u > " + tmp + ".org");
     return StringVector (tmp + ".org", (size_t) 100);  // PAR
   }
@@ -244,8 +247,8 @@ struct ThisApplication : ShellApplication
 
 
     Stderr stderr (quiet);
-    stderr << "Running "<< getCommandLine () << '\n';
-    stderr << "Software directory: " << execDir << "\n";
+    stderr << "Running: "<< getCommandLine () << '\n';
+    stderr << "Software directory: " << shellQuote (execDir) << "\n";
 
     if (threads_max < threads_max_min)
       throw runtime_error ("Number of threads cannot be less than " + to_string (threads_max_min));
@@ -314,26 +317,27 @@ struct ThisApplication : ShellApplication
       if (! dbDir. items. empty () && dbDir. items. back () == "latest")
       {
         prog2dir ["amrfinder_update"] = execDir;
-  		  exec (fullProg ("amrfinder_update") + " -d " + dbDir. getParent () + ifS (quiet, " -q") + ifS (qc_on, " --debug") + " > " + logFName, logFName);
+  		  exec (fullProg ("amrfinder_update") + " -d " + shellQuote (dbDir. getParent ()) + ifS (quiet, " -q") + ifS (qc_on, " --debug") + " > " + logFName, logFName);
       }
       else
         cout << "WARNING: Updating database directory works only for databases with the default data directory format." << endl
              << "Please see https://github.com/ncbi/amr/wiki for details." << endl
-             << "Current database directory is: " << strQuote (dbDir. get ()) << endl
-             << "New database directories will be created as subdirectories of " << strQuote (dbDir. getParent ()) << endl;
+             << "Current database directory is: " << dbDir. get () << endl
+             << "New database directories will be created as subdirectories of " << dbDir. getParent () << endl;
 		}
 
 
     const string downloadLatestInstr ("\nTo download the latest version to the default directory run amrfinder -u");
 
 		if (! directoryExists (db))  // PD-2447
 		  throw runtime_error ("No valid AMRFinder database found." + ifS (! update, downloadLatestInstr));
-		stderr << "Database directory: " << db << "\n";
+		stderr << "Database directory: " << shellQuote (db) << "\n";		
+    exec ("ln -s " + shellQuote (path2canonical (db)) + " " + tmp + ".db");
 
 
     if (list_organisms)
     {
-      const StringVector organisms (db2organisms (db));
+      const StringVector organisms (db2organisms ());
       cout << "Possible organisms: " + organisms. toString (", ") << endl;
       return;
     }    		  
@@ -403,11 +407,11 @@ struct ThisApplication : ShellApplication
       if (! emptyArg (dna)  && ! getFileSize (unQuote (dna)))   emptyFiles << dna;
       if (! emptyArg (gff)  && ! getFileSize (unQuote (gff)))   emptyFiles << gff;      
 
-      stderr << "AMRFinder " << searchMode << " search with database " << db;
+      stderr << "AMRFinder " << searchMode << " search with database " << shellQuote (db);
       {
         const string link (file2link (db));
         if (! link. empty ())
-          stderr << ": " << link;
+          stderr << ": " << shellQuote (link);
       }
       stderr << "\n";
 
@@ -468,7 +472,7 @@ struct ThisApplication : ShellApplication
       }
       if (! organism1. empty ())
       {
-        const StringVector organisms (db2organisms (db));
+        const StringVector organisms (db2organisms ());
         if (! organisms. contains (organism1))
           throw runtime_error ("Possible organisms: " + organisms. toString (", "));
       }
@@ -561,14 +565,14 @@ struct ThisApplication : ShellApplication
     			string num_threads;
     			if (blastThreadable ("blastp") && prot_threads > 1)
     			  num_threads = " -num_threads " + to_string (prot_threads);
-    			th. exec (fullProg ("blastp") + " -query " + prot + " -db " + db + "/AMRProt  " 
+    			th. exec (fullProg ("blastp") + " -query " + prot + " -db " + tmp + ".db/AMRProt" +"  " 
     			  + blastp_par + num_threads + " " BLAST_FMT " -out " + tmp + ".blastp > /dev/null 2> /dev/null", prot_threads);
 
     			stderr << "Running hmmsearch...\n";
     			string cpu;
     			if (prot_threads > 1)
     			  cpu = "--cpu " + to_string (prot_threads);
-    			th. exec (fullProg ("hmmsearch") + " --tblout " + tmp + ".hmmsearch  --noali  --domtblout " + tmp + ".dom  --cut_tc  -Z 10000  " + cpu + " " + db + "/AMR.LIB " + prot + " > /dev/null 2> /dev/null", prot_threads);
+    			th. exec (fullProg ("hmmsearch") + " --tblout " + tmp + ".hmmsearch  --noali  --domtblout " + tmp + ".dom  --cut_tc  -Z 10000  " + cpu + " " + tmp + ".db/AMR.LIB" + " " + prot + " > /dev/null 2> /dev/null", prot_threads);
   		  }
   		  else
   		  {
@@ -601,13 +605,13 @@ struct ThisApplication : ShellApplication
       		  FileItemGenerator fig (false, true, tmp + ".chunk");
       		  string item;
       		  while (fig. next (item))
-        			th << thread (exec, fullProg ("blastx") + "  -query " + tmp + ".chunk/" + item + " -db " + db + "/AMRProt  "
+        			th << thread (exec, fullProg ("blastx") + "  -query " + tmp + ".chunk/" + item + " -db " + tmp + ".db/AMRProt" + "  "
         			  + blastx_par + to_string (gencode) + " " BLAST_FMT
         			  " -out " + tmp + ".blastx_dir/" + item + " > /dev/null 2> /dev/null", string ());
       		  blastxChunks = true;
     		  }
     		  else
-      			th. exec (fullProg ("blastx") + "  -query " + dna + " -db " + db + "/AMRProt  "
+      			th. exec (fullProg ("blastx") + "  -query " + dna + " -db " + tmp + ".db/AMRProt" + "  "
       			  + blastx_par + to_string (gencode) + " " BLAST_FMT
       			  " -out " + tmp + ".blastx > /dev/null 2> /dev/null", threadsAvailable);
     		  amr_report_blastx = "-blastx " + tmp + ".blastx  -dna_len " + tmp + ".len";
@@ -627,7 +631,7 @@ struct ThisApplication : ShellApplication
     			findProg ("blastn");
     			prog2dir ["dna_mutation"] = execDir;
     			stderr << "Running blastn...\n";
-    			exec (fullProg ("blastn") + " -query " + dna + " -db " + db + "/AMR_DNA-" + organism1 + " -evalue 1e-20  -dust no  "
+    			exec (fullProg ("blastn") + " -query " + dna + " -db " + tmp + ".db/AMR_DNA-" + organism1 + " -evalue 1e-20  -dust no  "
     			  BLAST_FMT " -out " + tmp + ".blastn > " + logFName + " 2> " + logFName, logFName);
     		}
     		else
@@ -661,8 +665,8 @@ struct ThisApplication : ShellApplication
     // ".amr"
     const string mutation_allS (mutation_all. empty () ? "" : ("-mutation_all " + mutation_all));
     const string coreS (add_plus ? "" : " -core");
-		exec (fullProg ("amr_report") + " -fam " + db + "/fam.tab  " + amr_report_blastp + "  " + amr_report_blastx
-		  + "  -organism " + strQuote (organism1) + "  -mutation " + db + "/AMRProt-mutation.tab " + mutation_allS + " "
+		exec (fullProg ("amr_report") + " -fam " + shellQuote (db + "/fam.tab") + "  " + amr_report_blastp + "  " + amr_report_blastx
+		  + "  -organism " + strQuote (organism1) + "  -mutation " + shellQuote (db + "/AMRProt-mutation.tab") + " " + mutation_allS + " "
 		  + force_cds_report + " -pseudo" + coreS
 		  + (ident == -1 ? string () : "  -ident_min "    + toString (ident)) 
 		  + "  -coverage_min " + toString (cov)
@@ -673,7 +677,7 @@ struct ThisApplication : ShellApplication
 		    && fileExists (db + "/AMR_DNA-" + organism1)
 		   )
 		{
-			exec (fullProg ("dna_mutation") + tmp + ".blastn " + db + "/AMR_DNA-" + organism1 + ".tab" + qcS + " -log " + logFName + " > " + tmp + ".amr-snp", logFName);
+			exec (fullProg ("dna_mutation") + tmp + ".blastn " + shellQuote (db + "/AMR_DNA-" + organism1 + ".tab") + qcS + " -log " + logFName + " > " + tmp + ".amr-snp", logFName);
 			exec ("tail -n +2 " + tmp + ".amr-snp >> " + tmp + ".amr");
 	  }
 

diff --git a/amrfinder_update.cpp b/amrfinder_update.cpp
@@ -286,7 +286,7 @@ Requirements:\n\
 
 
     Stderr stderr (quiet);
-    stderr << "Running "<< getCommandLine () << '\n';
+    stderr << "Running: "<< getCommandLine () << '\n';
     const Verbose vrb (qc_on);
 
     string mainDirS;
@@ -324,23 +324,16 @@ Requirements:\n\
 
     // Users's files  
     if (! directoryExists (mainDirS))
-      exec ("mkdir -p " + mainDirS);
+      exec ("mkdir -p " + shellQuote (mainDirS));
 
     const string latestDir (mainDirS + latest_version + "/");
     if (directoryExists (latestDir))
-      stderr << latestDir << " already exists, overwriting what was there\n";
+      stderr << shellQuote (latestDir) << " already exists, overwriting what was there\n";
     else
-      exec ("mkdir -p " + latestDir);
+      exec ("mkdir -p " + shellQuote (latestDir));
 
-    {    
-      const string latestLink (mainDirS + "latest");
-      if (directoryExists (latestLink))
-        removeFile (latestLink);
-      exec ("ln -s " + latest_version + " " + latestLink);
-    }
-
 
-    stderr << "Downloading AMRFinder database version " << latest_version << " into " << latestDir << "\n";
+    stderr << "Downloading AMRFinder database version " << latest_version << " into " << shellQuote (latestDir) << "\n";
     const string urlDir (URL + curMinor + "/" + latest_version + "/");
     fetchAMRFile (curl, urlDir, latestDir, "AMR.LIB");
     fetchAMRFile (curl, urlDir, latestDir, "AMRProt");
@@ -377,11 +370,20 @@ Requirements:\n\
     fetchAMRFile (curl, urlDir, latestDir, "changes.txt");
 
     stderr << "Indexing" << "\n";
-    exec (fullProg ("hmmpress") + " -f " + latestDir + "AMR.LIB > /dev/null 2> /dev/null");
-	  exec (fullProg ("makeblastdb") + " -in " + latestDir + "AMRProt  -dbtype prot  -logfile /dev/null");  
-	  exec (fullProg ("makeblastdb") + " -in " + latestDir + "AMR_CDS  -dbtype nucl  -logfile /dev/null");  
+    exec (fullProg ("hmmpress") + " -f " + shellQuote (latestDir + "AMR.LIB") + " > /dev/null 2> /dev/null");
+    exec ("ln -s " + shellQuote (path2canonical (latestDir)) + " " + tmp + ".db");
+	  exec (fullProg ("makeblastdb") + " -in " + tmp + ".db/AMRProt" + "  -dbtype prot  -logfile /dev/null");  
+	  exec (fullProg ("makeblastdb") + " -in " + tmp + ".db/AMR_CDS" + "  -dbtype nucl  -logfile /dev/null");  
     for (const string& dnaPointMut : dnaPointMuts)
-  	  exec (fullProg ("makeblastdb") + " -in " + latestDir + "AMR_DNA-" + dnaPointMut + "  -dbtype nucl  -logfile /dev/null");
+  	  exec (fullProg ("makeblastdb") + " -in " + tmp + ".db/AMR_DNA-" + dnaPointMut + "  -dbtype nucl  -logfile /dev/null");
+
+    {    
+      const string latestLink (mainDirS + "latest");
+      if (directoryExists (latestLink))
+        removeFile (latestLink);
+      exec ("ln -s " + shellQuote (path2canonical (latestDir)) + " " + shellQuote (latestLink));
+    }
+
   }
 };
 

diff --git a/common.cpp b/common.cpp
@@ -91,7 +91,7 @@ string getCommandLine ()
     if (! commandLine. empty ())
       commandLine += ' ';
     if (bad)
-      commandLine += strQuote (s, '\'');
+      commandLine += shellQuote (s);
     else
       commandLine += s;
   }
@@ -817,7 +817,7 @@ streampos getFileSize (const string &fName)
 {
   ifstream f (fName, ifstream::binary);
   if (! f. good ())
-    throw runtime_error ("Cannot open file " + strQuote (fName));
+    throw runtime_error ("Cannot open file " + shellQuote (fName));
 
   const streampos start = f. tellg ();
   QC_ASSERT (start >= 0); 
@@ -829,7 +829,7 @@ streampos getFileSize (const string &fName)
   QC_ASSERT (end >= 0); 
 
   if (end < start)
-    throw runtime_error ("Bad file " + strQuote (fName));    
+    throw runtime_error ("Bad file " + shellQuote (fName));    
   return end - start; 
 }
 
@@ -1206,7 +1206,7 @@ StringVector::StringVector (const string &fName,
   }
   catch (const exception &e)
   {
-    throw runtime_error ("Loading file " + strQuote (fName) + "\n" + e. what ());
+    throw runtime_error ("Loading file " + shellQuote (fName) + "\n" + e. what ());
   }  
 }
 
@@ -1294,9 +1294,9 @@ Input::Input (const string &fName,
 , prog (0, displayPeriod)  
 { 
   if (! ifs. good ())
-    throw runtime_error ("Cannot open file " + strQuote (fName));
+    throw runtime_error ("Cannot open file " + shellQuote (fName));
   if (! ifs. rdbuf () -> pubsetbuf (buf. get (), (long) bufSize))
-  	throw runtime_error ("Cannot allocate buffer to file " + strQuote (fName));
+  	throw runtime_error ("Cannot allocate buffer to file " + shellQuote (fName));
 }
 
 
@@ -1787,7 +1787,7 @@ void OFStream::open (const string &dirName,
 	ofstream::open (pathName);
 
 	if (! good ())
-	  throw runtime_error ("Cannot create file " + strQuote (pathName));
+	  throw runtime_error ("Cannot create file " + shellQuote (pathName));
 }
 
 
@@ -2082,7 +2082,7 @@ JsonMap::JsonMap (const string &fName)
   CharInput in (fName);
   const Token token (in);
   if (! token. isDelimiter ('{'))
-    in. error ("Json file " + strQuote (fName) + ": text should start with '{'", false);
+    in. error ("Json file " + shellQuote (fName) + ": text should start with '{'", false);
   parse (in);
 }
 
@@ -2871,7 +2871,7 @@ void ShellApplication::initEnvironment ()
     string path (execDir + programName);
     for (;;)
     {
-      const string path_new (realpath (path. c_str (), nullptr));
+      const string path_new (path2canonical (path));
       if (path == path_new)
         break;
       path = path_new;
@@ -2902,7 +2902,7 @@ string ShellApplication::which (const string &progName) const
 	if (tmp. empty ())
 	  throw runtime_error ("Temporary file is needed");
 
-	try { exec ("which " + progName + " 1> " + tmp + ".src 2> /dev/null"); }
+	try { exec ("which " + shellQuote (progName) + " 1> " + tmp + ".src 2> /dev/null"); }
 	  catch (const runtime_error &)
 	    { return string (); }
 
@@ -2940,9 +2940,9 @@ string ShellApplication::fullProg (const string &progName) const
 {
 	string dir;
 	if (! find (prog2dir, progName, dir))
-	  throw runtime_error ("Program " + strQuote (progName) + " is not found");
+	  throw runtime_error ("Program " + shellQuote (progName) + " is not found");
 	ASSERT (isRight (dir, "/"));
-	return dir + progName + " ";
+	return shellQuote (dir + progName) + " ";
 }
 #endif