Merge pull request #451 from ANGSD/Preparing_master_to_banded_pull

merging the master branch into the banded so the banded is up to date

.github/workflows/build-tests.yml

@@ -0,0 +1,32 @@
+name: build-tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-18.04 ]
+        compiler: [ clang, gcc ]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Checkout submodules
+      run: git submodule update --init --recursive
+
+    - name: Install dependencies    
+      run: |
+        sudo apt-get install -y libbz2-dev liblzma-dev libcurl4-openssl-dev
+        git clone --depth=5 git://github.com/ANGSD/smallBam.git
+
+    - name: Run CMake
+      run: |
+        make
+        make test BAMDIR=./smallBam/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "htslib"]
+	path = htslib
+	url = https://github.com/samtools/htslib.git

Makefile

@@ -4,16 +4,50 @@ CXX ?= g++
 LIBS = -lz -lm -lbz2 -llzma -lpthread -lcurl
 CRYPTOLIB = -lcrypto
 
-# Adjust $(HTSSRC) to point to your top-level htslib directory
+#if htslib source is defined
 ifdef HTSSRC
+
+#if hts source is set to systemwide
+ifeq ($(HTSSRC),systemwide)
+$(info HTSSRC set to systemwide; assuming systemwide installation)
+LIBS += -lhts
+
+else
+
+#if hts source path is given
+# Adjust $(HTSSRC) to point to your top-level htslib directory
 $(info HTSSRC defined: $(HTSSRC))
 CPPFLAGS += -I"$(realpath $(HTSSRC))"
-LIBS := "$(realpath $(HTSSRC))/libhts.a" $(LIBS)
+LIBHTS := $(HTSSRC)/libhts.a
+LIBS := $(LIBHTS) $(LIBS)
+
+endif
+
+#if htssrc not defined
 else
-$(info HTSSRC not defined, assuming systemwide installation)
-LIBS += -lhts
+
+$(info HTSSRC not defined; using htslib submodule)
+$(info Use `make HTSSRC=/path/to/htslib` to build angsd using a local htslib installation)
+$(info Use `make HTSSRC=systemwide` to build angsd using the systemwide htslib installation)
+
+
+HTSSRC := $(CURDIR)/htslib
+CPPFLAGS += -I$(HTSSRC)
+LIBHTS := $(HTSSRC)/libhts.a
+LIBS := $(LIBHTS) $(LIBS)
+
+all: .activate_module
+
 endif
 
+.PHONY: .activate_module 
+
+.activate_module:
+	git submodule update --init --recursive
+	$(MAKE) -C $(HTSSRC)
+
+
+
 #modied from htslib makefile
 FLAGS = -O3
 FLAGS2 = $(CPPFLAGS) $(FLAGS) $(LDFLAGS)
@@ -50,7 +84,7 @@ all: $(PROGRAMS) misc
 BAMDIR=""
 BDIR=$(realpath $(BAMDIR))
 
-PACKAGE_VERSION  = 0.935
+PACKAGE_VERSION  = 0.937
 
 ifneq "$(wildcard .git)" ""
 PACKAGE_VERSION := $(shell git describe --always --dirty)
@@ -75,9 +109,11 @@ misc: analysisFunction.o bfgs.o prep_sites.o
 	$(CXX) -c  $(CXXFLAGS) $*.cpp
 	$(CXX) -MM $(CXXFLAGS) $*.cpp >$*.d
 
+
 angsd: version.h $(OBJ)
 	$(CXX) $(FLAGS) -o angsd *.o $(LIBS) $(CRYPTOLIB)
 
+
 testclean:
 	rm -rf test/sfstest/output test/tajima/output test/*.log version.h test/temp.txt
 

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.com/ANGSD/angsd.svg?branch=master)](https://travis-ci.com/ANGSD/angsd)
+[![Build Status](https://img.shields.io/github/workflow/status/angsd/angsd/build-tests)](https://github.com/ANGSD/angsd)
 angsd:
 
 =====
@@ -25,6 +25,7 @@ cd angsd;make
 Notes
 ====
 * I've switched over to using htslib for parsing single reads (to allow for CRAM reading, while avoid having to write my own CRAM parser). I'm still using my own readpools. Users should therefore also download and install htslib.
+* If you are on a mac computer and the compilation process complains about a missnig crybtolib library then do 'make CRYPTOLIB=""'
 
 Program has a paper
 =====

abcCounts.cpp

@@ -273,7 +273,7 @@ abcCounts::abcCounts(const char *outfiles,argStruct *arguments,int inputtype){
     if(dumpCounts!=5)
       ksprintf(&bufstr,"chr\tpos\ttotDepth\n");
     else
-      ksprintf(&bufstr,"chr\tpos\tmeanReadLength\n");
+      ksprintf(&bufstr,"chr\tpos\tmeanReadLength\tdepth\n");
     aio::bgzf_write(oFileCountsPos,bufstr.s,bufstr.l);bufstr.l=0;
     if(dumpCounts>1)
       oFileCountsBin = aio::openFileBG(outfiles,postfix2);
@@ -460,7 +460,7 @@ void abcCounts::print(funkyPars *pars){
 	  nreads++;
 	}
       }
-      ksprintf(&bpos, "%s\t%d\t%.2f\n",header->target_name[pars->refId],pars->posi[s]+1,readlensSum/nreads);
+      ksprintf(&bpos, "%s\t%d\t%.2f\t%d\n",header->target_name[pars->refId],pars->posi[s]+1,readlensSum/nreads,(int)nreads);
       //      fprintf(stderr,"nraads: %f readlenssum: %f\n",nreads,readlensSum);
     }
 

abcFreq.cpp

@@ -82,7 +82,7 @@ void abcFreq::printArg(FILE *argFile){
   fprintf(argFile,"\t-minMaf  \t%f\t(Remove sites with MAF below)\n",minMaf);
   fprintf(argFile,"\t-SNP_pval\t%f\t(Remove sites with a pvalue larger)\n",SNP_pval);
   fprintf(argFile,"\t-rmTriallelic\t%f\t(Remove sites with a pvalue lower)\n",rmTriallelic);
-  fprintf(argFile,"\t-forceMaf\t%f\t(Write .mafs file when running -doAsso (by default does not output .mafs file with -doAsso))\n",forceMaf);
+  fprintf(argFile,"\t-forceMaf\t%d\t(Write .mafs file when running -doAsso (by default does not output .mafs file with -doAsso))\n",forceMaf);
   fprintf(argFile,"\t-skipMissing\t%d\t(Set post to 0.33 if missing (do not use freq as prior))\n",skipMissing);
   fprintf(argFile,"Extras:\n");
   fprintf(argFile,"\t-ref\t%s\t(Filename for fasta reference)\n",refName);

abcScounts.cpp

@@ -33,8 +33,8 @@ aMap readvcf(const char *fname){
     if(4!=sscanf(buf,"%s\t%d\t%c\t%c\n",chr,&pos,&al1,&al2)){
       fprintf(stderr,"\t-> problem parsing line: %d which looks like: %s\n",at,buf);
     }
-    char tmpnam[1024];
-    sprintf(tmpnam,"%s %d",chr,pos);
+    char tmpnam[2048];
+    snprintf(tmpnam,2048,"%s %d",chr,pos);
     aMap::iterator it=am.find(tmpnam);
     if(it!=am.end()){
       fprintf(stderr,"\t-> Problem with duplicate positions: %s \n",tmpnam);

abcTemplate.cpp

@@ -478,7 +478,8 @@ void abcTemplate::run(funkyPars *pars){
 
 	// Reference base:
 	int refB = -1;
-	if (pars->ref == 0) fprintf(outfile,"  Warning: Ref not defined! %s \n", pars->ref);
+	if (pars->ref == NULL)
+	  fprintf(outfile,"\t-> Warning: Ref not defined!  \n");
 	else refB = refToInt[pars->ref[s]];
 
 

abcWriteBcf.cpp

@@ -128,6 +128,7 @@ void abcWriteBcf::clean(funkyPars *pars){
 }
 
 void abcWriteBcf::print(funkyPars *pars){
+  assert(bcf_hdr_nsamples(hdr)>0);
   if(doBcf==0)
     return;
 
@@ -143,7 +144,7 @@ void abcWriteBcf::print(funkyPars *pars){
     rec->rid = bcf_hdr_name2id(hdr,header->target_name[pars->refId]);
     rec->pos = pars->posi[s];//<- maybe one index?
     //    bcf_update_id(hdr, rec, "rs6054257");
-    if(domcall!=NULL){
+    if(domcall>0&&mcall!=NULL){
       // 
       char alleles[16];
       int goa =0;
@@ -376,7 +377,9 @@ abcWriteBcf::abcWriteBcf(const char *outfiles_a,argStruct *arguments,int inputty
   fp=aio::openFileHtsBcf(outfiles,".bcf");
 
   rec    = bcf_init1();
-  if(arguments->inputtype==INPUT_BAM){
+  whatIsTheInput(arguments->inputtype);
+
+  if(arguments->inputtype==INPUT_BAM||arguments->inputtype==INPUT_GLF||arguments->inputtype==INPUT_GLF3||arguments->inputtype==INPUT_GLF10_TEXT){
     hdr = bcf_hdr_init("w");
     print_bcf_header(fp,hdr,args,buf,header,domcall);
   } else if(arguments->inputtype==INPUT_VCF_GP||arguments->inputtype==INPUT_VCF_GL){
@@ -396,9 +399,11 @@ abcWriteBcf::abcWriteBcf(const char *outfiles_a,argStruct *arguments,int inputty
     buf.l=0;
     if ( bcf_hdr_write(fp, hdr)!=0 )
       fprintf(stderr,"Failed to write bcf %s:%d\n",__FILE__,__LINE__);
+
+    if(buf.s)
+      free(buf.s);
   }
-  if(buf.s)
-    free(buf.s);
+
 }
 
 

aio.cpp

@@ -19,7 +19,7 @@ FILE *aio::openFile(const char* a,const char* b){
     fprintf(stderr,"[%s] %s %s",__FUNCTION__,a,b);
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   //  fprintf(stderr,"\t-> Dumping file: %s\n",c);
   dumpedFiles.push_back(strdup(c));
   FILE *fp = NULL;
@@ -36,7 +36,7 @@ BGZF *aio::openFileBG(const char* a,const char* b){
 
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   dumpedFiles.push_back(strdup(c));
   BGZF *fp = bgzf_open(c,"w6h");
   delete [] c;
@@ -47,7 +47,7 @@ htsFile *aio::openFileHts(const char* a,const char* b){
 
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   dumpedFiles.push_back(strdup(c));
   htsFile *fp = hts_open(c,"w");
   delete [] c;
@@ -58,7 +58,7 @@ htsFile *aio::openFileHtsBcf(const char* a,const char* b){
 
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   dumpedFiles.push_back(strdup(c));
   htsFile *fp = hts_open(c,"wb");
   delete [] c;

analysisFunction.cpp

@@ -39,12 +39,12 @@ double angsd::addProtectN(double a[],int len){
   //function does: log(sum(exp(a))) while protecting for underflow
   double maxVal = a[0];
 
-  for(int i=1;i<10;i++)
+  for(int i=1;i<len;i++)
     if(maxVal<a[i])
       maxVal=a[i];
 
   double sumVal = 0;
-  for(int i=1;i<10;i++)
+  for(int i=1;i<len;i++)
     sumVal += exp(a[i]-maxVal);
 
   return log(sumVal) + maxVal;

argStruct.h

@@ -78,3 +78,4 @@ namespace angsd {
   std::vector<char*> getFilenames(const char * name,int nInd);
 }
 
+void whatIsTheInput(int type);

bgenReader.cpp

@@ -137,7 +137,7 @@ bgenLine *bgenReader::parseline(FILE *fp,header *hd){
   unsigned short Lchr_l;
   assert(fread(&Lchr_l,sizeof(unsigned short),1,fp)==1);
   bgen->Lchr =(char*) calloc(Lchr_l+1,sizeof(char));
-  fread(bgen->Lchr,sizeof(char),Lchr_l,fp);
+  assert(fread(bgen->Lchr,sizeof(char),Lchr_l,fp)==Lchr_l);
 
   assert(fread(&bgen->vpos,sizeof(unsigned),1,fp)==1);
   assert(fread(&bgen->nal,sizeof(unsigned short),1,fp)==1);

doc/formats.tex

@@ -149,7 +149,7 @@ \subsubsection{Theta definitions}
 Let $\eta$ be the site frequency spectra. Then $\eta_i$ is the posterior probablity of being in frequency $i$. 
 \begin{itemize}
 \item[Watterson] $\sum_{i=1}^{n-1}\eta_i/a^{-1}, a=\sum_{i=1}^{n-1}i$
-\item[$\pi$] $ {{n}\choose{2}}^{-1}\sum_{i=1}^{n-1}i(n-1)\eta_i$
+\item[$\pi$] $ {{n}\choose{2}}^{-1}\sum_{i=1}^{n-1}i(n-i)\eta_i$
 \item[FuLi] $\eta_1$
 \item[FayH] $ {{n}\choose{2}}^{-1}\sum_{i=1}^{n-1}i^2\eta_i$ 
 \item[L] $ {n-1}^{-1}\sum_{i=1}^{n-1}i\eta_i$ 

makeReadPool.cpp

@@ -260,7 +260,7 @@ void read_reads_noStop(htsFile *fp,int nReads,int &isEof,readPool &ret,int refTo
       rdObjRegionDone =1;
       break;
     }else if(ret.first[ret.l+i-1]>b->core.pos){
-      fprintf(stderr,"[%s] unsorted file detected will exit new(-1)=%d new=%lld,ret.l=%d i=%d\n",__FUNCTION__,ret.first[ret.l+i-1],b->core.pos,ret.l,i);
+      fprintf(stderr,"[%s] unsorted file detected will exit new(-1)=%d new=%ld,ret.l=%d i=%d\n",__FUNCTION__,ret.first[ret.l+i-1],b->core.pos,ret.l,i);
       exit(0);
     }
 

misc/fstreader.cpp

@@ -110,7 +110,7 @@ perfst * perfst_init(char *fname){
 #endif
   fclose(fp);
   char *tmp =(char*)calloc(strlen(fname)+100,1);//that should do it
-  tmp=strncpy(tmp,fname,strlen(fname)-3);
+  memcpy(tmp,fname,strlen(fname)-3);
   // fprintf(stderr,"tmp:%s\n",tmp);
 
   char *tmp2 = (char*)calloc(strlen(fname)+100,1);//that should do it

misc/header.cpp

@@ -28,7 +28,7 @@ BGZF *openFileBG(const char* a,const char* b){
 
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   BGZF *fp = bgzf_open(c,"wb");
   delete [] c;
   return fp;
@@ -38,7 +38,7 @@ FILE *openFile(const char* a,const char* b){
     fprintf(stderr,"[%s] %s %s",__FUNCTION__,a,b);
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   FILE *fp = fopen(c,"w");
   delete [] c;
   return fp;

misc/ibs.cpp

@@ -78,7 +78,7 @@ std::vector<char *> dumpedFiles;
 FILE *openFile(const char* a,const char* b){
   char *c = new char[strlen(a)+strlen(b)+1];
   strcpy(c,a);
-  strncat(c,b,strlen(b));
+  strcat(c,b);
   fprintf(stderr,"\t-> Dumping file: %s\n",c);
 
   dumpedFiles.push_back(strdup(c));