Merge branch 'release-12.0.0'

.circleci/config.yml

@@ -6,17 +6,18 @@ jobs:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
     docker:
-      - image: circleci/openjdk:8-jdk
+      # was circleci/openjdk:8-jdk but something changed and it tests failed on fork
+      - image: circleci/openjdk@sha256:3640c4f42886e796e805c23af48b0d7348dc1d3fa8dae9a365e1f023f913c795
     steps:
       - checkout
       - run: chmod +x mvnw
       - restore_cache:
-          key: exomiser-{{ checksum "pom.xml" }}
+          key: exomiser-{{ arch }}-{{ checksum "pom.xml" }}
       - run: ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
       - save_cache:
           paths:
             - ~/.m2
-          key: exomiser-{{ checksum "pom.xml" }}
+          key: exomiser-{{ arch }}-{{ checksum "pom.xml" }}
       - run: ./mvnw test -B
       - run: ./mvnw sonar:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.login=$SONAR_TOKEN
       #Save junit xml

.mvn/wrapper/maven-wrapper.properties

@@ -1 +1,21 @@
-distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip
+#
+# The Exomiser - A tool to annotate and prioritize genomic variants
+#
+# Copyright (c) 2016-2018 Queen Mary University of London.
+# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip

exomiser-cli/.mvn/wrapper/maven-wrapper.properties

@@ -1 +1,21 @@
-distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip
+#
+# The Exomiser - A tool to annotate and prioritize genomic variants
+#
+# Copyright (c) 2016-2018 Queen Mary University of London.
+# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip

exomiser-cli/CHANGELOG.md

@@ -1,5 +1,11 @@
 # The Exomiser Command Line Executable - Changelog
 
+## 12.0.0 2019-02-28
+This release contains significant diagnostic performance improvements due to the inclusion of a high-quality ClinVar whitelist and 'second generation' pathogenicity scores.
+
+- Added new ```PathogenicitySource``` sources - ```M_CAP, MPC, MVP, PRIMATE_AI```. Be aware that these may not be free for commercial use. _Check the licencing before use!_
+- Added new variant whitelist feature which enables flagging of variants on a whitelist and bypassing of ```FrequencyFilter``` and ```VariantEffectFilter```. By default this will use ClinVar variants listed as ```Pathogenic``` or ```Likely_pathogenic``` and with a review status of ```criteria provided, single submitter``` or better. See https://www.ncbi.nlm.nih.gov/clinvar/docs/review_status/ for an explanation of the ClinVar review status. 
+
 ## 11.0.0 2018-09-21
 - Removed ```analysisMode: SPARSE``` option - this will default to ```PASS_ONLY```
 - Removed ```phenixPrioritiser: {}``` option - we recommend using ```hiPhivePrioritiser: {runParams: 'human'}``` for human-only model comparisons

exomiser-cli/pom.xml

@@ -1,7 +1,7 @@
 <!--
   ~ The Exomiser - A tool to annotate and prioritize genomic variants
   ~
-  ~ Copyright (c) 2016-2018 Queen Mary University of London.
+  ~ Copyright (c) 2016-2019 Queen Mary University of London.
   ~ Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
   ~
   ~ This program is free software: you can redistribute it and/or modify
@@ -32,7 +32,7 @@
     <parent>
         <groupId>org.monarchinitiative.exomiser</groupId>
         <artifactId>exomiser</artifactId>
-        <version>11.0.0</version>
+        <version>12.0.0</version>
     </parent>
 
     <properties>
@@ -65,6 +65,7 @@
                 <!--Use filtering so that maven will replace placeholders with values from the pom e.g. ${project.version} -->
                 <filtering>true</filtering>
                 <excludes>
+                    <exclude>application.properties</exclude>
                     <exclude>distribution/*</exclude>
                     <exclude>examples/*</exclude>
                     <exclude>vcf/*</exclude>
@@ -94,7 +95,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <version>3.0.2</version>
+                <version>3.1.1</version>
                 <executions>
                     <execution>
                         <id>copy-dependencies</id>

exomiser-cli/src/main/java/org/monarchinitiative/exomiser/cli/ExomiserCommandLineRunner.java

@@ -1,7 +1,7 @@
 /*
  * The Exomiser - A tool to annotate and prioritize genomic variants
  *
- * Copyright (c) 2016-2018 Queen Mary University of London.
+ * Copyright (c) 2016-2019 Queen Mary University of London.
  * Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -36,6 +36,8 @@
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 
 /**
@@ -83,12 +85,16 @@ private void runAnalyses(CommandLine commandLine) {
             Path analysisBatchFile = Paths.get(commandLine.getOptionValue("analysis-batch"));
             List<Path> analysisScripts = BatchFileReader.readPathsFromBatchFile(analysisBatchFile);
             logger.info("Running {} analyses from analysis batch file.", analysisScripts.size());
+            Instant timeStart = Instant.now();
             //this *could* be run in parallel using parallelStream() at the expense of RAM in order to hold all the variants in memory.
             //HOWEVER there may be threading issues so this needs investigation.
             analysisScripts.forEach(analysis ->{
                 logger.info("Running analysis: {}", analysis);
                 runAnalysisFromScript(analysis);
             });
+            Duration duration = Duration.between(timeStart, Instant.now());
+            long ms = duration.toMillis();
+            logger.info("Finished batch of {} samples in {}m {}s ({} ms)", analysisScripts.size(), (ms / 1000) / 60 % 60, ms / 1000 % 60, ms);
         }
     }
 

exomiser-cli/src/main/resources/application.properties

@@ -1,7 +1,7 @@
 #
 # The Exomiser - A tool to annotate and prioritize genomic variants
 #
-# Copyright (c) 2016-2018 Queen Mary University of London.
+# Copyright (c) 2016-2019 Queen Mary University of London.
 # Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
 #
 # This program is free software: you can redistribute it and/or modify
@@ -22,10 +22,12 @@
 #it is assumed that all the files required by exomiser listed in this properties file
 #will be found in the data directory unless specifically overridden here.
 #exomiser.data-directory=
+
 ### hg19 assembly ###
-exomiser.hg19.data-version=1805
+exomiser.hg19.data-version=1902
 #transcript source will default to ensembl. Can define as ucsc/ensembl/refseq
 #exomiser.hg19.transcript-source=ensembl
+
 #exomiser.hg19.data-directory=${exomiser.data-directory}/${exomiser.hg19.data-version}_hg19
 #location of CADD/REMM Tabix files - you will need these for analysis of non-coding variants.
 #CADD can be downloaded from http://cadd.gs.washington.edu/download - v1.3 has been tested.
@@ -38,15 +40,20 @@ exomiser.hg19.data-version=1805
 #exomiser.hg19.cadd-in-del-path=${exomiser.hg19.data-directory}/InDels.tsv.gz
 #exomiser.hg19.remm-path=${exomiser.hg19.data-directory}/remmData.tsv.gz
 #exomiser.hg19.local-frequency-path=${exomiser.hg19.data-directory}/local_frequency_test.tsv.gz
+
+exomiser.hg19.variant-white-list-path=1902_hg19_clinvar_whitelist.tsv.gz
+
 ### hg38 assembly ###
 # To enable analysis of samples called against the hg38 assembly copy the hg19 above and just replace the hg19 with hg38
-#exomiser.hg38.data-version=1805
+#exomiser.hg38.data-version=1902
+
 ### phenotypes ###
-exomiser.phenotype.data-version=1807
+exomiser.phenotype.data-version=1902
 #exomiser.phenotype.data-directory=${exomiser.data-directory}/${exomiser.phenotype.data-version}_phenotype
 #String random walk data file
-#exomiser.phenotype.random-walk-file-name=rw_string_9_05.gz
+#exomiser.phenotype.random-walk-file-name=rw_string_10.mv
 #exomiser.phenotype.random-walk-index-file-name=rw_string_9_05_id2index.gz
+
 ### caching ###
 #If you're running exomiser in batch mode there might be some performance benefit
 #if you enable caching. The 'simple' option will continue to store data in memory *without*
@@ -55,4 +62,7 @@ exomiser.phenotype.data-version=1807
 #to your requirements
 #none/simple/caffeine
 #spring.cache.type=none
-#spring.cache.caffeine.spec=maximumSize=60000
+#spring.cache.caffeine.spec=maximumSize=60000
+
+### logging ###
+#logging.file=logs/exomiser.log

exomiser-cli/src/test/resources/logback-test.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ The Exomiser - A tool to annotate and prioritize genomic variants
+  ~
+  ~ Copyright (c) 2016-2018 Queen Mary University of London.
+  ~ Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
+  ~
+  ~ This program is free software: you can redistribute it and/or modify
+  ~ it under the terms of the GNU Affero General Public License as
+  ~ published by the Free Software Foundation, either version 3 of the
+  ~ License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU Affero General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Affero General Public License
+  ~ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+  -->
+
+<configuration>
+    <include resource="org/springframework/boot/logging/logback/base.xml" />
+    <logger name="org.springframework" level="ERROR"/>
+    <logger name="org.thymeleaf" level="ERROR"/>
+</configuration>

exomiser-core/.mvn/wrapper/maven-wrapper.properties

@@ -1 +1,21 @@
-distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip
+#
+# The Exomiser - A tool to annotate and prioritize genomic variants
+#
+# Copyright (c) 2016-2018 Queen Mary University of London.
+# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip

exomiser-core/CHANGELOG.md

@@ -1,5 +1,69 @@
 # The Exomiser - Core Library Changelog
 
+## 12.0.0 2019-02-28
+Its Rare-Disease Day 2019! Although we're not officially releasing this on Feb 28, we're code-complete and undergoing final performance and quality tests, so it's as good as released. This was unplanned, and therefore proves that the universe can have a sense of humour/appropriateness.
+
+API breaking changes:
+- Removed FREQUENCY_SOURCE_MAP from FrequencySource
+- Changed ```Frequency```, ```RsId``` and ```PathogenicityScore``` static ```valueOf()``` constructor to ```of()```
+- Removed deprecated ```IntervalFilter.getGeneticInterval()```
+- Changed visibility of ```PhenodigmMatchRawScore``` from public to package private and made immutable
+- Changed visibility of ```CrossSpeciesPhenotypeMatcher``` from public to package private and added static ```of()``` constructor
+- Replaced redundant ```Default*DaoMvStoreProto``` classes with new ```AllelePropertiesDaoMvStore```
+- Added ```OntologyService``` as constructor argument to ```AnalysisFactory```, ```AnalysisParser``` and ```AnalysisBuilder```
+- Replaced ```BasePathogenicityScore.compareTo()``` method with default ```PathogenicityScore.compareTo()```
+- ```GeneticInterval``` no longer accepts ```ReferenceDictionary``` as a constructor argument
+
+New APIs:
+- Added CADD and REMM to data-genome ```AlleleProperty```
+- Moved ```JannovarDataSourceLoader``` from autoconfigure to core module
+- Added ```AllelePosition.isSymbolic()``` method
+- Added ```Variant.isCodingVariant()``` method
+- Added ```AnalysisBuilder.addIntervalFilter(Collection<ChromosomalRegion> chromosomalRegions)``` method
+- Added new non-public ```FilterStats``` class for more accurate filtering statistics 
+- Added new ```AllelePropertiesDao``` interface
+- Added new ```AllelePropertiesDaoMvStore``` implementation
+- Added new ```AllelePropertiesDaoAdapter``` to fix issue of Spring cache proxy not being able to intercept internal calls
+- Added new ```HpoIdChecker``` class to return current HPO id/terms for an input id/term
+- Added new ```HumanPhenotypeOntologyDao.getIdToPhenotypeTerms()``` method
+- Added new ```OntologyService.getCurrentHpoIds()``` method
+- Added new ```SampleGenotype.isEmpty()``` method
+- Added new experimental ```VcfCodecs``` class for de/serialising VCF lines
+- Added new ```JannovarDataProtoSerialiser.loadProto()``` method for loading intermediate ```JannovarProto.JannovarData``` 
+- Added new ```VariantWhiteList``` and ```InMemoryVariantWhiteList``` implementation
+- Added new ```VariantEvaluation.isWhiteListed()``` method and relevant builder methods
+- Added new ```JannovarDataFactory``` for a simple programmatic API to build ```JannovarData``` objects
+- Added new ```TranscriptSource``` enum 
+- Added new ```PathogenicityScore.of()``` static factory constructor
+- Added new ```PathogenicityScore.getRawScore()``` method
+- Added default ```PathogenicityScore.compareTo()``` method
+- Added new static ```PathogenicityScore.compare()``` method
+- Added new ```ScaledPathogenicityScore``` class
+- Added new ```MpcScore``` class
+- Add new ```Contig``` class for converting contig names to integer-based id 
+
+Other changes:
+- Updated Spring Boot to version 2.1.3
+- Updated Jannovar to version 0.28
+- Updated HTSJDK to version 2.18.2
+- Refactored ```FrequencyData``` to use array-based backing for 5-10% memory usage improvement and lower GC especially when nearing max memory
+- Refactored ```AnalysisParser``` to utilise ```AnalysisBuilder``` directly reducing code duplication
+- Refactored ```AnalysisRunner``` classes to to utilise new ```FilterStats``` class
+- Refactored ```QueryPhenotypeMatch``` to store and return input queryPhenotypeMatches argument
+- Refactored ```VariantDataServiceImpl``` to use new AllelePropertiesDao
+- Refactored ```VariantDataServiceImpl``` for better readability and performance
+- Added check for obsolete HPO id input in ```AnalysisBuilder.hpoIds()```
+- Re-enabled ```PhenixPrioritiser``` in ```AnalysisParser```
+- Refactored ```VariantEvaluation.getSampleGenotypeString()``` implementation to use ```SampleGenotype``` instead of ```VariantContext```
+- Refactored ```VariantEffectCounter``` internals with ```VariantEvaluation``` calls in place of ```VariantContext```
+- Enabled flagging of variants on a whitelist and bypassing of ```FrequencyFilter``` and ```VariantEffectFilter```
+- Changed ```DefaultDiseaseDao``` to only return diseases marked as having known disease-gene association or copy-number/structural causes
+- Added range check to ```BasePathogenicityScore``` constructor
+- Updated ```CaddScore``` and ```SiftScore``` to extend ```ScaledPathogenicityScore```
+- Updated ```CaddDao``` to use CADD phred scaled score directly
+- Replaced production use of ```ReferenceDictionary``` from ```HG19RefDictBuilder``` with ```Contig```
+- Added new ```PathogenicitySource``` sources - ```M_CAP, MPC, MVP, PRIMATE_AI```. Be aware that these may not be free for commercial use.
+
 ## 11.0.0 2018-09-21
 API breaking changes:
 - Removed unused ```VariantSerialiser```

exomiser-core/pom.xml

@@ -2,7 +2,7 @@
 <!--
   ~ The Exomiser - A tool to annotate and prioritize genomic variants
   ~
-  ~ Copyright (c) 2016-2018 Queen Mary University of London.
+  ~ Copyright (c) 2016-2019 Queen Mary University of London.
   ~ Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd.
   ~
   ~ This program is free software: you can redistribute it and/or modify
@@ -30,7 +30,7 @@
     <parent>
         <groupId>org.monarchinitiative.exomiser</groupId>
         <artifactId>exomiser</artifactId>
-        <version>11.0.0</version>
+        <version>12.0.0</version>
     </parent>
 
     <dependencies>
@@ -50,6 +50,12 @@
             <groupId>com.github.samtools</groupId>
             <artifactId>htsjdk</artifactId>
             <version>${htsjdk.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.jblas</groupId>
@@ -127,14 +133,18 @@
         <!-- Groovy is required for logback.groovy processing-->
         <dependency>
             <groupId>org.codehaus.groovy</groupId>
-            <artifactId>groovy-all</artifactId>
+            <artifactId>groovy</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.skyscreamer</groupId>
             <artifactId>jsonassert</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+        </dependency>
     </dependencies>
 
     <build>