Merge pull request #255 from geneontology/owl-cmdline-issue254

Owl cmdline issue254

.gitignore

@@ -7,3 +7,6 @@ minerva-server/blazegraph.jnl
 /bin/
 *.icloud
 minerva-cli/bin/reactome-entities-blazegraph.jnl
+*.jnl
+minerva-cli/go-cam-schema.shex
+minerva-cli/go-cam-shapes.shapeMap

minerva-cli/src/main/java/org/geneontology/minerva/cli/MinervaCommandRunner.java

@@ -5,15 +5,20 @@
 import com.bigdata.rdf.sail.BigdataSailRepository;
 import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
 import com.google.common.base.Optional;
+import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.BlazegraphMolecularModelManager;
 import org.geneontology.minerva.GafToLegoIndividualTranslator;
+import org.geneontology.minerva.ModelContainer;
+import org.geneontology.minerva.UndoAwareMolecularModelManager;
 import org.geneontology.minerva.curie.CurieHandler;
 import org.geneontology.minerva.curie.CurieMappings;
 import org.geneontology.minerva.curie.DefaultCurieHandler;
@@ -25,14 +30,22 @@
 import org.geneontology.minerva.legacy.LegoToGeneAnnotationTranslator;
 import org.geneontology.minerva.legacy.sparql.GPADSPARQLExport;
 import org.geneontology.minerva.lookup.ExternalLookupService;
+import org.geneontology.minerva.server.StartUpTool;
+import org.geneontology.minerva.server.inferences.InferenceProviderCreator;
+import org.geneontology.minerva.server.validation.MinervaShexValidator;
 import org.geneontology.minerva.util.BlazegraphMutationCounter;
 import org.geneontology.minerva.validation.Enricher;
 import org.geneontology.minerva.validation.ShexValidationReport;
 import org.geneontology.minerva.validation.ShexValidator;
+import org.geneontology.minerva.validation.ValidationResultSet;
+import org.geneontology.rules.engine.WorkingMemory;
+import org.geneontology.rules.util.Bridge;
 import org.openrdf.query.MalformedQueryException;
 import org.openrdf.query.QueryLanguage;
 import org.openrdf.query.UpdateExecutionException;
 import org.openrdf.repository.RepositoryException;
+import org.openrdf.rio.RDFHandlerException;
+import org.openrdf.rio.RDFParseException;
 import org.semanticweb.owlapi.apibinding.OWLManager;
 import org.semanticweb.owlapi.formats.FunctionalSyntaxDocumentFormat;
 import org.semanticweb.owlapi.formats.ManchesterSyntaxDocumentFormat;
@@ -54,6 +67,7 @@
 import owltools.gaf.io.GafWriter;
 import owltools.gaf.io.GpadWriter;
 import owltools.graph.OWLGraphWrapper;
+import scala.collection.JavaConverters;
 import uk.ac.manchester.cs.owlapi.modularity.ModuleType;
 import uk.ac.manchester.cs.owlapi.modularity.SyntacticLocalityModuleExtractor;
 
@@ -62,6 +76,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.stream.Collectors;
 
 public class MinervaCommandRunner extends JsCommandRunner {
 
@@ -390,6 +405,190 @@ else if (opts.nextEq("-m|--minimize")) {
 		}
 	}
 
+	/**
+	 * Output whether each model provided is logically consistent or not
+	 * example parameters for invocation:
+	 *   --owlcheck-go-cams -i /GitHub/GO_Shapes/test_ttl/go_cams/should_pass/ -c ./catalog-no-import.xml
+	 * @param opts
+	 * @throws Exception
+	 */
+	@CLIMethod("--owlcheck-go-cams")
+	public void owlCheckGoCams(Opts opts) throws Exception {
+		Logger LOG = Logger.getLogger(BlazegraphMolecularModelManager.class);
+		LOG.setLevel(Level.ERROR);
+		LOGGER.setLevel(Level.INFO);
+		String catalog = null;
+		String modelIdPrefix = "http://model.geneontology.org/";
+		String modelIdcurie = "gomodel";
+		String inputDB = "blazegraph.jnl";
+		String explanationOutputFile = null;
+		String basicOutputFile = null;
+		String ontologyIRI = "http://purl.obolibrary.org/obo/go/extensions/go-lego.owl";
+		String shexFileUrl = "https://raw.githubusercontent.com/geneontology/go-shapes/master/shapes/go-cam-shapes.shex";
+		String goshapemapFileUrl = "https://raw.githubusercontent.com/geneontology/go-shapes/master/shapes/go-cam-shapes.shapeMap";
+		String shexpath = null;
+		String shapemappath = null;
+		String input = null;
+		boolean checkShex = false;
+		Map<String, String> modelid_filename = new HashMap<String, String>();
+		while (opts.hasOpts()) {
+			if (opts.nextEq("-i|--input")) {
+				input = opts.nextOpt();
+			}
+			else if (opts.nextEq("--shex")) {
+				checkShex = true;
+			}
+			else if (opts.nextEq("--output-file")) {
+				basicOutputFile = opts.nextOpt();
+			}
+			else if (opts.nextEq("--explanation-output-file")) {
+				explanationOutputFile = opts.nextOpt();
+			}
+			else if (opts.nextEq("--model-id-prefix")) {
+				modelIdPrefix = opts.nextOpt();
+			}
+			else if (opts.nextEq("--model-id-curie")) {
+				modelIdcurie = opts.nextOpt();
+			}
+			else if (opts.nextEq("--ontology")) {
+				ontologyIRI = opts.nextOpt();
+			}
+			else if (opts.nextEq("-c|--catalog")) {
+				catalog = opts.nextOpt();
+			}
+			else {
+				break;
+			}
+		}
+		if(basicOutputFile==null) {
+			LOGGER.error("please specific an output file with --output-file ");
+			System.exit(-1);
+		}
+		if(input==null) {
+			LOGGER.error("please provide an input file - either a directory of ttl files or a blazegraph journal");
+			System.exit(-1);
+		}
+		if(input.endsWith(".jnl")) {
+			inputDB = input;
+			LOGGER.info("loaded blazegraph journal: "+input);
+		}else {
+			LOGGER.info("no journal found, trying as directory: "+input);
+			File i = new File(input);
+			if(i.exists()) {
+				//remove anything that existed earlier
+				File bgdb = new File(inputDB);
+				if(bgdb.exists()) {
+					bgdb.delete();
+				}
+				//load everything into a bg journal
+				OWLOntology dummy = OWLManager.createOWLOntologyManager().createOntology(IRI.create("http://example.org/dummy"));
+				CurieHandler curieHandler = new MappedCurieHandler();
+				BlazegraphMolecularModelManager<Void> m3 = new BlazegraphMolecularModelManager<>(dummy, curieHandler, modelIdPrefix, inputDB, null);
+				if(i.isDirectory()) {
+					FileUtils.listFiles(i, null, true).parallelStream().parallel().forEach(file-> {
+						//for (File file : 
+						//m3.getAvailableModelIds().stream().parallel().forEach(modelIRI -> {
+						if(file.getName().endsWith(".ttl")||file.getName().endsWith("owl")) {
+							LOGGER.info("Loading " + file);
+							try {
+								String modeluri = m3.importModelToDatabase(file, true);
+								modelid_filename.put(modeluri, file.getName());
+							} catch (OWLOntologyCreationException | RepositoryException | RDFParseException
+									| RDFHandlerException | IOException e) {
+								// TODO Auto-generated catch block
+								e.printStackTrace();
+							}
+						} 
+					});
+				}else {
+					LOGGER.info("Loading " + i);
+					m3.importModelToDatabase(i, true);
+				}
+				LOGGER.info("loaded files into blazegraph journal: "+input);
+				m3.dispose();
+			}
+		}
+		LOGGER.info("loading tbox ontology: "+ontologyIRI);
+		OWLOntologyManager ontman = OWLManager.createOWLOntologyManager();
+		if(catalog!=null) {
+			LOGGER.info("using catalog: "+catalog);
+			ontman.setIRIMappers(Sets.newHashSet(new owltools.io.CatalogXmlIRIMapper(catalog)));
+		}else {
+			LOGGER.info("no catalog, resolving all ontology uris directly");
+		}
+		OWLOntology tbox_ontology = ontman.loadOntology(IRI.create(ontologyIRI));
+		tbox_ontology = StartUpTool.forceMergeImports(tbox_ontology, tbox_ontology.getImports());
+		LOGGER.info("ontology axioms loaded: "+tbox_ontology.getAxiomCount());
+		LOGGER.info("building model manager and structural reasoner");
+		CurieMappings localMappings = new CurieMappings.SimpleCurieMappings(Collections.singletonMap(modelIdcurie, modelIdPrefix));
+		CurieHandler curieHandler = new MappedCurieHandler(DefaultCurieHandler.loadDefaultMappings(), localMappings);
+		UndoAwareMolecularModelManager m3 = new UndoAwareMolecularModelManager(tbox_ontology, curieHandler, modelIdPrefix, inputDB, null);
+		String reasonerOpt = "arachne"; 
+		LOGGER.info("tbox reasoner for shex "+m3.getTbox_reasoner().getReasonerName());		
+		if(shexpath==null) {
+			//fall back on downloading from shapes repo
+			URL shex_schema_url = new URL(shexFileUrl);
+			shexpath = "./go-cam-schema.shex";
+			File shex_schema_file = new File(shexpath);
+			org.apache.commons.io.FileUtils.copyURLToFile(shex_schema_url, shex_schema_file);			
+			System.err.println("-s .No shex schema provided, using: "+shexFileUrl);
+		}
+		if(shapemappath==null) {
+			URL shex_map_url = new URL(goshapemapFileUrl);
+			shapemappath = "./go-cam-shapes.shapeMap";
+			File shex_map_file = new File(shapemappath);
+			org.apache.commons.io.FileUtils.copyURLToFile(shex_map_url, shex_map_file);
+			System.err.println("-m .No shape map file provided, using: "+goshapemapFileUrl);
+		}
+		MinervaShexValidator shex = new MinervaShexValidator(shexpath, shapemappath, curieHandler, m3.getTbox_reasoner());
+		if(checkShex) {
+			if(checkShex) {
+				shex.setActive(true);
+			}else {
+				shex.setActive(false);
+			}
+		}
+		LOGGER.info("Building OWL inference provider: "+reasonerOpt);
+		InferenceProviderCreator ipc = StartUpTool.createInferenceProviderCreator(reasonerOpt, m3, shex);
+		LOGGER.info("Validating models: "+reasonerOpt);
+
+		FileWriter basic_output = new FileWriter(basicOutputFile);
+		try {
+			basic_output.write("filename\tmodel_id\tOWL_consistent\tshex_valid\n");
+			final boolean shex_output = checkShex;
+			m3.getAvailableModelIds().stream().forEach(modelIRI -> {
+				try {
+					String filename = modelid_filename.get(modelIRI.toString());
+					boolean isConsistent = true;
+					boolean isConformant = true;
+					LOGGER.info("processing "+filename+"\t"+modelIRI);
+					ModelContainer mc = m3.getModel(modelIRI);		
+					//not reporting OWL errors ?
+					InferenceProvider ip = ipc.create(mc);
+					isConsistent = ip.isConsistent();
+					if(shex_output) {
+						ValidationResultSet validations = ip.getValidation_results();
+						isConformant = validations.allConformant();	
+						LOGGER.info(filename+"\t"+modelIRI+"\tOWL:"+isConsistent+"\tshex:"+isConformant);
+						basic_output.write(filename+"\t"+modelIRI+"\t"+isConsistent+"\t"+isConformant+"\n");
+					}else {
+						LOGGER.info(filename+"\t"+modelIRI+"\tOWL:"+isConsistent+"\tshex:not checked");
+						basic_output.write(filename+"\t"+modelIRI+"\t"+isConsistent+"\tnot checked\n");						
+					}
+				} catch (InconsistentOntologyException e) {
+					LOGGER.error("Inconsistent model: " + modelIRI);
+				} catch (Exception e) {
+					// TODO Auto-generated catch block
+					e.printStackTrace();
+				} 
+			});
+			m3.dispose();
+		}finally{
+			basic_output.close();
+		}
+	}
+
+
 	/**
 	 * Will test a go-cam model or directory of models against a shex schema and shape map to test conformance
 	 * Example invocation: --validate-go-cams -s /Users/bgood/Documents/GitHub/GO_Shapes/shapes/go-cam-shapes.shex -m /Users/bgood/Documents/GitHub/GO_Shapes/shapes/go-cam-shapes.shapeMap -f /Users/bgood/Documents/GitHub/GO_Shapes/test_ttl/go_cams/should_pass/ -e -r /Users/bgood/Desktop/shapely_report.txt
@@ -407,12 +606,12 @@ public void validateGoCams(Opts opts) throws Exception {
 		String shapemappath = null;
 		String model_file = null;//"../test_ttl/go_cams/should_pass/typed_reactome-homosapiens-Acetylation.ttl";
 		String report_file = null;
-		boolean addSuperClasses = false;
+		boolean addSuperClasses = true;//this always needs to be on unless we have a major shex refactor
 		boolean addSuperClassesLocal = false;
 		boolean travisMode = false; 
 		boolean shouldPass = true;
 		String extra_endpoint = null;
-		String local_go_lego = null;
+		String catalog = null;
 		Map<String, Model> name_model = new HashMap<String, Model>();
 
 		while (opts.hasOpts()) {
@@ -435,17 +634,14 @@ else if(opts.nextEq("-m|--shapemap")) {
 			else if(opts.nextEq("-r|--report")) { 
 				report_file = opts.nextOpt();
 			}
-			else if(opts.nextEq("-expand")) { 
-				addSuperClasses = true;
-			}
-			else if(opts.nextEq("-downloadtbox")) { 
-				addSuperClassesLocal = true;
+			else if(opts.nextEq("-localtbox")) { 
+				addSuperClassesLocal = true; //this will download the beast unless there is a catalogue file
 			}
 			else if(opts.nextEq("-extra_endpoint")) { 
 				extra_endpoint = opts.nextOpt();
 			}
-			else if(opts.nextEq("-local-go-lego")) {
-				local_go_lego = opts.nextOpt();
+			else if (opts.nextEq("-c|--catalog")) {
+				catalog = opts.nextOpt();
 			}
 			else {
 				break;
@@ -472,39 +668,44 @@ else if(opts.nextEq("-local-go-lego")) {
 		}
 		if(!addSuperClassesLocal) {
 			System.out.println("Using RDF endpoint for super class expansion: "+Enricher.go_endpoint);
-			System.out.println("add -downloadtbox to retrieve the ontology from http://purl.obolibrary.org/obo/go/extensions/go-lego.owl "
-					+ "and do the inferences locally.  This is slower and uses much more memory (set to 8g), but at least you know what you are getting.");
+			System.out.println("add -localtbox to retrieve the ontology from http://purl.obolibrary.org/obo/go/extensions/go-lego.owl "
+					+ "and do the inferences locally.  This is slower and uses much more memory, but at least you know what you are getting. "
+					+ "You can use a catalog file by adding -c yourcatalog.xml "
+					+ "- catalog files allow you to map URIs to local files to change "
+					+ "what they resolve to and to reduce network traffic time.");
 		}
 		//requirements
 		if(model_file==null) {
 			System.err.println("-f .No go-cam file or directory provided to validate.");
 			exit(-1);
 		}else {
-			validator = new ShexValidator(shexpath, shapemappath);
 			FileWriter w = new FileWriter(report_file);
 			int good = 0; int bad = 0;
 			Enricher enrich = new Enricher(extra_endpoint, null);
 			if(addSuperClassesLocal) {
-				File tbox_file = new File("./target/go-lego.owl");
-				if(local_go_lego!=null) {
-					tbox_file = new File(local_go_lego);
-					System.out.println("using local tbox "+local_go_lego);
+				OWLOntologyManager ontman = OWLManager.createOWLOntologyManager();	
+				if(catalog!=null) {
+					System.out.println("using catalog: "+catalog);
+					ontman.setIRIMappers(Sets.newHashSet(new owltools.io.CatalogXmlIRIMapper(catalog)));
 				}else {
-					URL tbox_location = new URL(url_for_tbox);
-					System.out.println("downloading tbox ontology from "+url_for_tbox);
-					org.apache.commons.io.FileUtils.copyURLToFile(tbox_location, tbox_file);
-					System.out.println("loading tbox ontology from "+tbox_file.getAbsolutePath());
-				}
-				OWLOntologyManager ontman = OWLManager.createOWLOntologyManager();					
-				OWLOntology tbox = ontman.loadOntologyFromOntologyDocument(tbox_file);
+					System.out.println("no catalog, resolving all import ontology uris directly, be patient...");
+				}				
+				OWLOntology tbox_ontology = ontman.loadOntology(IRI.create(url_for_tbox));
+				tbox_ontology = StartUpTool.forceMergeImports(tbox_ontology, tbox_ontology.getImports());
+				System.out.println("ontology axioms loaded: "+tbox_ontology.getAxiomCount());
+				System.out.println("building model manager");
+
 				System.out.println("done loading, building structural reasoner");
 				OWLReasonerFactory reasonerFactory = new StructuralReasonerFactory();
-				OWLReasoner tbox_reasoner = reasonerFactory.createReasoner(tbox);
+				OWLReasoner tbox_reasoner = reasonerFactory.createReasoner(tbox_ontology);
+				validator = new ShexValidator(shexpath, shapemappath, tbox_reasoner);
 				enrich = new Enricher(null, tbox_reasoner);
+			}else {
+				validator = new ShexValidator(shexpath, shapemappath, null);
 			}
 			for(String name : name_model.keySet()) {
 				Model test_model = name_model.get(name);
-				if(addSuperClasses||addSuperClassesLocal) {
+				if(addSuperClasses) {
 					test_model = enrich.enrichSuperClasses(test_model);
 				}
 				if(validator.GoQueryMap!=null){

minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java

@@ -597,8 +597,9 @@ public void addPostLoadOntologyFilter(PostLoadOntologyFilter filter) {
 	 * @throws IOException
 	 * @throws RepositoryException
 	 */
-	public void importModelToDatabase(File file, boolean skipMarkedDelete) throws OWLOntologyCreationException, RepositoryException, IOException, RDFParseException, RDFHandlerException {
+	public String importModelToDatabase(File file, boolean skipMarkedDelete) throws OWLOntologyCreationException, RepositoryException, IOException, RDFParseException, RDFHandlerException {
 		synchronized(repo) {
+			String modeliri = null;
 			final BigdataSailRepositoryConnection connection = repo.getUnisolatedConnection();
 			try {
 				connection.begin();
@@ -617,6 +618,7 @@ public void importModelToDatabase(File file, boolean skipMarkedDelete) throws OW
 							//FIXME Turtle format is hard-coded here
 							connection.add(file, "", RDFFormat.TURTLE, graph);
 							connection.commit();
+							modeliri = graph.toString();
 						} else {
 							throw new OWLOntologyCreationException("Detected anonymous ontology; must have IRI");
 						}
@@ -628,7 +630,9 @@ public void importModelToDatabase(File file, boolean skipMarkedDelete) throws OW
 			} finally {
 				connection.close();
 			}
+			return modeliri;
 		}
+
 	}
 
 	/**