Tp info

src/main/scala/org/broadinstitute/hail/Utils.scala

@@ -210,7 +210,7 @@ class RichRDD[T](val r: RDD[T]) extends AnyVal {
   def writeTable(filename: String, header: String = null) {
     if (header != null)
       writeTextFile(filename + ".header", r.sparkContext.hadoopConfiguration) {_.write(header)}
-    hadoopDelete(filename, r.sparkContext.hadoopConfiguration, true)
+    hadoopDelete(filename, r.sparkContext.hadoopConfiguration, recursive = true)
     r.saveAsTextFile(filename)
   }
 }
@@ -240,6 +240,7 @@ class RichOption[T](val o: Option[T]) extends AnyVal {
 class RichStringBuilder(val sb: mutable.StringBuilder) extends AnyVal {
   def tsvAppend[T](v: Option[T]) {
     v match {
+      case Some(d: Double) => sb.append(stringFormatDouble(d))
       case Some(x) => sb.append(x)
       case None => sb.append("NA")
     }
@@ -427,6 +428,10 @@ object Utils {
     if (!p) throw new AssertionError
   }
 
+  def stringFormatDouble(d: Double): String = {
+    d.formatted("%.4e")
+  }
+
   // FIXME Would be nice to have a version that averages three runs, perhaps even discarding an initial run. In this case the code block had better be functional!
   def printTime[T](block: => T) = {
     val timed = time(block)

src/main/scala/org/broadinstitute/hail/annotations/AnnotationSignature.scala

@@ -0,0 +1,8 @@
+package org.broadinstitute.hail.annotations
+
+abstract class AnnotationSignature {
+  def buildCaseClasses: String
+  def conversion: String
+  def getType: String
+
+}

src/main/scala/org/broadinstitute/hail/annotations/Annotations.scala

@@ -0,0 +1,160 @@
+package org.broadinstitute.hail.annotations
+
+case class Annotations[T](maps: Map[String, Map[String, T]], vals: Map[String, T]) extends Serializable {
+
+  def nAttrs: Int = {
+    var i = 0
+    maps.foreach {
+      case (id, m) =>
+        i += m.size
+    }
+    i += vals.size
+    i
+  }
+
+  def hasMap(str: String): Boolean = maps.contains(str)
+
+  def contains(str: String): Boolean = vals.contains(str.toLowerCase)
+
+  def contains(parent: String, str: String): Boolean = hasMap(parent) && maps(parent).contains(str)
+
+  def get(str: String): Option[T] = vals.get(str)
+
+  def get(parent: String, str: String): Option[T] = {
+    if (!hasMap(parent))
+      None
+    else
+      maps(parent).get(str)
+  }
+
+  def getOrElse(parent: String, str: String, default: T): T = {
+    if (!hasMap(parent) || !contains(parent, str))
+      default
+    else
+      maps(parent)(str)
+  }
+
+  def getOrElse(str: String, default: T): T = {
+    if (!contains(str))
+      default
+    else
+      vals(str)
+  }
+
+  def addMap(name: String, m: Map[String, T]): Annotations[T] = {
+    Annotations(maps
+      .-(name)
+      .+((name, m)), vals)
+  }
+
+  def addMaps(newMaps: Map[String, Map[String, T]]): Annotations[T] = {
+    Annotations(maps
+      .--(newMaps.keys)
+      .++(newMaps), vals)
+  }
+
+  def addVal(name: String, mapping: T): Annotations[T] = {
+    Annotations(maps, vals
+      .-(name)
+      .+((name, mapping)))
+  }
+
+  def addVals(newVals: Map[String, T]): Annotations[T] = {
+    Annotations(maps, vals
+      .--(newVals.keys)
+      .++(newVals))
+  }
+}
+
+object EmptyAnnotationSignatures {
+  def apply(): AnnotationSignatures = {
+    Annotations(Map.empty[String, Map[String, AnnotationSignature]], Map.empty[String, AnnotationSignature])
+  }
+}
+
+object EmptyAnnotations {
+  def apply(): AnnotationData = {
+    Annotations(Map.empty[String, Map[String, String]], Map.empty[String, String])
+  }
+}
+
+object EmptySampleAnnotations {
+  def apply(nSamples: Int): IndexedSeq[AnnotationData] = {
+    (0 until nSamples)
+      .map(i => Annotations(Map.empty[String, Map[String, String]], Map.empty[String, String]))
+  }
+}
+
+object AnnotationUtils {
+
+  def annotationToString(ar: AnyRef): String = {
+    ar match {
+      case iter: Iterable[_] => iter.map(_.toString).reduceRight(_ + ", " + _)
+      case _ => ar.toString
+    }
+  }
+
+  def parseAnnotationType(str: String): String = {
+    str match {
+      case "Flag" => "Boolean"
+      case "Integer" => "Int"
+      case "Float" => "Double"
+      case "String" => "String"
+      case _ => throw new UnsupportedOperationException("unexpected annotation type")
+    }
+  }
+}
+
+object AnnotationClassBuilder {
+
+  def signatures(sigs: AnnotationSignatures, hiddenClassName: String): String = {
+    val internalClasses = sigs.maps.map {
+      case (subclass, subMap) =>
+        s"class __${subclass}Annotations(subMap: Map[String, String]) extends Serializable {\n" +
+          subMap.map { case (k, sig) =>
+//            s"""  val $k: $kType = subMap.getOrElse("$k", \"false\").$kMethod\n"""
+            val default = getDefault(sig.getType)
+            s"""  val $k: ${sig.getType} = subMap.getOrElse("$k", "$default").${sig.conversion}\n"""
+          }
+            .foldRight[String]("")(_ + _) + "}\n"
+    }
+      .foldRight[String]("")(_ + _)
+
+    val hiddenClass = s"class ${hiddenClassName}Annotations" +
+      s"(annot: org.broadinstitute.hail.annotations.AnnotationData) extends Serializable {\n" +
+      sigs.maps.map { case (subclass, subMap) =>
+        s"""  val $subclass =  new __${subclass}Annotations(annot.maps(\"$subclass\"))\n""" }
+        .foldRight[String]("")(_ + _) +
+      sigs.vals.map { case (k, sig) =>
+        val default = getDefault(sig.getType)
+        s"""  val $k: ${sig.getType} = annot.vals.getOrElse("$k", "$default").${sig.conversion} \n"""
+      }
+        .foldRight[String]("")(_ + _) + "}\n"
+
+    "\n" + internalClasses + hiddenClass
+  }
+
+  def instantiate(exposedName: String, hiddenClassName: String): String = {
+    s"val $exposedName = new ${hiddenClassName}Annotations($hiddenClassName)\n"
+  }
+
+  def makeIndexedSeq(hiddenOutputName: String, hiddenClassName: String, hiddenAnnotationArrayName: String): String = {
+    s"val $hiddenOutputName: IndexedSeq[${hiddenClassName}Annotations] = " +
+      s"$hiddenAnnotationArrayName.map(new ${hiddenClassName}Annotations(_))\n"
+  }
+
+  val arrayRegex = """Array\[(\w+)\]""".r
+  val optionRegex = """Option\[(\w+)\]""".r
+  private def getDefault(typeStr: String): String = {
+    if (typeStr == "Int" || typeStr == "Double")
+      "0"
+    else if (typeStr == "Boolean")
+      "false"
+    else
+      typeStr match {
+        case optionRegex(subType) => "None"
+        case arrayRegex(subType) => getDefault(subType)
+        case _ => ""
+      }
+  }
+}

src/main/scala/org/broadinstitute/hail/annotations/SimpleSignature.scala

@@ -0,0 +1,11 @@
+package org.broadinstitute.hail.annotations
+
+case class SimpleSignature(scalaType: String, conversionMethod: String, description: String)
+  extends AnnotationSignature {
+
+  def buildCaseClasses: String = ""
+
+  def conversion: String = conversionMethod
+
+  def getType: String = scalaType
+}

src/main/scala/org/broadinstitute/hail/annotations/package.scala

@@ -0,0 +1,6 @@
+package org.broadinstitute.hail
+
+package object annotations {
+  type AnnotationSignatures = Annotations[AnnotationSignature]
+  type AnnotationData = Annotations[String]
+}

src/main/scala/org/broadinstitute/hail/driver/ExportGenotypes.scala

@@ -0,0 +1,76 @@
+package org.broadinstitute.hail.driver
+
+import org.broadinstitute.hail.Utils._
+import org.broadinstitute.hail.methods._
+import org.broadinstitute.hail.variant._
+import org.broadinstitute.hail.annotations._
+import org.kohsuke.args4j.{Option => Args4jOption}
+
+object ExportGenotypes extends Command {
+
+  class Options extends BaseOptions {
+
+    @Args4jOption(required = true, name = "-o", aliases = Array("--output"),
+      usage = "path of output tsv")
+    var output: String = _
+
+    @Args4jOption(required = true, name = "-c", aliases = Array("--condition"),
+      usage = "Comma-separated list of fields to be printed to tsv")
+    var condition: String = _
+
+    @Args4jOption(required = false, name = "--missing",
+      usage = "Format of missing values (Default: 'NA')")
+    var missing = "NA"
+  }
+
+  def newOptions = new Options
+
+  def name = "exportgenotypes"
+
+  def description = "Export list of sample-variant information to tsv"
+
+  def run(state: State, options: Options): State = {
+    val vds = state.vds
+
+    val cond = options.condition
+
+    val output = options.output
+
+    val vas: AnnotationSignatures = state.vds.metadata.variantAnnotationSignatures
+    val sas: AnnotationSignatures = state.vds.metadata.sampleAnnotationSignatures
+    val sa = state.vds.metadata.sampleAnnotations
+
+    val makeString: IndexedSeq[AnnotationData] => ((Variant, AnnotationData) =>
+      ((Int, Sample, Genotype) => String)) = try {
+      val cf = new ExportGenotypeEvaluator(options.condition, vas, sas, sa, options.missing)
+      cf.typeCheck()
+      cf.apply
+    }
+    catch {
+      case e: scala.tools.reflect.ToolBoxError =>
+        /* e.message looks like:
+           reflective compilation has failed:
+
+           ';' expected but '.' found. */
+        fatal("parse error in condition: " + e.message.split("\n").last)
+    }
+
+    val sampleIdsBc = state.sc.broadcast(state.vds.sampleIds)
+
+    val stringVDS = vds.mapValuesWithAll((v: Variant, va: AnnotationData, s: Int, g: Genotype) =>
+      makeString(sa)(v, va)(s, Sample(sampleIdsBc.value(s)), g))
+
+    writeTextFile(output + ".header", state.hadoopConf) { s =>
+      s.write(cond.split(",").map(_.split("\\.").last).reduceRight(_ + "\t" + _))
+      s.write("\n")
+    }
+
+    hadoopDelete(output, state.hadoopConf, recursive = true)
+
+    stringVDS.rdd
+      .flatMap { case (v, va, strings) => strings}
+      .saveAsTextFile(output)
+
+    state
+  }
+}

src/main/scala/org/broadinstitute/hail/driver/ExportSamples.scala

@@ -0,0 +1,68 @@
+package org.broadinstitute.hail.driver
+
+import org.broadinstitute.hail.Utils._
+import org.broadinstitute.hail.methods._
+import org.broadinstitute.hail.variant._
+import org.broadinstitute.hail.annotations._
+import org.kohsuke.args4j.{Option => Args4jOption}
+
+object ExportSamples extends Command {
+
+  class Options extends BaseOptions {
+
+    @Args4jOption(required = true, name = "-o", aliases = Array("--output"),
+      usage = "path of output tsv")
+    var output: String = _
+
+    @Args4jOption(required = true, name = "-c", aliases = Array("--condition"),
+      usage = "Comma-separated list of fields to be printed to tsv")
+    var condition: String = _
+
+    @Args4jOption(required = false, name = "--missing",
+      usage = "Format of missing values (Default: 'NA')")
+    var missing = "NA"
+  }
+
+  def newOptions = new Options
+
+  def name = "exportsamples"
+
+  def description = "Export list of sample information to tsv"
+
+  def run(state: State, options: Options): State = {
+    val vds = state.vds
+
+    val cond = options.condition
+
+    val output = options.output
+
+    val sas = vds.metadata.sampleAnnotationSignatures
+    val makeString: (Sample, Annotations[String]) => String = {
+      try {
+        val ese = new ExportSamplesEvaluator(cond, sas, options.missing)
+        ese.typeCheck()
+        ese.apply
+      } catch {
+        case e: scala.tools.reflect.ToolBoxError =>
+          /* e.message looks like:
+             reflective compilation has failed:
+
+             ';' expected but '.' found. */
+          fatal("parse error in condition: " + e.message.split("\n").last)
+      }
+    }
+
+    writeTextFile(output + ".header", state.hadoopConf) { s =>
+      s.write(cond.split(",").map(_.split("\\.").last).reduceRight(_ + "\t" + _))
+      s.write("\n")
+    }
+
+    hadoopDelete(output, state.hadoopConf, recursive = true)
+
+    vds.sparkContext.parallelize(vds.sampleIds.map(Sample).zip(vds.metadata.sampleAnnotations))
+      .map { case (s, sa) => makeString(s, sa)}
+      .saveAsTextFile(output)
+
+    state
+  }
+}