SampleRank.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie._\n",
      "import cc.factorie.directed._\n",
      "import la.{DenseTensor2, Tensor2, DenseTensor1, Tensor1, Tensor}\n",
      "import cc.factorie.directed.{MaximizeMultivariateGaussianCovariance, MaximizeMultivariateGaussianMean, MultivariateGaussian, Gaussian}\n",
      "//import collection.mutable.{Map}\n",
      "import collection.mutable.{ArrayBuffer}\n",
      "import cc.factorie.variable.{CategoricalDomain, CategoricalVariable, DoubleVariable, MutableDiscreteVar}\n",
      "import cc.factorie.model.{Parameters, Model, Factor, DotFamilyWithStatistics1, DotFamilyWithStatistics2, Weights2, WeightsMap,WeightsSet}\n",
      "import cc.factorie.la.{DenseTensor1, DenseTensor2}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 137
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "implicit val random = scala.util.Random"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 138,
       "text": [
        "scala.util.Random$@45297e"
       ]
      }
     ],
     "prompt_number": 138
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "object spinDomain extends CategoricalDomain[Char](List('a', 'r', 'n', 'd', 'c', 'q', 'e', 'g', 'h', 'i', 'l', 'k', 'm', 'f', 'p', 's', 't', 'w', 'y', 'v', '-'))  "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 139,
       "text": [
        "CategoricalDomain[](21)"
       ]
      }
     ],
     "prompt_number": 139
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class GREMLINWeightGenerator(\n",
      "    val domain: CategoricalDomain[Char], \n",
      "    val numSites: Int, \n",
      "    val edgeProbability: Double,\n",
      "    val loopFree: Boolean,\n",
      "    val gamma_shape: Double = 1,\n",
      "    val gamma_scale: Double = 2,\n",
      "    val localMean: Double = 0,\n",
      "    val localVariance: Double = 1.0,\n",
      "    val pairwiseMean: Double = 0,\n",
      "    val pairwiseVariance: Double = 3.0)(implicit val random: scala.util.Random)  {\n",
      "    \n",
      "    \n",
      "    //allEdges contains all permutations of our\n",
      "    val allEdges = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield (i, j)).toSet\n",
      "    val activeEdges = if(loopFree) {\n",
      "                                    drawFromSetLoopFree(edgeProbability, allEdges)\n",
      "                      } else {\n",
      "                              drawFromSet(edgeProbability, allEdges)\n",
      "                      }\n",
      "                              \n",
      "    \n",
      "    def drawFromSet[A](p: Double, s: Set[A]): Set[A] = {\n",
      "        val drawn = for(i <- s if random.nextDouble < p) yield i\n",
      "        drawn\n",
      "    }\n",
      "    \n",
      "    def drawFromSetLoopFree(p: Double, s: Set[(Int, Int)]): Set[(Int, Int)] = {\n",
      "        val drawn: ArrayBuffer[(Int, Int)] = new ArrayBuffer\n",
      "        var already_connected: Set[Int] = Set()\n",
      "        for(e <- s) {\n",
      "            if(random.nextDouble < 0.3) {\n",
      "                if(!((already_connected contains e._1) && (already_connected contains e._2))) {\n",
      "                    drawn += e\n",
      "                    already_connected += e._1\n",
      "                    already_connected += e._2\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "        drawn.toSet\n",
      "    } \n",
      "    \n",
      "    def generateLocalMassesGamma: IndexedSeq[DenseTensor1] = {\n",
      "        implicit val model = DirectedModel()\n",
      "        \n",
      "        val shape = new DoubleVariable(gamma_shape)\n",
      "        val scale = new DoubleVariable(gamma_scale)\n",
      "        \n",
      "        val localMasses = for(i <- 0 until numSites) yield new DenseTensor1(domain.size)\n",
      "        for(massTensor <- localMasses){\n",
      "            for(i <- 0 until massTensor.dim1){\n",
      "                val mass = new DoubleVariable :~ Gamma(shape, scale)\n",
      "                massTensor(i) = mass.value\n",
      "            }\n",
      "        }\n",
      "        localMasses\n",
      "    }\n",
      "    \n",
      "    def generateLocalMassesNormal: IndexedSeq[DenseTensor1] = {\n",
      "        implicit val model = DirectedModel()\n",
      "        \n",
      "        val mean = new DoubleVariable(localMean)\n",
      "        val variance = new DoubleVariable(localVariance)\n",
      "        \n",
      "        val localMasses = for(i <- 0 until numSites) yield new DenseTensor1(domain.size)\n",
      "        for(massTensor <- localMasses){\n",
      "            for(i <- 0 until massTensor.dim1){\n",
      "                val mass = new DoubleVariable :~ Gaussian(mean, variance)\n",
      "                massTensor(i) = mass.value\n",
      "            }\n",
      "        }\n",
      "        localMasses\n",
      "    }\n",
      "    \n",
      "    def generatePairwiseMassesNormal: Map[(Int, Int), DenseTensor2] = {\n",
      "        val pairMasses: Map[(Int, Int), DenseTensor2] =(for(e <- activeEdges) yield (e -> generatePairwiseTensorNormal)).toMap\n",
      "        pairMasses\n",
      "    }\n",
      "    \n",
      "    def generatePairwiseTensorNormal: DenseTensor2 = {\n",
      "        implicit val model = DirectedModel()\n",
      "        \n",
      "        val mean = new DoubleVariable(pairwiseMean)\n",
      "        val variance = new DoubleVariable(pairwiseVariance)\n",
      "        \n",
      "        val tensor = new DenseTensor2(domain.size, domain.size)\n",
      "        for(a1 <- 0 until domain.size; a2 <- 0 until domain.size if a1 <= a2) {\n",
      "        //for(a1 <- 0 until domain.size; a2 <- 0 until domain.size) {\n",
      "            val mass = new DoubleVariable :~ Gaussian(mean, variance)\n",
      "            //our masses are symmetric, as we only have one factor for (i, j) and (j, i)\n",
      "            tensor(a1, a2) = mass.value\n",
      "            tensor(a2, a1) = mass.value\n",
      "        }\n",
      "        tensor\n",
      "    }\n",
      "        \n",
      "    def generatePairwiseMassesGamma: Map[(Int, Int), DenseTensor2] = {\n",
      "        val pairMasses: Map[(Int, Int), DenseTensor2] =(for(e <- activeEdges) yield (e -> generatePairwiseTensorGamma)).toMap\n",
      "        pairMasses\n",
      "    }\n",
      "    \n",
      "    def generatePairwiseTensorGamma: DenseTensor2 = {\n",
      "        implicit val model = DirectedModel()\n",
      "        \n",
      "        val shape = new DoubleVariable(gamma_shape)\n",
      "        val scale = new DoubleVariable(gamma_scale)\n",
      "        \n",
      "        val tensor = new DenseTensor2(domain.size, domain.size)\n",
      "        for(a1 <- 0 until domain.size; a2 <- 0 until domain.size if a1 <= a2) {\n",
      "            val mass = new DoubleVariable :~ Gamma(shape, scale)\n",
      "            //our weights are \n",
      "            tensor(a1, a2) = mass.value\n",
      "            tensor(a2, a1) = mass.value\n",
      "        }\n",
      "        tensor\n",
      "    }\n",
      "}\n",
      "\n",
      "object GREMLINWeightGenerator {\n",
      "    def apply(domain: CategoricalDomain[Char], n: Int, p: Double, lf: Boolean) = new GREMLINWeightGenerator(domain, n, p, lf)\n",
      "    def apply(domain: CategoricalDomain[Char], p: Double, lf: Boolean) = new GREMLINWeightGenerator(domain, 10, p, lf)\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 140,
       "text": [
        "GREMLINWeightGenerator$@16446942"
       ]
      }
     ],
     "prompt_number": 140
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class Spin(c: Char, val i: Int, d: CategoricalDomain[Char]) extends CategoricalVariable[Char](c){\n",
      "    \n",
      "    //This is how FACTORIE assigns domains to variables.\n",
      "    def domain = d\n",
      "    \n",
      "    //This is our own construct, to enable a spin to know what sample it is part of\n",
      "    var cont: IndexedSeq[Spin] = null\n",
      "    def setContainer(container: IndexedSeq[Spin]): Unit = {this.cont = container}\n",
      "    \n",
      "    override def toString = this.i.toString + \": \" + this.value.toString  \n",
      "}\n",
      "\n",
      "//Scalas companion objects allow for convenient Factory methods\n",
      "object Spin {\n",
      "    \n",
      "    //The standard factory method, removes the need to use 'new' when creating instances.\n",
      "    def apply(c: Char, i: Int, d: CategoricalDomain[Char]): Spin = new Spin(c, i, d)\n",
      "    \n",
      "    \n",
      "    //Some factory methods vor creating spin vectors ie. protein samples.\n",
      "    \n",
      "    //This factory is used for to read in a single sample from the dataset.\n",
      "    def vector(s: String, d: CategoricalDomain[Char]): IndexedSeq[Spin] = {\n",
      "        val vector = for((c, i) <- s.zipWithIndex) yield Spin(c, i, d)\n",
      "        vector.foreach(_.setContainer(vector))\n",
      "        vector\n",
      "    }\n",
      "    \n",
      "    //Initialized a vector with all spins having the same value. Used for creating fake data.\n",
      "    def vector(c: Char, numSites: Int, d: CategoricalDomain[Char]): IndexedSeq[Spin] = {\n",
      "        val vector = for(i <- 0 until numSites) yield Spin(c, i, d)\n",
      "        vector.foreach(_.setContainer(vector))\n",
      "        vector\n",
      "    }\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 141,
       "text": [
        "Spin$@2d97d5cc"
       ]
      }
     ],
     "prompt_number": 141
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class PottsModel(\n",
      "    val domain: CategoricalDomain[Char], \n",
      "    val localMasses: IndexedSeq[DenseTensor1],\n",
      "    val pairwiseMasses: Map[(Int, Int), DenseTensor2])\n",
      "    extends Model with Parameters {\n",
      "    \n",
      "    /* We initialize our 'families' which contain an inner class to create factors on the fly.\n",
      "     * There is a 1:1 relationship between the factors in our model and the families.\n",
      "     * Wether or not two spins are connected is not defined in this class, it simply creates Families\n",
      "     * for all the tensors it has received in its constructor argumetn pairwiseMasses. Those are created in a \n",
      "     * factory method in the companion object below.\n",
      "     */\n",
      "    val localFamilies: IndexedSeq[DotFamilyWithStatistics1[Spin]] = \n",
      "        for(m <- localMasses) yield new DotFamilyWithStatistics1[Spin] { val weights = Weights(m) }\n",
      "                                       \n",
      "    val pairwiseFamilies: Map[(Int, Int), DotFamilyWithStatistics2[Spin, Spin]] =\n",
      "        for((k, m) <- pairwiseMasses) yield (k -> new DotFamilyWithStatistics2[Spin, Spin] { val weights = Weights(m) })\n",
      "    \n",
      "    \n",
      "    /* The essantial method that has to be implemented by a any model. Here is where the 'imperative freedom' of IDF happens.\n",
      "     * We chose to include a reference of all possible neighbours in the Spin.cont member. We then iterate over them \n",
      "     * and see if our model contains a family for that neighbour, create a factor if it does and add it to the results.\n",
      "     */\n",
      "    def factors(variables: Iterable[Var]): Iterable[Factor] = {\n",
      "        val factors = new ArrayBuffer[Factor]\n",
      "        \n",
      "        //a simple cast of varriables from Var to Spin\n",
      "        val possibleNeighbours = variables collect {case s: Spin => s}\n",
      "        \n",
      "        for(s <- possibleNeighbours){\n",
      "                val localFamily = localFamilies(s.i)\n",
      "                factors += localFamily.Factor(s)\n",
      "            \n",
      "                val existingConnections = pairwiseFamilies.keys.filter((k: (Int, Int)) => k._1 == s.i || k._2 == s.i)\n",
      "                for(k <- existingConnections) {\n",
      "                    val pairwiseFamily = pairwiseFamilies(k)\n",
      "                    factors += pairwiseFamily.Factor(s.cont(k._1), s.cont(k._2))\n",
      "                }\n",
      "        }\n",
      "        factors\n",
      "    }\n",
      "}\n",
      "    \n",
      "object PottsModel {\n",
      "    \n",
      "    /* If we receive pairwises and local masses we create initialize the model. They might come from a synthetic model generator \n",
      "     * or from the other factory methods below.\n",
      "     */\n",
      "    def apply(localMasses: IndexedSeq[DenseTensor1], pairwiseMasses: Map[(Int, Int), DenseTensor2], domain: CategoricalDomain[Char]): PottsModel = {\n",
      "        new PottsModel(domain, localMasses, pairwiseMasses)\n",
      "    }\n",
      "    /* If just the number of variables and the domain is supplied, we create a completely connected model with weights initialized to zero.\n",
      "     */\n",
      "    def apply(numSites: Int, domain: CategoricalDomain[Char]): PottsModel = {\n",
      "        val localMasses = for(i <- 0 until numSites) yield new DenseTensor1(domain.size)\n",
      "        val pairwiseMasses = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield ((i, j), new DenseTensor2(domain.size, domain.size))).toMap\n",
      "        \n",
      "        new PottsModel(domain, localMasses, pairwiseMasses)\n",
      "    }\n",
      "    \n",
      "    /* This factory initializes a completely connected model, but initializes the local masses to be the the logarithms of  \n",
      "     * the local freuencies counted in a dataset. We add a pseudocount to deter limited sampling size effects.\n",
      "     */\n",
      "    def logFreqsAsLocalWeights(samples: Seq[IndexedSeq[Spin]], domain: CategoricalDomain[Char]): PottsModel = {\n",
      "        val PSEUDOCOUNT = 2.\n",
      "        assert(samples.forall(_.length == samples(0).length), \"Samples must all be of same length.\")\n",
      "        val numSites = samples(0).length\n",
      "        val numSamples = samples.toList.length\n",
      "        val model = PottsModel(numSites, domain)\n",
      "        \n",
      "        //\n",
      "        for ((family, i) <- model.localFamilies.zipWithIndex) {\n",
      "            val weightTensor = model.parameters(family.weights)\n",
      "            \n",
      "            val pseudoCountTensor = weightTensor.copy\n",
      "            for(i <- 0 until pseudoCountTensor.asArray.length) pseudoCountTensor.update(i, PSEUDOCOUNT)\n",
      "            weightTensor += pseudoCountTensor\n",
      "            \n",
      "            for(s <- samples) {\n",
      "                val factor = family.Factor(s(i))\n",
      "                weightTensor += factor.currentStatistics\n",
      "            }\n",
      "            weightTensor *= 1./(numSamples + (domain.size * PSEUDOCOUNT))\n",
      "            \n",
      "            for(i <- 0 until weightTensor.length) weightTensor.update(i, math.log(weightTensor(i)))\n",
      "        }\n",
      "        model\n",
      "    }      \n",
      "    \n",
      "    \n",
      "    /* We here abuse the tensor/factor infrastructure to collecte empirical frequencies from a dataset.\n",
      "     * The \"weights\" of the this model are then used to estimate mutual information in the Experiment class.\n",
      "     */\n",
      "    def frequenciesAsWeights(samples: Seq[IndexedSeq[Spin]], domain: CategoricalDomain[Char]): PottsModel = {\n",
      "        val PSEUDOCOUNT = 2.\n",
      "        assert(samples.forall(_.length == samples(0).length), \"Samples must all be of same length.\")\n",
      "        val numSites = samples(0).length\n",
      "        val numSamples = samples.toList.length\n",
      "        val model = PottsModel(numSites, domain)\n",
      "        \n",
      "        for ((key, family) <- model.pairwiseFamilies) {\n",
      "            val weightTensor = model.parameters(family.weights)\n",
      "            \n",
      "            //we add a pseudocount as in [Weight08]\n",
      "            val pseudoCountTensor = weightTensor.copy\n",
      "            for(i <- 0 until pseudoCountTensor.asArray.length) pseudoCountTensor.update(i, (PSEUDOCOUNT/domain.size))\n",
      "            weightTensor += pseudoCountTensor\n",
      "            \n",
      "            for(s <- samples) {\n",
      "                val factor = family.Factor(s(key._1), s(key._2))    \n",
      "                weightTensor += factor.currentStatistics\n",
      "            }\n",
      "            //we correct our denominator by domain.size to account for the pseudocount\n",
      "            weightTensor *= 1./(numSamples + (domain.size * PSEUDOCOUNT))\n",
      "            for(i <- 0 until weightTensor.length) weightTensor.update(i, weightTensor(i))\n",
      "        }\n",
      "        for ((family, i) <- model.localFamilies.zipWithIndex) {\n",
      "            val weightTensor = model.parameters(family.weights)\n",
      "            \n",
      "            val pseudoCountTensor = weightTensor.copy\n",
      "            for(i <- 0 until pseudoCountTensor.asArray.length) pseudoCountTensor.update(i, PSEUDOCOUNT)\n",
      "            weightTensor += pseudoCountTensor\n",
      "            \n",
      "            for(s <- samples) {\n",
      "                val factor = family.Factor(s(i))\n",
      "                weightTensor += factor.currentStatistics\n",
      "            }\n",
      "            weightTensor *= 1./(numSamples + (domain.size * PSEUDOCOUNT))\n",
      "            for(i <- 0 until weightTensor.length) weightTensor.update(i, weightTensor(i))\n",
      "        }\n",
      "        model\n",
      "    }      \n",
      "\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 142,
       "text": [
        "PottsModel$@33bfe5e9"
       ]
      }
     ],
     "prompt_number": 142
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 142
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val smallD = new CategoricalDomain[Char](List('a', 'b'))\n",
      "val samples = List(Spin.vector(\"baab\", smallD), Spin.vector(\"aaab\", smallD), Spin.vector(\"aaaa\", smallD) ,Spin.vector(\"aaaa\", smallD))\n",
      "val m = PottsModel.frequenciesAsWeights(samples, smallD)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 143,
       "text": [
        "PottsModel@46791f6a"
       ]
      }
     ],
     "prompt_number": 143
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m.pairwiseFamilies(0, 1).weights.value"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 144,
       "text": [
        "DenseTensor2(0.5,0.125,0.25,0.125)"
       ]
      }
     ],
     "prompt_number": 144
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.infer.{VariableSettingsSampler, InferByGibbsSampling, GibbsSampler}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      }
     ],
     "prompt_number": 145
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def generateSampleStrings(N:Int, model: PottsModel, variables: IndexedSeq[Spin], burnIn: Int = 1000, thinning: Int = 10): Seq[String] = {\n",
      "    implicit val random = new scala.util.Random(0)                                                                                                   \n",
      "    val sampler = new GibbsSampler(model)\n",
      "    val strings = new ArrayBuffer[String]\n",
      "    sampler.processAll(variables, burnIn)\n",
      "    for(i <- 0 until N) {\n",
      "        \n",
      "        //if(i > burnin || i % thinning == 0) strings += variables.map(_.value).mkString\n",
      "        sampler.processAll(variables)\n",
      "        strings += variables.map(_.value).mkString\n",
      "    }\n",
      "    strings\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 146
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "object Timer {\n",
      "    var t: Long = 0\n",
      "    def start: Unit = { t = System.currentTimeMillis }\n",
      "    def stop: Long = { System.currentTimeMillis - t}\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 147,
       "text": [
        "Timer$@4ec2c69f"
       ]
      }
     ],
     "prompt_number": 147
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.optimize._\n",
      "import cc.factorie.infer.{InferByBPLoopy, InferByGibbsSampling}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      }
     ],
     "prompt_number": 148
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "implicit val random = scala.util.Random"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 149,
       "text": [
        "scala.util.Random$@45297e"
       ]
      }
     ],
     "prompt_number": 149
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 149
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "//we add a pseudocount of 1\n",
      "class Counter(var value: Double=1) {\n",
      "\n",
      "    def divide(t: Double): Unit = {value *= 1/t}\n",
      "    override def toString: String = {value.toString}\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 150
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "//merge this mith PottsModel.frequenciesAsWeights\n",
      "/*class MutualInformation(val samples: Seq[String], domain: CategoricalDomain[Char]) {\n",
      "    val total = samples.length\n",
      "    val numSites = samples(0).length\n",
      "    val categories = domain.categories\n",
      "    \n",
      "    val localFreqs = countLocalFrequencies\n",
      "    val pairwiseFreqs = countPairwiseFrequencies\n",
      "    \n",
      "    \n",
      "    def makeLocalCountsMap: Map[Char, Counter] = (for(c <- categories) yield (c, new Counter)).toMap\n",
      "    def countLocalFrequencies: IndexedSeq[Map[Char, Counter]] = {\n",
      "        val sitewiseCountMaps: IndexedSeq[Map[Char, Counter]] = for(i <- 0 until numSites) yield makeLocalCountsMap\n",
      "        \n",
      "        for(sample <- samples; (category, i) <- sample.zipWithIndex) sitewiseCountMaps(i)(category).add(1)\n",
      "        \n",
      "        for(countMap <- sitewiseCountMaps; c <- categories) countMap(c).divide(total)\n",
      "        sitewiseCountMaps\n",
      "    }\n",
      "    \n",
      "    def makePairwiseCountsMap: Map[(Char, Char), Counter] = (for(c1 <- categories; c2 <- categories) yield ((c1, c2), new Counter)).toMap\n",
      "    def countPairwiseFrequencies: Map[(Int, Int), Map[(Char, Char), Counter]] = {\n",
      "   \n",
      "        val pairwiseCountMaps: Map[(Int, Int), Map[(Char, Char), Counter]] = (for(i <- 0 until numSites; j <- 0 until numSites) yield ((i, j), makePairwiseCountsMap)).toMap\n",
      "    \n",
      "        for(sample <- samples; (c_i, i) <- sample.zipWithIndex; (c_j, j) <- sample.zipWithIndex) pairwiseCountMaps(i, j)(c_i, c_j).add(1)\n",
      "        for(countMap <- pairwiseCountMaps.values; c_i <- categories; c_j <- categories) countMap(c_i, c_j).divide(total)\n",
      "        pairwiseCountMaps\n",
      "    }\n",
      "    \n",
      "    def MI(i: Int, j: Int): Double = { \n",
      "        val f_i = localFreqs(i)\n",
      "        val f_j = localFreqs(j)\n",
      "        val f_ij = pairwiseFreqs(i, j)\n",
      "        \n",
      "        (for(k <- categories; l <- categories) yield (f_ij(k, l).value*math.log(f_ij(k,l).value/(f_i(k).value*f_j(l).value)))).sum\n",
      "        \n",
      "    }\n",
      "    val allMIs: Map[(Int, Int), Double] = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield ((i, j), MI(i, j))).toMap\n",
      "}\n",
      "object MutualInformation {\n",
      "        def apply(samples: Seq[String], domain: CategoricalDomain[Char]) = new MutualInformation(samples, domain)\n",
      "}\n",
      "*/"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "incomplete"
       ]
      }
     ],
     "prompt_number": 151
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.infer.Sampler"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 152
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class ConnectionStrengths(val strengths: Map[(Int, Int), Double], val sourceName: String, val source: Any = null){\n",
      "        def apply(key: (Int, Int)): Double = strengths(key)\n",
      "        val sortedByStrength = strengths.toList.sortBy(_._2).reverse\n",
      "}\n",
      "object ConnectionStrengths{\n",
      "               \n",
      "        def apply(model: PottsModel, name: String) = {\n",
      "            val numSites = model.localFamilies.length\n",
      "            val allEdges = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield (i, j)).toSet\n",
      "            val families = model.pairwiseFamilies\n",
      "            var strengths: Map[(Int, Int), Double] = Map()\n",
      "            for(edge <- allEdges) {\n",
      "                if(families.keys.toSet contains edge) strengths += (edge -> fNorm(families(edge).weights.value))\n",
      "                else strengths += (edge -> 0)\n",
      "            }\n",
      "            new ConnectionStrengths(strengths, name, model)\n",
      "        }\n",
      "        \n",
      "        def MI(samples: Seq[IndexedSeq[Spin]], d: CategoricalDomain[Char], name: String = \"MI\") = {\n",
      "            val freqModel = PottsModel.frequenciesAsWeights(samples, d)\n",
      "            val numSites = freqModel.localFamilies.length\n",
      "            val allEdges = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield (i, j)).toSet\n",
      "            \n",
      "            \n",
      "            \n",
      "            def localFreqs(i: Int) = freqModel.localFamilies(i).weights.value\n",
      "            def pairwiseFreqs(i: Int, j: Int) = freqModel.pairwiseFamilies(i, j).weights.value.asInstanceOf[DenseTensor2]\n",
      "            def singleMI(i: Int, j: Int): Double = { \n",
      "                val f_i = localFreqs(i)\n",
      "                val f_j = localFreqs(j)\n",
      "                val f_ij = pairwiseFreqs(i, j)\n",
      "            (for(k <- d.categories; l <- d.categories) yield (f_ij(d.index(k), d.index(l)) * math.log(f_ij(d.index(k), d.index(l)) / (f_i(d.index(k))*f_j(d.index(l)))))).sum\n",
      "            }\n",
      "            val allMIs: Map[(Int, Int), Double] = (for(edge <- allEdges) yield (edge, singleMI(edge._1, edge._2))).toMap\n",
      "            new ConnectionStrengths(allMIs, name, freqModel)\n",
      "        }\n",
      "            \n",
      "        private def fNorm(w: Tensor): Double = {\n",
      "            val t = w.asInstanceOf[DenseTensor2]\n",
      "            def averageOverRow(t: DenseTensor2, k: Int): Double = {\n",
      "                // J_ij(k, .) k being a row\n",
      "                val row = for(l <- 0 until t.dim2) yield t(k, l)\n",
      "                row.sum/row.length\n",
      "            }    \n",
      "            def averageOverCol(t: DenseTensor2, l: Int): Double = {\n",
      "                // J_ij(., l) l being a column\n",
      "                val col = for(k <- 0 until t.dim1) yield t(k, l)\n",
      "                col.sum/col.length\n",
      "            }\n",
      "            val totalAverage: Double = t.sum/t.length\n",
      "            val transformed: DenseTensor2 = t.copy\n",
      "            for(k <- 0 until t.dim1; l <- 0 until t.dim2) {\n",
      "                transformed(k, l) = t(k, l) - averageOverRow(t, k) - averageOverCol(t, l) + totalAverage\n",
      "            }\n",
      "            math.sqrt(transformed.twoNorm)\n",
      "        }\n",
      "}\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 153,
       "text": [
        "ConnectionStrengths$@4c96c84d"
       ]
      }
     ],
     "prompt_number": 153
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val mimod = PottsModel.frequenciesAsWeights(samples, domain)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 154,
       "text": [
        "PottsModel@6d992755"
       ]
      }
     ],
     "prompt_number": 154
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class Experiment(val samples: Seq[Seq[Spin]], val details: Map[String, Any]) {\n",
      "    \n",
      "    assert(samples.forall(_.length == samples(0).length), \"Samples must all be of same length.\")\n",
      "    val numSites = samples(0).length\n",
      "    val allEdges = (for(i <- 0 until numSites; j <- 0 until numSites if i < j) yield (i, j)).toSet\n",
      "    \n",
      "    var rankings: Map[String, ConnectionStrengths] = Map()\n",
      "    def addRanking(r: ConnectionStrengths): Unit = { \n",
      "        assert(r.strengths.keys.forall(allEdges contains _), \"Result doesn't fit dataset\")                                     \n",
      "        rankings += (r.sourceName -> r) \n",
      "    } \n",
      "    \n",
      "    def TPRate(p: Int, estimatedRankingName: String, trueRankingName: String = \"true\", threshold: Double = 0): Double = {\n",
      "        val estimatedOrdering = rankings(estimatedRankingName).sortedByStrength.map(_._1)\n",
      "        val trueStrengths = rankings(trueRankingName)\n",
      "        var tp = 0.0\n",
      "        for(i <- 0 until p) if(trueStrengths(estimatedOrdering(i)) > threshold) tp += 1.0\n",
      "        tp/p\n",
      "    }\n",
      "    \n",
      "    def TPRates(maxP: Int, estimatedRankingName: String, trueRankingName: String = \"true\"): Seq[Double] = {\n",
      "        val tps = for(p <- 1 to maxP) yield TPRate(p, estimatedRankingName, trueRankingName)\n",
      "        tps.toList\n",
      "    }\n",
      "        \n",
      "    \n",
      "    \n",
      "    def printOrderedBy(name: String): Unit = {\n",
      "        assert(rankings.keys.toSet contains name, \"Ranking name not in experiment.\")\n",
      "        val order = rankings(name).sortedByStrength.map(_._1)\n",
      "        val maxStrengthStringLengths = (for(name <- rankings.keys) yield (name, rankings(name).strengths.toList.map(_._2).map(_.toString.length).max)).toMap\n",
      "        val maxEdgeStringLength = order.map(_.toString.length).max\n",
      "        \n",
      "        //print table header\n",
      "        for(i <- 0 to maxEdgeStringLength) print(\" \")\n",
      "        for(name <- rankings.keys) {\n",
      "            \n",
      "            print(name)\n",
      "            for(i <- 0 to (maxStrengthStringLengths(name) - name.length)) print(\" \")\n",
      "        }\n",
      "        println(\"\")\n",
      "        \n",
      "        //print rows\n",
      "        for(edge <- order) {\n",
      "            \n",
      "            print(edge)\n",
      "            for(i <- 0 to (maxEdgeStringLength - edge.toString.length)) print(\" \")\n",
      "            for(name <- rankings.keys) {\n",
      "                val strength = rankings(name).strengths(edge)\n",
      "                print(strength)\n",
      "                for(i <- 0 to (maxStrengthStringLengths(name) - strength.toString.length)) print(\" \")\n",
      "            }\n",
      "            println(\"\")\n",
      "        }\n",
      "    }\n",
      "        \n",
      "}\n",
      "object Experiment{\n",
      "        def apply(samples: Seq[Seq[Spin]], domain: CategoricalDomain[Char], name: String) = {\n",
      "            new Experiment(samples, Map(\"name\" -> name, \"domain\" -> domain))\n",
      "        }\n",
      "}\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 155,
       "text": [
        "Experiment$@4fe152d5"
       ]
      }
     ],
     "prompt_number": 155
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.la.WeightsMapAccumulator\n",
      "import cc.factorie.util.DoubleAccumulator\n",
      "import cc.factorie.model.DotFamily\n",
      "class ContrastiveDivergenceExampleVector[C](val contexts: Iterable[C], model: Model with Parameters, val sampler: Sampler[C], val k: Int = 1) extends Example {\n",
      "  // NOTE: this assumes that variables are set to the ground truth when this method is called\n",
      "  def accumulateValueAndGradient(value: DoubleAccumulator, gradient: WeightsMapAccumulator): Unit = {\n",
      "    require(gradient != null, \"The ContrastiveDivergenceExample needs a gradient accumulator\")\n",
      "    //val proposalDiff = new DiffList\n",
      "    //repeat(k) { proposalDiff ++= sampler.process(context) }\n",
      "    val proposalDiff = sampler.processAll(contexts, returnDiffs=true)\n",
      "    model.factorsOfFamilyClass[DotFamily](proposalDiff).foreach(f => gradient.accumulate(f.family.weights, f.currentStatistics, -1.0))\n",
      "    proposalDiff.undo()\n",
      "    model.factorsOfFamilyClass[DotFamily](proposalDiff).foreach(f => gradient.accumulate(f.family.weights, f.currentStatistics))\n",
      "  }\n",
      "}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 155
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.optimize.{ContrastiveDivergenceExample, ConstantLearningRate}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 156
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "val domain = spinDomain\n",
      "//val domain = new CategoricalDomain[Char](List('a', 'b'))\n",
      "val numSites = 20\n",
      "val connectedness = 0.3\n",
      "val g = GREMLINWeightGenerator(domain, numSites, connectedness, false)\n",
      "val localWeights = g.generateLocalMassesNormal\n",
      "val pairwiseWeights = g.generatePairwiseMassesNormal\n",
      "val trueModel = PottsModel(localWeights, pairwiseWeights, domain)\n",
      "val spins = Spin.vector('a', numSites, domain)\n",
      "val strings = generateSampleStrings(1000, trueModel, spins)\n",
      "val samples = strings.map(Spin.vector(_, domain))\n",
      "trueModel.pairwiseFamilies.keys.toList.length"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 186,
       "text": [
        "68"
       ]
      }
     ],
     "prompt_number": 186
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "samples.length"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 187,
       "text": [
        "1000"
       ]
      }
     ],
     "prompt_number": 187
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cc.factorie.optimize.AdaGradRDA"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 286
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=0.1, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))    \n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29799\n",
        "0.38235294117647056\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29983\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.4264705882352941\n",
        "0.47058823529411764\n"
       ]
      }
     ],
     "prompt_number": 194
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=0.5, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "30460\n",
        "0.4117647058823529\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29042\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.4264705882352941\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 195
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=1, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29259\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.4411764705882353\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29653\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.4264705882352941\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 196
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=10, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29924\n",
        "0.6176470588235294\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "30305\n",
        "0.4264705882352941\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 197
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=0.05, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29452\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.38235294117647056\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29750\n",
        "0.4117647058823529\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 198
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "val learnedModel = PottsModel.logFreqsAsLocalWeights(samples, domain)\n",
      "val sampler = new GibbsSampler(learnedModel)\n",
      "val samp2 = sampler.asInstanceOf[Sampler[Spin]]\n",
      "var CDexamples2: ArrayBuffer[ContrastiveDivergenceExampleVector[Spin]] = ArrayBuffer()\n",
      "for(sample <- samples) {\n",
      "     CDexamples2 += new ContrastiveDivergenceExampleVector(sample, learnedModel, samp2)\n",
      "}\n",
      "val opt2 = new AdaGradRDA(l1=0.008, numExamples=1000)\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val e = Experiment(samples,  domain, \"20, complete\")\n",
      "e.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "e.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "e.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(e.TPRate(68, \"learned\"))\n",
      "println(e.TPRate(68, \"MI\"))\n",
      "Timer.start\n",
      "Trainer.onlineTrain(learnedModel.parameters, CDexamples2, optimizer=opt2, useParallelTrainer=true)\n",
      "println(Timer.stop)\n",
      "val b = Experiment(samples,  domain, \"20, complete\")\n",
      "b.addRanking(ConnectionStrengths(learnedModel, \"learned\"))\n",
      "b.addRanking(ConnectionStrengths(trueModel, \"true\"))\n",
      "b.addRanking(ConnectionStrengths.MI(samples, domain))\n",
      "println(b.TPRate(68, \"learned\"))\n",
      "println(b.TPRate(68, \"MI\"))    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29340\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.39705882352941174\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29777\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.4117647058823529\n",
        "0.47058823529411764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": []
      }
     ],
     "prompt_number": 199
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}