Merge pull request #537 from jyundt/add_gpu_support

Add gpu support

app/collins/models/AssetMeta.scala

@@ -206,9 +206,15 @@ object AssetMeta extends Schema with AnormAdapter[AssetMeta] with AssetMetaKeys
     val BaseProduct = findOrCreateFromName("BASE_PRODUCT")
     val BaseVendor = findOrCreateFromName("BASE_VENDOR")
     val BaseSerial = findOrCreateFromName("BASE_SERIAL")
+    val GpuProduct= findOrCreateFromName("GPU_PRODUCT")
+    val GpuVendor = findOrCreateFromName("GPU_VENDOR")
 
     def getValues(): Seq[AssetMeta] = {
-      Seq(BaseDescription, BaseProduct, BaseVendor, BaseSerial)
+      Seq(BaseDescription, BaseProduct, BaseVendor, BaseSerial, GpuProduct, GpuVendor)
+    }
+
+    def getLshwValues(): Set[AssetMeta] = {
+      Set(BaseDescription, BaseProduct, BaseVendor, BaseSerial, GpuProduct, GpuVendor)
     }
   }
 }

app/collins/models/LshwHelper.scala

@@ -3,6 +3,7 @@ package collins.models
 import collins.models.AssetMeta.DynamicEnum._
 import collins.models.AssetMeta.Enum._
 import collins.models.lshw.Cpu
+import collins.models.lshw.Gpu
 import collins.models.lshw.Disk
 import collins.models.lshw.Memory
 import collins.models.lshw.Nic
@@ -34,8 +35,11 @@ object LshwHelper extends CommonHelper[LshwRepresentation] {
     DiskStorageTotal
   )
 
+  override val managedDynamicTags = AssetMeta.DynamicEnum.getLshwValues()
+
   def construct(asset: Asset, lshw: LshwRepresentation): Seq[AssetMetaValue] = {
     collectCpus(asset, lshw) ++
+      collectGpus(asset, lshw) ++
       collectMemory(asset, lshw) ++
       collectNics(asset, lshw) ++
       collectDisks(asset, lshw) ++
@@ -45,11 +49,12 @@ object LshwHelper extends CommonHelper[LshwRepresentation] {
   def reconstruct(asset: Asset, assetMeta: Seq[MetaWrapper]): Reconstruction = {
     val metaMap = assetMeta.groupBy { _.getGroupId }
     val (cpus,postCpuMap) = reconstructCpu(metaMap)
-    val (memory,postMemoryMap) = reconstructMemory(postCpuMap)
+    val (gpus,postGpuMap) = reconstructGpu(postCpuMap)
+    val (memory,postMemoryMap) = reconstructMemory(postGpuMap)
     val (nics,postNicMap) = reconstructNics(postMemoryMap)
     val (disks,postDiskMap) = reconstructDisks(postNicMap)
     val (base,postBaseMap) = reconstructBase(postDiskMap)
-    (LshwRepresentation(cpus, memory, nics, disks, base.headOption.getOrElse(ServerBase())), postBaseMap.values.flatten.toSeq)
+    (LshwRepresentation(cpus, gpus, memory, nics, disks, base.headOption.getOrElse(ServerBase())), postBaseMap.values.flatten.toSeq)
   }
 
   protected def reconstructCpu(meta: Map[Int, Seq[MetaWrapper]]): FilteredSeq[Cpu] = {
@@ -72,6 +77,7 @@ object LshwHelper extends CommonHelper[LshwRepresentation] {
     }
     (cpuSeq, filteredMeta)
   }
+
   protected def collectCpus(asset: Asset, lshw: LshwRepresentation): Seq[AssetMetaValue] = {
     if (lshw.cpuCount < 1) {
       return Seq()
@@ -86,6 +92,43 @@ object LshwHelper extends CommonHelper[LshwRepresentation] {
     )
   }
 
+  protected def reconstructGpu(meta: Map[Int, Seq[MetaWrapper]]): FilteredSeq[Gpu] = {
+    val gpuSeq = meta.foldLeft(Seq[Gpu]()) { case (seq, map) =>
+      val groupId = map._1
+      val wrapSeq = map._2
+      val product = amfinder(wrapSeq, GpuProduct, _.toString, "")
+      val vendor= amfinder(wrapSeq, GpuVendor, _.toString, "")
+      if (product.isEmpty || vendor.isEmpty) {
+        seq
+      } else {
+        Gpu("", product, vendor) +: seq
+      }
+    }
+    val filteredMeta = meta.map { case(groupId, metaSeq) =>
+      val newSeq = filterNot(
+        metaSeq,
+        Set(GpuProduct.id, GpuVendor.id)
+      )
+      groupId -> newSeq
+    }
+    (gpuSeq, filteredMeta)
+  }
+
+  protected def collectGpus(asset: Asset, lshw: LshwRepresentation): Seq[AssetMetaValue] = {
+    if (lshw.gpuCount < 1) {
+      return Seq()
+    }
+    lshw.gpus.foldLeft((0,Seq[AssetMetaValue]())) { case (run,gpu) =>
+      val groupId = run._1
+      val total = run._2
+      val res: Seq[AssetMetaValue] = Seq(
+        AssetMetaValue(asset, GpuProduct.id, groupId, gpu.product.toString),
+        AssetMetaValue(asset, GpuVendor.id, groupId, gpu.vendor.toString)
+      )   
+      (groupId + 1, total ++ res)
+    }._2
+  }
+
   protected def reconstructMemory(meta: Map[Int, Seq[MetaWrapper]]): FilteredSeq[Memory] = {
     if (!meta.contains(0)) {
       return (Seq[Memory](), meta)

app/collins/models/lshw/Gpu.scala

@@ -0,0 +1,27 @@
+package collins.models.lshw
+
+import play.api.libs.json.Format
+import play.api.libs.json.JsObject
+import play.api.libs.json.JsSuccess
+import play.api.libs.json.JsValue
+import play.api.libs.json.Json
+
+object Gpu {
+
+  implicit object GpuFormat extends Format[Gpu] {
+    override def reads(json: JsValue) = JsSuccess(Gpu(
+      (json \ "DESCRIPTION").as[String],
+      (json \ "PRODUCT").as[String],
+      (json \ "VENDOR").as[String]))
+    override def writes(gpu: Gpu) = JsObject(Seq(
+      "DESCRIPTION" -> Json.toJson(gpu.description),
+      "PRODUCT" -> Json.toJson(gpu.product),
+      "VENDOR" -> Json.toJson(gpu.vendor)))
+  }
+}
+
+case class Gpu(
+    description: String, product: String, vendor: String) extends LshwAsset {
+  import Gpu._
+  override def toJsValue() = Json.toJson(this)
+}

app/collins/models/shared/CommonHelper.scala

@@ -10,6 +10,7 @@ trait CommonHelper[T] {
   type FilteredSeq[T1] = Tuple2[Seq[T1], Map[Int, Seq[MetaWrapper]]]
 
   val managedTags: Set[AssetMeta.Enum]
+  val managedDynamicTags: Set[AssetMeta] = Set()
 
   /**
    * Construct an appropriate AssetMetaValue sequence from the representation
@@ -27,7 +28,7 @@ trait CommonHelper[T] {
   def updateAsset(asset: Asset, rep: T): Boolean = {
     val mvs = construct(asset, rep)
     if (!managedTags.isEmpty) {
-      AssetMetaValue.deleteByAssetAndMetaId(asset, managedTags.map(_.id.toLong))
+      AssetMetaValue.deleteByAssetAndMetaId(asset, ((managedTags.map(_.id.toLong) ++ managedDynamicTags.map(_.id))))
     }
 
     mvs.size == AssetMetaValue.create(mvs)

app/collins/util/LshwRepresentation.scala

@@ -8,6 +8,8 @@ import play.api.libs.json.Json
 
 import collins.models.lshw.Cpu
 import collins.models.lshw.Cpu.CpuFormat
+import collins.models.lshw.Gpu
+import collins.models.lshw.Gpu.GpuFormat
 import collins.models.lshw.Disk
 import collins.models.lshw.Disk.DiskFormat
 import collins.models.lshw.Memory
@@ -19,23 +21,26 @@ import collins.models.lshw.ServerBase.ServerbaseFormat
 
 object LshwRepresentation {
   def empty(): LshwRepresentation = {
-    new LshwRepresentation(Seq(), Seq(), Seq(), Seq(), new ServerBase)
+    new LshwRepresentation(Seq(), Seq(), Seq(), Seq(), Seq(), new ServerBase)
   }
   implicit object LshwFormat extends Format[LshwRepresentation] {
     import Cpu._
+    import Gpu._
     import Disk._
     import Memory._
     import Nic._
     import ServerBase._
     import Json.toJson
     override def reads(json: JsValue) = JsSuccess(LshwRepresentation(
       (json \ "CPU").as[Seq[Cpu]],
+      (json \ "GPU").as[Seq[Gpu]],
       (json \ "MEMORY").as[Seq[Memory]],
       (json \ "NIC").as[Seq[Nic]],
       (json \ "DISK").as[Seq[Disk]],
       (json \ "BASE").as[ServerBase]))
     override def writes(lshw: LshwRepresentation) = JsObject(Seq(
       "CPU" -> Json.toJson(lshw.cpus),
+      "GPU" -> Json.toJson(lshw.gpus),
       "MEMORY" -> Json.toJson(lshw.memory),
       "NIC" -> Json.toJson(lshw.nics),
       "DISK" -> Json.toJson(lshw.disks),
@@ -45,6 +50,7 @@ object LshwRepresentation {
 
 case class LshwRepresentation(
     cpus: Seq[Cpu],
+    gpus: Seq[Gpu],
     memory: Seq[Memory],
     nics: Seq[Nic],
     disks: Seq[Disk],
@@ -61,6 +67,8 @@ case class LshwRepresentation(
   }
   def cpuSpeed: Double = cpus.sortBy(_.speedGhz).lastOption.map(_.speedGhz).getOrElse(0.0)
 
+  def gpuCount: Int = gpus.size
+
   def totalMemory: ByteStorageUnit = memory.foldLeft(new ByteStorageUnit(0)) {
     case (total, mem) =>
       new ByteStorageUnit(total.bytes + mem.size.bytes)

app/collins/util/config/GpuConfig.scala

@@ -0,0 +1,12 @@
+package collins.util.config
+
+object GpuConfig extends Configurable {
+  override val namespace = "gpu"
+  override val referenceConfigFilename = "gpu_reference.conf"
+
+  def supportedVendorStrings= getStringSet("supportedVendorStrings", Set("NVIDIA Corporation"))
+
+  override protected def validateConfig() {
+    supportedVendorStrings
+  }
+}

app/collins/util/parsers/LshwParser.scala

@@ -2,12 +2,14 @@ package collins.util.parsers
 
 import collins.models.lshw.LshwAsset
 import collins.models.lshw.Cpu
+import collins.models.lshw.Gpu
 import collins.models.lshw.Memory
 import collins.models.lshw.Disk
 import collins.models.lshw.Nic
 import collins.models.lshw.ServerBase
 
 import collins.util.config.LshwConfig
+import collins.util.config.GpuConfig
 import collins.util.LshwRepresentation
 import collins.util.ByteStorageUnit
 import collins.util.BitStorageUnit
@@ -25,10 +27,11 @@ class LshwParser(txt: String) extends CommonParser[LshwRepresentation](txt) {
 
   val wildcard: PartialFunction[NodeSeq, LshwAsset] = { case _ => null }
   lazy val matcher = cpuMatcher.orElse(
-    memMatcher.orElse(
-      diskMatcher.orElse(
-        nicMatcher.orElse(
-          wildcard))))
+    gpuMatcher.orElse(
+      memMatcher.orElse(
+        diskMatcher.orElse(
+          nicMatcher.orElse(
+            wildcard)))))
 
   override def parse(): Either[Throwable, LshwRepresentation] = {
     val xml = try {
@@ -40,10 +43,11 @@ class LshwParser(txt: String) extends CommonParser[LshwRepresentation](txt) {
     }
     val rep = try {
       val base = getBaseInfo(xml)
-      getCoreNodes(xml).foldLeft(LshwRepresentation(Nil, Nil, Nil, Nil, base)) {
+      getCoreNodes(xml).foldLeft(LshwRepresentation(Nil, Nil, Nil, Nil, Nil, base)) {
         case (holder, node) =>
           matcher(node) match {
             case c: Cpu    => holder.copy(cpus = c +: holder.cpus)
+            case g: Gpu    => holder.copy(gpus = g +: holder.gpus)
             case m: Memory => holder.copy(memory = m.copy(bank = holder.memory.size) +: holder.memory)
             case d: Disk   => holder.copy(disks = d +: holder.disks)
             case n: Nic    => holder.copy(nics = n +: holder.nics)
@@ -79,6 +83,13 @@ class LshwParser(txt: String) extends CommonParser[LshwRepresentation](txt) {
       Cpu(cores, threads, speed, asset.description, asset.product, asset.vendor)
   }
 
+  val gpuMatcher: PartialFunction[NodeSeq, Gpu] = {
+    case n if ((n \ "@class" text) == "display" && GpuConfig.supportedVendorStrings.exists(s => (n \\ "vendor" text).contains(s))) => {
+      val asset = getAsset(n)
+      Gpu(asset.description, asset.product, asset.vendor)
+    }
+  }
+
   val memMatcher: PartialFunction[NodeSeq, Memory] = {
     case n if (n \ "@class" text) == "memory" && (n \ "@id" text).contains("bank:") =>
       val asset = getAsset(n)

app/views/asset/show_hwdetails.scala.html

@@ -73,6 +73,26 @@ <h4>CPU <small>Collected CPU Information</small></h4>
     </tbody>
   </table>
 
+  @if(aa.lshw.gpuCount > 0) {
+  <h4>GPU <small>Collected GPU Information</small></h4>
+  <table class="table table-bordered table-hover table-condensed">
+    <thead>
+      <tr>
+        <th>Id</th><th>Description</th><th>Vendor</th>
+      </tr>
+    </thead>
+    <tbody>
+    @aa.lshw.gpus.zipWithIndex.map { case(gpu,id) =>
+      <tr>
+        <th>@id</th>
+        <td>@gpu.product</td>
+        <td>@gpu.vendor</td>
+      </tr>
+    }
+    </tbody>
+  </table>
+  }
+
   <h4>Memory <small>Collected Memory Information</small></h4>
   <table class="table table-bordered table-hover table-condensed">
     <thead>

app/views/asset/show_overview.scala.html

@@ -204,6 +204,15 @@ <h3>Hardware Summary <small>Summary of system components reported by LSHW</small
           <td>@{if (aa.lshw.hasHyperthreadingEnabled) "Yes" else "No"}</td>
         </tr>
 
+        <tr>
+          <th colspan="3">GPU</th>
+        </tr>
+        <tr>
+          <td></td>
+          <td>Total GPUs</td>
+          <td>@aa.lshw.gpuCount</td>
+        </tr>
+
         <tr>
           <th colspan="3">Memory</th>
         </tr>

conf/docker/validations.conf

@@ -15,6 +15,7 @@ config.validations = [
   collins.util.config.CryptoConfig,
   collins.util.config.IpmiConfig,
   collins.util.config.LshwConfig,
+  collins.util.config.GpuConfig,
   collins.util.config.LldpConfig,
   collins.util.config.NodeclassifierConfig,
   collins.util.power.PowerConfiguration,

conf/evolutions/collins/14.sql

@@ -0,0 +1,11 @@
+# --- Add GPU information to asset_meta
+
+# --- !Ups
+
+INSERT INTO asset_meta (name, priority, label, description) VALUES ('GPU_PRODUCT', -1, 'GPU Product', 'GPU product (description of GPU)');
+INSERT INTO asset_meta (name, priority, label, description) VALUES ('GPU_VENDOR', -1, 'GPU Vendor', 'GPU vendor');
+
+# --- !Downs
+
+DELETE FROM asset_meta WHERE name ='GPU_VENDOR'
+DELETE FROM asset_meta WHERE name ='GPU_PRODUCT'

conf/reference/gpu_reference.conf

@@ -0,0 +1,5 @@
+gpu {
+  # A list of hardware vendors that should be parsed by lshw as "gpu vendors"
+  gpuVendors = []
+
+}

conf/validations.conf

@@ -18,6 +18,7 @@ config.validations = [
   collins.util.config.LshwConfig,
   collins.util.config.LldpConfig,
   collins.util.config.NodeclassifierConfig,
+  collins.util.config.GpuConfig,
   collins.util.power.PowerConfiguration,
   collins.util.security.AuthenticationProviderConfig,
   collins.util.security.FileAuthenticationProviderConfig,

support/ruby/collins-client/lib/collins/asset.rb

@@ -193,6 +193,15 @@ def cpus
       extract(extras, "HARDWARE", "CPU") || []
     end
 
+    # @return [Fixnum] Number of GPU's found
+    def gpu_count
+      (extract(extras, "HARDWARE", "GPU") || []).length
+    end
+    # @return [Array<Hash>] GPU information
+    def gpus
+      extract(extras, "HARDWARE", "GPU") || []
+    end
+
     # @return [Array<Hash>] Disk information
     def disks
       extract(extras, "HARDWARE", "DISK") || []

support/ruby/collins-client/spec/collins/asset_spec.rb

@@ -96,6 +96,14 @@
     end
   end
 
+  context "assets with GPUs should have" do
+    subject { Collins::Asset.from_json(CollinsFixture.full_asset_with_gpu(true)) }
+
+    it "#gpu_count" do
+      subject.gpu_count.should == 2
+    end
+  end
+
   context "Update" do
     it "lshw is not an attribute" do
       ["lshw","LSHW"].each do |name|

support/ruby/collins-client/spec/fixtures/full_asset_with_gpu.json

@@ -0,0 +1 @@
+{"status":"success:ok","data":{"ASSET":{"ID":2,"TAG":"U000006","STATE":{"ID":1,"STATUS":null,"NAME":"NEW","LABEL":"New","DESCRIPTION":"A service in this state is inactive. It does minimal work and consumes minimal resources."},"STATUS":"New","TYPE":"SERVER_NODE","CREATED":"2017-04-26T16:52:36","UPDATED":"2017-04-26T16:52:38","DELETED":null},"HARDWARE":{"CPU":[{"CORES":6,"THREADS":6,"SPEED_GHZ":2.3,"DESCRIPTION":"AMD Opteron(tm) Processor 4174 HE Hynix Semiconductor (Hyundai Electronics)","PRODUCT":"","VENDOR":""},{"CORES":6,"THREADS":6,"SPEED_GHZ":2.3,"DESCRIPTION":"AMD Opteron(tm) Processor 4174 HE Hynix Semiconductor (Hyundai Electronics)","PRODUCT":"","VENDOR":""}],"GPU":[{"DESCRIPTION":"GM200GL [Quadro M6000] - NVIDIA Corporation","PRODUCT":"","VENDOR":""},{"DESCRIPTION":"GM200GL [Quadro M6000] - NVIDIA Corporation","PRODUCT":"","VENDOR":""}],"MEMORY":[{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":0,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":1,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":8589934592,"SIZE_S":"8589934592","SIZE_HUMAN":"8.00 GB","BANK":2,"DESCRIPTION":"DIMM DDR3 Synchronous 1333 MHz (0.8 ns) - Hyundai HMT31GR7BFR4A-H9","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":3,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":4,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":8589934592,"SIZE_S":"8589934592","SIZE_HUMAN":"8.00 GB","BANK":5,"DESCRIPTION":"DIMM DDR3 Synchronous 1333 MHz (0.8 ns) - Hyundai HMT31GR7BFR4A-H9","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":6,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":7,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":8589934592,"SIZE_S":"8589934592","SIZE_HUMAN":"8.00 GB","BANK":8,"DESCRIPTION":"DIMM DDR3 Synchronous 1333 MHz (0.8 ns) - Hyundai HMT31GR7BFR4A-H9","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":9,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":0,"SIZE_S":"0","SIZE_HUMAN":"0 Bytes","BANK":10,"DESCRIPTION":"Empty Memory Bank","PRODUCT":"","VENDOR":""},{"SIZE":8589934592,"SIZE_S":"8589934592","SIZE_HUMAN":"8.00 GB","BANK":11,"DESCRIPTION":"DIMM DDR3 Synchronous 1333 MHz (0.8 ns) - Hyundai HMT31GR7BFR4A-H9","PRODUCT":"","VENDOR":""}],"NIC":[],"DISK":[],"BASE":{"DESCRIPTION":"Rack Mount Chassis","PRODUCT":"PowerEdge C6105 (N/A)","VENDOR":"Winbond Electronics","SERIAL":"FZ22YQ1"}},"LLDP":{"INTERFACES":[{"NAME":"eth0","CHASSIS":{"NAME":"accessB07.corp.uhq.ua.tc","ID":{"TYPE":"mac","VALUE":"ec:3e:f7:1e:77:c0"},"DESCRIPTION":"Juniper Networks, Inc. ex4300-48p Ethernet Switch, kernel JUNOS 14.1X53-D30.3, Build date: 2015-10-02 12:40:24 UTC Copyright (c) 1996-2015 Juniper Networks, Inc."},"PORT":{"ID":{"TYPE":"local","VALUE":"529"},"DESCRIPTION":"ge-0/0/12"},"VLANS":[{"ID":40,"NAME":"vlan-40"}]}]},"IPMI":{"ASSET_ID":2,"ASSET_TAG":"U000006","IPMI_USERNAME":"root","IPMI_PASSWORD":"YCM0PCt6y37uJsHb","IPMI_GATEWAY":"172.16.32.1","IPMI_ADDRESS":"172.16.32.20","IPMI_NETMASK":"255.255.240.0","ID":2},"ADDRESSES":[],"POWER":[],"ATTRIBS":{"0":{"CHASSIS_TAG":"U000006"}}}}

support/ruby/collins-client/spec/support/collins_fixture.rb

@@ -30,6 +30,9 @@ def partial_asset_no_state json = false
   def basic_log json=false
     get_fixture_data 'basic_log.json', json
   end
+  def full_asset_with_gpu json=false
+    get_fixture_data 'full_asset_with_gpu.json', json
+  end
 
   def data name
     File.read(fixture_file(name))