Skip to content

Commit

Permalink
Memory efficiency for used names collection
Browse files Browse the repository at this point in the history
  - Build the maps in the used name relation in a mutable map,
    then use a HashMap builder in a second pass to build the
    resulting maps. In Scala 2.12.12+ or 2.13.0+, use of the
    builder is fastest as the builder mutates the structure
    internally.
  - Intern the keys and values, so we don't have a separate
    instance of `UsedName("toString", EnumSet.of(UseScope.Default)`
    for each call site.
  • Loading branch information
retronym committed Apr 28, 2020
1 parent 31e862c commit 21f554c
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ private final class AnalysisCallback(
private[this] val objectApis = new TrieMap[String, ApiInfo]
private[this] val classPublicNameHashes = new TrieMap[String, Array[NameHash]]
private[this] val objectPublicNameHashes = new TrieMap[String, Array[NameHash]]
private[this] val usedNames = new TrieMap[String, ConcurrentSet[UsedName]]
private[this] val usedNames = new RelationBuilder[String, UsedName]
private[this] val unreporteds = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[Problem]]
private[this] val reporteds = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[Problem]]
private[this] val mainClasses = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[String]]
Expand Down Expand Up @@ -367,8 +367,11 @@ private final class AnalysisCallback(
()
}

def usedName(className: String, name: String, useScopes: util.EnumSet[UseScope]) =
add(usedNames, className, UsedName(name, useScopes))
def usedName(className: String, name: String, useScopes: util.EnumSet[UseScope]) = {
usedNames.synchronized {
usedNames(className) = UsedName(name, useScopes)
}
}

override def enabled(): Boolean = options.enabled

Expand All @@ -381,12 +384,9 @@ private final class AnalysisCallback(
def getOrNil[A, B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten
def addCompilation(base: Analysis): Analysis =
base.copy(compilations = base.compilations.add(compilation))
def addUsedNames(base: Analysis): Analysis = usedNames.foldLeft(base) {
case (a, (className, names)) =>
import scala.collection.JavaConverters._
names.asScala.foldLeft(a) {
case (a, name) => a.copy(relations = a.relations.addUsedName(className, name))
}
def addUsedNames(base: Analysis): Analysis = {
assert(base.relations.names.size == 0)
base.copy(relations = base.relations.addUsedNames(usedNames.result()))
}

private def companionsWithHash(className: String): (Companions, HashAPI.Hash, HashAPI.Hash) = {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright Lightbend, Inc. and Mark Harrah
*
* Licensed under Apache License 2.0
* (http://www.apache.org/licenses/LICENSE-2.0).
*
* See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.
*/

package sbt.internal.inc

import sbt.internal.util.Relation

import scala.collection.{ immutable, mutable }

final class RelationBuilder[A <: AnyRef, B <: AnyRef]() {
private[this] val forward =
new java.util.HashMap[A, (A, mutable.Builder[B, immutable.HashSet[B]])]()
private[this] val reverse =
new java.util.HashMap[B, (B, mutable.Builder[A, immutable.HashSet[A]])]()

def update(a: A, b: B): Unit = {
val (internedB, asBuilder) =
reverse.computeIfAbsent(b, (b => (b, immutable.HashSet.newBuilder[A])))
val (internedA, bsBuilder) =
forward.computeIfAbsent(a, (a => (a, immutable.HashSet.newBuilder[B])))
asBuilder += internedA
bsBuilder += internedB
}

def +=(other: Relation[A, B]): Unit = {
for ((a, bs) <- other.forwardMap.iterator) {
for (b <- bs) {
update(a, b)
}
}
}

def result(): Relation[A, B] = {
def toImmutable[K, V](
map: java.util.HashMap[K, (K, mutable.Builder[V, immutable.HashSet[V]])]
): immutable.HashMap[K, immutable.HashSet[V]] = {
val builder = immutable.HashMap.newBuilder[K, immutable.HashSet[V]]
map.entrySet().forEach(e => builder.+=((e.getKey, e.getValue._2.result())))
builder.result()
}
Relation.make[A, B](toImmutable(forward), toImmutable(reverse))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ trait Relations {
deps: Iterable[(VirtualFileRef, String, XStamp)]
): Relations

private[inc] def addUsedName(className: String, name: UsedName): Relations
private[inc] def addUsedNames(data: Relation[String, UsedName]): Relations

/** Concatenates the two relations. Acts naively, i.e., doesn't internalize external deps on added files. */
def ++(o: Relations): Relations
Expand Down Expand Up @@ -636,17 +636,18 @@ private class MRelationsNameHashing(
productClassName = productClassName
)

override private[inc] def addUsedName(className: String, name: UsedName): Relations =
private[inc] def addUsedNames(data: Relation[String, UsedName]): Relations = {
new MRelationsNameHashing(
srcProd,
libraryDep,
libraryClassName,
internalDependencies = internalDependencies,
externalDependencies = externalDependencies,
classes,
names = names + (className, name),
names = if (names.forwardMap.isEmpty) data else names ++ data,
productClassName = productClassName
)
}

override def inheritance: ClassDependencies =
new ClassDependencies(
Expand Down

0 comments on commit 21f554c

Please sign in to comment.