From 9cad96e56856814844024cd0f643119ae8d9fd85 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Fri, 13 Dec 2019 15:40:26 +0300
Subject: [PATCH 01/12] Split libraries into separate jsons

---
 build.gradle                                  |  14 +-
 config.json                                   | 184 ------------------
 kernelspec/kernel.json.template               |   2 +-
 libraries/gral.json                           |  21 ++
 libraries/klaxon.json                         |  12 ++
 libraries/kmath.json                          |  17 ++
 libraries/koma.json                           |  17 ++
 libraries/kotlin-statistics.json              |  12 ++
 libraries/krangl.json                         |  21 ++
 libraries/kravis.json                         |  18 ++
 libraries/lets-plot.json                      |  31 +++
 libraries/spark.json                          |  46 +++++
 readme.md                                     |   8 +-
 .../org/jetbrains/kotlin/jupyter/config.kt    | 129 +++++++++++-
 .../org/jetbrains/kotlin/jupyter/ikotlin.kt   |  58 +-----
 .../org/jetbrains/kotlin/jupyter/repl.kt      |  10 +-
 .../kotlin/jupyter/test/replTests.kt          |  63 +++---
 17 files changed, 377 insertions(+), 286 deletions(-)
 delete mode 100644 config.json
 create mode 100644 libraries/gral.json
 create mode 100644 libraries/klaxon.json
 create mode 100644 libraries/kmath.json
 create mode 100644 libraries/koma.json
 create mode 100644 libraries/kotlin-statistics.json
 create mode 100644 libraries/krangl.json
 create mode 100644 libraries/kravis.json
 create mode 100644 libraries/lets-plot.json
 create mode 100644 libraries/spark.json

diff --git a/build.gradle b/build.gradle
index c62c6fde7..d93d9dbd7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -44,7 +44,6 @@ allprojects {
             project.getProperty('installPath') :
             Paths.get(System.properties['user.home'].toString(), ".ipython", "kernels", "kotlin").toAbsolutePath().toString()
     ext.debugPort = 1044
-    ext.configFile = "config.json"
     ext.debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort"
 }
 
@@ -69,6 +68,7 @@ dependencies {
 
     compile "org.apache.maven:maven-core:3.0.3"
     compile 'org.slf4j:slf4j-api:1.7.25'
+    compile "khttp:khttp:1.0.0"
     compile 'org.zeromq:jeromq:0.3.5'
     compile 'com.beust:klaxon:5.2'
     runtime 'org.slf4j:slf4j-simple:1.7.25'
@@ -117,7 +117,7 @@ void createTaskForSpecs(Boolean debug) {
             } .join(File.pathSeparator)
             spec = substitute(spec, "RUNTIME_CLASSPATH", libsCp)
             spec = substitute(spec, "DEBUGGER_CONFIG", debug ? "\"$debuggerConfig\"," : "")
-            spec = substitute(spec, "LIBRARIES_PATH", "$installPath$sep$configFile")
+            spec = substitute(spec, "KERNEL_HOME", "$installPath")
             File installDir = new File("$installPath")
             if (!installDir.exists()) {
                 installDir.mkdirs();
@@ -138,9 +138,9 @@ static String substitute(String spec, String template, String val) {
     return spec.replace("\${$template}", val.replace("\\", "\\\\"))
 }
 
-task copyLibrariesConfig(type: Copy, dependsOn: cleanInstallDir) {
-    from configFile
-    into installPath
+task copyLibraries(type: Copy, dependsOn: cleanInstallDir) {
+    from "libraries"
+    into Paths.get(installPath, "libraries").toString()
 }
 
 createTaskForSpecs(true)
@@ -151,8 +151,8 @@ task installLibs(type: Copy, dependsOn: cleanInstallDir) {
     from configurations.deploy
 }
 
-task install(dependsOn: [installKernel, installLibs, createSpecs, copyLibrariesConfig]) {
+task install(dependsOn: [installKernel, installLibs, createSpecs, copyLibraries]) {
 }
 
-task installDebug(dependsOn: [installKernel, installLibs, createDebugSpecs, copyLibrariesConfig]) {
+task installDebug(dependsOn: [installKernel, installLibs, createDebugSpecs, copyLibraries]) {
 }
diff --git a/config.json b/config.json
deleted file mode 100644
index 46c31cee6..000000000
--- a/config.json
+++ /dev/null
@@ -1,184 +0,0 @@
-{
-  "repositories": [
-    "https://jcenter.bintray.com/",
-    "https://repo.maven.apache.org/maven2/",
-    "https://jitpack.io"
-  ],
-  "libraries": [
-    {
-      "name": "klaxon(v=5.2)",
-      "dependencies": [
-        "com.beust:klaxon:$v"
-      ],
-      "imports": [
-        "com.beust.klaxon.*"
-      ],
-      "link": "https://github.com/cbeust/klaxon"
-    },
-    {
-      "name": "lets-plot",
-      "link": "https://github.com/JetBrains/lets-plot-kotlin",
-      "repositories": [
-        "https://jetbrains.bintray.com/lets-plot-maven"
-      ],
-      "dependencies": [
-        "org.jetbrains.lets-plot:lets-plot-common:1.0.1-SNAPSHOT",
-        "org.jetbrains.lets-plot:lets-plot-kotlin-api:0.0.8-SNAPSHOT",
-        "org.jetbrains.lets-plot:kotlin-frontend-api:0.0.8-SNAPSHOT",
-        "org.jetbrains.lets-plot:lets-plot-jfx:1.0.1-SNAPSHOT"
-      ],
-      "imports": [
-        "jetbrains.letsPlot.*",
-        "jetbrains.letsPlot.geom.*",
-        "jetbrains.letsPlot.stat.*"
-      ],
-      "init": [
-        "fun jetbrains.letsPlot.intern.Plot.getHtml() = jetbrains.letsPlot.intern.frontendContext.FrontendContextUtil.getHtml(this)",
-        "DISPLAY(HTML(jetbrains.datalore.jupyter.configureScript()))"
-      ],
-      "renderers": [
-        {
-          "class": "jetbrains.letsPlot.intern.Plot",
-          "result": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())"
-        }
-      ]
-    },
-    {
-      "name": "krangl(v=-SNAPSHOT)",
-      "dependencies": [
-        "com.github.holgerbrandl:krangl:$v"
-      ],
-      "imports": [
-        "krangl.*"
-      ],
-      "init": [
-        "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"<html><body>\")\nsb.append(\"<table><tr>\")\ncols.forEach { sb.append(\"<th style=\\\"text-align:left\\\">${it.name}</th>\") }\nsb.append(\"</tr>\")\nrows.take(limit).forEach {\n    sb.append(\"<tr>\")\n    it.values.map{it.toString()}.forEach { \n        val truncated = if (truncate > 0 && it.length > truncate) {\n                if (truncate < 4) it.substring(0, truncate)\n                else it.substring(0, truncate - 3) + \"...\"\n            } else {\n                it\n            }\n        sb.append(\"\"\"<td style=\"text-align:left\" title=\"$it\">$truncated</td>\"\"\") \n    }\n    sb.append(\"</tr>\")\n}\nsb.append(\"</table>\")\nif(limit < rows.count())\n  sb.append(\"<p>... only showing top $limit rows</p>\")\nsb.append(\"</body></html>\")\nreturn sb.toString()}"
-      ],
-      "renderers": [
-        {
-          "class": "krangl.SimpleDataFrame",
-          "result": "HTML($it.toHTML())"
-        }
-      ],
-      "link": "https://github.com/holgerbrandl/krangl"
-    },
-    {
-      "name": "kotlin-statistics(v=-SNAPSHOT)",
-      "dependencies": [
-        "com.github.thomasnield:kotlin-statistics:$v"
-      ],
-      "imports": [
-        "org.nield.kotlinstatistics.*"
-      ],
-      "link": "https://github.com/thomasnield/kotlin-statistics"
-    },
-    {
-      "name": "kravis(v=-SNAPSHOT)",
-      "dependencies": [
-        "com.github.holgerbrandl:kravis:$v"
-      ],
-      "imports": [
-        "kravis.*"
-      ],
-      "renderers": [
-        {
-          "class": "kravis.GGPlot",
-          "result": "$it.show()"
-        }
-      ],
-      "link": "https://github.com/holgerbrandl/kravis"
-    },
-    {
-      "name": "spark(scala=2.11.12,spark=2.4.4)",
-      "dependencies": [
-        "org.apache.spark:spark-mllib_2.11:$spark",
-        "org.apache.spark:spark-sql_2.11:$spark",
-        "org.apache.spark:spark-repl_2.11:$spark",
-        "org.apache.spark:spark-streaming-flume-assembly_2.11:$spark",
-        "org.apache.spark:spark-graphx_2.11:$spark",
-        "org.apache.spark:spark-launcher_2.11:$spark",
-        "org.apache.spark:spark-catalyst_2.11:$spark",
-        "org.apache.spark:spark-streaming_2.11:$spark",
-        "org.apache.spark:spark-core_2.11:$spark",
-        "org.scala-lang:scala-library:$scala",
-        "org.scala-lang:scala-reflect:$scala",
-        "org.scala-lang:scala-compiler:$scala",
-        "org.scala-lang.modules:scala-xml_2.11:1.2.0",
-        "commons-io:commons-io:2.5"
-      ],
-      "imports": [
-        "org.apache.spark.sql.*",
-        "org.apache.spark.api.java.*",
-        "org.apache.spark.ml.feature.*",
-        "org.apache.spark.sql.functions.*"
-      ],
-      "init": [
-        "org.apache.log4j.Logger.getLogger(\"org\").setLevel(org.apache.log4j.Level.OFF)",
-        "org.apache.log4j.Logger.getLogger(\"akka\").setLevel(org.apache.log4j.Level.OFF)",
-        "val spark = SparkSession\n        .builder()\n        .appName(\"Spark example\")\n        .master(\"local\")\n        .getOrCreate()",
-        "val sc = spark.sparkContext()",
-        "%dumpClassesForSpark",
-        "fun Dataset<Row>.toHTML(limit: Int = 20, truncate: Int = 50): String {\n    val sb = StringBuilder()\n\n    sb.append(\"<html><body>\")\n    sb.append(\"\"\"<table><tr>\"\"\")\n    sb.append(schema().fieldNames().map { \"<th style=\\\"text-align:left\\\">${it}</th>\"}.joinToString(\"\"))\n    sb.append(\"</tr>\")\n\n    limit(limit).collectAsList().forEach { row ->\n        sb.append(\"<tr>\")\n        (0 until row.size()).map {\n            row[it].toString()\n        }.forEach {\n            val truncated = if (truncate > 0 && it.length > truncate) {\n                if (truncate < 4) it.substring(0, truncate)\n                else it.substring(0, truncate - 3) + \"...\"\n            } else {\n                it\n            }\n            sb.append(\"\"\"<td style=\"text-align:left\" title=\"$it\">$truncated</td>\"\"\")\n        }\n        sb.append(\"</tr>\")\n    }\n    sb.append(\"</table>\")\n    if(limit < count())\n        sb.append(\"<p>... only showing top $limit rows</p>\")\n    sb.append(\"</body></html>\")\n    return sb.toString()\n}"
-      ],
-      "initCell": [
-        "scala.Console.setOut(System.out)",
-        "scala.Console.setErr(System.err)"
-      ],
-      "renderers": [
-        {
-          "class": "org.apache.spark.sql.Dataset",
-          "result": "HTML($it.toHTML())"
-        }
-      ]
-    },
-    {
-      "name": "gral(v=0.11)",
-      "link": "https://github.com/eseifert/gral",
-      "dependencies": [
-        "de.erichseifert.gral:gral-core:$v"
-      ],
-      "imports": [
-        "de.erichseifert.gral.data.*",
-        "de.erichseifert.gral.data.filters.*",
-        "de.erichseifert.gral.graphics.*",
-        "de.erichseifert.gral.plots.*",
-        "de.erichseifert.gral.plots.lines.*",
-        "de.erichseifert.gral.plots.points.*",
-        "de.erichseifert.gral.util.*"
-      ],
-      "init": [
-        "fun<T: Drawable> T.show(sizeX: Double, sizeY: Double): Any {\n    val writer = de.erichseifert.gral.io.plots.DrawableWriterFactory.getInstance().get(\"image/svg+xml\")\n\n    val buf = java.io.ByteArrayOutputStream()\n\n    writer.write(this, buf, sizeX, sizeY)\n\n    return MIME(writer.mimeType to buf.toString())\n}"
-      ]
-    },
-    {
-      "name": "kmath(v=0.1.3)",
-      "link": "https://github.com/mipt-npm/kmath",
-      "repositories": [
-        "https://dl.bintray.com/mipt-npm/scientifik"
-      ],
-      "dependencies": [
-        "scientifik:kmath-core-jvm:$v"
-      ],
-      "imports": [
-        "scientifik.kmath.linear.*",
-        "scientifik.kmath.operations.*",
-        "scientifik.kmath.structures.*"
-      ]
-    },
-    {
-      "name": "koma(v=0.12)",
-      "link": "https://koma.kyonifer.com/index.html",
-      "repositories": [
-        "https://dl.bintray.com/kyonifer/maven"
-      ],
-      "dependencies": [
-        "com.kyonifer:koma-core-ejml:$v",
-        "com.kyonifer:koma-plotting:$v"
-      ],
-      "imports": [
-        "koma.*",
-        "koma.extensions.*"
-      ]
-    }
-  ]
-}
diff --git a/kernelspec/kernel.json.template b/kernelspec/kernel.json.template
index 8660d2f16..fdf4a8922 100644
--- a/kernelspec/kernel.json.template
+++ b/kernelspec/kernel.json.template
@@ -1,5 +1,5 @@
 {
-  "argv": [ "java", "-jar", ${DEBUGGER_CONFIG} "${KERNEL_JAR_PATH}", "{connection_file}", "-cp=${RUNTIME_CLASSPATH}", "-libs=${LIBRARIES_PATH}"],
+  "argv": [ "java", "-jar", ${DEBUGGER_CONFIG} "${KERNEL_JAR_PATH}", "{connection_file}", "-cp=${RUNTIME_CLASSPATH}", "-home=${KERNEL_HOME}"],
   "display_name": "Kotlin",
   "language": "kotlin"
 }
\ No newline at end of file
diff --git a/libraries/gral.json b/libraries/gral.json
new file mode 100644
index 000000000..b23a241d0
--- /dev/null
+++ b/libraries/gral.json
@@ -0,0 +1,21 @@
+{
+  "arguments": [
+    "v=0.11"
+  ],
+  "link": "https://github.com/eseifert/gral",
+  "dependencies": [
+    "de.erichseifert.gral:gral-core:$v"
+  ],
+  "imports": [
+    "de.erichseifert.gral.data.*",
+    "de.erichseifert.gral.data.filters.*",
+    "de.erichseifert.gral.graphics.*",
+    "de.erichseifert.gral.plots.*",
+    "de.erichseifert.gral.plots.lines.*",
+    "de.erichseifert.gral.plots.points.*",
+    "de.erichseifert.gral.util.*"
+  ],
+  "init": [
+    "fun<T: Drawable> T.show(sizeX: Double, sizeY: Double): Any {\n    val writer = de.erichseifert.gral.io.plots.DrawableWriterFactory.getInstance().get(\"image/svg+xml\")\n\n    val buf = java.io.ByteArrayOutputStream()\n\n    writer.write(this, buf, sizeX, sizeY)\n\n    return MIME(writer.mimeType to buf.toString())\n}"
+  ]
+}
diff --git a/libraries/klaxon.json b/libraries/klaxon.json
new file mode 100644
index 000000000..0965d18c5
--- /dev/null
+++ b/libraries/klaxon.json
@@ -0,0 +1,12 @@
+{
+  "arguments": [
+    "v=5.2"
+  ],
+  "link": "https://github.com/cbeust/klaxon",
+  "dependencies": [
+    "com.beust:klaxon:$v"
+  ],
+  "imports": [
+    "com.beust.klaxon.*"
+  ]
+}
diff --git a/libraries/kmath.json b/libraries/kmath.json
new file mode 100644
index 000000000..1a747430b
--- /dev/null
+++ b/libraries/kmath.json
@@ -0,0 +1,17 @@
+{
+  "arguments": [
+    "v=0.1.3"
+  ],
+  "link": "https://github.com/mipt-npm/kmath",
+  "repositories": [
+    "https://dl.bintray.com/mipt-npm/scientifik"
+  ],
+  "dependencies": [
+    "scientifik:kmath-core-jvm:$v"
+  ],
+  "imports": [
+    "scientifik.kmath.linear.*",
+    "scientifik.kmath.operations.*",
+    "scientifik.kmath.structures.*"
+  ]
+}
diff --git a/libraries/koma.json b/libraries/koma.json
new file mode 100644
index 000000000..c01ac2647
--- /dev/null
+++ b/libraries/koma.json
@@ -0,0 +1,17 @@
+{
+  "arguments": [
+    "v=0.12"
+  ],
+  "link": "http://koma.kyonifer.com/index.html",
+  "repositories": [
+    "https://dl.bintray.com/kyonifer/maven"
+  ],
+  "dependencies": [
+    "com.kyonifer:koma-core-ejml:$v",
+    "com.kyonifer:koma-plotting:$v"
+  ],
+  "imports": [
+    "koma.*",
+    "koma.extensions.*"
+  ]
+}
diff --git a/libraries/kotlin-statistics.json b/libraries/kotlin-statistics.json
new file mode 100644
index 000000000..1de0c9ecd
--- /dev/null
+++ b/libraries/kotlin-statistics.json
@@ -0,0 +1,12 @@
+{
+  "arguments": [
+    "v=-SNAPSHOT"
+  ],
+  "link": "https://github.com/thomasnield/kotlin-statistics",
+  "dependencies": [
+    "com.github.thomasnield:kotlin-statistics:$v"
+  ],
+  "imports": [
+    "org.nield.kotlinstatistics.*"
+  ]
+}
diff --git a/libraries/krangl.json b/libraries/krangl.json
new file mode 100644
index 000000000..301ca1f4b
--- /dev/null
+++ b/libraries/krangl.json
@@ -0,0 +1,21 @@
+{
+  "arguments": [
+    "v=-SNAPSHOT"
+  ],
+  "link": "https://github.com/holgerbrandl/krangl",
+  "dependencies": [
+    "com.github.holgerbrandl:krangl:$v"
+  ],
+  "imports": [
+    "krangl.*"
+  ],
+  "init": [
+    "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"<html><body>\")\nsb.append(\"<table><tr>\")\ncols.forEach { sb.append(\"<th style=\\\"text-align:left\\\">${it.name}</th>\") }\nsb.append(\"</tr>\")\nrows.take(limit).forEach {\n    sb.append(\"<tr>\")\n    it.values.map{it.toString()}.forEach { \n        val truncated = if (truncate > 0 && it.length > truncate) {\n                if (truncate < 4) it.substring(0, truncate)\n                else it.substring(0, truncate - 3) + \"...\"\n            } else {\n                it\n            }\n        sb.append(\"\"\"<td style=\"text-align:left\" title=\"$it\">$truncated</td>\"\"\") \n    }\n    sb.append(\"</tr>\")\n}\nsb.append(\"</table>\")\nif(limit < rows.count())\n  sb.append(\"<p>... only showing top $limit rows</p>\")\nsb.append(\"</body></html>\")\nreturn sb.toString()}"
+  ],
+  "renderers": [
+    {
+      "class": "krangl.SimpleDataFrame",
+      "result": "HTML($it.toHTML())"
+    }
+  ]
+}
diff --git a/libraries/kravis.json b/libraries/kravis.json
new file mode 100644
index 000000000..0b3af5125
--- /dev/null
+++ b/libraries/kravis.json
@@ -0,0 +1,18 @@
+{
+  "arguments": [
+    "v=-SNAPSHOT"
+  ],
+  "link": "https://github.com/holgerbrandl/kravis",
+  "dependencies": [
+    "com.github.holgerbrandl:kravis:$v"
+  ],
+  "imports": [
+    "kravis.*"
+  ],
+  "renderers": [
+    {
+      "class": "kravis.GGPlot",
+      "result": "$it.show()"
+    }
+  ]
+}
diff --git a/libraries/lets-plot.json b/libraries/lets-plot.json
new file mode 100644
index 000000000..62b516aab
--- /dev/null
+++ b/libraries/lets-plot.json
@@ -0,0 +1,31 @@
+{
+  "arguments": [
+    "core=1.0.1-SNAPSHOT",
+    "kotlin=0.0.8-SNAPSHOT"
+  ],
+  "link": "https://github.com/JetBrains/lets-plot-kotlin",
+  "repositories": [
+    "https://jetbrains.bintray.com/lets-plot-maven"
+  ],
+  "dependencies": [
+    "org.jetbrains.lets-plot:lets-plot-common:$core",
+    "org.jetbrains.lets-plot:lets-plot-kotlin-api:$kotlin",
+    "org.jetbrains.lets-plot:kotlin-frontend-api:$kotlin",
+    "org.jetbrains.lets-plot:lets-plot-jfx:$core"
+  ],
+  "imports": [
+    "jetbrains.letsPlot.*",
+    "jetbrains.letsPlot.geom.*",
+    "jetbrains.letsPlot.stat.*"
+  ],
+  "init": [
+    "fun jetbrains.letsPlot.intern.Plot.getHtml() = jetbrains.letsPlot.intern.frontendContext.FrontendContextUtil.getHtml(this)",
+    "DISPLAY(HTML(jetbrains.datalore.jupyter.configureScript()))"
+  ],
+  "renderers": [
+    {
+      "class": "jetbrains.letsPlot.intern.Plot",
+      "result": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())"
+    }
+  ]
+}
diff --git a/libraries/spark.json b/libraries/spark.json
new file mode 100644
index 000000000..49549b570
--- /dev/null
+++ b/libraries/spark.json
@@ -0,0 +1,46 @@
+{
+  "arguments": [
+    "scala=2.11.12",
+    "spark=2.4.4"
+  ],
+  "dependencies": [
+    "org.apache.spark:spark-mllib_2.11:$spark",
+    "org.apache.spark:spark-sql_2.11:$spark",
+    "org.apache.spark:spark-repl_2.11:$spark",
+    "org.apache.spark:spark-streaming-flume-assembly_2.11:$spark",
+    "org.apache.spark:spark-graphx_2.11:$spark",
+    "org.apache.spark:spark-launcher_2.11:$spark",
+    "org.apache.spark:spark-catalyst_2.11:$spark",
+    "org.apache.spark:spark-streaming_2.11:$spark",
+    "org.apache.spark:spark-core_2.11:$spark",
+    "org.scala-lang:scala-library:$scala",
+    "org.scala-lang:scala-reflect:$scala",
+    "org.scala-lang:scala-compiler:$scala",
+    "org.scala-lang.modules:scala-xml_2.11:1.2.0",
+    "commons-io:commons-io:2.5"
+  ],
+  "imports": [
+    "org.apache.spark.sql.*",
+    "org.apache.spark.api.java.*",
+    "org.apache.spark.ml.feature.*",
+    "org.apache.spark.sql.functions.*"
+  ],
+  "init": [
+    "org.apache.log4j.Logger.getLogger(\"org\").setLevel(org.apache.log4j.Level.OFF)",
+    "org.apache.log4j.Logger.getLogger(\"akka\").setLevel(org.apache.log4j.Level.OFF)",
+    "val spark = SparkSession\n        .builder()\n        .appName(\"Spark example\")\n        .master(\"local\")\n        .getOrCreate()",
+    "val sc = spark.sparkContext()",
+    "%dumpClassesForSpark",
+    "fun Dataset<Row>.toHTML(limit: Int = 20, truncate: Int = 50): String {\n    val sb = StringBuilder()\n\n    sb.append(\"<html><body>\")\n    sb.append(\"\"\"<table><tr>\"\"\")\n    sb.append(schema().fieldNames().map { \"<th style=\\\"text-align:left\\\">${it}</th>\"}.joinToString(\"\"))\n    sb.append(\"</tr>\")\n\n    limit(limit).collectAsList().forEach { row ->\n        sb.append(\"<tr>\")\n        (0 until row.size()).map {\n            row[it].toString()\n        }.forEach {\n            val truncated = if (truncate > 0 && it.length > truncate) {\n                if (truncate < 4) it.substring(0, truncate)\n                else it.substring(0, truncate - 3) + \"...\"\n            } else {\n                it\n            }\n            sb.append(\"\"\"<td style=\"text-align:left\" title=\"$it\">$truncated</td>\"\"\")\n        }\n        sb.append(\"</tr>\")\n    }\n    sb.append(\"</table>\")\n    if(limit < count())\n        sb.append(\"<p>... only showing top $limit rows</p>\")\n    sb.append(\"</body></html>\")\n    return sb.toString()\n}"
+  ],
+  "initCell": [
+    "scala.Console.setOut(System.out)",
+    "scala.Console.setErr(System.err)"
+  ],
+  "renderers": [
+    {
+      "class": "org.apache.spark.sql.Dataset",
+      "result": "HTML($it.toHTML())"
+    }
+  ]
+}
diff --git a/readme.md b/readme.md
index 2e991374b..8ae667e39 100644
--- a/readme.md
+++ b/readme.md
@@ -88,7 +88,7 @@ List of supported libraries:
  - [koma](https://koma.kyonifer.com/index.html) - Scientific computing library
  - [kmath](https://github.com/mipt-npm/kmath) - Kotlin mathematical library analogous to NumPy
 
-*The list of all supported libraries can be found in [config file](config.json)*
+*The list of all supported libraries can be found in ['libraries' directory](libraries)*
 
 A definition of supported library may have a list of optional arguments that can be overriden when library is included.
 The major use case for library arguments is to specify particular version of library. Most library definitions default to `-SNAPSHOT` version that may be overriden in `%use` magic.     
@@ -131,10 +131,10 @@ Press `TAB` to get the list of suggested items for completion.
 
 ### Support new libraries
 
-You are welcome to add support for new `Kotlin` libraries by contributing to [config.json](config.json) file. 
+You are welcome to add support for new `Kotlin` libraries by adding `<libname>.json` file with library descriptor to ['libraries'](libraries) directory. 
 
 Library descriptor has the following fields:
-- `name`: short name of the library with optional arguments. All library arguments must have default value specified. Syntax: `<name>(<arg1>=<default1>, <arg2>=<default2>)`
+- `arguments`: an ordered list of library arguments. All arguments must have default value specified. Argument syntax: `<arg>=<default>`
 - `link`: a link to library homepage. This link will be displayed in `:help` command
 - `repositories`: a list of maven or ivy repositories to search for dependencies
 - `dependencies`: a list of library dependencies
@@ -143,6 +143,8 @@ Library descriptor has the following fields:
 - `initCell`: a list of code snippets to be executed before execution of any cell
 - `renderers`: a list of type converters for special rendering of particular types
 
+All fields are optional
+
 Fields for type renderer:
 - `class`: fully-qualified class name for the type to be rendered 
 - `result`: expression to produce output value. Source object is referenced as `$it`
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 974eec1ab..edb57a0e9 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -1,9 +1,23 @@
 package org.jetbrains.kotlin.jupyter
 
+import com.beust.klaxon.JsonObject
+import com.beust.klaxon.Parser
+import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import java.io.File
+import java.io.StringReader
+import java.nio.file.Paths
 import kotlin.script.experimental.dependencies.RepositoryCoordinates
 
+val LibrariesDir = "libraries"
+
+val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kotlin").toString()
+
+val GitHubApiHost = "api.github.com"
+val GitHubRepoOwner = "kotlin"
+val GitHubRepoName = "kotlin-jupyter"
+val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName/contents/"
+
 internal val log by lazy { LoggerFactory.getLogger("ikotlin") }
 
 enum class JupyterSockets {
@@ -14,6 +28,18 @@ enum class JupyterSockets {
     iopub
 }
 
+data class KernelConfig(
+        val ports: Array<Int>,
+        val transport: String,
+        val signatureScheme: String,
+        val signatureKey: String,
+        val pollingIntervalMillis: Long = 100,
+        val scriptClasspath: List<File> = emptyList(),
+        val resolverConfig: ResolverConfig?
+)
+
+val protocolVersion = "5.3"
+
 data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?)
 
 data class Variable(val name: String?, val value: String?)
@@ -27,16 +53,99 @@ class LibraryDefinition(val dependencies: List<String>,
                         val renderers: List<TypeRenderer>,
                         val link: String?)
 
-data class ResolverConfig(val repositories: List<RepositoryCoordinates>, val libraries: Map<String, LibraryDefinition>)
+data class ResolverConfig(val repositories: List<RepositoryCoordinates>,
+                          val libraries: Map<String, LibraryDefinition>)
 
-data class KernelConfig(
-        val ports: Array<Int>,
-        val transport: String,
-        val signatureScheme: String,
-        val signatureKey: String,
-        val pollingIntervalMillis: Long = 100,
-        val scriptClasspath: List<File> = emptyList(),
-        val resolverConfig: ResolverConfig?
+fun readJson(path: String) =
+        Parser.default().parse(path) as JsonObject
+
+fun JSONObject.toJsonObject() = Parser.default().parse(StringReader(toString())) as JsonObject
+
+fun <T> catchAll(body: () -> T): T? = try {
+    body()
+} catch (e: Exception) {
+    null
+}
+
+fun parseLibraryArgument(str: String): Variable {
+    val eq = str.indexOf('=')
+    return if (eq == -1) Variable(str.trim(), null)
+    else Variable(str.substring(0, eq).trim(), str.substring(eq + 1).trim())
+}
+
+fun parseLibraryName(str: String): Pair<String, List<Variable>> {
+    val brackets = str.indexOf('(')
+    if (brackets == -1) return str.trim() to emptyList()
+    val name = str.substring(0, brackets).trim()
+    val args = str.substring(brackets + 1, str.indexOf(')', brackets))
+            .split(',')
+            .map(::parseLibraryArgument)
+    return name to args
+}
+
+fun readLibraries(basePath: String? = null, filter: (File) -> Boolean = { true }): List<Pair<String, JsonObject>> {
+    val parser = Parser.default()
+    return File(basePath, LibrariesDir)
+            .listFiles()?.filter { it.extension == "json" && filter(it) }
+            ?.map {
+                log.info("Loading '${it.nameWithoutExtension}' descriptor from '${it.canonicalPath}'")
+                it.nameWithoutExtension to parser.parse(it.canonicalPath) as JsonObject
+            }
+            .orEmpty()
+}
+
+fun getLibrariesJsons(homeDir: String): Map<String, JsonObject> {
+
+    val librariesMap = readLibraries(LocalSettingsPath).toMap().orEmpty().toMutableMap()
+
+    val address = GitHubApiPrefix + LibrariesDir
+    val response = catchAll { khttp.get(address) }
+    if (response != null && response.statusCode == 200) {
+        response.jsonArray.forEach {
+            val o = it as JSONObject
+            val filename = o["name"] as String
+            if (filename.endsWith(".json")) {
+                val libName = filename.substring(0, filename.length - 5)
+                if (!librariesMap.containsKey(libName)) {
+                    val url = o["download_url"].toString()
+                    val res = catchAll { khttp.get(url) }
+                    if (res != null && res.statusCode == 200) {
+                        log.info("Loading '$libName' descriptor from '$url'")
+                        librariesMap[libName] = res.jsonObject.toJsonObject()
+                    }
+                }
+            }
+        }
+    }
+
+    readLibraries(homeDir) { !librariesMap.containsKey(it.nameWithoutExtension) }
+            .forEach { librariesMap.put(it.first, it.second) }
+
+    return librariesMap
+}
+
+fun loadResolverConfig(homeDir: String) = parseResolverConfig(getLibrariesJsons(homeDir))
+
+val defaultRepositories = arrayOf(
+        "https://jcenter.bintray.com/",
+        "https://repo.maven.apache.org/maven2/",
+        "https://jitpack.io"
 )
 
-val protocolVersion = "5.3"
+fun parseResolverConfig(libJsons: Map<String, JsonObject>): ResolverConfig {
+    val repos = defaultRepositories.map { RepositoryCoordinates(it) }.orEmpty()
+    return ResolverConfig(repos, libJsons.mapValues {
+        LibraryDefinition(
+                dependencies = it.value.array<String>("dependencies")?.toList().orEmpty(),
+                variables = it.value.array<String>("arguments")?.map(::parseLibraryArgument).orEmpty(),
+                imports = it.value.array<String>("imports")?.toList().orEmpty(),
+                repositories = it.value.array<String>("repositories")?.toList().orEmpty(),
+                init = it.value.array<String>("init")?.toList().orEmpty(),
+                initCell = it.value.array<String>("initCell")?.toList().orEmpty(),
+                renderers = it.value.array<JsonObject>("renderers")?.map {
+                    TypeRenderer(it.string("class")!!, it.string("display"), it.string("result"))
+                }?.toList().orEmpty(),
+                link = it.value.string("link")
+        )
+    })
+}
\ No newline at end of file
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt
index b1d87bf8d..f342e9d13 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt
@@ -5,27 +5,24 @@ import com.beust.klaxon.Parser
 import java.io.File
 import java.util.concurrent.atomic.AtomicLong
 import kotlin.concurrent.thread
-import kotlin.script.experimental.dependencies.RepositoryCoordinates
 import kotlin.script.experimental.jvm.util.classpathFromClassloader
 
-val DefaultConfigFile = "config.json"
-
 data class KernelArgs(val cfgFile: File,
                       val scriptClasspath: List<File>,
-                      val libs: File?)
+                      val homeDir: File?)
 
 private fun parseCommandLine(vararg args: String): KernelArgs {
     var cfgFile: File? = null
     var classpath: List<File>? = null
-    var libsJson: File? = null
+    var homeDir: File? = null
     args.forEach {
         when {
             it.startsWith("-cp=") || it.startsWith("-classpath=") -> {
                 if (classpath != null) throw IllegalArgumentException("classpath already set to ${classpath!!.joinToString(File.pathSeparator)}")
                 classpath = it.substringAfter('=').split(File.pathSeparator).map { File(it) }
             }
-            it.startsWith("-libs=") -> {
-                libsJson = File(it.substringAfter('='))
+            it.startsWith("-home=") -> {
+                homeDir = File(it.substringAfter('='))
             }
             else -> {
                 if (cfgFile != null) throw IllegalArgumentException("config file already set to $cfgFile")
@@ -35,7 +32,7 @@ private fun parseCommandLine(vararg args: String): KernelArgs {
     }
     if (cfgFile == null) throw IllegalArgumentException("config file is not provided")
     if (!cfgFile!!.exists() || !cfgFile!!.isFile) throw IllegalArgumentException("invalid config file $cfgFile")
-    return KernelArgs(cfgFile!!, classpath ?: emptyList(), libsJson)
+    return KernelArgs(cfgFile!!, classpath ?: emptyList(), homeDir)
 }
 
 fun printClassPath() {
@@ -48,49 +45,12 @@ fun printClassPath() {
         log.info("Current classpath: " + cp.joinToString())
 }
 
-fun parseLibraryName(str: String): Pair<String, List<Variable>> {
-    val pattern = """\w+(\w+)?""".toRegex().matches(str)
-    val brackets = str.indexOf('(')
-    if (brackets == -1) return str.trim() to emptyList()
-    val name = str.substring(0, brackets).trim()
-    val args = str.substring(brackets + 1, str.indexOf(')', brackets))
-            .split(',')
-            .map {
-                val eq = it.indexOf('=')
-                if (eq == -1) Variable(it.trim(), null)
-                else Variable(it.substring(0, eq).trim(), it.substring(eq + 1).trim())
-            }
-    return name to args
-}
-
-fun readResolverConfig(file: File = File(DefaultConfigFile)): ResolverConfig =
-        parseResolverConfig(Parser().parse(file.canonicalPath) as JsonObject)
-
-fun parseResolverConfig(json: JsonObject): ResolverConfig {
-    val repos = json.array<String>("repositories")?.map { RepositoryCoordinates(it) }.orEmpty()
-    val artifacts = json.array<JsonObject>("libraries")?.map {
-        val (name, variables) = parseLibraryName(it.string("name")!!)
-        name to LibraryDefinition(
-                dependencies = it.array<String>("dependencies")?.toList().orEmpty(),
-                variables = variables,
-                imports = it.array<String>("imports")?.toList().orEmpty(),
-                repositories = it.array<String>("repositories")?.toList().orEmpty(),
-                init = it.array<String>("init")?.toList().orEmpty(),
-                initCell = it.array<String>("initCell")?.toList().orEmpty(),
-                renderers = it.array<JsonObject>("renderers")?.map {
-                    TypeRenderer(it.string("class")!!, it.string("display"), it.string("result"))
-                }?.toList().orEmpty(),
-                link = it.string("link")
-        )
-    }?.toMap()
-    return ResolverConfig(repos, artifacts.orEmpty())
-}
-
 fun main(vararg args: String) {
     try {
         log.info("Kernel args: "+ args.joinToString { it })
-        val (cfgFile, scriptClasspath, librariesConfigFile) = parseCommandLine(*args)
-        val cfgJson = Parser().parse(cfgFile.canonicalPath) as JsonObject
+        val (cfgFile, scriptClasspath, homeDir) = parseCommandLine(*args)
+        val rootPath = homeDir!!.toString()
+        val cfgJson = Parser.default().parse(cfgFile.canonicalPath) as JsonObject
         fun JsonObject.getInt(field: String): Int = int(field) ?: throw RuntimeException("Cannot find $field in $cfgFile")
 
         val sigScheme = cfgJson.string("signature_scheme")
@@ -102,7 +62,7 @@ fun main(vararg args: String) {
                 signatureScheme = sigScheme ?: "hmac1-sha256",
                 signatureKey = if (sigScheme == null || key == null) "" else key,
                 scriptClasspath = scriptClasspath,
-                resolverConfig = librariesConfigFile?.let { readResolverConfig(it) }
+                resolverConfig = loadResolverConfig(rootPath)
         ))
     } catch (e: Exception) {
         log.error("exception running kernel with args: \"${args.joinToString()}\"", e)
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
index dd76390b2..022e74608 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
@@ -88,9 +88,10 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
             val kt = KotlinType(receiver.javaClass.canonicalName)
             implicitReceivers.invoke(listOf(kt))
 
-            val classes = listOf(/*receiver.javaClass,*/ ScriptTemplateWithDisplayHelpers::class.java)
-            val classPath = classes.asSequence().map { it.protectionDomain.codeSource.location.path }.joinToString(":")
-            compilerOptions.invoke(listOf("-classpath", classPath, "-jvm-target", "1.8"))
+            //val classes = listOf(/*receiver.javaClass,*/ ScriptTemplateWithDisplayHelpers::class.java)
+            //val classPath = classes.asSequence().map { it.protectionDomain.codeSource.location.path }.joinToString(":")
+            log.info("Classpath for compiler options: none")
+            compilerOptions.invoke(listOf("-jvm-target", "1.8"))
         }
     }
 
@@ -123,8 +124,11 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
             val scriptClassloader = URLClassLoader(scriptClasspath.map { it.toURI().toURL() }.toTypedArray(), filteringClassLoader)
             baseClassLoader(scriptClassloader)
         }
+        constructorArgs()
     }
 
+    val newEvalConfig = evaluatorConfiguration.with { constructorArgs() }
+
     private var executionCounter = 0
 
     private val compiler: ReplCompiler by lazy {
diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
index 47bab98fc..31369b899 100644
--- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
+++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
@@ -2,11 +2,8 @@ package org.jetbrains.kotlin.jupyter.test
 
 import com.beust.klaxon.JsonObject
 import com.beust.klaxon.Parser
-import jupyter.kotlin.DisplayResult
 import jupyter.kotlin.MimeTypedResult
-import org.jetbrains.kotlin.jupyter.ReplForJupyter
-import org.jetbrains.kotlin.jupyter.parseResolverConfig
-import org.jetbrains.kotlin.jupyter.readResolverConfig
+import org.jetbrains.kotlin.jupyter.*
 import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResultSuccess
 import org.junit.Assert
 import org.junit.Test
@@ -16,6 +13,8 @@ import kotlin.test.assertNotNull
 
 class ReplTest {
 
+    fun replWithResolver() = ReplForJupyter(classpath, parseResolverConfig(readLibraries().toMap()))
+
     @Test
     fun TestRepl() {
         val repl = ReplForJupyter(classpath)
@@ -73,14 +72,15 @@ class ReplTest {
 
     @Test
     fun TestUseMagic() {
-        val config = """
-            {
-                "libraries": [
+        val lib1 = "mylib" to """
                     {
-                        "name": "mylib(v1, v2=2.3)",
+                        "arguments": [
+                            "v1=0.2", 
+                            "v2=2.3"
+                        ],
                         "dependencies": [
-                            "artifact1:""" + "\$v1" + """",
-                            "artifact2:""" + "\$v2" + """"
+                            "artifact1:${'$'}v1",
+                            "artifact2:${'$'}v2"
                         ],
                         "imports": [
                             "package1",
@@ -90,23 +90,28 @@ class ReplTest {
                             "code1",
                             "code2"
                         ]
-                    },
-                    {
-                        "name": "other(a=temp, b=test)",
-                        "dependencies": [
-                            "path-""" + "\$a" + """",
-                            "path-""" + "\$b" + """"
-                        ],
-                        "imports": [
-                            "otherPackage"
-                        ]
-                    }
-                ]
-            }
+                    }""".trimIndent()
+        val lib2 = "other" to """
+                                {
+                                    "arguments": [
+                                        "a=temp", 
+                                        "b=test"
+                                    ],
+                                    "dependencies": [
+                                        "path-${'$'}a",
+                                        "path-${'$'}b"
+                                    ],
+                                    "imports": [
+                                        "otherPackage"
+                                    ]
+                                }
         """.trimIndent()
-        val json = Parser().parse(StringBuilder(config)) as JsonObject
-        val replConfig = parseResolverConfig(json)
-        val repl = ReplForJupyter(classpath, replConfig)
+        val parser = Parser.default()
+
+        val libJsons = arrayOf(lib1, lib2).map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap()
+        val resolverConfig = parseResolverConfig(libJsons)
+
+        val repl = ReplForJupyter(classpath, resolverConfig)
         val res = repl.preprocessCode("%use mylib(1.0), other(b=release, a=debug)").trimIndent()
         val libs = repl.librariesCodeGenerator.getProcessedLibraries()
         assertEquals("", res)
@@ -132,7 +137,7 @@ class ReplTest {
 
     @Test
     fun TestLetsPlot() {
-        val repl = ReplForJupyter(classpath, readResolverConfig())
+        val repl = replWithResolver()
         val code1 = "%use lets-plot"
         val code2 = """lets_plot(mapOf<String, Any>("cat" to listOf("a", "b")))"""
         val res1 = repl.eval(code1)
@@ -149,7 +154,7 @@ class ReplTest {
 
     @Test
     fun TestTwoLibrariesInUse() {
-        val repl = ReplForJupyter(classpath, readResolverConfig())
+        val repl = replWithResolver()
         val code = "%use lets-plot, krangl"
         val res = repl.eval(code)
         assertEquals(1, res.displayValues.count())
@@ -158,7 +163,7 @@ class ReplTest {
     @Test
     //TODO: https://github.com/Kotlin/kotlin-jupyter/issues/25
     fun TestKranglImportInfixFun() {
-        val repl = ReplForJupyter(classpath, readResolverConfig())
+        val repl = replWithResolver()
         val code = """%use krangl
                         "a" to {it["a"]}"""
         val res = repl.eval(code)

From 968b9706d2071188cb5eacaabceedfd799006cb2 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Thu, 19 Dec 2019 00:04:27 +0300
Subject: [PATCH 02/12] Add local cache for downloaded library descriptors

---
 readme.md                                     |  28 +++-
 .../org/jetbrains/kotlin/jupyter/config.kt    | 143 +++++++++++++-----
 .../org/jetbrains/kotlin/jupyter/repl.kt      |   2 -
 .../org/jetbrains/kotlin/jupyter/util.kt      |  35 +++++
 4 files changed, 165 insertions(+), 43 deletions(-)
 create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt

diff --git a/readme.md b/readme.md
index 8ae667e39..a679879b7 100644
--- a/readme.md
+++ b/readme.md
@@ -127,13 +127,13 @@ Press `TAB` to get the list of suggested items for completion.
 2. Run `jupyter-notebook`
 3. Attach remote debugger to JVM with specified port 
 
-## Contributing
+## Adding new libraries
 
-### Support new libraries
+To support new `JVM` library and make it available via `%use` magic command you need to create a library descriptor for it.
 
-You are welcome to add support for new `Kotlin` libraries by adding `<libname>.json` file with library descriptor to ['libraries'](libraries) directory. 
+Check ['libraries'](libraries) directory to see examples of library descriptors.
 
-Library descriptor has the following fields:
+Library descriptor is a `json` file with the following fields:
 - `arguments`: an ordered list of library arguments. All arguments must have default value specified. Argument syntax: `<arg>=<default>`
 - `link`: a link to library homepage. This link will be displayed in `:help` command
 - `repositories`: a list of maven or ivy repositories to search for dependencies
@@ -143,10 +143,24 @@ Library descriptor has the following fields:
 - `initCell`: a list of code snippets to be executed before execution of any cell
 - `renderers`: a list of type converters for special rendering of particular types
 
-All fields are optional
+*All fields are optional
 
 Fields for type renderer:
 - `class`: fully-qualified class name for the type to be rendered 
-- `result`: expression to produce output value. Source object is referenced as `$it`
+- `result`: expression that produces output value. Source object is referenced as `$it`
 
-Library arguments can be referenced in any parts of library descriptor as `$arg`
\ No newline at end of file
+Library arguments can be referenced in any parts of library descriptor as `$arg`
+
+There are two places where you can put new library descriptor:
+1. For private usage: into local settings folder `<UserHome>/.jupyter_kotlin/libraries`
+2. For sharing your library with community: checkout repository, put descriptor into ['libraries'](libraries) directory and create pull request.
+
+If you are maintaining some library and want to update your library descriptor, just create pull request with your update. After your request is accepted, 
+new version of your library will be available to all Kotlin Jupyter users on next kernel startup (no kernel update is needed).
+
+Kotlin Kernel collects library descriptors on startup in the following order:
+1. Local settings folder (highest priority)
+2. ['libraries'](libraries) folder at the latest master branch of `https://github.com/Kotlin/kotlin-jupyter` repository
+3. Kernel installation directory
+
+If you don't want some library to be updated automatically, put fixed version of its library descriptor into local settings folder.
\ No newline at end of file
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index edb57a0e9..0411ee0cc 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -5,18 +5,23 @@ import com.beust.klaxon.Parser
 import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import java.io.File
-import java.io.StringReader
+import java.nio.file.Files
 import java.nio.file.Paths
 import kotlin.script.experimental.dependencies.RepositoryCoordinates
 
 val LibrariesDir = "libraries"
+val LocalCacheDir = "cache"
+val CachedLibrariesFootprintFile = "libsCommit"
 
 val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kotlin").toString()
 
 val GitHubApiHost = "api.github.com"
 val GitHubRepoOwner = "kotlin"
 val GitHubRepoName = "kotlin-jupyter"
-val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName/contents/"
+val GitHubBranchName = "remote_config"
+val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
+
+val LibraryDescriptorExt = "json"
 
 internal val log by lazy { LoggerFactory.getLogger("ikotlin") }
 
@@ -40,6 +45,8 @@ data class KernelConfig(
 
 val protocolVersion = "5.3"
 
+val libraryDescriptorFormatVersion = "1.0"
+
 data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?)
 
 data class Variable(val name: String?, val value: String?)
@@ -56,17 +63,6 @@ class LibraryDefinition(val dependencies: List<String>,
 data class ResolverConfig(val repositories: List<RepositoryCoordinates>,
                           val libraries: Map<String, LibraryDefinition>)
 
-fun readJson(path: String) =
-        Parser.default().parse(path) as JsonObject
-
-fun JSONObject.toJsonObject() = Parser.default().parse(StringReader(toString())) as JsonObject
-
-fun <T> catchAll(body: () -> T): T? = try {
-    body()
-} catch (e: Exception) {
-    null
-}
-
 fun parseLibraryArgument(str: String): Variable {
     val eq = str.indexOf('=')
     return if (eq == -1) Variable(str.trim(), null)
@@ -86,7 +82,7 @@ fun parseLibraryName(str: String): Pair<String, List<Variable>> {
 fun readLibraries(basePath: String? = null, filter: (File) -> Boolean = { true }): List<Pair<String, JsonObject>> {
     val parser = Parser.default()
     return File(basePath, LibrariesDir)
-            .listFiles()?.filter { it.extension == "json" && filter(it) }
+            .listFiles()?.filter { it.extension == LibraryDescriptorExt && filter(it) }
             ?.map {
                 log.info("Loading '${it.nameWithoutExtension}' descriptor from '${it.canonicalPath}'")
                 it.nameWithoutExtension to parser.parse(it.canonicalPath) as JsonObject
@@ -94,32 +90,111 @@ fun readLibraries(basePath: String? = null, filter: (File) -> Boolean = { true }
             .orEmpty()
 }
 
-fun getLibrariesJsons(homeDir: String): Map<String, JsonObject> {
+fun getLatestCommitToLibraries(sinceTimestamp: String?): Pair<String, String>? =
+        log.catchAll {
+            var url = "$GitHubApiPrefix/commits?path=$LibrariesDir&sha=$GitHubBranchName"
+            if (sinceTimestamp != null)
+                url += "&since=$sinceTimestamp"
+            log.info("Checking for new commits to library descriptors at $url")
+            val arr = khttp.get(url).jsonArray
+            if (arr.length() == 0) {
+                if (sinceTimestamp != null)
+                    getLatestCommitToLibraries(null)
+                else {
+                    log.info("Didn't find any commits to '$LibrariesDir' at $url")
+                    null
+                }
+            } else {
+                val commit = arr[0] as JSONObject
+                val sha = commit["sha"] as String
+                val timestamp = ((commit["commit"] as JSONObject)["committer"] as JSONObject)["date"] as String
+                sha to timestamp
+            }
+        }
+
+/***
+ * Downloads library descriptors from GitHub to local cache if new commits in `libraries` directory were detected
+ */
+fun downloadNewLibraryDescriptors() {
+
+    // Read commit hash and timestamp for locally cached libraries.
+    // Timestamp is used as parameter for commits request to reduce output
+
+    val footprintFilePath = Paths.get(LocalSettingsPath, LocalCacheDir, CachedLibrariesFootprintFile).toString()
+    log.info("Reading commit info for which library descriptors were cached: '$footprintFilePath'")
+    val footprintFile = File(footprintFilePath)
+    val footprint = footprintFile.readIniConfig()
+    val timestampRegex = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""".toRegex()
+    val syncedCommitTimestamp = footprint?.get("timestamp")?.validOrNull { timestampRegex.matches(it) }
+    val syncedCommitSha = footprint?.get("sha")
+    log.info("Local libraries are cached for commit '$syncedCommitSha' at '$syncedCommitTimestamp'")
+
+    val (latestCommitSha, latestCommitTimestamp) = getLatestCommitToLibraries(syncedCommitTimestamp) ?: return
+    if (latestCommitSha.equals(syncedCommitSha)) {
+        log.info("No new commits to library descriptors were detected")
+        return
+    }
+
+    // Download library descriptors
+
+    log.info("New commits to library descriptors were detected. Downloading library descriptors for commit $latestCommitSha")
 
-    val librariesMap = readLibraries(LocalSettingsPath).toMap().orEmpty().toMutableMap()
+    val libraries = log.catchAll {
+        val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha"
+        log.info("Requesting the list of library descriptors at $url")
+        val response = khttp.get(url)
+        if (response.statusCode != 200)
+            throw Exception("Failed to get GitHub contents for '$LibrariesDir' from $url. Response = $response")
 
-    val address = GitHubApiPrefix + LibrariesDir
-    val response = catchAll { khttp.get(address) }
-    if (response != null && response.statusCode == 200) {
-        response.jsonArray.forEach {
+        response.jsonArray.mapNotNull {
             val o = it as JSONObject
             val filename = o["name"] as String
-            if (filename.endsWith(".json")) {
-                val libName = filename.substring(0, filename.length - 5)
-                if (!librariesMap.containsKey(libName)) {
-                    val url = o["download_url"].toString()
-                    val res = catchAll { khttp.get(url) }
-                    if (res != null && res.statusCode == 200) {
-                        log.info("Loading '$libName' descriptor from '$url'")
-                        librariesMap[libName] = res.jsonObject.toJsonObject()
-                    }
-                }
-            }
+            if ("""[\w-]+.$LibraryDescriptorExt""".toRegex().matches(filename)) {
+                val libUrl = o["download_url"].toString()
+                log.info("Downloading '$filename' from $libUrl")
+                val res = khttp.get(libUrl)
+                if (res.statusCode != 200)
+                    throw Exception("Failed to download '$filename' from $libUrl. Response = $res")
+                val text = res.jsonObject.toString()
+                filename to text
+            } else null
         }
+    } ?: return
+
+    // Save library descriptors to local cache
+
+    val librariesPath = Paths.get(LocalSettingsPath, LocalCacheDir, LibrariesDir)
+    val librariesDir = librariesPath.toFile()
+    log.info("Saving ${libraries.count()} library descriptors to local cache at '$librariesPath'")
+    try {
+        Files.createDirectories(librariesPath)
+        libraries.forEach {
+            File(librariesDir.toString(), it.first).writeText(it.second)
+        }
+        footprintFile.writeText("""
+            timestamp=$latestCommitTimestamp
+            sha=$latestCommitSha
+        """.trimIndent())
+    } catch (e: Exception) {
+        log.error("Failed to write downloaded library descriptors to local cache:", e)
+        log.catchAll { librariesDir.delete() }
     }
+}
 
-    readLibraries(homeDir) { !librariesMap.containsKey(it.nameWithoutExtension) }
-            .forEach { librariesMap.put(it.first, it.second) }
+fun getLibrariesJsons(homeDir: String): Map<String, JsonObject> {
+
+    downloadNewLibraryDescriptors()
+
+    val pathsToCheck = arrayOf(LocalSettingsPath,
+            Paths.get(LocalSettingsPath, LocalCacheDir).toString(),
+            homeDir)
+
+    val librariesMap = mutableMapOf<String, JsonObject>()
+
+    pathsToCheck.forEach {
+        readLibraries(it) { !librariesMap.containsKey(it.nameWithoutExtension) }
+                .forEach { librariesMap.put(it.first, it.second) }
+    }
 
     return librariesMap
 }
@@ -148,4 +223,4 @@ fun parseResolverConfig(libJsons: Map<String, JsonObject>): ResolverConfig {
                 link = it.value.string("link")
         )
     })
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
index 022e74608..a7e4c878d 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
@@ -88,8 +88,6 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
             val kt = KotlinType(receiver.javaClass.canonicalName)
             implicitReceivers.invoke(listOf(kt))
 
-            //val classes = listOf(/*receiver.javaClass,*/ ScriptTemplateWithDisplayHelpers::class.java)
-            //val classPath = classes.asSequence().map { it.protectionDomain.codeSource.location.path }.joinToString(":")
             log.info("Classpath for compiler options: none")
             compilerOptions.invoke(listOf("-jvm-target", "1.8"))
         }
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
new file mode 100644
index 000000000..4d84aa3bb
--- /dev/null
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
@@ -0,0 +1,35 @@
+package org.jetbrains.kotlin.jupyter
+
+import com.beust.klaxon.JsonObject
+import com.beust.klaxon.Parser
+import org.json.JSONObject
+import org.slf4j.Logger
+import java.io.File
+import java.io.StringReader
+
+fun <T> catchAll(body: () -> T): T? = try {
+    body()
+} catch (e: Exception) {
+    null
+}
+
+fun <T> Logger.catchAll(msg: String = "", body: () -> T): T? = try {
+    body()
+} catch (e: Exception) {
+    this.error(msg, e)
+    null
+}
+
+fun <T> T.validOrNull(predicate: (T) -> Boolean): T? = if (predicate(this)) this else null
+
+fun File.existsOrNull() = if (exists()) this else null
+
+fun File.readIniConfig() =
+        existsOrNull()?.let {
+            catchAll { it.readLines().map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() }
+        }
+
+fun readJson(path: String) =
+        Parser.default().parse(path) as JsonObject
+
+fun JSONObject.toJsonObject() = Parser.default().parse(StringReader(toString())) as JsonObject
\ No newline at end of file

From f1ddadcdbded1d7fc36ebc00d2793d56b1f2bbbb Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Thu, 19 Dec 2019 00:58:12 +0300
Subject: [PATCH 03/12] Download library descriptors asynchronously

---
 .../org/jetbrains/kotlin/jupyter/commands.kt  |  2 +-
 .../org/jetbrains/kotlin/jupyter/config.kt    | 19 ++++++++++++-------
 .../org/jetbrains/kotlin/jupyter/libraries.kt |  3 ++-
 .../org/jetbrains/kotlin/jupyter/repl.kt      | 15 +++++++++------
 .../org/jetbrains/kotlin/jupyter/util.kt      | 15 +++++++++++++++
 .../kotlin/jupyter/test/replTests.kt          |  8 +++++---
 6 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
index a5f7b847b..a84dc77fa 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt
@@ -32,7 +32,7 @@ fun runCommand(code: String, repl: ReplForJupyter?): ResponseWithMessage {
                 if (it.argumentsUsage != null) s += "\n        Usage: %${it.name} ${it.argumentsUsage}"
                 s
             }
-            val libraries = repl?.config?.libraries?.toList()?.joinToStringIndented {
+            val libraries = repl?.config?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented {
                 "${it.first} ${it.second.link ?: ""}"
             }
             ResponseWithMessage(ResponseState.Ok, textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries"))
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 0411ee0cc..79abb0663 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -2,6 +2,9 @@ package org.jetbrains.kotlin.jupyter
 
 import com.beust.klaxon.JsonObject
 import com.beust.klaxon.Parser
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
 import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import java.io.File
@@ -61,7 +64,7 @@ class LibraryDefinition(val dependencies: List<String>,
                         val link: String?)
 
 data class ResolverConfig(val repositories: List<RepositoryCoordinates>,
-                          val libraries: Map<String, LibraryDefinition>)
+                          val libraries: Deferred<Map<String, LibraryDefinition>>)
 
 fun parseLibraryArgument(str: String): Variable {
     val eq = str.indexOf('=')
@@ -199,17 +202,18 @@ fun getLibrariesJsons(homeDir: String): Map<String, JsonObject> {
     return librariesMap
 }
 
-fun loadResolverConfig(homeDir: String) = parseResolverConfig(getLibrariesJsons(homeDir))
+fun loadResolverConfig(homeDir: String) = ResolverConfig(defaultRepositories, GlobalScope.async {
+    parserLibraryDescriptors(getLibrariesJsons(homeDir))
+})
 
 val defaultRepositories = arrayOf(
         "https://jcenter.bintray.com/",
         "https://repo.maven.apache.org/maven2/",
         "https://jitpack.io"
-)
+).map { RepositoryCoordinates(it) }
 
-fun parseResolverConfig(libJsons: Map<String, JsonObject>): ResolverConfig {
-    val repos = defaultRepositories.map { RepositoryCoordinates(it) }.orEmpty()
-    return ResolverConfig(repos, libJsons.mapValues {
+fun parserLibraryDescriptors(libJsons: Map<String, JsonObject>): Map<String, LibraryDefinition> {
+    return libJsons.mapValues {
         LibraryDefinition(
                 dependencies = it.value.array<String>("dependencies")?.toList().orEmpty(),
                 variables = it.value.array<String>("arguments")?.map(::parseLibraryArgument).orEmpty(),
@@ -222,5 +226,6 @@ fun parseResolverConfig(libJsons: Map<String, JsonObject>): ResolverConfig {
                 }?.toList().orEmpty(),
                 link = it.value.string("link")
         )
-    })
+    }
 }
+
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
index ef29dce71..f21da7fa4 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
@@ -95,7 +95,8 @@ class LibrariesProcessor {
 
         splitLibraryCalls(arg).forEach {
             val (name, vars) = parseLibraryName(it)
-            val library = repl.config?.libraries?.get(name) ?: throw ReplCompilerException("Unknown library '$name'")
+            val library = repl.config?.libraries?.awaitBlocking()?.get(name)
+                    ?: throw ReplCompilerException("Unknown library '$name'")
 
             // treat single strings in parsed arguments as values, not names
             val arguments = vars.map { if (it.value == null) Variable(null, it.name) else it }
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
index a7e4c878d..1e26b868f 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt
@@ -42,7 +42,11 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
 
     private val resolver = JupyterScriptDependenciesResolver(config)
 
-    private val renderers = config?.let { it.libraries.flatMap { it.value.renderers } }?.map { it.className to it }?.toMap().orEmpty()
+    private val renderers = config?.let {
+        it.libraries.asyncLet {
+            it.flatMap { it.value.renderers }.map { it.className to it }.toMap()
+        }
+    }
 
     private val includedLibraries = mutableSetOf<LibraryDefinition>()
 
@@ -125,8 +129,6 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
         constructorArgs()
     }
 
-    val newEvalConfig = evaluatorConfiguration.with { constructorArgs() }
-
     private var executionCounter = 0
 
     private val compiler: ReplCompiler by lazy {
@@ -169,7 +171,7 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
 
     init {
         // TODO: to be removed after investigation of https://github.com/kotlin/kotlin-jupyter/issues/24
-        eval("1")
+        doEval("1")
     }
 
     fun eval(code: String, jupyterId: Int = -1): EvalResult {
@@ -228,8 +230,9 @@ class ReplForJupyter(val scriptClasspath: List<File> = emptyList(),
                     }
                 }
 
-                if (result != null) {
-                    renderers[result.javaClass.canonicalName]?.let {
+                if (result != null && renderers != null) {
+                    val resultType = result.javaClass.canonicalName
+                    renderers.awaitBlocking()[resultType]?.let {
                         it.displayCode?.let {
                             doEval(it.replace("\$it", "res$replId")).value?.let(displays::add)
                         }
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
index 4d84aa3bb..2600b9937 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
@@ -2,10 +2,15 @@ package org.jetbrains.kotlin.jupyter
 
 import com.beust.klaxon.JsonObject
 import com.beust.klaxon.Parser
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
 import org.json.JSONObject
 import org.slf4j.Logger
 import java.io.File
 import java.io.StringReader
+import javax.xml.bind.JAXBElement
 
 fun <T> catchAll(body: () -> T): T? = try {
     body()
@@ -22,8 +27,18 @@ fun <T> Logger.catchAll(msg: String = "", body: () -> T): T? = try {
 
 fun <T> T.validOrNull(predicate: (T) -> Boolean): T? = if (predicate(this)) this else null
 
+fun <T> T.asDeferred(): Deferred<T> = this.let { GlobalScope.async { it } }
+
 fun File.existsOrNull() = if (exists()) this else null
 
+fun <T, R> Deferred<T>.asyncLet(selector: suspend (T) -> R): Deferred<R> = this.let {
+    GlobalScope.async {
+        selector(it.await())
+    }
+}
+
+fun <T> Deferred<T>.awaitBlocking(): T = if (isCompleted) getCompleted() else runBlocking { await() }
+
 fun File.readIniConfig() =
         existsOrNull()?.let {
             catchAll { it.readLines().map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() }
diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
index 31369b899..8bdf6f03a 100644
--- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
+++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
@@ -3,6 +3,8 @@ package org.jetbrains.kotlin.jupyter.test
 import com.beust.klaxon.JsonObject
 import com.beust.klaxon.Parser
 import jupyter.kotlin.MimeTypedResult
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
 import org.jetbrains.kotlin.jupyter.*
 import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResultSuccess
 import org.junit.Assert
@@ -13,7 +15,8 @@ import kotlin.test.assertNotNull
 
 class ReplTest {
 
-    fun replWithResolver() = ReplForJupyter(classpath, parseResolverConfig(readLibraries().toMap()))
+    fun replWithResolver() = ReplForJupyter(classpath, ResolverConfig(defaultRepositories,
+            parserLibraryDescriptors(readLibraries().toMap()).asDeferred()))
 
     @Test
     fun TestRepl() {
@@ -109,9 +112,8 @@ class ReplTest {
         val parser = Parser.default()
 
         val libJsons = arrayOf(lib1, lib2).map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap()
-        val resolverConfig = parseResolverConfig(libJsons)
 
-        val repl = ReplForJupyter(classpath, resolverConfig)
+        val repl = ReplForJupyter(classpath, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asDeferred()))
         val res = repl.preprocessCode("%use mylib(1.0), other(b=release, a=debug)").trimIndent()
         val libs = repl.librariesCodeGenerator.getProcessedLibraries()
         assertEquals("", res)

From d547ad8a475831d8a0698750d325a08a4ea578a4 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Thu, 19 Dec 2019 15:49:16 +0300
Subject: [PATCH 04/12] Add .properties file with current library descriptor
 format version

---
 libraries/.properties                         |  1 +
 .../org/jetbrains/kotlin/jupyter/config.kt    | 50 +++++++++++++++----
 .../org/jetbrains/kotlin/jupyter/util.kt      |  7 ++-
 3 files changed, 45 insertions(+), 13 deletions(-)
 create mode 100644 libraries/.properties

diff --git a/libraries/.properties b/libraries/.properties
new file mode 100644
index 000000000..045e6eae5
--- /dev/null
+++ b/libraries/.properties
@@ -0,0 +1 @@
+formatVersion=1
\ No newline at end of file
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 79abb0663..0f31d1794 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -2,9 +2,11 @@ package org.jetbrains.kotlin.jupyter
 
 import com.beust.klaxon.JsonObject
 import com.beust.klaxon.Parser
+import khttp.responses.Response
 import kotlinx.coroutines.Deferred
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.async
+import org.jetbrains.kotlin.konan.parseKonanVersion
 import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import java.io.File
@@ -21,10 +23,12 @@ val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kot
 val GitHubApiHost = "api.github.com"
 val GitHubRepoOwner = "kotlin"
 val GitHubRepoName = "kotlin-jupyter"
-val GitHubBranchName = "remote_config"
+val GitHubBranchName = "master"
 val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName"
 
 val LibraryDescriptorExt = "json"
+val LibraryPropertiesFile = ".properties"
+val libraryDescriptorFormatVersion = 1
 
 internal val log by lazy { LoggerFactory.getLogger("ikotlin") }
 
@@ -48,8 +52,6 @@ data class KernelConfig(
 
 val protocolVersion = "5.3"
 
-val libraryDescriptorFormatVersion = "1.0"
-
 data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?)
 
 data class Variable(val name: String?, val value: String?)
@@ -99,7 +101,7 @@ fun getLatestCommitToLibraries(sinceTimestamp: String?): Pair<String, String>? =
             if (sinceTimestamp != null)
                 url += "&since=$sinceTimestamp"
             log.info("Checking for new commits to library descriptors at $url")
-            val arr = khttp.get(url).jsonArray
+            val arr = getHttp(url).jsonArray
             if (arr.length() == 0) {
                 if (sinceTimestamp != null)
                     getLatestCommitToLibraries(null)
@@ -115,6 +117,25 @@ fun getLatestCommitToLibraries(sinceTimestamp: String?): Pair<String, String>? =
             }
         }
 
+fun getHttp(url: String): Response {
+    val response = khttp.get(url)
+    if (response.statusCode != 200)
+        throw Exception("Http request failed. Url = $url. Response = $response")
+    return response
+}
+
+fun getLibraryDescriptorVersion(commitSha: String) =
+        log.catchAll {
+            val url = "$GitHubApiPrefix/contents/$LibrariesDir/$LibraryPropertiesFile?ref=$commitSha"
+            log.info("Checking current library descriptor format version from $url")
+            val response = getHttp(url)
+            val downloadUrl = response.jsonObject["download_url"].toString()
+            val downloadResult = getHttp(downloadUrl)
+            val result = downloadResult.text.parseIniConfig()["formatVersion"]!!.toInt()
+            log.info("Current library descriptor format version: $result")
+            result
+        }
+
 /***
  * Downloads library descriptors from GitHub to local cache if new commits in `libraries` directory were detected
  */
@@ -126,7 +147,7 @@ fun downloadNewLibraryDescriptors() {
     val footprintFilePath = Paths.get(LocalSettingsPath, LocalCacheDir, CachedLibrariesFootprintFile).toString()
     log.info("Reading commit info for which library descriptors were cached: '$footprintFilePath'")
     val footprintFile = File(footprintFilePath)
-    val footprint = footprintFile.readIniConfig()
+    val footprint = footprintFile.tryReadIniConfig()
     val timestampRegex = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""".toRegex()
     val syncedCommitTimestamp = footprint?.get("timestamp")?.validOrNull { timestampRegex.matches(it) }
     val syncedCommitSha = footprint?.get("sha")
@@ -138,6 +159,17 @@ fun downloadNewLibraryDescriptors() {
         return
     }
 
+    // Download library descriptor version
+
+    val descriptorVersion = getLibraryDescriptorVersion(latestCommitSha) ?: return
+    if (descriptorVersion != libraryDescriptorFormatVersion) {
+        if (descriptorVersion < libraryDescriptorFormatVersion)
+            log.error("Incorrect library descriptor version in GitHub repository: $descriptorVersion")
+        else
+            log.warn("Kotlin Kernel needs to be updated to the latest version. Couldn't download new library descriptors from GitHub repository because their format was changed")
+        return
+    }
+
     // Download library descriptors
 
     log.info("New commits to library descriptors were detected. Downloading library descriptors for commit $latestCommitSha")
@@ -145,9 +177,7 @@ fun downloadNewLibraryDescriptors() {
     val libraries = log.catchAll {
         val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha"
         log.info("Requesting the list of library descriptors at $url")
-        val response = khttp.get(url)
-        if (response.statusCode != 200)
-            throw Exception("Failed to get GitHub contents for '$LibrariesDir' from $url. Response = $response")
+        val response = getHttp(url)
 
         response.jsonArray.mapNotNull {
             val o = it as JSONObject
@@ -155,9 +185,7 @@ fun downloadNewLibraryDescriptors() {
             if ("""[\w-]+.$LibraryDescriptorExt""".toRegex().matches(filename)) {
                 val libUrl = o["download_url"].toString()
                 log.info("Downloading '$filename' from $libUrl")
-                val res = khttp.get(libUrl)
-                if (res.statusCode != 200)
-                    throw Exception("Failed to download '$filename' from $libUrl. Response = $res")
+                val res = getHttp(libUrl)
                 val text = res.jsonObject.toString()
                 filename to text
             } else null
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
index 2600b9937..c5550e40c 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt
@@ -39,9 +39,12 @@ fun <T, R> Deferred<T>.asyncLet(selector: suspend (T) -> R): Deferred<R> = this.
 
 fun <T> Deferred<T>.awaitBlocking(): T = if (isCompleted) getCompleted() else runBlocking { await() }
 
-fun File.readIniConfig() =
+fun String.parseIniConfig() =
+        split("\n").map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap()
+
+fun File.tryReadIniConfig() =
         existsOrNull()?.let {
-            catchAll { it.readLines().map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() }
+            catchAll { it.readText().parseIniConfig() }
         }
 
 fun readJson(path: String) =

From 959ece9584f15c02badf0be526bedb6ea71cb654 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Thu, 19 Dec 2019 16:44:18 +0300
Subject: [PATCH 05/12] Replace 'http' with 'https' in 'koma.json'

---
 libraries/koma.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/koma.json b/libraries/koma.json
index c01ac2647..8a517931b 100644
--- a/libraries/koma.json
+++ b/libraries/koma.json
@@ -2,7 +2,7 @@
   "arguments": [
     "v=0.12"
   ],
-  "link": "http://koma.kyonifer.com/index.html",
+  "link": "https://koma.kyonifer.com/index.html",
   "repositories": [
     "https://dl.bintray.com/kyonifer/maven"
   ],

From 4d8c2c3d4c7efd834f92b5c951593e553a727097 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Sun, 22 Dec 2019 22:47:39 +0300
Subject: [PATCH 06/12] Add logging for resolver

---
 .../org/jetbrains/kotlin/jupyter/resolver.kt  | 38 +++++++++++++++----
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt
index c5fb74c77..3aaa421ff 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt
@@ -4,13 +4,22 @@ import jupyter.kotlin.DependsOn
 import jupyter.kotlin.Repository
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.kotlin.mainKts.impl.IvyResolver
+import org.slf4j.LoggerFactory
 import java.io.File
 import kotlin.script.dependencies.ScriptContents
-import kotlin.script.experimental.api.*
-import kotlin.script.experimental.dependencies.*
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.ScriptDiagnostic
+import kotlin.script.experimental.api.asSuccess
+import kotlin.script.experimental.api.makeFailureResult
+import kotlin.script.experimental.dependencies.CompoundDependenciesResolver
+import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
+import kotlin.script.experimental.dependencies.FileSystemDependenciesResolver
+import kotlin.script.experimental.dependencies.tryAddRepository
 
 open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) {
 
+    private val log by lazy { LoggerFactory.getLogger("resolver") }
+
     private val resolver: ExternalDependenciesResolver
 
     init {
@@ -33,17 +42,30 @@ open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) {
         script.annotations.forEach { annotation ->
             when (annotation) {
                 is Repository -> {
+                    log.info("Adding repository: ${annotation.value}")
                     if (!resolver.tryAddRepository(annotation.value))
                         throw IllegalArgumentException("Illegal argument for Repository annotation: $annotation")
                 }
                 is DependsOn -> {
-                    val result = runBlocking { resolver.resolve(annotation.value) }
-                    when (result) {
-                        is ResultWithDiagnostics.Failure -> scriptDiagnostics.add(ScriptDiagnostic("Failed to resolve dependencies:\n" + result.reports.joinToString("\n") { it.message }))
-                        is ResultWithDiagnostics.Success -> {
-                            addedClasspath.addAll(result.value)
-                            classpath.addAll(result.value)
+                    log.info("Resolving ${annotation.value}")
+                    try {
+                        val result = runBlocking { resolver.resolve(annotation.value) }
+                        when (result) {
+                            is ResultWithDiagnostics.Failure -> {
+                                val diagnostics = ScriptDiagnostic("Failed to resolve ${annotation.value}:\n" + result.reports.joinToString("\n") { it.message })
+                                log.warn(diagnostics.message, diagnostics.exception)
+                                scriptDiagnostics.add(diagnostics)
+                            }
+                            is ResultWithDiagnostics.Success -> {
+                                log.info("Resolved: " + result.value.joinToString())
+                                addedClasspath.addAll(result.value)
+                                classpath.addAll(result.value)
+                            }
                         }
+                    } catch (e: Exception) {
+                        val diagnostic = ScriptDiagnostic("Unhandled exception during resolve", exception = e)
+                        log.error(diagnostic.message, e)
+                        scriptDiagnostics.add(diagnostic)
                     }
                 }
                 else -> throw Exception("Unknown annotation ${annotation.javaClass}")

From 77a7cfeb984a8785f2f58749a8530a33f7f8c695 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Sun, 22 Dec 2019 23:19:29 +0300
Subject: [PATCH 07/12] Rename 'arguments' to 'properties' in library
 descriptor format, treat them as unordered dictionary instead of an ordered
 list. Unnamed argument is allowed only if library has single property
 declaration

---
 libraries/gral.json                           |  6 ++---
 libraries/klaxon.json                         |  6 ++---
 libraries/kmath.json                          |  6 ++---
 libraries/koma.json                           |  6 ++---
 libraries/kotlin-statistics.json              |  6 ++---
 libraries/krangl.json                         |  6 ++---
 libraries/kravis.json                         |  6 ++---
 libraries/lets-plot.json                      |  8 +++---
 libraries/spark.json                          |  8 +++---
 readme.md                                     | 18 +++++++------
 .../org/jetbrains/kotlin/jupyter/config.kt    |  6 ++---
 .../org/jetbrains/kotlin/jupyter/libraries.kt | 26 ++++++++-----------
 .../jupyter/test/parseArgumentsTests.kt       |  8 +++---
 .../kotlin/jupyter/test/replTests.kt          | 19 +++++++-------
 14 files changed, 66 insertions(+), 69 deletions(-)

diff --git a/libraries/gral.json b/libraries/gral.json
index b23a241d0..83eeaf2d8 100644
--- a/libraries/gral.json
+++ b/libraries/gral.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=0.11"
-  ],
+  "properties": {
+    "v": "0.11"
+  },
   "link": "https://github.com/eseifert/gral",
   "dependencies": [
     "de.erichseifert.gral:gral-core:$v"
diff --git a/libraries/klaxon.json b/libraries/klaxon.json
index 0965d18c5..f95a3d460 100644
--- a/libraries/klaxon.json
+++ b/libraries/klaxon.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=5.2"
-  ],
+  "properties": {
+    "v": "5.2"
+  },
   "link": "https://github.com/cbeust/klaxon",
   "dependencies": [
     "com.beust:klaxon:$v"
diff --git a/libraries/kmath.json b/libraries/kmath.json
index 1a747430b..c3e9fe295 100644
--- a/libraries/kmath.json
+++ b/libraries/kmath.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=0.1.3"
-  ],
+  "properties": {
+    "v": "0.1.3"
+  },
   "link": "https://github.com/mipt-npm/kmath",
   "repositories": [
     "https://dl.bintray.com/mipt-npm/scientifik"
diff --git a/libraries/koma.json b/libraries/koma.json
index 8a517931b..86ffbacc6 100644
--- a/libraries/koma.json
+++ b/libraries/koma.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=0.12"
-  ],
+  "properties": {
+    "v": "0.13"
+  },
   "link": "https://koma.kyonifer.com/index.html",
   "repositories": [
     "https://dl.bintray.com/kyonifer/maven"
diff --git a/libraries/kotlin-statistics.json b/libraries/kotlin-statistics.json
index 1de0c9ecd..b2b34706a 100644
--- a/libraries/kotlin-statistics.json
+++ b/libraries/kotlin-statistics.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=-SNAPSHOT"
-  ],
+  "properties": {
+    "v": "-SNAPSHOT"
+  },
   "link": "https://github.com/thomasnield/kotlin-statistics",
   "dependencies": [
     "com.github.thomasnield:kotlin-statistics:$v"
diff --git a/libraries/krangl.json b/libraries/krangl.json
index 301ca1f4b..0d0d77b08 100644
--- a/libraries/krangl.json
+++ b/libraries/krangl.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=-SNAPSHOT"
-  ],
+  "properties": {
+    "v": "-SNAPSHOT"
+  },
   "link": "https://github.com/holgerbrandl/krangl",
   "dependencies": [
     "com.github.holgerbrandl:krangl:$v"
diff --git a/libraries/kravis.json b/libraries/kravis.json
index 0b3af5125..c13301f4b 100644
--- a/libraries/kravis.json
+++ b/libraries/kravis.json
@@ -1,7 +1,7 @@
 {
-  "arguments": [
-    "v=-SNAPSHOT"
-  ],
+  "properties": {
+    "v": "-SNAPSHOT"
+  },
   "link": "https://github.com/holgerbrandl/kravis",
   "dependencies": [
     "com.github.holgerbrandl:kravis:$v"
diff --git a/libraries/lets-plot.json b/libraries/lets-plot.json
index 62b516aab..a5ae93ee2 100644
--- a/libraries/lets-plot.json
+++ b/libraries/lets-plot.json
@@ -1,8 +1,8 @@
 {
-  "arguments": [
-    "core=1.0.1-SNAPSHOT",
-    "kotlin=0.0.8-SNAPSHOT"
-  ],
+  "properties": {
+    "core": "1.0.1-SNAPSHOT",
+    "kotlin": "0.0.8-SNAPSHOT"
+  },
   "link": "https://github.com/JetBrains/lets-plot-kotlin",
   "repositories": [
     "https://jetbrains.bintray.com/lets-plot-maven"
diff --git a/libraries/spark.json b/libraries/spark.json
index 49549b570..488a42f27 100644
--- a/libraries/spark.json
+++ b/libraries/spark.json
@@ -1,8 +1,8 @@
 {
-  "arguments": [
-    "scala=2.11.12",
-    "spark=2.4.4"
-  ],
+  "properties": {
+    "scala": "2.11.12",
+    "spark": "2.4.4"
+  },
   "dependencies": [
     "org.apache.spark:spark-mllib_2.11:$spark",
     "org.apache.spark:spark-sql_2.11:$spark",
diff --git a/readme.md b/readme.md
index a679879b7..1980acfd2 100644
--- a/readme.md
+++ b/readme.md
@@ -133,8 +133,8 @@ To support new `JVM` library and make it available via `%use` magic command you
 
 Check ['libraries'](libraries) directory to see examples of library descriptors.
 
-Library descriptor is a `json` file with the following fields:
-- `arguments`: an ordered list of library arguments. All arguments must have default value specified. Argument syntax: `<arg>=<default>`
+Library descriptor is a `<libName>.json` file with the following fields:
+- `properties`: a dictionary of properties that are used within library descriptor
 - `link`: a link to library homepage. This link will be displayed in `:help` command
 - `repositories`: a list of maven or ivy repositories to search for dependencies
 - `dependencies`: a list of library dependencies
@@ -149,16 +149,18 @@ Fields for type renderer:
 - `class`: fully-qualified class name for the type to be rendered 
 - `result`: expression that produces output value. Source object is referenced as `$it`
 
-Library arguments can be referenced in any parts of library descriptor as `$arg`
+Name of the file is a library name that is passed to '%use' command
 
-There are two places where you can put new library descriptor:
-1. For private usage: into local settings folder `<UserHome>/.jupyter_kotlin/libraries`
-2. For sharing your library with community: checkout repository, put descriptor into ['libraries'](libraries) directory and create pull request.
+Library properties can be used in any parts of library descriptor as `$property`
+
+To register new library descriptor:
+1. For private usage - add it to local settings folder `<UserHome>/.jupyter_kotlin/libraries`
+2. For sharing with community - commit it to ['libraries'](libraries) directory and create pull request.
 
 If you are maintaining some library and want to update your library descriptor, just create pull request with your update. After your request is accepted, 
-new version of your library will be available to all Kotlin Jupyter users on next kernel startup (no kernel update is needed).
+new version of your library will be available to all Kotlin Jupyter users immediately on next kernel startup (no kernel update is needed).
 
-Kotlin Kernel collects library descriptors on startup in the following order:
+If a library descriptor with the same name is found in several locations, the following resolution priority is used:
 1. Local settings folder (highest priority)
 2. ['libraries'](libraries) folder at the latest master branch of `https://github.com/Kotlin/kotlin-jupyter` repository
 3. Kernel installation directory
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 0f31d1794..38bb92c6c 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -54,7 +54,7 @@ val protocolVersion = "5.3"
 
 data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?)
 
-data class Variable(val name: String?, val value: String?)
+data class Variable(val name: String, val value: String)
 
 class LibraryDefinition(val dependencies: List<String>,
                         val variables: List<Variable>,
@@ -70,7 +70,7 @@ data class ResolverConfig(val repositories: List<RepositoryCoordinates>,
 
 fun parseLibraryArgument(str: String): Variable {
     val eq = str.indexOf('=')
-    return if (eq == -1) Variable(str.trim(), null)
+    return if (eq == -1) Variable("", str.trim())
     else Variable(str.substring(0, eq).trim(), str.substring(eq + 1).trim())
 }
 
@@ -244,7 +244,7 @@ fun parserLibraryDescriptors(libJsons: Map<String, JsonObject>): Map<String, Lib
     return libJsons.mapValues {
         LibraryDefinition(
                 dependencies = it.value.array<String>("dependencies")?.toList().orEmpty(),
-                variables = it.value.array<String>("arguments")?.map(::parseLibraryArgument).orEmpty(),
+                variables = it.value.obj("properties")?.map { Variable(it.key, it.value.toString()) }.orEmpty(),
                 imports = it.value.array<String>("imports")?.toList().orEmpty(),
                 repositories = it.value.array<String>("repositories")?.toList().orEmpty(),
                 init = it.value.array<String>("init")?.toList().orEmpty(),
diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
index f21da7fa4..103f42e40 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
@@ -20,24 +20,21 @@ class LibrariesProcessor {
      * @return A name-to-value map of library arguments
      */
     private fun substituteArguments(parameters: List<Variable>, arguments: List<Variable>): Map<String, String> {
-        val firstNamed = arguments.indexOfFirst { it.name != null }
-        if (firstNamed != -1 && arguments.asSequence().drop(firstNamed).any { it.name == null })
-            throw ReplCompilerException("Mixing named and positional arguments is not allowed")
-        val parameterNames = parameters.map { it.name!! }.toSet()
         val result = mutableMapOf<String, String>()
-        for (i in 0 until arguments.count()) {
-            if (i >= parameters.count())
+        if (arguments.any { it.name.isEmpty() }) {
+            if (parameters.count() != 1)
+                throw ReplCompilerException("Unnamed argument is allowed only if library has a single property")
+            if (arguments.count() != 1)
                 throw ReplCompilerException("Too many arguments")
-            val name = arguments[i].name?.also {
-                if (!parameterNames.contains(it)) throw ReplCompilerException("Can not find parameter with name '$it'")
-            } ?: parameters[i].name!!
+            result[parameters[0].name] = arguments[0].value
+            return result
+        }
 
-            if (result.containsKey(name)) throw ReplCompilerException("An argument for parameter '$name' is already passed")
-            result[name] = arguments[i].value!!
+        arguments.forEach {
+            result[it.name] = it.value
         }
         parameters.forEach {
-            if (!result.containsKey(it.name!!)) {
-                if (it.value == null) throw ReplCompilerException("No value passed for parameter '${it.name}'")
+            if (!result.containsKey(it.name)) {
                 result[it.name] = it.value
             }
         }
@@ -99,8 +96,7 @@ class LibrariesProcessor {
                     ?: throw ReplCompilerException("Unknown library '$name'")
 
             // treat single strings in parsed arguments as values, not names
-            val arguments = vars.map { if (it.value == null) Variable(null, it.name) else it }
-            val mapping = substituteArguments(library.variables, arguments)
+            val mapping = substituteArguments(library.variables, vars)
 
             processedLibraries.add(LibraryWithCode(library, generateCode(repl, library, mapping)))
         }
diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt
index 8e05ce7a8..ee7cc57bf 100644
--- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt
+++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt
@@ -18,18 +18,18 @@ class ParseArgumentsTests {
         val (name, args) = parseLibraryName("lib(arg1)")
         Assert.assertEquals("lib", name)
         Assert.assertEquals(1, args.count())
-        Assert.assertEquals("arg1", args[0].name)
-        Assert.assertNull(args[0].value)
+        Assert.assertEquals("arg1", args[0].value)
+        Assert.assertEquals("", args[0].name)
     }
 
     @Test
     fun test3() {
-        val (name, args) = parseLibraryName("lib (arg1 = 1.2, arg2)")
+        val (name, args) = parseLibraryName("lib (arg1 = 1.2, arg2 = val2)")
         Assert.assertEquals("lib", name)
         Assert.assertEquals(2, args.count())
         Assert.assertEquals("arg1", args[0].name)
         Assert.assertEquals("1.2", args[0].value)
         Assert.assertEquals("arg2", args[1].name)
-        Assert.assertNull(args[1].value)
+        Assert.assertEquals("val2", args[1].value)
     }
 }
\ No newline at end of file
diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
index 8bdf6f03a..e08145d33 100644
--- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
+++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt
@@ -77,13 +77,12 @@ class ReplTest {
     fun TestUseMagic() {
         val lib1 = "mylib" to """
                     {
-                        "arguments": [
-                            "v1=0.2", 
-                            "v2=2.3"
-                        ],
+                        "properties": {
+                            "v1": "0.2"
+                        },
                         "dependencies": [
                             "artifact1:${'$'}v1",
-                            "artifact2:${'$'}v2"
+                            "artifact2:${'$'}v1"
                         ],
                         "imports": [
                             "package1",
@@ -96,10 +95,10 @@ class ReplTest {
                     }""".trimIndent()
         val lib2 = "other" to """
                                 {
-                                    "arguments": [
-                                        "a=temp", 
-                                        "b=test"
-                                    ],
+                                    "properties": {
+                                        "a": "temp", 
+                                        "b": "test"
+                                    },
                                     "dependencies": [
                                         "path-${'$'}a",
                                         "path-${'$'}b"
@@ -121,7 +120,7 @@ class ReplTest {
         arrayOf(
                 """
                     @file:DependsOn("artifact1:1.0")
-                    @file:DependsOn("artifact2:2.3")
+                    @file:DependsOn("artifact2:1.0")
                     import package1
                     import package2
                     code1

From e9121ad53c762e446a96d8ec6035a4bc4ced484b Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Sun, 22 Dec 2019 23:30:39 +0300
Subject: [PATCH 08/12] Clear 'cache/libraries' before writing downloaded files
 to it

---
 src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 38bb92c6c..f671c08bd 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -6,6 +6,7 @@ import khttp.responses.Response
 import kotlinx.coroutines.Deferred
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.async
+import org.apache.commons.io.FileUtils
 import org.jetbrains.kotlin.konan.parseKonanVersion
 import org.json.JSONObject
 import org.slf4j.LoggerFactory
@@ -198,6 +199,7 @@ fun downloadNewLibraryDescriptors() {
     val librariesDir = librariesPath.toFile()
     log.info("Saving ${libraries.count()} library descriptors to local cache at '$librariesPath'")
     try {
+        FileUtils.deleteDirectory(librariesDir)
         Files.createDirectories(librariesPath)
         libraries.forEach {
             File(librariesDir.toString(), it.first).writeText(it.second)
@@ -208,7 +210,7 @@ fun downloadNewLibraryDescriptors() {
         """.trimIndent())
     } catch (e: Exception) {
         log.error("Failed to write downloaded library descriptors to local cache:", e)
-        log.catchAll { librariesDir.delete() }
+        log.catchAll { FileUtils.deleteDirectory(librariesDir) }
     }
 }
 

From 00293b0d54007f6b02e7334032e7fd39287e94f1 Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Sun, 22 Dec 2019 23:35:53 +0300
Subject: [PATCH 09/12] Remove obsolete comment

---
 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
index 103f42e40..feb762bdd 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt
@@ -95,7 +95,6 @@ class LibrariesProcessor {
             val library = repl.config?.libraries?.awaitBlocking()?.get(name)
                     ?: throw ReplCompilerException("Unknown library '$name'")
 
-            // treat single strings in parsed arguments as values, not names
             val mapping = substituteArguments(library.variables, vars)
 
             processedLibraries.add(LibraryWithCode(library, generateCode(repl, library, mapping)))

From d54a6f74d7c434e5313fbf3fece8069bb5e8131a Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Mon, 23 Dec 2019 11:16:05 +0300
Subject: [PATCH 10/12] Fix regexp for library descriptor filename

---
 src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index f671c08bd..7d1d2be86 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -183,7 +183,7 @@ fun downloadNewLibraryDescriptors() {
         response.jsonArray.mapNotNull {
             val o = it as JSONObject
             val filename = o["name"] as String
-            if ("""[\w-]+.$LibraryDescriptorExt""".toRegex().matches(filename)) {
+            if ("""[\w-]+\.$LibraryDescriptorExt""".toRegex().matches(filename)) {
                 val libUrl = o["download_url"].toString()
                 log.info("Downloading '$filename' from $libUrl")
                 val res = getHttp(libUrl)

From 41f0211e93d412b99231648fd938b0f87d0d826b Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Mon, 23 Dec 2019 11:19:50 +0300
Subject: [PATCH 11/12] Move filenameRegex out of cycle

---
 src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 7d1d2be86..4eb524bd4 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -179,11 +179,12 @@ fun downloadNewLibraryDescriptors() {
         val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha"
         log.info("Requesting the list of library descriptors at $url")
         val response = getHttp(url)
+        val filenameRegex = """[\w-]+\.$LibraryDescriptorExt""".toRegex()
 
         response.jsonArray.mapNotNull {
             val o = it as JSONObject
             val filename = o["name"] as String
-            if ("""[\w-]+\.$LibraryDescriptorExt""".toRegex().matches(filename)) {
+            if (filenameRegex.matches(filename)) {
                 val libUrl = o["download_url"].toString()
                 log.info("Downloading '$filename' from $libUrl")
                 val res = getHttp(libUrl)

From d5eb6fc2e60adf0af7d10d488eda241c0005866b Mon Sep 17 00:00:00 2001
From: Anatoly Nikitin <Anatoly.Nikitin@jetbrains.com>
Date: Mon, 23 Dec 2019 11:23:14 +0300
Subject: [PATCH 12/12] Allow '.' in lib name

---
 src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
index 4eb524bd4..e2fe9e0fa 100644
--- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
+++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt
@@ -179,7 +179,7 @@ fun downloadNewLibraryDescriptors() {
         val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha"
         log.info("Requesting the list of library descriptors at $url")
         val response = getHttp(url)
-        val filenameRegex = """[\w-]+\.$LibraryDescriptorExt""".toRegex()
+        val filenameRegex = """[\w.-]+\.$LibraryDescriptorExt""".toRegex()
 
         response.jsonArray.mapNotNull {
             val o = it as JSONObject