diff --git a/build.py b/build.py old mode 100644 new mode 100755 diff --git a/build_config.py b/build_config.py old mode 100644 new mode 100755 diff --git a/docs/style.css b/docs/css/style.css similarity index 100% rename from docs/style.css rename to docs/css/style.css diff --git a/docs/hail-20190225-1051-0.2.7-8839a658eb10.log b/docs/hail-20190225-1051-0.2.7-8839a658eb10.log deleted file mode 100644 index 4a4a6a9634e..00000000000 --- a/docs/hail-20190225-1051-0.2.7-8839a658eb10.log +++ /dev/null @@ -1,111 +0,0 @@ -2019-02-25 10:51:25 SparkContext: INFO: Running Spark version 2.2.0 -2019-02-25 10:51:25 NativeCodeLoader: WARN: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2019-02-25 10:51:26 SparkContext: INFO: Submitted application: Hail -2019-02-25 10:51:26 SparkContext: INFO: Spark configuration: -spark.app.name=Hail -spark.driver.extraClassPath=/Users/labbott/hail/hail/build/libs/hail-all-spark.jar -spark.executor.extraClassPath=./hail-all-spark.jar -spark.hadoop.io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec,is.hail.io.compress.BGzipCodec,is.hail.io.compress.BGzipCodecTbi,org.apache.hadoop.io.compress.GzipCodec -spark.hadoop.mapreduce.input.fileinputformat.split.minsize=1048576 -spark.jars=file:/Users/labbott/hail/hail/build/libs/hail-all-spark.jar -spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator -spark.logConf=true -spark.master=local[*] -spark.serializer=org.apache.spark.serializer.KryoSerializer -spark.submit.deployMode=client -spark.ui.showConsoleProgress=false -2019-02-25 10:51:26 SecurityManager: INFO: Changing view acls to: labbott -2019-02-25 10:51:26 SecurityManager: INFO: Changing modify acls to: labbott -2019-02-25 10:51:26 SecurityManager: INFO: Changing view acls groups to: -2019-02-25 10:51:26 SecurityManager: INFO: Changing modify acls groups to: -2019-02-25 10:51:26 SecurityManager: INFO: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(labbott); groups with view permissions: Set(); users with modify permissions: Set(labbott); groups with modify permissions: Set() -2019-02-25 10:51:26 Utils: INFO: Successfully started service 'sparkDriver' on port 57138. -2019-02-25 10:51:26 SparkEnv: INFO: Registering MapOutputTracker -2019-02-25 10:51:26 SparkEnv: INFO: Registering BlockManagerMaster -2019-02-25 10:51:26 BlockManagerMasterEndpoint: INFO: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information -2019-02-25 10:51:26 BlockManagerMasterEndpoint: INFO: BlockManagerMasterEndpoint up -2019-02-25 10:51:26 DiskBlockManager: INFO: Created local directory at /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/blockmgr-1c70d358-0651-4fd5-bb3e-b3fcb214a7e0 -2019-02-25 10:51:26 MemoryStore: INFO: MemoryStore started with capacity 366.3 MB -2019-02-25 10:51:27 SparkEnv: INFO: Registering OutputCommitCoordinator -2019-02-25 10:51:27 log: INFO: Logging initialized @3597ms -2019-02-25 10:51:27 Server: INFO: jetty-9.3.z-SNAPSHOT -2019-02-25 10:51:27 Server: INFO: Started @3773ms -2019-02-25 10:51:27 AbstractConnector: INFO: Started ServerConnector@205caa1{HTTP/1.1,[http/1.1]}{0.0.0.0:4040} -2019-02-25 10:51:27 Utils: INFO: Successfully started service 'SparkUI' on port 4040. -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@5eff1999{/jobs,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@47bb303a{/jobs/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@25b4fb0e{/jobs/job,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@69252f36{/jobs/job/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6fe10e21{/stages,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1363bde6{/stages/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@4146a781{/stages/stage,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@3c89f63d{/stages/stage/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@22f76faa{/stages/pool,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@209a221c{/stages/pool/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@3066036b{/storage,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1a65d5b9{/storage/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1e77bb77{/storage/rdd,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@350a5387{/storage/rdd/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@5c31251d{/environment,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@e33e6bc{/environment/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@50664474{/executors,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@38f96cd1{/executors/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@b25f567{/executors/threadDump,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@22377812{/executors/threadDump/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@7d2a1978{/static,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@b263d5c{/,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@73cdc358{/api,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@467302b7{/jobs/job/kill,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@d0132ad{/stages/stage/kill,null,AVAILABLE,@Spark} -2019-02-25 10:51:27 SparkUI: INFO: Bound SparkUI to 0.0.0.0, and started at http://10.1.2.93:4040 -2019-02-25 10:51:27 SparkContext: INFO: Added JAR file:/Users/labbott/hail/hail/build/libs/hail-all-spark.jar at spark://10.1.2.93:57138/jars/hail-all-spark.jar with timestamp 1551109887516 -2019-02-25 10:51:27 Executor: INFO: Starting executor ID driver on host localhost -2019-02-25 10:51:27 Utils: INFO: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57139. -2019-02-25 10:51:27 NettyBlockTransferService: INFO: Server created on 10.1.2.93:57139 -2019-02-25 10:51:27 BlockManager: INFO: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy -2019-02-25 10:51:27 BlockManagerMaster: INFO: Registering BlockManager BlockManagerId(driver, 10.1.2.93, 57139, None) -2019-02-25 10:51:27 BlockManagerMasterEndpoint: INFO: Registering block manager 10.1.2.93:57139 with 366.3 MB RAM, BlockManagerId(driver, 10.1.2.93, 57139, None) -2019-02-25 10:51:27 BlockManagerMaster: INFO: Registered BlockManager BlockManagerId(driver, 10.1.2.93, 57139, None) -2019-02-25 10:51:27 BlockManager: INFO: Initialized BlockManager: BlockManagerId(driver, 10.1.2.93, 57139, None) -2019-02-25 10:51:27 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@733086fb{/metrics/json,null,AVAILABLE,@Spark} -2019-02-25 10:51:28 Hail: INFO: SparkUI: http://10.1.2.93:4040 -2019-02-25 10:51:28 Hail: INFO: Running Hail version 0.2.7-8839a658eb10 -2019-02-25 10:51:29 root: ERROR: IOException: No FileSystem for scheme: gs -From java.io.IOException: No FileSystem for scheme: gs - at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2660) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2667) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703) - at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295) - at is.hail.utils.richUtils.RichHadoopConfiguration$.fileSystem$extension(RichHadoopConfiguration.scala:19) - at is.hail.utils.richUtils.RichHadoopConfiguration$.listStatus$extension(RichHadoopConfiguration.scala:53) - at is.hail.utils.Py4jUtils$class.ls(Py4jUtils.scala:63) - at is.hail.utils.package$.ls(package.scala:26) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:498) - at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) - at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) - at py4j.Gateway.invoke(Gateway.java:280) - at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) - at py4j.commands.CallCommand.execute(CallCommand.java:79) - at py4j.GatewayConnection.run(GatewayConnection.java:214) - at java.lang.Thread.run(Thread.java:748) - - - -2019-02-25 11:00:29 SparkContext: INFO: Invoking stop() from shutdown hook -2019-02-25 11:00:29 AbstractConnector: INFO: Stopped Spark@205caa1{HTTP/1.1,[http/1.1]}{0.0.0.0:4040} -2019-02-25 11:00:29 SparkUI: INFO: Stopped Spark web UI at http://10.1.2.93:4040 -2019-02-25 11:00:29 MapOutputTrackerMasterEndpoint: INFO: MapOutputTrackerMasterEndpoint stopped! -2019-02-25 11:00:29 MemoryStore: INFO: MemoryStore cleared -2019-02-25 11:00:29 BlockManager: INFO: BlockManager stopped -2019-02-25 11:00:29 BlockManagerMaster: INFO: BlockManagerMaster stopped -2019-02-25 11:00:29 OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: INFO: OutputCommitCoordinator stopped! -2019-02-25 11:00:29 SparkContext: INFO: Successfully stopped SparkContext -2019-02-25 11:00:29 ShutdownHookManager: INFO: Shutdown hook called -2019-02-25 11:00:29 ShutdownHookManager: INFO: Deleting directory /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/spark-27e91ae9-3b3a-4c82-ab52-8574f6289fdd/pyspark-1c2d0b8b-595e-49c6-9567-67f344dfbf5e -2019-02-25 11:00:29 ShutdownHookManager: INFO: Deleting directory /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/spark-27e91ae9-3b3a-4c82-ab52-8574f6289fdd diff --git a/docs/hail-20190225-1418-0.2.7-8839a658eb10.log b/docs/hail-20190225-1418-0.2.7-8839a658eb10.log deleted file mode 100644 index 7861d72ec71..00000000000 --- a/docs/hail-20190225-1418-0.2.7-8839a658eb10.log +++ /dev/null @@ -1,639 +0,0 @@ -2019-02-25 14:18:03 SparkContext: INFO: Running Spark version 2.2.0 -2019-02-25 14:18:03 NativeCodeLoader: WARN: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2019-02-25 14:18:03 SparkContext: INFO: Submitted application: Hail -2019-02-25 14:18:03 SparkContext: INFO: Spark configuration: -spark.app.name=Hail -spark.driver.extraClassPath=/Users/labbott/hail/hail/build/libs/hail-all-spark.jar -spark.executor.extraClassPath=./hail-all-spark.jar -spark.hadoop.io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec,is.hail.io.compress.BGzipCodec,is.hail.io.compress.BGzipCodecTbi,org.apache.hadoop.io.compress.GzipCodec -spark.hadoop.mapreduce.input.fileinputformat.split.minsize=1048576 -spark.jars=file:/Users/labbott/hail/hail/build/libs/hail-all-spark.jar -spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator -spark.logConf=true -spark.master=local[*] -spark.serializer=org.apache.spark.serializer.KryoSerializer -spark.submit.deployMode=client -spark.ui.showConsoleProgress=false -2019-02-25 14:18:03 SecurityManager: INFO: Changing view acls to: labbott -2019-02-25 14:18:03 SecurityManager: INFO: Changing modify acls to: labbott -2019-02-25 14:18:03 SecurityManager: INFO: Changing view acls groups to: -2019-02-25 14:18:03 SecurityManager: INFO: Changing modify acls groups to: -2019-02-25 14:18:03 SecurityManager: INFO: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(labbott); groups with view permissions: Set(); users with modify permissions: Set(labbott); groups with modify permissions: Set() -2019-02-25 14:18:04 Utils: INFO: Successfully started service 'sparkDriver' on port 51333. -2019-02-25 14:18:04 SparkEnv: INFO: Registering MapOutputTracker -2019-02-25 14:18:04 SparkEnv: INFO: Registering BlockManagerMaster -2019-02-25 14:18:04 BlockManagerMasterEndpoint: INFO: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information -2019-02-25 14:18:04 BlockManagerMasterEndpoint: INFO: BlockManagerMasterEndpoint up -2019-02-25 14:18:04 DiskBlockManager: INFO: Created local directory at /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/blockmgr-252405cd-da71-49bd-b304-d0995520a9b9 -2019-02-25 14:18:04 MemoryStore: INFO: MemoryStore started with capacity 366.3 MB -2019-02-25 14:18:04 SparkEnv: INFO: Registering OutputCommitCoordinator -2019-02-25 14:18:04 log: INFO: Logging initialized @3265ms -2019-02-25 14:18:04 Server: INFO: jetty-9.3.z-SNAPSHOT -2019-02-25 14:18:04 Server: INFO: Started @3401ms -2019-02-25 14:18:04 AbstractConnector: INFO: Started ServerConnector@195f89ef{HTTP/1.1,[http/1.1]}{0.0.0.0:4040} -2019-02-25 14:18:04 Utils: INFO: Successfully started service 'SparkUI' on port 4040. -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@5e342b9{/jobs,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@3011c1f8{/jobs/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@12800481{/jobs/job,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@7fab3e15{/jobs/job/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1485c062{/stages,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@75b3f5c5{/stages/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@4e5894c1{/stages/stage,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@5ef71a2f{/stages/stage/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@ab8a48a{/stages/pool,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@23231fbc{/stages/pool/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@5a8cd810{/storage,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@3ecfb0c1{/storage/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6a1e6ac2{/storage/rdd,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@78661b9b{/storage/rdd/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@445ead15{/environment,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@50f04adc{/environment/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@40f92605{/executors,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1b02bdd6{/executors/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@13e40608{/executors/threadDump,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6148020c{/executors/threadDump/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6e25db6e{/static,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@9ae1a91{/,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@35f501b1{/api,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@1be0f837{/jobs/job/kill,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@2dda2133{/stages/stage/kill,null,AVAILABLE,@Spark} -2019-02-25 14:18:05 SparkUI: INFO: Bound SparkUI to 0.0.0.0, and started at http://10.1.2.93:4040 -2019-02-25 14:18:05 SparkContext: INFO: Added JAR file:/Users/labbott/hail/hail/build/libs/hail-all-spark.jar at spark://10.1.2.93:51333/jars/hail-all-spark.jar with timestamp 1551122285118 -2019-02-25 14:18:05 Executor: INFO: Starting executor ID driver on host localhost -2019-02-25 14:18:05 Utils: INFO: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51338. -2019-02-25 14:18:05 NettyBlockTransferService: INFO: Server created on 10.1.2.93:51338 -2019-02-25 14:18:05 BlockManager: INFO: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy -2019-02-25 14:18:05 BlockManagerMaster: INFO: Registering BlockManager BlockManagerId(driver, 10.1.2.93, 51338, None) -2019-02-25 14:18:05 BlockManagerMasterEndpoint: INFO: Registering block manager 10.1.2.93:51338 with 366.3 MB RAM, BlockManagerId(driver, 10.1.2.93, 51338, None) -2019-02-25 14:18:05 BlockManagerMaster: INFO: Registered BlockManager BlockManagerId(driver, 10.1.2.93, 51338, None) -2019-02-25 14:18:05 BlockManager: INFO: Initialized BlockManager: BlockManagerId(driver, 10.1.2.93, 51338, None) -2019-02-25 14:18:05 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@30468786{/metrics/json,null,AVAILABLE,@Spark} -2019-02-25 14:18:06 Hail: INFO: SparkUI: http://10.1.2.93:4040 -2019-02-25 14:18:06 Hail: INFO: Running Hail version 0.2.7-8839a658eb10 -2019-02-25 14:20:23 root: INFO: optimize: before: -(GetField rows - (TableCollect - (TableHead 11 - (TableMapRows - (TableMapRows - (TableOrderBy (Aidx) - (TableKeyBy () False - (TableMapRows - (TableKeyBy () False - (TableMapRows - (TableRange 10 4) - (InsertFields - (SelectFields (idx) - (Ref row)) - (test - (MakeStruct - (yep - (I32 1))))))) - (InsertFields - (SelectFields (idx test) - (Ref row)))))) - (InsertFields - (MakeStruct) - (idx - (GetField idx - (Ref row))) - (`test.yep` - (GetField yep - (GetField test - (Ref row)))))) - (InsertFields - (MakeStruct) - (idx - (Let __uid_1 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField idx - (Ref row)))) - (Apply str - (GetField idx - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_1)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_1) - (I32 83)) - (Str "...")) - (Ref __uid_1)))) - (`test.yep` - (Let __uid_2 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField `test.yep` - (Ref row)))) - (Apply str - (GetField `test.yep` - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_2)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_2) - (I32 83)) - (Str "...")) - (Ref __uid_2))))))))) -2019-02-25 14:20:23 root: INFO: optimize: after: -(GetField rows - (TableCollect - (TableMapRows - (TableMapRows - (TableMapRows - (TableExplode row - (TableKeyByAndAggregate 1 10 - (TableKeyBy () False - (TableMapRows - (TableRange 10 4) - (InsertFields - (Ref row) - (test - (Literal Struct{yep:Int32} ))))) - (MakeStruct - (row - (ApplyAggOp TakeBy - ( (I32 11)) - None - ( (Ref row) - (MakeStruct - (idx - (GetField idx - (Ref row)))))))) - (MakeStruct))) - (GetField row - (Ref row))) - (MakeStruct - (idx - (GetField idx - (Ref row))) - (test.yep - (GetField yep - (GetField test - (Ref row)))))) - (MakeStruct - (idx - (Let __uid_1 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField idx - (Ref row)))) - (Apply str - (GetField idx - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_1)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_1) - (I32 83)) - (Str "...")) - (Ref __uid_1)))) - (test.yep - (Let __uid_2 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField `test.yep` - (Ref row)))) - (Apply str - (GetField `test.yep` - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_2)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_2) - (I32 83)) - (Str "...")) - (Ref __uid_2)))))))) -2019-02-25 14:20:23 root: INFO: optimize: before: -(GetField rows - (TableCollect - (TableMapRows - (TableMapRows - (TableMapRows - (TableExplode row - (TableKeyByAndAggregate 1 10 - (TableKeyBy () False - (TableMapGlobals - (TableMapRows - (TableMapGlobals - (TableRange 10 4) - (InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} )))) - (InsertFields - (Ref row) - (test - (GetField __iruid_13 - (Ref global))))) - (SelectFields () - (Ref global)))) - (MakeStruct - (row - (ApplyAggOp TakeBy - ( (I32 11)) - None - ( (Ref row) - (MakeStruct - (idx - (GetField idx - (Ref row)))))))) - (MakeStruct))) - (GetField row - (Ref row))) - (MakeStruct - (idx - (GetField idx - (Ref row))) - (test.yep - (GetField yep - (GetField test - (Ref row)))))) - (MakeStruct - (idx - (Let __uid_1 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField idx - (Ref row)))) - (Apply str - (GetField idx - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_1)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_1) - (I32 83)) - (Str "...")) - (Ref __uid_1)))) - (test.yep - (Let __uid_2 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField `test.yep` - (Ref row)))) - (Apply str - (GetField `test.yep` - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_2)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_2) - (I32 83)) - (Str "...")) - (Ref __uid_2)))))))) -2019-02-25 14:20:23 root: INFO: optimize: after: -(GetField rows - (TableCollect - (TableMapRows - (TableMapRows - (TableMapRows - (TableExplode row - (TableKeyByAndAggregate 1 10 - (TableKeyBy () False - (TableMapGlobals - (TableMapRows - (TableMapGlobals - (TableRange 10 4) - (InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} )))) - (InsertFields - (Ref row) - (test - (GetField __iruid_13 - (Ref global))))) - (SelectFields () - (Ref global)))) - (MakeStruct - (row - (ApplyAggOp TakeBy - ( (I32 11)) - None - ( (Ref row) - (MakeStruct - (idx - (GetField idx - (Ref row)))))))) - (MakeStruct))) - (GetField row - (Ref row))) - (MakeStruct - (idx - (GetField idx - (Ref row))) - (test.yep - (GetField yep - (GetField test - (Ref row)))))) - (MakeStruct - (idx - (Let __uid_1 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField idx - (Ref row)))) - (Apply str - (GetField idx - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_1)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_1) - (I32 83)) - (Str "...")) - (Ref __uid_1)))) - (test.yep - (Let __uid_2 - (If - (ApplyUnaryPrimOp Bang - (IsNA - (GetField `test.yep` - (Ref row)))) - (Apply str - (GetField `test.yep` - (Ref row))) - (Str "NA")) - (If - (ApplyComparisonOp GT - (StringLength - (Ref __uid_2)) - (I32 86)) - (Apply `+` - (ApplyIR `[:*]` - (Ref __uid_2) - (I32 83)) - (Str "...")) - (Ref __uid_2)))))))) -2019-02-25 14:20:23 root: INFO: optimize: before: -(InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} ))) -2019-02-25 14:20:23 root: INFO: optimize: after: -(InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} ))) -2019-02-25 14:20:23 root: INFO: optimize: before: -(InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} ))) -2019-02-25 14:20:23 root: INFO: optimize: after: -(InsertFields - (Ref global) - (__iruid_13 - (Literal Struct{yep:Int32} ))) -2019-02-25 14:20:23 MemoryStore: INFO: Block broadcast_0 stored as values in memory (estimated size 120.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_0_piece0 stored as bytes in memory (estimated size 75.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_0_piece0 in memory on 10.1.2.93:51338 (size: 75.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 0 from broadcast at BroadcastValue.scala:15 -2019-02-25 14:20:24 root: INFO: initop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C0. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C0.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C0.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C0.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: seqop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C1. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C1.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C1.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C1.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C2. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C2.apply instruction count: 111 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C2.apply instruction count: 28 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C2.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_1 stored as values in memory (estimated size 1464.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_1_piece0 stored as bytes in memory (estimated size 494.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_1_piece0 in memory on 10.1.2.93:51338 (size: 494.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 1 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 root: INFO: optimize: before: -(SelectFields () - (Ref global)) -2019-02-25 14:20:24 root: INFO: optimize: after: -(SelectFields () - (Ref global)) -2019-02-25 14:20:24 root: INFO: optimize: before: -(SelectFields () - (Ref global)) -2019-02-25 14:20:24 root: INFO: optimize: after: -(SelectFields () - (Ref global)) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C3. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C3.apply instruction count: 61 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C3.apply instruction count: 7 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C3.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: initop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C4. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C4.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C4.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C4.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: seqop (Begin - (SeqOp (TakeBy (Int32) None (Struct{idx:Int32,test:Struct{yep:Int32}} Struct{idx:Int32})) - (I32 0) - ( (Ref row) - (MakeStruct - (idx - (GetField idx - (Ref row))))))) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C5. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C5.apply instruction count: 23 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C5.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C5.method1 instruction count: 114 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C5.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C6. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C6.apply instruction count: 76 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C6.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C6.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_2 stored as values in memory (estimated size 56.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_2_piece0 stored as bytes in memory (estimated size 67.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_2_piece0 in memory on 10.1.2.93:51338 (size: 67.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 2 from broadcast at BroadcastValue.scala:15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C7. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C7.apply instruction count: 23 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C7.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C7.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C8. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C8.apply instruction count: 110 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C8.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C8.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C9. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C9.apply instruction count: 186 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C9.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C9.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_3 stored as values in memory (estimated size 640.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_3_piece0 stored as bytes in memory (estimated size 360.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_3_piece0 in memory on 10.1.2.93:51338 (size: 360.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 3 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_4 stored as values in memory (estimated size 640.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_4_piece0 stored as bytes in memory (estimated size 360.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_4_piece0 in memory on 10.1.2.93:51338 (size: 360.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 4 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 root: INFO: initop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C10. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C10.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C10.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C10.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: seqop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C11. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C11.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C11.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C11.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C12. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C12.apply instruction count: 37 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C12.apply instruction count: 28 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C12.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_5 stored as values in memory (estimated size 640.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_5_piece0 stored as bytes in memory (estimated size 360.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_5_piece0 in memory on 10.1.2.93:51338 (size: 360.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 5 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 root: INFO: initop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C13. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C13.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C13.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C13.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: seqop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C14. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C14.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C14.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C14.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C15. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C15.apply instruction count: 45 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C15.apply instruction count: 28 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C15.method1 instruction count: 118 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C15.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_6 stored as values in memory (estimated size 640.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_6_piece0 stored as bytes in memory (estimated size 360.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_6_piece0 in memory on 10.1.2.93:51338 (size: 360.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 6 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 root: INFO: initop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C16. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C16.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C16.apply instruction count: 16 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C16.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: seqop (Begin) -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C17. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C17.apply instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C17.apply instruction count: 22 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C17.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.apply instruction count: 45 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.apply instruction count: 28 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.method1 instruction count: 721 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.method2 instruction count: 28 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.method3 instruction count: 37 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.method4 instruction count: 15 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C18.setPartitionIndex instruction count: 14 -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_7 stored as values in memory (estimated size 640.0 B, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_7_piece0 stored as bytes in memory (estimated size 360.0 B, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_7_piece0 in memory on 10.1.2.93:51338 (size: 360.0 B, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 7 from broadcast at RVDPartitioner.scala:80 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C19. instruction count: 3 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C19.apply instruction count: 143 -2019-02-25 14:20:24 root: INFO: is/hail/codegen/generated/C19.apply instruction count: 12 -2019-02-25 14:20:24 SparkContext: INFO: Starting job: collect at RVD.scala:646 -2019-02-25 14:20:24 DAGScheduler: INFO: Registering RDD 9 (mapPartitions at ContextRDD.scala:147) -2019-02-25 14:20:24 DAGScheduler: INFO: Got job 0 (collect at RVD.scala:646) with 1 output partitions -2019-02-25 14:20:24 DAGScheduler: INFO: Final stage: ResultStage 1 (collect at RVD.scala:646) -2019-02-25 14:20:24 DAGScheduler: INFO: Parents of final stage: List(ShuffleMapStage 0) -2019-02-25 14:20:24 DAGScheduler: INFO: Missing parents: List(ShuffleMapStage 0) -2019-02-25 14:20:24 DAGScheduler: INFO: Submitting ShuffleMapStage 0 (MapPartitionsRDD[9] at mapPartitions at ContextRDD.scala:147), which has no missing parents -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_8 stored as values in memory (estimated size 23.5 KB, free 366.3 MB) -2019-02-25 14:20:24 MemoryStore: INFO: Block broadcast_8_piece0 stored as bytes in memory (estimated size 11.1 KB, free 366.3 MB) -2019-02-25 14:20:24 BlockManagerInfo: INFO: Added broadcast_8_piece0 in memory on 10.1.2.93:51338 (size: 11.1 KB, free: 366.3 MB) -2019-02-25 14:20:24 SparkContext: INFO: Created broadcast 8 from broadcast at DAGScheduler.scala:1006 -2019-02-25 14:20:24 DAGScheduler: INFO: Submitting 4 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[9] at mapPartitions at ContextRDD.scala:147) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) -2019-02-25 14:20:24 TaskSchedulerImpl: INFO: Adding task set 0.0 with 4 tasks -2019-02-25 14:20:25 TaskSetManager: INFO: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, PROCESS_LOCAL, 4702 bytes) -2019-02-25 14:20:25 TaskSetManager: INFO: Starting task 1.0 in stage 0.0 (TID 1, localhost, executor driver, partition 1, PROCESS_LOCAL, 4702 bytes) -2019-02-25 14:20:25 TaskSetManager: INFO: Starting task 2.0 in stage 0.0 (TID 2, localhost, executor driver, partition 2, PROCESS_LOCAL, 4702 bytes) -2019-02-25 14:20:25 TaskSetManager: INFO: Starting task 3.0 in stage 0.0 (TID 3, localhost, executor driver, partition 3, PROCESS_LOCAL, 4702 bytes) -2019-02-25 14:20:25 Executor: INFO: Running task 3.0 in stage 0.0 (TID 3) -2019-02-25 14:20:25 Executor: INFO: Running task 0.0 in stage 0.0 (TID 0) -2019-02-25 14:20:25 Executor: INFO: Running task 1.0 in stage 0.0 (TID 1) -2019-02-25 14:20:25 Executor: INFO: Running task 2.0 in stage 0.0 (TID 2) -2019-02-25 14:20:25 Executor: INFO: Finished task 2.0 in stage 0.0 (TID 2). 995 bytes result sent to driver -2019-02-25 14:20:25 Executor: INFO: Finished task 3.0 in stage 0.0 (TID 3). 995 bytes result sent to driver -2019-02-25 14:20:25 Executor: INFO: Finished task 0.0 in stage 0.0 (TID 0). 995 bytes result sent to driver -2019-02-25 14:20:25 Executor: INFO: Finished task 1.0 in stage 0.0 (TID 1). 995 bytes result sent to driver -2019-02-25 14:20:25 TaskSetManager: INFO: Finished task 0.0 in stage 0.0 (TID 0) in 437 ms on localhost (executor driver) (1/4) -2019-02-25 14:20:25 TaskSetManager: INFO: Finished task 2.0 in stage 0.0 (TID 2) in 416 ms on localhost (executor driver) (2/4) -2019-02-25 14:20:25 TaskSetManager: INFO: Finished task 1.0 in stage 0.0 (TID 1) in 420 ms on localhost (executor driver) (3/4) -2019-02-25 14:20:25 TaskSetManager: INFO: Finished task 3.0 in stage 0.0 (TID 3) in 412 ms on localhost (executor driver) (4/4) -2019-02-25 14:20:25 TaskSchedulerImpl: INFO: Removed TaskSet 0.0, whose tasks have all completed, from pool -2019-02-25 14:20:25 DAGScheduler: INFO: ShuffleMapStage 0 (mapPartitions at ContextRDD.scala:147) finished in 0.473 s -2019-02-25 14:20:25 DAGScheduler: INFO: looking for newly runnable stages -2019-02-25 14:20:25 DAGScheduler: INFO: running: Set() -2019-02-25 14:20:25 DAGScheduler: INFO: waiting: Set(ResultStage 1) -2019-02-25 14:20:25 DAGScheduler: INFO: failed: Set() -2019-02-25 14:20:25 DAGScheduler: INFO: Submitting ResultStage 1 (MapPartitionsRDD[27] at mapPartitions at ContextRDD.scala:147), which has no missing parents -2019-02-25 14:20:25 MemoryStore: INFO: Block broadcast_9 stored as values in memory (estimated size 53.8 KB, free 366.2 MB) -2019-02-25 14:20:25 MemoryStore: INFO: Block broadcast_9_piece0 stored as bytes in memory (estimated size 23.7 KB, free 366.2 MB) -2019-02-25 14:20:25 BlockManagerInfo: INFO: Added broadcast_9_piece0 in memory on 10.1.2.93:51338 (size: 23.7 KB, free: 366.3 MB) -2019-02-25 14:20:25 SparkContext: INFO: Created broadcast 9 from broadcast at DAGScheduler.scala:1006 -2019-02-25 14:20:25 DAGScheduler: INFO: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[27] at mapPartitions at ContextRDD.scala:147) (first 15 tasks are for partitions Vector(0)) -2019-02-25 14:20:25 TaskSchedulerImpl: INFO: Adding task set 1.0 with 1 tasks -2019-02-25 14:20:25 TaskSetManager: INFO: Starting task 0.0 in stage 1.0 (TID 4, localhost, executor driver, partition 0, ANY, 4621 bytes) -2019-02-25 14:20:25 Executor: INFO: Running task 0.0 in stage 1.0 (TID 4) -2019-02-25 14:20:25 ShuffleBlockFetcherIterator: INFO: Getting 4 non-empty blocks out of 4 blocks -2019-02-25 14:20:25 ShuffleBlockFetcherIterator: INFO: Started 0 remote fetches in 23 ms -2019-02-25 14:20:25 Executor: INFO: Finished task 0.0 in stage 1.0 (TID 4). 1218 bytes result sent to driver -2019-02-25 14:20:25 TaskSetManager: INFO: Finished task 0.0 in stage 1.0 (TID 4) in 209 ms on localhost (executor driver) (1/1) -2019-02-25 14:20:25 TaskSchedulerImpl: INFO: Removed TaskSet 1.0, whose tasks have all completed, from pool -2019-02-25 14:20:25 DAGScheduler: INFO: ResultStage 1 (collect at RVD.scala:646) finished in 0.210 s -2019-02-25 14:20:25 DAGScheduler: INFO: Job 0 finished: collect at RVD.scala:646, took 0.947350 s -2019-02-25 14:26:25 SparkContext: INFO: Invoking stop() from shutdown hook -2019-02-25 14:26:25 AbstractConnector: INFO: Stopped Spark@195f89ef{HTTP/1.1,[http/1.1]}{0.0.0.0:4040} -2019-02-25 14:26:25 SparkUI: INFO: Stopped Spark web UI at http://10.1.2.93:4040 -2019-02-25 14:26:25 MapOutputTrackerMasterEndpoint: INFO: MapOutputTrackerMasterEndpoint stopped! -2019-02-25 14:26:25 MemoryStore: INFO: MemoryStore cleared -2019-02-25 14:26:25 BlockManager: INFO: BlockManager stopped -2019-02-25 14:26:25 BlockManagerMaster: INFO: BlockManagerMaster stopped -2019-02-25 14:26:25 OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: INFO: OutputCommitCoordinator stopped! -2019-02-25 14:26:25 SparkContext: INFO: Successfully stopped SparkContext -2019-02-25 14:26:25 ShutdownHookManager: INFO: Shutdown hook called -2019-02-25 14:26:25 ShutdownHookManager: INFO: Deleting directory /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/spark-027e15db-e078-48fc-ba96-0eb45df2e213 -2019-02-25 14:26:25 ShutdownHookManager: INFO: Deleting directory /private/var/folders/k9/030l_7js6634cphk5k4rg63c940c2d/T/spark-027e15db-e078-48fc-ba96-0eb45df2e213/pyspark-3ee52666-d0b9-41a3-b47e-bf345f947811 diff --git a/docs/index.html b/docs/index.html index a3385ad2401..4399bd297a0 100644 --- a/docs/index.html +++ b/docs/index.html @@ -15,12 +15,6 @@ - - - - - -
@@ -111,6 +105,7 @@