Skip to content

Commit

Permalink
Merge branch 'master' into SPARK-27355
Browse files Browse the repository at this point in the history
  • Loading branch information
uncleGen authored May 7, 2019
2 parents 7cfd180 + eec1a3c commit a489327
Show file tree
Hide file tree
Showing 539 changed files with 11,123 additions and 4,608 deletions.
4 changes: 2 additions & 2 deletions LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ Python Software Foundation License
----------------------------------

pyspark/heapq3.py

python/docs/_static/copybutton.js

BSD 3-Clause
------------
Expand Down Expand Up @@ -258,4 +258,4 @@ data/mllib/images/kittens/29.5.a_b_EGDP022204.jpg
data/mllib/images/kittens/54893.jpg
data/mllib/images/kittens/DP153539.jpg
data/mllib/images/kittens/DP802813.jpg
data/mllib/images/multi-channel/chr30.4.184.jpg
data/mllib/images/multi-channel/chr30.4.184.jpg
2 changes: 0 additions & 2 deletions LICENSE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,6 @@ com.google.code.gson:gson
com.google.inject:guice
com.google.inject.extensions:guice-servlet
com.twitter:parquet-hadoop-bundle
commons-beanutils:commons-beanutils-core
commons-cli:commons-cli
commons-dbcp:commons-dbcp
commons-io:commons-io
Expand Down Expand Up @@ -490,7 +489,6 @@ Eclipse Distribution License (EDL) 1.0
org.glassfish.jaxb:jaxb-runtime
jakarta.xml.bind:jakarta.xml.bind-api
com.sun.istack:istack-commons-runtime
jakarta.activation:jakarta.activation-api


Mozilla Public License (MPL) 1.1
Expand Down
12 changes: 10 additions & 2 deletions R/pkg/R/functions.R
Original file line number Diff line number Diff line change
Expand Up @@ -3589,6 +3589,8 @@ setMethod("element_at",

#' @details
#' \code{explode}: Creates a new row for each element in the given array or map column.
#' Uses the default column name \code{col} for elements in the array and
#' \code{key} and \code{value} for elements in the map unless specified otherwise.
#'
#' @rdname column_collection_functions
#' @aliases explode explode,Column-method
Expand Down Expand Up @@ -3649,7 +3651,9 @@ setMethod("sort_array",

#' @details
#' \code{posexplode}: Creates a new row for each element with position in the given array
#' or map column.
#' or map column. Uses the default column name \code{pos} for position, and \code{col}
#' for elements in the array and \code{key} and \code{value} for elements in the map
#' unless specified otherwise.
#'
#' @rdname column_collection_functions
#' @aliases posexplode posexplode,Column-method
Expand Down Expand Up @@ -3790,7 +3794,8 @@ setMethod("repeat_string",
#' \code{explode}: Creates a new row for each element in the given array or map column.
#' Unlike \code{explode}, if the array/map is \code{null} or empty
#' then \code{null} is produced.
#'
#' Uses the default column name \code{col} for elements in the array and
#' \code{key} and \code{value} for elements in the map unless specified otherwise.
#'
#' @rdname column_collection_functions
#' @aliases explode_outer explode_outer,Column-method
Expand All @@ -3815,6 +3820,9 @@ setMethod("explode_outer",
#' \code{posexplode_outer}: Creates a new row for each element with position in the given
#' array or map column. Unlike \code{posexplode}, if the array/map is \code{null} or empty
#' then the row (\code{null}, \code{null}) is produced.
#' Uses the default column name \code{pos} for position, and \code{col}
#' for elements in the array and \code{key} and \code{value} for elements in the map
#' unless specified otherwise.
#'
#' @rdname column_collection_functions
#' @aliases posexplode_outer posexplode_outer,Column-method
Expand Down
2 changes: 1 addition & 1 deletion bin/docker-image-tool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ do
if ! minikube status 1>/dev/null; then
error "Cannot contact minikube. Make sure it's running."
fi
eval $(minikube docker-env)
eval $(minikube docker-env --shell bash)
;;
u) SPARK_UID=${OPTARG};;
esac
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@
*/
public class RetryingBlockFetcherSuite {

ManagedBuffer block0 = new NioManagedBuffer(ByteBuffer.wrap(new byte[13]));
ManagedBuffer block1 = new NioManagedBuffer(ByteBuffer.wrap(new byte[7]));
ManagedBuffer block2 = new NioManagedBuffer(ByteBuffer.wrap(new byte[19]));
private final ManagedBuffer block0 = new NioManagedBuffer(ByteBuffer.wrap(new byte[13]));
private final ManagedBuffer block1 = new NioManagedBuffer(ByteBuffer.wrap(new byte[7]));
private final ManagedBuffer block2 = new NioManagedBuffer(ByteBuffer.wrap(new byte[19]));

@Test
public void testNoFailures() throws IOException, InterruptedException {
Expand Down Expand Up @@ -291,7 +291,7 @@ private static void performInteractions(List<? extends Map<String, Object>> inte
}

assertNotNull(stub);
stub.when(fetchStarter).createAndStart(any(), anyObject());
stub.when(fetchStarter).createAndStart(any(), any());
String[] blockIdArray = blockIds.toArray(new String[blockIds.size()]);
new RetryingBlockFetcher(conf, fetchStarter, blockIdArray, listener).start();
}
Expand Down
2 changes: 1 addition & 1 deletion core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@
<dependency>
<groupId>net.razorvine</groupId>
<artifactId>pyrolite</artifactId>
<version>4.13</version>
<version>4.23</version>
<exclusions>
<exclusion>
<groupId>net.razorvine</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -573,19 +573,31 @@ public boolean hasNext() {

@Override
public void loadNext() throws IOException {
synchronized (this) {
loaded = true;
if (nextUpstream != null) {
// Just consumed the last record from in memory iterator
if (lastPage != null) {
freePage(lastPage);
lastPage = null;
MemoryBlock pageToFree = null;
try {
synchronized (this) {
loaded = true;
if (nextUpstream != null) {
// Just consumed the last record from in memory iterator
if(lastPage != null) {
// Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
// method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
// sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
// `SpillableIterator` in sequence, which may happen in
// `TaskMemoryManager.acquireExecutionMemory`.
pageToFree = lastPage;
lastPage = null;
}
upstream = nextUpstream;
nextUpstream = null;
}
upstream = nextUpstream;
nextUpstream = null;
numRecords--;
upstream.loadNext();
}
} finally {
if (pageToFree != null) {
freePage(pageToFree);
}
numRecords--;
upstream.loadNext();
}
}

Expand Down

This file was deleted.

Large diffs are not rendered by default.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit a489327

Please sign in to comment.