forked from apache-spark-on-k8s/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #12 from yifeih/yh/ess-metadata-v1
ESS mapoutput metadata
- Loading branch information
Showing
27 changed files
with
390 additions
and
131 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
core/src/main/java/org/apache/spark/shuffle/api/CommittedPartition.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
package org.apache.spark.shuffle.api; | ||
|
||
import org.apache.spark.storage.ShuffleLocation; | ||
|
||
import java.util.Optional; | ||
|
||
public interface CommittedPartition { | ||
|
||
/** | ||
* Indicates the number of bytes written in a committed partition. | ||
* Note that returning the length is mainly for backwards compatibility | ||
* and should be removed in a more polished variant. After this method | ||
* is called, the writer will be discarded; it's expected that the | ||
* implementation will close any underlying resources. | ||
*/ | ||
long length(); | ||
|
||
/** | ||
* Indicates the shuffle location to which this partition was written. | ||
* Some implementations may not need to specify a shuffle location. | ||
*/ | ||
Optional<ShuffleLocation> shuffleLocation(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 32 additions & 0 deletions
32
core/src/main/java/org/apache/spark/shuffle/external/ExternalCommittedPartition.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
package org.apache.spark.shuffle.external; | ||
|
||
import org.apache.spark.shuffle.api.CommittedPartition; | ||
import org.apache.spark.storage.ShuffleLocation; | ||
|
||
import java.util.Optional; | ||
|
||
public class ExternalCommittedPartition implements CommittedPartition { | ||
|
||
private final long length; | ||
private final Optional<ShuffleLocation> shuffleLocation; | ||
|
||
public ExternalCommittedPartition(long length) { | ||
this.length = length; | ||
this.shuffleLocation = Optional.empty(); | ||
} | ||
|
||
public ExternalCommittedPartition(long length, ShuffleLocation shuffleLocation) { | ||
this.length = length; | ||
this.shuffleLocation = Optional.of(shuffleLocation); | ||
} | ||
|
||
@Override | ||
public long length() { | ||
return length; | ||
} | ||
|
||
@Override | ||
public Optional<ShuffleLocation> shuffleLocation() { | ||
return shuffleLocation; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
core/src/main/java/org/apache/spark/shuffle/external/ExternalShuffleLocation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
package org.apache.spark.shuffle.external; | ||
|
||
import org.apache.hadoop.mapreduce.task.reduce.Shuffle; | ||
import org.apache.spark.network.protocol.Encoders; | ||
import org.apache.spark.storage.ShuffleLocation; | ||
|
||
import java.io.*; | ||
|
||
public class ExternalShuffleLocation implements ShuffleLocation { | ||
|
||
private String shuffleHostname; | ||
private int shufflePort; | ||
|
||
public ExternalShuffleLocation() { /* for serialization */ } | ||
|
||
public ExternalShuffleLocation(String shuffleHostname, int shufflePort) { | ||
this.shuffleHostname = shuffleHostname; | ||
this.shufflePort = shufflePort; | ||
} | ||
|
||
@Override | ||
public void writeExternal(ObjectOutput out) throws IOException { | ||
out.writeUTF(shuffleHostname); | ||
out.writeInt(shufflePort); | ||
} | ||
|
||
@Override | ||
public void readExternal(ObjectInput in) throws IOException { | ||
this.shuffleHostname = in.readUTF(); | ||
this.shufflePort = in.readInt(); | ||
} | ||
|
||
public String getShuffleHostname() { | ||
return this.shuffleHostname; | ||
} | ||
|
||
public int getShufflePort() { | ||
return this.shufflePort; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.