Skip to content

Commit

Permalink
init zeph artifact evaluation
Browse files Browse the repository at this point in the history
  • Loading branch information
@nicolas-kuechler
nicolas-kuechler committed Apr 1, 2021
0 parents commit 77c6705
Show file tree
Hide file tree
Showing 337 changed files with 49,296 additions and 0 deletions.
120 changes: 120 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/data
/results/old
/results/review-results
/dashboard
/notebooks
/aws
.ipynb_checkpoints/

# Avro Generated Files
/zeph-shared/src/main/avro/ch/ethz/infk/pps/shared/avro

# keys
ansible/roles/common/files
*.pem

#*.pdf

### Ansible ###
*.retry
.boto

# Avro
*.avro

# Compiled class file
*.class

# Log file
*.log

# BlueJ files
*.ctxt

# Mobile Tools for Java (J2ME)
.mtj.tmp/

# Package Files #
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar

# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*


# Maven
target/
pom.xml.tag
pom.xml.releaseBackup
pom.xml.versionsBackup
pom.xml.next
release.properties
dependency-reduced-pom.xml
buildNumber.properties
.mvn/timing.properties
# https://github.com/takari/maven-wrapper#usage-without-binary-jar
.mvn/wrapper/maven-wrapper.jar


# Eclipse

*.project
*.classpath

.metadata
bin/
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.settings/
.loadpath
.recommenders

# External tool builders
.externalToolBuilders/

# Locally stored "Eclipse launch configurations"
*.launch


# CDT- autotools
.autotools

# Java annotation processor (APT)
.factorypath

# PDT-specific (PHP Development Tools)
.buildpath

# sbteclipse plugin
.target

# Tern plugin
.tern-project

# TeXlipse plugin
.texlipse

# STS (Spring Tool Suite)
.springBeans

# Code Recommenders
.recommenders/

# Annotation Processing
.apt_generated/
.apt_generated_test/

# Scala IDE specific (Scala & Java development for Eclipse)
.cache-main
.scala_dependencies
.worksheet

221 changes: 221 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"compounds": [
{
"name": "E2E Client - 3 Producer + 3 Controller",
"configurations": ["E2E Controller Partition", "E2E Producer Partition"]
}
],
"inputs": [
{
"type": "promptString",
"id": "experimentId",
"description": "Experiment ID?",
"default": "exp1"
},
{
"type": "promptString",
"id": "universeId",
"description": "Universe ID?"
},

],
"configurations": [
{
"type": "java",
"name": "Launch DigestOp",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.shared.DigestOp",
"projectName": "zeph-shared"
},
{
"type": "java",
"name": "Launch DataLoader",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.crypto.util.DataLoader",
"projectName": "zeph-crypto"
},
{
"type": "java",
"name": "Zeph Server: Master",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.server.master.MasterApp",
"projectName": "zeph-server-master",
"args": [
"--bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--interactive-queries-server=127.0.0.1:1234",
"--state-dir=${config:zeph.stateDir}",
"--stream-threads=4",
"--time-to-commit=6000",
"--universe-partitions=3",
"--universe-replications=1"
]
},
{
"type": "java",
"name": "Zeph Server: Worker",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.server.worker.WorkerApp",
"projectName": "zeph-server-worker",
"args": [
"--bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--state-dir=${config:zeph.stateDir}",
"--stream-threads=4",
"--grace-size=5000",
"--partitions=3",
"--replications=1",
"--retention-time=24",
"--universe-id=${input:universeId}",
"--universe-size=${config:zeph.minUniverseSize}",
"--window-size=${config:zeph.e2e.windowSize}"
]
},
{
"type": "java",
"name": "E2E Producer Partition",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.benchmark.macro.Main",
"projectName": "zeph-macrobenchmarks",
"args": [
"--end-to-end",
"--zeph",
"--zeph",
"--zeph",
"--e2e-producer",
"--e2e-producer",
"--e2e-producer",
"--experiment-id=${input:experimentId}",
"--experiment-id=${input:experimentId}",
"--experiment-id=${input:experimentId}",
"--test-time=${config:zeph.e2e.testTime}",
"--test-time=${config:zeph.e2e.testTime}",
"--test-time=${config:zeph.e2e.testTime}",
"--out-file=${input:experimentId}_out.json",
"--kafka-bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--universes=${input:universeId}",
"--universe-size=3",
"--window-size=${config:zeph.e2e.windowSize}",
"--producer-partition=0",
"--producer-partition-size=3",
"--producer-partition-count=1",
"--expo-delay-mean=${config:zeph.e2e.expoDelayMean}",
"--data-dir=${config:zeph.dataDir}",
"--controller-partition=0",
"--controller-partition-size=3",
"--controller-partition-count=1",
"--universe-min-size=${config:zeph.minUniverseSize}",
"--alpha=0.5",
"--delta=0.00005",
"--producers-in-controller=1",
"--universes-in-controller=1"
]
},
{
"type": "java",
"name": "E2E Controller Partition",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.benchmark.macro.Main",
"projectName": "zeph-macrobenchmarks",
"args": [
"--end-to-end",
"--zeph",
"--zeph",
"--zeph",
"--e2e-controller",
"--e2e-controller",
"--e2e-controller",
"--experiment-id=${input:experimentId}",
"--experiment-id=${input:experimentId}",
"--experiment-id=${input:experimentId}",
"--test-time=${config:zeph.e2e.testTime}",
"--test-time=${config:zeph.e2e.testTime}",
"--test-time=${config:zeph.e2e.testTime}",
"--out-file=${input:experimentId}_out.json",
"--kafka-bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--universes=${input:universeId}",
"--universe-size=3",
"--window-size=${config:zeph.e2e.windowSize}",
"--producer-partition=0",
"--producer-partition-size=3",
"--producer-partition-count=1",
"--expo-delay-mean=${config:zeph.e2e.expoDelayMean}",
"--data-dir=${config:zeph.dataDir}",
"--controller-partition=0",
"--controller-partition-size=3",
"--controller-partition-count=1",
"--universe-min-size=3",
"--alpha=0.5",
"--delta=0.00005",
"--producers-in-controller=1",
"--universes-in-controller=1"
]
},
{
"type": "java",
"name": "Plaintext Server",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.server.worker.WorkerApp",
"projectName": "zeph-server-worker",
"args": [
"plaintext",
"--bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--state-dir=${config:zeph.stateDir}",
"--stream-threads=4",
"--grace-size=5000",
"--partitions=3",
"--replications=1",
"--retention-time=24",
"--universe-id=${input:universeId}",
"--universe-size=${config:zeph.minUniverseSize}",
"--window-size=${config:zeph.e2e.windowSize}"
]
},
{
"type": "java",
"name": "E2E Plaintext Partition",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.benchmark.macro.Main",
"projectName": "zeph-macrobenchmarks",
"args": [
"--end-to-end",
"--plaintext",
"--e2e-producer",
"--experiment-id=${input:experimentId}",
"--test-time=${config:zeph.e2e.testTime}",
"--out-file=${input:experimentId}_out.json",
"--kafka-bootstrap-server=${config:zeph.kafkaBootstrapServers}",
"--universes=${input:universeId}",
"--universe-size=3",
"--window-size=${config:zeph.e2e.windowSize}",
"--producer-partition=0",
"--producer-partition-size=3",
"--producer-partition-count=1",
"--expo-delay-mean=${config:zeph.e2e.expoDelayMean}",
"--data-dir=${config:zeph.dataDir}"
]
},
{
"type": "java",
"name": "Debug (Launch) - Current File",
"request": "launch",
"mainClass": "${file}"
},
{
"type": "java",
"name": "Debug (Launch)-ClientDemo<zeph-client-demo>",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.clientdemo.ClientDemo",
"projectName": "zeph-client-demo"
},
{
"type": "java",
"name": "Debug (Launch)-PlaintextApp<zeph-server-worker>",
"request": "launch",
"mainClass": "ch.ethz.infk.pps.zeph.server.worker.PlaintextApp",
"projectName": "zeph-server-worker"
}
]
}
24 changes: 24 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{

"kafkapath": "/opt/kafka_2.13-2.6.0",
"kafkalogspath": "/var/lib/kafka-logs",

"zeph.kafkaBootstrapServers": "127.0.0.1:9092", // <Host:Port>
"zeph.dataDir": "data",
"zeph.stateDir": "logs",
"zeph.e2e.testTime": "60", // secs
"zeph.e2e.windowSize": "10000", // millis
"zeph.e2e.expoDelayMean": "0.1",
"zeph.minUniverseSize": "3",

"awsPrivateKeyPath": "~/.ssh/kunicola.pem",



"files.exclude": {
"**/.classpath": true,
"**/.project": true,
"**/.settings": true,
"**/.factorypath": true
}
}
Loading

0 comments on commit 77c6705

Please sign in to comment.