-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
264 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "net" | ||
| "net/http" | ||
| "os" | ||
| "testing" | ||
| "time" | ||
|
|
||
| pinot "github.com/startreedata/pinot-client-go/pinot" | ||
| "github.com/stretchr/testify/assert" | ||
|
|
||
| log "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| // getEnv retrieves the value of the environment variable named by the key. | ||
| // It returns the value, which will be the default value if the variable is not present. | ||
| func getEnv(key, defaultValue string) string { | ||
| if value, exists := os.LookupEnv(key); exists { | ||
| return value | ||
| } | ||
| return defaultValue | ||
| } | ||
|
|
||
| var ( | ||
| zookeeperPort = getEnv("ZOOKEEPER_PORT", "2123") | ||
| controllerPort = getEnv("CONTROLLER_PORT", "9000") | ||
| brokerPort = getEnv("BROKER_PORT", "8000") | ||
| ) | ||
|
|
||
| func getPinotClientFromZookeeper() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromZookeeper([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster") | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func getPinotClientFromController() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromController("localhost:" + controllerPort) | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func getPinotClientFromBroker() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromBrokerList([]string{"localhost:" + brokerPort}) | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func getCustomHttpClient() *http.Client { | ||
| httpClient := &http.Client{ | ||
| Timeout: 15 * time.Second, | ||
| Transport: &http.Transport{ | ||
| MaxIdleConns: 100, // Max idle connections in total | ||
| MaxIdleConnsPerHost: 10, // Max idle connections per host | ||
| IdleConnTimeout: 90 * time.Second, | ||
| DialContext: (&net.Dialer{ | ||
| Timeout: 30 * time.Second, | ||
| KeepAlive: 30 * time.Second, | ||
| }).DialContext, | ||
| // You may add other settings like TLS configuration, Proxy, etc. | ||
| }, | ||
| } | ||
| return httpClient | ||
| } | ||
|
|
||
| func getPinotClientFromZookeeperAndCustomHttpClient() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromZookeeperAndClient([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster", getCustomHttpClient()) | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func getPinotClientFromControllerAndCustomHttpClient() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromControllerAndClient("localhost:"+controllerPort, getCustomHttpClient()) | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func getPinotClientFromBrokerAndCustomHttpClient() *pinot.Connection { | ||
| pinotClient, err := pinot.NewFromBrokerListAndClient([]string{"localhost:" + brokerPort}, getCustomHttpClient()) | ||
| if err != nil { | ||
| log.Fatalln(err) | ||
| } | ||
| return pinotClient | ||
| } | ||
|
|
||
| func TestSendingQueriesToPinot(t *testing.T) { | ||
| pinotClients := []*pinot.Connection{ | ||
| getPinotClientFromZookeeper(), | ||
| getPinotClientFromController(), | ||
| getPinotClientFromBroker(), | ||
| getPinotClientFromZookeeperAndCustomHttpClient(), | ||
| getPinotClientFromControllerAndCustomHttpClient(), | ||
| getPinotClientFromBrokerAndCustomHttpClient(), | ||
| } | ||
|
|
||
| table := "baseballStats" | ||
| pinotQueries := []string{ | ||
| "select count(*) as cnt from baseballStats limit 1", | ||
| } | ||
|
|
||
| log.Printf("Querying SQL") | ||
| for _, query := range pinotQueries { | ||
| for i := 0; i < 100; i++ { | ||
| log.Printf("Trying to query Pinot: %v\n", query) | ||
| brokerResp, err := pinotClients[i%len(pinotClients)].ExecuteSQL(table, query) | ||
| assert.Nil(t, err) | ||
| assert.Equal(t, int64(97889), brokerResp.ResultTable.GetLong(0, 0)) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Set the Pinot version | ||
| if [ -z "${PINOT_VERSION}" ]; then | ||
| echo "PINOT_VERSION is not set. Using default version 1.0.0" | ||
| PINOT_VERSION="1.0.0" | ||
| fi | ||
|
|
||
| # Set the download URL | ||
| DOWNLOAD_URL="https://archive.apache.org/dist/pinot/apache-pinot-${PINOT_VERSION}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" | ||
|
|
||
| # Set the destination directory | ||
| if [ -z "${PINOT_HOME}" ]; then | ||
| echo "PINOT_HOME is not set. Using default directory /tmp/pinot" | ||
| PINOT_HOME="/tmp/pinot" | ||
| fi | ||
|
|
||
| # Create the destination directory | ||
| mkdir -p "${PINOT_HOME}" | ||
|
|
||
| # Download the Pinot package | ||
| curl -L "${DOWNLOAD_URL}" -o "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" | ||
|
|
||
| # Extract the downloaded package | ||
| tar -xzf "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" -C "${PINOT_HOME}" | ||
|
|
||
| # Remove the downloaded package | ||
| rm "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" | ||
|
|
||
| # Start the Pinot cluster | ||
| ${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin/bin/pinot-admin.sh QuickStart -type MULTI_STAGE & | ||
| PID=$! | ||
|
|
||
| # Print the JVM settings | ||
| jps -lvm | ||
|
|
||
| ### --------------------------------------------------------------------------- | ||
| ### Ensure Pinot cluster started correctly. | ||
| ### --------------------------------------------------------------------------- | ||
|
|
||
| echo "Ensure Pinot cluster started correctly" | ||
|
|
||
| # Wait at most 5 minutes to reach the desired state | ||
| for i in $(seq 1 150) | ||
| do | ||
| SUCCEED_TABLE=0 | ||
| for table in "airlineStats" "baseballStats" "dimBaseballTeams" "githubComplexTypeEvents" "githubEvents" "starbucksStores"; | ||
| do | ||
| QUERY="select count(*) from ${table} limit 1" | ||
| QUERY_REQUEST='curl -s -X POST -H '"'"'Accept: application/json'"'"' -d '"'"'{"sql": "'${QUERY}'"}'"'"' http://localhost:'${BROKER_PORT_FORWARD}'/query/sql' | ||
| echo ${QUERY_REQUEST} | ||
| QUERY_RES=`eval ${QUERY_REQUEST}` | ||
| echo ${QUERY_RES} | ||
|
|
||
| if [ $? -eq 0 ]; then | ||
| COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'` | ||
| if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -gt 0 ]; then | ||
| SUCCEED_TABLE=$((SUCCEED_TABLE+1)) | ||
| fi | ||
| fi | ||
| echo "QUERY: ${QUERY}, QUERY_RES: ${QUERY_RES}" | ||
| done | ||
| echo "SUCCEED_TABLE: ${SUCCEED_TABLE}" | ||
| if [ "${SUCCEED_TABLE}" -eq 6 ]; then | ||
| break | ||
| fi | ||
| sleep 2 | ||
| done | ||
|
|
||
| if [ "${SUCCEED_TABLE}" -lt 6 ]; then | ||
| echo 'Quickstart failed: Cannot confirmed count-star result from quickstart table in 5 minutes' | ||
| exit 1 | ||
| fi | ||
| echo "Pinot cluster started correctly" |