Skip to content

Commit

Permalink
Merge pull request #47 from open-estuary/caliper_integ
Browse files Browse the repository at this point in the history 
Caliper integration branch merge to Main branch.
  • Loading branch information
@shyjupv
shyjupv authored Nov 3, 2016
2 parents a8dd845 + 2031e6a commit 17049ee
Show file tree
Hide file tree
Showing 64 changed files with 2,110 additions and 507 deletions.
108 changes: 90 additions & 18 deletions caliper
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ import signal
import datetime
import commands
import subprocess
import glob

from argparse import ArgumentParser

from caliper.client.parser.dictionary import openssl
from caliper.client.shared.caliper_path import folder_ope as FOLDER
try:
import caliper.common as common
Expand All @@ -37,20 +40,64 @@ from caliper.client.shared import send_mails
from caliper.server import summary
from caliper.server.parser_process import parser as parser_engine
from caliper.server.parser_process import normalize
from caliper.server.parser_process import openssl
from caliper.server.parser_process import hardware
from caliper.server.parser_process import performance_functional
from caliper.client.shared.caliper_path import intermediate


logging.basicConfig(level=logging.INFO)
def build_all_tests(remote_host,f_option):
def build_all_tests(remote_host,f_option,clear):
try:
logging.info("Begining to build Caliper for the remote host")
result1 = build.build_for_target(remote_host,f_option)
result1 = build.build_for_target(remote_host,f_option,clear)
except Exception, e:
logging.info(e)
raise
else:
return result1

def populate_excels():
template_dir = caliper_path.TEMPLATE_DATA_DIR
input_cov = caliper_path.COV_DATA_DIR_INPUT
output_cov = caliper_path.COV_DATA_DIR_OUTPUT
input_path = caliper_path.CONSOLIDATED_DATA_DIR_INPUT
output_path = caliper_path.EXCEL_DATA_DIR_OUTPUT
file_list = []


col = performance_functional.col_start

cov_files_list = performance_functional.get_cov_file_list(input_cov)

for item in cov_files_list:
file_name = ".".join(item[0].split("/")[-1].split(".")[0:-1])
output_excel_path = os.path.join(output_cov, file_name)
if not os.path.exists(output_excel_path):
os.makedirs(output_excel_path)
try:
col_end = performance_functional.populate_excel(item, output_excel_path, template_dir,1)
performance_functional.mean_cov_populate( output_excel_path, col_end,file_name)#for each platforms in the Input_Cov folder generate the Iteration excel
performance_functional.get_COV_excel(output_excel_path,file_name,output_cov,col,template_dir)#copy the COV values to final COV excel
except Exception as e:
logging.info(e)
col += 1


for files in glob.glob(os.path.join(input_path, "*yaml")):
file_list.append(files)
col_end = performance_functional.populate_excel(file_list,output_path,template_dir,0)


try:
openssl.populate_excel()

hardware.populate_excel()
except Exception as e:
logging.info(e)



def run_caliper(remote_host,f_option):
try:
logging.debug("begining to run Caliper")
Expand Down Expand Up @@ -167,9 +214,11 @@ def normalize_scores():
if __name__=="__main__":
parser = ArgumentParser()
parser.add_argument("-b", "--build", action="store_true",
dest="build", default=True,help="select to build the selected test tools")
dest="build", default=True,help="select to incrementally build the selected test tools")
parser.add_argument("-B", "--nobuild", action="store_false",
dest="build", help="select not to build process of test tools")
dest="build", help="select not to incrementally build process of test tools")
parser.add_argument("-c", "--cleanbuild", action="store_true",
dest="cleanbuild", help="select to clean build the selected tools")
parser.add_argument("-r", "--run", action="store_true", dest="run", default=True,
help="select to run the selected test tools")
parser.add_argument("-R", "--norun", action="store_false", dest="run", help=
Expand All @@ -188,11 +237,29 @@ if __name__=="__main__":
default=False, help="Select to send mail to the receivers or not")
parser.add_argument( "-f", "--folder", action="store", dest="folder",
default="", help="assign a folder to store the results")
parser.add_argument("-c", "--config", action="store", dest="config_file",
parser.add_argument("-C", "--config", action="store", dest="config_file",
default="", help="specify the location of config file")
args = parser.parse_args()
start_time = datetime.datetime.now()

f_option = 0
if args.folder:
f_option = 1
caliper_path.folder_ope.workspace = args.folder
caliper_path.folder_ope.set_up_path()
from caliper.client.shared.caliper_path import folder_ope as FOLDER
if os.path.exists(FOLDER.caliper_log_file):
fp = open(FOLDER.caliper_log_file,'a+')
fp.write("re-execution with -f .It may be execution of all the tools or some specific tools as specified in the config file \n")
fp.close()

if not os.path.exists(caliper_path.folder_ope.workspace):
try:
os.makedirs(caliper_path.folder_ope.workspace)
except Exception as e:
print e
sys.exit(1)

if args.config_file:
caliper_path.config_files.name = args.config_file
caliper_path.config_files.setup_path()
Expand All @@ -206,23 +273,27 @@ if __name__=="__main__":
caliper_path.config_files.config_dir,
'client_config.cfg'
))
remote_host = get_remote_host()
f_option = 0
if args.folder:
f_option = 1
caliper_path.folder_ope.name = args.folder
caliper_path.folder_ope.set_up_path()
if os.path.exists(FOLDER.caliper_log_file):
fp = open(FOLDER.caliper_log_file,'a+')
fp.write("re-execution with -f .It may be execution of all the tools or some specific tools as specified in the config file \n")
fp.close()
if not os.path.exists(caliper_path.folder_ope.name):
os.makedirs(caliper_path.folder_ope.name)
else:
caliper_path.config_files.name = caliper_path.folder_ope.workspace
caliper_path.config_files.setup_path()
if not os.path.exists(caliper_path.folder_ope.workspace +"/test_cases_cfg"):
shutil.copytree(caliper_path.caliper_test_def, caliper_path.folder_ope.workspace + "/test_cases_cfg")

if not os.path.exists(caliper_path.folder_ope.workspace + "/config"):
shutil.copytree(caliper_path.caliper_config_file, caliper_path.folder_ope.workspace + "/config")

settings.set_config_files(os.path.join(
caliper_path.config_files.config_dir,
'client_config.cfg'
))
remote_host = get_remote_host()
caliper_path.create_dir()
if args.build:
try:
result1 = build_all_tests(remote_host,f_option)
clear = 0
if args.cleanbuild:
clear = 1
result1 = build_all_tests(remote_host,f_option,clear)
if result1:
sys.exit()
except Exception, e:
Expand Down Expand Up @@ -269,6 +340,7 @@ if __name__=="__main__":

if args.webpage and not args.send_email:
try:
populate_excels()
flag = normalize_scores()
write_results.normalize_caliper()
normalize.normalise()
Expand Down
138 changes: 119 additions & 19 deletions client/shared/caliper_path.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
import os
import sys
import subprocess
import datetime
import shutil
import ConfigParser
import logging


def judge_caliper_installed():
Expand All @@ -22,41 +26,133 @@ def judge_caliper_installed():
return 1
else:
return 0



CURRENT_PATH = os.path.dirname(sys.modules[__name__].__file__)
CALIPER_DIR = os.path.abspath(os.path.join(CURRENT_PATH, '..', '..'))
PARSER_DIR = os.path.abspath(os.path.join(CALIPER_DIR, 'client', 'parser'))
FRONT_TMP_DIR = os.path.join(CALIPER_DIR, 'frontend')



caliper_output = os.path.join(os.environ['HOME'], 'caliper_output', 'configuration')
caliper_config_file = os.path.join(caliper_output,'config')
caliper_test_def = os.path.join(caliper_output,'test_cases_cfg')
TIMP_STAMP = datetime.datetime.now().strftime("%y-%m-%d_%H-%M-%S")



# FETCHING THE TARGET HOST NAME AND ARCH FOR CREATING THE WORKSPACE
def ConfigValue(path=None,section=None,key=None,action='get',value=None):
config = ConfigParser.ConfigParser()
config.read(path)
if action == 'get':
return config.get(section, key)
else:
return config.set(section,key,value)

client_ip = ConfigValue(path=os.path.join(caliper_output,'config','client_config.cfg'), section='CLIENT', key='ip',action='get')
client_user = ConfigValue(path=os.path.join(caliper_output,'config','client_config.cfg'), section='CLIENT', key='user',action='get')
platForm_name = ConfigValue(path=os.path.join(caliper_output,'config','client_config.cfg'), section='CLIENT', key='Platform_name',action='get')

if not platForm_name:
try:
hostName = subprocess.Popen('ssh '+str(client_user)+"@"+str(client_ip)+" 'hostname'", shell=True,
stdout=subprocess.PIPE)
hostName = hostName.communicate()
except Exception as e:
logging.error(e)
sys.exit(1)
WORKSPACE = os.path.join(os.environ['HOME'],'caliper_output', str(hostName[0].strip()) + '_WS_'+ TIMP_STAMP)
else:
WORKSPACE = os.path.join(os.environ['HOME'], 'caliper_output',
str(platForm_name) + '_WS_' + TIMP_STAMP)
intermediate = 0

def create_folder(folder, mode=0755):
if os.path.exists(folder):
shutil.rmtree(folder)
try:
os.mkdir(folder, mode)
except OSError:
os.makedirs(folder, mode)

def create_dir():

if not os.path.exists(FRONT_END_DIR):
shutil.copytree(FRONT_TMP_DIR,
FRONT_END_DIR)
if not os.path.exists(HTML_DATA_DIR_INPUT):
create_folder(HTML_DATA_DIR_INPUT)
if not os.path.exists(HTML_DATA_DIR_OUTPUT):
create_folder(HTML_DATA_DIR_OUTPUT)

if not os.path.exists(DATA_DIR_INPUT):
create_folder(DATA_DIR_INPUT)
if not os.path.exists(OPENSSL_DATA_DIR_INPUT):
create_folder(OPENSSL_DATA_DIR_INPUT)
if not os.path.exists(COV_DATA_DIR_INPUT):
create_folder(COV_DATA_DIR_INPUT)
for i in range(1,6):
if not os.path.exists(os.path.join(COV_DATA_DIR_INPUT,str(i))):
create_folder(os.path.join(COV_DATA_DIR_INPUT,str(i)))
if not os.path.exists(CONSOLIDATED_DATA_DIR_INPUT):
create_folder(CONSOLIDATED_DATA_DIR_INPUT)
if not os.path.exists(HW_DATA_DIR_INPUT):
create_folder(HW_DATA_DIR_INPUT)
if not os.path.exists(HTML_DATA_DIR):
create_folder(HTML_DATA_DIR)
if not os.path.exists(COV_DATA_DIR_OUTPUT):
create_folder(COV_DATA_DIR_OUTPUT)
if not os.path.exists(EXCEL_DATA_DIR_OUTPUT):
create_folder(EXCEL_DATA_DIR_OUTPUT)
if not os.path.exists(TEMPLATE_DATA_DIR):
create_folder(TEMPLATE_DATA_DIR)








if not judge_caliper_installed():
# This means caliper is not installed and execution will be local.
# Output folders are created with in the local directory structure.
# This will allow multi instance of caliper to execute.
# fixme CALIPER_REPORT_HOME ??? replace it with CALIPER_REPORT_HOME
CALIPER_REPORT_HOME = os.path.join(CALIPER_DIR, 'caliper_output')
if not os.path.exists(CALIPER_REPORT_HOME):
os.mkdir(CALIPER_REPORT_HOME)
BENCHS_DIR = os.path.join(CALIPER_DIR, 'benchmarks')
else:
# This means that the caliper is already installed. Only instance can
# execute as the updatation of the results will happen under
# ~/home/user/caliper_workspace
CALIPER_TMP_DIR = os.path.join(os.environ['HOME'], 'caliper_output')
if not os.path.exists(CALIPER_TMP_DIR):
os.mkdir(CALIPER_TMP_DIR)
CALIPER_REPORT_HOME = CALIPER_TMP_DIR
BENCHS_DIR = os.path.join(os.environ['HOME'], '.caliper', 'benchmarks')

BUILD_FILE = 'build.sh'
BUILD_TIME = os.path.join(CALIPER_DIR,"server","build","building_timing.yaml")
SOURCE_BUILD_FILE = os.path.join(CALIPER_DIR, 'server', 'build', 'build.sh')
TMP_DIR = os.path.join('/tmp', 'caliper.tmp')
TMP_DIR = os.path.join('/tmp', 'caliper.tmp'+ "_" + TIMP_STAMP)
GEN_DIR = os.path.join(CALIPER_REPORT_HOME, 'binary')

FRONT_END_DIR = os.path.join(CALIPER_REPORT_HOME, 'frontend')
BUILD_LOGS = os.path.join(os.environ['HOME'],'.caliper_build_logs')
FRONT_END_DIR = os.path.join(CALIPER_REPORT_HOME,'frontend')
HTML_DATA_DIR = os.path.join(FRONT_END_DIR, 'frontend', 'data_files')
HTML_DATA_DIR_INPUT = os.path.join(HTML_DATA_DIR, 'Input_Logs')

DATA_DIR_INPUT = os.path.join(HTML_DATA_DIR, 'Input_Logs')
HTML_DATA_DIR_INPUT = os.path.join(DATA_DIR_INPUT, 'Input_Report')
OPENSSL_DATA_DIR_INPUT = os.path.join(DATA_DIR_INPUT,'Input_Openssl')
COV_DATA_DIR_INPUT = os.path.join(DATA_DIR_INPUT,'Input_Cov')
CONSOLIDATED_DATA_DIR_INPUT = os.path.join(DATA_DIR_INPUT,'Input_Consolidated')
HW_DATA_DIR_INPUT = os.path.join(DATA_DIR_INPUT,'Input_Hardware')
HW_DATA_DIR_OUTPUT = os.path.join(FRONT_END_DIR, 'polls', 'static', 'TargetInfo')
HTML_DATA_DIR_OUTPUT = os.path.join(HTML_DATA_DIR, 'Normalised_Logs')
COV_DATA_DIR_OUTPUT = os.path.join(FRONT_END_DIR, 'polls', 'static', 'TestInfo','Iterations')
EXCEL_DATA_DIR_OUTPUT = os.path.join(FRONT_END_DIR, 'polls', 'static', 'TestInfo','Report-Data')
TEMPLATE_DATA_DIR = os.path.join(FRONT_END_DIR,'polls','templates','polls')

HTML_PICTURE_DIR = os.path.join(FRONT_END_DIR, 'polls', 'static', 'polls',
'pictures')

Expand All @@ -81,6 +177,7 @@ def __new__(cls, *args, **kwargs):


class Folder(Singleton):
workspace = ''
name = ''
build_dir = ''
exec_dir = ''
Expand All @@ -92,26 +189,29 @@ class Folder(Singleton):

def __init__(self, folder=""):
if folder:
self.name = folder
self.workspace = folder
if not folder:
self.name = '_'.join(['output', str(get_caliper_num())])
self.name = os.path.join(CALIPER_REPORT_HOME, self.name)
self.workspace = WORKSPACE
self.name = os.path.join('output')

def set_up_path(self):
self.build_dir = os.path.join(CALIPER_REPORT_HOME, self.name,
self.workspace = os.path.join(os.path.join(os.environ['HOME'], 'caliper_output', self.workspace))
self.name = os.path.join('output')
self.build_dir = os.path.join(self.workspace, self.name,
'caliper_build')
self.exec_dir = os.path.join(CALIPER_REPORT_HOME, self.name,
self.exec_dir = os.path.join(self.workspace, self.name,
'caliper_exec')
self.results_dir = os.path.join(CALIPER_REPORT_HOME, self.name,
self.results_dir = os.path.join(self.workspace, self.name,
'results')
self.caliper_log_file = os.path.join(CALIPER_REPORT_HOME, self.name,
self.caliper_log_file = os.path.join(self.workspace, self.name,
'caliper_exe.log')
self.summary_file = os.path.join(CALIPER_REPORT_HOME, self.name,
self.summary_file = os.path.join(self.workspace, self.name,
'results_summary.log')
self.final_parser = os.path.join(CALIPER_REPORT_HOME, self.name,'final_parsing_logs.yaml')
self.final_parser = os.path.join(self.workspace, self.name,'final_parsing_logs.yaml')
self.yaml_dir = os.path.join(self.results_dir, 'yaml')
self.html_dir = os.path.join(self.results_dir, 'html')
self.name = os.path.join(CALIPER_REPORT_HOME, self.name)



folder_ope = Folder()
folder_ope.set_up_path()
Expand All @@ -127,7 +227,7 @@ def __init__(self, folder=""):
self.name = os.path.abspath(folder)
else:
if judge_caliper_installed():
self.name = os.path.join('/etc', 'caliper')
self.name = caliper_output
else:
self.name = CALIPER_DIR

Expand Down
Loading

0 comments on commit 17049ee

Please sign in to comment.