diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py index 97f86dd428032f..d850936ad7f852 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py @@ -7,13 +7,14 @@ sys.path.append('./') from test_util import getExpectedCommit,\ - getBordersByTestData, getActualCommit + getBordersByTestData, getActualCommit, getCSOutput from utils.break_validator import validateBMOutput, BmValidationError from test_data import FirstBadVersionData, FirstValidVersionData,\ BmStableData, BmValidatorSteppedBreakData, BmValidatorSteppedBreakData2,\ BenchmarkAppDataUnstable, BenchmarkAppDataStable, BenchmarkAppNoDegradationData,\ BenchmarkAppUnstableDevData, BenchmarkAppWrongPathData, BenchmarkAppPathFoundData,\ - BenchmarkFirstFixedAppData, AcModeData, BenchmarkMetricData, CustomizedLogData + BenchmarkFirstFixedAppData, AcModeData, BenchmarkMetricData, CustomizedLogData, \ + MultiConfigData class CommitSliderTest(TestCase): @@ -39,6 +40,16 @@ def testCustomizedLog(self): actualCommit, _ = getActualCommit(updatedData) self.assertEqual(breakCommit, actualCommit) + @skip_commit_slider_devtest + def testMultiConfig(self): + breakCommit, updatedData = getExpectedCommit( + MultiConfigData()) + # actualCommit, _ = getActualCommit(updatedData) + out = getCSOutput(updatedData) + print(out) + # self.assertEqual(breakCommit, actualCommit) + self.assertTrue(True) + @skip_commit_slider_devtest def testBmUnstable(self): _, updatedData = getExpectedCommit( diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py index 78ed3b1328a090..24724d0b2de054 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py @@ -68,7 +68,8 @@ class TestCase(Enum): BmFirstFixed = 12, BmLatencyMetric = 13, ACModeData = 14, - CustomizedLog = 15 + CustomizedLog = 15, + MultiConfig = 16 def requireTestData(self, reqLambda): # mapping json to test data holder @@ -106,6 +107,19 @@ def __init__(self): requireBinarySearchData ) +class MultiConfigData(TestData): + def getTestCase(): + return TestData.TestCase.CustomizedLog + + def getTestName(self): + return "MultiConfig" + + def __init__(self): + from test_util import requireBinarySearchData + self.requireTestData( + requireBinarySearchData + ) + class BenchmarkAppDataUnstable(TestData): def getTestCase(): return TestData.TestCase.BmBinarySearchUnstable diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/test_util.py b/src/plugins/intel_cpu/tools/commit_slider/tests/test_util.py index ed82ca592ac335..d06448546816a8 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/test_util.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/test_util.py @@ -150,6 +150,31 @@ def getExpectedCommit(td: TestData): return breakCommit, td def getActualCommit(td: TestData): + sliderOutput = runCS(td) + rejectReason, foundCommit = parseSliderOutput(sliderOutput) + # clear temp data + [shutil.rmtree(dir) for dir in [ + td.repoName, + td.userCachePath, + td.userLogPath + ]] + os.remove(td.testCfg) + + return foundCommit, rejectReason + +def getCSOutput(td: TestData): + sliderOutput = runCS(td) + # clear temp data + [shutil.rmtree(dir) for dir in [ + td.repoName, + td.userCachePath, + td.userLogPath + ]] + os.remove(td.testCfg) + + return sliderOutput + +def runCS(td: TestData): if not td.actualDataReceived: raise TestError("Running actual commit before expected.") @@ -159,34 +184,25 @@ def getActualCommit(td: TestData): td.userLogPath, "testDir", os.getcwd()) td.userCachePath = CfgManager.singlestepStrFormat( td.userCachePath, "testDir", os.getcwd()) + td.testCfg = "test_cfg.json" for key in [ 'userLogPath', 'clearLogsAposteriori', 'userCachePath', 'clearCache' ]: cfg[key] = getattr(td, key) - testCfg = "test_cfg.json" - with open(testCfg, "w+") as customCfg: + with open(td.testCfg, "w+") as customCfg: customCfg.truncate(0) json.dump(cfg, customCfg) customCfg.close() # run slider and check output sliderOutput = runCmd( - "python3.8 commit_slider.py -cfg tests/{}".format(testCfg), + "python3.8 commit_slider.py -cfg tests/{}".format(td.testCfg), "../") sliderOutput = '\n'.join(map(str, sliderOutput)) - rejectReason, foundCommit = parseSliderOutput(sliderOutput) - # clear temp data - [shutil.rmtree(dir) for dir in [ - td.repoName, - td.userCachePath, - td.userLogPath - ]] - os.remove(testCfg) - - return foundCommit, rejectReason + return sliderOutput def parseSliderOutput(sliderOutput: str): rejectReason, foundCommit, matcher = None, None, None diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json index 0ec0a105752f32..bd9242d099da08 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json @@ -57,6 +57,26 @@ } } }, + "MultiConfig": { + "repoName": "firstBadVersion", + "patchedFile": "tests_res/fbv_main.cpp", + "testCfg": { + "appCmd" : "{appCmd}", + "appPath": "{appPath}", + "gitPath" : "{gitPath}", + "userLogPath" : "{testDir}/test_log/sub_log/", + "clearLogsAposteriori" : false, + "buildPath" : "{buildPath}", + "verboseOutput": false, + "runConfig" : { + "commitList" : { + "getCommitListCmd" : "git log {start}..{end} --boundary --pretty=\"%h\"" + }, + "mode" : "nop", + "traversal" : "bruteForce" + } + } + }, "BmBinarySearchUnstable": { "repoName": "BmBinarySearchUnstable", "patchedFile": "tests_res/fbv_bm_unstable_main.cpp", diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/common_mode.py b/src/plugins/intel_cpu/tools/commit_slider/utils/common_mode.py index a2bd490c608280..d0726414687627 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/common_mode.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/common_mode.py @@ -35,6 +35,9 @@ def __init__(self, cfg) -> None: self.commonLogger = util.setupLogger( "commonLogger", logPath, "common_log.log" ) + self.outLogger = util.setupLogger( + "outLogger", logPath, "out_log.log" + ) self.checkOutPathPattern = "check_output_cache.json" def isPerformanceBased(self): @@ -238,17 +241,23 @@ def setOutputInfo(self, pathCommit): def printResult(self): if not self.commitPath.metaInfo["preValidationPassed"]: - print("Preliminary check failed, reason: {}".format( + msg = "Preliminary check failed, reason: {}".format( self.commitPath.metaInfo["reason"] - )) + ) + print(msg) + self.outLogger.info(msg) elif not self.commitPath.metaInfo["postValidationPassed"]: - print("Output results invalid, reason: {}".format( + msg = "Output results invalid, reason: {}".format( self.commitPath.metaInfo["reason"] - )) + ) + print(msg) + self.outLogger.info(msg) else: for pathcommit in self.commitPath.getList(): if pathcommit.state is not Mode.CommitPath.CommitState.DEFAULT: - print(self.getCommitInfo(pathcommit)) + commitInfo = self.getCommitInfo(pathcommit) + print(commitInfo) + self.outLogger.info(commitInfo) def getCommitInfo(self, commit): # override if you need more details in output representation diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py index 177551403f96f6..a2493464de0f03 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py @@ -32,6 +32,11 @@ def getPseudoMetric(self, commit, cfg): commitLogger.info(checkOut) return + def printResult(self): + msg = "Hi there" + print(msg) + self.outLogger.info(msg) + class CheckOutputMode(Mode): def __init__(self, cfg):