diff --git a/.github/workflows/run-matlab-tests.yml b/.github/workflows/run-matlab-tests.yml index dd855bc..82843b4 100644 --- a/.github/workflows/run-matlab-tests.yml +++ b/.github/workflows/run-matlab-tests.yml @@ -1,4 +1,4 @@ -name: MATLAB matrix tests +name: Generate Test and Coverage Artifacts on GitHub-Hosted Runner on: push: pull_request: @@ -6,11 +6,11 @@ on: schedule: - cron: '24 16 * * 2' # schedule a weekly build to keep caches warm jobs: - test-job: + my-job: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-13, macos-14] - name: Run MATLAB Tests and Generate Artifacts + name: Build Toolbox runs-on: ${{ matrix.os }} steps: - name: Support long paths @@ -24,6 +24,7 @@ jobs: uses: matlab-actions/setup-matlab@v2 with: cache: true + release: R2024b products: | Deep_Learning_Toolbox Computer_Vision_Toolbox @@ -38,13 +39,23 @@ jobs: Deep_Learning_Toolbox_Model_for_ResNet-50_Network Deep_Learning_Toolbox_Model_for_VGG-16_Network Deep_Learning_Toolbox_Model_for_VGG-19_Network - - name: Run tests and generate artifacts - uses: matlab-actions/run-tests@v2 - with: - test-results-junit: test-results/results.xml - logging-level: detailed - - name: Upload Test Results for ${{ matrix.os }} + - name: Run MATLAB build + uses: matlab-actions/run-build@v2 + - name: Upload Test and Coverage Results uses: actions/upload-artifact@v4 with: name: ${{ matrix.os }} Results path: test-results + - name: Upload Code Analysis Results + if: ${{ matrix.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v4 + with: + name: Results + path: analysis-results + - name: Upload Toolbox (mltbx) + if: ${{ matrix.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v4 + with: + name: Tool Validation Kit + path: release/Tool Validation Kit.mltbx + \ No newline at end of file diff --git a/.gitignore b/.gitignore index d7ad1ae..761a343 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ # List of untracked files to ignore +derived/ \ No newline at end of file diff --git a/UnitTestFolder/ComputerVisionToolbox/DeepLabV3PlusTests.m b/UnitTestFolder/ComputerVisionToolbox/DeepLabV3PlusTests.m index d803344..30bc882 100644 --- a/UnitTestFolder/ComputerVisionToolbox/DeepLabV3PlusTests.m +++ b/UnitTestFolder/ComputerVisionToolbox/DeepLabV3PlusTests.m @@ -21,11 +21,11 @@ 'xception','block14_sepconv2_act'); ExpAdditionalLayer = struct(... - 'resnet18', 33, ... - 'inceptionresnetv2', 33, ... - 'mobilenetv2', 39', ... - 'resnet50', 33, ... - 'xception', 39); + 'resnet18', 32, ... + 'inceptionresnetv2', 32, ... + 'mobilenetv2', 38', ... + 'resnet50', 32, ... + 'xception', 38); end %% Test Methods @@ -36,7 +36,7 @@ function checkDeepLabv3PlusLayersTest(test, SpPkgName, LayerMatch, ExpAdditional % Check the layer is created/replaced properly imageSize = [512 512]; numClasses = 21; - lgraph = deeplabv3plusLayers(imageSize, numClasses, SpPkgName, "Downsampling", 8); + lgraph = deeplabv3plus(imageSize, numClasses, SpPkgName, "Downsampling", 8); % Verify default values dlayerNames = {lgraph.Layers.Name}; diff --git a/UnitTestFolder/ComputerVisionToolbox/UnetTests.m b/UnitTestFolder/ComputerVisionToolbox/UnetTests.m index 5e41a51..d28f56c 100644 --- a/UnitTestFolder/ComputerVisionToolbox/UnetTests.m +++ b/UnitTestFolder/ComputerVisionToolbox/UnetTests.m @@ -98,11 +98,8 @@ function testDefaultValues(test) for i=1:length(idxConvLayer) paddingModes{i} = unet1.Layers(idxConvLayer(i)).PaddingMode; end - if strcmp(padding, "valid") - test.verifyEqual(paddingModes,repmat({'manual'},[1,i])); - else - test.verifyEqual(paddingModes,repmat({padding},[1,i])); - end + + test.verifyEqual(paddingModes,repmat({padding},[1,i])); end function verifyEncoderAndDecoderSections(test) @@ -110,10 +107,10 @@ function verifyEncoderAndDecoderSections(test) % of Encoder and Decoder sections. encoderDepth = 4; - unet = unetLayers([96 96], 2, 'EncoderDepth', encoderDepth); + unet1 = unetLayers([96 96], 2, 'EncoderDepth', encoderDepth); % Get the names of the layers. - layerNames = {unet.Layers(:).Name}; + layerNames = {unet1.Layers(:).Name}; encDecFlag = false([1 encoderDepth]); @@ -143,7 +140,7 @@ function verifyNumChannelsInEncoderAndDecoderSections(test) % Testpoint to verify same number of channels for last % convolution layer of each encoder and decoder section. numFirstEncoderFilters = 128; - unet = unetLayers([96 96], 2, 'NumFirstEncoderFilters', ... + unet1 = unetLayers([96 96], 2, 'NumFirstEncoderFilters', ... numFirstEncoderFilters); encoderLevel = 1; @@ -152,19 +149,19 @@ function verifyNumChannelsInEncoderAndDecoderSections(test) encoderChannel = []; decoderChannel = []; - for i = 1:length(unet.Layers) + for i = 1:length(unet1.Layers) - if contains(unet.Layers(i).Name, ['Encoder-Stage-' ... + if contains(unet1.Layers(i).Name, ['Encoder-Stage-' ... num2str(encoderLevel) '-Conv-2']) encoderChannel = [encoderChannel ... - unet.Layers(i).NumChannels]; + unet1.Layers(i).NumChannels]; encoderLevel = encoderLevel+1; end - if contains(unet.Layers(i).Name, ['Decoder-Stage-' ... + if contains(unet1.Layers(i).Name, ['Decoder-Stage-' ... num2str(decoderLevel) '-Conv-2']) - decoderChannel = [unet.Layers(i).NumChannels... + decoderChannel = [unet1.Layers(i).NumChannels... decoderChannel]; decoderLevel = decoderLevel+1; @@ -180,14 +177,14 @@ function verifyValidConvolutionPadding(test) encoderDepth = 1; numFirstEncoderFilters = 32; padding = 'valid'; - unet = unetLayers([100 100 3], 3, 'EncoderDepth', ... + unet1 = unetLayers([100 100 3], 3, 'EncoderDepth', ... encoderDepth, 'FilterSize', filterSize, ... 'NumFirstEncoderFilters', numFirstEncoderFilters, ... 'ConvolutionPadding', padding); decoderLevel = 1; numCrop2dLayers = 0; - for i = 1:length(unet.Layers) - if contains(unet.Layers(i).Name, ... + for i = 1:length(unet1.Layers) + if contains(unet1.Layers(i).Name, ... ['Crop2d-' num2str(decoderLevel)]) decoderLevel = decoderLevel+1; numCrop2dLayers = numCrop2dLayers + 1; @@ -201,7 +198,7 @@ function testDifferentParameters(test, InputSize, EncoderDepth, ... flag = true; try - unet = unetLayers(InputSize, 4,... + unet1 = unetLayers(InputSize, 4,... 'EncoderDepth', EncoderDepth, ... 'NumFirstEncoderFilters', ... NumFirstEncoderFilters, 'FilterSize', FilterSize); %#ok diff --git a/UnitTestFolder/DeepLearningToolbox/ImageDataAugmenterTests.m b/UnitTestFolder/DeepLearningToolbox/ImageDataAugmenterTests.m index a1f2683..22f6edd 100644 --- a/UnitTestFolder/DeepLearningToolbox/ImageDataAugmenterTests.m +++ b/UnitTestFolder/DeepLearningToolbox/ImageDataAugmenterTests.m @@ -43,11 +43,10 @@ function testDefaultValues(testcase) function basicBehaviorRGB(testcase) - augmenter = imageDataAugmenter('RandRotation',[-20 20]); + augmenter = instrumentedImageDataAugmenter('RandRotation',[-20 20]); A = testcase.peppersImage; out = augment(augmenter,testcase.peppersImage); - - exp = imwarp(A,affine2d(augmenter.AffineTransforms),'OutputView',imref2d(size(A))); + exp = imwarp(A,affine2d(augmenter.getAffineTransform),'OutputView',imref2d(size(A))); % Use SSIM to give us some wiggle room re: resizing algorithm, % intermediate datatype, etc. We just want to know if the same @@ -60,25 +59,16 @@ function basicBehaviorRGB(testcase) function basicBehaviorGrayscale(testcase) - augmenter = imageDataAugmenter('RandRotation',[-20 20],'RandXTranslation',[0 10]); + augmenter = instrumentedImageDataAugmenter('RandRotation',[-20 20],'RandXTranslation',[0 10]); A = testcase.cameramanImage; out = augment(augmenter,A); - rot = augmenter.Rotation; - xtrans = augmenter.XTranslation; - - testcase.verifyGreaterThanOrEqual(rot,-20); - testcase.verifyLessThanOrEqual(rot,20); - testcase.verifyGreaterThanOrEqual(xtrans,0); - testcase.verifyLessThanOrEqual(xtrans,10); - - exp = imwarp(A,affine2d(augmenter.AffineTransforms),'OutputView',imref2d(size(A))); + exp = imwarp(A,affine2d(augmenter.getAffineTransform()),'OutputView',imref2d(size(A))); ssimObserved = ssim(out,exp); - testcase.verifyGreaterThan(ssimObserved,0.9,'Incorrect augmentation for RGB rotation.'); - + testcase.verifyGreaterThan(ssimObserved,0.9,'Incorrect augmentation for RGB rotation.'); end function testRotation(testcase) @@ -152,27 +142,67 @@ function testScale(testcase) end - function testReflection(testcase) - + function testXReflection(testcase) + + import matlab.unittest.constraints.IsEqualTo; + A = testcase.cameramanImage; augmenter = imageDataAugmenter('RandXReflection',true); - - rng(1); - act = augmenter.augment(A); - exp = fliplr(A); - - testcase.verifyTrue(isequal(act,exp),'Incorrect augmentation for X reflection.'); - + + % Every result should be either flipped or not. + Aflipped = fliplr(A); + con = IsEqualTo(A) | IsEqualTo(Aflipped); + + % Use a fixed random sequence to ensure test repeatability. + orig = rng(1); + testcase.addTeardown(@rng, orig) + + flipped = false(1,100); + for n = 1:100 + act = augmenter.augment(A); + testcase.verifyThat(act, con, 'Incorrect augmentation for X reflection.'); + + flipped(n) = all(act==Aflipped, 'all'); + end + + % The number flipped ought to be close to the expected mean, 50. The exact + % number is repeatable in this test due to the fixed seed, but may alter if + % the imageDataAugmenter changes how it samples from the random stream. To + % allow for this we test that the flipping happens within the 2 sigma + % level. + testcase.verifyLessThanOrEqual(abs(sum(flipped)-50), 10, 'Incorrect augmentation for X reflection.'); + end + + function testYReflection(testcase) + + import matlab.unittest.constraints.IsEqualTo; + + A = testcase.cameramanImage; augmenter = imageDataAugmenter('RandYReflection',true); - rng(1); - act = augmenter.augment(A); - exp = flipud(A); - - testcase.verifyTrue(isequal(act,exp),'Incorrect augmentation for Y reflection.'); - + Aflipped = flipud(A); + con = IsEqualTo(A) | IsEqualTo(Aflipped); + + % Use a fixed random sequence to ensure test repeatability. + orig = rng(1); + testcase.addTeardown(@rng, orig) + + flipped = false(1,100); + for n = 1:100 + act = augmenter.augment(A); + testcase.verifyThat(act, con, 'Incorrect augmentation for Y reflection.'); + + flipped(n) = all(act==Aflipped, 'all'); + end + + % The number flipped ought to be close to the expected mean, 50. The exact + % number is repeatable in this test due to the fixed seed, but may alter if + % the imageDataAugmenter changes how it samples from the random stream. To + % allow for this we test that the flipping happens within the 2 sigma + % level. + testcase.verifyLessThanOrEqual(abs(sum(flipped)-50), 10, 'Incorrect augmentation for Y reflection.'); end - + function testShear(testcase) A = testcase.peppersImage; diff --git a/UnitTestFolder/DeepLearningToolbox/TrainingOptionsTests.m b/UnitTestFolder/DeepLearningToolbox/TrainingOptionsTests.m index 724c672..98275c6 100644 --- a/UnitTestFolder/DeepLearningToolbox/TrainingOptionsTests.m +++ b/UnitTestFolder/DeepLearningToolbox/TrainingOptionsTests.m @@ -41,7 +41,7 @@ function passingInvalidSolverNameThrowsError(test) % be thrown. invalidSolverName = 'NotARealSolverName'; - errorID = 'nnet_cnn:trainingOptions:InvalidSolverName'; + errorID = 'nnet_cnn:trainingOptions:InvalidTrainingOptions'; try trainingOptions(invalidSolverName); diff --git a/UnitTestFolder/DeepLearningToolbox/instrumentedImageDataAugmenter.m b/UnitTestFolder/DeepLearningToolbox/instrumentedImageDataAugmenter.m new file mode 100644 index 0000000..34aec50 --- /dev/null +++ b/UnitTestFolder/DeepLearningToolbox/instrumentedImageDataAugmenter.m @@ -0,0 +1,27 @@ +classdef instrumentedImageDataAugmenter < imageDataAugmenter + % imageDataAugmenter subclass which provides access to intermediate + % transform values for testing. + + % Copyright 2024 The MathWorks, Inc. + + methods + function obj = instrumentedImageDataAugmenter(varargin) + obj@imageDataAugmenter(varargin{:}); + end + + function T = getAffineTransform(obj, varargin) + %getAffineTransform Return the affine transform matrix + % + % getAffineTransform(obj) returns the full 3x3 matrix that the + % imageDataAugmenter used for the most recent augment() call. + % + % getAffineTransform(obj, rowidx, colidx, pageidx) indexes the above + % matrix and returns only the requested elements. + + T = obj.AffineTransforms; + if nargin>1 + T = T(varargin{:}); + end + end + end +end \ No newline at end of file diff --git a/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/regionpropsResults.mat b/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/regionpropsResults.mat index 6af86a7..1174193 100644 --- a/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/regionpropsResults.mat +++ b/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/regionpropsResults.mat @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 oid sha256:75d1c74ce45bdf85413869e34af64be8679f3f1d3f09001c6030079a580212aa -size 204 \ No newline at end of file +size 204 diff --git a/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/strelResults.mat b/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/strelResults.mat index a42d6e7..78803b6 100644 --- a/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/strelResults.mat +++ b/UnitTestFolder/ImageProcessingToolbox/ImageProcessingTest/strelResults.mat @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 oid sha256:8a2d412eb77310b627637b70b23bfdd98e521e1ffb5af9a6e5f8a366119cabd9 -size 3586 \ No newline at end of file +size 3586 diff --git a/app/Tool Validation Kit App.prj b/app/Tool Validation Kit App.prj index a14e3eb..02be3a0 100644 --- a/app/Tool Validation Kit App.prj +++ b/app/Tool Validation Kit App.prj @@ -1,5 +1,5 @@ - + Tool Validation Kit App MathWorks Consulting @@ -13,12 +13,12 @@ - 4.2.1 + 4.2.2 - ${PROJECT_ROOT} + C:\projects\tool-validation-kit\release 22279144-5ed8-4032-b62b-bf792b595b21 @@ -29,7 +29,6 @@ - ${PROJECT_ROOT}\runToolValidationKitApp.m @@ -54,11 +53,11 @@ - C:\projects\tool-validation-kit\app + C:\projects\tool-validation-kit\release - C:\Program Files\MATLAB\R2023b + C:\Program Files\MATLAB\R2024b diff --git a/buildfile.m b/buildfile.m new file mode 100644 index 0000000..5000a1b --- /dev/null +++ b/buildfile.m @@ -0,0 +1,44 @@ +function plan = buildfile +%BUILDFILE build tasks for automation + +import matlab.buildtool.tasks.* + +plan = buildplan(localfunctions); + +plan("clean") = CleanTask; + +plan("check") = CodeIssuesTask(... + SourceFiles="source", ... + IncludeSubfolders=true, ... + Results="analysis-results/codeIssues.sarif",... + WarningThreshold=38); + +plan("test") = TestTask(... + SourceFiles="source", ... + TestResults=["test-results/results.xml" "test-results/test-report.html"], ... + LoggingLevel="detailed") ... + .addCodeCoverage(... + ["test-results/coverage.xml", "test-results/coverage-report/index.html"], ... + MetricLevel="mcdc"); + +plan("toolbox").Dependencies = ["check" "test"]; +plan("toolbox").Inputs = plan.RootFolder; +plan("toolbox").Outputs = "release/Tool Validation Kit.mltbx"; + +plan.DefaultTasks = "toolbox"; +end + +function toolboxTask(ctx) +opts = matlab.addons.toolbox.ToolboxOptions(ctx.Plan.RootFolder,"266fc400-0ecc-42c5-ad93-c19364dbaf03", ... + ToolboxVersion="4.2.2", ... + AuthorName="MathWorks Consulting", ... + AuthorCompany="MathWorks, Inc.", ... + MinimumMatlabRelease="R2024b", ... + ToolboxFiles=["app", "documentation", "icons", "source", ... + "documents", "release/Tool Validation Kit App.mlappinstall", ... + "release/Tool Validation Kit.prj", "demos", "license.txt", ... + "SECURITY.md"], ... + OutputFile=ctx.Task.Outputs.paths); + +matlab.addons.toolbox.packageToolbox(opts); +end diff --git a/release/Tool Validation Kit App.mlappinstall b/release/Tool Validation Kit App.mlappinstall index 46812c0..c9f1f8a 100644 Binary files a/release/Tool Validation Kit App.mlappinstall and b/release/Tool Validation Kit App.mlappinstall differ diff --git a/release/Tool Validation Kit.mltbx b/release/Tool Validation Kit.mltbx deleted file mode 100644 index aa8262f..0000000 Binary files a/release/Tool Validation Kit.mltbx and /dev/null differ diff --git a/release/Tool Validation Kit.prj b/release/Tool Validation Kit.prj index e2874f9..31a352d 100644 --- a/release/Tool Validation Kit.prj +++ b/release/Tool Validation Kit.prj @@ -7,7 +7,7 @@ - 4.2.1 + 4.2.2 ${PROJECT_ROOT}\Tool Validation Kit.mltbx @@ -162,8 +162,7 @@ resources C:\projects\TVKMain\toolvalidationkit\documents C:\projects\TVKMain\toolvalidationkit\icons C:\projects\TVKMain\toolvalidationkit\license.txt - C:\projects\TVKMain\toolvalidationkit\MATLAB_Tool_Validation_Report_20240304T152955.docx - C:\projects\TVKMain\toolvalidationkit\README.asv + C:\projects\TVKMain\toolvalidationkit\MATLAB_Tool_Validation_Report_20240325T163939.docx C:\projects\TVKMain\toolvalidationkit\release C:\projects\TVKMain\toolvalidationkit\SECURITY.md C:\projects\TVKMain\toolvalidationkit\source @@ -176,7 +175,7 @@ resources - C:\Program Files\MATLAB\R2023b + C:\Program Files\MATLAB\R2024b diff --git a/resources/project/Root.type.Files/UnitTestFolder.type.File/DeepLearningToolbox.type.File/instrumentedImageDataAugmenter.m.type.File.xml b/resources/project/Root.type.Files/UnitTestFolder.type.File/DeepLearningToolbox.type.File/instrumentedImageDataAugmenter.m.type.File.xml new file mode 100644 index 0000000..7a6326b --- /dev/null +++ b/resources/project/Root.type.Files/UnitTestFolder.type.File/DeepLearningToolbox.type.File/instrumentedImageDataAugmenter.m.type.File.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/resources/project/Root.type.Files/app.type.File/TVKApp.mlapp.type.File.xml b/resources/project/Root.type.Files/app.type.File/TVKApp.mlapp.type.File.xml index 7a6326b..99772b4 100644 --- a/resources/project/Root.type.Files/app.type.File/TVKApp.mlapp.type.File.xml +++ b/resources/project/Root.type.Files/app.type.File/TVKApp.mlapp.type.File.xml @@ -1,4 +1,4 @@ - +