Difference between revisions of "2009 Winter Project Week Command Line Program Testing"
(→Status) |
(→Status) |
||
Line 59: | Line 59: | ||
Add | Add | ||
| | | | ||
− | Regression | + | <span style="color:#339933">Regression</span> |
| | | | ||
${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/AddTest.nhdr | ${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/AddTest.nhdr | ||
Line 66: | Line 66: | ||
AffineRegistration | AffineRegistration | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 89: | Line 89: | ||
CheckerBoard | CheckerBoard | ||
| | | | ||
− | Regression | + | <span style="color:#339933">Regression</span> |
| | | | ||
--checkerPattern 5,3,2 | --checkerPattern 5,3,2 | ||
Line 99: | Line 99: | ||
ConfidenceConnected | ConfidenceConnected | ||
| | | | ||
− | None (Regression) | + | <span style="color:#ff0000">None</span> (<span style="color:#339933">Regression</span>) |
| | | | ||
--smoothingIterations 1 | --smoothingIterations 1 | ||
Line 114: | Line 114: | ||
CurvatureAnisotropicDiffusion | CurvatureAnisotropicDiffusion | ||
| | | | ||
− | None (Regression) | + | <span style="color:#ff0000">None</span> (<span style="color:#339933">Regression</span>) |
| | | | ||
--conductance 2 | --conductance 2 | ||
Line 125: | Line 125: | ||
DicomToNrrdConverter | DicomToNrrdConverter | ||
| | | | ||
− | None (Smoke) | + | <span style="color:#ff0000">None</span> (Smoke) |
| | | | ||
${TEST_DATA}/CTHeadAxialDicom | ${TEST_DATA}/CTHeadAxialDicom | ||
Line 133: | Line 133: | ||
DiffusionTensorEstimation | DiffusionTensorEstimation | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 139: | Line 139: | ||
DiffusionTensorMathematics | DiffusionTensorMathematics | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 145: | Line 145: | ||
DiffusionTensorTest | DiffusionTensorTest | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 151: | Line 151: | ||
DiffusionWeightedTest | DiffusionWeightedTest | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 157: | Line 157: | ||
DWIDicomLoad | DWIDicomLoad | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 163: | Line 163: | ||
ExecutionModelTour | ExecutionModelTour | ||
| | | | ||
− | None (Smoke) | + | <span style="color:#ff0000">None</span> (Smoke) |
| | | | ||
--integer 30 | --integer 30 | ||
Line 179: | Line 179: | ||
ExtractSkeleton/ExtractSkeleton | ExtractSkeleton/ExtractSkeleton | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 185: | Line 185: | ||
FreesurferSurfaceSectionExtraction | FreesurferSurfaceSectionExtraction | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 191: | Line 191: | ||
GaussianBlurImageFilter | GaussianBlurImageFilter | ||
| | | | ||
− | Regression | + | <span style="color:#339933">Regression</span> |
| | | | ||
--sigma 2.5 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/GaussianBlurImageFilterTest.nhdr | --sigma 2.5 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/GaussianBlurImageFilterTest.nhdr | ||
Line 198: | Line 198: | ||
GradientAnisotropicDiffusion | GradientAnisotropicDiffusion | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 204: | Line 204: | ||
GrayscaleFillHoleImageFilter | GrayscaleFillHoleImageFilter | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 210: | Line 210: | ||
GrayscaleGrindPeakImageFilter | GrayscaleGrindPeakImageFilter | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 216: | Line 216: | ||
GrayscaleModelMaker | GrayscaleModelMaker | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 222: | Line 222: | ||
HistogramMatching | HistogramMatching | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 228: | Line 228: | ||
ImageLabelCombine | ImageLabelCombine | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 234: | Line 234: | ||
ImageReadDicomWrite | ImageReadDicomWrite | ||
| | | | ||
− | None (Regression) | + | <span style="color:#ff0000">None</span> (<span style="color:#339933">Regression</span>) |
| | | | ||
--patientName Austrialian | --patientName Austrialian | ||
Line 255: | Line 255: | ||
LabelMapSmoothing | LabelMapSmoothing | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 277: | Line 277: | ||
MedianImageFilter | MedianImageFilter | ||
| | | | ||
− | Regression | + | <span style="color:#339933">Regression</span> |
| | | | ||
--neighborhood 1,2,3 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/MedianImageFilterTest.nhdr | --neighborhood 1,2,3 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/MedianImageFilterTest.nhdr | ||
Line 284: | Line 284: | ||
ModelMaker | ModelMaker | ||
| | | | ||
− | None (Smoke) | + | <span style="color:#ff0000">None</span> (Smoke) |
| | | | ||
--labels 2 | --labels 2 | ||
Line 293: | Line 293: | ||
MultipleModelsExample | MultipleModelsExample | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 299: | Line 299: | ||
OrientImage | OrientImage | ||
| | | | ||
− | Regression | + | <span style="color:#339933">Regression</span> |
| | | | ||
--orientation Axial ${MRML_DATA}/fixed.nrrd ${TEMP}/OrientImageTestAxial.nrrd | --orientation Axial ${MRML_DATA}/fixed.nrrd ${TEMP}/OrientImageTestAxial.nrrd | ||
Line 308: | Line 308: | ||
OtsuThresholdImageFilter | OtsuThresholdImageFilter | ||
| | | | ||
− | None | + | <span style="color:#ff0000">None</span> |
| | | | ||
|- | |- | ||
Line 314: | Line 314: | ||
OtsuThresholdSegmentation | OtsuThresholdSegmentation | ||
| | | | ||
− | None (Regression) | + | <span style="color:#773333">None</span> (<span style="color:#339933">Regression</span>) |
| | | | ||
--brightObjects | --brightObjects | ||
Line 326: | Line 326: | ||
ProbeVolumeWithModel | ProbeVolumeWithModel | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 332: | Line 332: | ||
Realign/Realign | Realign/Realign | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 338: | Line 338: | ||
ResampleVolume2 | ResampleVolume2 | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 344: | Line 344: | ||
ResampleVolume | ResampleVolume | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 350: | Line 350: | ||
RigidRegistration | RigidRegistration | ||
| | | | ||
− | None (--help) | + | <span style="color:#ff0000">None</span> (--help) |
| | | | ||
|- | |- | ||
Line 356: | Line 356: | ||
Subtract | Subtract | ||
| | | | ||
− | None (--help) | + | <span style="color:#773333">None</span> (<span style="color:#ff5555">--help</span>) |
| | | | ||
|- | |- | ||
Line 362: | Line 362: | ||
TestGridTransformRegistration | TestGridTransformRegistration | ||
| | | | ||
− | Smoke | + | <span style="color:#808000">Smoke</span> |
| | | | ||
--gridSize 5 | --gridSize 5 | ||
Line 371: | Line 371: | ||
VotingBinaryHoleFillingImageFilter | VotingBinaryHoleFillingImageFilter | ||
| | | | ||
− | None (Regression) | + | <span style="color:#ff0000">None</span> (<span style="color:#339933">Regression</span>) |
| | | | ||
--radius 3,3,3 | --radius 3,3,3 | ||
Line 380: | Line 380: | ||
${TEMP}/VotingBinaryHoleFillingImageFilterTest.nhdr | ${TEMP}/VotingBinaryHoleFillingImageFilterTest.nhdr | ||
|- | |- | ||
+ | | style="background:#339933; color:black"| | ||
+ | ZeroCrossingBasedEdgeDetectionImageFilter | ||
| | | | ||
− | + | <span style="color:#ff0000">None</span> (<span style="color:#339933">Regression</span>) | |
− | |||
− | None (Regression) | ||
| | | | ||
${TEST_DATA}/CTHeadResampled.nhdr | ${TEST_DATA}/CTHeadResampled.nhdr |
Revision as of 21:50, 2 January 2009
Home < 2009 Winter Project Week Command Line Program Testing
Key Investigators
- Bill Lorensen (His Basement)
Objective
There is NA-MIC funded infrastructure in place to regression test the command line programs found in Applications/CLI. Surprisingly, of the over 50 command line programs in the current Slicer3 distribution, only 11 have tests. This project will illustrate the importance of testing and show how easy it is to create tests for command line programs using the existing infrastructure. The project will document the process, create tests and help others to create tests.
Approach, Plan
Using the existing infrastructure, the project will create regression tests for many of the existing command line programs in Applications/CLI. The project will provide expert assistance to those who feel that creating a test is difficult. A Wiki page will provide step-by-step procedures to test the command line programs.
Progress
There is really no excuse for the lack of testing in Applications/CLI. The infrastructure that has been in place for a few years makes it almost painless to test command line programs. Better documentation and peer pressure will help improve the situation.
Tests have been created for all of the command line programs in Applications/CLI (not including sub directories). Three types of tests exist:
- --help tests just run the command line program with the --help flag. These tests are as basic as a test can be. All they really test is whether or not the command line program and build, link and report help.
- Smoke tests run the program put do not check the results of the program. These are one step above the --help tests and should be the minimum test for each program.
- Regression tests run the program and compare the resulting output with a baseline output. Currently only image regression testing is available. Each command line program that produces an image should have a regression test.
Documentation: TODO
Status
The table shows each command line program and its testing status. Under Test Type, the item in parentheses is a test that has been added as part of this project.
Program | Test Type | Command Line |
---|---|---|
Add |
Regression |
${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/AddTest.nhdr |
AffineRegistration |
None (--help) |
|
BSplineDeformableRegistration |
Smoke |
--resampledmovingfilename ${TEMP}/BSplineDeformableRegistrationTest00.nhdr --outputtransform ${TEMP}/BSplineDeformableRegistrationTest00Transform.txt --default 0 --maximumDeformation 1.0 --constrain --spatialsamples 10000 --histogrambins 32 --gridSize 5 --iterations 50 ${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr |
CheckerBoard |
Regression |
--checkerPattern 5,3,2 ${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxialDoubled.nhdr ${TEMP}/CheckerBoardTest.nhdr |
ConfidenceConnected |
None (Regression) |
--smoothingIterations 1 --timestep 0.0625 --iterations 5 --multiplier 2.5 --neighborhood 1 --labelvalue 2 --seed 9.71622,-8.1,14.2189 --seed -12.9459,-8.1,18.577 ${TEST_DATA}/MRHeadResampled.nhdr ${TEMP}/ConfidenceConnectedTest.nhdr |
CurvatureAnisotropicDiffusion |
None (Regression) |
--conductance 2 --timeStep 0.0625 --iterations 2 ${TEST_DATA}/MRHeadResampled.nhdr ${TEMP}/CurvatureAnisotropicDiffusionTest.nhdr |
DicomToNrrdConverter |
None (Smoke) |
${TEST_DATA}/CTHeadAxialDicom ${TEMP} CTHeadAxialFromDicom.nhdr |
DiffusionTensorEstimation |
None (--help) |
|
DiffusionTensorMathematics |
None (--help) |
|
DiffusionTensorTest |
None (--help) |
|
DiffusionWeightedTest |
None (--help) |
|
DWIDicomLoad |
None (--help) |
|
ExecutionModelTour |
None (Smoke) |
--integer 30 --double 30 -f 1.3,2,-14 --string_vector foo,bar,foobar --enumeration Bill --boolean1 --transform1 ${TEST_DATA}/ExecutionModelTourTest.mrml\#vtkMRMLLinearTransformNode1 --transform2 ${TEST_DATA}/ExecutionModelTourTest.mrml\#vtkMRMLLinearTransformNode2 ${TEST_DATA}/MRHeadResampled.nhdr ${TEST_DATA}/CTHeadAxial.nhdr |
ExtractSkeleton/ExtractSkeleton |
None (--help) |
|
FreesurferSurfaceSectionExtraction |
None (--help) |
|
GaussianBlurImageFilter |
Regression |
--sigma 2.5 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/GaussianBlurImageFilterTest.nhdr |
GradientAnisotropicDiffusion |
None (--help) |
|
GrayscaleFillHoleImageFilter |
None (--help) |
|
GrayscaleGrindPeakImageFilter |
None (--help) |
|
GrayscaleModelMaker |
None (--help) |
|
HistogramMatching |
None (--help) |
|
ImageLabelCombine |
None (--help) |
|
ImageReadDicomWrite |
None (Regression) |
--patientName Austrialian --patientID 8775070 --patientComments "A volunteer" --studyID 123456 --studyDate 20090102 --studyComments Resampled --studyDescription None --modality CT --manufacturer "GE Medical Systems" --model CT8800 --seriesNumber 123456 --seriesDescription None --dicomDirectory ${TEMP} --dicomPrefix CTHeadAxialDicom ${TEST_DATA}/CTHeadAxial.nhdr |
LabelMapSmoothing |
None (--help) |
|
LinearRegistration |
Smoke |
--resampledmovingfilename ${TEMP}/LinearRegistrationTest00.nhdr --outputtransform ${TEMP}/LinearRegistrationTest00Transform.txt --initialtransform ${TEST_DATA}/itkAffineTransform00.txt --spatialsamples 10000 --histogrambins 32 --iterations 50 --learningrate 0.01 --translationscale 100 ${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr |
MedianImageFilter |
Regression |
--neighborhood 1,2,3 ${TEST_DATA}/CTHeadAxial.nhdr ${TEMP}/MedianImageFilterTest.nhdr |
ModelMaker |
None (Smoke) |
--labels 2 --modelSceneFile ${TEMP}/ModelMakerTest.mrml\#vtkMRMLModelHierarchyNode1 ${MRML_TEST_DATA}/helix-roi-lable2.nrrd |
MultipleModelsExample |
None (--help) |
|
OrientImage |
Regression |
--orientation Axial ${MRML_DATA}/fixed.nrrd ${TEMP}/OrientImageTestAxial.nrrd --orientation Coronal ${MRML_DATA}/fixed.nrrd ${TEMP}/OrientImageTestCoronal.nrrd --orientation Sagittal ${MRML_DATA}/fixed.nrrd ${TEMP}/OrientImageTestSagittal.nrrd |
OtsuThresholdImageFilter |
None |
|
OtsuThresholdSegmentation |
None (Regression) |
--brightObjects --numberOfBins 128 --faceConnected --minimumObjectSize 1000 ${TEST_DATA}/CTHeadResampled.nhdr ${TEMP}/OtsuThresholdImageFilterTest.nhdr |
ProbeVolumeWithModel |
None (--help) |
|
Realign/Realign |
None (--help) |
|
ResampleVolume2 |
None (--help) |
|
ResampleVolume |
None (--help) |
|
RigidRegistration |
None (--help) |
|
Subtract |
None (--help) |
|
TestGridTransformRegistration |
Smoke |
--gridSize 5 --outputtransform ${TEMP}/TestGridTransformRegistrationTest00Transform.txt ${TEST_DATA}/CTHeadAxial.nhdr ${TEST_DATA}/CTHeadAxial.nhdr |
VotingBinaryHoleFillingImageFilter |
None (Regression) |
--radius 3,3,3 --majorityThreshold 1 --background 0 --foreground 1 ${TEST_DATA}/CTHeadResampledOtsuSegmented.nhdr ${TEMP}/VotingBinaryHoleFillingImageFilterTest.nhdr |
ZeroCrossingBasedEdgeDetectionImageFilter |
None (Regression) |
${TEST_DATA}/CTHeadResampled.nhdr ${TEMP}/ZeroCrossingBasedEdgeDetectionImageFilterTest.nhdr |