View Javadoc

1   /**
2    * <copyright>
3    *  Copyright 1997-2002 BBNT Solutions, LLC
4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5    *
6    *  This program is free software; you can redistribute it and/or modify
7    *  it under the terms of the Cougaar Open Source License as published by
8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
9    *
10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
19   * </copyright>
20   *
21   * Created on Aug 26, 2002
22   */
23  package net.sourceforge.pmd.stat;
24  
25  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.MINIMUM_DESCRIPTOR;
26  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.SIGMA_DESCRIPTOR;
27  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.TOP_SCORE_DESCRIPTOR;
28  import static org.junit.Assert.assertEquals;
29  import static org.junit.Assert.assertTrue;
30  
31  import java.util.ArrayList;
32  import java.util.Collections;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Random;
36  
37  import junit.framework.AssertionFailedError;
38  import net.sourceforge.pmd.Report;
39  import net.sourceforge.pmd.Rule;
40  import net.sourceforge.pmd.RuleContext;
41  import net.sourceforge.pmd.lang.Language;
42  import net.sourceforge.pmd.lang.java.ast.DummyJavaNode;
43  import net.sourceforge.pmd.lang.java.symboltable.SourceFileScope;
44  import net.sourceforge.pmd.lang.rule.stat.StatisticalRule;
45  import net.sourceforge.pmd.stat.DataPoint;
46  import net.sourceforge.pmd.stat.Metric;
47  
48  import org.junit.Before;
49  import org.junit.Ignore;
50  import org.junit.Test;
51  
52  /**
53   * This class tests the Statistical Rules in PMD.
54   * <p/>
55   * The idea is, that we fill up 999 datapoints into
56   * the Stat Rule, and then throw random parameters
57   * at it.
58   * <p/>
59   * The three parameters which are checked are:
60   * sigma - # Sigmas over the mean.
61   * topscore - Only the top 5 or so items.
62   * minimum - Only things of score 10 or better
63   * <p/>
64   * When more than one parameter is lumped together, then
65   * we expect the one which would return the fewest to
66   * determine what gets sent back.
67   * <p/>
68   * So, we throw each collection of parameters, where each
69   * one is a different order into the system.  We check the
70   * results off of what the smallest value should be.
71   * <p/>
72   * If you are going to work with StatisticalRule any, please
73   * bump the "NUM_TESTS" number up to something like 128.  That
74   * way you are more likely to identify problems.  It is set low
75   * now to make building and running tests easier (when we aren't
76   * touching the file.)
77   * <p/>
78   * Note also, that when verifying the Sigma, I wasn't quite able
79   * to determine how many results it would return (it would vary
80   * from -2 to 2 of what I expected.)  That is what the delta
81   * parameter on the verify method takes.  If you can figure it
82   * out exactly, (without stealing code from the StatRule) then
83   * feel free to change it and tighten the deltas.
84   */
85  public class StatisticalRuleTest  {
86  
87      private static final int POINTS = 100;
88  
89      private DataPoint[] points = new DataPoint[POINTS];
90      private MockStatisticalRule IUT = null;
91      private String testName = "";//FIXME - why/when was this added. It was never set.
92      private Random random = new Random();
93  
94      public static final double MAX_MINIMUM = POINTS;
95      public static final double NO_MINIMUM = -1.0;
96      public static final double MAX_SIGMA = 5.0;
97      public static final double NO_SIGMA = -1.0;
98      public static final int MIN_TOPSCORE = 0;
99      public static final int NO_TOPSCORE = -1;
100 
101 
102     public static final double MEAN = 49.5;
103     public static final double SIGMA = 29.0115;
104     public static final int NUM_TESTS = 1;
105 
106     public static final double DELTA = 0.005;
107 
108 
109     @Before
110     public void setUp() {
111         IUT = new MockStatisticalRule();
112         if (testName.endsWith("0")) {
113             for (int i = 0; i < POINTS; i++) {
114                 points[i] = new DataPoint();
115                 points[i].setScore(1.0 * i);
116                 DummyJavaNode s = new DummyJavaNode(1);
117                 s.setScope(new SourceFileScope("foo"));
118                 s.testingOnly__setBeginLine(i);
119                 s.testingOnly__setBeginColumn(1);
120                 points[i].setNode(s);
121                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
122 
123                 IUT.addDataPoint(points[i]);
124             }
125         } else if (testName.endsWith("1")) {
126             for (int i = POINTS - 1; i >= 0; i--) {
127                 points[i] = new DataPoint();
128                 points[i].setScore(1.0 * i);
129                 DummyJavaNode s = new DummyJavaNode(1);
130                 s.setScope(new SourceFileScope("foo"));
131                 s.testingOnly__setBeginLine(i);
132                 s.testingOnly__setBeginColumn(1);
133                 points[i].setNode(s);
134                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
135 
136                 IUT.addDataPoint(points[i]);
137             }
138         } else {
139             List<DataPoint> lPoints = new ArrayList<DataPoint>();
140             for (int i = 0; i < POINTS; i++) {
141                 points[i] = new DataPoint();
142                 points[i].setScore(1.0 * i);
143                 DummyJavaNode s = new DummyJavaNode(1);
144                 s.setScope(new SourceFileScope("foo"));
145                 s.testingOnly__setBeginLine(i);
146                 s.testingOnly__setBeginColumn(1);
147                 s.testingOnly__setBeginColumn(1);
148                 points[i].setNode(s);
149                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
150 
151                 lPoints.add(points[i]);
152             }
153 
154             Collections.shuffle(lPoints);
155             for (int i = 0; i < POINTS; i++) {
156                 IUT.addDataPoint(lPoints.get(i));
157             }
158         }
159 
160     }
161 
162     /**
163      * This test verifies that the Stat rule creates a Metric,
164      * with the proper values.
165      */
166     @Test
167     public void testMetrics() throws Throwable {
168         Report report = makeReport(IUT);
169         Iterator metrics = report.metrics();
170 
171         assertTrue(metrics.hasNext());
172         Object o = metrics.next();
173 
174         assertTrue(o instanceof Metric);
175         Metric m = (Metric) o;
176 
177         assertEquals("net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
178 
179         assertEquals(0.0, m.getLowValue(), 0.05);
180         assertEquals(POINTS - 1.0, m.getHighValue(), 0.05);
181         assertEquals(MEAN, m.getAverage(), 0.05);
182         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
183     }
184 
185     /**
186      * This returns a Random value for Sigma which will
187      * return some values.
188      */
189     public double randomSigma() {
190         return random.nextDouble() * 1.0;
191     }
192 
193     /**
194      * This returns a Random value for Sigma which value
195      * is greater than the parameter.
196      */
197     public double randomSigma(int minimum) {
198         double minSigma = ((POINTS - 1 - minimum) - MEAN) / SIGMA;
199 
200         if ((minSigma <= 0) || (minSigma > 2))
201             return randomSigma();
202 
203         return minSigma + (random.nextDouble() * (2 - minSigma));
204     }
205 
206     /**
207      * This returns the expected number of results when
208      * the Sigma rating is the smallest.
209      */
210     public int expectedSigma(double sigma) {
211         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
212 
213         if (((POINTS - 1) - expectedMin) < 0)
214             return 0;
215         return (POINTS - 1) - (int) expectedMin;
216     }
217 
218     /**
219      * This generates a random minimum value for testing.
220      */
221     public double randomMinimum() {
222         return random.nextDouble() * (POINTS - 1);
223     }
224 
225     /**
226      * This generates a random minimum value for which fewer
227      * results would be returned.
228      */
229     public double randomMinimum(int minimum) {
230         double diffTarget = 1.0 * (POINTS - 1 - minimum);
231         return (random.nextDouble() * minimum) + diffTarget;
232     }
233 
234     /**
235      * This returns the expected number of reports.
236      * <p/>
237      * If the Minimum comes in at 521.569 then we expect
238      * 522, 523, ... 999 will pass.
239      */
240     public int expectedMinimum(double minimum) {
241         Double d = Double.valueOf(minimum);
242         return POINTS - 1 - d.intValue();
243     }
244 
245     @Test
246     public void testExpectedMinimum() {
247         for (int i = 0; i < POINTS - 1; i++) {
248             assertEquals("Integer Min", POINTS - 1 - i, expectedMinimum(i * 1.0));
249             assertEquals("Double Min", POINTS - 1 - i, expectedMinimum((i * 1.0) + 0.5));
250         }
251     }
252 
253     /**
254      * This returns a random value for Top Score.
255      */
256     public int randomTopScore() {
257         return random.nextInt(POINTS - 1);
258     }
259 
260     /**
261      * This will return a random value for the Top Score
262      * which will return more than the minimum provided.
263      */
264     public int randomTopScore(double target) {
265         if (target < 0)
266             return 0;
267 
268         return random.nextInt(Double.valueOf(target).intValue());
269     }
270 
271     /**
272      * This will return the expected number of results
273      * with the given Top Score.
274      */
275     public int expectedTopScore(int target) {
276         return target;
277     }
278 
279     // Test Single Datapoint
280     @Test
281     public void testSingleDatapoint() {
282         StatisticalRule IUT = new MockStatisticalRule();
283 
284         DataPoint point = new DataPoint();
285         point.setScore(POINTS + 1.0);
286         DummyJavaNode s = new DummyJavaNode(1);
287         s.setScope(new SourceFileScope("foo"));
288         s.testingOnly__setBeginLine(POINTS + 1);
289         s.testingOnly__setBeginColumn(1);
290         point.setNode(s);
291         point.setMessage("SingleDataPoint");
292 
293         IUT.setProperty(MINIMUM_DESCRIPTOR, (double)POINTS);
294 
295         IUT.addDataPoint(point);
296 
297         Report report = makeReport(IUT);
298 
299         assertEquals("Expecting only one result", 1, report.size());
300     }
301 
302     // Okay, we have three properties we need to
303     // test in Combination:
304     //  S = Sigma
305     //  T = Top Score
306     //  M = Minimum
307     //
308     // They are listed in decreasing order of what
309     // to expect.
310     //
311     // Thus testSM() should have the Sigma less than
312     // the minimum, so we expect the Minimum # of results.
313     //
314 
315     @Test
316     public void testS() throws Throwable {
317         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
318 
319         for (int i = 0; i < NUM_TESTS; i++) {
320             double sigma = randomSigma();
321             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
322         }
323     }
324 
325     @Test
326     public void testS1() throws Throwable {
327         testS();
328     }
329 
330     @Test
331     public void testS2() throws Throwable {
332         testS();
333     }
334 
335     @Test
336     public void testS3() throws Throwable {
337         testS();
338     }
339 
340     @Test
341     public void testS4() throws Throwable {
342         testS();
343     }
344 
345     @Test
346     public void testS5() throws Throwable {
347         testS();
348     }
349 
350 
351     @Test
352     public void testT() throws Throwable {
353         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
354 
355         for (int i = 0; i < NUM_TESTS; i++) {
356             int topScore = randomTopScore();
357             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
358         }
359     }
360 
361     @Test
362     public void testT1() throws Throwable {
363         testT();
364     }
365 
366     @Test
367     public void testT2() throws Throwable {
368         testT();
369     }
370 
371     @Test
372     public void testT3() throws Throwable {
373         testT();
374     }
375 
376     @Test
377     public void testT4() throws Throwable {
378         testT();
379     }
380 
381     @Test
382     public void testT5() throws Throwable {
383         testT();
384     }
385 
386     @Test
387     public void testM() throws Throwable {
388         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
389 
390         for (int i = 0; i < NUM_TESTS; i++) {
391             double minimum = randomMinimum();
392             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
393         }
394     }
395 
396     @Test
397     public void testM1() throws Throwable {
398         testM();
399     }
400 
401     @Test
402     public void testM2() throws Throwable {
403         testM();
404     }
405 
406     @Test
407     public void testM3() throws Throwable {
408         testM();
409     }
410 
411     @Test
412     public void testM4() throws Throwable {
413         testM();
414     }
415 
416     @Test
417     public void testM5() throws Throwable {
418         testM();
419     }
420 
421     @Test
422     public void testST() throws Throwable {
423         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
424 
425         for (int i = 0; i < NUM_TESTS; i++) {
426             double sigma = randomSigma();
427             int topScore = randomTopScore(expectedSigma(sigma));
428 
429             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
430         }
431     }
432 
433     @Test
434     public void testST1() throws Throwable {
435         testST();
436     }
437 
438     @Test
439     public void testST2() throws Throwable {
440         testST();
441     }
442 
443     @Test
444     public void testST3() throws Throwable {
445         testST();
446     }
447 
448     @Test
449     public void testST4() throws Throwable {
450         testST();
451     }
452 
453     @Test
454     public void testST5() throws Throwable {
455         testST();
456     }
457 
458     @Test
459     public void testTS() throws Throwable {
460         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
461 
462         for (int i = 0; i < NUM_TESTS; i++) {
463             int topScore = randomTopScore();
464             double sigma = randomSigma(expectedTopScore(topScore));
465 
466             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
467         }
468     }
469 
470     @Test
471     public void testTS1() throws Throwable {
472         testTS();
473     }
474 
475     @Test
476     public void testTS2() throws Throwable {
477         testTS();
478     }
479 
480     @Test
481     public void testTS3() throws Throwable {
482         testTS();
483     }
484 
485     @Test
486     public void testTS4() throws Throwable {
487         testTS();
488     }
489 
490     @Test
491     public void testTS5() throws Throwable {
492         testTS();
493     }
494 
495     @Test
496     public void testSM() throws Throwable {
497         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
498         for (int i = 0; i < NUM_TESTS; i++) {
499             double sigma = randomSigma();
500             double minimum = randomMinimum(expectedSigma(sigma));
501 
502             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
503         }
504 
505     }
506 
507     @Test
508     public void testSM1() throws Throwable {
509         testSM();
510     }
511 
512     @Test
513     public void testSM2() throws Throwable {
514         testSM();
515     }
516 
517     @Test
518     public void testSM3() throws Throwable {
519         testSM();
520     }
521 
522     @Test
523     public void testSM4() throws Throwable {
524         testSM();
525     }
526 
527     @Test
528     public void testSM5() throws Throwable {
529         testSM();
530     }
531 
532 
533     @Test
534     public void testMS() throws Throwable {
535         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
536         for (int i = 0; i < NUM_TESTS; i++) {
537             double minimum = randomMinimum();
538             double sigma = randomSigma(expectedMinimum(minimum));
539 
540             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
541         }
542     }
543 
544     @Test
545     public void testMS1() throws Throwable {
546         testMS();
547     }
548 
549     @Test
550     public void testMS2() throws Throwable {
551         testMS();
552     }
553 
554     @Test
555     public void testMS3() throws Throwable {
556         testMS();
557     }
558 
559     @Test
560     public void testMS4() throws Throwable {
561         testMS();
562     }
563 
564     @Test
565     public void testMS5() throws Throwable {
566         testMS();
567     }
568 
569 
570     @Test
571     public void testTM() throws Throwable {
572         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
573         for (int i = 0; i < NUM_TESTS; i++) {
574             int topScore = randomTopScore();
575             double minimum = randomMinimum(expectedTopScore(topScore));
576 
577             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
578         }
579     }
580 
581     @Test
582     public void testTM1() throws Throwable {
583         testTM();
584     }
585 
586     @Test
587     public void testTM2() throws Throwable {
588         testTM();
589     }
590 
591     @Test
592     public void testTM3() throws Throwable {
593         testTM();
594     }
595 
596     @Test
597     public void testTM4() throws Throwable {
598         testTM();
599     }
600 
601     @Test
602     public void testTM5() throws Throwable {
603         testTM();
604     }
605 
606 
607     @Test
608     public void testMT() throws Throwable {
609         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
610         for (int i = 0; i < NUM_TESTS; i++) {
611             double minimum = randomMinimum();
612             int topScore = randomTopScore(expectedMinimum(minimum));
613 
614             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
615         }
616     }
617 
618     @Test
619     public void testMT1() throws Throwable {
620         testMT();
621     }
622 
623     @Test
624     public void testMT2() throws Throwable {
625         testMT();
626     }
627 
628     @Test
629     public void testMT3() throws Throwable {
630         testMT();
631     }
632 
633     @Test
634     public void testMT4() throws Throwable {
635         testMT();
636     }
637 
638     @Test
639     public void testMT5() throws Throwable {
640         testMT();
641     }
642 
643 
644     @Test
645     public void testSTM() throws Throwable {
646         double sigma = randomSigma();
647         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
648 
649         for (int i = 0; i < NUM_TESTS; i++) {
650             sigma = randomSigma();
651             int topScore = randomTopScore(expectedSigma(sigma));
652             double minimum = randomMinimum(expectedTopScore(topScore));
653 
654             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
655         }
656     }
657 
658     @Test
659     public void testSTM1() throws Throwable {
660         testSTM();
661     }
662 
663     @Test
664     public void testSTM2() throws Throwable {
665         testSTM();
666     }
667 
668     @Test
669     public void testSTM3() throws Throwable {
670         testSTM();
671     }
672 
673     @Test
674     public void testSTM4() throws Throwable {
675         testSTM();
676     }
677 
678     @Test
679     public void testSTM5() throws Throwable {
680         testSTM();
681     }
682 
683     @Test
684     public void testSMT() throws Throwable {
685         double sigma = randomSigma();
686         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
687 
688         for (int i = 0; i < NUM_TESTS; i++) {
689             sigma = randomSigma();
690             double minimum = randomMinimum(expectedSigma(sigma));
691             int topScore = randomTopScore(expectedMinimum(minimum));
692 
693             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
694         }
695     }
696 
697     @Test
698     public void testSMT1() throws Throwable {
699         testSMT();
700     }
701 
702     @Test
703     public void testSMT2() throws Throwable {
704         testSMT();
705     }
706 
707     @Test
708     public void testSMT3() throws Throwable {
709         testSMT();
710     }
711 
712     @Test
713     public void testSMT4() throws Throwable {
714         testSMT();
715     }
716 
717     @Test
718     public void testSMT5() throws Throwable {
719         testSMT();
720     }
721 
722     @Test
723     // because of random failures during continuous integration,
724     // tests are disabled in regress mode until somebody figures out
725     // what the tests are supposed to measure and why they sometime fail
726     @Ignore("random failures during continuous integration")
727     public void testTSM() throws Throwable {
728         int topScore = randomTopScore();
729         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
730 
731         for (int i = 0; i < NUM_TESTS; i++) {
732             topScore = randomTopScore();
733             double sigma = randomSigma(expectedTopScore(topScore));
734             double minimum = randomMinimum(expectedSigma(sigma));
735 
736             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
737         }
738     }
739 
740     @Test
741     @Ignore("random failures during continuous integration")
742     public void testTSM1() throws Throwable {
743         testTSM();
744     }
745 
746     @Test
747     @Ignore("random failures during continuous integration")
748     public void testTSM2() throws Throwable {
749         testTSM();
750     }
751 
752     @Test
753     @Ignore("random failures during continuous integration")
754     public void testTSM3() throws Throwable {
755         testTSM();
756     }
757 
758     @Test
759     @Ignore("random failures during continuous integration")
760     public void testTSM4() throws Throwable {
761         testTSM();
762     }
763 
764     @Test
765     @Ignore("random failures during continuous integration")
766     public void testTSM5() throws Throwable {
767         testTSM();
768     }
769 
770     @Test
771     public void testTMS() throws Throwable {
772         int topScore = randomTopScore();
773         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
774 
775         for (int i = 0; i < NUM_TESTS; i++) {
776             topScore = randomTopScore();
777             double minimum = randomMinimum(expectedTopScore(topScore));
778             double sigma = randomSigma(expectedMinimum(minimum));
779 
780             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
781         }
782     }
783 
784     @Test
785     public void testTMS1() throws Throwable {
786         testTMS();
787     }
788 
789     @Test
790     public void testTMS2() throws Throwable {
791         testTMS();
792     }
793 
794     @Test
795     public void testTMS3() throws Throwable {
796         testTMS();
797     }
798 
799     @Test
800     public void testTMS4() throws Throwable {
801         testTMS();
802     }
803 
804     @Test
805     public void testTMS5() throws Throwable {
806         testTMS();
807     }
808 
809     /**
810      * Verifies what happens when you pass these parameters
811      * into the thing.  DELTA is the amount of error allowed.
812      * Usually DELTA is only used for Sigma, as we really can't
813      * calculate it exactly.
814      */
815 
816     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
817         try {
818             setUp();
819             if (sigma >= 0) {
820             	IUT.setProperty(SIGMA_DESCRIPTOR, sigma);
821             }
822 
823             if (minimum >= 0) {
824             	IUT.setProperty(MINIMUM_DESCRIPTOR, minimum);
825             }
826 
827             if (topScore >= 0) {
828                 IUT.setProperty(TOP_SCORE_DESCRIPTOR, topScore);
829             }
830 
831             Report report = makeReport(IUT);
832             if (delta == 0) {
833                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
834             } else {
835                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
836 
837                 assertTrue(assertStr, report.size() >= (expected - delta));
838                 assertTrue(assertStr, report.size() <= (expected + delta));
839             }
840         } catch (AssertionFailedError afe) {
841             System.err.println("******** " + testName + " ***********");
842             if (sigma != NO_SIGMA) {
843                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
844             }
845 
846             if (minimum != NO_MINIMUM) {
847                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
848             }
849 
850             if (topScore != NO_TOPSCORE) {
851                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
852             }
853 
854             throw afe;
855 
856         }
857     }
858 
859     public Report makeReport(Rule IUT) {
860         List list = new ArrayList();
861         Report report = new Report();
862 
863         RuleContext ctx = new RuleContext();
864         ctx.setReport(report);
865         ctx.setSourceCodeFilename(testName);
866         ctx.setLanguageVersion(Language.JAVA.getDefaultVersion());
867 
868         IUT.apply(list, ctx);
869 
870         return report;
871     }
872 
873     public static junit.framework.Test suite() {
874         return new junit.framework.JUnit4TestAdapter(StatisticalRuleTest.class);
875     }
876 }