-
Notifications
You must be signed in to change notification settings - Fork 9
/
testing.bib
5700 lines (4749 loc) · 237 KB
/
testing.bib
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
%%% testing.bib -- Testing bibliography (selective)
@unpublished{EWD:EWD303,
author = "Edsger W. Dijkstra",
title = "EWD303: On the reliability of programs",
year = "n.d.",
url = "http://www.cs.utexas.edu/users/EWD/ewd03xx/EWD303.PDF",
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Impact and cost of poor quality software and testing
%%%
@TechReport{NIST02-3,
author = "{Research Triangle Institute}",
title = "The Economic Impacts of Inadequate Infrastructure for Software Testing",
institution = "National Institute of Standards and Technology",
year = "2002",
OPTkey = "",
type = "NIST Planning Report",
number = "02-3",
OPTaddress = "",
month = may,
OPTnote = "",
OPTannote = ""
}
@InProceedings{Hartman2002,
author = "A. Hartman",
title = "Is {ISSTA} research relevant to industry?",
crossref = "ISSTA2002",
pages = "205--206",
}
@InProceedings{HooimeijerW2007,
author = "Hooimeijer, Pieter and Weimer, Westley",
title = "Modeling bug report quality",
crossref = "ASE2007",
pages = "34-43",
}
@Article{AlyA1988,
author = "Nael A. E. Aly and Adel A. Aly",
title = "Measures of testability for automatic diagnostic systems",
journal = IEEETR,
year = 1988,
volume = 37,
number = 5,
pages = "531-538",
month = dec,
}
@Book{Beizer90,
Author = {Beizer, B},
title = {Software Testing Techniques},
publisher = {Boston: International Thomson Computer Press},
year = {1990},
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Empirical evaluation of unit testing & TDD (test-driven development)
%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Software reliability and fault injection models
%%%
@Article{goel85a,
key = "goel85a",
author = "Amrit L. Goel",
title = "Software Reliability Models: Assumptions, Limitations, and
Applicability",
journal = TSE,
month = dec,
year = "1985",
volume = "SE-11",
number = "12",
pages = "1411--1423"
}
@InProceedings{1989:icse:ohba,
author = "Mitsuru Ohba and Xiao-Mei Chou",
title = "Does Imperfect Debugging Affect Software Reliability
Growth?",
crossref = "ICSE89",
pages = "237--244",
ISSN = "0270-5257",
ISBN = "0-89791-258-6 (IEEE), 0-8186-1941-4 (ACM)",
genterms = "DESIGN, LANGUAGES, MEASUREMENT, RELIABILITY",
categories = "D.2.5 Software, SOFTWARE ENGINEERING, Testing and
Debugging. D.2.8 Software, SOFTWARE ENGINEERING,
Metrics, Performance measures.",
abstract = "This paper discusses the improvement of conventional
software reliability growth models by elimination of
the unreasonable assumption that errors or faults in a
program can be perfectly removed when they are
detected. The results show that exponential-type
software reliability growth models that deal with
error-counting data could be used even if the perfect
debugging assumption were not held, in which case the
interpretation of the model parameters should be
changed. An analysis of real project data is
presented.",
annote = "incomplete",
}
@Article{EickGKMM2001,
author = "Stephen G. Eick and Todd L. Graves and Alan F. Karr and J. S. Marron and Audris Mockus",
title = "Does code decay? Assessing the evidence from change
management data",
journal = TSE,
year = 2001,
volume = 27,
number = 1,
pages = "1--12",
month = jan,
abstract =
"A central feature of the evolution of large software systems is that change
--- which is necessary to add new functionality, accommodate new hardware
and repair faults --- becomes increasingly difficult over time. In this
paper we approach this phenomenon, which we term code decay, scientifically
and statistically. We define code decay, and propose a number of
measurements (code decay indices) on software, and on the organizations
that produce it, that serve as symptoms, risk factors and predictors of
decay. Using an unusually rich data set (the fifteen-plus year change
history of the millions of lines of software for a telephone switching
system), we find mixed but on the whole persuasive statistical evidence of
code decay, which is corroborated by developers of the code. Suggestive,
but not yet fully assimilated, indications that perfective maintenance can
retard code decay are also discussed.",
}
@Article{GravesKMS2000,
author = "Todd L. Graves and Alan F. Karr and J. S. Marron and Harvey Siy",
title = "Predicting fault incidence using software change history",
journal = TSE,
year = 2000,
volume = 26,
number = 7,
pages = "653--661",
month = jul,
abstract =
"This paper is an attempt to understand the processes by which software
ages. We define code to be aged or decayed if its structure makes it too
difficult to understand or change, and we measure the extent of decay by
counting the number of faults in code in a period of time. Using change
management data from a very large, long-lived software system, we explore
the extent to which measurements from the change history are successful in
predicting the distribution over modules of these incidences of faults. In
general, process measures based on the change history are more useful in
predicting fault rates than product metrics of the code: for instance, the
number of times code has been changed is a better indication of how many
faults it will contain than is its length. We also compare the fault rates
of code of various ages, finding that if a module is on the average a year
older than an otherwise similar module, the older module will have roughly
a third fewer faults. Our most successful model measures the fault
potential of a module as a sum of contributions from all of the times the
module has been changed, with large, recent changes receiving the most
weight."
}
@Article{ChristensonH96,
author = "Dennis A. Christenson and Steel T. Huang",
title = "Estimating the fault content of software using the fix-on-fix model",
journal = "Bell Labs Technical Journal",
year = 1996,
volume = 1,
number = 1,
pages = "130--137",
month = "Summer",
abstract =
"In statistical theory, the percentage of defects in a randomly
drawn sample is an estimate of the percentage of defects in the
entire population. When this concept is applied to the process of
fixing faults during software development, a new fix-on-fix model
results. Such a model can predict the number of software faults,
thus providing a useful quality assessment. The model discussed in
this paper implements the concepts of BF and FOF, which have been
used in the 5ESS\textregistered-2000 switch project for several years. The FOF
model is similar to error seeding models in which predetermined
errors are planted in the code. The number of remaining errors can
be predicted based on the number of original errors seeded and the
number of both seeded and nonseeded errors found during
testing. The model may initiate a new approach to software quality
prediction, and it has the advantage of being independent of
testing intensity, methodology, and environment. The FOF model is
applicable to any software product in which BF and FOF rates can be
measured from source-code management systems.",
}
@Article{YuSD88,
author = "T.-J. Yu and V. Y. Shen and H. E. Dunsmore",
title = "An analysis of several software defect models",
journal = TSE,
year = 1988,
volume = 14,
number = 9,
pages = "1261--1270",
month = sep,
abstract =
"Results are presented of an analysis of several defect models using data
collected from two large commercial projects. Traditional models typically
use either program matrices (i.e. measurements from software products) or
testing time or combinations of these as independent variables. The
limitations of such models have been well-documented. The models considered
use the number of defects detected in the earlier phases of the development
process as the independent variable. This number can be used to predict the
number of defects to be detected later, even in modified software
products. A strong correlation between the number of earlier defects and
that of later ones was found. Using this relationship, a mathematical model
was derived which may be used to estimate the number of defects remaining
in software. This defect model may also be used to guide software
developers in evaluating the effectiveness of the software development and
testing processes.",
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Effectiveness of various testing strategies
%%%
@Article{1998:tse:frankl,
title = "Evaluating Testing Methods by Delivered Reliability",
author = "Phyllis G. Frankl and Richard G. Hamlet and Bev
Littlewood and Lorenzo Strigini",
pages = "586--601",
journal = TSE,
ISSN = "0098-5589",
year = "1998",
volume = "24",
month = aug,
number = "8",
abstract = "There are two main goals in testing software: 1) to
achieve adequate quality (\emph{debug testing}); the
objective is to probe the software for defects so that
these can be removed and 2) to assess existing quality
(\emph{operational testing}); the objective is to gain
confidence that the software is reliable. The names are
arbitrary, and most testing techniques address both
goals to some degree. However, debug methods tend to
ignore random selection of test data from an
operational profile, while for operational methods this
selection is all-important. Debug methods are thought,
without any real proof, to be good at uncovering
defects so that these can be repaired, but having done
so they do not provide a technically defensible
assessment of the reliability that results. On the
other hand, operational methods provide accurate
assessment, but may not be as useful for achieving
reliability. This paper examines the relationship
between the two testing goals, using a probabilistic
analysis. We define simple models of programs and their
testing, and try to answer theoretically the question
of how to attain program reliability: Is it better to
test by probing for defects as in debug testing, or to
assess reliability directly as in operational testing,
uncovering defects by accident, so to speak? There is
no simple answer, of course. Testing methods are
compared in a model where program failures are detected
and the software changed to eliminate them. The
``better'' method delivers higher reliability after all
test failures have been eliminated. This comparison
extends previous work, where the measure was the
probability of detecting a failure. Revealing special
cases are exhibited in which each kind of testing is
superior. Preliminary analysis of the distribution of
the delivered reliability indicates that even simple
models have unusual statistical properties, suggesting
caution in interpreting theoretical comparisons.",
keywords = "Reliability, debugging, software testing, statistical
testing theory",
correctedby = "\cite{1999:tse:frankl}",
note = "Special Section: International Conference on Software
Engineering (ICSE~'97)",
annote = "incomplete",
}
@Article{1999:tse:frankl,
title = "Correction to: Evaluating Testing Methods by Delivered
Reliability",
author = "Phyllis Frankl and Dick Hamlet and Bev Littlewood and
Lorenzo Strigini",
pages = "286",
journal = TSE,
ISSN = "0098-5589",
year = "1999",
volume = "25",
month = mar # "/" # apr,
number = "2",
corrects = "\cite{1998:tse:frankl}",
references = "\cite{1998:tse:frankl}",
annote = "checked",
}
@misc{ irvine-effectiveness,
author = "A. Irvine and A. Offutt",
title = "The Effectiveness of Category-Partition Testing of Object-Oriented Software",
text = "Alisa Irvine and A. Jefferson Offutt, The Effectiveness of Category-Partition
Testing of Object-Oriented Software, ISSE Department George Mason University,
Fairfax, VA 22030.",
url = "citeseer.nj.nec.com/irvine95effectiveness.html"
}
@misc{ tewary-empirical,
author = "A. Jefferson Offutt, Kanupriya Tewary",
title = "Empirical Comparisons of Data Flow and Mutation Testing",
url = "citeseer.nj.nec.com/offutt92empirical.html"
}
@misc{ offutt94experiments,
author = "A. Offutt and J. Pan and K. Tewary and T. Zhang",
title = "Experiments with data flow and mutation testing",
text = "A. J. Offutt, J. Pan, K. Tewary, and T. Zhang. Experiments with
data flow and mutation testing. Technical Report
ISSE-TR-94-105, Department of Information and Software Systems
Engineering, George Mason University, Fairfax, Virginia, 1994.",
year = "1994",
url = "citeseer.nj.nec.com/offutt94experiments.html"
}
@Article{HowdenH95,
author = "W. E. Howden and Yudong Huang",
title = "Software trustability analysis",
journal = TOSEM,
year = 1995,
volume = 4,
number = 1,
pages = "36--64",
month = jan
}
@InProceedings{BohmeP2014,
author = "B{\"o}hme, Marcel and Paul, Soumya",
authorASCII = "Bohme, Marcel and Paul, Soumya",
authorASCII = "Boehme, Marcel and Paul, Soumya",
title = "On the efficiency of automated testing",
crossref = "FSE2014",
pages = "632--642",
}
%%%
%%% Random testing
%%%
@InCollection{Hamlet94,
author = "Dick Hamlet",
title = "Random Testing",
booktitle = "Encyclopedia of Software Engineering",
publisher = "John Wiley and Sons",
year = 1994
}
@Article{1990:tse:hamlet,
title = "Partition Testing Does Not Inspire Confidence",
author = "Dick Hamlet and Ross Taylor",
pages = "1402--1411",
journal = TSE,
ISSN = "0098-5589",
year = "1990",
volume = "16",
month = dec,
number = "12",
referencedby = "\cite{1997:icse:bernot}, \cite{1997:icse:frankl},
\cite{1998:tse:frankl}, \cite{1999:tosem:podgurski}",
annote = "incomplete",
abstract = "Partition testing, in which a program's input domain is
divided according to some rule and tests conducted within
the subdomains, enjoys a good reputation. However,
comparison between testing that observes subdomain
boundaries and random sampling that ignores the partition
gives the counterintuitive result that partitioning is of
little value. In this paper we improve the negative
results published about partition testing, and try to
reconcile them with its intuitive value.
Theoretical models allow us to study partition testing in
the abstract, and to describe the circumstances under
which it should perform well at failure detection.
Partition testing is shown to be more valuable when the
partitions are narrowly based on expected failures and
there is a good chance that failures occur. For gaining
confidence from successful tests, partition testing as
usually practiced has little value."
}
@Article{1984:tse:duran,
title = "An Evaluation of Random Testing",
author = "Joe W. Duran and Simeon C. Ntafos",
pages = "438--444",
journal = TSE,
ISSN = "0098-5589",
year = "1984",
volume = "10",
month = jul,
number = "4",
referencedby = "\cite{1997:icse:bernot}, \cite{1997:icse:frankl},
\cite{1998:tse:frankl}, \cite{1999:tosem:podgurski}",
annote = "incomplete",
}
@Article{MillerMNPNMV92,
author = "Keith W. Miller and Larry J. Morell and Robert E. Noonan
and Stephen K. Park and David M. Nichol and Branson
W. Murrill and Jeffrey M. Voas",
title = "Estimating the probability of failure when testing reveals
no failures",
journal = TSE,
year = 1992,
volume = 18,
number = 1,
pages = "33--43",
month = jan
}
@Article{MillerFS90,
author = "Barton P. Miller and Louis Fredriksen and Bryan So",
title = "An empirical study of the reliability of {UNIX} utilities",
journal = CACM,
year = 1990,
volume = 33,
number = 12,
pages = "32--44",
month = dec,
doi = "https://doi.acm.org/10.1145/96267.96279",
}
@InProceedings{Xie2006,
author = "Tao Xie",
title = "Augmenting automatically generated unit-test suites with
regression oracle checking",
crossref = "ECOOP2006",
pages = "380--403",
}
@InProceedings{JiangZCT2009,
author = "Jiang, Bo and Zhang, Zhenyu and Chan, W. K. and Tse, T. H.",
title = "Adaptive Random Test Case Prioritization",
crossref = "ASE2009",
pages = "233--244",
}
@Article{SoaresGSM2010,
author = "Soares, Gustavo and Gheyi, Rohit and Serey, Dalton and Massoni, Tiago",
title = "Making program refactoring safer",
journal = IEEESoftware,
year = 2010,
volume = 27,
number = 4,
pages = "52--57",
month = jul # "/" # aug,
}
@InProceedings{KleinFF2010,
author = "Klein, Casey and Flatt, Matthew and Findler, Robert Bruce",
title = "Random testing for higher-order, stateful programs",
crossref = "OOPSLA2010",
pages = "555--566",
}
@InProceedings{PradelG2012,
author = "Pradel, Michael and Gross, Thomas R.",
title = "Leveraging test generation and specification mining for automated bug detection without false positives",
crossref = "ICSE2012",
pages = "288--298",
}
@InProceedings{GaneshLR2009,
author = "Ganesh, Vijay and Leek, Tim and Rinard, Martin",
title = "Taint-based directed whitebox fuzzing",
crossref = "ICSE2009",
pages = "474--484",
}
@InProceedings{TanMTL2012,
author = "Shin Hwei Tan and Darko Marinov and Lin Tan and Gary T. Leavens",
title = "{@tComment}: Testing {Javadoc} Comments to Detect Comment-Code Inconsistencies",
crossref = "ICST2012",
pages = "260--269",
}
@inproceedings{Garg2013,
abstract = {In industry, software testing and coverage-based metrics are the
predominant techniques to check correctness of software. This paper addresses
automatic unit test generation for programs written in C/C++. The main idea is to
improve the coverage obtained by feedback-directed random test generation
methods, by utilizing concolic execution on the generated test drivers.
Furthermore, for programs with numeric computations, we employ non-linear solvers
in a lazy manner to generate new test inputs. These techniques significantly
improve the coverage provided by a feedback-directed random unit testing
framework, while retaining the benefits of full automation. We have implemented
these techniques in a prototype platform, and describe promising experimental
results on a number of C/C++ open source benchmarks. © 2013 IEEE.},
author = {Garg, Pranav and Ivancic, Franjo and Balakrishnan, Gogul and Maeda,
Naoto and Gupta, Aarti},
booktitle = {2013 35th International Conference on Software Engineering (ICSE)},
doi = {10.1109/ICSE.2013.6606559},
isbn = {978-1-4673-3076-3},
issn = {02705257},
mendeley-groups = {randoop-reimplementation},
month = {may},
pages = {132--141},
publisher = {IEEE},
title = {Feedback-directed unit test generation for {C/C++} using concolic
execution},
url = {http://www.scopus.com/inward/record.url?eid=2-s2.0-84886426731
{\&}partnerID=tZOtx3y1},
year = {2013}
}
@Article{MajumdarN2018,
author = "Rupak Majumdar and Filip Niksic",
title = "Why is random testing effective for partition tolerance bugs?",
journal = PACMPL,
year = 2017,
volume = 2,
number = "POPL",
pages = "46:1--46:24",
month = dec,
articleno = 46,
numpages = 24,
}
@InProceedings{CumminsPML2018,
author = "Cummins, Chris and Petoumenos, Pavlos and Murray, Alastair and Leather, Hugh",
title = "Compiler fuzzing through deep learning",
crossref = "ISSTA2018",
pages = "95--105",
}
@inproceedings{Lemieux:2018:PAG:3213846.3213874,
author = {Lemieux, Caroline and Padhye, Rohan and Sen, Koushik and Song, Dawn},
title = {PerfFuzz: Automatically Generating Pathological Inputs},
booktitle = {Proceedings of the 27th ACM SIGSOFT International Symposium on Software Testing and Analysis},
series = {ISSTA 2018},
year = {2018},
isbn = {978-1-4503-5699-2},
location = {Amsterdam, Netherlands},
pages = {254--265},
numpages = {12},
url = {https://doi.acm.org/10.1145/3213846.3213874},
doi = {10.1145/3213846.3213874},
acmid = {3213874},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {algorithmic complexity, fuzz testing, performance, worst-case},
}
@misc{randoop-tool-2020,
key = {Randoop},
year = {2020},
howpublished = "\url{https://github.com/randoop/randoop}"
}
% Use: \citepalias{randoop-tool}
% then cite with: \citepalias{randoop-tool}
@misc{randoop-issue-tracker-2020,
key = {Randoop issue tracker},
year = {2010--2020},
howpublished = "\url{https://github.com/randoop/randoop/issues}"
}
@misc{randoop-mailing-lists-2020,
key = {Randoop mailing lists},
year = {2010--2020},
howpublished =
"\url{https://groups.google.com/forum/\#!forum/randoop-developers} and \url{https://groups.google.com/forum/\#!forum/randoop-discuss}"
}
@misc{randoop-manual-4.2.3,
key = {Randoop Manual},
year = {2020},
month = mar,
note = "version 4.2.3",
howpublished =
"\url{https://randoop.github.io/randoop/manual/index.html}"
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Specification-based test suite generation
%%%
@PhdThesis{Meudec98,
author = "Christophe Meudec",
title = "Automatic Generation of Software Test Cases From Formal
Specifications",
school = "Queen's University of Belfast",
year = 1998,
OPTkey = "",
OPTtype = "",
OPTaddress = "",
OPTmonth = "",
OPTnote = "",
OPTannote = "",
abstract = "Software testing consumes a large percentage of total
software development costs. Yet, it is still usually
performed manually in a non rigorous fashion. While
techniques, and limited automatic support, for the
generation of test data from the actual code of the
system under test have been well researched, test cases
generation from a high level specification of the
intended behaviour of the system being developed has
hardly been addressed. In this thesis we present a
rationale for using tests derived from high level formal
specifications and then set to find an efficient
technique for the generation of adequate test sets from
specifications written in our study language, VDM-SL. In
this..."
}
@InProceedings{ChangR99,
author = "Juei Chang and Debra J. Richardson",
title = "Structural Specification-Based Testing: Automated
Support and Experimental Evaluation",
crossref = "FSE99",
pages = "285--302",
abstract = "In this paper, we describe a testing technique, called
structural specification-based testing (SST), which
utilizes the formal specification of a program unit as
the basis for test selection and test coverage
measurement. We also describe an automated testing tool,
called ADLscope, which supports SST for program units
specified in Sun Microsystems' Assertion Definition
Language (ADL). ADLscope automatically generates coverage
conditions from a program's ADL specification. While the
program is tested, ADLscope determines which of these
conditions are covered by the tests. An uncovered
condition exhibits aspects of the specification
inadequately exercised during testing. The tester uses
this information to develop new test data to exercise the
uncovered conditions.
\par
We provide an overview of SST's specification-based test
criteria and describe the design and implementation of
ADLscope. Specification-based testing is guided by a
specification, whereby the testing activity is directly
related to what a component under test is supposed to do,
rather than what it actually does. Specification-based
testing is a significant advance in testing, because it
is often more straightforward to accomplish and it can
reveal failures that are often missed by traditional
code-based testing techniques. As an initial evaluation
of the capabilities of specification-based testing, we
conducted an experiment to measure defect detection
capabilities, code coverage and usability of
SST/ADLscope; we report here on the results."
}
@InProceedings{ChangRS96:ISSTA,
author = "Juei Chang and Debra J. Richardson and Sriram Sankar",
title = "Structural Specification-based Testing with {ADL}",
OPTeditor = "Steven J. Ziel",
crossref = "ISSTA96",
pages = "62--70",
genterms = "DESIGN, VERIFICATION",
categories = "D.2.1 Software, SOFTWARE ENGINEERING,
Requirements/Specifications. D.2.5 Software, SOFTWARE
ENGINEERING, Testing and Debugging. F.3.1 Theory of
Computation, LOGICS AND MEANINGS OF PROGRAMS,
Specifying and Verifying and Reasoning about Programs,
Specification techniques.",
annote = "incomplete",
abstract = "This paper describes a specification-based black-box
technique for testing program units. The main
contribution is the method that we have developed to
derive test conditions, which are descriptions of test
cases, from the formal specification of each program
unit. The derived test conditions are used to guide test
selection and to measure comprehensiveness of existing
test suites. Our technique complements traditional
code-based techniques such as statement coverage and
branch coverage. It allows the tester to quickly develop
a black-box test suite.
\par
In particular, this paper presents techniques for
deriving test conditions from specifications written in
the Assertion Definition Language (ADL) [SH94], a
predicate logic-based language that is used to describe
the relationships between inputs and outputs of a program
unit. Our technique is fully automatable, and we are
currently implementing a tool based on the techniques
presented in this paper."
}
@TechReport{HayesS94,
author = "Roger Hayes and Sriram Sankar",
title = "Specifying and Testing Software Components using {ADL}",
institution = "Sun Microsystems Research",
year = 1994,
number = "TR-94-23",
address = "Palo Alto, CA, USA",
month = apr,
abstract = "This paper presents a novel approach to unit testing of
software components. This approach uses the specification
language ADL, that is particularly well-suited for
testing, to formally document the intended behavior of
software components. Another related language, TDD, is
used to systematically describe the test-data on which
the software components will be tested.
\par
This paper gives a detailed overview of the ADL language,
and a brief presentation of the TDD language. Some
details of the actual test system are also presented,
along with some significant results.",
URL = "http://www.sun.com/research/techrep/1994/smli_tr-94-23.ps"
}
@InProceedings{1989:tav:balcer,
author = "Marc J. Balcer and William M. Hasling and Thomas J.
Ostrand",
title = "Automatic Generation of Test Scripts from Formal Test
Specifications",
OMITeditor = "Richard A. Kemmerer",
crossref = "TAV89",
pages = "210--218",
genterms = "LANGUAGES, RELIABILITY, VERIFICATION",
categories = "I.2.2 Computing Methodologies, ARTIFICIAL
INTELLIGENCE, Automatic Programming, Program
transformation. D.2.5 Software, SOFTWARE ENGINEERING,
Testing and Debugging, Testing tools (e.g., data
generators, coverage testing).",
annote = "incomplete",
}
@InProceedings{RichardsonOT89:TAV,
author = "Debra J. Richardson and Owen O'Malley and Cindy
Tittle",
title = "Approaches to Specification-Based Testing",
IGNOREeditor = "Richard A. Kemmerer",
crossref = "TAV89",
pages = "86--96",
genterms = "LANGUAGES, RELIABILITY, VERIFICATION",
categories = "D.3.2 Software, PROGRAMMING LANGUAGES, Language
Classifications, Larch. D.2.1 Software, SOFTWARE
ENGINEERING, Requirements/Specifications, ANNA. D.2.1
Software, SOFTWARE ENGINEERING,
Requirements/Specifications, Languages. D.2.5 Software,
SOFTWARE ENGINEERING, Testing and Debugging. D.2.4
Software, SOFTWARE ENGINEERING, Software/Program
Verification, Validation.",
annote = "incomplete",
abstract = "Current software testing practices focus, almost
exclusively, on the implementation, despite widely
acknowledged benefits of testing based on software
specifications. We propose approaches to
specification-based testing by extending a wide variety of
implementation-based testing techniques to be applicable
to formal specification languages. We demonstrate these
approaches for the Anna and Larch specification languages."
}
@Article{Offutt:1999:DDR,
author = "A. Jefferson Offutt and Zhenyi Jin and Jie Pan",
title = "The dynamic domain reduction procedure for test data
generation",
journal = j-SPE,
volume = "29",
number = "2",
pages = "167--193",
month = feb,
year = "1999",
coden = "SPEXBL",
ISSN = "0038-0644",
bibdate = "Thu Jul 29 15:12:12 MDT 1999",
url = "http://www3.interscience.wiley.com/cgi-bin/fulltext?ID=55000306&;PLACEBO=IE.pdf;
http://www3.interscience.wiley.com/cgi-bin/abstract?ID=55000306",
acknowledgement = ack-nhfb,
}
@Article{1991:jsi:offutt,
author = "A. Jefferson Offutt",
title = "An integrated automatic test data generation system",
journal = "Journal of Systems Integration",
volume = "1",
year = "1991",
number = "3",
month = nov,
pages = "391--409",
referencedby = "\cite{1992:tosem:offutt}, \cite{1993:tosem:demillo}",
annote = "incomplete",
abstract =
"The Godzilla automatic test data generator is an integrated collection of
tools that implements a relatively new test data generation method,
constraint-based testing, that is based on mutation
analysis. Constraint-based testing integrates mutation analysis with
several other testing techniques, including statement coverage, branch
coverage, domain perturbation and symbolic evaluation. Because Godzilla
uses a rule-based approach to generate test data, it is easily extendible
to allow new testing techniques to be integrated into the current
system. This paper describes the system that has been built to implement
constraint-based testing. Godzilla's design emphasizes orthogonality and
modularity, allowing relatively easy extensions. Godzilla's internal
structure and algorithms are described with emphasis on internal structures
of the system, and the engineering problems that were solved during the
implementation.",
}
@Article{OffuttL99,
author = "A. Jefferson Offutt and Shaoying Liu",
title = "Generating test data from {SOFL} specifications",
journal = "The Journal of Systems and Software",
volume = "49",
number = "1",
pages = "49--62",
day = "15",
month = dec,
year = "1999",
coden = "JSSODM",
ISSN = "0164-1212",
bibdate = "Tue Oct 10 10:06:05 MDT 2000",
url = "http://www.elsevier.nl/inca/publications/store/5/0/5/7/3/2/505732.pub.htt",
acknowledgement = ack-nhfb,
abstract = "Software testing can only be formalized and quantified
when a solid basis for test generation can be defined.
Tests are commonly generated from the source code,
control flow graphs, design representations, and
specifications/requirements. Formal specifications
represent a significant opportunity for testing because
they precisely describe what functions the software is
supposed to provide in a form that can be easily
manipulated. This paper presents a new method for
generating tests from formal specifications. This method
is comprehensive in specification coverage, applies at
several levels of abstraction, and can be highly
automated. The paper applies the method to SOFL
specifications, describes the technique, and demonstrates
the application on a case study. A preliminary evaluation
using a code-level coverage criterion (mutation testing),
indicates that the method can result in very effective
tests."
}
@TechReport{Burton99,
author = "Simon Burton",
title = "Towards automated unit testing of statechart implementations",
institution = "Department of Computer Science, University of York, UK",
year = "1999",
OPTkey = "",
OPTtype = "",
OPTnumber = "YCS 319",
OPTaddress = "",
OPTmonth = aug # "~2,",
OPTnote = "",
OPTannote = "",
url = "http://www.cs.york.ac.uk/ftpdir/reports/YCS-99-319.ps.gz",
abstract = "This report describes an automated method of unit test
design based on requirements specified in a subset of the
statechart notation. The behaviour under test is first
extracted from the requirements and specified in the Z
notation. Existing methods and tools are then applied to
this specification to derive the tests. Using Z to model
the requirements and specify the tests allows for a
deductive approach to verifying test satisfiability, test
result correctness and certain properties of the
requirements. An examination of the specification
coverage achieved by the tests is provided and the report
concludes with an evaluation of the work to date and a
set of directions for future work.",
}
@InProceedings{Donat97,
author = {Michael R. Donat},
title = {Automating Formal Specification-based Testing},
crossref = "TAPSOFT97",
pages = {833--847},
abstract = "This paper presents a technique for automatically generating
logical schemata that specify groups of black-box test cases
from formal specifications containing universal and
existential quantification. These schemata are called test
frames. Previous automated techniques have dealt with
languages based on propositional logic. Since this new
technique deals with quantification it can be applied to more
expressive specifications. This makes the technique
applicable to specifications written at the system
requirements level. The limitations imposed by quantification
are discussed. Industrial needs are addressed by the
capabilities of recognizing and augmenting existing test
frames and by accommodating a range of specification-coverage
schemes. The coverage scheme taxonomy introduced in this
paper provides a standard for controlling the number of test
frames produced. This technique is intended to automate
portions of what is done manually by practitioners. Basing
this technique on formal rules of logical derivation ensures
that the test frames produced are logical consequences of the
specification. It is expected that deriving test frames