Blame | Last modification | View Log | RSS feed
#!/usr/bin/python
#
#Script: stdriver.py
#Author: Patrick Boyd
#
#Purpose: Run point-valued python test scripts and generate a grade sheet
#
#Description:
#Runs a set of python tests as determined by the contents of the input file.
#The contents of the .txt file should appear as follows:
# = test group 1
# 5 test_worth_5.py
# 10 test_worth 10.py
# 15 test_worth_15.py
# = test group 2
# 20 test_worth 20.py
# 25 test_worth 25.py
# ...
#
#These tests will then be spawned into their own subprocesses and ran,
#and their results will be recorded. If the verbose option is enabled,
#error information will be displayed.
#
#
#
import getopt, os, sys, subprocess, re
# add directory in which script is located to python path
script_dir = "/".join(__file__.split("/")[:-1])
if script_dir == "":
script_dir = "."
if script_dir not in sys.path:
sys.path.append(script_dir)
verbose = False
output_spec_file = "./eshoutput.py"
plugin_directory = ""
test_base = "./eshtests/"
milestone_test_file = "milestone.tst"
basic_test_file = "basic.tst"
reduced_test_file = "reduced.tst"
advanced_test_file = "advanced.tst"
testfilter = None
def usage():
print """
Usage: python stdriver.py [options] [tests1.tst tests2.tst] ...
-m Run milestone tests
-p plugin_dir Run plugins from the supplied directory
-b Run basic tests
-v Verbose
-h Show help
-o outputspec Run using this output spec file
-t testname Run only tests whose names contains 'testname'
-l List available tests (not implemented)
-a Run advanced tests
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "ahvmrbo:p:t:", \
["help", "outputspec=", "test="])
except getopt.GetoptError, err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-t", "--test"):
testfilter = a
elif o in ("-m"):
args = [test_base + milestone_test_file]
elif o in ("-b"):
args = [test_base + basic_test_file]
elif o in ("-a"):
args = [test_base + advanced_test_file]
elif o in ("-r"):
#args = [test_base + reduced_test_file]
print "-r is no longer allowed"
usage()
sys.exit()
elif o in ("-p"):
plugin_directory = a
elif o in ("-o"):
output_spec_file = a
else:
assert False, "unhandled option"
if plugin_directory != "" and not os.access(plugin_directory, os.R_OK):
print "Directory ", plugin_directory, " is not readable"
print "You must create this plugin directory if you wish to test plugins"
sys.exit(1)
else:
plugin_directory = " -p " + plugin_directory
if not os.access(output_spec_file, os.R_OK):
print "File ", output_spec_file, " is not readable"
print "You must create this file and adapt it to match the output of your shell"
sys.exit(1)
full_testlist = []
if len(args) == 0:
usage()
sys.exit(1)
for testlist_filename in args:
try:
testlist_file = open(testlist_filename, 'r')
test_dir = os.path.dirname(testlist_filename)
if test_dir == "":
test_dir = './'
else:
test_dir = test_dir + '/'
except:
print 'Error: Tests list file: ''%s'' could not be opened.'% testlist_filename
sys.exit(-1)
#File input, read in the test filenames
#test information is placed into tuples that are placed onto a 2d list
for line in testlist_file:
grps = re.match("^= (.+)", line)
if grps:
testset = { 'name' : grps.group(1),
'tests' : [ ],
'dir' : test_dir }
full_testlist.append(testset)
else:
grps = re.match("(\d+) (.+)", line)
if grps:
points, testname = int(grps.group(1)), grps.group(2)
if not testfilter or testname.find(testfilter) != -1:
testset['tests'].append((points, testname))
testlist_file.close()
# print full_testlist
process_list = []
#Run through each test set in the list
for testset in full_testlist:
print testset['name']
print '-------------------------'
#Run through each test in the set
testresults = [ ]
for (points, testname) in testset['tests']:
print str(points) + '\t' + testname + ':',
sys.stdout.flush()
# run test
child_process = subprocess.Popen(["python", testset['dir'] + testname, \
output_spec_file, plugin_directory],\
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
test_passed = child_process.wait() == 0
# build result list
testresults.append((points, testname, child_process, test_passed))
#if: test passed
if test_passed:
print ' PASS'
else:
print ' FAIL'
print ""
testset['tests'] = testresults
#Grade sheet printing. It prints each individual test and the points awarded
#from that test, the points awarded from each set of tests, and the total
#points awarded over the entire test.
#Run through each set of tests in the list
testset_points = 0
testset_points_earned = 0
#Run through each test in the set
for (points, testname, child_process, test_passed) in testset['tests']:
testset_points += points
points_earned = points
if not test_passed:
points_earned = 0
print '\t%s\t%d/%d' % (testname, points_earned, points)
testset_points_earned += points_earned
print '-------------------------'
print testset['name'] + '\t' + str(testset_points_earned) + '/' + \
str(testset_points)
#Verbose printing. If the verbose option was enabled, print the error
#information from the tests that failed.
if verbose:
print '\nError Output'
print '-------------------------'
for testset in full_testlist:
for (points, testname, child_process, test_passed) in testset['tests']:
if not test_passed:
print testname + ':'
(stdout, stderr) = child_process.communicate()
print stdout, stderr