Subversion Repositories Code-Repo

#!/usr/bin/python

#

#Script: stdriver.py

#Author: Patrick Boyd

#

#Purpose: Run point-valued python test scripts and generate a grade sheet

#

#Description:

#Runs a set of python tests as determined by the contents of the input file.

#The contents of the .txt file should appear as follows:

#   = test group 1

#   5 test_worth_5.py

#   10 test_worth 10.py

#   15 test_worth_15.py

#   = test group 2

#   20 test_worth 20.py

#   25 test_worth 25.py

#   ...

#

#These tests will then be spawned into their own subprocesses and ran,

#and their results will be recorded.  If the verbose option is enabled,

#error information will be displayed.

#

#

#

import getopt, os, sys, subprocess, re

# add directory in which script is located to python path

script_dir = "/".join(__file__.split("/")[:-1])

if script_dir == "":

    script_dir = "."

if script_dir not in sys.path:

    sys.path.append(script_dir)

verbose = False

output_spec_file = "./eshoutput.py"

plugin_directory = ""

test_base = "./eshtests/"

milestone_test_file = "milestone.tst"

basic_test_file = "basic.tst"

reduced_test_file = "reduced.tst"

advanced_test_file = "advanced.tst"

testfilter = None

def usage():

    print """

Usage: python stdriver.py [options] [tests1.tst tests2.tst] ... 

    -m              Run milestone tests 

    -p plugin_dir   Run plugins from the supplied directory

    -b              Run basic tests

    -v              Verbose 

    -h              Show help 

    -o outputspec   Run using this output spec file

    -t testname     Run only tests whose names contains 'testname'

    -l              List available tests (not implemented)

    -a              Run advanced tests

    """

try:

    opts, args = getopt.getopt(sys.argv[1:], "ahvmrbo:p:t:", \

        ["help", "outputspec=", "test="])

except getopt.GetoptError, err:

    # print help information and exit:

    print str(err) # will print something like "option -a not recognized"

    usage()

    sys.exit(2)

for o, a in opts:

    if o == "-v":

        verbose = True

    elif o in ("-h", "--help"):

        usage()

        sys.exit()

    elif o in ("-t", "--test"):

        testfilter = a

    elif o in ("-m"):

        args = [test_base + milestone_test_file]

    elif o in ("-b"):

        args = [test_base + basic_test_file]

    elif o in ("-a"):

        args = [test_base + advanced_test_file]

    elif o in ("-r"):

        #args = [test_base + reduced_test_file]

        print "-r is no longer allowed"

        usage()

        sys.exit()

    elif o in ("-p"):

        plugin_directory = a

    elif o in ("-o"):

        output_spec_file = a

    else:

        assert False, "unhandled option"

if plugin_directory != "" and not os.access(plugin_directory, os.R_OK):

    print "Directory ", plugin_directory, " is not readable"

    print "You must create this plugin directory if you wish to test plugins"

    sys.exit(1)

else:

    plugin_directory = " -p " + plugin_directory

if not os.access(output_spec_file, os.R_OK):

    print "File ", output_spec_file, " is not readable"

    print "You must create this file and adapt it to match the output of your shell"

    sys.exit(1)

full_testlist = []

if len(args) == 0:

    usage()

    sys.exit(1)

for testlist_filename in args:

    try:

        testlist_file = open(testlist_filename, 'r')

        test_dir = os.path.dirname(testlist_filename)

        if test_dir == "":

            test_dir = './'

        else:

            test_dir = test_dir + '/'

    except:

        print 'Error: Tests list file: ''%s'' could not be opened.'% testlist_filename

        sys.exit(-1)

    #File input, read in the test filenames

    #test information is placed into tuples that are placed onto a 2d list

    for line in testlist_file:

        grps = re.match("^= (.+)", line)

        if grps:

            testset = { 'name' : grps.group(1), 

                        'tests' : [ ], 

                        'dir' : test_dir }

            full_testlist.append(testset)

        else:

            grps = re.match("(\d+) (.+)", line)

            if grps:

                points, testname = int(grps.group(1)), grps.group(2)

                if not testfilter or testname.find(testfilter) != -1:

                    testset['tests'].append((points, testname))

    testlist_file.close()

# print full_testlist

process_list = []

#Run through each test set in the list

for testset in full_testlist:

    print testset['name']

    print '-------------------------'

    #Run through each test in the set

    testresults = [ ]

    for (points, testname) in testset['tests']:

        print str(points) + '\t' + testname + ':',

        sys.stdout.flush()

        # run test

        child_process = subprocess.Popen(["python", testset['dir'] + testname, \

                             output_spec_file, plugin_directory],\

                             stderr=subprocess.PIPE, stdout=subprocess.PIPE)

        test_passed = child_process.wait() == 0

        # build result list

        testresults.append((points, testname, child_process, test_passed))

        #if: test passed

        if test_passed:

            print ' PASS'

        else:

            print ' FAIL'

    print ""

    testset['tests'] = testresults

    #Grade sheet printing.  It prints each individual test and the points awarded

    #from that test, the points awarded from each set of tests, and the total

    #points awarded over the entire test.

    #Run through each set of tests in the list

    testset_points = 0

    testset_points_earned = 0

    #Run through each test in the set

    for (points, testname, child_process, test_passed) in testset['tests']:

        testset_points += points

        points_earned = points

        if not test_passed:

            points_earned = 0

        print '\t%s\t%d/%d' % (testname, points_earned, points)

        testset_points_earned += points_earned

    print '-------------------------'

    print testset['name'] + '\t' + str(testset_points_earned) + '/' + \

                                                        str(testset_points)

#Verbose printing.  If the verbose option was enabled, print the error

#information from the tests that failed.

if verbose:

    print '\nError Output'

    print '-------------------------'

    for testset in full_testlist:

        for (points, testname, child_process, test_passed) in testset['tests']:

            if not test_passed:

                print testname + ':'

                (stdout, stderr) = child_process.communicate()

                print stdout, stderr