gavo.rscdef.regtest

1 """ 2 A framework for regression tests within RDs. 3 4 The basic idea is that there's small pieces of python almost-declaratively 5 defining tests for a given piece of data. These things can then be 6 run while (or rather, after) executing gavo val. 7 """ 8 9 #c Copyright 2008-2019, the GAVO project 10 #c 11 #c This program is free software, covered by the GNU GPL. See the 12 #c COPYING file in the source distribution. 13 14 15 from __future__ import print_function 16 17 import argparse 18 import collections 19 import cPickle as pickle 20 import httplib 21 import os 22 import Queue 23 import random 24 import re 25 import sys 26 import time 27 import threading 28 import traceback 29 import unittest 30 import urllib 31 import urlparse 32 from cStringIO import StringIO 33 from email.Message import Message 34 from email.MIMEMultipart import MIMEMultipart 35 36 try: 37 from lxml import etree as lxtree 38 except ImportError: 39 # don't fail here, fail when running the test. 40 # TODO: add functionality to skip tests with dependencies not satisfied. 41 pass 42 43 from gavo import base 44 from gavo import votable 45 from gavo import utils 46 from gavo.utils import EqualingRE #noflake: published name 47 from . import common 48 from . import procdef

49 50 ################## Utilities 51 52 @utils.memoized 53 -def _loadCreds():

54 """returns a dictionary of auth keys to user/password pairs from 55 ~/.gavo/test.creds 56 """ 57 res = {} 58 try: 59 with open(os.path.join(os.environ["HOME"], ".gavo", "test.creds")) as f: 60 for ln in f: 61 authKey, user, pw = ln.strip().split() 62 res[authKey] = (user, pw) 63 except IOError: 64 pass 65 return res

66

67 68 -def getAuthFor(authKey):

69 """returns a header dictionary to authenticate for authKey. 70 71 authKey is a key into ~/.gavo/test.creds. 72 """ 73 try: 74 user, pw = _loadCreds()[authKey] 75 except KeyError: 76 raise base.NotFoundError(authKey, "Authorization info", 77 "~/.gavo/test.creds") 78 return {'Authorization': "Basic "+( 79 "%s:%s"%(user, pw)).encode("base64").strip()}

80

81 82 -def doHTTPRequest(method, host, path, query, 83 payload, headers, timeout):

84 """creates the HTTP request and retrieves the result. 85 """ 86 conn = httplib.HTTPConnection(host, timeout=timeout) 87 conn.connect() 88 try: 89 if query: 90 path = path+"?"+query 91 conn.request(method, path, payload, headers) 92 resp = conn.getresponse() 93 respHeaders = resp.getheaders() 94 content = resp.read() 95 finally: 96 conn.close() 97 return resp.status, respHeaders, content

98

99 100 -def getHeaderValue(headers, key):

101 """returns the value for key in the httplib headers. 102 103 Matching is case-insensitive as required by HTTP. Missing keys 104 raise KeyErrors. 105 """ 106 for hKey, hValue in headers: 107 if hKey.lower()==key.lower(): 108 return hValue 109 raise KeyError(key)

110

111 112 -class Keywords(argparse.Action):

113 """A class encapsulating test selection keywords. 114 115 There's a match method that takes a string and returns true if either 116 no keywords are defined or all keywords are present in other (after 117 case folding). 118 119 This doubles as an argparse action and as such is "self-parsing" if you 120 will. 121 """

122 - def __init__(self, *args, **kwargs):

123 argparse.Action.__init__(self, *args, **kwargs) 124 self.keywords = set()

125

126 - def __call__(self, parser, namespace, values, option_string=None):

127 self.keywords = set(values.lower().split()) 128 setattr(namespace, self.dest, self)

129

130 - def match(self, other):

131 if not self.keywords: 132 return True 133 134 return not self.keywords-set( 135 re.sub("[^\w\s]+", "", other).lower().split())

136

137 138 139 ################## RD elements 140 141 -class DynamicOpenVocAttribute(base.AttributeDef):

142 """an attribute that collects arbitrary attributes in a sequence 143 of pairs. 144 145 The finished sequence is available as a freeAttrs attribute on the 146 embedding instance. No parsing is done, everything is handled as 147 a string. 148 """ 149 typeDesc_ = "any attribute not otherwise used" 150

151 - def __init__(self, name, **kwargs):

152 base.AttributeDef.__init__(self, name, **kwargs)

153

154 - def feedObject(self, instance, value):

155 if not hasattr(instance, "freeAttrs"): 156 instance.freeAttrs = [] 157 instance.freeAttrs.append((self.name_, value))

158

159 - def feed(self, ctx, instance, value):

160 self.feedObject(instance, value)

161

162 - def getCopy(self, instance, newParent):

163 raise NotImplementedError("This needs some thought")

164

165 - def makeUserDoc(self):

166 return "(ignore)"

167

168 - def iterParentMethods(self):

169 def getAttribute(self, name): 170 # we need an instance-private attribute dict here: 171 if self.managedAttrs is self.__class__.managedAttrs: 172 self.managedAttrs = self.managedAttrs.copy() 173 174 try: 175 return base.Structure.getAttribute(self, name) 176 except base.StructureError: # no "real" attribute, it's a macro def 177 self.managedAttrs[name] = DynamicOpenVocAttribute(name) 178 # that's a decoy to make Struct.validate see a value for the attribute 179 setattr(self, name, None) 180 return self.managedAttrs[name]

181 yield "getAttribute", getAttribute

182

183 184 -class _FormData(MIMEMultipart):

185 """is a container for multipart/form-data encoded messages. 186 187 This is usually used for file uploads. 188 """

189 - def __init__(self):

190 MIMEMultipart.__init__(self, "form-data") 191 self.epilogue = ""

192

193 - def addFile(self, paramName, fileName, data):

194 """attaches the contents of fileName under the http parameter name 195 paramName. 196 """ 197 msg = Message() 198 msg.set_type("application/octet-stream") 199 msg["Content-Disposition"] = "form-data" 200 msg.set_param("name", paramName, "Content-Disposition") 201 msg.set_param("filename", fileName, "Content-Disposition") 202 msg.set_payload(data) 203 self.attach(msg)

204

205 - def addParam(self, paramName, paramVal):

206 """adds a form parameter paramName with the (string) value paramVal 207 """ 208 msg = Message() 209 msg["Content-Disposition"] = "form-data" 210 msg.set_param("name", paramName, "Content-Disposition") 211 msg.set_payload(paramVal) 212 self.attach(msg)

213

214 215 -class Upload(base.Structure):

216 """An upload going with a URL. 217 """ 218 name_ = "httpUpload" 219 220 _src = common.ResdirRelativeAttribute("source", 221 default=base.NotGiven, 222 description="Path to a file containing the data to be uploaded.", 223 copyable=True) 224 225 _name = base.UnicodeAttribute("name", 226 default=base.Undefined, 227 description="Name of the upload parameter", 228 copyable=True) 229 230 _filename = base.UnicodeAttribute("fileName", 231 default=None, 232 description="Remote file name for the uploaded file.", 233 copyable=True) 234 235 _content = base.DataContent(description="Inline data to be uploaded" 236 " (conflicts with source)") 237 238 @property

239 - def rd(self):

240 return self.parent.rd

241

242 - def addToForm(self, form):

243 """sets up a _Form instance to upload the data. 244 """ 245 if self.content_: 246 data = self.content_ 247 else: 248 with open(self.source) as f: 249 data = f.read() 250 form.addFile(self.name, self.fileName, data)

251

252 - def validate(self):

253 if (self.content_ and self.source 254 or not (self.content_ or self.source)): 255 raise base.StructureError("Exactly one of element content and source" 256 " attribute must be given for an upload.")

257

258 259 -class DataURL(base.Structure):

260 """A source document for a regression test. 261 262 As string URLs, they specify where to get data from, but the additionally 263 let you specify uploads, authentication, headers and http methods, 264 while at the same time saving you manual escaping of parameters. 265 266 The bodies is the path to run the test against. This is 267 interpreted as relative to the RD if there's no leading slash, 268 relative to the server if there's a leading slash, and absolute 269 if there's a scheme. 270 271 The attributes are translated to parameters, except for a few 272 pre-defined names. If you actually need those as URL parameters, 273 should at us and we'll provide some way of escaping these. 274 275 We don't actually parse the URLs coming in here. GET parameters 276 are appended with a & if there's a ? in the existing URL, with a ? 277 if not. Again, shout if this is too dumb for you (but urlparse 278 really isn't all that robust either...) 279 """ 280 name_ = "url" 281 282 # httpURL will be set to the URL actually used in retrieveResource 283 # Only use this to report the source of the data for, e.g., failing 284 # tests. 285 httpURL = "(not retrieved)" 286 287 _base = base.DataContent(description="Base for URL generation; embedded" 288 " whitespace will be removed, so you're free to break those whereever" 289 " you like.", 290 copyable=True) 291 292 _httpMethod = base.UnicodeAttribute("httpMethod", 293 description="Request method; usually one of GET or POST", 294 default="GET") 295 296 _httpPost = common.ResdirRelativeAttribute("postPayload", 297 default=base.NotGiven, 298 description="Path to a file containing material that should go" 299 " with a POST request (conflicts with additional parameters).", 300 copyable=True) 301 302 _parset = base.EnumeratedUnicodeAttribute("parSet", 303 description="Preselect a default parameter set; form gives what" 304 " our framework adds to form queries.", default=base.NotGiven, 305 validValues=["form", "TAP"], 306 copyable=True) 307 308 _httpHeaders = base.DictAttribute("httpHeader", 309 description="Additional HTTP headers to pass.", 310 copyable=True) 311 312 _httpAuthKey = base.UnicodeAttribute("httpAuthKey", 313 description="A key into ~/.gavo/test.creds to find a user/password" 314 " pair for this request.", 315 default=base.NotGiven, 316 copyable=True) 317 318 _httpUploads = base.StructListAttribute("uploads", 319 childFactory=Upload, 320 description='HTTP uploads to add to request (must have httpMethod="POST")', 321 copyable=True) 322 323 _httpHonorRedirects = base.BooleanAttribute("httpHonorRedirects", 324 default=False, 325 description="Follow 30x redirects instead of just using" 326 " status, headers, and payload of the initial request.", 327 copyable="True") 328 329 _rd = common.RDAttribute() 330 331 _open = DynamicOpenVocAttribute("open") 332 333

334 - def getValue(self, serverURL):

335 """returns a pair of full request URL and postable payload for this 336 test. 337 """ 338 urlBase = re.sub(r"\s+", "", self.content_) 339 if "://" in urlBase: 340 # we believe there's a scheme in there 341 pass 342 elif urlBase.startswith("/"): 343 urlBase = serverURL+urlBase 344 else: 345 urlBase = serverURL+"/"+self.parent.rd.sourceId+"/"+urlBase 346 347 if self.httpMethod=="POST": 348 return urlBase 349 else: 350 return self._addParams(urlBase, urllib.urlencode(self.getParams()))

351

352 - def getParams(self):

353 """returns the URL parameters as a sequence of kw, value pairs. 354 """ 355 params = getattr(self, "freeAttrs", []) 356 357 if self.parSet=="form": 358 params.extend([("__nevow_form__", "genForm"), ("submit", "Go"), 359 ("_charset_", "UTF-8")]) 360 361 elif self.parSet=='TAP': 362 params.extend([("LANG", "ADQL"), ("REQUEST", "doQuery")]) 363 364 return params

365 366

367 - def retrieveResource(self, serverURL, timeout):

368 """returns a triple of status, headers, and content for retrieving 369 this URL. 370 """ 371 self.httpURL, payload = self.getValue(serverURL), None 372 headers = { 373 "user-agent": "DaCHS regression tester"} 374 headers.update(self.httpHeader) 375 376 if self.httpMethod=="POST": 377 if self.postPayload: 378 with open(self.postPayload) as f: 379 payload = f.read() 380 381 elif self.uploads: 382 form = _FormData() 383 for key, value in self.getParams(): 384 form.addParam(key, value) 385 for upload in self.uploads: 386 upload.addToForm(form) 387 boundary = "========== roughtest deadbeef" 388 form.set_param("boundary", boundary) 389 headers["Content-Type"] = form.get_content_type( 390 )+'; boundary="%s"'%boundary 391 payload = form.as_string() 392 393 else: 394 payload = urllib.urlencode(self.getParams()) 395 headers["Content-Type"] = "application/x-www-form-urlencoded" 396 397 scheme, host, path, _, query, _ = urlparse.urlparse(str(self.httpURL)) 398 assert scheme=="http" 399 400 if self.httpAuthKey is not base.NotGiven: 401 headers.update(getAuthFor(self.httpAuthKey)) 402 status, respHeaders, content = doHTTPRequest(str(self.httpMethod), 403 host, path, query, payload, headers, timeout) 404 405 while self.httpHonorRedirects and status in [301, 302, 303]: 406 scheme, host, path, _, query, _ = urlparse.urlparse( 407 getHeaderValue(respHeaders, "location")) 408 status, respHeaders, content = doHTTPRequest("GET", 409 host, path, query, None, {}, timeout) 410 411 return status, respHeaders, content

412

413 - def _addParams(self, urlBase, params):

414 """a brief hack to add query parameters to GET-style URLs. 415 416 This is a workaround for not trusting urlparse and is fairly easy to 417 fool. 418 419 Params must already be fully encoded. 420 """ 421 if not params: 422 return urlBase 423 424 if "?" in urlBase: 425 return urlBase+"&"+params 426 else: 427 return urlBase+"?"+params

428

429 - def validate(self):

430 if self.postPayload is not base.NotGiven: 431 if self.getParams(): 432 raise base.StructureError("No parameters (or parSets) are" 433 " possible with postPayload") 434 if self.httpMethod!="POST": 435 raise base.StructureError("Only POST is allowed as httpMethod" 436 " together with postPayload") 437 438 if self.uploads: 439 if self.httpMethod!="POST": 440 raise base.StructureError("Only POST is allowed as httpMethod" 441 " together with upload") 442 443 self._validateNext(DataURL)

444

445 446 -class RegTest(procdef.ProcApp, unittest.TestCase):

447 """A regression test. 448 """ 449 name_ = "regTest" 450 requiredType = "regTest" 451 formalArgs = "self" 452 453 runCount = 1 454 455 additionalNamesForProcs = { 456 "EqualingRE": EqualingRE} 457 458 _title = base.NWUnicodeAttribute("title", 459 default=base.Undefined, 460 description="A short, human-readable phrase describing what this" 461 " test is exercising.") 462 463 _url = base.StructAttribute("url", 464 childFactory=DataURL, 465 default=base.NotGiven, 466 description="The source from which to fetch the test data.") 467 468 _tags = base.StringSetAttribute("tags", 469 description="A list of (free-form) tags for this test. Tagged tests" 470 " are only run when the runner is constructed with at least one" 471 " of the tags given. This is mainly for restricting tags to production" 472 " or development servers.") 473 474 _rd = common.RDAttribute() 475

476 - def __init__(self, *args, **kwargs):

477 unittest.TestCase.__init__(self, "fakeForPyUnit") 478 procdef.ProcApp.__init__(self, *args, **kwargs)

479

480 - def fakeForPyUnit(self):

481 raise AssertionError("This is not a pyunit test right now")

482 483 @property

484 - def description(self):

485 source = "" 486 if self.rd: 487 id = self.rd.sourceId 488 source = " (%s)"%id 489 return self.title+source

490

491 - def retrieveData(self, serverURL, timeout):

492 """returns headers and content when retrieving the resource at url. 493 494 Sets the headers and data attributes of the test instance. 495 """ 496 if self.url is base.NotGiven: 497 self.status, self.headers, self.data = None, None, None 498 else: 499 self.status, self.headers, self.data = self.url.retrieveResource( 500 serverURL, timeout=timeout)

501

502 - def getDataSource(self):

503 """returns a string pointing people to where data came from. 504 """ 505 if self.url is base.NotGiven: 506 return "(Unconditional)" 507 else: 508 return self.url.httpURL

509

510 - def pointNextToLocation(self, addToPath=""):

511 """arranges for the value of the location header to become the 512 base URL of the next test. 513 514 addToPath, if given, is appended to the location header. 515 516 If no location header was provided, the test fails. 517 518 All this of course only works for tests in sequential regSuites. 519 """ 520 if not hasattr(self, "followUp"): 521 raise AssertionError("pointNextToLocation only allowed within" 522 " sequential regSuites") 523 524 for key, value in self.headers: 525 if key=='location': 526 self.followUp.url.content_ = value+addToPath 527 break 528 else: 529 raise AssertionError("No location header in redirect")

530 531 @utils.document

532 - def assertHasStrings(self, *strings):

533 """checks that all its arguments are found within content. 534 """ 535 for phrase in strings: 536 assert phrase in self.data, "%s missing"%repr(phrase)

537 538 @utils.document

539 - def assertLacksStrings(self, *strings):

540 """checks that all its arguments are *not* found within content. 541 """ 542 for phrase in strings: 543 assert phrase not in self.data, "Unexpected: '%s'"%repr(phrase)

544 545 @utils.document

546 - def assertHTTPStatus(self, expectedStatus):

547 """checks whether the request came back with expectedStatus. 548 """ 549 assert expectedStatus==self.status, ("Bad status received, %s instead" 550 " of %s"%(self.status, expectedStatus))

551 552 @utils.document

553 - def assertValidatesXSD(self):

554 """checks whether the returned data are XSD valid. 555 556 This uses DaCHS built-in XSD validator with the built-in schema 557 files; it hence will in general not retrieve schema files from 558 external sources. 559 """ 560 from gavo.helpers import testtricks 561 msgs = testtricks.getXSDErrors(self.data) 562 if msgs: 563 raise AssertionError("Response not XSD valid. Validator output" 564 " starts with\n%s"%(msgs[:160]))

565 566 XPATH_NAMESPACE_MAP = { 567 "v": "http://www.ivoa.net/xml/VOTable/v1.3", 568 "v2": "http://www.ivoa.net/xml/VOTable/v1.2", 569 "v1": "http://www.ivoa.net/xml/VOTable/v1.1", 570 "o": "http://www.openarchives.org/OAI/2.0/", 571 "h": "http://www.w3.org/1999/xhtml", 572 } 573 574 @utils.document

575 - def assertXpath(self, path, assertions):

576 """checks an xpath assertion. 577 578 path is an xpath (as understood by lxml), with namespace 579 prefixes statically mapped; there's currently v2 (VOTable 580 1.2), v1 (VOTable 1.1), v (whatever VOTable version 581 is the current DaCHS default), h (the namespace of the 582 XHTML elements DaCHS generates), and o (OAI-PMH 2.0). 583 If you need more prefixes, hack the source and feed back 584 your changes (monkeypatching self.XPATH_NAMESPACE_MAP 585 is another option). 586 587 path must match exactly one element. 588 589 assertions is a dictionary mapping attribute names to 590 their expected value. Use the key None to check the 591 element content, and match for None if you expect an 592 empty element. 593 594 If you need an RE match rather than equality, there's 595 EqualingRE in your code's namespace. 596 """ 597 if not hasattr(self, "cached parsed tree"): 598 setattr(self, "cached parsed tree", lxtree.fromstring(self.data)) 599 tree = getattr(self, "cached parsed tree") 600 res = tree.xpath(path, namespaces=self.XPATH_NAMESPACE_MAP) 601 if len(res)==0: 602 raise AssertionError("Element not found: %s"%path) 603 elif len(res)!=1: 604 raise AssertionError("More than one item matched for %s"%path) 605 606 el = res[0] 607 for key, val in assertions.iteritems(): 608 if key is None: 609 foundVal = el.text 610 else: 611 foundVal = el.attrib[key] 612 assert val==foundVal, "Trouble with %s: %s (%s, %s)"%( 613 key or "content", path, repr(val), repr(foundVal))

614 615 @utils.document

616 - def getXpath(self, path, element=None):

617 """returns the equivalent of tree.xpath(path) for an lxml etree 618 of the current document or in element, if passed in. 619 620 This uses the same namespace conventions as assertXpath. 621 """ 622 if element is None: 623 if not hasattr(self, "_parsedTree"): 624 self._parsedTree = lxtree.fromstring(self.data) 625 element = self._parsedTree 626 627 return element.xpath(path, namespaces=self.XPATH_NAMESPACE_MAP)

628 629 @utils.document

630 - def assertHeader(self, key, value):

631 """checks that header key has value in the response headers. 632 633 keys are compared case-insensitively, values are compared literally. 634 """ 635 try: 636 foundValue = getHeaderValue(self.headers, key) 637 self.assertEqual(value, foundValue) 638 except KeyError: 639 raise AssertionError("Header %s not found in %s"%( 640 key, self.headers))

641 642 @utils.document

643 - def getFirstVOTableRow(self):

644 """interprets data as a VOTable and returns the first row as a dictionary 645 646 In test use, make sure the VOTable returned is sorted, or you will get 647 randomly failing tests. Ideally, you'll constrain the results to just 648 one match; database-querying cores (which is where order is an 649 issue) also honor _DBOPTIONS_ORDER). 650 """ 651 data, metadata = votable.loads(self.data) 652 for row in metadata.iterDicts(data): 653 return row

654 655 @utils.document

656 - def getVOTableRows(self):

657 """parses the first table in a result VOTable and returns the contents 658 as a sequence of dictionaries. 659 """ 660 data, metadata = votable.loads(self.data) 661 return list(metadata.iterDicts(data))

662

663 664 -class RegTestSuite(base.Structure):

665 """A suite of regression tests. 666 """ 667 name_ = "regSuite" 668 669 _tests = base.StructListAttribute("tests", 670 childFactory=RegTest, 671 description="Tests making up this suite", 672 copyable=False) 673 674 _title = base.NWUnicodeAttribute("title", 675 description="A short, human-readable phrase describing what this" 676 " suite is about.") 677 678 _sequential = base.BooleanAttribute("sequential", 679 description="Set to true if the individual tests need to be run" 680 " in sequence.", 681 default=False) 682

683 - def itertests(self, tags, keywords):

684 for test in self.tests: 685 if test.tags and not test.tags&tags: 686 continue 687 if keywords and not keywords.match(test.title): 688 continue 689 yield test

690

691 - def completeElement(self, ctx):

692 if self.title is None: 693 self.title = "Test suite from %s"%self.parent.sourceId 694 self._completeElementNext(base.Structure, ctx)

695

696 - def expand(self, *args, **kwargs):

697 """hand macro expansion to the RD. 698 """ 699 return self.parent.expand(*args, **kwargs)

700

701 702 #################### Running Tests 703 704 -class TestStatistics(object):

705 """A statistics gatherer/reporter for the regression tests. 706 """

707 - def __init__(self, verbose=True):

708 self.verbose = False 709 self.runs = [] 710 self.oks, self.fails, self.total = 0, 0, 0 711 self.globalStart = time.time() 712 self.lastTimestamp = time.time()+1 713 self.timeSum = 0

714

715 - def add(self, status, runTime, title, payload, srcRD):

716 """adds a test result to the statistics. 717 718 status is either OK, FAIL, or ERROR, runTime is the time 719 spent in running the test, title is the test's title, 720 and payload is "something" associated with failures that 721 should help diagnosing them. 722 """ 723 if status=="OK": 724 self.oks += 1 725 else: 726 if self.verbose: 727 print(">>>>>>>>", status) 728 self.fails += 1 729 self.total += 1 730 self.timeSum += runTime 731 #XXX TODO: Payload can use a lot of memory -- I'm nuking it for now 732 # -- maybe use an on-disk database to store this and allow later debugging? 733 self.runs.append((runTime, status, title, 734 None, #str(payload), 735 srcRD)) 736 self.lastTimestamp = time.time()

737

738 - def getReport(self):

739 """returns a string representation of a short report on how the tests 740 fared. 741 """ 742 try: 743 return ("%d of %d bad. avg %.2f, min %.2f, max %.2f. %.1f/s, par %.1f" 744 )%(self.fails, self.fails+self.oks, self.timeSum/len(self.runs), 745 min(self.runs)[0], max(self.runs)[0], float(self.total)/( 746 self.lastTimestamp-self.globalStart), 747 self.timeSum/(self.lastTimestamp-self.globalStart)) 748 except ZeroDivisionError: 749 return "No tests run (probably did not find any)."

750

751 - def getFailures(self):

752 """returns a string containing some moderately verbose info on the 753 failures collected. 754 """ 755 failures = {} 756 for runTime, status, title, payload, srcRD in self.runs: 757 if status!="OK": 758 failures.setdefault(srcRD, []).append("%s %s"%(status, title)) 759 760 return "\n".join("From %s:\n %s\n\n"%(srcRD, 761 "\n ".join(badTests)) 762 for srcRD, badTests in failures.iteritems())

763

764 - def save(self, target):

765 """saves the entire test statistics to target. 766 767 This is a pickle of basically what's added with add. No tools 768 for doing something with this are provided so far. 769 """ 770 with open(target, "w") as f: 771 pickle.dump(self.runs, f)

772

773 774 -class TestRunner(object):

775 """A runner for regression tests. 776 777 It is constructed with a sequence of suites (RegTestSuite instances) 778 and allows running these in parallel. It honors the suites' wishes 779 as to being executed sequentially. 780 """ 781 782 # The real trick here are the test suites with state (sequential=True. For 783 # those, the individual tests must be serialized, which happens using the magic 784 # followUp attribute on the tests. 785

786 - def __init__(self, suites, serverURL=None, 787 verbose=True, dumpNegative=False, tags=None, 788 timeout=45, failFile=None, nRepeat=1, 789 execDelay=0, nThreads=8, printTitles=False, 790 keywords=None):

791 self.verbose, self.dumpNegative = verbose, dumpNegative 792 self.failFile, self.nRepeat = failFile, nRepeat 793 self.printTitles = printTitles 794 if tags: 795 self.tags = tags 796 else: 797 self.tags = frozenset() 798 self.timeout = timeout 799 self.execDelay = execDelay 800 self.nThreads = nThreads 801 self.keywords = keywords 802 803 self.serverURL = serverURL or base.getConfig("web", "serverurl") 804 self.curRunning = {} 805 self.threadId = 0 806 self._makeTestList(suites) 807 self.stats = TestStatistics(verbose=self.verbose) 808 self.resultsQueue = Queue.Queue()

809 810 @classmethod

811 - def fromRD(cls, rd, **kwargs):

812 """constructs a TestRunner for a single ResourceDescriptor. 813 """ 814 return cls(rd.tests, **kwargs)

815 816 @classmethod

817 - def fromSuite(cls, suite, **kwargs):

818 """constructs a TestRunner for a RegTestSuite suite 819 """ 820 return cls([suite], **kwargs)

821 822 @classmethod

823 - def fromTest(cls, test, **kwargs):

824 """constructs a TestRunner for a single RegTest 825 """ 826 return cls([base.makeStruct(RegTestSuite, tests=[test], 827 parent_=test.parent.parent)], 828 **kwargs)

829

830 - def _makeTestList(self, suites):

831 """puts all individual tests from all test suites in a deque. 832 """ 833 self.testList = collections.deque() 834 for suite in suites: 835 if suite.sequential: 836 self._makeTestsWithState(suite) 837 else: 838 self.testList.extend(suite.itertests(self.tags, self.keywords))

839

840 - def _makeTestsWithState(self, suite):

841 """helps _makeTestList by putting suite's test in a way that they are 842 executed sequentially. 843 """ 844 # technically, this is done by just entering the suite's "head" 845 # and have that pull all the other tests in the suite behind it. 846 tests = list(suite.itertests(self.tags, self.keywords)) 847 if tests: 848 firstTest = tests.pop(0) 849 self.testList.append(firstTest) 850 for test in tests: 851 firstTest.followUp = test 852 firstTest = test

853

854 - def _spawnThread(self):

855 """starts a new test in a thread of its own. 856 """ 857 test = self.testList.popleft() 858 if self.printTitles: 859 sys.stderr.write(" <%s> "%test.title) 860 sys.stderr.flush() 861 862 newThread = threading.Thread(target=self.runOneTest, 863 args=(test, self.threadId, self.execDelay)) 864 newThread.description = test.description 865 newThread.setDaemon(True) 866 self.curRunning[self.threadId] = newThread 867 self.threadId += 1 868 newThread.start() 869 870 if test.runCount<self.nRepeat: 871 test.runCount += 1 872 self.testList.append(test)

873

874 - def runOneTest(self, test, threadId, execDelay):

875 """runs test and puts the results in the result queue. 876 877 This is usually run in a thread. However, threadId is only 878 used for reporting, so you may run this without threads. 879 880 To support sequential execution, if test has a followUp attribute, 881 this followUp is queued after the test has run. 882 883 If the execDelay argument is non-zero, the thread delays its execution 884 by that many seconds. 885 """ 886 if execDelay: 887 time.sleep(execDelay) 888 startTime = time.time() 889 try: 890 try: 891 test.retrieveData(self.serverURL, timeout=self.timeout) 892 test.compile()(test) 893 self.resultsQueue.put(("OK", test, None, None, time.time()-startTime)) 894 895 except KeyboardInterrupt: 896 raise 897 898 except AssertionError as ex: 899 self.resultsQueue.put(("FAIL", test, ex, None, 900 time.time()-startTime)) 901 # races be damned 902 if self.dumpNegative: 903 print("Content of failing test:\n%s\n"%test.data) 904 if self.failFile: 905 with open(self.failFile, "w") as f: 906 f.write(test.data) 907 908 except Exception as ex: 909 if self.failFile and getattr(test, "data", None) is not None: 910 with open(self.failFile, "w") as f: 911 f.write(test.data) 912 913 f = StringIO() 914 traceback.print_exc(file=f) 915 self.resultsQueue.put(("ERROR", test, ex, f.getvalue(), 916 time.time()-startTime)) 917 918 finally: 919 if hasattr(test, "followUp"): 920 self.resultsQueue.put(("addTest", test.followUp, None, None, 0)) 921 922 if threadId is not None: 923 self.resultsQueue.put(("collectThread", threadId, None, None, 0))

924

925 - def _printStat(self, state, test, payload, traceback):

926 """gives feedback to the user about the result of a test. 927 """ 928 if not self.verbose: 929 return 930 if state=="FAIL": 931 print("**** Test failed: %s -- %s\n"%( 932 test.title, test.getDataSource())) 933 print(">>>>", payload) 934 elif state=="ERROR": 935 print("**** Internal Failure: %s -- %s\n"%(test.title, 936 test.url.httpURL)) 937 print(traceback)

938

939 - def _runTestsReal(self, showDots=False):

940 """executes the tests, taking tests off the queue and spawning 941 threads until the queue is empty. 942 943 showDots, if True, instructs the runner to push one dot to stderr 944 per test spawned. 945 """ 946 while self.testList or self.curRunning: 947 while len(self.curRunning)<self.nThreads and self.testList: 948 self._spawnThread() 949 950 evType, test, payload, traceback, dt = self.resultsQueue.get( 951 timeout=self.timeout) 952 if evType=="addTest": 953 self.testList.appendleft(test) 954 elif evType=="collectThread": 955 deadThread = self.curRunning.pop(test) 956 deadThread.join() 957 else: 958 self.stats.add(evType, dt, test.title, "", test.rd.sourceId) 959 if showDots: 960 if evType=="OK": 961 sys.stderr.write(".") 962 else: 963 sys.stderr.write("E") 964 sys.stderr.flush() 965 self._printStat(evType, test, payload, traceback) 966 967 if showDots: 968 sys.stderr.write("\n")

969

970 - def runTests(self, showDots=False):

971 """executes the tests in a random order and in parallel. 972 """ 973 random.shuffle(self.testList) 974 try: 975 self._runTestsReal(showDots=showDots) 976 except Queue.Empty: 977 sys.stderr.write("******** Hung jobs\nCurrently executing:\n") 978 for thread in self.curRunning.values(): 979 sys.stderr.write("%s\n"%thread.description)

980

981 - def runTestsInOrder(self):

982 """runs all tests sequentially and in the order they were added. 983 """ 984 for test in self.testList: 985 self.runOneTest(test, None, self.execDelay) 986 try: 987 while True: 988 evType, test, payload, traceback, dt = self.resultsQueue.get(False) 989 if evType=="addTest": 990 self.testList.appendleft(test) 991 else: 992 self.stats.add(evType, dt, test.title, "", test.rd.sourceId) 993 self._printStat(evType, test, payload, traceback) 994 except Queue.Empty: 995 pass

996

997 998 ################### command line interface 999 1000 1001 -def urlToURL():

1002 """converts HTTP (GET) URLs to URL elements. 1003 """ 1004 # This is what's invoked by the makeTestURLs command. 1005 while True: 1006 parts = urlparse.urlparse(raw_input()) 1007 print("<url %s>%s</url>"%( 1008 " ".join('%s="%s"'%(k,v[0]) 1009 for k,v in urlparse.parse_qs(parts.query).iteritems()), 1010 parts.path))

1011

1012 1013 -def _getRunnerForAll(runnerArgs):

1014 from gavo.registry import publication 1015 from gavo import api 1016 1017 suites = [] 1018 for rdId in publication.findAllRDs(): 1019 try: 1020 rd = api.getRD(rdId, doQueries=False) 1021 except Exception as msg: 1022 base.ui.notifyError("Error loading RD %s (%s). Ignoring."%( 1023 rdId, utils.safe_str(msg))) 1024 suites.extend(rd.tests) 1025 1026 return TestRunner(suites, **runnerArgs)

1027

1028 1029 -def _getRunnerForSingle(testId, runnerArgs):

1030 from gavo import api 1031 1032 testElement = common.getReferencedElement(testId, doQueries=False) 1033 1034 if isinstance(testElement, api.RD): 1035 runner = TestRunner.fromRD(testElement, **runnerArgs) 1036 elif isinstance(testElement, RegTestSuite): 1037 runner = TestRunner.fromSuite(testElement, **runnerArgs) 1038 elif isinstance(testElement, RegTest): 1039 runner = TestRunner.fromTest(testElement, **runnerArgs) 1040 else: 1041 raise base.ReportableError("%s is not a testable element."%testId, 1042 hint="Only RDs, regSuites, or regTests are eligible for testing.") 1043 return runner

1044

1045 1046 -def parseCommandLine(args=None):

1047 """parses the command line for main() 1048 """ 1049 parser = argparse.ArgumentParser(description="Run tests embedded in RDs") 1050 parser.add_argument("id", type=str, 1051 help="RD id or cross-RD identifier for a testable thing.") 1052 parser.add_argument("-v", "--verbose", help="Dump info on failed test", 1053 action="store_true", dest="verbose") 1054 parser.add_argument("-V", "--titles", help="Write title when starting" 1055 " a test.", 1056 action="store_true", dest="printTitles") 1057 parser.add_argument("-d", "--dump-negative", help="Dump the content of" 1058 " failing tests to stdout", 1059 action="store_true", dest="dumpNegative") 1060 parser.add_argument("-t", "--tag", help="Also run tests tagged with TAG.", 1061 action="store", dest="tag", default=None, metavar="TAG") 1062 parser.add_argument("-R", "--n-repeat", help="Run each test N times", 1063 action="store", dest="nRepeat", type=int, default=None, metavar="N") 1064 parser.add_argument("-T", "--timeout", help="Abort and fail requests" 1065 " after inactivity of SECONDS", 1066 action="store", dest="timeout", type=int, default=15, metavar="SECONDS") 1067 parser.add_argument("-D", "--dump-to", help="Dump the content of" 1068 " last failing test to FILE", metavar="FILE", 1069 action="store", type=str, dest="failFile", 1070 default=None) 1071 parser.add_argument("-w", "--wait", help="Wait SECONDS before executing" 1072 " a request", metavar="SECONDS", action="store", 1073 dest="execDelay", type=int, default=0) 1074 parser.add_argument("-u", "--serverURL", help="URL of the DaCHS root" 1075 " at the server to test", 1076 action="store", type=str, dest="serverURL", 1077 default=base.getConfig("web", "serverURL")) 1078 parser.add_argument("-n", "--number-par", help="Number of requests" 1079 " to be run in parallel", 1080 action="store", type=int, dest="nThreads", 1081 default=8) 1082 parser.add_argument("--seed", help="Seed the RNG with this number." 1083 " Note that this doesn't necessarily make the execution sequence" 1084 " predictable, just the submission sequence.", 1085 action="store", type=int, dest="randomSeed", default=None) 1086 parser.add_argument("-k", "--keywords", help="Only run tests" 1087 " with descriptions containing all (whitespace-separated) keywords." 1088 " Sequential tests will be run in full, nevertheless, if their head test" 1089 " matches.", 1090 action=Keywords, type=str, dest="keywords") 1091 1092 return parser.parse_args(args)

1093

1094 1095 -def main(args=None):

1096 """user interaction for gavo test. 1097 """ 1098 tags = None 1099 args = parseCommandLine(args) 1100 if args.randomSeed: 1101 random.seed(args.randomSeed) 1102 if args.tag: 1103 tags = set([args.tag]) 1104 if args.serverURL: 1105 args.serverURL = args.serverURL.rstrip("/") 1106 1107 runnerArgs = { 1108 "verbose": args.verbose, 1109 "dumpNegative": args.dumpNegative, 1110 "serverURL": args.serverURL, 1111 "tags": tags, 1112 "failFile": args.failFile, 1113 "nRepeat": args.nRepeat, 1114 "timeout": args.timeout, 1115 "execDelay": args.execDelay, 1116 "nThreads": args.nThreads, 1117 "printTitles": args.printTitles, 1118 "keywords": args.keywords, 1119 } 1120 1121 if args.id=="ALL": 1122 runner = _getRunnerForAll(runnerArgs) 1123 else: 1124 runner = _getRunnerForSingle(args.id, runnerArgs) 1125 1126 runner.runTests(showDots=True) 1127 print(runner.stats.getReport()) 1128 if runner.stats.fails: 1129 print(runner.stats.getFailures()) 1130 sys.exit(1)

1131

Source Code for Module gavo.rscdef.regtest