gavo.web.vodal

1 """ 2 Support for IVOA DAL and registry protocols. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import datetime 12 import os 13 14 from nevow import appserver 15 from nevow import inevow 16 17 from twisted.internet import defer 18 19 from zope.interface import implements 20 21 from gavo import base 22 from gavo import formats 23 from gavo import registry 24 from gavo import rscdef 25 from gavo import rsc 26 from gavo import svcs 27 from gavo import utils 28 from gavo import votable 29 from gavo.protocols import dali 30 from gavo.protocols import dlasync 31 from gavo.protocols import ssap 32 from gavo.svcs import streaming 33 from gavo.votable import V 34 from gavo.web import grend 35 36 37 MS = base.makeStruct 38 39 40 __docformat__ = "restructuredtext en"

41 42 43 -class DALRenderer(grend.ServiceBasedPage):

44 """is a base class for renderers for the usual IVOA DAL protocols. 45 46 This is for simple, GET-based DAL renderers (where we allow POST as 47 well). They work using nevow forms, but with standard-compliant error 48 reporting (i.e., in VOTables). 49 50 Since DALRenderer mixes in FormMixin, it always has the form genFrom. 51 """ 52 53 implements(inevow.ICanHandleException) 54 55 resultType = base.votableType 56 urlUse = "base" 57 standardId = None 58

59 - def __init__(self, ctx, *args, **kwargs):

60 reqArgs = inevow.IRequest(ctx).args 61 # XXX TODO: Do away with _FORMAT in general, move to RESPONSEFORMAT 62 reqArgs["_FORMAT"] = ["VOTable"] 63 64 # see _writeErrorTable 65 self.saneResponseCodes = False 66 grend.ServiceBasedPage.__init__(self, ctx, *args, **kwargs) 67 self.defaultLimit = base.getConfig("ivoa", "dalDefaultLimit")

68 69 @classmethod

70 - def makeAccessURL(cls, baseURL):

71 return "%s/%s?"%(baseURL, cls.name)

72 73 @classmethod

74 - def isBrowseable(self, service):

75 return False

76

77 - def renderHTTP(self, ctx):

78 queryMeta = svcs.QueryMeta.fromContext(ctx, 79 defaultLimit=self.defaultLimit) 80 if (queryMeta["dbLimit"]==0 81 or queryMeta.ctxArgs.get("FORMAT", "").lower()=="metadata"): 82 return self._renderMetadata(ctx, queryMeta) 83 84 return defer.maybeDeferred(self._runService, ctx, queryMeta 85 ).addErrback(self._handleInputErrors, ctx 86 ).addErrback(self._handleRandomFailure, ctx)

87

88 - def renderSync(self, ctx):

89 # This does essentially what renderHTTP does, but synchronously. 90 # This *should* work for DAL protocols, but it's really only 91 # intended for unit testing. It's not a bug in a renderer or 92 # core if renderSync doesn't work for it. 93 queryMeta = svcs.QueryMeta.fromContext(ctx) 94 try: 95 request = inevow.IRequest(ctx) 96 dali.mangleUploads(request) 97 res = self.processData(request.args, queryMeta) 98 except Exception: 99 raise # TODO: do _handleInputErrors and _handleRandomFailures 100 return self._formatOutput(res, ctx, queryMeta, stream=False)

101

102 - def _getMetadataData(self, queryMeta):

103 """returns a SIAP-style metadata data item. 104 """ 105 inputFields = [] 106 for param in self.service.getInputKeysFor(self): 107 # Caution: UPLOAD mangling is a *renderer* thing -- the core 108 # doesn't know anything about it. Hence, parameter adaption 109 # is *not* done by adapting the core. Instead: 110 if param.type=="file": 111 inputFields.append(dali.getUploadKeyFor(param)) 112 else: 113 inputFields.append(param.change(name="INPUT:"+param.name)) 114 outputTD = self.service.core.outputTable.change(id="results") 115 for param in outputTD.params: 116 param.name = "OUTPUT:"+param.name 117 118 nullRowmaker = MS(rscdef.RowmakerDef) 119 dataDesc = MS(rscdef.DataDescriptor, makes=[ 120 MS(rscdef.Make, table=outputTD, rowmaker=nullRowmaker)], 121 params=inputFields, 122 parent_=self.service.rd) 123 124 data = rsc.makeData(dataDesc) 125 data.tables["results"].votCasts = self._outputTableCasts 126 data.setMeta("_type", "results") 127 data.addMeta("info", 128 base.getMetaText(self.service, "title") or "Unnamed", 129 infoName="serviceInfo", 130 infoValue=str(self.service.getURL(self.name))) 131 132 return data

133

134 - def _renderMetadata(self, ctx, queryMeta):

135 metaData = self._getMetadataData(queryMeta) 136 metaData.addMeta("info", "OK", 137 infoName="QUERY_STATUS", infoValue="OK") 138 request = inevow.IRequest(ctx) 139 request.setHeader("content-type", "text/xml") 140 votLit = formats.getFormatted("votable", metaData) 141 # maybe provide a better way to attach stylesheet info? 142 splitPos = votLit.find("?>")+2 143 return votLit[:splitPos]+("<?xml-stylesheet href='/static" 144 "/xsl/meta-votable-to-html.xsl' type='text/xsl'?>" 145 )+votLit[splitPos:]

146

147 - def _runService(self, ctx, queryMeta):

148 request = inevow.IRequest(ctx) 149 150 # don't enable this for now. If a validator actually starts validating 151 # against the whole VERSION catastrophe, re-think this. 152 # if "VERSION" in request.args: 153 # if request.args["VERSION"]!=self.version: 154 # raise ValidationError("Version mismatch -- version supported by" 155 # " this service: %s"%self.version) 156 157 dali.mangleUploads(request) 158 return self.runService(request.args, queryMeta 159 ).addCallback(self._formatOutput, ctx, queryMeta)

160

161 - def _writeErrorTable(self, ctx, errmsg, code=200, queryStatus="ERROR"):

162 request = inevow.IRequest(ctx) 163 164 # Unfortunately, most legacy DAL specs say the error messages must 165 # be delivered with a 200 response code. I hope this is going 166 # to change at some point, so I let renderers order sane response 167 # codes. 168 if not self.saneResponseCodes: 169 request.setResponseCode(code) 170 result = self._makeErrorTable(ctx, errmsg, queryStatus) 171 request.setHeader("content-type", base.votableType) 172 votable.write(result, request) 173 return "\n"

174

175 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

176 data.original.addMeta("info", "", infoName="QUERY_STATUS", 177 infoValue=base.getMetaText(data.original.getPrimaryTable(), 178 "_queryStatus")) 179 180 if self.standardId: 181 data.original.addMeta("info", 182 "Written by DaCHS %s %s"%(base.getVersion(), self.__class__.__name__), 183 infoName="standardID", 184 infoValue=self.standardId) 185 186 request = inevow.IRequest(ctx) 187 destFormat = base.votableType 188 if "RESPONSEFORMAT" in queryMeta.ctxArgs: 189 # This is our DALI RESPONSEFORMAT implementation; the corresponding 190 # parameter is declared automatically in Service.completeElement 191 # from pql#DALIPars. 192 destFormat = queryMeta.ctxArgs["RESPONSEFORMAT"] 193 requestedType = formats.getMIMEFor(destFormat, destFormat) 194 else: 195 requestedType = self.resultType 196 197 request.setHeader("content-type", requestedType) 198 request.setHeader('content-disposition', 199 'attachment; filename="result.%s"'% 200 formats.getExtensionFor(requestedType)) 201 202 formatterArgs = {} 203 204 if stream: 205 206 def writeStuff(outputFile): 207 formats.formatData(destFormat, 208 data.original, outputFile, acquireSamples=False, **formatterArgs)

209 210 return streaming.streamOut(writeStuff, request) 211 212 else: 213 return formats.getFormatted(destFormat, 214 data.original, acquireSamples=False, **formatterArgs)

215 216

217 - def _handleRandomFailure(self, failure, ctx):

218 if base.DEBUG: 219 base.ui.notifyFailure(failure) 220 return self._writeErrorTable(ctx, 221 "Unexpected failure, error message: %s"%failure.getErrorMessage(), 222 500)

223

224 - def _handleInputErrors(self, failure, ctx):

225 queryStatus = "ERROR" 226 227 base.ui.notifyFailure(failure) 228 229 if isinstance(failure.value, base.EmptyData): 230 inevow.IRequest(ctx).setResponseCode(400) 231 queryStatus = "EMPTY" 232 233 return self._writeErrorTable(ctx, failure.getErrorMessage(), 234 queryStatus=queryStatus)

235

236 237 -class SCSRenderer(DALRenderer):

238 """ 239 A renderer for the Simple Cone Search protocol. 240 241 These do their error signaling in the value attribute of an 242 INFO child of RESOURCE. 243 244 You must set the following metadata items on services using 245 this renderer if you want to register them: 246 247 * testQuery.ra, testQuery.dec -- A position for which an object is present 248 within 0.001 degrees. 249 """ 250 name = "scs.xml" 251 version = "1.0" 252 parameterStyle = "pql" 253 standardId = "ivo://ivoa.net/std/ConeSearch" 254 255 # move the ucdCasts from formatOutput here. 256 _outputTableCasts = {} 257

258 - def __init__(self, ctx, *args, **kwargs):

259 reqArgs = inevow.IRequest(ctx).args 260 if "RESPONSEFORMAT" not in reqArgs: 261 reqArgs["RESPONSEFORMAT"] = ["votable1.1"] 262 self.defaultLimit = base.getConfig("ivoa", "dalDefaultLimit")*10 263 DALRenderer.__init__(self, ctx, *args, **kwargs)

264

265 - def _writeErrorTable(self, ctx, msg, code=200, queryStatus="ERROR"):

266 request = inevow.IRequest(ctx) 267 request.setHeader("content-type", base.votableType) 268 votable.write(V.VOTABLE11[ 269 V.DESCRIPTION[base.getMetaText(self.service, "description")], 270 V.INFO(ID="Error", name="Error", 271 value=str(msg).replace('"', '\\"'))], request) 272 request.write("\n") 273 return ""

274

275 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

276 """makes output SCS 1.02 compatible or causes the service to error out. 277 278 This comprises mapping meta.id;meta.main to ID_MAIN and 279 pos.eq* to POS_EQ*. 280 """ 281 ucdCasts = { 282 "meta.id;meta.main": {"ucd": "ID_MAIN", "datatype": "char", 283 "arraysize": "*"}, 284 "pos.eq.ra;meta.main": {"ucd": "POS_EQ_RA_MAIN", 285 "datatype": "double"}, 286 "pos.eq.dec;meta.main": {"ucd": "POS_EQ_DEC_MAIN", 287 "datatype": "double"}, 288 } 289 realCasts = {} 290 table = data.original.getPrimaryTable() 291 for ind, ofield in enumerate(table.tableDef.columns): 292 if ofield.ucd in ucdCasts: 293 realCasts[ofield.name] = ucdCasts.pop(ofield.ucd) 294 if ucdCasts: 295 return self._writeErrorTable(ctx, "Table cannot be formatted for" 296 " SCS. Column(s) with the following new UCD(s) were missing in" 297 " output table: %s"%', '.join(ucdCasts)) 298 299 # allow integers as ID_MAIN [HACK -- this needs to become saner. 300 # conditional cast functions?] 301 idCol = table.tableDef.getColumnByUCD("meta.id;meta.main") 302 if idCol.type in set(["integer", "bigint", "smallint"]): 303 realCasts[idCol.name]["castFunction"] = str 304 table.votCasts = realCasts 305 306 return DALRenderer._formatOutput(self, 307 data, ctx, queryMeta, 308 stream=stream)

309

310 311 -class SIAPRenderer(DALRenderer):

312 """A renderer for a the Simple Image Access Protocol. 313 314 These have errors in the content of an info element, and they support 315 metadata queries. 316 317 For registration, services using this renderer must set the following 318 metadata items: 319 320 - sia.type -- one of Cutout, Mosaic, Atlas, Pointed, see SIAP spec 321 322 You should set the following metadata items: 323 324 - testQuery.pos.ra, testQuery.pos.dec -- RA and Dec for a query that 325 yields at least one image 326 - testQuery.size.ra, testQuery.size.dec -- RoI extent for a query that 327 yields at least one image. 328 329 You can set the following metadata items (there are defaults on them 330 that basically communicate there are no reasonable limits on them): 331 332 - sia.maxQueryRegionSize.(long|lat) 333 - sia.maxImageExtent.(long|lat) 334 - sia.maxFileSize 335 - sia.maxRecord (default dalHardLimit global meta) 336 """ 337 version = "1.0" 338 name = "siap.xml" 339 parameterStyle = "pql" 340 standardId = "ivo://ivoa.net/std/sia" 341

342 - def __init__(self, ctx, *args, **kwargs):

343 DALRenderer.__init__(self, ctx, *args, **kwargs)

344

345 - def renderHTTP(self, ctx):

346 args = inevow.IRequest(ctx).args 347 try: 348 metadataQuery = args["FORMAT"][0].lower()=="metadata" 349 except (IndexError, KeyError): 350 metadataQuery = False 351 if metadataQuery: 352 return self._renderMetadata(ctx, svcs.QueryMeta.fromContext(ctx)) 353 354 return DALRenderer.renderHTTP(self, ctx)

355 356 _outputTableCasts = { 357 "pixelScale": {"datatype": "double", "arraysize": "*"}, 358 "wcs_cdmatrix": {"datatype": "double", "arraysize": "*"}, 359 "wcs_refValues": {"datatype": "double", "arraysize": "*"}, 360 "bandpassHi": {"datatype": "double"}, 361 "bandpassLo": {"datatype": "double"}, 362 "bandpassRefval": {"datatype": "double"}, 363 "wcs_refPixel": {"datatype": "double", "arraysize": "*"}, 364 "wcs_projection": {"arraysize": "3", "castFunction": lambda s: s[:3]}, 365 "mime": {"ucd": "VOX:Image_Format"}, 366 "accref": {"ucd": "VOX:Image_AccessReference"}, 367 "accsize": {"datatype": "int"}, 368 } 369

370 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

371 data.original.setMeta("_type", "results") 372 data.original.getPrimaryTable().votCasts = self._outputTableCasts 373 return DALRenderer._formatOutput(self, 374 data, ctx, queryMeta, 375 stream=stream)

376

377 - def _makeErrorTable(self, ctx, msg, queryStatus="ERROR"):

378 return V.VOTABLE11[ 379 V.RESOURCE(type="results")[ 380 V.INFO(name="QUERY_STATUS", value=queryStatus)[ 381 str(msg)]]]

382

383 384 -class UnifiedDALRenderer(DALRenderer):

385 """A renderer for new-style simple DAL protocols. 386 387 All input processing (e.g., metadata queries and the like) are considered 388 part of the individual protocol and thus left to the core. 389 390 The error style is that of SSAP (which, hopefully, will be kept 391 for the other DAL2 protocols, too). 392 393 To define actual renderers, inherit from this and set the name attribute 394 (plus _outputTableCasts if necessary). Also, explain any protocol-specific 395 metadata in the docstring. 396 """ 397 398 _outputTableCasts = {} 399

400 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

401 request = inevow.IRequest(ctx) 402 if isinstance(data.original, tuple): 403 # core returned a complete document (mime and string) 404 mime, payload = data.original 405 request.setHeader("content-type", mime) 406 return streaming.streamOut(lambda f: f.write(payload), request) 407 else: 408 request.setHeader("content-type", "text/xml+votable") 409 data.original.setMeta("_type", "results") 410 data.original.getPrimaryTable().votCasts = self._outputTableCasts 411 return DALRenderer._formatOutput(self, 412 data, ctx, queryMeta, 413 stream=stream)

414

415 - def _makeErrorTable(self, ctx, msg, queryStatus="ERROR"):

416 return V.VOTABLE11[ 417 V.RESOURCE(type="results")[ 418 V.INFO(name="QUERY_STATUS", value=queryStatus)[ 419 str(msg)]]]

420

421 422 -class SIAP2Renderer(UnifiedDALRenderer):

423 """A renderer for SIAPv2. 424 425 In general, if you want a SIAP2 service, you'll need something like the 426 obscore view in the underlying table. 427 """ 428 parameterStyle = "dali" 429 name = "siap2.xml" 430 standardId = "ivo://ivoa.net/std/sia" 431

432 - def _makeErrorTable(self, ctx, msg, queryStatus="ERROR"):

433 # FatalFault, DefaultFault 434 return V.VOTABLE[ 435 V.RESOURCE(type="results")[ 436 V.INFO(name="QUERY_STATUS", value=queryStatus)[ 437 str(msg)]]]

438

439 - def _handleRandomFailure(self, failure, ctx):

440 if base.DEBUG: 441 base.ui.notifyFailure(failure) 442 return self._writeErrorTable(ctx, 443 "DefaultFault: "+failure.getErrorMessage(), 444 500)

445

446 - def _handleInputErrors(self, failure, ctx):

447 queryStatus = "ERROR" 448 449 if isinstance(failure.value, base.EmptyData): 450 inevow.IRequest(ctx).setResponseCode(400) 451 queryStatus = "EMPTY" 452 453 return self._writeErrorTable(ctx, "UsageFault: "+failure.getErrorMessage(), 454 queryStatus=queryStatus)

455

456 457 -class SSAPRenderer(UnifiedDALRenderer):

458 """A renderer for the simple spectral access protocol. 459 460 For registration, you must set the following metadata 461 for the ssap.xml renderer: 462 463 - ssap.dataSource -- survey, pointed, custom, theory, artificial 464 - ssap.testQuery -- a query string that returns some data; REQUEST=queryData 465 is added automatically 466 467 Other SSA metadata includes: 468 469 - ssap.creationType -- archival, cutout, filtered, mosaic, 470 projection, spectralExtraction, catalogExtraction (defaults to archival) 471 - ssap.complianceLevel -- set to "query" when you don't deliver 472 SDM compliant spectra; otherwise don't say anything, DaCHS will fill 473 in the right value. 474 475 It is recommended to set this metadata globally on the RD, as the 476 SSA mixin can use that metadata to fill tables with sensible values 477 without operator intervention. 478 479 Properties supported by this renderer: 480 481 - datalink -- if present, this must be the id of a datalink service 482 that can work with the pubDIDs in this table (don't use this any more, 483 datalink is handled through table-level metadata now) 484 - defaultRequest -- by default, requests without a REQUEST parameter 485 will be rejected. If you set defaultRequest to querydata, such 486 requests will be processed as if REQUEST were given (which is of 487 course sane but is a violation of the standard). 488 """ 489 version = "1.04" 490 name = "ssap.xml" 491 parameterStyle = "pql" 492 standardId = "ivo://ivoa.net/std/ssap" 493

494 - def __init__(self, ctx, *args, **kwargs):

495 reqArgs = inevow.IRequest(ctx).args 496 if "RESPONSEFORMAT" not in reqArgs: 497 reqArgs["RESPONSEFORMAT"] = ["votabletd"] 498 UnifiedDALRenderer.__init__(self, ctx, *args, **kwargs)

499

500 - def _getMetadataData(self, queryMeta):

501 data = UnifiedDALRenderer._getMetadataData(self, queryMeta) 502 data.dd.addMeta( 503 "info", "SSAP", infoName="SERVICE_PROTOCOL", infoValue="1.04") 504 return data

505

506 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

507 # for SSA, we need some funny attributes on the root resource 508 data.original.setMeta("_type", "results") 509 data.original.addMeta("_votableRootAttributes", 510 'xmlns:ssa="http://www.ivoa.net/xml/DalSsap/v1.0"') 511 data.original.addMeta("info", "SSAP", 512 infoName="SERVICE_PROTOCOL", 513 infoValue=self.version) 514 515 # In SSA, we want a "direct" SODA block if we have a dlget-capable 516 # datalink service. 517 sodaGenerators = [] 518 for table in data.original: 519 if not table.rows: 520 continue 521 for svcMeta in table.iterMeta("_associatedDatalinkService"): 522 dlService = base.resolveId(table.tableDef.rd, 523 base.getMetaText(svcMeta, "serviceId")) 524 if "dlget" not in dlService.allowed: 525 continue 526 if utils.looksLikeURLPat.match(table.rows[0]["accref"]): 527 # we need the product table for direct processing 528 # (if we ever want this, we'd have to discover the 529 # descriptor class from the datalink service) 530 continue 531 532 # endpoint 0 hopefully is the sync service. TODO: make this a bit 533 # more robust. 534 core = ssap.getDatalinkCore(dlService, table) 535 sodaGenerators.append( 536 lambda ctx, core=core, svcMeta=svcMeta, table=table: 537 core.datalinkEndpoints[0].asVOT( 538 ctx, 539 dlService.getURL(core.datalinkEndpoints[0].rendName), 540 linkIdTo=ctx.getOrMakeIdFor( 541 table.tableDef.getByName( 542 base.getMetaText(svcMeta, "idColumn"))))) 543 if sodaGenerators: 544 data.original.sodaGenerators = sodaGenerators 545 546 return UnifiedDALRenderer._formatOutput(self, 547 data, ctx, queryMeta, 548 stream=stream)

549

550 551 -class SLAPRenderer(UnifiedDALRenderer):

552 """A renderer for the simple line access protocol SLAP. 553 554 For registration, you must set the following metadata on services 555 using the slap.xml renderer: 556 557 There's two mandatory metadata items for these: 558 559 - slap.dataSource -- one of observational/astrophysical, 560 observational/laboratory, or theoretical 561 - slap.testQuery -- parameters that lead to a non-empty response. 562 The way things are written in DaCHS, MAXREC=1 should in general 563 work. 564 """ 565 version = "1.0" 566 name = "slap.xml" 567 parameterStyle = "pql" 568 standardId = "ivo://ivoa.net/std/ssap" 569

570 - def _formatOutput(self, data, ctx, queryMeta, stream=True):

571 data.original.addMeta("_votableRootAttributes", 572 'xmlns:ssldm="http://www.ivoa.net/xml/SimpleSpectrumLineDM' 573 '/SimpleSpectrumLineDM-v1.0.xsd"') 574 return UnifiedDALRenderer._formatOutput(self, 575 data, ctx, queryMeta, 576 stream=stream)

577

578 579 -class APIRenderer(UnifiedDALRenderer):

580 """A renderer that works like a VO standard renderer but that doesn't 581 actually follow a given protocol. 582 583 Use this for improvised APIs. The default output format is a VOTable, 584 and the errors come in VOSI VOTables. The renderer does, however, 585 evaluate basic DALI parameters. You can declare that by 586 including <FEED source="//pql#DALIPars"/> in your service. 587 588 These will return basic serice metadata if passed MAXREC=0. 589 """ 590 name = "api" 591 parameterStyle = "dali"

592

593 594 -class RegistryRenderer(grend.ServiceBasedPage):

595 """A renderer that works with registry.oaiinter to provide an OAI-PMH 596 interface. 597 598 The core is expected to return a stanxml tree. 599 """ 600 name = "pubreg.xml" 601 urlUse = "base" 602 resultType = "text/xml" 603

604 - def renderHTTP(self, ctx):

605 # Make a robust (unchecked) pars dict for error rendering; real 606 # parameter checking happens in getPMHResponse 607 inData = {"args": [inevow.IRequest(ctx).args]} 608 return self.runService(inData, 609 queryMeta=svcs.QueryMeta.fromContext(ctx) 610 ).addCallback(self._renderResponse, ctx 611 ).addErrback(self._renderError, ctx, inData["args"][0])

612

613 - def _renderResponse(self, svcResult, ctx):

614 return self._renderXML(svcResult.original, ctx)

615

616 - def _renderXML(self, stanxml, ctx):

617 # XXX TODO: this can be pretty large -- do we want async operation 618 # here? Stream this? 619 request = inevow.IRequest(ctx) 620 request.setHeader("content-type", "text/xml") 621 return utils.xmlrender(stanxml, 622 "<?xml-stylesheet href='/static/xsl/oai.xsl' type='text/xsl'?>")

623

624 - def _getErrorTree(self, exception, pars):

625 """returns an ElementTree containing an OAI-PMH error response. 626 627 If exception is one of "our" exceptions, we translate them to error messages. 628 Otherwise, we reraise the exception to an enclosing 629 function may "handle" it. 630 631 Contrary to the recommendation in the OAI-PMH spec, this will only 632 return one error at a time. 633 """ 634 from gavo.registry.model import OAI 635 636 if isinstance(exception, registry.OAIError): 637 code = exception.__class__.__name__ 638 code = code[0].lower()+code[1:] 639 message = str(exception) 640 else: 641 code = "badArgument" # Why the hell don't they have a serverError? 642 message = "Internal Error: "+str(exception) 643 return OAI.PMH[ 644 OAI.responseDate[datetime.datetime.utcnow().strftime( 645 utils.isoTimestampFmt)], 646 OAI.request(metadataPrefix=pars.get("metadataPrefix", [None])[0]), 647 OAI.error(code=code)[ 648 message 649 ] 650 ]

651

652 - def _renderError(self, failure, ctx, pars):

653 try: 654 if not isinstance(failure.value, 655 (registry.OAIError, base.ValidationError)): 656 base.ui.notifyFailure(failure) 657 return self._renderXML(self._getErrorTree(failure.value, pars), 658 ctx) 659 except: 660 base.ui.notifyError("Cannot create registry error document") 661 request = inevow.IRequest(ctx) 662 request.setResponseCode(400) 663 request.setHeader("content-type", "text/plain") 664 request.write("Internal error. Please notify site maintainer") 665 request.finishRequest(False) 666 return appserver.errorMarker

667

668 669 -def addAttachmentHeaders(request, mime=None):

670 """adds a content-disposition header to request with a filename guessed 671 based on datalink arguments. 672 673 This tries a number of heuristics to try and preserve a bit of the 674 provenance in the name, mainly to make saving these things simpler. 675 """ 676 try: 677 if mime is None: 678 mime = request.getHeader('content-type') 679 if mime is None: 680 # this is an error, really. This should only be called when 681 # request is ready for serving. But let's be generous. 682 mime = "application/octet-stream" 683 ext = formats.getExtensionFor(mime) 684 685 basicId = request.args["ID"][0] 686 if '?' in basicId: 687 # presumably the query part is the path 688 stem = os.path.splitext( 689 basicId.split('?')[-1].replace("/", "_"))[0] 690 else: 691 # who knows? Whatever comes after the last slash is 692 # probably a better bet than anything else 693 stem = basicId.split("/")[-1] 694 695 if len(stem)<3: # this cannot be right 696 stem = "result"+stem 697 698 # if there's any arguments we suspect have changed the contents 699 argkeys = [k for k in 700 set(request.args.keys()) - set(["ID", "RESPONSEFORMAT"]) 701 if request.args[k]] 702 if argkeys: 703 stem = stem+"_proc" 704 705 request.setHeader('content-disposition', 706 'attachment; filename=%s%s'%(stem, ext)) 707 except: 708 # if all fails, use a safe, if dumb, fallback 709 request.setHeader('content-disposition', 710 'attachment; filename=result.dat')

711

712 713 -class _DatalinkRendererBase(grend.ServiceBasedPage):

714 """the base class of the two datalink sync renderers. 715 """ 716 urlUse = "base" 717 718 # send out files as attachments with separate file names? 719 attachResult = False 720

721 - def renderHTTP(self, ctx):

722 request = inevow.IRequest(ctx) 723 return self.runService(request.args, svcs.QueryMeta.fromContext(ctx) 724 ).addCallback(self._formatData, request 725 ).addErrback(self._reportError, request)

726

727 - def _formatData(self, svcResult, request):

728 # the core returns mime, data or a resource. So, if it's a pair, 729 # to something myself, else let twisted sort it out 730 data = svcResult.original 731 732 if isinstance(data, tuple): 733 # XXX TODO: the same thing is in formrender. Refactor; since this is 734 # something most renderers should be able to do, ServiceBasedPage would be 735 # a good place 736 mime, payload = data 737 request.setHeader("content-type", mime) 738 739 if self.attachResult: 740 addAttachmentHeaders(request, mime) 741 742 return streaming.streamOut(lambda f: f.write(payload), 743 request) 744 745 else: 746 if self.attachResult: 747 # the following getattr is for when data is a nevow.static.File 748 addAttachmentHeaders(request, getattr(data, "type", None)) 749 750 return data

751 752 failureNameMap = { 753 'ValidationError': 'UsageError', 754 'MultiplicityError': 'MultiValuedParamNotSupported', 755 } 756

757 - def _reportError(self, failure, request):

758 # Do not trap svcs.WebRedirect here! 759 failure.trap(base.ValidationError, 760 base.ExecutiveAction) 761 762 request.setHeader("content-type", "text/plain") 763 764 if hasattr(failure.value, "responseCode"): 765 request.setResponseCode(failure.value.responseCode) 766 else: 767 request.setResponseCode(422) 768 769 if hasattr(failure.value, "responsePayload"): 770 return failure.value.responsePayload 771 else: 772 return "%s: %s\n"%( 773 self.failureNameMap.get(failure.value.__class__.__name__, "Error"), 774 utils.safe_str(failure.value))

775

776 777 -def _doDatalinkXSLT(bytes, _cache={}):

778 """a temporary hack to do server-side XSLT while the browser implementations 779 apparently suck. 780 781 Remove this once we've worked out how to make the datalink-to-xml 782 stylesheet compatible with actual browers. 783 """ 784 if "etree" not in _cache: 785 from lxml import etree as lxmletree 786 _cache["etree"] = lxmletree 787 if "style" not in _cache: 788 with base.openDistFile("web/xsl/datalink-to-html.xsl") as f: 789 _cache["style"] = _cache["etree"].XSLT( 790 _cache["etree"].XML(f.read())) 791 return str(_cache["style"](_cache["etree"].XML(bytes)))

792

793 794 -class DatalinkGetDataRenderer(_DatalinkRendererBase):

795 """A renderer for data processing by datalink cores. 796 797 This must go together with a datalink core, nothing else will do. 798 799 This renderer will actually produce the processed data. It must be 800 complemented by the dlmeta renderer which allows retrieving metadata. 801 """ 802 name = "dlget" 803 attachResult = True 804 standardId = "ivo://ivoa.net/std/soda#sync-1.0"

805 # This shouldn't have parameterStyle for now, as it would

806 # add DALI parameters (MAXREC etc) which are probably inappropriate 807 # here. 808 809 810 -class DatalinkGetMetaRenderer(_DatalinkRendererBase):

811 """A renderer for data processing by datalink cores. 812 813 This must go together with a datalink core, nothing else will do. 814 815 This renderer will return the links and services applicable to 816 one or more pubDIDs. 817 818 See `Datalink and SODA`_ for more information. 819 """ 820 name = "dlmeta" 821 resultType = "application/x-votable+xml;content=datalink" 822 standardId = "ivo://ivoa.net/std/datalink" 823 parameterStyle = "dali" 824

825 - def _formatData(self, svcResult, request):

826 # this is a (hopefully temporary) hack that does XSLT server-side 827 # if we think we're talking to a browser. The reason I'm doing 828 # this is that several browsers were confused when doing both 829 # XSLT and non-trivial javascript. 830 # 831 # remove this method once we've figured out how to placate these browsers. 832 mime, data = svcResult.original 833 if "Mozilla" in (request.getHeader("user-agent") or ""): 834 # it's a browser, do server-side XSLT 835 request.setHeader("content-type", "text/html;charset=utf-8") 836 return _doDatalinkXSLT(data) 837 838 else: 839 # no browser, do the right thing 840 request.setHeader("content-type", mime) 841 return data

842

843 844 -class AsyncRendererBase(grend.ServiceBasedPage):

845 """An abstract renderer for things running in a UWS. 846 847 To make these concrete, they need a name and a workerSystem attribute. 848 """ 849 parameterStyle = "pql" 850

851 - def renderHTTP(self, ctx):

852 return self.locateChild(ctx, ())[0]

853

854 - def locateChild(self, ctx, segments):

855 from gavo.protocols import uwsactions 856 from gavo.web import asyncrender 857 858 # no trailing slashes here, ever (there probably should be central 859 # code for this somewhere, as this is done in taprender and 860 # possibly in other places, too) 861 if segments and not segments[-1]: # trailing slashes are forbidden here 862 newSegments = "/".join(segments[:-1]) 863 if newSegments: 864 newSegments = "/"+newSegments 865 raise svcs.WebRedirect(self.service.getURL(self.name)+newSegments) 866 867 uwsactions.lowercaseProtocolArgs(inevow.IRequest(ctx).args) 868 return asyncrender.getAsyncResource(ctx, 869 self.workerSystem, 870 self.name, 871 self.service, 872 segments), ()

873

874 875 -class DatalinkAsyncRenderer(AsyncRendererBase):

876 """A renderer for asynchronous datalink. 877 """ 878 # TODO: I suspect this should go somewhere else, presumably together 879 # with the stripped-down TAP renderer. 880 name = "dlasync" 881 workerSystem = dlasync.DL_WORKER

882

883 884 -class UWSAsyncRenderer(AsyncRendererBase):

885 """A renderer speaking UWS. 886 887 This is for asynchronous exection of larger jobs. Operators will normally 888 use this together with a custom core or a python core. 889 890 See `Custom UWSes`_ for details. 891 """ 892 name = "uws.xml" 893 894 @property

895 - def workerSystem(self):

896 return self.service.getUWS()

897

898 899 -def _test():

900 import doctest, vodal 901 doctest.testmod(vodal)

902 903 904 if __name__=="__main__": 905 _test() 906

Source Code for Module gavo.web.vodal