gavo.protocols.datalink

1 """ 2 The datalink core and its numerous helper classes. 3 4 More on this in "Datalink Cores" in the reference documentation. 5 """ 6 7 #c Copyright 2008-2019, the GAVO project 8 #c 9 #c This program is free software, covered by the GNU GPL. See the 10 #c COPYING file in the source distribution. 11 12 13 import itertools 14 import inspect 15 import os 16 import urllib 17 18 from gavo import base 19 from gavo import formats 20 from gavo import rsc 21 from gavo import rscdef 22 from gavo import svcs 23 from gavo import utils 24 from gavo.protocols import products 25 from gavo.protocols import soda 26 from gavo.protocols.soda import (FormatNow, DeliverNow, DatalinkFault, 27 DEFAULT_SEMANTICS) 28 from gavo.formats import votablewrite 29 from gavo.utils import fitstools 30 from gavo.utils import pyfits 31 from gavo.votable import V, modelgroups 32 33 from nevow import inevow 34 from nevow import rend 35 from nevow import static 36 37 from twisted.internet import defer 38 39 40 MS = base.makeStruct

41 42 43 -class ProductDescriptor(object):

44 """An encapsulation of information about some "product" (i.e., file). 45 46 This is basically equivalent to a line in the product table; the 47 arguments of the constructor are all available as same-named attributes. 48 49 It also has an attribute data defaulting to None. DataGenerators 50 set it, DataFilters potentially change it. 51 52 If you inherit from this method and you have a way to guess the 53 size of what the descriptor describes, override the estimateSize() 54 method. The default will return a file size if accessPath points 55 to an existing file, None otherwise. 56 """ 57 data = None 58

59 - def __init__(self, pubDID, accref, accessPath, mime, 60 owner=None, embargo=None, sourceTable=None, datalink=None, 61 preview=None, preview_mime=None):

62 self.pubDID = pubDID 63 self.accref, self.accessPath, self.mime = accref, accessPath, mime 64 self.owner, self.embargo, self.sourceTable = owner, embargo, sourceTable 65 self.preview, self.previewMime = preview, preview_mime

66 67 @classmethod

68 - def fromAccref(cls, pubDID, accref, accrefPrefix=None, **moreKWs):

69 """returns a product descriptor for an access reference. 70 71 If an accrefPrefix is passed in, an AuthenticationFault (for want 72 of something better fitting) is returned when the accref doesn't 73 start with accrefPrefix. 74 """ 75 if accrefPrefix and not accref.startswith(accrefPrefix): 76 return DatalinkFault.AuthenticationFault(pubDID, 77 "This Datalink service not available" 78 " with this pubDID", semantics="#this") 79 80 kwargs = products.RAccref(accref).productsRow 81 kwargs.update(moreKWs) 82 return cls(pubDID, **kwargs)

83

84 - def estimateSize(self):

85 if isinstance(self.accessPath, basestring): 86 candPath = os.path.join(base.getConfig("inputsDir"), self.accessPath) 87 try: 88 return os.path.getsize(candPath) 89 except: 90 # fall through to returning None 91 pass

92

93 - def makeLink(self, url, **kwargs):

94 """returns a LinkDef for this descriptor for url. 95 96 kwargs are passed on to LinkDef and include, in particular, 97 semantics, contentType, contentLength, and description. 98 """ 99 return LinkDef(self.pubDID, url, **kwargs)

100

101 - def makeLinkFromFile(self, localPath, description, semantics, 102 service=None, contentType=None, suppressMissing=False):

103 """returns a LinkDef for a local file. 104 105 Arguments are as for LinkDef.fromFile, except you don't have 106 to pass in service if you're using the datalink service itself 107 to access the file; this method will try to find the service by 108 itself. 109 """ 110 if service is None: 111 try: 112 service = inspect.currentframe().f_back.f_locals["self"].parent 113 except (KeyError, AttributeError): 114 raise base.StructureError("Cannot infer service for datalink" 115 " file link. Pass an appropriate service manually.") 116 return LinkDef.fromFile(localPath, description, semantics, 117 service=service, contentType=None, suppressMissing=suppressMissing)

118

119 120 -class FITSProductDescriptor(ProductDescriptor):

121 """A SODA descriptor for FITS files. 122 123 On top of the normal product descriptor, this has an attribute hdr 124 containing a copy of the image header, and a method 125 changingAxis (see there). 126 127 There's also an attribute dataIsPristine that must be set to false 128 if changes have been made. The formatter will spit out the original 129 data otherwise, ignoring your changes. 130 131 Finally, there's a slices attribute provided explained in 132 soda#fits_doWCSCutout that can be used by data functions running before 133 it to do cutouts. 134 135 The FITSProductDescriptor is constructed like a normal ProductDescriptor. 136 """ 137

138 - def __init__(self, *args, **kwargs):

139 qnd = kwargs.pop("qnd", True) 140 ProductDescriptor.__init__(self, *args, **kwargs) 141 142 self.imageExtind = 0 143 144 if qnd and self.imageExtind==0: 145 with open(os.path.join( 146 base.getConfig("inputsDir"), self.accessPath)) as f: 147 self.hdr = utils.readPrimaryHeaderQuick(f, 148 maxHeaderBlocks=100) 149 150 else: 151 hdus = pyfits.open(os.path.join( 152 base.getConfig("inputsDir"), self.accessPath)) 153 self.imageExtind = fitstools.fixImageExtind( 154 hdus, self.imageExtind) 155 self.hdr = hdus[self.imageExtind].header 156 hdus.close() 157 158 self.slices = [] 159 self.dataIsPristine = True 160 self._axesTouched = set()

161

162 - def changingAxis(self, axisIndex, parName):

163 """must be called before cutting out along axisIndex. 164 165 axIndex is a FITS (1-based) axis index axIndex, parName the name of the 166 parameter that causes the cutout. 167 168 This will simply return if nobody has called changingAxis with that index 169 before and raise a ValidationError otherwise. Data functions doing a cutout 170 must call this before doing so; if they don't the cutout will probably be 171 wrong when two conflicting constraints are given. 172 """ 173 if axisIndex in self._axesTouched: 174 raise base.ValidationError("Attempt to cut out along axis %d that" 175 " has been modified before."%axisIndex, parName) 176 self._axesTouched.add(axisIndex)

177

178 179 -class DLFITSProductDescriptor(FITSProductDescriptor):

180 """A SODA descriptor for FITS files with datalink product paths. 181 182 Use is as descClass in //soda#fits_genDesc when the product table 183 has a datalink as the product. 184 """

185 - def __init__(self, *args, **kwargs):

186 kwargs["accessPath"] = os.path.join( 187 base.getConfig("inputsDir"), 188 kwargs["accref"]) 189 FITSProductDescriptor.__init__(self, *args, **kwargs)

190

191 192 -def getFITSDescriptor(pubDID, accrefPrefix=None, 193 cls=FITSProductDescriptor, qnd=False):

194 """returns a datalink descriptor for a FITS file. 195 196 This is the implementation of fits_genDesc and should probably reused 197 when making more specialised descriptors. 198 """ 199 try: 200 accref = rscdef.getAccrefFromStandardPubDID(pubDID) 201 except ValueError: 202 return DatalinkFault.NotFoundFault(pubDID, 203 "Not a pubDID from this site.") 204 205 return cls.fromAccref(pubDID, accref, accrefPrefix, qnd=qnd)

206

207 208 -class _File(static.File):

209 """A nevow static.File with a pre-determined type. 210 """

211 - def __init__(self, path, mediaType):

212 static.File.__init__(self, path) 213 self.type = mediaType 214 self.encoding = None

215

216 217 -class _TemporaryFile(_File):

218 """A nevow resource that spits out a file and then deletes it. 219 220 This is a helper class for DataFunctions and DataFormatters, available 221 there as TemporaryFile. 222 """

223 - def renderHTTP(self, ctx):

224 return defer.maybeDeferred(_File.renderHTTP, self, ctx).addBoth( 225 self._cleanup)

226

227 - def _cleanup(self, result):

228 self.fp.remove() 229 return result

230

231 232 -class DescriptorGenerator(rscdef.ProcApp):

233 """A procedure application for making product descriptors for PUBDIDs 234 235 Despite the name, a descriptor generator has to *return* (not yield) 236 a descriptor instance. While this could be anything, it is recommended 237 to derive custom classes from prodocols.datalink.ProductDescrpitor, which 238 exposes essentially the columns from DaCHS' product table as attributes. 239 This is what you get when you don't define a descriptor generator 240 in your datalink core. 241 242 The following names are available to the code: 243 244 - pubDID -- the pubDID to be resolved 245 - args -- all the arguments that came in from the web 246 (these should not ususally be necessary for making the descriptor 247 and are completely unparsed at this point) 248 - FITSProductDescriptor -- the base class of FITS product descriptors 249 - DLFITSProductDescriptor -- the same, just for when the product table 250 has a datalink. 251 - ProductDescriptor -- a base class for your own custom descriptors 252 - DatalinkFault -- use this when flagging failures 253 - soda -- contents of the soda module for convenience 254 255 If you made your pubDID using the ``getStandardPubDID`` rowmaker function, 256 and you need no additional logic within the descriptor, 257 the default (//soda#fromStandardPubDID) should do. 258 259 If you need to derive custom descriptor classes, you can see the base 260 class under the name ProductDescriptor; there's also 261 FITSProductDescriptor and DatalinkFault in each proc's namespace. 262 If your Descriptor does not actually refer to something in the 263 product table, it is likely that you want to set the descriptor's 264 ``suppressAutoLinks`` attribute to True. This will stop DaCHS 265 from attempting to add automatic #this and #preview links. 266 """ 267 name_ = "descriptorGenerator" 268 requiredType = "descriptorGenerator" 269 formalArgs = "pubDID, args" 270 271 additionalNamesForProcs = { 272 "FITSProductDescriptor": FITSProductDescriptor, 273 "DLFITSProductDescriptor": DLFITSProductDescriptor, 274 "ProductDescriptor": ProductDescriptor, 275 "getFITSDescriptor": getFITSDescriptor, 276 "DatalinkFault": DatalinkFault, 277 "soda": soda, 278 }

279

280 281 -class LinkDef(object):

282 """A definition of a datalink related document. 283 284 These are constructed at least with: 285 286 - the pubDID (as a string) 287 - the access URL (as a string) 288 289 In addition, we accept the remaining column names from 290 //datalink#dlresponse as keyword arguments. 291 292 In particular, do set semantics with a term from 293 http://www.ivoa.net/rdf/datalink/core. This includes #this, #preview, 294 #calibration, #progenitor, #derivation 295 """

296 - def __init__(self, pubDID, accessURL, 297 serviceType=None, 298 errorMessage=None, 299 description=None, 300 semantics=DEFAULT_SEMANTICS, 301 contentType=None, 302 contentLength=None):

303 ID = pubDID #noflake: used in locals() 304 del pubDID 305 self.dlRow = locals()

306 307 @classmethod

308 - def fromFile(cls, localPath, description, semantics, 309 service, contentType=None, suppressMissing=False):

310 """constructs a LinkDef based on a local file. 311 312 You must give localPath (which may be resdir-relative), description and 313 semantics are mandatory. ContentType and contentSize will normally be 314 determined by DaCHS. 315 316 You must also pass in the service used to retrieve the file. This 317 must allow the static renderer and have a staticData property. It should 318 normally be the datalink service itself, which in a metaMaker 319 is accessible as self.parent.parent. It is, however, legal 320 to reference other suitable services (use self.parent.rd.getById or 321 base.resolveCrossId) 322 323 If you pass suppressMissing=True, a link to a non-existing file 324 will be skipped rather than create a missing datalink. 325 """ 326 baseDir = service.rd.resdir 327 localPath = os.path.join(baseDir, localPath) 328 pubDID = utils.stealVar("descriptor").pubDID 329 staticPath = os.path.join(baseDir, 330 service.getProperty("staticData")) 331 332 if not os.path.isfile(localPath): 333 if suppressMissing: 334 return None 335 else: 336 return DatalinkFault.NotFoundFault(pubDID, "No file" 337 " for linked item", semantics=semantics, description=description) 338 elif not os.access(localPath, os.R_OK): 339 return DatalinkFault.AutorizationFault(pubDID, "Linked" 340 " item not readable", semantics=semantics, description=description) 341 342 try: 343 svcPath = utils.getRelativePath(localPath, staticPath) 344 except ValueError: 345 return LinkDef(pubDID, errorMessage="FatalFault: Linked item" 346 " not accessible through the given service", 347 semantics=semantics, description=description) 348 349 ext = os.path.splitext(localPath)[-1] 350 contentType = (contentType 351 or static.File.contentTypes.get(ext, "application/octet-stream")) 352 353 return cls(pubDID, 354 service.getURL("static")+"/"+svcPath, 355 description=description, semantics=semantics, 356 contentType=contentType, 357 contentLength=os.path.getsize(localPath))

358

359 - def asDict(self):

360 """returns the link definition in a form suitable for ingestion 361 in //datalink#dlresponse. 362 """ 363 return { 364 "ID": self.dlRow["ID"], 365 "access_url": self.dlRow["accessURL"], 366 "service_def": self.dlRow["serviceType"], 367 "error_message": self.dlRow["errorMessage"], 368 "description": self.dlRow["description"], 369 "semantics": self.dlRow["semantics"], 370 "content_type": self.dlRow["contentType"], 371 "content_length": self.dlRow["contentLength"]}

372

373 374 -class _ServiceDescriptor(object):

375 """An internal descriptor for one of our services. 376 377 These are serialized into service resources in VOTables. 378 Basically, these collect input keys, a pubDID, as well as any other 379 data we might need in service definition. 380 """

381 - def __init__(self, pubDID, inputKeys, rendName, description):

382 self.pubDID, self.inputKeys = pubDID, inputKeys 383 self.rendName, self.description = rendName, description 384 if self.pubDID: 385 # if we're fixed to a specific pubDID, reflect that in the ID 386 # field -- this is how clients know which dataset to pull 387 # from datalink documents. 388 for index, ik in enumerate(self.inputKeys): 389 if ik.name=="ID": 390 ik = ik.copy(None) 391 ik.set(pubDID) 392 self.inputKeys[index] = ik

393

394 - def asVOT(self, ctx, accessURL, linkIdTo=None):

395 """returns VOTable stanxml for a description of this service. 396 397 This is a RESOURCE as required by Datalink. 398 399 linkIdTo is used to support data access descriptors embedded 400 in descovery queries. It is the id of the column containing 401 the identifiers. SSA can already provide this. 402 """ 403 paramsByName, stcSpecs = {}, set() 404 for param in self.inputKeys: 405 paramsByName[param.name] = param 406 if param.stc: 407 stcSpecs.add(param.stc) 408 409 def getIdFor(colRef): 410 colRef.toParam = True 411 return ctx.getOrMakeIdFor(paramsByName[colRef.dest], 412 suggestion=colRef.dest)

413 414 res = V.RESOURCE(ID=ctx.getOrMakeIdFor(self, suggestion="proc_svc"), 415 name="proc_svc", type="meta", utype="adhoc:service")[ 416 V.DESCRIPTION[self.description], 417 [modelgroups.marshal_STC(ast, getIdFor)[0] 418 for ast in stcSpecs], 419 V.PARAM(arraysize="*", datatype="char", 420 name="accessURL", ucd="meta.ref.url", 421 value=accessURL)] 422 423 standardId = { 424 "dlasync": "ivo://ivoa.net/std/SODA#async-1.0", 425 "dlget": "ivo://ivoa.net/std/SODA#sync-1.0"}.get(self.rendName) 426 if standardId: 427 res[ 428 V.PARAM(arraysize="*", datatype="char", 429 name="standardID", value=standardId)] 430 431 inputParams = V.GROUP(name="inputParams") 432 res = res[inputParams] 433 434 for ik in self.inputKeys: 435 param = ctx.addID(ik, 436 votablewrite.makeFieldFromColumn(ctx, V.PARAM, ik)) 437 if linkIdTo and ik.name=="ID": 438 param = param(ref=linkIdTo) 439 inputParams[param] 440 441 return res

442

443 444 -class MetaMaker(rscdef.ProcApp):

445 """A procedure application that generates metadata for datalink services. 446 447 The code must be generators (i.e., use yield statements) producing either 448 svcs.InputKeys or protocols.datalink.LinkDef instances. 449 450 metaMaker see the data descriptor of the input data under the name 451 descriptor. 452 453 The data attribute of the descriptor is always None for metaMakers, so 454 you cannot use anything given there. 455 456 Within MetaMakers' code, you can access InputKey, Values, Option, and 457 LinkDef without qualification, and there's the MS function to build 458 structures. Hence, a metaMaker returning an InputKey could look like this:: 459 460 <metaMaker> 461 <code> 462 yield MS(InputKey, name="format", type="text", 463 description="Output format desired", 464 values=MS(Values, 465 options=[MS(Option, content_=descriptor.mime), 466 MS(Option, content_="text/plain")])) 467 </code> 468 </metaMaker> 469 470 (of course, you should give more metadata -- ucds, better description, 471 etc) in production). 472 473 It's ok to yield None; this will suppress a Datalink and is convenient 474 when some component further down figures out that a link doesn't exist 475 (e.g., because a file isn't there). Note that in many cases, it's 476 more helpful to client components to handle such situations by 477 yielding a DatalinkFault.NotFoundFault. 478 479 In addition to the usual names available to ProcApps, meta makers have: 480 - MS -- function to make DaCHS structures 481 - InputKey -- the class to make for input parameters 482 - Values -- the class to make for input parameters' values attributes 483 - Options -- used by Values 484 - LinkDef -- a class to define further links within datalink services. 485 - DatalinkFault -- a container of datalink error generators 486 - soda -- the soda module. 487 """ 488 name_ = "metaMaker" 489 requiredType = "metaMaker" 490 formalArgs = "self, descriptor" 491 492 additionalNamesForProcs = { 493 "MS": base.makeStruct, 494 "InputKey": svcs.InputKey, 495 "Values": rscdef.Values, 496 "Option": rscdef.Option, 497 "LinkDef": LinkDef, 498 "DatalinkFault": DatalinkFault, 499 "soda": soda, 500 }

501

502 503 -class DataFunction(rscdef.ProcApp):

504 """A procedure application that generates or modifies data in a processed 505 data service. 506 507 All these operate on the data attribute of the product descriptor. 508 The first data function plays a special role: It *must* set the data 509 attribute (or raise some appropriate exception), or a server error will 510 be returned to the client. 511 512 What is returned depends on the service, but typcially it's going to 513 be a table or products.*Product instance. 514 515 Data functions can shortcut if it's evident that further data functions 516 can only mess up (i.e., if the do something bad with the data attribute); 517 you should not shortcut if you just *think* it makes no sense to 518 further process your output. 519 520 To shortcut, raise either of FormatNow (falls though to the formatter, 521 which is usually less useful) or DeliverNow (directly returns the 522 data attribute; this can be used to return arbitrary chunks of data). 523 524 The following names are available to the code: 525 - descriptor -- whatever the DescriptorGenerator returned 526 - args -- all the arguments that came in from the web. 527 528 In addition to the usual names available to ProcApps, data functions have: 529 - FormatNow -- exception to raise to go directly to the formatter 530 - DeliverNow -- exception to raise to skip all further formatting 531 and just deliver what's currently in descriptor.data 532 - File(path, type) -- if you just want to return a file on disk, pass 533 its path and media type to File and assign the result to 534 descriptor.data. 535 - TemporaryFile(path,type) -- as File, but the disk file is 536 unlinked after use 537 - makeData -- the rsc.makeData function 538 - soda -- the protocols.soda module 539 """ 540 name_ = "dataFunction" 541 requiredType = "dataFunction" 542 formalArgs = "descriptor, args" 543 544 additionalNamesForProcs = { 545 "FormatNow": FormatNow, 546 "DeliverNow": DeliverNow, 547 "File": _File, 548 "TemporaryFile": _TemporaryFile, 549 "makeData": rsc.makeData, 550 "soda": soda, 551 }

552

553 554 -class DataFormatter(rscdef.ProcApp):

555 """A procedure application that renders data in a processed service. 556 557 These play the role of the renderer, which for datalink is ususally 558 trivial. They are supposed to take descriptor.data and return 559 a pair of (mime-type, bytes), which is understood by most renderers. 560 561 When no dataFormatter is given for a core, it will return descriptor.data 562 directly. This can work with the datalink renderer itself if 563 descriptor.data will work as a nevow resource (i.e., has a renderHTTP 564 method, as our usual products do). Consider, though, that renderHTTP 565 runs in the main event loop and thus most not block for extended 566 periods of time. 567 568 The following names are available to the code: 569 - descriptor -- whatever the DescriptorGenerator returned 570 - args -- all the arguments that came in from the web. 571 572 In addition to the usual names available to ProcApps, data formatters have: 573 - Page -- base class for resources with renderHTTP methods. 574 - IRequest -- the nevow interface to make Request objects with. 575 - File(path, type) -- if you just want to return a file on disk, pass 576 its path and media type to File and return the result. 577 - TemporaryFile(path, type) -- as File, but the disk file is unlinked 578 after use 579 - soda -- the protocols.soda module 580 """ 581 name_ = "dataFormatter" 582 requiredType = "dataFormatter" 583 formalArgs = "descriptor, args" 584 585 additionalNamesForProcs = { 586 "Page": rend.Page, 587 "IRequest": inevow.IRequest, 588 "File": _File, 589 "TemporaryFile": _TemporaryFile, 590 "soda": soda, 591 }

592

593 594 -class DatalinkCoreBase(svcs.Core, base.ExpansionDelegator):

595 """Basic functionality for datalink cores. 596 597 This is pulled out of the datalink core proper as it is used without 598 the complicated service interface sometimes, e.g., by SSAP. 599 """ 600 601 _descriptorGenerator = base.StructAttribute("descriptorGenerator", 602 default=base.NotGiven, 603 childFactory=DescriptorGenerator, 604 description="Code that takes a PUBDID and turns it into a" 605 " product descriptor instance. If not given," 606 " //soda#fromStandardPubDID will be used.", 607 copyable=True) 608 609 _metaMakers = base.StructListAttribute("metaMakers", 610 childFactory=MetaMaker, 611 description="Code that takes a data descriptor and either" 612 " updates input key options or yields related data.", 613 copyable=True) 614 615 _dataFunctions = base.StructListAttribute("dataFunctions", 616 childFactory=DataFunction, 617 description="Code that generates of processes data for this" 618 " core. The first of these plays a special role in that it" 619 " must set descriptor.data, the others need not do anything" 620 " at all.", 621 copyable=True) 622 623 _dataFormatter = base.StructAttribute("dataFormatter", 624 default=base.NotGiven, 625 childFactory=DataFormatter, 626 description="Code that turns descriptor.data into a nevow resource" 627 " or a mime, content pair. If not given, the renderer will be" 628 " returned descriptor.data itself (which will probably not usually" 629 " work).", 630 copyable=True) 631 632 _inputKeys = rscdef.ColumnListAttribute("inputKeys", 633 childFactory=svcs.InputKey, 634 description="A parameter to one of the proc apps (data functions," 635 " formatters) active in this datalink core; no specific relation" 636 " between input keys and procApps is supposed; all procApps are passed" 637 " all argments. Conventionally, you will write the input keys in" 638 " front of the proc apps that interpret them.", 639 copyable=True) 640 641 rejectExtras = True 642

643 - def completeElement(self, ctx):

644 if self.descriptorGenerator is base.NotGiven: 645 self.descriptorGenerator = MS(DescriptorGenerator, 646 procDef=base.resolveCrossId("//soda#fromStandardPubDID")) 647 648 if self.dataFormatter is base.NotGiven: 649 self.dataFormatter = MS(DataFormatter, 650 procDef=base.caches.getRD("//soda").getById("trivialFormatter")) 651 652 self.inputKeys.append(MS(svcs.InputKey, name="ID", type="text", 653 ucd="meta.id;meta.main", 654 multiplicity="multiple", 655 std=True, 656 description="The pubisher DID of the dataset of interest")) 657 658 if self.inputTable is base.NotGiven: 659 self.inputTable = MS(svcs.InputTD, inputKeys=self.inputKeys) 660 661 # this is a cheat for service.getTableSet to pick up the datalink 662 # table. If we fix this for TAP, we should fix it here, too. 663 self.queriedTable = base.caches.getRD("//datalink").getById( 664 "dlresponse") 665 666 self._completeElementNext(DatalinkCoreBase, ctx)

667

668 - def getMetaForDescriptor(self, descriptor):

669 """returns a pair of linkDefs, inputKeys for a datalink desriptor 670 and this core. 671 """ 672 linkDefs, inputKeys, errors = [], self.inputKeys[:], [] 673 674 for metaMaker in self.metaMakers: 675 try: 676 for item in metaMaker.compile(self)(self, descriptor): 677 if isinstance(item, LinkDef): 678 linkDefs.append(item) 679 elif isinstance(item, DatalinkFault): 680 errors.append(item) 681 elif item is None: 682 pass 683 else: 684 inputKeys.append(item) 685 except Exception as ex: 686 if base.DEBUG: 687 base.ui.notifyError("Error in datalink meta generator %s: %s"%( 688 metaMaker, repr(ex))) 689 base.ui.notifyError("Failing source: \n%s"%metaMaker.getFuncCode()) 690 errors.append(DatalinkFault.Fault(descriptor.pubDID, 691 "Unexpected failure while creating" 692 " datalink: %s"%utils.safe_str(ex))) 693 694 return linkDefs, inputKeys, errors

695

696 - def getDatalinksResource(self, ctx, service):

697 """returns a VOTable RESOURCE element with the data links. 698 699 This does not contain the actual service definition elements, but it 700 does contain references to them. 701 702 You must pass in a VOTable context object ctx (for the management 703 of ids). If this is the entire content of the VOTable, use 704 votablewrite.VOTableContext() there. 705 """ 706 internalLinks = [] 707 708 internalLinks.extend( 709 LinkDef( 710 s.pubDID, 711 None, 712 serviceType=ctx.getOrMakeIdFor(s, suggestion="procsvc"), 713 description=s.description, 714 semantics="#proc") 715 for s in self.datalinkEndpoints) 716 717 for d in self.descriptors: 718 # for all descriptors that are products, make a full dataset 719 # available through the data access, possibly also adding a preview. 720 if not isinstance(d, ProductDescriptor): 721 continue 722 if hasattr(d, "suppressAutoLinks"): 723 continue 724 725 # if the accref is a datalink document, go through dlget itself. 726 if d.mime=="application/x-votable+xml;content=datalink": 727 if isinstance(d, DLFITSProductDescriptor): 728 # this is perhaps a bit insane, but I like image/fits 729 # for non-tabular FITS files, and that's what DLFITS 730 # deals with. 731 mediaType = "image/fits" 732 else: 733 mediaType = formats.guessMediaType(d.accref), 734 internalLinks.append(LinkDef(d.pubDID, 735 service.getURL("dlget")+"?ID=%s"%urllib.quote(d.pubDID), 736 description="The full dataset.", 737 contentType=mediaType, 738 contentLength=d.estimateSize(), 739 semantics="#this")) 740 741 else: 742 internalLinks.append(LinkDef(d.pubDID, 743 products.makeProductLink(d.accref), 744 description="The full dataset.", 745 contentType=d.mime, 746 contentLength=d.estimateSize(), 747 semantics="#this")) 748 749 if getattr(d, "preview", None): 750 if d.preview.startswith("http"): 751 # deliver literal links directly 752 previewLink = d.preview 753 754 else: 755 # It's either AUTO or a local path; in both cases, let 756 # the products infrastructure worry about it. 757 previewLink = products.makeProductLink( 758 products.RAccref(d.accref, 759 inputDict={"preview": True})) 760 761 internalLinks.append(LinkDef(d.pubDID, 762 previewLink, description="A preview for the dataset.", 763 contentType=d.previewMime, semantics="#preview")) 764 765 data = rsc.makeData( 766 base.caches.getRD("//datalink").getById("make_response"), 767 forceSource=self.datalinkLinks+internalLinks+self.errors) 768 data.setMeta("_type", "results") 769 770 return votablewrite.makeResource( 771 votablewrite.VOTableContext(tablecoding="td"), 772 data)

773

774 775 -class DatalinkCore(DatalinkCoreBase):

776 """A core for processing datalink and processed data requests. 777 778 The input table of this core is dynamically generated from its 779 metaMakers; it makes no sense at all to try and override it. 780 781 See `Datalink and SODA`_ for more information. 782 783 In contrast to "normal" cores, one of these is made (and destroyed) 784 for each datalink request coming in. This is because the interface 785 of a datalink service depends on the request's value(s) of ID. 786 787 The datalink core can produce both its own metadata and data generated. 788 It is the renderer's job to tell them apart. 789 """ 790 name_ = "datalinkCore" 791 792 datalinkType = "application/x-votable+xml;content=datalink" 793 794 # the core will be specially and non-cacheably adapted for these 795 # renderers (ssap.xml is in here for legacy getData): 796 datalinkAdaptingRenderers = frozenset([ 797 "form", "dlget", "dlmeta", "dlasync", "ssap.xml"]) 798

799 - def _getPubDIDs(self, args):

800 """returns a list of pubDIDs from args["ID"]. 801 802 args is supposed to be a nevow request.args-like dict, where the PubDIDs 803 are taken from the ID parameter. If it's atomic, it'll be expanded into 804 a list. If it's not present, a ValidationError will be raised. 805 """ 806 pubDIDs = args.get("ID") 807 if not pubDIDs: 808 pubDIDs = [] 809 elif not isinstance(pubDIDs, list): 810 pubDIDs = [pubDIDs] 811 return pubDIDs

812

813 - def adaptForDescriptors(self, renderer, descriptors):

814 """returns a core for renderer and a sequence of ProductDescriptors. 815 816 This method is mainly for helping adaptForRenderer. Do read the 817 docstring there. 818 """ 819 try: 820 allowedForSvc = set(utils.stealVar("allowedRendsForStealing")) 821 except ValueError: 822 allowedForSvc = [] 823 824 linkDefs, endpoints, errors = [], [], [] 825 for descriptor in descriptors: 826 if isinstance(descriptor, DatalinkFault): 827 errors.append(descriptor) 828 829 else: 830 lds, inputKeys, lerrs = self.getMetaForDescriptor(descriptor) 831 linkDefs.extend(lds) 832 errors.extend(lerrs) 833 834 # ssap expects the first renderer here to be dlget, so don't 835 # remove it or move it back. 836 for rendName in ["dlget", "dlasync"]: 837 if rendName in allowedForSvc: 838 endpoints.append( 839 _ServiceDescriptor( 840 descriptor.pubDID, 841 inputKeys, 842 rendName, 843 base.getMetaText(self.parent, 844 "dlget.description", 845 default="An interactive service on this dataset."))) 846 847 # dispatch on whether we're making metadata (case 1) or actual 848 # data (case 2) 849 inputKeys = self.inputKeys[:] 850 if renderer.name=="dlmeta": 851 pass # RESPONSEFORMAT and friends added through pql#DALIPars 852 853 else: 854 # we're a data generating core; inputKeys are the core's plus 855 # possibly those of actual processors. Right now, we assume they're 856 # all the same, so we take the last one as representative 857 # 858 # TODO: this restricts the use of the core to dlget and dlasync 859 # (see endpoint creation above). It's not clear that's what we 860 # want, as e.g. form may work fine as well. 861 if not descriptors: 862 raise base.ValidationError("ID is mandatory with dlget", 863 "ID") 864 if endpoints: 865 inputKeys.extend(endpoints[-1].inputKeys) 866 if isinstance(descriptors[-1], DatalinkFault): 867 descriptors[-1].raiseException() 868 869 res = self.change(inputTable=MS(svcs.InputTD, 870 inputKeys=inputKeys, exclusive=True)) 871 872 # again dispatch on meta or data, this time as regards what to run. 873 if renderer.name=="dlmeta": 874 res.run = res.runForMeta 875 else: 876 res.run = res.runForData 877 878 res.nocache = True 879 res.datalinkLinks = linkDefs 880 res.datalinkEndpoints = endpoints 881 res.descriptors = descriptors 882 res.errors = errors 883 return res

884

885 - def adaptForRenderer(self, renderer):

886 """returns a core for a specific product. 887 888 The ugly thing about datalink in DaCHS' architecture is that its 889 interface (in terms of, e.g., inputKeys' values children) depends 890 on the arguments themselves, specifically the pubDID. 891 892 The workaround is to abuse the renderer-specific getCoreFor, 893 ignore the renderer and instead steal an "args" variable from 894 somewhere upstack. Nasty, but for now an acceptable solution. 895 896 It is particularly important to never let service cache the 897 cores returned for the dl* renderers; hence to "nocache" magic. 898 899 This tries to generate all datalink-relevant metadata in one go 900 and avoid calling the descriptorGenerator(s) more than once per 901 pubDID. It therefore adds datalinkLinks, datalinkEndpoints, 902 and datalinkDescriptors attributes. These are used later 903 in either metadata generation or data processing. 904 905 The latter will in general use only the last pubDID passed in. 906 Therefore, this last pubDID determines the service interface 907 for now. Perhaps we should be joining the inputKeys in some way, 908 though, e.g., if we want to allow retrieving multiple datasets 909 in a tar file? Or to re-use the same service for all pubdids? 910 """ 911 # if we're not speaking real datalink, return right away (this will 912 # be cached, so this must never happen for actual data) 913 if not renderer.name in self.datalinkAdaptingRenderers: 914 return self 915 916 try: 917 args = utils.stealVar("args") 918 if not isinstance(args, dict): 919 # again, we're not being called in a context with a pubdid 920 raise ValueError("No pubdid") 921 except ValueError: 922 # no arguments found: decide later on whether to fault out. 923 args = {"ID": []} 924 925 pubDIDs = self._getPubDIDs(args) 926 descGen = self.descriptorGenerator.compile(self) 927 descriptors = [] 928 for pubDID in pubDIDs: 929 try: 930 descriptors.append(descGen(pubDID, args)) 931 except svcs.RedirectBase: 932 # let through redirects even for dlmeta; these are rendered 933 # further up in the call chain. 934 raise 935 936 except Exception as ex: 937 # if we're dlget, just let exceptions through (e.g., authentication), 938 # also with a view to pushing out useful error messages. 939 if renderer.name!="dlmeta": 940 raise 941 else: 942 # In dlmeta, convert to fault rows. 943 if isinstance(ex, base.NotFoundError): 944 descriptors.append(DatalinkFault.NotFoundFault(pubDID, 945 utils.safe_str(ex))) 946 else: 947 if base.DEBUG: 948 base.ui.notifyError("Error in datalink descriptor generator: %s"% 949 utils.safe_str(ex)) 950 descriptors.append(DatalinkFault.Fault(pubDID, 951 utils.safe_str(ex))) 952 953 return self.adaptForDescriptors(renderer, descriptors)

954

955 - def _iterAccessResources(self, ctx, service):

956 """iterates over the VOTable RESOURCE elements necessary for 957 the datalink rows produced by service. 958 """ 959 for dlSvc in self.datalinkEndpoints: 960 yield dlSvc.asVOT(ctx, service.getURL(dlSvc.rendName))

961

962 - def runForMeta(self, service, inputTable, queryMeta):

963 """returns a rendered VOTable containing the datalinks. 964 """ 965 try: 966 ctx = votablewrite.VOTableContext(tablecoding="td") 967 vot = V.VOTABLE[ 968 self.getDatalinksResource(ctx, service), 969 self._iterAccessResources(ctx, service)] 970 971 if "text/html" in queryMeta["accept"]: 972 # we believe it's a web browser; let it do stylesheet magic 973 destMime = "text/xml" 974 else: 975 destMime = self.datalinkType 976 977 destMime = str(inputTable.getParam("RESPONSEFORMAT") or destMime) 978 if destMime=="votable": 979 destMime = self.datalinkType 980 981 res = (destMime, "<?xml-stylesheet href='/static/xsl/" 982 "datalink-to-html.xsl' type='text/xsl'?>"+vot.render()) 983 return res 984 finally: 985 self.finalize()

986

987 - def runForData(self, service, inputTable, queryMeta):

988 """returns a data set processed according to inputTable's parameters. 989 """ 990 try: 991 args = inputTable.getParamDict() 992 if not self.dataFunctions: 993 raise base.DataError("This datalink service cannot process data") 994 995 descriptor = self.descriptors[-1] 996 self.dataFunctions[0].compile(self)(descriptor, args) 997 998 if descriptor.data is None: 999 raise base.ReportableError("Internal Error: a first data function did" 1000 " not create data.") 1001 1002 for func in self.dataFunctions[1:]: 1003 try: 1004 func.compile(self)(descriptor, args) 1005 except FormatNow: 1006 break 1007 except DeliverNow: 1008 return descriptor.data 1009 1010 res = self.dataFormatter.compile(self)(descriptor, args) 1011 return res 1012 finally: 1013 self.finalize()

1014

1015 - def finalize(self):

1016 """breaks circular references to make the garbage collector's job 1017 easier. 1018 1019 The core will no longer function once this has been called. 1020 """ 1021 utils.forgetMemoized(self) 1022 for proc in itertools.chain(self.metaMakers, self.dataFunctions): 1023 utils.forgetMemoized(proc) 1024 utils.forgetMemoized(self.descriptorGenerator) 1025 if self.dataFormatter: 1026 utils.forgetMemoized(self.dataFormatter) 1027 self.breakCircles() 1028 self.run = None

1029

1030 1031 -def makeDatalinkServiceDescriptor(ctx, service, tableDef, columnName):

1032 """returns a datalink descriptor for a datalink (dlmeta) service). 1033 1034 What's returned is gavo.votable elements. 1035 1036 ctx is a votablewrite VOTableContext that manages the IDs of the elements 1037 involved, service is the datalink service as a svc.Service instance, 1038 tableDef is the rscdef.TableDef instance with the rows the datalink service 1039 operates on, and columnName names the column within table to take 1040 the datalink's ID parameter from. 1041 """ 1042 return V.RESOURCE(type="meta", utype="adhoc:service", 1043 name=base.getMetaText(service, "name", default="links", propagate=False))[ 1044 V.DESCRIPTION[base.getMetaText(service, "description", 1045 default="A Datalink service to retrieve the data set" 1046 " as well as additional related files, plus possibly services" 1047 " for server-side processing.", propagate=False)], 1048 V.PARAM(name="standardID", datatype="char", arraysize="*", 1049 value="ivo://ivoa.net/std/DataLink#links-1.0"), 1050 V.PARAM(name="accessURL", datatype="char", arraysize="*", 1051 value=service.getURL("dlmeta")), 1052 V.GROUP(name="inputParams")[ 1053 V.PARAM(name="ID", datatype="char", arraysize="*", 1054 ref=ctx.getOrMakeIdFor( 1055 tableDef.getColumnByName(columnName), suggestion=columnName), 1056 ucd="meta.id;meta.main")]]

1057

Source Code for Module gavo.protocols.datalink