Source Code for Module gavo.svcs.service

   1  """ 
   2  Services, i.e. combinations of a core and at least one renderer. 
   3   
   4  A Service is something that receives some sort of structured data (typically, 
   5  a nevow context, processes it into input data using a grammar (default is 
   6  the contextgrammar), pipes it through a core to receive a data set and 
   7  optionally tinkers with that data set. 
   8  """ 
   9   
  10  #c Copyright 2008-2019, the GAVO project 
  11  #c 
  12  #c This program is free software, covered by the GNU GPL.  See the 
  13  #c COPYING file in the source distribution. 
  14   
  15   
  16  import os 
  17  import urllib 
  18   
  19  from nevow import inevow 
  20  from nevow import rend 
  21   
  22  from nevow import tags as T, entities as E #noflake: for custom render fcts 
  23   
  24  from zope.interface import implements 
  25   
  26  from gavo import base 
  27  from gavo import rsc 
  28  from gavo import rscdef 
  29  from gavo import utils 
  30  from gavo.rsc import table 
  31  from gavo.rscdef import rmkdef 
  32  from gavo.svcs import common 
  33  from gavo.svcs import core 
  34  from gavo.svcs import inputdef 
  35  from gavo.svcs import outputdef 
  36  from gavo.svcs import renderers 
  37   
  38   
  39  MS = base.makeStruct 
  40   
  41   
  42 -def adaptTable(origTable, newColumns): 
  43          """returns a Data instance created from origTable with columns taken from 
  44          newColumns. 
  45   
  46          The adaptation works like this: 
  47   
  48          (1) if names and units of newColumns are a subset of origTable.columns, 
  49          return a table with the rows from origTable and the columns from 
  50          newColumns. 
  51   
  52          (1a) if origTable has a noPostprocess attribute, proceed to (4) 
  53   
  54          (2) if names of newColumns are a subset of origTable.columns match 
  55          but one or more units don't, set up a conversion routine and create 
  56          new rows, combining them with newColumns to the result. 
  57   
  58          (3) else raise an error. 
  59   
  60          (4) Finally, stick the whole thing into a data container. 
  61   
  62          This stinks.  I'm plotting to do away with it. 
  63          """ 
  64          if hasattr(origTable, "noPostprocess"): 
  65                  colDiffs = None 
  66                  newTd = origTable.tableDef 
  67   
  68          else: 
  69                  colDiffs = base.computeColumnConversions( 
  70                          newColumns, origTable.tableDef.columns) 
  71                  newTd = origTable.tableDef.copy(origTable.tableDef.parent) 
  72                  newTd.columns = newColumns 
  73   
  74          if not colDiffs: 
  75                  newTable = table.InMemoryTable(newTd, rows=origTable.rows) 
  76                  newTable.meta_ = origTable.meta_ 
  77                  newTable._params = origTable._params 
  78   
  79          else: # we need to do work 
  80                  newTd.copyMetaFrom(origTable.tableDef) 
  81                  rmk = rscdef.RowmakerDef(None) 
  82                  for col in newColumns: 
  83                          exprStart = "" 
  84                          if col.name in colDiffs: 
  85                                  exprStart = "%s*"%colDiffs[col.name] 
  86                          rmk.feedObject("map", rmkdef.MapRule(rmk, dest=col.name, 
  87                                  content_="%svars[%s]"%(exprStart, repr(col.name)) 
  88                                  ).finishElement(None)) 
  89                  newTable = table.InMemoryTable(newTd, validate=False) 
  90                  mapper = rmk.finishElement(None).compileForTableDef(newTd) 
  91                  for r in origTable: 
  92                          newTable.addRow(mapper(r, newTable)) 
  93                  newTable._params = origTable._params 
  94   
  95          return rsc.wrapTable(newTable, rdSource=origTable.tableDef) 
  96   
  97   
  98 -class PreparsedInput(dict): 
  99          """a sentinel class signalling to the service that its input already 
 100          is parsed. 
 101   
 102          This is for for stuff coming from nevow formal rather than request.args, 
 103          and to be fed into service.run. 
 104   
 105          Construct with a dictionary. 
 106          """ 
 107   
 108   
 109 -class SvcResult(rend.DataFactory): 
 110          """A nevow.IContainer that has the result and also makes the 
 111          input dataset accessible. 
 112   
 113          It is constructed with an InMemoryTable instance coreResult, 
 114          a table instance inputTable, the current querymeta, and a service 
 115          instance. 
 116   
 117          If a service is defined, SvcResult adapts coreResult to the 
 118          columns defined by the service (getCurOutputFields). 
 119   
 120          Original currently either is an InMemory table, in which case it gets 
 121          adapted to what the service expects, or a data.Data instance (or 
 122          something else; but few renderers will be able to handle "something else"),  
 123          which is left alone. 
 124   
 125          SvcResult also makes queryMeta, inputTable and the service available.   This 
 126          should give renderers access to basically all the information they need.  The 
 127          resultmeta data item collects some of those. 
 128   
 129          SvcResult objects must be able to fall back to sensible behaviour 
 130          without a service.  This may be necessary in error handling. 
 131          """ 
 132          implements(inevow.IContainer) 
 133           
 134 -        def __init__(self, coreResult, inputTable, queryMeta, service=None): 
 135                  self.inputTable = inputTable 
 136                  self.queryMeta = queryMeta 
 137                  self.service = service 
 138                  if (service and isinstance(coreResult, rsc.BaseTable)): 
 139                          coreResult = adaptTable(coreResult,  
 140                                  service.getCurOutputFields(queryMeta)) 
 141                  self.original = coreResult 
 142   
 143 -        def data_resultmeta(self, ctx, data): 
 144                  resultmeta = { 
 145                          "itemsMatched": self.queryMeta.get("Matched",  
 146                                  len(self.original.getPrimaryTable())), 
 147                          "message": "", 
 148                  } 
 149                  return resultmeta 
 150   
 151 -        def data_inputRec(self, ctx, data): 
 152                  return self.inputTable.getParamDict() 
 153   
 154 -        def data_table(self, ctx, data): 
 155                  return self.original.getPrimaryTable() 
 156   
 157 -        def data_tableWithRole(self, role): 
 158                  """returns the table with role. 
 159   
 160                  If no such table is available, this will return an empty string. 
 161                  """ 
 162                  def _(ctx, data): 
 163                          try: 
 164                                  return data.original.getTableWithRole(role) 
 165                          except (AttributeError, base.DataError): 
 166                                  return "" 
 167   
 168                  return _ 
 169   
 170 -        def getParam(self, paramName): 
 171                  """returns getParam of the core result or that of its primary table. 
 172                  """ 
 173                  try: 
 174                          val = self.original.getParam(paramName) 
 175                          if val is not None: 
 176                                  return val 
 177                  except (KeyError, base.NotFoundError): 
 178                          pass 
 179   
 180                  return self.original.getPrimaryTable().getParam(paramName) 
 181   
 182   
 183 -class Publication(base.Structure, base.ComputedMetaMixin): 
 184          """A specification of how a service should be published. 
 185   
 186          This contains most of the metadata for what is an interface in 
 187          registry speak. 
 188          """ 
 189          name_ = "publish" 
 190   
 191          _rd = rscdef.RDAttribute() 
 192          _render = base.UnicodeAttribute("render", default=base.Undefined, 
 193                  description="The renderer the publication will point at.", 
 194                  copyable=True) 
 195          _sets = base.StringSetAttribute("sets",  
 196                  description="Comma-separated list of sets this service will be" 
 197                          " published in.  Predefined are: local=publish on front page," 
 198                          " ivo_managed=register with the VO registry.  If you leave it" 
 199                          " empty, 'local' publication is assumed.", 
 200                          copyable="True") 
 201          _service = base.ReferenceAttribute("service", default=base.NotGiven, 
 202                  description="Reference for a service actually implementing the" 
 203                          " capability corresponding to this publication.  This is" 
 204                          " mainly when there is a vs:WebBrowser service accompanying a VO" 
 205                          " protocol service, and this other service should be published" 
 206                          " in the same resource record.  See also the operator's guide.", 
 207                          copyable="True") 
 208          _auxiliary = base.BooleanAttribute("auxiliary", default=False, 
 209                  description="Auxiliary publications are for capabilities" 
 210                          " not intended to be picked up for all-VO queries, typically" 
 211                          " because they are already registered with other services." 
 212                          " This is mostly used internally; you probably have no reason" 
 213                          " to touch it.") 
 214   
 215 -        def completeElement(self, ctx): 
 216                  if self.render is base.Undefined: 
 217                          self.render = "form" 
 218                  if not self.sets: 
 219                          self.sets.add("local") 
 220                  if self.service is base.NotGiven: 
 221                          self.service = self.parent 
 222                  self.setMetaParent(self.service) 
 223                  self._completeElementNext(Publication, ctx) 
 224   
 225 -        def validate(self): 
 226                  self._validateNext(Publication) 
 227                  try: 
 228                          renderers.getRenderer(self.render) 
 229                  except KeyError: 
 230                          raise base.StructureError("Unknown renderer: %s"%self.render) 
 231   
 232 -        def _meta_accessURL(self): 
 233                  return self.service.getURL(self.render, canonical=True) 
 234   
 235 -        def _meta_urlUse(self): 
 236                  return renderers.getRenderer(self.render).urlUse 
 237   
 238 -        def _meta_requestMethod(self): 
 239                  return renderers.getRenderer(self.render).preferredMethod 
 240           
 241 -        def _meta_resultType(self): 
 242                  return renderers.getRenderer(self.render).resultType 
 243   
 244   
 245 -class CustomPageFunction(base.Structure, base.RestrictionMixin): 
 246          """An abstract base for nevow.rend.Page-related functions on services. 
 247          """ 
 248          _name = base.UnicodeAttribute("name", default=base.Undefined, 
 249                  description="Name of the render function (use this in the" 
 250                          " n:render or n:data attribute in custom templates).",  
 251                          copyable=True, strip=True) 
 252          _code = base.DataContent(description="Function body of the renderer; the" 
 253                  " arguments are named ctx and data.", copyable=True) 
 254   
 255 -        def onElementComplete(self): 
 256                  self._onElementCompleteNext(CustomPageFunction) 
 257                  vars = globals().copy() 
 258                  vars["service"] = self.parent 
 259                  exec ("def %s(ctx, data):\n%s"%(self.name, 
 260                                  utils.fixIndentation(self.content_, newIndent="  ", 
 261                                          governingLine=1).rstrip())) in vars 
 262                  self.func = vars[self.name] 
 263   
 264   
 265 -class CustomRF(CustomPageFunction): 
 266          """A custom render function for a service. 
 267   
 268          Custom render functions can be used to expose certain aspects of a service 
 269          to Nevow templates.  Thus, their definition usually only makes sense with 
 270          custom templates, though you could, in principle, override built-in 
 271          render functions. 
 272   
 273          In the render functions, you have the names ctx for nevow's context and 
 274          data for whatever data the template passes to the renderer.  
 275   
 276          You can return anything that can be in a stan DOM.  Usually, this will be 
 277          a string.  To return HTML, use the stan DOM available under the T namespace. 
 278   
 279          As an example, the following code returns the current data as a link:: 
 280   
 281                  return ctx.tag[T.a(href=data)[data]] 
 282           
 283          You can access the embedding service as service, the embedding 
 284          RD as service.rd. 
 285          """ 
 286          name_ = "customRF" 
 287   
 288   
 289 -class CustomDF(CustomPageFunction): 
 290          """A custom data function for a service. 
 291   
 292          Custom data functions can be used to expose certain aspects of a service 
 293          to Nevow templates.  Thus, their definition usually only makes sense with 
 294          custom templates, though you could, in principle, override built-in 
 295          render functions. 
 296   
 297          In the data functions, you have the names ctx for nevow's context and 
 298          data for whatever data the template passes to the renderer.  
 299   
 300          You can access the embedding service as service, the embedding 
 301          RD as service.rd. 
 302   
 303          You can return arbitrary python objects -- whatever the render functions 
 304          can deal with.  You could, e.g., write:: 
 305   
 306                  <customDF name="now"> 
 307                          return datetime.datetime.utcnow() 
 308                  </customDF> 
 309           
 310          You also see a nevow context within the function.  You can use that to 
 311          access a query paramter ``order`` like this:: 
 312   
 313            args = inevow.IRequest(ctx).args 
 314                  sortOrder = args.get("order", ["authors"]) 
 315          """ 
 316          name_ = "customDF" 
 317   
 318   
 319 -class CoreAttribute(base.ReferenceAttribute): 
 320 -        def __init__(self): 
 321                  base.ReferenceAttribute.__init__(self, "core",  
 322                          description="The core that does the computations for this service." 
 323                          "  Instead of a reference, you can use an immediate element" 
 324                          " of some registred core.",  
 325                          forceType=core.Core, copyable=True, aliases=core.CORE_REGISTRY.keys()) 
 326           
 327 -        def _makeChild(self, name, parent): 
 328                  return core.getCore(name)(parent) 
 329   
 330   
 331  @utils.memoized 
 332 -def getDALIServiceKeys(): 
 333          """returns a list of the service keys defined in //pql#DALIPars. 
 334   
 335          This is always the same object, so if you really have to change anything 
 336          in here, be sure to make copies before touching either the list or 
 337          the items. 
 338          """ 
 339          nullSvc = base.parseFromString(Service,  
 340                  """<service><nullCore/><FEED source="//pql#DALIPars"/></service>""") 
 341   
 342          res = [] 
 343          for inputKey in nullSvc.serviceKeys: 
 344                  # let's orphan them in a way that they won't be reparented 
 345                  inputKey.parent_ = base.NotGiven 
 346                  res.append(inputKey) 
 347   
 348          return res 
 349   
 350   
 351 -class Service(base.Structure, base.ComputedMetaMixin,  
 352                  base.StandardMacroMixin, rscdef.IVOMetaMixin): 
 353          """A service definition. 
 354   
 355          A service is a combination of a core and one or more renderers.  They 
 356          can be published, and they carry the metadata published into the VO. 
 357   
 358          You can set the defaultSort property on the service to a name of an 
 359          output column to preselect a sort order.  Note again that this will 
 360          slow down responses for all but the smallest tables unless there is 
 361          an index on the corresponding column. 
 362   
 363          Properties evaluated: 
 364   
 365          * defaultSort -- a key to sort on by default with the form renderer.   
 366            This differs from the dbCore's sortKey in that this does not suppress the 
 367            widget itself, it just sets a default for its value.  Don't use this unless 
 368            you have to; the combination of sort and limit can have disastrous effects 
 369            on the run time of queries. 
 370          * votableRespectsOutputTable -- usually, VOTable output puts in 
 371            all columns from the underlying database table with low enough 
 372            verbLevel (essentially).  When this property is "True" (case-sensitive), 
 373                  that's not done and only the service's output table is evaluated. 
 374          """ 
 375          name_ = "service" 
 376   
 377          _core = CoreAttribute() 
 378          _templates = base.DictAttribute("templates", description="Custom" 
 379                  ' nevow templates for this service; use key "form" to replace the Form' 
 380                  " renderer's standard template.  Start the path with two slashes to" 
 381                  " access system templates.",  
 382                  itemAttD=rscdef.ResdirRelativeAttribute( 
 383                          "template", description="resdir-relative path to a nevow template" 
 384                          " used for the function given in key."), copyable=True) 
 385          _publications = base.StructListAttribute("publications", 
 386                  childFactory=Publication, description="Sets and renderers this service" 
 387                          " is published with.") 
 388          _limitTo = base.UnicodeAttribute("limitTo", default=None, 
 389                  description="Limit access to the group given; the empty default disables" 
 390                  " access control.", copyable="True") 
 391          _customPage = rscdef.ResdirRelativeAttribute("customPage", default=None, 
 392                  description="resdir-relative path to custom page code.  It is used" 
 393                  " by the 'custom' renderer", copyable="True") 
 394          _allowedRenderers = base.StringSetAttribute("allowed", 
 395                  description="Names of renderers allowed on this service; leave emtpy" 
 396                  " to allow the form renderer only.", copyable=True) 
 397          _customRF = base.StructListAttribute("customRFs", 
 398                  description="Custom render functions for use in custom templates.", 
 399                  childFactory=CustomRF, copyable=True) 
 400          _customDF = base.StructListAttribute("customDFs", 
 401                  description="Custom data functions for use in custom templates.", 
 402                  childFactory=CustomDF, copyable=True) 
 403          _outputTable = base.StructAttribute("outputTable", default=base.NotGiven, 
 404                  childFactory=outputdef.OutputTableDef, copyable=True, description= 
 405                  "The output fields of this service.") 
 406          _serviceKeys = base.UniquedStructListAttribute("serviceKeys", 
 407                  uniqueAttribute="name", policy="drop", 
 408                  childFactory=inputdef.InputKey, description="Input widgets for" 
 409                          " processing by the service, e.g. output sets.", copyable=True) 
 410          _defaultRenderer = base.UnicodeAttribute("defaultRenderer", 
 411                  default=None, description="A name of a renderer used when" 
 412                  " none is provided in the URL (lets you have shorter URLs).") 
 413   
 414          _rd = rscdef.RDAttribute() 
 415          _props = base.PropertyAttribute() 
 416          _original = base.OriginalAttribute() 
 417   
 418          metaModel = ("title(1), creationDate(1), description(1)," 
 419                  "subject, referenceURL(1), shortName(!)") 
 420   
 421          # formats that should query the same fields as HTML (the others behave 
 422          # like VOTables and offer a "verbosity" widget in forms). 
 423          htmlLikeFormats = ["HTML", "tar"] 
 424   
 425          ####################### Housekeeping methods 
 426   
 427 -        def __repr__(self): 
 428                  return "<Service %s at %s>"%(self.id, self.getSourcePosition()) 
 429   
 430 -        def completeElement(self, ctx): 
 431                  self._completeElementNext(Service, ctx) 
 432                  if not self.allowed: 
 433                          self.allowed.add("form") 
 434   
 435                  if self.core is base.Undefined: 
 436                          # undefined cores are only allowed with custom pages 
 437                          # (Deprecated) 
 438                          if self.customPage: 
 439                                  self.core = core.getCore("nullCore")(self.rd).finishElement(None) 
 440                                  base.ui.notifyWarning("Custom page service %s without nullCore." 
 441                                          "  This is deprecated, please fix"%self.id) 
 442                          else: 
 443                                  raise base.StructureError("Services must have cores (add <nullCore/>" 
 444                                          " if you really do not want a core, e.g., with fixed renderers).") 
 445   
 446                  # if there's only one renderer on this service, make it the default 
 447                  if self.defaultRenderer is None and len(self.allowed)==1: 
 448                          self.defaultRenderer = list(self.allowed)[0] 
 449   
 450                  # if there's a DALI-compliant renderer on this service, declare 
 451                  # its service parameters (MAXREC, RESPONSEFORMAT, etc). 
 452                  # Of course, other renderers might ignore them, but presumably 
 453                  # the metadata generated from this are only really evaluated 
 454                  # by VO/DALI components. 
 455                  # "DALI-complicance" is currently determined by a parameter 
 456                  # style of dali or pql. 
 457                  for rendName in self.allowed: 
 458                          try: 
 459                                  ps = renderers.getRenderer(rendName).parameterStyle  
 460                          except base.NotFoundError: 
 461                                  # letting this exc through could lead to confusing error messages 
 462                                  # on cross-RD reference, so we just warn: 
 463                                  base.ui.notifyWarning("Invalid renderer spec '%s' on service at %s"% 
 464                                          (rendName, self.getSourcePosition())) 
 465                                  continue 
 466   
 467                          if ps in ["dali", "pql"]: 
 468                                  for inputKey in getDALIServiceKeys(): 
 469                                          self._serviceKeys.addStruct(self, inputKey) 
 470                                  break 
 471   
 472                  # cache all kinds of things expensive to create and parse 
 473                  self._coresCache = {} 
 474                  self._loadedTemplates = {} 
 475                   
 476                  # Schedule the capabilities to be added when the parse is 
 477                  # done (i.e., the RD is complete) 
 478                  ctx.addExitFunc(lambda rd, ctx: self._addAutomaticCapabilities()) 
 479                           
 480 -        def onElementComplete(self): 
 481                  self._onElementCompleteNext(Service) 
 482                   
 483                  # Index custom render/data functions 
 484                  self.nevowRenderers = {} 
 485                  for customRF in self.customRFs: 
 486                          self.nevowRenderers[customRF.name] = customRF.func 
 487                  self.nevowDataFunctions = {} 
 488                  for customDF in self.customDFs: 
 489                          self.nevowDataFunctions[customDF.name] = customDF.func 
 490   
 491                  self._compileCustomPage() 
 492   
 493                  self._computeResourceType() 
 494   
 495 -        def _compileCustomPage(self): 
 496                  if self.customPage: 
 497                          try: 
 498                                  modNs, moddesc = utils.loadPythonModule(self.customPage) 
 499                                  modNs.RD = self.rd 
 500                                  getattr(modNs, "initModule", lambda: None)() 
 501                                  page = modNs.MainPage 
 502                          except ImportError: 
 503                                  raise base.ui.logOldExc( 
 504                                          base.LiteralParseError("customPage", self.customPage,  
 505                                          hint="This means that an exception was raised while DaCHS" 
 506                                                  " tried to import the renderer module.  If DaCHS ran" 
 507                                                  " with --debug, the original traceback is available" 
 508                                                  " in the logs.")) 
 509                          self.customPageCode = page, (os.path.basename(self.customPage),)+moddesc 
 510   
 511 -        def getTemplate(self, key): 
 512                  """returns the nevow template for the function key on this service. 
 513                  """ 
 514                  if key not in self._loadedTemplates: 
 515                          from nevow import loaders 
 516                          tp = self.templates[key] 
 517                          if tp.startswith("//"): 
 518                                  self._loadedTemplates[key] = common.loadSystemTemplate(tp[2:]) 
 519                          else: 
 520                                  self._loadedTemplates[key] = loaders.xmlfile( 
 521                                          os.path.join(self.rd.resdir, tp)) 
 522                  return self._loadedTemplates[key] 
 523   
 524 -        def getUWS(self): 
 525                  """returns a user UWS instance for this service. 
 526   
 527                  This is a service for the UWSAsyncRenderer. 
 528                  """ 
 529                  if not hasattr(self, "uws"): 
 530                          from gavo.protocols import useruws 
 531                          self.uws = useruws.makeUWSForService(self) 
 532                  return self.uws 
 533   
 534          ################### Registry and related methods. 
 535   
 536          @property 
 537 -        def isVOPublished(self, renderer=None): 
 538                  """is true if there is any ivo_managed publication on this 
 539                  service. 
 540   
 541                  If renderer is non-None, only publications with this renderer name 
 542                  count. 
 543                  """ 
 544                  for pub in self.publications: 
 545                          if "ivo_managed" in pub.sets: 
 546                                  if renderer: 
 547                                          if pub.render==renderer: 
 548                                                  return True 
 549                                  else: 
 550                                          return True 
 551                  return False 
 552   
 553 -        def _computeResourceType(self): 
 554                  """sets the resType attribute. 
 555   
 556                  Services are resources, and the registry code wants to know what kind. 
 557                  This method ventures a guess.  You can override this decision by setting 
 558                  the resType meta item. 
 559                  """ 
 560                  if (self.core.outputTable.columns  
 561                          or "tap" in self.allowed): 
 562                          self.resType = "catalogService" 
 563                  else: # no output table defined, we're a plain service 
 564                          self.resType = "nonTabularService" 
 565   
 566 -        def _iterAutomaticCapabilities(self): 
 567                  """helps _addAutomaticCapabilities. 
 568   
 569                  Actually, we also use it to generate VOSI capabilties for 
 570                  unpublished services. 
 571                  """ 
 572                  vosiSet = set(["ivo_managed"]) 
 573   
 574                  # All actual services get VOSI caps 
 575                  if not isinstance(self.core, core.getCore("nullCore")): 
 576                          yield base.makeStruct(Publication,  
 577                                          render="availability",  
 578                                          sets=vosiSet, 
 579                                          parent_=self) 
 580                          yield base.makeStruct(Publication,  
 581                                          render="capabilities",  
 582                                          sets=vosiSet, 
 583                                          parent_=self) 
 584                          yield base.makeStruct(Publication,  
 585                                          render="tableMetadata",  
 586                                          sets=vosiSet, 
 587                                          parent_=self) 
 588   
 589                  # things querying tables get a TAP relationship if  
 590                  # their table is adql-queriable 
 591                  if isinstance(self.core, core.getCore("dbCore")): 
 592                          if self.core.queriedTable.adql: 
 593                                  tapService = base.resolveCrossId("//tap#run")  
 594                                  yield base.makeStruct(Publication,  
 595                                                  render="tap",  
 596                                                  sets=vosiSet, 
 597                                                  auxiliary=True,  
 598                                                  service=tapService, 
 599                                                  parent_=self) 
 600   
 601                  # things with examples meta get an examples capability 
 602                  try: 
 603                          self.getMeta("_example", raiseOnFail=True) 
 604                          yield base.makeStruct(Publication,  
 605                                          render="examples",  
 606                                          sets=utils.AllEncompassingSet(), 
 607                                          parent_=self) 
 608                  except base.NoMetaKey: 
 609                          pass 
 610   
 611 -        def _addAutomaticCapabilities(self): 
 612                  """adds some publications that are automatic for certain types 
 613                  of services. 
 614   
 615                  For services with ivo_managed publications and with useful cores 
 616                  (this keeps out doc-like publications, which shouldn't have VOSI 
 617                  resources), artificial VOSI publications are added. 
 618   
 619                  If there is _example meta, an examples publication is added. 
 620   
 621                  If this service exposes a table (i.e., a DbCore with a queriedTable) 
 622                  and that table is adql-readable, also add an auxiliary TAP publication 
 623                  if going to the VO. 
 624   
 625                  This is being run as an exit function from the parse context as 
 626                  we want the RD to be complete at this point (e.g., _examples 
 627                  meta might come from it).  This also lets us liberally resolve 
 628                  references anywhere. 
 629                  """ 
 630                  if not self.isVOPublished: 
 631                          return 
 632   
 633                  for pub in self._iterAutomaticCapabilities(): 
 634                          self._publications.feedObject(self, pub) 
 635   
 636            # This is no longer a good place to do this, but TAP-published 
 637                  # tables need a servedBy, and we since it's automatic, let's 
 638                  # do it here. 
 639                  # According to the "discovering dependent" note, we don't 
 640                  # do the reverse relationship lest the TAP service 
 641                  # gets too related... 
 642                  if isinstance(self.core, core.getCore("dbCore")): 
 643                          if self.core.queriedTable.adql: 
 644                                  tapService = base.resolveCrossId("//tap#run") 
 645                                  self.addMeta("servedBy",  
 646                                          base.getMetaText(tapService, "title"), 
 647                                          ivoId=base.getMetaText(tapService, "identifier")) 
 648   
 649 -        def _iterAssociatedDatalinkServices(self): 
 650                  """yields instances for datalink services appropriate for this 
 651                  service. 
 652   
 653                  The datalink services are taken from the core's queriedTable 
 654                  attribute, if available.  As a legacy fallback, for now 
 655                  the datalink property on the service is supported as well for now, 
 656                  but I'll try to get rid of that. 
 657                  """ 
 658                  queriedTable = getattr(self.core, "queriedTable", None) 
 659                  # if there's not queried table, there's not much point in 
 660                  # having a datalink service. 
 661                  linkGenerated = False 
 662                  if queriedTable: 
 663                          for svcRef in queriedTable.iterMeta( 
 664                                          "_associatedDatalinkService.serviceId"): 
 665                                  yield base.resolveId(self.rd, str(svcRef)) 
 666                                  linkGenerated = True 
 667   
 668                  # Legacy fallback in case no datalink services are found in the table: 
 669                  # the datalink property 
 670                  if not linkGenerated: 
 671                          svcRef = self.getProperty("datalink", None) 
 672                          if svcRef: 
 673                                  yield self.rd.getById(svcRef) 
 674   
 675 -        def getPublicationsForSet(self, names): 
 676                  """returns publications for a set of set names (the names argument). 
 677   
 678                  In the special case names=None, all allowed renderers are treated 
 679                  as published. 
 680                  """ 
 681                  addAllAllowed = False 
 682                  if names is None: 
 683                          names = utils.AllEncompassingSet() 
 684                          addAllAllowed = True 
 685   
 686                  # the or in the list comprehension is because I can't see a way 
 687                  # to make AllEmcompassingSet work on the right side of the 
 688                  # operand, but it can occur on both sides. 
 689                  result = [pub for pub in self.publications  
 690                          if pub.sets & names or names & pub.sets] 
 691   
 692                  # for ivo_managed, also return a datalink endpoints if they're 
 693                  # there; the specs imply that might be useful some day. 
 694                  for dlSvc in self._iterAssociatedDatalinkServices(): 
 695                          if "dlget" in dlSvc.allowed: 
 696                                  result.append(base.makeStruct(Publication, 
 697                                          render="dlget", 
 698                                          sets="ivo_managed", 
 699                                          service=dlSvc)) 
 700   
 701                          if "dlasync" in dlSvc.allowed: 
 702                                  result.append(base.makeStruct(Publication, 
 703                                          render="dlasync", 
 704                                          sets="ivo_managed", 
 705                                          service=dlSvc)) 
 706   
 707                          if "dlmeta" in dlSvc.allowed: 
 708                                  result.append(base.makeStruct(Publication, 
 709                                          render="dlmeta", 
 710                                          sets="ivo_managed", 
 711                                          service=dlSvc)) 
 712   
 713                  if addAllAllowed: # name=None was passed in, 
 714                          # add publications for everything allowed but not yet marked 
 715                          # as published (for VOSI) 
 716                          alreadyPublished = set(p.render for p in result) 
 717                          for rendName in self.allowed: 
 718                                  if not rendName in alreadyPublished: 
 719                                          result.append(base.makeStruct(Publication, 
 720                                                  render=rendName, sets="vosi", service=self)) 
 721                          # these will only be added here for unpublished services 
 722                          for pub in self._iterAutomaticCapabilities(): 
 723                                  if not pub.render in alreadyPublished: 
 724                                          result.append(pub) 
 725   
 726                  return result 
 727   
 728   
 729 -        def getURL(self, rendName, absolute=True, canonical=False, **kwargs): 
 730                  """returns the full canonical access URL of this service together  
 731                  with renderer. 
 732   
 733                  rendName is the name of the intended renderer in the registry 
 734                  of renderers. 
 735   
 736                  With absolute, a fully qualified URL is being returned. 
 737   
 738                  Further keyword arguments are translated into URL parameters in the 
 739                  query part. 
 740                  """ 
 741                  basePath = "%s%s/%s"%(base.getConfig("web", "nevowRoot"), 
 742                          self.rd.sourceId, self.id) 
 743                  if absolute: 
 744                          basePath = base.makeAbsoluteURL(basePath, canonical=canonical) 
 745                  res = renderers.getRenderer(rendName 
 746                          ).makeAccessURL(basePath) 
 747   
 748                  if kwargs: 
 749                          res = res+"?"+urllib.urlencode(kwargs) 
 750                  return res 
 751   
 752          # used by getBrowserURL; keep external higher than form as long as 
 753          # we have mess like Potsdam CdC. 
 754          _browserScores = {"form": 10, "external": 12, "fixed": 15, 
 755                  "custom": 3, "img.jpeg": 2, "static": 1} 
 756   
 757 -        def getBrowserURL(self, fq=True): 
 758                  """returns a published URL that's suitable for a web browser or None if 
 759                  no such URL can be guessed. 
 760   
 761                  If you pass fq=False, you will get a path rather than a URL. 
 762                  """ 
 763                  # There can be multiple candidates for browser URLs (like when a service 
 764                  # has both form, static, and external renderers).  If so, we select 
 765                  # by plain scores. 
 766                  browseables = [] 
 767                  for rendName in self.allowed: 
 768                          if self.isBrowseableWith(rendName): 
 769                                  browseables.append((self._browserScores.get(rendName, -1), rendName)) 
 770                  if browseables: 
 771                          return self.getURL(max(browseables)[1], absolute=fq) 
 772                  else: 
 773                          return None 
 774                   
 775 -        def isBrowseableWith(self, rendName): 
 776                  """returns true if rendering this service through rendName results  
 777                  in something pretty in a web browser. 
 778                  """ 
 779                  try: 
 780                          return bool(renderers.getRenderer(rendName).isBrowseable(self)) 
 781                  except base.NotFoundError: # renderer name not known 
 782                          return False 
 783   
 784 -        def getTableSet(self): 
 785                  """returns a list of table definitions that have something to do with 
 786                  this service. 
 787   
 788                  This is for VOSI-type requests.  Usually, that's just the core's 
 789                  queried table or an output table, except when there is a TAP renderer on 
 790                  the service. 
 791   
 792                  All this is a bit heuristic; but then again, there's no rigorous  
 793                  definition for what's to be in a tables endpoint either. 
 794                  """ 
 795                  tables = [] 
 796   
 797                  qt = getattr(self.core, "queriedTable", None) 
 798                  if qt is None or not qt.columns: 
 799                          qt = self.core.outputTable 
 800                          if qt is None or not qt.columns: 
 801                                  if self.outputTable and self.outputTable.columns: 
 802                                          qt = self.outputTable 
 803   
 804                  if qt is not None: 
 805                          tables.append(qt) 
 806   
 807                  # XXX TODO: This stinks big time.  It's because we got TAP factorization 
 808                  # wrong.  Sync and async should be renderers, and there should 
 809                  # be a core that then could say this kind of thing.  That's not 
 810                  # yet the case, so: 
 811                  if "tap" in self.allowed: 
 812                          # tap never has "native" tables, so start afresh 
 813                          tables = [] 
 814   
 815                          mth = base.caches.getMTH(None) 
 816                          with base.getTableConn() as conn: 
 817                                  for tableName, in conn.query("SELECT table_name" 
 818                                                  " FROM TAP_SCHEMA.tables"): 
 819                                          try: 
 820                                                  tables.append(mth.getTableDefForTable(tableName)) 
 821                                          except: 
 822                                                  base.ui.notifyError("Failure trying to retrieve table definition" 
 823                                                          " for table %s.  Please fix the corresponding RD."%tableName) 
 824   
 825                  return [t for t in tables if t is not None and t.rd is not None] 
 826   
 827 -        def declareServes(self, data): 
 828                  """adds meta to self and data indicating that data is served by 
 829                  service. 
 830   
 831                  This is used by table/@adql and the publish element on data. 
 832                  """ 
 833                  if data.registration: 
 834                          self.addMeta("serviceFor",  
 835                                  base.getMetaText(data, "title", default="Anonymous"), 
 836                                  ivoId=base.getMetaText(data, "identifier")) 
 837                          data.addMeta("servedBy",  
 838                                  base.getMetaText(self, "title"), 
 839                                  ivoId=base.getMetaText(self, "identifier")) 
 840   
 841                          # Since this is always initiated by the data, the dependency 
 842                          # must show up in its RD to be properly added on publication 
 843                          # and to be removed when the data is removed. 
 844                          data.rd.addDependency(self.rd, data.rd) 
 845   
 846          ########################## Output field selection 
 847   
 848          _allSet = set(["ALL"]) 
 849   
 850 -        def _getFilteredColumns(self, columnSource, queryMeta): 
 851                  """filters columns in columnSource according to verbosity and colum 
 852                  set given in queryMeta. 
 853   
 854                  Actually, we only evaluate verbosity and requireSet. 
 855                  """ 
 856                  if queryMeta["columnSet"]: 
 857                          columnSource = [f for f in columnSource 
 858                                  if f.sets==self._allSet or queryMeta["columnSet"]&f.sets] 
 859   
 860                  verbLevel = queryMeta.get("verbosity", 20) 
 861                  return [f for f in columnSource 
 862                          if f.verbLevel<=verbLevel and not f.hidden] 
 863   
 864 -        def _getVOTableOutputFields(self, queryMeta): 
 865                  """returns a list of OutputFields suitable for a VOTable 
 866                  response described by queryMeta. 
 867   
 868                  This is the set of all columns in the source table below 
 869                  the verbosity defined in queryMeta, except that columns 
 870                  with a displayHint of noxml present are thrown out, too. 
 871   
 872                  When the service sets the votableRespectsOutputTable property to 
 873                  "True", the column source is the service's output table rather 
 874                  than the core's one. 
 875                  """ 
 876                  if queryMeta.get("verbosity")=="HTML": 
 877                          # SAMP transfers, typically: Pretend we're HTML 
 878                          columnSource = self.getHTMLOutputFields(queryMeta) 
 879   
 880                  elif (self.getProperty("votableRespectsOutputTable", "").lower()=="true" 
 881                                  or queryMeta["columnSet"]): 
 882                          columnSource = self.outputTable 
 883   
 884                  else: 
 885                          columnSource = self.getAllOutputFields() 
 886   
 887                  fields = self._getFilteredColumns( 
 888                          [f for f in columnSource  
 889                                  if f.displayHint.get("noxml")!="true"], 
 890                          queryMeta) 
 891   
 892                  return rscdef.ColumnList(fields) 
 893   
 894   
 895 -        def getHTMLOutputFields(self, queryMeta, ignoreAdditionals=False, 
 896                          raiseOnUnknown=True): 
 897                  """returns a list of OutputFields suitable for an HTML response described 
 898                  by queryMeta. 
 899   
 900                  This is the service's output table if given, else the core's output 
 901                  table at verbLevel 2.  Additional fields can be set by the user. 
 902   
 903                  raiseOnUnknown is used by customwidgets to avoid exceptions because of 
 904                  bad additional fields during form construction (when they aren't 
 905                  properly caught). 
 906                  """ 
 907                  if self.outputTable: 
 908                          columnSource = self.outputTable.columns 
 909                  else: 
 910                          columnSource = self.core.outputTable.columns 
 911   
 912                  fields = self._getFilteredColumns(columnSource, queryMeta) 
 913   
 914                  # add user-selected fields 
 915                  if not ignoreAdditionals and queryMeta["additionalFields"]: 
 916                          try: 
 917                                  for fieldName in queryMeta["additionalFields"]: 
 918                                          col = self.core.outputTable.getColumnByName(fieldName) 
 919                                          if isinstance(col, outputdef.OutputField): 
 920                                                  fields.append(col) 
 921                                          else: 
 922                                                  fields.append(outputdef.OutputField.fromColumn(col)) 
 923                          except base.NotFoundError as msg: 
 924                                  if raiseOnUnknown: 
 925                                          raise base.ValidationError("The additional field %s you requested" 
 926                                                  " does not exist"%repr(msg.lookedFor), colName="_OUTPUT") 
 927   
 928                  return rscdef.ColumnList(fields) 
 929   
 930 -        def getCurOutputFields(self, queryMeta=None, raiseOnUnknown=True): 
 931                  """returns a list of desired output fields for query meta. 
 932   
 933                  This is for both the core and the formatter to figure out the 
 934                  structure of the tables passed. 
 935                  """ 
 936                  queryMeta = queryMeta or common.emptyQueryMeta 
 937                  format = queryMeta.get("format", "HTML") 
 938                  if format in self.htmlLikeFormats: 
 939                          return self.getHTMLOutputFields(queryMeta, raiseOnUnknown=raiseOnUnknown) 
 940                  else: 
 941                          return self._getVOTableOutputFields(queryMeta) 
 942   
 943 -        def getAllOutputFields(self): 
 944                  """Returns a sequence of all available output fields. 
 945   
 946                  This is what the core gives, and this is what will be declared 
 947                  to the registry.  Depending on the output format, the verbosity 
 948                  level and perhaps other user settings, the actuall columns produced 
 949                  will be different. 
 950                  """ 
 951                  return self.core.outputTable.columns 
 952   
 953          ################### running and input computation. 
 954   
 955 -        def getCoreFor(self, renderer): 
 956                  """returns a core tailored for renderer. 
 957   
 958                  See svcs.core's module docstring.  
 959                   
 960                  The argument can be a renderer or a renderer name. 
 961                  """ 
 962                  if isinstance(renderer, basestring): 
 963                          renderer = renderers.getRenderer(renderer) 
 964   
 965                  # non-checked renderers use the core for info purposes only; don't 
 966                  # bother for those 
 967                  if not renderer.checkedRenderer: 
 968                          return self.core 
 969   
 970                  if renderer.name not in self._coresCache: 
 971                          # Bad Hack: Tell datalink core what renderers are allowed on 
 972                          # this service 
 973                          allowedRendsForStealing = self.allowed #noflake: for stealVar downstack 
 974   
 975                          res = self.core.adaptForRenderer(renderer) 
 976   
 977                          # Hack: let the polymorphous datalink core suppress caching 
 978                          if getattr(res, "nocache", False): 
 979                                  return res 
 980   
 981                          self._coresCache[renderer.name] = res 
 982                  return self._coresCache[renderer.name] 
 983   
 984 -        def getContextGrammarFor(self, renderer, core=None): 
 985                  """returns an ContextGrammar apropriate for this renderer. 
 986   
 987                  Pass in the core if you already have it as an optimisation (in 
 988                  particular for datalink, where cores aren't automatically cached); 
 989                  if you don't the core will be computed from the renderer. 
 990   
 991                  In either case, the context grammar simply is built from the core's 
 992                  inputTable. 
 993                  """ 
 994                  if isinstance(renderer, basestring): 
 995                          renderer = renderers.getRenderer(renderer) 
 996                  if core is None: 
 997                          core = self.getCoreFor(renderer) 
 998   
 999                  serviceKeys = list(inputdef.filterInputKeys(self.serviceKeys, 
1000                          renderer.name, inputdef.getRendererAdaptor(renderer))) 
1001   
1002                  return MS(inputdef.ContextGrammar, 
1003                                  inputTD=core.inputTable, 
1004                                  inputKeys=serviceKeys) 
1005   
1006 -        def getInputKeysFor(self, renderer): 
1007                  """returns a sequence of input keys, adapted for renderer. 
1008   
1009                  The renderer argument may either be a renderer name, a renderer 
1010                  class or a renderer instance. 
1011   
1012                  This is the main interface for external entities to discover. 
1013                  service metadata. 
1014                  """ 
1015                  if isinstance(renderer, basestring): 
1016                          renderer = renderers.getRenderer(renderer) 
1017                  return list(self.getContextGrammarFor(renderer).iterInputKeys()) 
1018   
1019 -        def _runWithInputTable(self, core, inputTable, queryMeta): 
1020                  """runs the core and formats an SvcResult. 
1021   
1022                  This is an internal method. 
1023                  """ 
1024                  coreRes = core.run(self, inputTable, queryMeta) 
1025                  res = SvcResult(coreRes, inputTable, queryMeta, self) 
1026                  return res 
1027   
1028 -        def run(self, renderer, args, queryMeta=None): 
1029                  """runs the service, returning an SvcResult. 
1030   
1031                  This is the main entry point for protocol renderers; args is 
1032                  a dict of lists as provided by request.args. 
1033   
1034                  Pass in queryMeta if convenient or if args is not simply request.args 
1035                  (but, e.g., nevow formal data).  Otherwise, it will be constructed 
1036                  from args. 
1037                  """ 
1038                  if isinstance(renderer, basestring): 
1039                          renderer = renderers.getRenderer(renderer) 
1040                  if queryMeta is None: 
1041                          queryMeta = common.QueryMeta.fromNevowArgs(args) 
1042   
1043                  core = self.getCoreFor(renderer) 
1044                  coreArgs = inputdef.CoreArgs.fromRawArgs( 
1045                          core.inputTable, args, self.getContextGrammarFor(renderer, core)) 
1046   
1047                  return self._runWithInputTable(core, coreArgs, queryMeta) 
1048   
1049   
1050          #################### meta and such 
1051   
1052 -        def _meta_available(self): 
1053  # XXX TODO: have this ask the core 
1054                  return "true" 
1055   
1056 -        def macro_tablesForTAP(self): 
1057                  """returns a list of table names available for TAP querying. 
1058   
1059                  This, really, is an implementation detail for the TAP service and 
1060                  might go away anytime. 
1061                  """ 
1062                  # this is only used by tap.rd -- maybe it 
1063                  # should go there? 
1064                  from gavo.protocols import tap 
1065                   
1066                  schemas = {} 
1067                  for qname in tap.getAccessibleTables(): 
1068                          try: 
1069                                  schema, name = qname.split(".") 
1070                          except: # weird name 
1071                                  continue 
1072                          schemas.setdefault(schema, []).append(name) 
1073   
1074                  return ", ".join("%s from the %s schema"%(", ".join(tables), schema) 
1075                          for schema, tables in schemas.iteritems()) 
1076           
1077 -        def _meta_examplesLink(self): 
1078                  """returns a link to a examples for this service if any 
1079                  are available. 
1080                  """ 
1081                  try: 
1082                          self.getMeta("_example", raiseOnFail=True) 
1083                          return base.META_CLASSES_FOR_KEYS["_related"]( 
1084                                  self.getURL("examples", False), 
1085                                  title="DALI examples") 
1086                  except base.NoMetaKey: 
1087                          return None 
1088           
1089 -        def _meta_howtociteLink(self): 
1090                  """returns a link to a how-to-cite page for this service as an URL 
1091                  meta. 
1092                  """ 
1093                  if self.getMeta("creationDate"): 
1094                          return base.META_CLASSES_FOR_KEYS["_related"]( 
1095                                  self.getURL("howtocite", False), 
1096                                  title="Advice on citing this resource") 
1097