gavo.rscdef.common

1 """ 2 Common items used by resource definition objects. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import datetime 12 import imp 13 import os 14 import re 15 import urllib 16 17 from gavo import base 18 from gavo import utils 19 20 21 # The following is a flag for initdachs (and initdachs exclusively) 22 # to prevent resource metadata reading from database tables during 23 # the dachst init. 24 _BOOTSTRAPPING = False

25 26 27 -class RDAttribute(base.AttributeDef):

28 """an attribute that gives access to the current rd. 29 30 The attribute is always called rd. There is no default, but on 31 the first access, we look for an ancestor with an rd attribute and 32 use that if it exists, otherwise rd will be None. There currently 33 is no way to reset the rd. 34 35 These attributes cannot (yet) be fed, so rd="xxx" won't work. 36 If we need this, the literal would probably be an id. 37 """ 38 computed_ = True 39 typeDesc_ = "reference to a resource descriptor" 40

41 - def __init__(self):

42 base.AttributeDef.__init__(self, "rd", None, "The parent" 43 " resource descriptor; never set this manually, the value will" 44 " be filled in by the software.")

45

46 - def iterParentMethods(self):

47 def _getRD(self): 48 if self.parent is None: # not yet adopted, we may want to try again later 49 return None 50 try: 51 return self.__rd 52 except AttributeError: 53 parent = self.parent 54 while parent: 55 if hasattr(parent, "rd") and parent.rd is not None: 56 self.__rd = parent.rd 57 break 58 parent = parent.parent 59 else: # a parent hasn't been adopted yet, try again later. 60 return None 61 return self.__rd

62 yield ("rd", property(_getRD)) 63 64 def getFullId(self): 65 if self.rd is None: 66 return self.id 67 return "%s#%s"%(self.rd.sourceId, self.id)

68 yield ("getFullId", getFullId) 69

70 - def makeUserDoc(self):

71 return None # don't metion it in docs -- users can't and mustn't set it.

72

73 74 -class ResdirRelativeAttribute(base.FunctionRelativePathAttribute):

75 """is a path that is interpreted relative to the current RD's resdir. 76 77 The parent needs an RDAttribute. 78 """

79 - def __init__(self, name, default=None, description="Undocumented", **kwargs):

80 base.FunctionRelativePathAttribute.__init__(self, name, 81 baseFunction=self.getResdir, 82 default=default, description=description, **kwargs)

83

84 - def getResdir(self, instance):

85 if instance.rd is None: 86 # we don't have a parent yet, but someone wants to see our 87 # value. This can happen if an element is validated before 88 # it is adopted (which we probably should forbid). Here, we 89 # hack around it and hope nobody trips over it 90 return None 91 return instance.rd.resdir

92

93 94 -class ProfileListAttribute(base.AtomicAttribute):

95 """An attribute containing a comma separated list of profile names. 96 97 There's the special role name "defaults" for whatever default this 98 profile list was constructed with. 99 """ 100 typeDesc_ = "Comma separated list of profile names." 101

102 - def __init__(self, name, default, description):

103 base.AtomicAttribute.__init__(self, name, base.Computed, description) 104 self.realDefault = default

105 106 @property

107 - def default_(self):

108 return self.realDefault.copy()

109

110 - def parse(self, value):

111 pNames = set() 112 for pName in value.split(","): 113 pName = pName.strip() 114 if not pName: 115 continue 116 if pName=="defaults": 117 pNames = pNames|self.default_ 118 else: 119 pNames.add(pName) 120 return pNames

121

122 - def unparse(self, value):

123 # It would be nice to reconstruct "defaults" here, but right now it's 124 # certainly not worth the effort. 125 return ", ".join(value)

126

127 128 -class PrivilegesMixin(object):

129 """A mixin for structures declaring access to database objects (tables, 130 schemas). 131 132 Access is managed on the level of database profiles. Thus, the names 133 here are not directly role names in the database. 134 135 We have two types of privileges: "All" means at least read and write, 136 and "Read" meaning at least read and lookup. 137 """ 138 _readProfiles = ProfileListAttribute("readProfiles", 139 default=base.getConfig("db", "queryProfiles"), 140 description="A (comma separated) list of profile names through" 141 " which the object can be read.") 142 _allProfiles = ProfileListAttribute("allProfiles", 143 default=base.getConfig("db", "maintainers"), 144 description="A (comma separated) list of profile names through" 145 " which the object can be written or administred (oh, and the" 146 " default is not admin, msdemlei but is the value of [db]maintainers)")

147

148 149 -class IVOMetaMixin(object):

150 """A mixin for resources aspiring to have IVO ids. 151 152 All those need to have an RDAttribute. Also, for some data this accesses 153 the servicelist database, so the class should really be in registry, where 154 that stuff is defined. But it can't be there, since it's needed for 155 the definition of tabledefs. 156 """

157 - def _meta_referenceURL(self):

158 return base.META_CLASSES_FOR_KEYS["referenceURL"]( 159 self.getURL("info"), 160 title="Service info")

161

162 - def _meta_identifier(self):

163 if "identifier" in self.meta_: 164 return self.meta_["identifier"] 165 # if we're called without an RD, that's probably while we're 166 # copied. Code there knows what to do when we return None 167 if self.rd is None: 168 return None 169 170 return "ivo://%s/%s/%s"%(base.getConfig("ivoa", "authority"), 171 self.rd.sourceId, self.id)

172

173 - def __getFromDB(self, metaKey):

174 # If we do the DB queries while the python module import lock is 175 # held, we'll get ugly races. Unfortunately, some resources 176 # may do metadata operations while we're being imported. For 177 # those, we check the import lock and return Nones. Similarly, 178 # during the initial import, this is suppressed because //services 179 # may not yet be present. 180 if imp.lock_held() or _BOOTSTRAPPING: 181 return None 182 183 try: # try to used cached data 184 if self.__dbRecord is None: 185 raise base.NoMetaKey(metaKey, carrier=self) 186 return self.__dbRecord[metaKey] 187 except AttributeError: 188 # fetch data from DB 189 pass 190 res = None 191 192 if self.rd: 193 # We're not going through servicelist since we don't want to depend 194 # on the registry subpackage. 195 with base.getTableConn() as conn: 196 res = list( 197 conn.query("SELECT dateUpdated, recTimestamp, setName" 198 " FROM dc.resources_join WHERE sourceRD=%(rdId)s AND resId=%(id)s", 199 {"rdId": self.rd.sourceId, "id": self.id})) 200 201 if res: 202 self.__dbRecord = { 203 "sets": list(set(row[2] for row in res)), 204 "recTimestamp": res[0][1].strftime(utils.isoTimestampFmt) 205 } 206 else: 207 self.__dbRecord = { 208 'sets': ['unpublished'], 209 'recTimestamp': datetime.datetime.utcnow().strftime( 210 utils.isoTimestampFmt) 211 } 212 return self.__getFromDB(metaKey)

213

214 - def _meta_dateUpdated(self):

215 if self.rd: 216 return self.rd.getMeta("dateUpdated")

217

218 - def _meta_datetimeUpdated(self):

219 if self.rd: 220 return self.rd.getMeta("datetimeUpdated")

221

222 - def _meta_recTimestamp(self):

223 return self.__getFromDB("recTimestamp")

224

225 - def _meta_sets(self):

226 return self.__getFromDB("sets")

227

228 - def _meta_status(self):

229 return "active"

230

231 232 -class Registration(base.Structure, base.MetaMixin):

233 """A request for registration of a data or table item. 234 235 This is much like publish for services, just for data and tables; 236 since they have no renderers, you can only have one register element 237 per such element. 238 239 Data registrations may refer to published services that make their 240 data available. 241 """ 242 name_ = "publish" 243 docName_ = "publish (data)" 244 aliases = ["register"] 245 246 _sets = base.StringSetAttribute("sets", default=frozenset(["ivo_managed"]), 247 description="A comma-separated list of sets this data will be" 248 " published in. To publish data to the VO registry, just" 249 " say ivo_managed here. Other sets probably don't make much" 250 " sense right now. ivo_managed also is the default.") 251 252 _servedThrough = base.ReferenceListAttribute("services", 253 description="A DC-internal reference to a service that lets users" 254 " query that within the data collection; tables with adql=True" 255 " are automatically declared to be servedBy the TAP service.") 256 257 # the following attribute is for compatibility with service.Publication 258 # in case someone manages to pass such a publication to the capability 259 # builder. 260 auxiliary = True 261

262 - def publishedForADQL(self):

263 """returns true if at least one table published is available for 264 TAP/ADQL. 265 """ 266 if getattr(self.parent, "adql", False): 267 # single table 268 return True 269 270 for t in getattr(self.parent, "iterTableDefs", lambda: [])(): 271 # data item with multiple tables 272 if t.adql: 273 return True 274 275 return False

276

277 - def register(self):

278 """adds servedBy and serviceFrom metadata to data, service pairs 279 in this registration. 280 """ 281 if self.publishedForADQL(): 282 tapSvc = base.caches.getRD("//tap").getById("run") 283 if not tapSvc in self.services: 284 self.services.append(tapSvc) 285 286 for srv in self.services: 287 srv.declareServes(self.parent)

288

289 290 -class ColumnList(list):

291 """A list of column.Columns (or derived classes) that takes 292 care that no duplicates (in name) occur. 293 294 If you add a field with the same dest to a ColumnList, the previous 295 instance will be overwritten. The idea is that you can override 296 ColumnList in, e.g., interfaces later on. 297 298 Also, two ColumnLists are considered equal if they contain the 299 same names. 300 301 After construction, you should set the withinId attribute to 302 something that will help make sense of error messages. 303 """

304 - def __init__(self, *args):

305 list.__init__(self, *args) 306 self.nameIndex = dict([(c.name, ct) for ct, c in enumerate(self)]) 307 self.withinId = "unnamed table"

308

309 - def __contains__(self, fieldName):

310 return fieldName in self.nameIndex

311

312 - def __eq__(self, other):

313 if isinstance(other, ColumnList): 314 myFields = set([f.name for f in self 315 if f.name not in self.internallyUsedFields]) 316 otherFields = set([f.name for f in other 317 if f.name not in self.internallyUsedFields]) 318 return myFields==otherFields 319 elif other==[] and len(self)==0: 320 return True 321 return False

322

323 - def deepcopy(self, newParent):

324 """returns a deep copy of self. 325 326 This means that all child structures are being copied. In that 327 process, they receive a new parent, which is why you need to 328 pass one in. 329 """ 330 return self.__class__([c.copy(newParent) for c in self])

331

332 - def getIdIndex(self):

333 try: 334 return self.__idIndex 335 except AttributeError: 336 self.__idIndex = dict((c.id, c) for c in self if c.id is not None) 337 return self.__idIndex

338

339 - def append(self, item):

340 """adds the Column item to the data field list. 341 342 It will overwrite a Column of the same name if such a thing is already 343 in the list. Indices are updated. 344 """ 345 key = item.name 346 if key in self.nameIndex: 347 nameInd = self.nameIndex[key] 348 assert self[nameInd].name==key, \ 349 "Someone tampered with ColumnList" 350 self[nameInd] = item 351 else: 352 self.nameIndex[item.name] = len(self) 353 list.append(self, item)

354

355 - def replace(self, oldCol, newCol):

356 ind = 0 357 while True: 358 if self[ind]==oldCol: 359 self[ind] = newCol 360 break 361 ind += 1 362 del self.nameIndex[oldCol.name] 363 self.nameIndex[newCol.name] = ind

364

365 - def remove(self, col):

366 del self.nameIndex[col.name] 367 list.remove(self, col)

368

369 - def extend(self, seq):

370 for item in seq: 371 self.append(item)

372

373 - def getColumnByName(self, name):

374 """returns the column with name. 375 376 It will raise a NotFoundError if no such column exists. 377 """ 378 try: 379 return self[self.nameIndex[name]] 380 except KeyError: 381 try: 382 return self[self.nameIndex[utils.QuotedName(name)]] 383 except KeyError: 384 raise base.NotFoundError(name, what="column", within=self.withinId)

385

386 - def getColumnById(self, id):

387 """returns the column with id. 388 389 It will raise a NotFoundError if no such column exists. 390 """ 391 try: 392 return self.getIdIndex()[id] 393 except KeyError: 394 raise base.NotFoundError(id, what="column", within=self.withinId)

395

396 - def getColumnByUtype(self, utype):

397 """returns the column having utype. 398 399 This should be unique, but this method does not check for uniqueness. 400 """ 401 utype = utype.lower() 402 for item in self: 403 if item.utype and item.utype.lower()==utype: 404 return item 405 raise base.NotFoundError(utype, what="column with utype", 406 within=self.withinId)

407

408 - def getColumnsByUCD(self, ucd):

409 """returns all columns having ucd. 410 """ 411 return [item for item in self if item.ucd==ucd]

412

413 - def getColumnByUCD(self, ucd):

414 """retuns the single, unique column having ucd. 415 416 It raises a ValueError if there is no such column or more than one. 417 """ 418 cols = self.getColumnsByUCD(ucd) 419 if len(cols)==1: 420 return cols[0] 421 elif cols: 422 raise ValueError("More than one column for %s"%ucd) 423 else: 424 raise ValueError("No column for %s"%ucd)

425

426 - def getColumnByUCDs(self, *ucds):

427 """returns the single, unique column having one of ucds. 428 429 This method has a confusing interface. It sole function is to 430 help when there are multiple possible UCDs that may be interesting 431 (like pos.eq.ra;meta.main and POS_EQ_RA_MAIN). It should only be 432 used for such cases. 433 """ 434 for ucd in ucds: 435 try: 436 return self.getColumnByUCD(ucd) 437 except ValueError: 438 pass 439 raise ValueError("No unique column for any of %s"%", ".join(ucds))

440

441 442 -class ColumnListAttribute(base.StructListAttribute):

443 """An adapter from a ColumnList to a structure attribute. 444 """ 445 @property

446 - def default_(self):

447 return ColumnList()

448

449 - def getCopy(self, instance, newParent, ctx):

450 return ColumnList(base.StructListAttribute.getCopy(self, 451 instance, newParent, ctx))

452

453 - def replace(self, instance, oldStruct, newStruct):

454 if oldStruct.name!=newStruct.name: 455 raise base.StructureError("Can only replace fields of the same" 456 " name in a ColumnList") 457 getattr(instance, self.name_).append(newStruct)

458

459 460 -class NamePathAttribute(base.AtomicAttribute):

461 """defines an attribute NamePath used for resolution of "original" 462 attributes. 463 464 The NamePathAttribute provides a resolveName method as expected 465 by base.OriginalAttribute. 466 """ 467 typeDesc_ = "id reference" 468

469 - def __init__(self, **kwargs):

470 if "description" not in kwargs: 471 kwargs["description"] = ("Reference to an element tried to" 472 " satisfy requests for names in id references of this" 473 " element's children.") 474 base.AtomicAttribute.__init__(self, name="namePath", **kwargs)

475

476 - def iterParentMethods(self):

477 def resolveName(instance, context, id): 478 if hasattr(instance, "parentTable"): 479 try: 480 return base.resolveNameBased(instance.parentTable, id) 481 except base.NotFoundError: 482 # try on real name path 483 pass 484 485 if hasattr(instance, "getByName"): 486 try: 487 return instance.getByName(id) 488 except base.NotFoundError: 489 pass 490 491 np = instance.namePath 492 if np is None and instance.parent: 493 np = getattr(instance.parent, "namePath", None) 494 if np is None: 495 raise base.NotFoundError(id, "Element with name", repr(self), 496 hint="No namePath here") 497 res = context.resolveId(np+"."+id) 498 return res

499 yield "resolveName", resolveName

500

501 - def parse(self, value):

502 return value

503

504 - def unparse(self, value):

505 return value

506 507 508 _atPattern = re.compile("@(%s)"%utils.identifierPattern.pattern[:-1])

509 510 -def replaceProcDefAt(src, dictName="vars"):

511 """replaces @<identifier> with <dictName>["<identifier>"] in src. 512 513 We do this to support this shortcut in the vicinity of rowmakers (i.e., 514 there and in procApps). 515 """ 516 return _atPattern.sub(r'%s["\1"]'%dictName, src)

517

518 519 # this is mainly here for lack of a better place. I don't want it in 520 # base.parsecontext as it needs config, and I don't want it 521 # in user.common as it might be useful for non-UI stuff. 522 -def getReferencedElement(refString, forceType=None, **kwargs):

523 """returns the element for the DaCHS reference ``refString``. 524 525 ``refString`` has the form ``rdId[#subRef]``; ``rdId`` can be 526 filesystem-relative, but the RD referenced must be below ``inputsDir`` 527 anyway. 528 529 You can pass a structure class into ``forceType``, and a ``StructureError`` 530 will be raised if what's pointed to by the id isn't of that type. 531 532 You should usually use ``base.resolveCrossId`` instead of this from *within* 533 DaCHS. This is intended for code handling RD ids from users. 534 535 This supports further keyword arguments to getRD. 536 """ 537 # get the inputs postfix now so we don't pollute the current exception later 538 try: 539 cwdInInputs = utils.getRelativePath(os.getcwd(), 540 base.getConfig("inputsDir"), liberalChars=True) 541 except ValueError: 542 # not in inputs 543 cwdInInputs = None 544 545 try: 546 return base.resolveCrossId(refString, forceType=forceType, **kwargs) 547 except base.RDNotFound: 548 if cwdInInputs: 549 return base.resolveCrossId("%s/%s"%(cwdInInputs, refString), 550 forceType=forceType) 551 raise

552

553 554 @utils.document 555 -def getStandardPubDID(path):

556 """returns the standard DaCHS PubDID for ``path``. 557 558 The publisher dataset identifier (PubDID) is important in protocols like 559 SSAP and obscore. If you use this function, the PubDID will be your 560 authority, the path compontent ~, and the inputs-relative path of 561 the input file as the parameter. 562 563 ``path`` can be relative, in which case it is interpreted relative to 564 the DaCHS ``inputsDir.`` 565 566 You *can* define your PubDIDs in a different way, but you'd then need 567 to provide a custom descriptorGenerator to datalink services (and 568 might need other tricks). If your data comes from plain files, use 569 this function. 570 571 In a rowmaker, you'll usually use the \\standardPubDID macro. 572 """ 573 # Why add inputsDir first and remove it again? Well, I want to keep 574 # getInputsRelativePath in the loop since it does some validation 575 # and may, at some point, do more. 576 if path[0]!="/": 577 path = os.path.join(base.getConfig("inputsDir"), path) 578 579 return "ivo://%s/~?%s"%( 580 base.getConfig("ivoa", "authority"), 581 getInputsRelativePath(path, liberalChars=True))

582

583 584 @utils.document 585 -def getDatalinkMetaLink(dlSvc, accref):

586 """returns a datalink URL for the product referenced through accref 587 with the datalink service dlSvc. 588 589 This assumes that dlSvc uses the standard DaCHS pubDIDs. dlSvc 590 needs to be the service element. 591 592 A typical use is in a metaMaker and would look like this:: 593 594 getDatalinkMetaLink(rd.getById("dl"), descriptor.accref) 595 """ 596 return '%s?ID=%s'%( 597 dlSvc.getURL("dlmeta"), 598 urllib.quote_plus(getStandardPubDID(accref)))

599

600 601 602 @utils.document 603 -def getAccrefFromStandardPubDID(pubdid, 604 authBase="ivo://%s/~?"%base.getConfig("ivoa", "authority")):

605 """returns an accref from a standard DaCHS PubDID. 606 607 This is basically the inverse of getStandardPubDID. It will raise 608 NotFound if pubdid "looks like a URI" (implementation detail: has a colon 609 in the first 10 characters) and does not start with ivo://<authority>/~?. 610 If it's not a URI, we assume it's a local accref and just return it. 611 612 The function does not check if the remaining characters are a valid 613 accref, much less whether it can be resolved. 614 615 authBase's default will reflect you system's settings on your installation, 616 which probably is not what's given in this documentation. 617 """ 618 if ":" not in pubdid[:10]: 619 return pubdid 620 621 if not pubdid.startswith(authBase): 622 raise base.NotFoundError(pubdid, 623 "The authority in the dataset identifier", 624 "the authorities managed here") 625 return pubdid[len(authBase):]

626

627 628 @utils.document 629 -def getInputsRelativePath(absPath, liberalChars=True):

630 """returns absath relative to the DaCHS inputsDir. 631 632 If ``absPath`` is not below ``inputsDir``, a ``ValueError`` results. On 633 ``liberalChars``, wee see the `function getRelativePath`_. 634 635 In rowmakers and rowfilters, you'll usually use the macro 636 ``\inputRelativePath`` that inserts the appropriate code. 637 """ 638 return utils.getRelativePath(absPath, 639 base.getConfig("inputsDir"), liberalChars=liberalChars)

640

Source Code for Module gavo.rscdef.common