gavo.registry.builders

Source Code for Module gavo.registry.builders

1 """ 2 Functions returning xmlstan for various OAI/VOR documents. 3 4 This comprises basic VOResource elements; capabilities and interfaces 5 (i.e. everything to do with renderers) are in registry.capabilities. 6 7 All this only becomes difficult when actually generating VOResource 8 metadata (OAI is plain). For every type of VO resource (CatalogService, 9 Registry, etc), there's a XYResourceMaker, all inheriting ResourceMaker. 10 11 The decision what VOResource type a given service has is passed 12 using common.getResType; this means the resType meta is tried first, 13 using resob.resType as a fallback. 14 """ 15 16 #c Copyright 2008-2019, the GAVO project 17 #c 18 #c This program is free software, covered by the GNU GPL. See the 19 #c COPYING file in the source distribution. 20 21 22 # DataCollection mess: In rev 3769, we experimentally pushed out 23 # CatalogService records instead of DataCollections, trying to have 24 # capabilities for them. That didn't turn out well even though 25 # we didn't do that for SIA and frieds: All-VO discovery of a 26 # given service type is a very typcial use case. So, we backed 27 # that out again in rev. 3883, though the support code remains. 28 # 29 # Then, in rev. 3891, we went back to CatalogService records, only 30 # this time we only pushed out capabilities with "auxiliary" standard ids. 31 32 33 from gavo import base 34 from gavo import stc 35 from gavo import svcs 36 from gavo import utils 37 from gavo.base import meta 38 from gavo.registry import common 39 from gavo.registry import capabilities 40 from gavo.registry import identifiers 41 from gavo.registry import tableset 42 from gavo.registry import servicelist 43 from gavo.registry.model import ( 44 OAI, VOR, VOG, DC, RI, VS, OAIDC, VSTD, DOC) 45 46 MS = base.makeStruct 47 48 SF = meta.stanFactory 49 _defaultSet = set(["ivo_managed"]) 50 # Set this to False to disable some lame "don't fail" error handlings; 51 # this will raise more exceptions and is not recommended in the actual 52 # OAI interface (where *some* info is better than none at all). 53 VALIDATING = False 54 55 56 ################## ModelBasedBuilders for simple metadata handling 57

58 -def _build_source(children, localattrs=None):

59 # in source, we try to recognize bibcodes automatically, hence we have 60 # this manual builder. 61 src = unicode(children[0]) 62 attrs = {} 63 if utils.couldBeABibcode(src): 64 attrs["format"] = "bibcode" 65 return VOR.source(**attrs)[src]

66 67

68 -def _build_dateFromNews(children, localattrs={}):

69 # _news was designed to be non-VOResource, but it turned out it's 70 # really useful to specify dates. So, contrary to our principle, there's 71 # an underscore-starting meta here, anyway. 72 if utils.dateRE.match(localattrs.get("date", "")): 73 return [VOR.date(role=localattrs.get("role", "updated"))[ 74 localattrs["date"]]]

75 76 77 _vrResourceBuilder = meta.ModelBasedBuilder([ 78 ('title', SF(VOR.title)), 79 ('shortName', SF(VOR.shortName)), 80 ('identifier', SF(VOR.identifier)), 81 ('doi', lambda args, localattrs=None: VOR.altIdentifier["doi:"+args[0]]), 82 (None, SF(VOR.curation), [ 83 ('publisher', SF(VOR.publisher), (), { 84 "ivoId": "ivoId"}), 85 ('creator', SF(VOR.creator), [ 86 ('name', SF(VOR.name)), 87 ('altIdentifier', SF(VOR.altIdentifier)), 88 ('logo', SF(VOR.logo)),]), 89 ('contributor', SF(VOR.contributor), (), { 90 "ivoId": "ivoId"}), 91 # We don't yet discriminate between updates to the RR and updates 92 # to the resource itself. We probably should, and most of 93 # what it takes is in place. IVOMetaMixin would be the place 94 # to add a "dataUpdated" meta or something like that. 95 ('datetimeUpdated', SF(VOR.date, role="updated")), 96 ('date', SF(VOR.date), (), { 97 "role": "role"}), 98 ('_news', _build_dateFromNews, (), { 99 "role": "role", 100 "date": "date"}), 101 ('version', SF(VOR.version)), 102 ('contact', SF(VOR.contact), [ 103 ('name', SF(VOR.name), (), { 104 "ivoId": "ivoId"}), 105 ('address', SF(VOR.address)), 106 ('email', SF(VOR.email)), 107 ('telephone', SF(VOR.telephone)),]),]), 108 (None, SF(VOR.content), [ 109 ('subject', SF(VOR.subject)), 110 ('description', SF(VOR.description)), 111 ('source', _build_source), 112 ('referenceURL', SF(VOR.referenceURL)), 113 ('type', SF(VOR.type)), 114 ('contentLevel', SF(VOR.contentLevel)),] + [ 115 # old-style relationship terms from VOResource 1.0 f 116 # plan: after a while, make the update mapping in here 117 (None, SF(lambda t=vorTerm: 118 VOR.relationship[VOR.relationshipType[t]]), [ 119 (metaName, SF(VOR.relatedResource), (), { 120 "ivoId": "ivoId"})]) 121 for metaName, vorTerm in [ 122 ("servedBy", "served-by"), 123 ("serviceFor", "service-for"), 124 ("derivedFrom", "dervived-from"), 125 ("relatedTo", "related-to"), 126 ("mirrorOf", "mirror-of"), 127 ("relatedTo", "related-to"), 128 ("uses", "Cites")] 129 ] + [ 130 # new-style relationship terms from VOResource 1.1 131 (None, SF(lambda term=term: 132 VOR.relationship[VOR.relationshipType[term]]), [ 133 (term[0].lower()+term[1:], SF(VOR.relatedResource), (), { 134 "ivoId": "ivoId"})]) 135 for term in [ 136 "Cites", 137 "IsSupplementTo", 138 "IsSupplementedBy", 139 "IsContinuedBy", 140 "Continues", 141 "IsNewVersionOf", 142 "IsPreviousVersionOf", 143 "IsPartOf", 144 "HasPart", 145 "IsSourceOf", 146 "IsDerivedFrom", 147 "IsIdenticalTo", 148 "IsServiceFor", 149 "IsServedBy"]]) 150 ]) 151 152 153 _dcBuilder = meta.ModelBasedBuilder([ 154 ('title', SF(DC.title)), 155 ('identifier', SF(DC.identifier)), 156 ('creator', None, [ 157 ('name', SF(DC.creator))]), 158 ('contributor', None, [ 159 ('name', SF(DC.contributor))]), 160 ('description', SF(DC.description)), 161 ('language', SF(DC.language)), 162 ('rights', SF(DC.rights), ()), 163 ('publisher', SF(DC.publisher)), 164 ]) 165 166 167 _oaiHeaderBuilder = meta.ModelBasedBuilder([ 168 ('identifier', SF(OAI.identifier)), 169 ('recTimestamp', SF(OAI.datestamp)), 170 ('sets', SF(OAI.setSpec))]) 171 172 173 _orgMetaBuilder = meta.ModelBasedBuilder([ 174 ('facility', SF(VOR.facility)),# XXX TODO: look up ivo-ids? 175 ('instrument', SF(VOR.instrument)), 176 ]) 177 178 179 _standardsMetaBuilder = meta.ModelBasedBuilder([ 180 ('endorsedVersion', SF(VSTD.endorsedVersion), (), { 181 'status': 'status', 182 'use': 'use'}), 183 ('deprecated', SF(VSTD.deprecated), ()), 184 ('key', SF(VSTD.key), [ 185 ('name', SF(VSTD.name), []), 186 ('description', SF(VSTD.description), [])])]) 187 188

189 -def _stcResourceProfile(metaValue, localattrs=None):

190 # This is a helper for the coverageMetaBuilder; it expects 191 # STC-S and will return an STC resource profile for literal 192 # embedding. 193 if not metaValue: 194 return None 195 try: 196 return stc.astToStan( 197 stc.parseSTCS(metaValue[0]), 198 stc.STC.STCResourceProfile) 199 except Exception as exc: 200 if VALIDATING: 201 raise 202 base.ui.notifyError("Coverage profile '%s' bad while generating " 203 " registry (%s). It is left out."%(metaValue, str(exc)))

204 205

206 -def _build_footprintURL(children, localattrs):

207 # a service that has a spatial coverage can spit out 208 # MOCs using the coverage renderer. 209 # 210 # Unfortunately, getting a URL from coverage information breaks 211 # our abstractions. I'd like to get rid of footprintURL anyway, 212 # so this is another case for stealing variables from upstack. 213 # It turns out the macroPackage in the enclosing frame actually 214 # is the service or table we're building for. And if that thing 215 # can build URLs, that's what we're looking for. If it can't 216 # it can't produce footprint MOCs in the first place, so we 217 # can return None. 218 carrier = utils.stealVar("macroPackage") 219 if isinstance(carrier, svcs.Service): 220 return VS.footprint(ivoId="ivo://ivoa.net/std/moc")[ 221 carrier.getURL("coverage")]

222 223 224 _coverageMetaBuilder = meta.ModelBasedBuilder([ 225 ('coverage', SF(VS.coverage), [ 226 ('profile', _stcResourceProfile), 227 ('spatial', SF(VS.spatial)), 228 ('temporal', SF(VS.temporal)), 229 ('spectral', SF(VS.spectral)), 230 # resources that have spatial coverage get a footprint url in 231 # addition to the spatial coverage at least for a while. 232 ('spatial', _build_footprintURL), 233 ('waveband', SF(VS.waveband)), 234 ('regionOfRegard', SF(VS.regionOfRegard)), 235 ])]) 236 237

238 -def getResourceArgs(resob):

239 """returns the mandatory attributes for constructing a Resource record 240 for service in a dictionary. 241 """ 242 return { 243 "created": base.getMetaText(resob, "creationDate", propagate=True), 244 "updated": base.getMetaText(resob, "datetimeUpdated", propagate=True), 245 "status": base.getMetaText(resob, "status"), 246 }

247 248

249 -def getOAIHeaderElementForRestup(restup):

250 if isinstance(restup, OAI.OAIElement): 251 return restup 252 status = None 253 if restup["deleted"]: 254 status = "deleted" 255 return OAI.header(status=status)[ 256 OAI.identifier[identifiers.computeIdentifierFromRestup(restup)], 257 OAI.datestamp[restup["recTimestamp"].strftime(utils.isoTimestampFmt)], 258 [ 259 OAI.setSpec[setName] 260 for setName in servicelist.getSetsForResource(restup)]]

261 262 263 ###################### Direct children of OAI.PMH 264

265 -def _getOAIURL(registryService):

266 """returns the OAI-PHM access URL for a registry service. 267 268 We don't want to just use getURL(pubreg) since the publication 269 may (and for the publishing registry does) have an accessURL meta. 270 """ 271 oaiAccessURL = registryService.getURL("pubreg.xml") 272 for pub in registryService.publications: 273 if pub.render=="pubreg.xml": 274 oaiAccessURL = base.getMetaText( 275 pub, "accessURL", macroPackage=pub.parent) 276 break 277 return oaiAccessURL

278 279

280 -def getIdentifyElement(registryService):

281 """returns OAI Identify stanxml. 282 283 registryService is the registry we're identifying, i.e. typically 284 __system__/services#registry 285 """ 286 return OAI.Identify[ 287 OAI.repositoryName[base.getMetaText(registryService, "title")], 288 OAI.baseURL[_getOAIURL(registryService)], 289 OAI.protocolVersion["2.0"], 290 OAI.adminEmail[base.getMetaText(registryService, "contact.email")], 291 OAI.earliestDatestamp["1970-01-01T00:00:00Z"], 292 OAI.deletedRecord["transient"], 293 OAI.granularity["YYYY-MM-DDThh:mm:ssZ"], 294 OAI.description[ 295 getVORMetadataElement(registryService), 296 ], 297 ]

298 299

300 -def getListIdentifiersElement(restups):

301 """returns an OAI ListIdentifiers element for the rec tuples recs. 302 """ 303 return OAI.ListIdentifiers[ 304 [getOAIHeaderElementForRestup(restup) for restup in restups], 305 ]

306 307

308 -def getListMetadataFormatsElement():

309 return OAI.ListMetadataFormats[[ 310 OAI.metadataFormat[ 311 OAI.metadataPrefix[prefix], 312 OAI.schema[schema], 313 OAI.metadataNamespace[ns], 314 ] for prefix, schema, ns in common.METADATA_PREFIXES] 315 ]

316 317

318 -def getListSetsElement():

319 return OAI.ListSets[[ 320 # XXX TODO: Add some kind of description, in particular when we define 321 # real local sets. 322 OAI.set[ 323 OAI.setSpec[set["setName"]], 324 OAI.setName[set["setName"]], 325 ] 326 for set in servicelist.getSets()]]

327 328

329 -def getResourceElement(resob, setNames, metadataMaker):

330 """helps get[VO|DC]ResourceElement. 331 """ 332 if isinstance(resob, OAI.OAIElement): 333 return resob 334 status = None 335 if base.getMetaText(resob, "status")=="deleted": 336 status = "deleted" 337 return OAI.record[ 338 OAI.header(status=status)[ 339 _oaiHeaderBuilder.build(resob)], 340 OAI.metadata[ 341 metadataMaker(resob, setNames) 342 ] 343 ]

344 345

346 -def getDCMetadataElement(resob, setNames):

347 return OAIDC.dc[_dcBuilder.build(resob)]

348 349

350 -def getDCResourceElement(resob, setNames=_defaultSet):

351 return getResourceElement(resob, setNames, getDCMetadataElement)

352 353

354 -def getDCListRecordsElement(resobs, setNames, 355 makeRecord=getDCResourceElement):

356 """returns stanxml for ListRecords in dublin core format. 357 358 resobs is a sequence of res objects. 359 makeRecord(resob, setNames) -> stanxml is a function that returns 360 an OAI.record element. For ivo_vor metadata prefixes, this is overridden. 361 by getVOListRecordsElement. 362 """ 363 recs = OAI.ListRecords() 364 for resob in resobs: 365 try: 366 recs[makeRecord(resob, setNames)] 367 except base.NoMetaKey as msg: 368 base.ui.notifyError("Cannot create registry record for %s#%s" 369 " because mandatory meta %s is missing"%( 370 resob.rd.sourceId, resob.id, msg)) 371 except Exception as msg: 372 base.ui.notifyError("Cannot create registry record %s. Reason: %s"%( 373 resob, msg)) 374 return recs

375 376

377 -def getDCGetRecordElement(resob):

378 return OAI.GetRecord[ 379 getDCResourceElement(resob)]

380 381 382 ################### VOResource metadata element creation 383

384 -class ResourceMaker(object):

385 """A base class for the generation of VOResource elements. 386 387 These have a resType attribute specifying which resource type 388 they work for. These types are computed by the getResourceType 389 helper function. 390 391 The makeResource function below tries the ResourceMakers in turn 392 for the "best" one that matches. 393 394 If you create new ResourceMakers, you will have to enter them 395 *in the correct sequence* in the _resourceMakers list below. 396 397 ResourceMaker instances are called with a resob argument and a set 398 of set names. You will want to override the _makeResource(resob) 399 -> xmlstan method and probably the resourceClass element. 400 """ 401 resourceClass = RI.Resource 402 resType = None 403

404 - def _loadDependencies(self, resob):

405 """loads all RDs dependent on resob.rd (if present). 406 407 The dependencies are taken from the dc.res_dependencies table. There, 408 they are typically introduced by served-by relationships (see also 409 service.declareServes. 410 """ 411 if not hasattr(resob.rd, "cached dependencies"): 412 deps = common.getDependencies(resob.rd.sourceId) 413 setattr(resob.rd, "cached dependencies", deps) 414 else: 415 deps = getattr(resob.rd, "cached dependencies") 416 for dep in deps: 417 base.caches.getRD(dep)

418 419

420 - def _makeResource(self, resob, setNames):

421 self._loadDependencies(resob) 422 res = self.resourceClass(**getResourceArgs(resob))[ 423 VOR.validationLevel(validatedBy=str(resob.getMeta("validatedBy")))[ 424 resob.getMeta("validationLevel")], 425 _vrResourceBuilder.build(resob),] 426 # Registry interface mandates ri:Resource (rather than, say, vr:Resource) 427 # even in OAI. No idea why, but let's just force it. 428 res._prefix = "ri" 429 return res

430

431 - def __call__(self, resob, setNames):

432 return self._makeResource(resob, setNames)

433 434 435 _rightsBuilder = meta.ModelBasedBuilder([ 436 ('rights', SF(VOR.rights), (), { 437 'rightsURI': 'rightsURI'}),]) 438 439

440 -class ServiceResourceMaker(ResourceMaker):

441 """A ResourceMaker adding rights and capabilities. 442 """ 443 resourceClass = VS.DataService 444 resType = "nonTabularService" 445

446 - def _makeResource(self, service, setNames):

447 res = ResourceMaker._makeResource(self, service, setNames)[ 448 _rightsBuilder.build(service)] 449 return res[[ 450 capabilities.getCapabilityElement(pub) 451 for pub in service.getPublicationsForSet(setNames)]]

452 453

454 -class DataServiceResourceMaker(ServiceResourceMaker):

455 """A ResourceMaker for DataServices. 456 457 These are services that may have instrument, facility, and coverage 458 metas but have no associated tables. This is not generated by the 459 service classifier currently since we always have a table. You can 460 force generation of such records via setMeta("resType", "dataService"). 461 """ 462 resourceClass = VS.DataService 463 resType = "dataService" 464

465 - def _makeResource(self, service, setNames):

466 return ServiceResourceMaker._makeResource(self, service, setNames)[ 467 _orgMetaBuilder.build(service), 468 _coverageMetaBuilder.build(service)]

469 470

471 -class CatalogServiceResourceMaker(DataServiceResourceMaker):

472 resourceClass = VS.CatalogService 473 resType = "catalogService"

474 - def _makeResource(self, service, setNames):

475 return DataServiceResourceMaker._makeResource(self, service, setNames)[ 476 tableset.getTablesetForService(service)]

477 478 479 _registryMetaBuilder = meta.ModelBasedBuilder([ 480 ('managedAuthority', SF(VOG.managedAuthority)),]) 481 482

483 -class RegistryResourceMaker(ServiceResourceMaker):

484 resourceClass = VOG.Resource 485 resType = "registry" 486

487 - def _makeResource(self, registry, setNames):

488 return ServiceResourceMaker._makeResource(self, registry, setNames) [ 489 VOG.full[base.getMetaText(registry, "full", "false")], 490 _registryMetaBuilder.build(registry)]

491 # not for now: let's see if we can get the schema changes tableset.getTablesetForService(registry)] 492 493

494 -class OrgResourceMaker(ResourceMaker):

495 resourceClass = VOR.Organisation 496 resType = "organization"

497 - def _makeResource(self, registry, setNames):

498 return ResourceMaker._makeResource(self, registry, setNames) [ 499 _orgMetaBuilder.build(registry)]

500 501

502 -class AuthResourceMaker(ResourceMaker):

503 resourceClass = VOG.Authority 504 resType = "authority"

505 - def _makeResource(self, registry, setNames):

506 return ResourceMaker._makeResource(self, registry, setNames) [ 507 VOG.managingOrg( 508 ivoId=base.getMetaText(registry, "managingOrg.ivo-id", default=None))[ 509 base.getMetaText(registry, "managingOrg")]]

510 511

512 -class StandardsResourceMaker(ResourceMaker):

513 resourceClass = VSTD.Standard 514 resType = "standard"

515 - def _makeResource(self, registry, setNames):

516 return ResourceMaker._makeResource(self, registry, setNames) [ 517 _standardsMetaBuilder.build(registry)]

518 519

520 -class DocResourceMaker(DataServiceResourceMaker):

521 resourceClass = DOC.Document 522 resType = "document"

523 524

525 -class DeletedResourceMaker(ResourceMaker):

526 resType = "deleted"

527 - def _makeResource(self, res, setNames):

528 return []

529 530

531 -class DataCollectionResourceMaker(ResourceMaker):

532 """A base class for Table- and DataResourceMaker. 533 """ 534 resourceClass = VS.CatalogResource 535

536 - def _makeTableset(self, schemas):

537 return tableset.getTablesetForSchemaCollection(schemas)

538

539 - def _makeResourceForSchemas(self, metaCarrier, schemas, setNames):

540 """returns xmlstan for schemas within metaCarrier. 541 542 metaCarrier has to provide all the VOR metadata. schemas is a 543 sequence of triples of (rd, tables); rd is used to define a 544 VODataService schema, tables is a sequence of TableDefs that 545 define the tables within that schema. 546 """ 547 res = ResourceMaker._makeResource(self, metaCarrier, setNames)[[ 548 capabilities.getCapabilityElement(pub) 549 for pub in metaCarrier.getPublicationsForSet(setNames)], 550 _orgMetaBuilder.build(metaCarrier), 551 _coverageMetaBuilder.build(metaCarrier), 552 self._makeTableset(schemas)] 553 return res

554 555

556 -class TableResourceMaker(DataCollectionResourceMaker):

557 """A ResourceMaker for rscdef.TableDef items (yielding reformed 558 DataCollections) 559 """ 560 resType = "table" 561

562 - def _makeResource(self, td, setNames):

563 return DataCollectionResourceMaker._makeResourceForSchemas( 564 self, td, [(td.rd, [td])], setNames)

565 566

567 -class DataResourceMaker(DataCollectionResourceMaker):

568 """A ResourceMaker for rscdef.DataDescriptor items (yielding reformed 569 DataCollections) 570 """ 571 resType = "data" 572

573 - def _makeResource(self, dd, setNames):

574 return DataCollectionResourceMaker._makeResourceForSchemas( 575 self, dd, [(dd.rd, set(dd.iterTableDefs()))], setNames)

576 577 578 _getResourceMaker = utils.buildClassResolver(ResourceMaker, 579 globals().values(), instances=True, 580 key=lambda obj: obj.resType) 581 582

583 -def getVORMetadataElement(resob, setNames=_defaultSet):

584 return _getResourceMaker(common.getResType(resob))(resob, setNames)

585 586

587 -def getVOResourceElement(resob, setNames=_defaultSet):

588 """returns a stanxml for Resource in VOR format. 589 590 There's trouble here in that we have set management on the level of 591 renderers (capabilities). Thus, to come up with capabilities for 592 a given ivorn, we have to know what set is queried. However, 593 OAI GetRecord doesn't specify sets. So, we provide a default 594 set of ivo_managed, assuming that the registry is only interested 595 in records actually VO-registred. This may fly into our face, 596 but I can't see a way around it given the way our services are 597 described. 598 """ 599 return getResourceElement(resob, setNames, getVORMetadataElement)

600 601

602 -def getVOListRecordsElement(resobs, setNames):

603 return getDCListRecordsElement(resobs, setNames, 604 getVOResourceElement)

605 606

607 -def getVOGetRecordElement(resob):

608 return OAI.GetRecord[ 609 getVOResourceElement(resob)]

610