gavo.rscdef.tabledef

1 """ 2 Description and definition of tables. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import functools 12 import re 13 14 from gavo import adql 15 from gavo import base 16 from gavo import dm 17 from gavo import stc 18 from gavo import utils 19 from gavo.rscdef import column 20 from gavo.rscdef import common 21 from gavo.rscdef import group 22 from gavo.rscdef import mixins 23 from gavo.rscdef import rmkfuncs 24 25 26 MS = base.makeStruct

27 28 29 @functools.total_ordering 30 -class DBIndex(base.Structure):

31 """A description of an index in the database. 32 33 In real databases, indices may be fairly complex things; still, the 34 most common usage here will be to just index a single column:: 35 36 <index columns="my_col"/> 37 38 To index over functions, use the character content; parentheses are added 39 by DaCHS, so don't have them in the content. An explicit specification 40 of the index expression is also necessary to allow RE pattern matches using 41 indices in character columns (outside of the C locale). That would be:: 42 43 <index columns="uri">uri text_pattern_ops</index> 44 45 (you still want to give columns so the metadata engine is aware of the 46 index). See section "Operator Classes and Operator Families" in 47 the Postgres documentation for details. 48 49 For pgsphere-valued columns, you at the time of writing need to specify 50 the method:: 51 52 <index columns="coverage" method="GIST"/> 53 54 To define q3c indices, use the ``//scs#q3cindex`` mixin; if you're 55 devious enough to require something more flexible, have a look at 56 that mixin's definition. 57 58 If indexed columns take part in a DaCHS-defined view, DaCHS will not 59 notice. You should still declare the indices so users will see them 60 in the metadata; writing:: 61 62 <index columns="col1, col2, col3"/> 63 64 is sufficent for that. 65 """ 66 name_ = "index" 67 68 _name = base.UnicodeAttribute("name", default=base.Undefined, 69 description="Name of the index. Defaults to something computed from" 70 " columns; the name of the parent table will be prepended in the DB." 71 " The default will *not* work if you have multiple indices on one" 72 " set of columns.", 73 copyable=True) 74 _columns = base.StringListAttribute("columns", description= 75 "Table columns taking part in the index (must be given even if there" 76 " is an expression building the index and mention all columns taking" 77 " part in the index generated by it", copyable=True) 78 _cluster = base.BooleanAttribute("cluster", default=False, 79 description="Cluster the table according to this index?", 80 copyable=True) 81 _code = base.DataContent(copyable=True, description= 82 "Raw SQL specifying an expression the table should be" 83 " indexed for. If not given, the expression will be generated from" 84 " columns (which is what you usually want).") 85 _method = base.UnicodeAttribute("method", default=None, 86 description="The indexing method, like an index type. In the 8.x," 87 " series of postgres, you need to set method=GIST for indices" 88 " over pgsphere columns; otherwise, you should not need to" 89 " worry about this.", copyable=True) 90

91 - def completeElement(self, ctx):

92 if self.content_ and getattr(ctx, "restricted", False): 93 raise base.RestrictedElement("index", hint="Free-form SQL on indices" 94 " is not allowed in restricted mode") 95 self._completeElementNext(DBIndex, ctx) 96 97 if not self.columns and not self.content_: 98 raise base.StructureError("Index without columns is verboten.") 99 100 if self.name is base.Undefined: 101 self.name = "%s"%(re.sub("[^\w]+", "_", "_".join(self.columns))) 102 103 if not self.content_: 104 self.content_ = "%s"%",".join(self.columns)

105 106 # indices are sorted by table; this should put them in an order 107 # optimal for importing. Right now, we only make sure indices 108 # with CLUSTER go as early as possible.

109 - def __eq__(self, other):

110 return id(self)==id(other)

111

112 - def __lt__(self, other):

113 othercluster = getattr(other, "cluster", None) 114 if ((self.cluster and othercluster) 115 or (not self.cluster and not othercluster)): 116 return id(self)<id(other) 117 else: 118 if self.cluster: 119 return True 120 else: 121 return False

122

123 - def iterCode(self):

124 destTableName = self.parent.getQName() 125 usingClause = "" 126 if self.method is not None: 127 usingClause = " USING %s"%self.method 128 yield self.parent.expand("CREATE INDEX %s ON %s%s (%s)"%( 129 self.dbname, destTableName, usingClause, self.content_)) 130 if self.cluster: 131 yield self.parent.expand( 132 "CLUSTER %s ON %s"%(self.dbname, destTableName))

133

134 - def create(self, querier):

135 """creates the index on the parent table if necessary. 136 137 querier is an object mixing in the DBMethodsMixin, usually the 138 DBTable object the index should be created on. 139 """ 140 if not querier.hasIndex(self.parent.getQName(), self.dbname): 141 if not self.parent.system: 142 base.ui.notifyIndexCreation( 143 self.parent.expand(self.dbname)) 144 for statement in self.iterCode(): 145 querier.query(statement)

146

147 - def drop(self, querier):

148 """drops the index if it exists. 149 150 querier is an object mixing in the DBMethodsMixin, usually the 151 DBTable object the index possibly exists on. 152 """ 153 iName = self.parent.expand(self.dbname) 154 if querier.hasIndex(self.parent.getQName(), iName): 155 querier.query("DROP INDEX %s.%s"%(self.parent.rd.schema, iName))

156 157 @property

158 - def dbname(self):

159 return "%s_%s"%(self.parent.id, self.name)

160

161 162 -class ColumnTupleAttribute(base.StringListAttribute):

163 """is a tuple of column names. 164 165 In a validate method, it checks that the names actually are in parent's 166 fields. 167 """

168 - def iterParentMethods(self):

169 """adds a getPrimaryIn method to the parent class. 170 171 This function will return the value of the primary key in a row 172 passed. The whole thing is a bit dense in that I want to compile 173 that method to avoid having to loop every time it is called. This 174 compilation is done in a descriptor -- ah well, probably it's a waste 175 of time anyway. 176 """ 177 def makeGetPrimaryFunction(instance): 178 funcSrc = ('def getPrimaryIn(row):\n' 179 ' return (%s)')%(" ".join(['row["%s"],'%name 180 for name in getattr(instance, self.name_)])) 181 return utils.compileFunction(funcSrc, "getPrimaryIn")

182 183 def getPrimaryIn(self, row): 184 try: 185 return self.__getPrimaryIn(row) 186 except AttributeError: 187 self.__getPrimaryIn = makeGetPrimaryFunction(self) 188 return self.__getPrimaryIn(row)

189 yield "getPrimaryIn", getPrimaryIn 190

191 - def validate(self, parent):

192 for colName in getattr(parent, self.name_): 193 try: 194 parent.getColumnByName(colName) 195 except base.NotFoundError: 196 raise base.ui.logOldExc(base.LiteralParseError(self.name_, colName, 197 hint="Column tuple component %s is not in parent table"%colName))

198

199 200 -class ForeignKey(base.Structure):

201 """A description of a foreign key relation between this table and another 202 one. 203 """ 204 name_ = "foreignKey" 205 206 _inTable = base.ReferenceAttribute("inTable", default=base.Undefined, 207 description="Reference to the table the foreign key points to.", 208 copyable=True) 209 _source = base.UnicodeAttribute("source", default=base.Undefined, 210 description="Comma-separated list of local columns corresponding" 211 " to the foreign key. No sanity checks are performed here.", 212 copyable=True) 213 _dest = base.UnicodeAttribute("dest", default=base.NotGiven, 214 description="Comma-separated list of columns in the target table" 215 " belonging to its key. No checks for their existence, uniqueness," 216 " etc. are done here. If not given, defaults to source.") 217 _metaOnly = base.BooleanAttribute("metaOnly", default=False, 218 description="Do not tell the database to actually create the foreign" 219 " key, just declare it in the metadata. This is for when you want" 220 " to document a relationship but don't want the DB to actually" 221 " enforce this. This is typically a wise thing to do when you have, say" 222 " a gigarecord of flux/density pairs and only several thousand metadata" 223 " records -- you may want to update the latter without having" 224 " to tear down the former.") 225

226 - def getDescription(self):

227 return "%s:%s -> %s:%s"%(self.parent.getQName(), ",".join(self.source), 228 self.destTableName, ".".join(self.dest))

229

230 - def _parseList(self, raw):

231 if isinstance(raw, list): 232 # we're being copied 233 return raw 234 return [s.strip() for s in raw.split(",") if s.strip()]

235

236 - def onElementComplete(self):

237 self.destTableName = self.inTable.getQName() 238 self.isADQLKey = self.inTable.adql and self.inTable.adql!='hidden' 239 240 self.source = self._parseList(self.source) 241 if self.dest is base.NotGiven: 242 self.dest = self.source 243 else: 244 self.dest = self._parseList(self.dest) 245 self._onElementCompleteNext(ForeignKey)

246

247 - def create(self, querier):

248 if self.metaOnly: 249 return 250 251 if not querier.foreignKeyExists(self.parent.getQName(), 252 self.destTableName, 253 self.source, 254 self.dest): 255 return querier.query("ALTER TABLE %s ADD FOREIGN KEY (%s)" 256 " REFERENCES %s (%s)" 257 " ON DELETE CASCADE" 258 " DEFERRABLE INITIALLY DEFERRED"%( 259 self.parent.getQName(), 260 ",".join(self.source), 261 self.destTableName, 262 ",".join(self.dest)))

263

264 - def delete(self, querier):

265 if self.metaOnly: 266 return 267 268 try: 269 constraintName = querier.getForeignKeyName(self.parent.getQName(), 270 self.destTableName, self.source, self.dest) 271 except (ValueError, base.DBError): # key does not exist. 272 return 273 querier.query("ALTER TABLE %s DROP CONSTRAINT %s"%(self.parent.getQName(), 274 constraintName))

275

276 - def getAnnotation(self, roleName, container, instance):

277 """returns a dm annotation for this foreign key. 278 """ 279 return dm.ForeignKeyAnnotation(roleName, self, instance)

280

281 282 -class STCDef(base.Structure):

283 """A definition of a space-time coordinate system using STC-S. 284 """ 285 # Programmatically, you have 286 # * compiled -- an AST of the entire specification 287 # * iterColTypes -- iterates over column name/utype pairs, for the 288 # embedding table only; all others must not touch it 289 290 name_ = "stc" 291 292 _source = base.DataContent(copyable=True, description="An STC-S string" 293 " with column references (using quote syntax) instead of values") 294

295 - def completeElement(self, ctx):

296 self._completeElementNext(STCDef, ctx) 297 try: 298 self.compiled = stc.parseQSTCS(self.content_) 299 except stc.STCSParseError as msg: 300 raise base.ui.logOldExc(base.StructureError( 301 "Bad stc definition: %s"%str(msg))) 302 self.compiled.stripUnits() 303 self._origFields = dict((value.dest, utype) 304 for utype, value in stc.getUtypes(self.compiled) 305 if isinstance(value, stc.ColRef))

306

307 - def iterColTypes(self):

308 return self._origFields.iteritems()

309

310 311 -class ADQLVisibilityAttribute(base.BooleanAttribute):

312 """An attribute that has values True/False and hidden. 313 """ 314 typeDesc_ = "boolean or 'hidden'" 315

316 - def feedObject(self, instance, value):

317 if value=='hidden': 318 instance._readProfiles.feed(None, instance, "defaults,untrustedquery") 319 value = False 320 base.BooleanAttribute.feedObject(self, instance, value)

321

322 - def parse(self, value):

323 if value.lower()=="hidden": 324 return "hidden" 325 return base.BooleanAttribute.parse(self, value)

326

327 - def unparse(self, value):

328 if value=="hidden": 329 return value 330 return base.BooleanAttribute.unparse(self, value)

331

332 333 -class PublishableDataMixin(object):

334 """A mixin with a few classes and attributes for data that can be 335 published to the VO registry. 336 337 In particular, this contains the publish element (registration attribute). 338 """ 339 _registration = base.StructAttribute("registration", 340 default=None, 341 childFactory=common.Registration, 342 copyable=False, 343 description="A registration (to the VO registry) of this table" 344 " or data collection.") 345

346 - def getPublicationsForSet(self, setNames):

347 """returns a sequence of publication elements for the data, suitable 348 for OAI responses for the sets setNames. 349 350 Essentially: if registration is None, or its sets don't match 351 setNames, return an emtpy sequence. 352 353 If the registration mentions services, we turn their publications 354 into auxiliary publications and yield them 355 356 Otherwise, if we're published for ADQL, return the TAP service 357 as an auxiliary publication. 358 """ 359 if (self.registration is None 360 or not self.registration.sets & setNames): 361 return 362 363 services = self.registration.services 364 if not services: 365 services = [base.resolveCrossId("//tap#run")] 366 367 for service in services: 368 for pub in service.getPublicationsForSet(setNames): 369 copy = pub.change(parent_=self, auxiliary=True) 370 copy.meta_ = self.registration.meta_ 371 yield copy

372

373 374 -class TableDef(base.Structure, base.ComputedMetaMixin, common.PrivilegesMixin, 375 common.IVOMetaMixin, base.StandardMacroMixin, PublishableDataMixin):

376 """A definition of a table, both on-disk and internal. 377 378 Some attributes are ignored for in-memory tables, e.g., roles or adql. 379 380 Properties for tables: 381 382 * supportsModel -- a short name of a data model supported through this 383 table (for TAPRegExt dataModel); you can give multiple names separated 384 by commas. 385 * supportsModelURI -- a URI of a data model supported through this table. 386 You can give multiple URIs separated by blanks. 387 388 If you give multiple data model names or URIs, the sequences of names and 389 URIs must be identical (in particular, each name needs a URI). 390 """ 391 name_ = "table" 392 393 resType = "table" 394 395 # We don't want to force people to come up with an id for all their 396 # internal tables but want to avoid writing default-named tables to 397 # the db. Thus, the default is not a valid sql identifier. 398 _id = base.IdAttribute("id", 399 default=base.NotGiven, 400 description="Name of the table (must be SQL-legal for onDisk tables)") 401 402 _cols = common.ColumnListAttribute("columns", 403 childFactory=column.Column, 404 description="Columns making up this table.", 405 copyable=True) 406 407 _params = common.ColumnListAttribute("params", 408 childFactory=column.Param, 409 description='Param ("global columns") for this table.', 410 copyable=True) 411 412 _viewStatement = base.UnicodeAttribute("viewStatement", 413 default=None, 414 description="A single SQL statement to create a view. Setting this" 415 " makes this table a view. The statement will typically be something" 416 " like CREATE VIEW \\\\qName AS (SELECT \\\\colNames FROM...).", 417 copyable=True) 418 419 # onDisk must not be copyable since queries might copy the tds and havoc 420 # would result if the queries were to end up on disk. 421 _onDisk = base.BooleanAttribute("onDisk", 422 default=False, 423 description="Table in the database rather than in memory?") 424 425 _temporary = base.BooleanAttribute("temporary", 426 default=False, 427 description="If this is an onDisk table, make it temporary?" 428 " This is mostly useful for custom cores and such.", 429 copyable=True) 430 431 _adql = ADQLVisibilityAttribute("adql", 432 default=False, 433 description="Should this table be available for ADQL queries? In" 434 " addition to True/False, this can also be 'hidden' for tables" 435 " readable from the TAP machinery but not published in the" 436 " metadata; this is useful for, e.g., tables contributing to a" 437 " published view. Warning: adql=hidden is incompatible with setting" 438 " readProfiles manually.") 439 440 _system = base.BooleanAttribute("system", 441 default=False, 442 description="Is this a system table? If it is, it will not be" 443 " dropped on normal imports, and accesses to it will not be logged.") 444 445 _forceUnique = base.BooleanAttribute("forceUnique", 446 default=False, 447 description="Enforce dupe policy for primary key (see dupePolicy)?") 448 449 _dupePolicy = base.EnumeratedUnicodeAttribute("dupePolicy", 450 default="check", 451 validValues=["check", "drop", "overwrite", "dropOld"], 452 description= "Handle duplicate rows with identical primary keys manually" 453 " by raising an error if existing and new rows are not identical (check)," 454 " dropping the new one (drop), updating the old one (overwrite), or" 455 " dropping the old one and inserting the new one (dropOld)?") 456 457 _primary = ColumnTupleAttribute("primary", 458 default=(), 459 description="Comma separated names of columns making up the primary key.", 460 copyable=True) 461 462 _indices = base.StructListAttribute("indices", 463 childFactory=DBIndex, 464 description="Indices defined on this table", 465 copyable=True) 466 467 _foreignKeys = base.StructListAttribute("foreignKeys", 468 childFactory=ForeignKey, 469 description="Foreign keys used in this table", 470 copyable=False) 471 472 _groups = base.StructListAttribute("groups", 473 childFactory=group.Group, 474 description="Groups for columns and params of this table", 475 copyable=True) 476 477 _nrows = base.IntAttribute("nrows", 478 description="Approximate number of rows in this table (usually," 479 " you want to use dachs limits to fill this out; write <nrows>0</nrows>" 480 " to enable that).") 481 482 # this actually induces an attribute annotations with the DM 483 # annotation instances 484 _annotations = dm.DataModelRolesAttribute() 485 486 _properties = base.PropertyAttribute() 487 488 # don't copy stc -- columns just keep the reference to the original 489 # stc on copy, and nothing should rely on column stc actually being 490 # defined in the parent tableDefs. 491 _stcs = base.StructListAttribute("stc", description="STC-S definitions" 492 " of coordinate systems.", childFactory=STCDef) 493 494 _rd = common.RDAttribute() 495 _mixins = mixins.MixinAttribute() 496 _original = base.OriginalAttribute() 497 _namePath = common.NamePathAttribute() 498 499 fixupFunction = None 500 501 metaModel = ("title(1), creationDate(1), description(1)," 502 "subject, referenceURL(1)") 503 504 @classmethod

505 - def fromColumns(cls, columns, **kwargs):

506 """returns a TableDef from a sequence of columns. 507 508 You can give additional constructor arguments. makeStruct is used 509 to build the instance, the mixin hack is applied. 510 511 Columns with identical names will be disambiguated. 512 """ 513 res = MS(cls, 514 columns=common.ColumnList(cls.disambiguateColumns(columns)), 515 **kwargs) 516 return res

517

518 - def __iter__(self):

519 return iter(self.columns)

520

521 - def __contains__(self, name):

522 try: 523 self.columns.getColumnByName(name) 524 except base.NotFoundError: 525 return False 526 return True

527

528 - def __repr__(self):

529 try: 530 return "<Table definition of %s>"%self.getQName() 531 except base.Error: 532 return "<Non-RD table %s>"%self.id

533

534 - def completeElement(self, ctx):

535 # we want a meta parent as soon as possible, and we always let it 536 # be our struct parent 537 if (not self.getMetaParent() 538 and self.parent 539 and hasattr(self.parent, "_getMeta")): 540 self.setMetaParent(self.parent) 541 542 # Make room for DM annotations (these are currently filled by 543 # gavo.dm.dmrd.DataModelRoles, but we might reconsider this) 544 self.annotations = [] 545 546 if self.viewStatement and getattr(ctx, "restricted", False): 547 raise base.RestrictedElement("table", hint="tables with" 548 " view creation statements are not allowed in restricted mode") 549 550 if self.registration and self.id is base.NotGiven: 551 raise base.StructureError("Published tables need an assigned id.") 552 if not self.id: 553 self._id.feed(ctx, self, utils.intToFunnyWord(id(self))) 554 555 # allow iterables to be passed in for columns and convert them 556 # to a ColumnList here 557 if not isinstance(self.columns, common.ColumnList): 558 self.columns = common.ColumnList(self.columns) 559 self._resolveSTC() 560 self._completeElementNext(TableDef, ctx) 561 self.columns.withinId = self.params.tableName = "table "+self.id

562

563 - def validate(self):

564 if self.id.upper() in adql.allReservedWords: 565 raise base.StructureError("Reserved word %s is not allowed as a table" 566 " name"%self.id) 567 self._validateNext(TableDef)

568

569 - def onElementComplete(self):

570 if self.adql: 571 self.readProfiles = (self.readProfiles | 572 base.getConfig("db", "adqlProfiles")) 573 self.dictKeys = [c.key for c in self] 574 575 self.indexedColumns = set() 576 for index in self.indices: 577 for col in index.columns: 578 if "\\" in col: 579 try: 580 self.indexedColumns.add(self.expand(col)) 581 except (base.Error, ValueError): # cannot expand yet, ignore 582 pass 583 else: 584 self.indexedColumns.add(col) 585 if self.primary: 586 self.indexedColumns |= set(self.primary) 587 588 self._defineFixupFunction() 589 590 self._onElementCompleteNext(TableDef) 591 592 if self.registration: 593 self.registration.register() 594 595 # if there's no DM annotation yet, there's still a chance that our 596 # columns and params brought some with them. Try that. 597 if not self.annotations: 598 self.updateAnnotationFromChildren() 599 600 # we sort the indices for better performance (and possibly, 601 # one day, so things work at all, if there's dependencies). 602 # Right now, this only sorts indices to be clustered to the front. 603 self.indices.sort()

604

605 - def getElementForName(self, name):

606 """returns the first of column and param having name name. 607 608 The function raises a NotFoundError if neiter column nor param with 609 name exists. 610 """ 611 try: 612 try: 613 return self.columns.getColumnByName(name) 614 except base.NotFoundError: 615 return self.params.getColumnByName(name) 616 except base.NotFoundError as ex: 617 ex.within = "table %s"%self.id 618 raise

619

620 - def _resolveSTC(self):

621 """adds STC related attributes to this tables' columns. 622 """ 623 for stcDef in self.stc: 624 for name, type in stcDef.iterColTypes(): 625 destCol = self.getColumnByName(name) 626 if destCol.stc is not None: 627 # don't warn -- this kind of annotation is done for the future, 628 # when we can handle it properly. 629 continue 630 # base.ui.notifyWarning("Column %s is referenced twice from STC" 631 # " in table %s is referenced twice in STC groups. This" 632 # " is currently not supported, the second reference is" 633 # " ignored."%(name, self.getQName())) 634 destCol.stc = stcDef.compiled 635 destCol.stcUtype = type

636

637 - def _defineFixupFunction(self):

638 """defines a function to fix up records from column's fixup attributes. 639 640 This will leave a fixupFunction attribute which will be None if 641 no fixups are defined. 642 """ 643 fixups = [] 644 for col in self: 645 if col.fixup is not None: 646 fixups.append((col.name, col.fixup)) 647 if fixups: 648 assignments = [] 649 for key, expr in fixups: 650 expr = expr.replace("___", "row['%s']"%key) 651 assignments.append(" row['%s'] = %s"%(key, expr)) 652 source = self.expand( 653 "def fixup(row):\n%s\n return row"%("\n".join(assignments))) 654 self.fixupFunction = rmkfuncs.makeProc("fixup", source, 655 "", None)

656

657 - def getQName(self):

658 if self.temporary: 659 return self.id 660 else: 661 if self.rd is None: 662 raise base.Error("TableDefs without resource descriptor" 663 " have no qualified names") 664 return "%s.%s"%(self.rd.schema, self.id)

665

666 - def validateRow(self, row):

667 """checks that row is complete and complies with all known constraints on 668 the columns 669 670 The function raises a ValidationError with an appropriate message 671 and the relevant field if not. 672 """ 673 for col in self: 674 if col.key not in row: 675 raise base.ValidationError("Column %s missing"%col.name, 676 col.name, row, hint="The table %s has a column named '%s'," 677 " but the input row %s does not give it. This typically means" 678 " bad input or a rowmaker failing on some corner case."%( 679 self.id, col.name, row)) 680 try: 681 col.validateValue(row[col.key]) 682 except base.ValidationError as ex: 683 ex.row = row 684 raise

685

686 - def getFieldIndex(self, fieldName):

687 """returns the index of the field named fieldName. 688 """ 689 return self.columns.getFieldIndex(fieldName)

690

691 - def getParamByName(self, name):

692 return self.params.getColumnByName(name)

693

694 - def getColumnByName(self, name):

695 return self.columns.getColumnByName(name)

696

697 - def getColumnById(self, id):

698 return self.columns.getColumnById(id)

699

700 - def getColumnsByUCD(self, ucd):

701 return self.columns.getColumnsByUCD(ucd)

702

703 - def getColumnByUCD(self, ucd):

704 return self.columns.getColumnByUCD(ucd)

705

706 - def getColumnByUCDs(self, *ucds):

707 return self.columns.getColumnByUCDs(*ucds)

708

709 - def getColumnsByUCDs(self, *ucds):

710 res = [] 711 for ucd in ucds: 712 res.extend(self.columns.getColumnsByUCD(ucd)) 713 return res

714

715 - def getByUtype(self, utype):

716 """returns the column or param with utype. 717 718 This is supposed to be unique, but the function will just return 719 the first matching item it finds. 720 """ 721 try: 722 return self.params.getColumnByUtype(utype) 723 except base.NotFoundError: 724 return self.columns.getColumnByUtype(utype)

725

726 - def getByUtypes(self, *utypes):

727 """returns the first param or column matching the first utype 728 matching anything. 729 """ 730 for utype in utypes: 731 try: 732 return self.getByUtype(utype) 733 except base.NotFoundError: 734 pass 735 raise base.NotFoundError(", ".join(utypes), 736 what="param or column with utype in", 737 within="table %s"%self.id)

738

739 - def getByName(self, name):

740 """returns the column or param with name. 741 742 There is nothing keeping you from having both a column and a param with 743 the same name. If that happens, you will only see the column. But 744 don't do it. 745 """ 746 try: 747 return self.columns.getColumnByName(name) 748 except base.NotFoundError: 749 return self.params.getColumnByName(name)

750

751 - def makeRowFromTuple(self, dbTuple):

752 """returns a row (dict) from a row as returned from the database. 753 """ 754 preRes = dict(zip(self.dictKeys, dbTuple)) 755 if self.fixupFunction: 756 return self.fixupFunction(preRes) 757 return preRes

758

759 - def getDefaults(self):

760 """returns a mapping from column names to defaults to be used when 761 making a row for this table. 762 """ 763 defaults = {} 764 for col in self: 765 if col.values: 766 defaults[col.name] = col.values.default 767 elif not col.required: 768 defaults[col.name] = None 769 return defaults

770

771 - def getSTCDefs(self):

772 """returns a set of all STC specs referenced in this table as ASTs. 773 """ 774 # Do not use our stc attribute -- the columns may come from different 775 # tables and carry stc from there. 776 stcObjects = utils.uniqueItems(col.stc for col in self) 777 if None in stcObjects: 778 stcObjects.remove(None) 779 return stcObjects

780

781 - def getNote(self, noteTag):

782 """returns the table note meta value for noteTag. 783 784 This will raise a NotFoundError if we don't have such a note. 785 786 You will not usually use this to retrieve meta items since columns 787 have the meta values in their note attributes. Columns, of course, 788 use this to get their note attribute value. 789 """ 790 mi = self.getMeta("note") or [] 791 for mv in mi: 792 if mv.tag==noteTag: 793 return mv 794 else: 795 raise base.NotFoundError(noteTag, what="note tag", 796 within="table %s"%self.id)

797

798 - def getURL(self, rendName, absolute=True):

799 """returns the URL DaCHS will show the table info page for this table 800 under. 801 802 Of course the URL is only valid for imported tables. 803 """ 804 basePath = "%stableinfo/%s"%( 805 base.getConfig("web", "nevowRoot"), 806 self.getQName()) 807 if absolute: 808 basePath = base.makeAbsoluteURL(basePath) 809 return basePath

810

811 - def getDDL(self):

812 """returns an SQL statement that creates the table. 813 """ 814 preTable = "" 815 if self.temporary: 816 preTable = "TEMP " 817 statement = "CREATE %sTABLE %s (%s)"%( 818 preTable, 819 self.getQName(), 820 ", ".join(column.getDDL() for column in self)) 821 return statement

822

823 - def getSimpleQuery(self, 824 selectClause=None, 825 fragments="", 826 postfix=""):

827 """returns a query against this table. 828 829 selectClause is a list of column names (in which case the names 830 are validated against the real column names and you can use 831 user input) or a literal string (in which case you must not provide 832 user input or have a SQL injection hole). 833 834 fragments (the WHERE CLAUSE) and postfix are taken as literal strings (so 835 they must not contain user input). 836 837 This is purely a string operation, so you'll have your normal 838 value references in fragments and postfix, and should maintain 839 the parameter dictionaries as usual. 840 841 All parts are optional, defaulting to pulling the entire table. 842 """ 843 parts = ["SELECT"] 844 845 if selectClause is None: 846 parts.append("*") 847 elif isinstance(selectClause, list): 848 parts.append(", ".join( 849 self.getColumnByName(colName).name for colName in selectClause)) 850 else: 851 parts.append(selectClause) 852 853 parts.append("FROM %s"%self.getQName()) 854 855 if fragments: 856 parts.append("WHERE %s"%fragments) 857 858 if postfix: 859 parts.append(postfix) 860 861 return " ".join(parts)

862 863 @property

864 - def caseFixer(self):

865 return dict((col.name.lower(), col.name) for col in self)

866

867 - def doSimpleQuery(self, 868 selectClause=None, 869 fragments="", 870 params=None, 871 postfix=""):

872 """runs a query generated via getSimpleQuery and returns a list 873 of rowdicts. 874 875 This uses a table connection and queryToDicts; the keys in the 876 dictionaries will have the right case for this table's columns, though. 877 878 params is a dictionary of fillers for fragments and postfix. 879 """ 880 with base.getTableConn() as conn: 881 return list( 882 conn.queryToDicts( 883 self.getSimpleQuery( 884 selectClause, 885 fragments, 886 postfix), 887 params, 888 caseFixer=self.caseFixer))

889

890 - def macro_colNames(self):

891 """returns an SQL-ready list of column names of this table. 892 """ 893 return ", ".join(c.name for c in self.columns)

894

895 - def macro_curtable(self):

896 """returns the qualified name of the current table. 897 898 (this is identical to the `macro qName`_, which you should prefer 899 in new RDs.) 900 """ 901 return self.getQName()

902

903 - def macro_qName(self):

904 """returns the qualified name of the current table. 905 """ 906 return self.getQName()

907

908 - def macro_tablename(self):

909 """returns the unqualified name of the current table. 910 911 In most contexts, you will probably need to use the `macro qName`_ 912 instead of this. 913 """ 914 return self.id

915

916 - def macro_nameForUCD(self, ucd):

917 """returns the (unique!) name of the field having ucd in this table. 918 919 If there is no or more than one field with the ucd in this table, 920 we raise a ValueError. 921 """ 922 return self.getColumnByUCD(ucd).name

923

924 - def macro_nameForUCDs(self, ucds):

925 """returns the (unique!) name of the field having one 926 of ucds in this table. 927 928 Ucds is a selection of ucds separated by vertical bars 929 (|). The rules for when this raises errors are so crazy 930 you don't want to think about them. This really is 931 only intended for cases where "old" and "new" standards 932 are to be supported, like with pos.eq.*;meta.main and 933 POS_EQ_*_MAIN. 934 935 If there is no or more than one field with the ucd in 936 this table, we raise an exception. 937 """ 938 return self.getColumnByUCDs(*(s.strip() for s in ucds.split("|"))).name

939

940 - def macro_getParam(self, parName, default=""):

941 """returns the string representation of the parameter parName. 942 943 This is the parameter as given in the table definition. Any changes 944 to an instance are not reflected here. 945 946 If the parameter named does not exist, an empty string is returned. 947 NULLs/Nones are rendered as NULL; this is mainly a convenience 948 for obscore-like applications and should not be exploited otherwise, 949 since it's ugly and might change at some point. 950 951 If a default is given, it will be returned for both NULL and non-existing 952 params. 953 """ 954 try: 955 param = self.params.getColumnByName(parName) 956 except base.NotFoundError: 957 return default 958 if param.content_ is base.NotGiven or param.value is None: 959 return default or "NULL" 960 else: 961 return param.content_

962 963 @staticmethod

964 - def disambiguateColumns(columns):

965 """returns a sequence of columns without duplicate names. 966 """ 967 newColumns, seenNames = [], set() 968 for c in columns: 969 while c.name in seenNames: 970 c.name = c.name+"_" 971 newColumns.append(c) 972 seenNames.add(c.name) 973 return newColumns

974

975 - def _meta_howtociteLink(self):

976 """returns a link to a how-to-cite page for this service as an URL 977 meta. 978 """ 979 if self.onDisk: 980 # we assume we're sufficiently long-lived to merit a tableinfo if 981 # we're on disk 982 return base.META_CLASSES_FOR_KEYS["_related"]( 983 self.getURL(None, True), 984 title="Advice on citing this resource")

985

986 - def _meta_referenceURL(self):

987 """returns a link to the table-info page. 988 """ 989 return base.META_CLASSES_FOR_KEYS["_related"]( 990 self.getURL(None, True), 991 title="Table information")

992

993 994 -def makeTDForColumns(name, cols, **moreKWs):

995 """returns a TableDef object named names and having the columns cols. 996 997 cols is some sequence of Column objects. You can give arbitrary 998 table attributes in keyword arguments. 999 """ 1000 kws = {"id": name, "columns": common.ColumnList(cols)} 1001 kws.update(moreKWs) 1002 return base.makeStruct(TableDef, **kws)

1003

Source Code for Module gavo.rscdef.tabledef