1 """
2 Some standard cores for services.
3
4 A core receives and "input table" (usually just containing the query
5 parameters in the doc rec) and returns an output data set (which is promoted
6 to a SvcResult in the service before being handed over to the renderer).
7
8 These then have to be picked up by renderers and formatted for delivery.
9 """
10
11
12
13
14
15
16
17 import sys
18
19 from gavo import base
20 from gavo import rsc
21 from gavo import rscdef
22 from gavo.base import sqlsupport
23 from gavo.svcs import core
24 from gavo.svcs import inputdef
25 from gavo.svcs import outputdef
26
27
28 -class Error(base.Error):
30
31
32 MS = base.makeStruct
36 """A procedure application for generating SQL expressions from input keys.
37
38 PhraseMaker code must *yield* SQL fragments that can occur in WHERE
39 clauses, i.e., boolean expressions (thus, they must be generator
40 bodies). The clauses yielded by a single condDesc are combined
41 with the joiner set in the containing CondDesc (default=OR).
42
43 The following names are available to them:
44
45 - inputKeys -- the list of input keys for the parent CondDesc
46 - inPars -- a dictionary mapping inputKey names to the values
47 provided by the user
48 - outPars -- a dictionary that is later used as the parameter
49 dictionary to the query.
50 - core -- the core to which this phrase maker's condDesc belongs
51
52 To get the standard SQL a single key would generate, say::
53
54 yield base.getSQLForField(inputKeys[0], inPars, outPars)
55
56 To insert some value into outPars, do not simply use some key into
57 outParse, since, e.g., the condDesc might be used multiple times.
58 Instead, use getSQLKey, maybe like this::
59
60 ik = inputKeys[0]
61 yield "%s BETWEEN %%(%s)s AND %%(%s)s"%(ik.name,
62 base.getSQLKey(ik.name, inPars[ik.name]-10, outPars),
63 base.getSQLKey(ik.name, inPars[ik.name]+10, outPars))
64
65 getSQLKey will make sure unique names in outPars are chosen and
66 enters the values there.
67 """
68 name_ = "phraseMaker"
69
70 requiredType = "phraseMaker"
71 formalArgs = "self, inputKeys, inPars, outPars, core"
72
75 """A query specification for cores talking to the database.
76
77 CondDescs define inputs as a sequence of InputKeys (see `Element InputKey`_).
78 Internally, the values in the InputKeys can be translated to SQL.
79 """
80 name_ = "condDesc"
81
82 _inputKeys = rscdef.ColumnListAttribute("inputKeys",
83 childFactory=inputdef.InputKey,
84 description="One or more InputKeys defining the condition's input.",
85 copyable=True)
86
87 _silent = base.BooleanAttribute("silent",
88 default=False,
89 description="Do not produce SQL from this CondDesc. This"
90 " can be used to convey meta information to the core. However,"
91 " in general, a service is a more appropriate place to deal with"
92 " such information, and thus you should prefer service InputKeys"
93 " to silent CondDescs.",
94 copyable=True)
95
96 _required = base.BooleanAttribute("required",
97 default=False,
98 description="Reject queries not filling the InputKeys of this CondDesc",
99 copyable=True)
100
101 _fixedSQL = base.UnicodeAttribute("fixedSQL",
102 default=None,
103 description="Always insert this SQL statement into the query. Deprecated.",
104 copyable=True)
105
106 _buildFrom = base.ReferenceAttribute("buildFrom",
107 description="A reference to a column or an InputKey to define"
108 " this CondDesc",
109 default=None)
110
111 _phraseMaker = base.StructAttribute("phraseMaker",
112 default=None,
113 description="Code to generate custom SQL from the input keys",
114 childFactory=PhraseMaker,
115 copyable=True)
116
117 _combining = base.BooleanAttribute("combining",
118 default=False,
119 description="Allow some input keys to be missing when others are given?"
120 " (you want this for pseudo-condDescs just collecting random input"
121 " keys)",
122 copyable="True")
123
124 _group = base.StructAttribute("group",
125 default=None,
126 childFactory=rscdef.Group,
127 description="Group child input keys in the input table (primarily"
128 " interesting for web forms, where this grouping is shown graphically;"
129 " Set the style property to compact to have a one-line group there)")
130
131 _joiner = base.UnicodeAttribute("joiner",
132 default="OR",
133 description="When yielding multiple fragments, join them"
134 " using this operator (probably the only thing besides OR is"
135 " AND).",
136 copyable=True)
137
138 _original = base.OriginalAttribute()
139
145
147 return "<CondDesc %s>"%",".join(ik.name for ik in self.inputKeys)
148
149 @classmethod
152
153 @classmethod
156
157 @property
159 """returns some key for uniqueness of condDescs.
160 """
161
162
163
164
165 return "+".join([f.name for f in self.inputKeys])
166
173
174 - def expand(self, *args, **kwargs):
175 """hands macro expansion requests (from phraseMakers) upwards.
176
177 This is to the queried table if the parent has one (i.e., we're
178 part of a core), or to the RD if not (i.e., we're defined within
179 an rd).
180 """
181 if hasattr(self.parent, "queriedTable"):
182 return self.parent.queriedTable.expand(*args, **kwargs)
183 else:
184 return self.parent.rd.expand(*args, **kwargs)
185
191
192
193
194
195
197 try:
198 return self.__compiledPhraseMaker
199 except AttributeError:
200 if self.phraseMaker is not None:
201 val = self.phraseMaker.compile()
202 else:
203 val = self._makePhraseDefault
204 self.__compiledPhraseMaker = val
205 return self.__compiledPhraseMaker
206 makePhrase = property(_getPhraseMaker)
207
209 """returns True if the dict inPars contains input to all our input keys.
210 """
211 for f in self.inputKeys:
212 if f.name not in inPars:
213 return False
214 return True
215
248
249 - def asSQL(self, inPars, sqlPars, queryMeta):
258
260 """returns a changed version of self if renderer suggests such a
261 change.
262
263 This only happens if buildFrom is non-None. The method must
264 return a "defused" version that has buildFrom None (or self,
265 which will do because core.adaptForRenderer stops adapting if
266 the condDescs are stable).
267
268 The adaptors may also raise a Replace exception and return a
269 full CondDesc; this is done, e.g., for spoints for the form
270 renderer, since they need two input keys and a completely modified
271 phrase.
272 """
273 if not self.buildFrom:
274 return self
275 adaptor = inputdef.getRendererAdaptor(renderer)
276 if adaptor is None:
277 return self
278
279 try:
280 newInputKeys = []
281 for ik in self.inputKeys:
282 newInputKeys.append(adaptor(ik))
283 if self.inputKeys==newInputKeys:
284 return self
285 else:
286 return self.change(inputKeys=newInputKeys, buildFrom=None)
287 except base.Replace as ex:
288 return ex.newOb
289
292 """translates exception into something we can display properly.
293 """
294
295
296 if getattr(excValue, "cursor", None) is not None:
297 base.ui.notifyWarning("Failed DB query: %s"%excValue.cursor.query)
298 if isinstance(excValue, sqlsupport.QueryCanceledError):
299 message = "Query timed out (took too long).\n"
300 if base.getConfig("maintainerAddress"):
301 message = message+("Unless you know why the query took that"
302 " long, please contact %s. Otherwise, use TAP's async mode.\n"%
303 base.getConfig("maintainerAddress"))
304 message += ("Meanwhile, if this failure happened with a cross match,"
305 " please try exchanging the large and the small catalog in POINT"
306 " and CIRCLE.\n")
307 raise base.ui.logOldExc(base.ValidationError(message, "query"))
308 elif isinstance(excValue, base.NotFoundError):
309 raise base.ui.logOldExc(base.ValidationError("Could not locate %s '%s'"%(
310 excValue.what, excValue.lookedFor), "query"))
311 elif isinstance(excValue, base.DBError):
312 raise base.ui.logOldExc(base.ValidationError(unicode(excValue), "query"))
313 else:
314 raise
315
318 """A core knowing a DB table it operates on and allowing the definition
319 of condDescs.
320 """
321 _queriedTable = base.ReferenceAttribute("queriedTable",
322 default=base.Undefined, description="A reference to the table"
323 " this core queries.", copyable=True, callbacks=["_fixNamePath"])
324 _condDescs = base.StructListAttribute("condDescs", childFactory=CondDesc,
325 description="Descriptions of the SQL and input generating entities"
326 " for this core; if not given, they will be generated from the"
327 " table columns.", copyable=True)
328 _namePath = rscdef.NamePathAttribute(description="Id of an element"
329 " that will be used to located names in id references. Defaults"
330 " to the queriedTable's id.")
331
345
367
369
370 if self.namePath is None:
371 self.namePath = qTable.getFullId()
372
374 """returns a where fragment and the appropriate parameters
375 for the query defined by inputTable and queryMeta.
376 """
377 sqlPars = {}
378 return base.joinOperatorExpr("AND",
379 [cd.asSQL(inputTable.args, sqlPars, queryMeta)
380 for cd in self.condDescs]), sqlPars
381
383 """furnishes resultTable with a DALI-compatible _queryStatus meta item.
384
385 This will also add a _warning meta in case of overflows.
386 """
387 isOverflowed = len(resultTable.rows)>=queryMeta.get("dbLimit", 1e10)
388 if isOverflowed:
389 del resultTable.rows[-1]
390 queryMeta["Matched"] = len(resultTable.rows)
391 if isOverflowed:
392 resultTable.addMeta("_warning",
393 "The query limit was reached. Increase it"
394 " to retrieve more matches. Note that unsorted truncated queries"
395 " are not reproducible (i.e., might return a different result set"
396 " at a later time).")
397 resultTable.setMeta("_queryStatus", "OVERFLOW")
398 else:
399 resultTable.setMeta("_queryStatus", "OK")
400
402 """returns a core tailored to renderer renderers.
403
404 This mainly means asking the condDescs to build themselves for
405 a certain renderer. If no polymorphuous condDescs are there,
406 self is returned.
407 """
408 newCondDescs = []
409 for cd in self.condDescs:
410 newCondDescs.append(cd.adaptForRenderer(renderer))
411 if newCondDescs!=self.condDescs:
412 return self.change(condDescs=newCondDescs, inputTable=base.NotGiven
413 ).adaptForRenderer(renderer)
414 else:
415 return core.Core.adaptForRenderer(self, renderer)
416
419 """A core executing a pre-specified query with fancy conditions.
420
421 Unless you select \*, you *must* define the outputTable here;
422 Weird things will happen if you don't.
423
424 The queriedTable attribute is ignored.
425 """
426 name_ = "fancyQueryCore"
427
428 _query = base.UnicodeAttribute("query", description="The query to"
429 " execute. It must contain exactly one %s where the generated"
430 " where clause is to be inserted. Do not write WHERE yourself."
431 " All other percents must be escaped by doubling them.",
432 default=base.Undefined,
433 copyable=True)
434 _timeout = base.FloatAttribute("timeout", default=5.,
435 description="Seconds until the query is aborted",
436 copyable=True)
437
438 - def run(self, service, inputTable, queryMeta):
458
459
460 -class DBCore(TableBasedCore):
461 """A core performing database queries on one table or view.
462
463 DBCores ask the service for the desired output schema and adapt their
464 output. The DBCore's output table, on the other hand, lists all fields
465 available from the queried table.
466 """
467 name_ = "dbCore"
468
469 _sortKey = base.UnicodeAttribute("sortKey",
470 description="A pre-defined sort order (suppresses DB options widget)."
471 " The sort key accepts multiple columns, separated by commas.",
472 copyable=True)
473 _limit = base.IntAttribute("limit", description="A pre-defined"
474 " match limit (suppresses DB options widget).", copyable=True)
475 _distinct = base.BooleanAttribute("distinct", description="Add a"
476 " 'distinct' modifier to the query?", default=False, copyable=True)
477 _groupBy = base.UnicodeAttribute("groupBy", description=
478 "A group by clause. You shouldn't generally need this, and if"
479 " you use it, you must give an outputTable to your core.",
480 default=None)
481
484
486 """returns the fields we need in the output table.
487
488 The normal DbBased core just returns whatever the service wants.
489 Derived cores, e.g., for special protocols, could override this
490 to make sure they have some fields in the result they depend on.
491 """
492 return service.getCurOutputFields(queryMeta)
493
494 - def _runQuery(self, resultTableDef, fragment, pars, queryMeta,
495 **kwargs):
496 with base.getTableConn() as conn:
497 queriedTable = rsc.TableForDef(self.queriedTable, nometa=True,
498 create=False, connection=conn)
499 queriedTable.setTimeout(queryMeta["timeout"])
500
501 if fragment and pars:
502 resultTableDef.addMeta("info", repr(pars),
503 infoName="queryPars", infoValue=fragment)
504
505 iqArgs = {"limits": queryMeta.asSQL(), "distinct": self.distinct,
506 "groupBy": self.groupBy}
507 iqArgs.update(kwargs)
508
509 try:
510 try:
511 res = queriedTable.getTableForQuery(
512 resultTableDef, fragment, pars, **iqArgs)
513 except:
514 mapDBErrors(*sys.exc_info())
515 finally:
516 queriedTable.close()
517
518 self._addQueryStatus(res, queryMeta)
519 return res
520
528
529 - def run(self, service, inputTable, queryMeta):
530 """does the DB query and returns an InMemoryTable containing
531 the result.
532 """
533 resultTableDef = self._makeResultTableDef(
534 service, inputTable, queryMeta)
535
536 resultTableDef.copyMetaFrom(self.queriedTable)
537 if not resultTableDef.columns:
538 raise base.ValidationError("No output columns with these settings."
539 "_OUTPUT")
540
541 sortKeys = None
542 if self.sortKey:
543 sortKeys = self.sortKey.split(",")
544
545 queryMeta.overrideDbOptions(limit=self.limit, sortKeys=sortKeys,
546 sortFallback=self.getProperty("defaultSortKey", None))
547 try:
548 fragment, pars = self._getSQLWhere(inputTable, queryMeta)
549 except base.LiteralParseError as ex:
550 raise base.ui.logOldExc(base.ValidationError(str(ex),
551 colName=ex.attName))
552 queryMeta["sqlQueryPars"] = pars
553 return self._runQuery(resultTableDef, fragment, pars, queryMeta)
554
557 """A core executing a predefined query.
558
559 This usually is not what you want, unless you want to expose the current
560 results of a specific query, e.g., for log or event data.
561 """
562 name_ = "fixedQueryCore"
563
564 _timeout = base.FloatAttribute("timeout", default=15.,
565 description="Seconds until the query is aborted", copyable=True)
566 _query = base.UnicodeAttribute("query", default=base.Undefined,
567 description="The query to be executed. You must define the"
568 " output fields in the core's output table. The query will"
569 " be macro-expanded in the resource descriptor.")
570 _writable = base.BooleanAttribute("writable", default=False,
571 description="Use a writable DB connection?")
572
577
578 - def run(self, service, inputTable, queryMeta):
598
601 """A core always returning None.
602
603 This core will not work with the common renderers. It is really
604 intended to go with coreless services (i.e. those in which the
605 renderer computes everthing itself and never calls service.runX).
606 As an example, the external renderer could go with this.
607 """
608
609 name_ = "nullCore"
610
611 inputTableXML = "<inputTable/>"
612 outputTableXML = "<outputTable/>"
613
614 - def run(self, service, inputTable, queryMeta):
616