Source Code for Module gavo.rscdef.scripting

  1  """ 
  2  Support code for attaching scripts to objects. 
  3   
  4  Scripts can be either in python or in SQL.  They always live on 
  5  make instances.  For details, see Scripting in the reference 
  6  documentation. 
  7  """ 
  8   
  9  #c Copyright 2008-2019, the GAVO project 
 10  #c 
 11  #c This program is free software, covered by the GNU GPL.  See the 
 12  #c COPYING file in the source distribution. 
 13   
 14   
 15  from pyparsing import ( 
 16          OneOrMore, ZeroOrMore, QuotedString, Forward, 
 17          SkipTo, StringEnd, Regex, Suppress, 
 18          Literal) 
 19   
 20  from gavo import base 
 21  from gavo import utils 
 22  from gavo.base import sqlsupport 
 23  from gavo.rscdef import rmkfuncs 
 24   
 25   
 26 -class Error(base.Error): 
 27          pass 
 28   
 29   
 30 -def _getSQLScriptGrammar(): 
 31          """returns a pyparsing ParserElement that splits SQL scripts into 
 32          individual commands. 
 33   
 34          The rules are: Statements are separated by semicolons, empty statements 
 35          are allowed. 
 36          """ 
 37          with utils.pyparsingWhitechars(" \t"): 
 38                  atom = Forward() 
 39                  atom.setName("Atom") 
 40   
 41                  sqlComment = Literal("--")+SkipTo("\n", include=True) 
 42                  cStyleComment = Literal("/*")+SkipTo("*/", include=True) 
 43                  comment = sqlComment | cStyleComment 
 44                  lineEnd = Literal("\n") 
 45   
 46                  simpleStr = QuotedString(quoteChar="'", escChar="\\",  
 47                          multiline=True, unquoteResults=False) 
 48                  quotedId = QuotedString(quoteChar='"', escChar="\\", unquoteResults=False) 
 49                  dollarQuoted = Regex(r"(?s)\$(\w*)\$.*?\$\1\$") 
 50                  dollarQuoted.setName("dollarQuoted") 
 51                  # well, quotedId is not exactly a string literal.  I hate it, and so 
 52                  # it's lumped in here. 
 53                  strLiteral = simpleStr | dollarQuoted | quotedId 
 54                  strLiteral.setName("strLiteral") 
 55   
 56                  other = Regex("[^;'\"$]+") 
 57                  other.setName("other") 
 58   
 59                  literalDollar = Literal("$") + ~ Literal("$") 
 60                  statementEnd = ( Literal(';') + ZeroOrMore(lineEnd) | StringEnd() ) 
 61   
 62                  atom <<  ( Suppress(comment) | other | strLiteral | literalDollar ) 
 63                  statement = OneOrMore(atom) + Suppress( statementEnd ) 
 64                  statement.setName("statement") 
 65                  statement.setParseAction(lambda s, p, toks: " ".join(toks)) 
 66   
 67                  script = OneOrMore( statement ) + StringEnd() 
 68                  script.setName("script") 
 69                  script.setParseAction(lambda s, p, toks: [t for t in toks.asList() 
 70                          if str(t).strip()]) 
 71   
 72                  if False: 
 73                          atom.setDebug(True) 
 74                          comment.setDebug(True) 
 75                          other.setDebug(True) 
 76                          strLiteral.setDebug(True) 
 77                          statement.setDebug(True) 
 78                          statementEnd.setDebug(True) 
 79                          dollarQuoted.setDebug(True) 
 80                          literalDollar.setDebug(True) 
 81                  return script 
 82   
 83   
 84  getSQLScriptGrammar = utils.CachedGetter(_getSQLScriptGrammar) 
 85   
 86   
 87 -class ScriptRunner(object): 
 88          """An object encapsulating the preparation and execution of 
 89          scripts. 
 90   
 91          They are constructed with instances of Script below and have 
 92          a method run(dbTable, **kwargs). 
 93   
 94          You probably should not override __init__ but instead override 
 95          _prepare(script) which is called by __init__. 
 96          """ 
 97 -        def __init__(self, script): 
 98                  self.name, self.notify = script.name, script.notify 
 99                  self._prepare(script) 
100           
101 -        def _prepare(self, script): 
102                  raise ValueError("Cannot instantate plain ScriptRunners") 
103   
104   
105 -class SQLScriptRunner(ScriptRunner): 
106          """A runner for SQL scripts. 
107   
108          These will always use the table's querier to execute the statements. 
109   
110          Keyword arguments to run are ignored. 
111          """ 
112 -        def _prepare(self, script): 
113                  self.statements = utils.pyparseString(getSQLScriptGrammar(),  
114                          script.getSource()) 
115           
116 -        def run(self, dbTable, **kwargs): 
117                  for statement in self.statements: 
118                          dbTable.query(statement.replace("%", "%%")) 
119   
120   
121 -class ACSQLScriptRunner(SQLScriptRunner): 
122          """A runner for "autocommitted" SQL scripts. 
123   
124          These are like SQLScriptRunners, except that for every statement, 
125          a savepoint is created, and for SQL errors, the savepoint is restored 
126          (in other words ACSQL scripts turn SQL errors into warnings). 
127          """ 
128 -        def run(self, dbTable, **kwargs): 
129                  for statement in self.statements: 
130                          try: 
131                                  dbTable.query("SAVEPOINT beforeStatement") 
132                                  try: 
133                                          dbTable.query(statement.replace("%", "%%")) 
134                                  except sqlsupport.DBError as msg: 
135                                          dbTable.query("ROLLBACK TO SAVEPOINT beforeStatement") 
136                                          base.ui.notifyError("Ignored error during script execution: %s"% 
137                                                  msg) 
138                          finally: 
139                                  dbTable.query("RELEASE SAVEPOINT beforeStatement") 
140   
141   
142 -class PythonScriptRunner(ScriptRunner): 
143          """A runner for python scripts. 
144   
145          The scripts can access the current table as table (and thus run 
146          SQL statements through table.query(query, pars)). 
147   
148          Additional keyword arguments are available under their names. 
149   
150          You are in the namespace of usual procApps (like procs, rowgens, and 
151          the like). 
152          """ 
153 -        def __init__(self, script): 
154                  # I need to memorize the script as I may need to recompile 
155                  # it if there's special arguments (yikes!) 
156                  self.code = ("def scriptFun(table, **kwargs):\n"+ 
157                          utils.fixIndentation(script.getSource(), "      ")+"\n") 
158                  ScriptRunner.__init__(self, script) 
159   
160 -        def _compile(self, moreNames={}): 
161                  return rmkfuncs.makeProc("scriptFun", self.code, "", self, 
162                          **moreNames) 
163   
164 -        def _prepare(self, script, moreNames={}): 
165                  self.scriptFun = self._compile() 
166           
167 -        def run(self, dbTable, **kwargs): 
168  # I want the names from kwargs to be visible as such in scriptFun -- if 
169  # given.  Since I do not want to manipulate func_globals, the only 
170  # way I can see to do this is to compile the script.  I don't think 
171  # this is going to be a major performance issue. 
172                  if kwargs: 
173                          func = self._compile(kwargs) 
174                  else: 
175                          func = self.scriptFun 
176                  func(dbTable, **kwargs) 
177   
178   
179  RUNNER_CLASSES = { 
180          "SQL": SQLScriptRunner, 
181          "python": PythonScriptRunner, 
182          "AC_SQL": ACSQLScriptRunner, 
183  } 
184   
185 -class Script(base.Structure, base.RestrictionMixin): 
186          """A script, i.e., some executable item within a resource descriptor. 
187   
188          The content of scripts is given by their type -- usually, they are 
189          either python scripts or SQL with special rules for breaking the 
190          script into individual statements (which are basically like python's). 
191   
192          The special language AC_SQL is like SQL, but execution errors are 
193          ignored.  This is not what you want for most data RDs (it's intended 
194          for housekeeping scripts). 
195   
196          See `Scripting`_. 
197          """ 
198          name_ = "script" 
199          typeDesc_ = "Embedded executable code with a type definition" 
200   
201          _lang = base.EnumeratedUnicodeAttribute("lang", default=base.Undefined, 
202                  description="Language of the script.",  
203                  validValues=["SQL", "python", "AC_SQL"], copyable=True) 
204          _type = base.EnumeratedUnicodeAttribute("type", default=base.Undefined, 
205                  description="Point of time at which script is to run.",  
206                  validValues=["preImport", "newSource", "preIndex", "preCreation", 
207                          "postCreation", 
208                          "beforeDrop", "sourceDone"], copyable=True) 
209          _name = base.UnicodeAttribute("name", default="anonymous", 
210                  description="A human-consumable designation of the script.", 
211                  copyable=True) 
212          _notify = base.BooleanAttribute("notify", default=True, 
213                  description="Send out a notification when running this" 
214                          " script.", copyable=True) 
215          _content = base.DataContent(copyable=True, description="The script body.") 
216          _original = base.OriginalAttribute() 
217   
218 -        def getSource(self): 
219                  """returns the content with all macros expanded. 
220                  """ 
221                  return self.parent.getExpander().expand(self.content_) 
222   
223   
224   
225 -class ScriptingMixin(object): 
226          """A mixin that gives objects a getRunner method and a script attribute. 
227   
228          Within the DC, this is only mixed into make. 
229   
230          The getRunner() method returns a callable that takes the current table 
231          (we expect db tables, really), the phase and possibly further keyword 
232          arguments, as appropriate for the phase. 
233   
234          Objects mixing this in must also support define a method 
235          getExpander() returning an object mixin in a MacroPackage. 
236          """ 
237          _scripts = base.StructListAttribute("scripts", childFactory=Script, 
238                  description="Code snippets attached to this object.  See Scripting_ .", 
239                  copyable=True) 
240   
241 -        def getRunner(self): 
242                  if not hasattr(self, "_runScriptsCache"): 
243                          runnersByPhase = {} 
244                          for rawScript in self.scripts: 
245                                  runner = RUNNER_CLASSES[rawScript.lang](rawScript) 
246                                  runnersByPhase.setdefault(rawScript.type, []).append(runner) 
247                                   
248                          def runScripts(table, phase, **kwargs): 
249                                  for runner in runnersByPhase.get(phase, []): 
250                                          if runner.notify: 
251                                                  base.ui.notifyScriptRunning(runner) 
252                                          runner.run(table, **kwargs) 
253   
254                          self._runScriptsCache = runScripts 
255   
256                  return self._runScriptsCache 
257