Source Code for Module gavo.svcs.uploadcores

  1  """ 
  2  Cores to alter the DB state from the Web. 
  3  """ 
  4   
  5  #c Copyright 2008-2019, the GAVO project 
  6  #c 
  7  #c This program is free software, covered by the GNU GPL.  See the 
  8  #c COPYING file in the source distribution. 
  9   
 10   
 11  import grp 
 12  import os 
 13   
 14  from gavo import base 
 15  from gavo import rsc 
 16  from gavo.svcs import core 
 17   
 18   
 19  MS = base.makeStruct 
 20   
 21  uploadOutputDef = """<outputTable> 
 22                                          <column name="nAffected" type="integer"  
 23                                                  tablehead="Number touched" required="True"/> 
 24                                  </outputTable>""" 
 25   
 26   
 27 -class UploadCore(core.Core): 
 28          """A core handling uploads of files to the database. 
 29   
 30          It allows users to upload individual files into a special staging 
 31          area (taken from the stagingDir property of the destination data descriptor) 
 32          and causes these files to be parsed using destDD.  Note that destDD 
 33          *must* have ``updating="True"`` for this to work properly (it will otherwise 
 34          drop the table on each update).  If uploads are the only way updates 
 35          into the table occur, source management is not necessary for these, though. 
 36   
 37          You can tell UploadCores to either insert or update the incoming data using 
 38          the "mode" input key. 
 39          """ 
 40          name_ = "uploadCore" 
 41   
 42          _destDD = base.ReferenceAttribute("destDD", default=base.Undefined, 
 43                  description="Reference to the data we are uploading into. The" 
 44                          " destination must be an updating data descriptor.") 
 45   
 46          inputTableXML = """ 
 47                  <inputTable id="inFields"> 
 48                          <inputKey name="File" type="file" required="True" 
 49                                  tablehead="Source to upload"/> 
 50                          <inputKey name="Mode" type="text" tablehead="Upload mode"  
 51                                  required="True" multiplicity="forced-single"> 
 52                                  <values default="i"> 
 53                                          <option title="Insert">i</option> 
 54                                          <option title="Update">u</option> 
 55                                  </values> 
 56                          </inputKey> 
 57                  </inputTable> 
 58                  """ 
 59          outputTableXML = uploadOutputDef 
 60   
 61 -        def _fixPermissions(self, fName): 
 62                  """tries to chmod the newly created file to 0664 and change the group 
 63                  to config.gavoGroup. 
 64                  """ 
 65                  try: 
 66                          os.chmod(fName, 0664) 
 67                          os.chown(fName, -1, grp.getgrnam(base.getConfig("gavoGroup"))[2]) 
 68                  except (KeyError, os.error):  # let someone else worry about it 
 69                          pass 
 70   
 71 -        def _writeFile(self, srcFile, fName): 
 72                  """writes the contents of srcFile to fName in destDD's staging dir. 
 73                  """ 
 74                  try: 
 75                          targetDir = os.path.join(self.rd.resdir,  
 76                                  self.destDD.getProperty("stagingDir")) 
 77                  except KeyError: 
 78                          raise base.ui.logOldExc(base.ValidationError("Uploading is only" 
 79                                  " supported for data having a staging directory.", "File")) 
 80                  if not os.path.exists(targetDir): 
 81                          raise base.ValidationError("Staging directory does not exist.", 
 82                                  "File") 
 83                  targetFName = fName.split("/")[-1].encode("iso-8859-1") 
 84                  if not targetFName: 
 85                          raise base.ValidationError("Bad file name", "File") 
 86                  targetPath = os.path.join(targetDir, targetFName) 
 87   
 88                  with open(targetPath, "w") as f: 
 89                          f.write(srcFile.read()) 
 90   
 91                  try: 
 92                          self._fixPermissions(targetPath) 
 93                  except os.error: 
 94                          # Nothing we can do, and it may not even hurt 
 95                          pass 
 96                  return targetPath 
 97   
 98 -        def _importData(self, sourcePath, mode): 
 99                  """parses the input file at sourcePath and writes the result to the DB. 
100                  """ 
101                  base.ui.notifyInfo("Web upload ingesting %s in %s mode"%(sourcePath, mode)) 
102                  try: 
103                          parseOptions = rsc.getParseOptions( 
104                                  validateRows=True,  
105                                  doTableUpdates=mode=="u") 
106                          with base.getWritableAdminConn() as conn: 
107                                  res = rsc.makeData(self.destDD, parseOptions=parseOptions,  
108                                          forceSource=sourcePath, connection=conn) 
109                  except Exception as msg: 
110                          raise base.ui.logOldExc(base.ValidationError("Cannot enter %s in" 
111                                  " database: %s"%(os.path.basename(sourcePath), str(msg)), "File")) 
112                  return res.nAffected 
113   
114 -        def _saveData(self, srcFile, fName, mode): 
115                  """saves data read from srcFile to both fNames staging dir and to the 
116                  database table(s) described by destDD. 
117   
118                  mode can be "u" (for update) or "i" for insert. 
119   
120                  If parsing or the database operations fail, the saved file will be removed. 
121                  Errors will ususally be base.ValidationErrors on either File or Mode. 
122   
123                  The function returns the number of items modified. 
124                  """ 
125                  targetPath = self._writeFile(srcFile, fName) 
126                  try: 
127                          nAffected = self._importData(targetPath, mode) 
128                  except: 
129                          os.unlink(targetPath) 
130                          raise 
131                  return nAffected 
132   
133 -        def run(self, service, inputTable, queryMeta): 
134                  totalAffected = 0 
135                  fName, srcFile = inputTable.getParam("File") 
136                  mode = inputTable.getParam("Mode") 
137                  totalAffected += self._saveData(srcFile, fName, mode) 
138                  return rsc.TableForDef(self.outputTable,  
139                          rows=[{"nAffected": totalAffected}]) 
140