Source Code for Module gavo.protocols.dali

  1  """ 
  2  Common code supporting functionality described in DALI. 
  3  """ 
  4   
  5  #c Copyright 2008-2019, the GAVO project 
  6  #c 
  7  #c This program is free software, covered by the GNU GPL.  See the 
  8  #c COPYING file in the source distribution. 
  9   
 10   
 11  import os 
 12   
 13  from gavo import base 
 14  from gavo import formats 
 15  from gavo import utils 
 16  from gavo.base import sqlmunge 
 17   
 18  # Upload stuff -- note that TAP uploads are somewhat different from the 
 19  # DALI ones, as TAP allows multiple uploads in one string. Hence, we 
 20  # have a different (and simpler) implementation here. 
 21   
 22   
 23 -def getUploadKeyFor(inputKey): 
 24          """returns an input key for file items in "PQL". 
 25   
 26          This is actually specified by DALI.  In that scheme, the parameter 
 27          is always called UPLOAD (there can thus only be one such parameter, 
 28          but it can be present multiple times if necessary, except we've 
 29          not figured out how to do the description right in that case). 
 30   
 31          It contains a comma-separated pair of (key,source) pairs, where 
 32          source is a URL; there's a special scheme param: for referring to  
 33          inline uploads by their name. 
 34   
 35          This is used exclusively for metadata generation, and there's special 
 36          code to handle it there.  There's also special code in  
 37          inputdef.ContextGrammar to magcially make UPLOAD into the file  
 38          parameters we use within DaCHS.  Sigh. 
 39          """ 
 40          return inputKey.change( 
 41                  name="INPUT:UPLOAD", 
 42                  type="pql-upload", 
 43                  description="An upload of the form '%s,URL'; the input for this" 
 44                          " parameter is then taken from URL, which may be param:name" 
 45                          " for pulling the content from the inline upload name.  Purpose" 
 46                          " of the upload: %s"%(inputKey.name, inputKey.description), 
 47                  values=None) 
 48   
 49  # pql-uploads never contribute to SQL queries 
 50  sqlmunge.registerSQLFactory("pql-upload", lambda field, val, sqlPars: None) 
 51   
 52   
 53 -def parseUploadString(uploadString): 
 54          """returns resourceName, uploadSource from a DALI upload string. 
 55          """ 
 56          try: 
 57                  destName, uploadSource = uploadString.split(",", 1) 
 58          except (TypeError, ValueError): 
 59                  raise base.ValidationError("Invalid UPLOAD string", 
 60                          "UPLOAD", hint="UPLOADs look like my_upload,http://foo.bar/up" 
 61                                  " or inline_upload,param:foo.") 
 62          return destName, uploadSource 
 63   
 64   
 65 -class URLUpload(object): 
 66          """a somewhat FieldStorage-compatible facade to an upload coming from 
 67          a URL. 
 68   
 69          The filename for now is the complete upload URL, but that's likely 
 70          to change. 
 71          """ 
 72 -        def __init__(self, uploadURL, uploadName): 
 73                  self.uploadURL, self.name = uploadURL, uploadName 
 74                  self.file = utils.urlopenRemote(self.uploadURL) 
 75                  self.filename = uploadURL 
 76                  self.headers = self.file.info() 
 77                  major, minor, parSet = formats.getMIMEKey(self.headers.get( 
 78                          "content-type", "*/*")) 
 79                  self.type = "%s/%s"%(major, minor) 
 80                  self.type_options = dict(parSet) 
 81           
 82          @property 
 83 -        def value(self): 
 84                  try: 
 85                          f = utils.urlopenRemote(self.uploadURL) 
 86                          return f.read() 
 87                  finally: 
 88                          f.close() 
 89   
 90   
 91 -def iterUploads(request): 
 92          """iterates over DALI uploads in request. 
 93   
 94          This yields pairs of (file name, file object), where file name 
 95          is the file name requested (sanitized to have no slashes and non-ASCII). 
 96          The UPLOAD and inline-file keys are removed from request's args 
 97          member.  file object is a cgi-style thing with file, filename, 
 98          etc. attributes. 
 99          """ 
100          # UWS auto-downcases things (it probably shouldn't) 
101          uploads = request.args.pop("UPLOAD", [])+request.args.pop("upload", []) 
102          if not uploads: 
103                  return 
104                   
105          for uploadString in uploads: 
106                  destName, uploadSource = parseUploadString(uploadString) 
107                  # mangle the future file name such that we hope it's representable 
108                  # in the file system 
109                  destName = str(destName).replace("/", "_") 
110                  try: 
111                          if uploadSource.startswith("param:"): 
112                                  fileKey = uploadSource[6:] 
113                                  upload = request.fields[fileKey] 
114                                  # remove upload in string form from args to remove clutter 
115                                  request.args.pop(fileKey, None)  
116                          else: 
117                                  upload = URLUpload(uploadSource, destName) 
118   
119                          yield destName, upload 
120                  except (KeyError, AttributeError): 
121                          raise base.ui.logOldExc(base.ValidationError( 
122                                  "%s references a non-existing" 
123                                  " file upload."%uploadSource, "UPLOAD",  
124                                  hint="If you pass UPLOAD=foo,param:x," 
125                                  " you must pass a file upload under the key x.")) 
126   
127   
128 -def mangleUploads(request): 
129          """manipulates request to turn DALI UPLOADs into what nevow formal 
130          produces for file uploads. 
131   
132          These are as in normal CGI: uploads are under "their names" (with 
133          DALI uploads, the resource names), with values being pairs of 
134          some name and a FieldStorage-compatible thing having name, filename, value, 
135          file, type, type_options, and headers. 
136   
137          ArgDict is manipulated in place. 
138          """ 
139          for fName, fObject in iterUploads(request): 
140                  request.args[fName] = (fObject.filename, fObject.file) 
141   
142   
143 -def writeUploadBytesTo(request, destDir): 
144          """writes a file corresponding to a DALI upload to destDir. 
145   
146          For the sake uws.UploadParameter, we return the names of the 
147          files we've been creating. 
148          """ 
149          created = [] 
150   
151          if not os.path.isdir(destDir): 
152                  os.mkdir(destDir) 
153   
154          for fName, fObject in iterUploads(request): 
155                  with open(os.path.join(destDir, fName), "w") as f: 
156                          utils.cat(fObject.file, f) 
157                  created.append(fName) 
158   
159          return created 
160