Source Code for Module gavo.grammars.csvgrammar

 1  """ 
 2  A grammar using python's csv module to parse files. 
 3  """ 
 4   
 5  #c Copyright 2008-2019, the GAVO project 
 6  #c 
 7  #c This program is free software, covered by the GNU GPL.  See the 
 8  #c COPYING file in the source distribution. 
 9   
10   
11  import csv 
12   
13  from gavo import base 
14  from gavo.grammars.common import( 
15          Grammar, FileRowIterator, FileRowAttributes, MapKeys) 
16   
17   
18 -class CSVIterator(FileRowIterator): 
19 -        def __init__(self, grammar, sourceToken, **kwargs): 
20                  FileRowIterator.__init__(self, grammar, sourceToken, **kwargs) 
21                  consArgs = { 
22                          "delimiter": str(self.grammar.delimiter), 
23                          "fieldnames": self.grammar.names, 
24                          "skipinitialspace": self.grammar.strip, 
25                          "restkey": "NOTASSIGNED", 
26                  } 
27   
28                  if self.grammar.topIgnoredLines: 
29                          for i in range(self.grammar.topIgnoredLines): 
30                                  self.inputFile.readline() 
31   
32                  self.csvSource = csv.DictReader(self.inputFile, **consArgs) 
33   
34 -        def _iterMappedRows(self): 
35                  for row in self.csvSource: 
36                          yield self.grammar.mapKeys.doMap(row) 
37   
38 -        def _iterRows(self): 
39                  if self.grammar.mapKeys: 
40                          return self._iterMappedRows() 
41                  else: 
42                          return self.csvSource 
43   
44 -        def getLocator(self): 
45                  return "line %s"%self.csvSource.line_num 
46   
47   
48 -class CSVGrammar(Grammar, FileRowAttributes): 
49          """A grammar that uses python's csv module to parse files. 
50   
51          Note that these grammars by default interpret the first line of 
52          the input file as the column names.  When your files don't follow 
53          that convention, you *must* give names (as in ``names='raj2000, 
54          dej2000, magV'``), or you'll lose the first line and have silly 
55          column names. 
56   
57          CSVGrammars currently do not support non-ASCII inputs. 
58          Contact the authors if you need that. 
59   
60          If data is left after filling the defind keys, it is available under 
61          the NOTASSIGNED key. 
62          """ 
63          name_ = "csvGrammar" 
64   
65          _delimiter = base.UnicodeAttribute("delimiter",  
66                  description="CSV delimiter", default=",", copyable=True) 
67   
68          _names = base.StringListAttribute("names", default=None, 
69                  description="Names for the parsed fields, in sequence of the" 
70                  " comma separated values.  The default is to read the field names" 
71                  " from the first line of the csv file.  You can use macros here," 
72                  r" e.g., \\colNames{someTable}.", expand=True, 
73                  copyable=True) 
74   
75          _strip = base.BooleanAttribute("strip", default=False, 
76                  description="If True, whitespace immediately following a delimiter" 
77                  " is ignored.", copyable=True) 
78   
79          _til = base.IntAttribute("topIgnoredLines", default=0, description= 
80                  "Skip this many lines at the top of each source file.") 
81   
82          _mapKeys = base.StructAttribute("mapKeys", childFactory=MapKeys, 
83                  default=None, copyable=True, description="Prescription for how to" 
84                  " map header keys to grammar dictionary keys") 
85   
86          rowIterator = CSVIterator 
87