1 """
2 Conversions between type systems.
3
4 The DC software has to deal with a quite a few type systems:
5
6 - Python
7 - SQL
8 - Postgres pg_type
9 - VOTable
10 - XSD
11 - Twisted formal
12 - numpy
13
14 Based on the (stinking) framework of utils.typeconversions, this
15 module contains converters between them as necessary. The
16 linuga franca of our type systems is SQL+extensions as laid
17 down in utils.typeconversions.
18 """
19
20
21
22
23
24
25
26 import numpy
27
28 from gavo import utils
29 from gavo.base import literals
30 from gavo.utils.typeconversions import (FromSQLConverter,
31 sqltypeToVOTable, voTableToSQLType, ConversionError)
32
33
35 typeSystem = "db"
36
37 simpleMap = {
38 ("short", '1'): "smallint",
39 ("int", '1'): "integer",
40 ("long", '1'): "bigint",
41 ("float", '1'): "real",
42 ("boolean", '1'): "boolean",
43 ("double", '1'): "double precision",
44 ("char", "*"): "text",
45 ("char", '1'): "char",
46 ("unsignedByte", '1'): "smallint",
47 ("raw", '1'): "raw",
48 }
49
50 xtypeMap = {
51 "adql:POINT": "spoint",
52 "adql:REGION": "spoly",
53 "adql:TIMESTAMP": "timestamp",
54 "timestamp": "timestamp",
55 "point": "spoint",
56 "circle": "scirle",
57 "polygon": "spoly",
58 }
59
60 - def convert(self, type, arraysize, xtype=None):
69
71 if arraysize=="*":
72 arraysize = ""
73 if type=="char":
74 return "text"
75 if type=="unicodeChar":
76 return "unicode"
77 if type=="unsignedByte" and arraysize!="1":
78 return "bytea[]"
79 if (type, '1') in self.simpleMap:
80 return "%s[%s]"%(self.simpleMap[type, '1'], arraysize)
81 raise ConversionError("No SQL type for %s, %s"%(type, arraysize))
82
83
85
86 typeSystem = "XSD"
87 simpleMap = {
88 "smallint": "short",
89 "integer": "int",
90 "bigint": "long",
91 "real": "float",
92 "boolean": "boolean",
93 "double precision": "double",
94 "text": "string",
95 "unicode": "string",
96 "char": "string",
97 "date": "date",
98 "timestamp": "dateTime",
99 "time": "time",
100 "raw": "string",
101 "vexpr-mjd": "string",
102 "vexpr-date": "string",
103 "vexpr-float": "string",
104 "vexpr-string": "string",
105 }
106
110
111
113
114 typeSystem = "numpy"
115 simpleMap = {
116 "smallint": numpy.int16,
117 "integer": numpy.int32,
118 "bigint": numpy.int64,
119 "real": numpy.float32,
120 "boolean": numpy.bool,
121 "double precision": numpy.float64,
122 "text": numpy.str,
123 "unicode": numpy.unicode,
124 "char": numpy.str,
125 "date": numpy.float32,
126 "timestamp": numpy.float64,
127 "time": numpy.float32,
128 }
129
133
134
136 """The base for converters turning dealing with turning "simple" literals
137 into python values.
138
139 These return the identity for most "complex" types that do not have
140 plain literals.
141
142 What is returned here is a name of a function turning a single literal
143 into an object of the desired type; all those reside within base.literals.
144
145 All such functions should be transparent to None (null value) and to
146 objects that already are of the desired type.
147 """
148 simpleMap = {
149 "smallint": "parseInt",
150 "integer": "parseInt",
151 "bigint": "parseInt",
152 "real": "parseFloat",
153 "boolean": "parseBooleanLiteral",
154 "double precision": "parseFloat",
155 "text": "parseUnicode",
156 "char": "parseUnicode",
157 "unicode": "parseUnicode",
158 "date": "parseDefaultDate",
159 "timestamp": "parseDefaultDatetime",
160 "time": "parseDefaultTime",
161 "spoint": "parseSPoint",
162 "scircle": "parseSimpleSTCS",
163 "spoly": "parseSimpleSTCS",
164 "sbox": "identity",
165 "smoc": "identity",
166 "bytea": "identity",
167 "raw": "identity",
168 "file": "identity",
169 "box": "identity",
170 "vexpr-mjd": "identity",
171 "vexpr-string": "identity",
172 "vexpr-float": "identity",
173 "vexpr-date": "identity",
174 "pql-string": "identity",
175 "pql-float": "identity",
176 "pql-int": "identity",
177 "pql-date": "identity",
178 "pql-upload": "identity",
179 "int4range": "parseInterval",
180 }
181
183 if type in self._charTypes:
184 return "parseUnicode"
185 else:
186 return "identity"
187
188
190 """returns code templates to turn literals in variables to python objects.
191
192 This is for the rowmakers' src="xx" specification, where no fancy literal
193 processing needs to be done.
194
195 The values of the map are simple string interpolation templates, with a
196 single %s for the name of the variable to be converted.
197
198 The code assumes whatever executes those literals has done the equvialent
199 of gavo.base.literals import * or use gavo.base.literals.defaultParsers()
200 """
201 typeSystem = "pythonsrc"
202
204 funcName = ToPythonBase.convert(self, sqlType)
205 if funcName=="identity":
206 return "%s"
207 return funcName+"(%s)"
208
209
211 """returns constructors making python values from strings.
212
213 This is only for non-fancy applications with controlled input. For
214 more general circumstances, you'll want to use the parsing infrastructure.
215
216 In particular, this will return the identity for most non-trivial stuff.
217 Maybe that's wrong, but it will only change as sane literals are defined.
218 """
219 typeSystem = "python"
220
224
225
227 """returns a function taking some python value and returning stuff that
228 can be parsed using ToPythonCodeConverter.
229 """
230 typeSystem = "literal"
231 simpleMap = {
232 "smallint": str,
233 "integer": str,
234 "bigint": str,
235 "real": str,
236 "boolean": str,
237 "double precision": str,
238 "text": str,
239 "char": str,
240 "unicode": unicode,
241 "date": lambda v: v.isoformat(),
242 "timestamp": lambda v: utils.formatISODT(v),
243 "time": lambda v: v.isoformat(),
244 "spoint": lambda v: "%f,%f"%(v.x/utils.DEG, v.y/utils.DEG),
245
246
247
248
249
250 }
251
256
257
259 """returns a callable that takes a type code from postgres' pg_type
260 that will raise a TypeError if the to types are deemed incompatible.
261 """
262 typeSystem = "Postgres pg_types validators"
263
265 if pgcode not in frozenset(['int2', 'int4', 'int8']):
266 raise TypeError("%s is not compatible with an integer column"%pgcode)
267
269 if pgcode not in frozenset(['float4', 'float8']):
270 raise TypeError("%s is not compatible with an integer column"%pgcode)
271
273 def checker(pgcode):
274 if expectedCode!=pgcode:
275 raise TypeError("Incompatible type in DB: Expected %s, found %s"%(
276 expectedCode, pgcode))
277 return checker
278
282
284 def beAlarmed(pgcode):
285 raise TypeError("Column with a non-db type %s mapped to db type %s"%(
286 typeName, pgcode))
287 return beAlarmed
288
289 simpleMap = {
290 "smallint": checkInt,
291 "integer": checkInt,
292 "bigint": checkInt,
293 "real": checkFloat,
294 "boolean": makeChecker("bool"),
295 "double precision": checkFloat,
296 "text": makeChecker("text"),
297 "char": makeChecker("bpchar"),
298 "date": makeChecker("date"),
299 "timestamp": makeChecker("timestamp"),
300 "time": makeChecker("time"),
301 "box": dontCheck,
302 "vexpr-mjd": makeAlarmer("vexpr-mjd"),
303 "vexpr-string": makeAlarmer("vexpr-string"),
304 "vexpr-date": makeAlarmer("vexpr-date"),
305 "vexpr-float": makeAlarmer("vexpr-float"),
306 "file": makeAlarmer("file"),
307 "pql-float": makeAlarmer("pql-float"),
308 "pql-string": makeAlarmer("pql-string"),
309 "pql-date": makeAlarmer("pql-date"),
310 "pql-int": makeAlarmer("pql-int"),
311 "pql-upload": makeAlarmer("pql-upload"),
312 "raw": makeAlarmer("raw"),
313 "bytea": makeChecker("bytea"),
314 "spoint": makeChecker("spoint"),
315 "scircle": makeChecker("scircle"),
316 "sbox": makeChecker("sbox"),
317 "spoly": makeChecker("spoly"),
318 "smap": makeChecker("smap"),
319 "smoc": makeChecker("smoc"),
320 "unicode": makeChecker("text"),
321 }
322
324 if (length is None or length==1) and type in self.simpleMap:
325 return self.simpleMap[type]
326
327
328
329 def check(pgcode):
330 if pgcode.startswith("_"):
331 pgcode = pgcode[1:]
332 self.simpleMap[type](pgcode)
333
334 return check
335
336
338 """returns a postgres type for one of our internal SQL types.
339
340 This is really only necessary because of our UNICODE hack. And it
341 shouldn't become more complex than this.
342
343 >>> sqltypeToPG("INT[23, 42]")
344 'INT[23, 42]'
345 >>> sqltypeToPG("uniCode[23]")
346 'TEXT[23]'
347 """
348 if type.upper().startswith("UNICODE"):
349 return "TEXT"+type[7:]
350 return type
351
352
353 sqltypeToXSD = ToXSDConverter().convert
354 sqltypeToNumpy = ToNumpyConverter().convert
355 sqltypeToPython = ToPythonConverter().convert
356 sqltypeToPythonCode = ToPythonCodeConverter().convert
357 sqltypeToPgValidator = ToPgTypeValidatorConverter().convert
358 pythonToLiteral = ToLiteralConverter().convert
359
360
364
365 if __name__=="__main__":
366 _test()
367
368 __all__ = ["sqltypeToVOTable", "sqltypeToXSD", "sqltypeToNumpy",
369 "sqltypeToPython", "sqltypeToPythonCode", "voTableToSQLType",
370 "ConversionError", "FromSQLConverter", "pythonToLiteral",
371 "sqltypeToPgValidator", "sqltypeToPG"]
372