1 """
2 Code dealing with spectra (the actual data), in particular in the spectral
3 data model (sdm).
4
5 This assumes Spectral Data Model Version 1, but doesn't use very much of it.
6 There's an sdm2 module for version 2 support. Generic, version-independent
7 code should go here.
8 """
9
10
11
12
13
14
15
16 import datetime
17 import os
18 from cStringIO import StringIO
19
20 from gavo import base
21 from gavo import formats
22 from gavo import rsc
23 from gavo import svcs
24 from gavo import utils
25 from gavo.formats import fitstable
26 from gavo.protocols import products
27
28
29
30
31
32 GETDATA_FORMATS = {
33 base.votableType: "votable",
34 "application/x-votable+xml;serialization=tabledata": "votabletd",
35 "text/plain": "tsv",
36 "text/csv": "csv",
37 "application/fits": None,
38 "application/x-votable+xml;content=spec2": None,
39 "application/x-votable+xml;version=1.4": "vodml"}
40
41
42 _SDM1_IRREGULARS = {
43 "Dataset.Type": "Spectrum.Type",
44 "Dataset.Length ": "Spectrum.Length",
45 "Dataset.TimeSI": "Spectrum.TimeSI",
46 "Dataset.SpectralSI": "Spectrum.SpectralSI",
47 "Dataset.FluxSI": "Spectrum.FluxSI",
48 "Access.Format": None,
49 "Access.Reference": None,
50 "Access.Size": None,
51 "AstroCoords.Position2D.Value2":
52 "Spectrum.Char.SpatialAxis.Coverage.Location.Value",
53 "Target.pos.spoint": "Spectrum.Target.pos",
54 }
55
57 """returns a utype from the spectrum data model for a utype of the ssa
58 data model.
59
60 For most utypes, this just removes a prefix and adds spec:Spectrum. Heaven
61 knows why these are two different data models anyway. There are some
62 (apparently random) differences, though.
63
64 For convenience, utype=None is allowed and returned as such.
65 """
66 if utype is None:
67 return None
68 localName = utype.split(":")[-1]
69 specLocal = _SDM1_IRREGULARS.get(localName, "Spectrum."+localName)
70 if specLocal:
71 return "spec:"+specLocal
72 else:
73 return None
74
75
76 _SDM2_IRREGULARS = {
77
78
79 "dataid.instrument": "ObsConfig.Instrument.Name",
80 "dataid.bandpass": "ObsConfig.Bandpass.Name",
81 "dataid.datasource": "ObsConfig.DataSource.Name",
82
83 "char.spatialaxis.samplingprecision.fillfactor":
84 "Char.SpatialAxis.SamplingPrecision.SamplingPrecisionRefval.fillFactor",
85 "char.spatialaxis.calibration": "Char.SpatialAxis.CalibrationStatus",
86 "char.spatialaxis.resolution": "Char.SpatialAxis.Resolution.refVal",
87
88 "char.spectralaxis.samplingprecision.fillfactor":
89 "Char.SpectralAxis.SamplingPrecision.SamplingPrecisionRefval.fillFactor",
90 "char.spectralaxis.calibration": "Char.SpectralAxis.CalibrationStatus",
91 "char.spectralaxis.resolution": "Char.SpectralAxis.Resolution.refVal",
92
93 "char.timeaxis.samplingprecision.fillfactor":
94 "Char.TimeAxis.SamplingPrecision.SamplingPrecisionRefval.fillFactor",
95 "char.timeaxis.calibration": "Char.TimeAxis.CalibrationStatus",
96 "char.timeaxis.resolution": "Char.TimeAxis.Resolution.refVal",
97
98 "char.fluxaxis.calibration": "Char.FluxAxis.CalibrationStatus",
99
100 "curation.publisherdid": "DataID.DatasetID",
101 }
102
103
105 """returns a utype from the spectral data model 2 for a utype of the ssa
106 data model.
107
108 For convenience, utype=None is allowed and returned as such.
109 """
110
111
112
113 if utype is None:
114 return None
115 localName = utype.split(":")[-1]
116 specLocal = _SDM2_IRREGULARS.get(localName.lower(), localName)
117 return "spec2:"+specLocal
118
119
120 _SDM_TO_SED_UTYPES = {
121 "spec:Spectrum.Data.SpectralAxis.Value":
122 "sed:Segment.Points.SpectralCoord.Value",
123 "spec2:Data.SpectralAxis.Value":
124 "sed:Segment.Points.SpectralCoord.Value",
125 "spec:Spectrum.Data.FluxAxis.Value": "sed:Segment.Points.Flux.Value",
126 "spec2:Data.FluxAxis.Value": "sed:Segment.Points.Flux.Value",
127 }
128
129
130 SDM1_TO_SDM2_IRREGULARS = {
131 "Char.FluxAxis.Calibration": "Char.FluxAxis.CalibrationStatus",
132 "Char.SpatialAxis.Calibration": "Char.SpatialAxis.CalibrationStatus",
133 "Char.SpatialAxis.Resolution": "Char.SpatialAxis.Resolution.refVal",
134 "Char.SpectralAxis.Calibration": "Char.SpectralAxis.CalibrationStatus",
135 "Char.SpectralAxis.Resolution": "Char.SpectralAxis.Resolution.refVal",
136 "Char.SpectralAxis.ResPower":
137 "Char.SpectralAxis.Resolution.ResolPower.refVal",
138 "Char.TimeAxis.Calibration": "Char.TimeAxis.CalibrationStatus",
139 "Char.TimeAxis.Resolution": "Char.TimeAxis.Resolution.refVal",
140 "Data.FluxAxis.Quality": "Data.FluxAxis.Accuracy.QualityStatus",
141 "Data.SpectralAxis.Resolution": "Data.SpectralAxis.Resolution.refVal",
142 "Type": "Dataset.Type",
143 "DataModel": "Dataset.DataModel.Name",
144 "Length": "Dataset.Length",
145 "TimeSI": "Dataset.TimeSI",
146 "SpectralSI": "Dataset.SpectralSI",
147 "FluxSI": "Dataset.FluxSI",
148 "DataID.Instrument": "ObsConfig.Instrument",
149 "DataID.Bandpass": "ObsConfig.Bandpass",
150 "DataID.DataSource": "ObsConfig.DataSource",
151 "Curation.PublisherDID": "DataID.DatasetID",
152 }
153
155 """returns a utype for the spectral data model 2 from a version 1 one.
156 """
157 if utype is None:
158 return None
159
160 if not utype.startswith("spec:Spectrum."):
161 return utype
162 localPart = SDM1_TO_SDM2_IRREGULARS.get(utype[14:], utype[14:])
163 return "spec2:%s"%localPart
164
165
196
197
198
199
200
241
242
245 """returns a rsc.Data instance containing an SDM compliant spectrum
246 for pubDID from ssaTable.
247
248 ssaTD is the definition of a table containg the SSA metadata,
249 spectrumData is a data element making a primary table containing
250 the spectrum data from an SSA row (typically, this is going to be
251 the tablesource property of an SSA service).
252 """
253 matchingRows = ssaTD.doSimpleQuery(fragments="ssa_pubdid=%(pubDID)s",
254 params={"pubDID": pubDID})
255 if not matchingRows:
256 raise svcs.UnknownURI("No spectrum with pubdid %s known here"%
257 pubDID)
258 return makeSDMDataForSSARow(
259 matchingRows[0],
260 spectrumData,
261 sdmVersion=sdmVersion)
262
263
264
265
271
272
278
279
281 """returns the fits field index that has colUtype as its utype within
282 header.
283
284 If no such field exists, this raises a KeyError
285 """
286 nCols = header.get("TFIELDS", 0)
287 for colInd in range(1, nCols+1):
288 key = "TUTYP%d"%colInd
289 if header.get(key, None)==colUtype:
290 return colInd
291 else:
292 raise KeyError(colUtype)
293
294
296 """returns a _SDM_HEADER_MAPPING for declaring TDMIN/MAXn and friends.
297
298 colUtype determines the n here.
299 """
300 def func(header, par):
301 try:
302 key = keyBase+str(getColIndForUtype(header, colUtype))
303 header.set(key, par.value, comment="[%s]"%par.unit)
304 except KeyError:
305
306 pass
307
308 return func
309
310
311
312
313
314
315 _SDM_HEADER_MAPPING = {
316 "datamodel": "VOCLASS",
317 "length": "DATALEN",
318 "type": "VOSEGT",
319 "coordsys.id": "VOCSID",
320 "coordsys.spaceframe.name": "RADECSYS",
321 "coordsys.spaceframe.equinox": "EQUINOX",
322 "coordsys.spaceframe.ucd": "SKY_UCD",
323 "coordsys.spaceframe.refpos": "SKY_REF",
324 "coordsys.timeframe.name": "TIMESYS",
325 "coordsys.timeframe.ucd": None,
326 "coordsys.timeframe.zero": "MJDREF",
327 "coordsys.timeframe.refpos": None,
328 "coordsys.spectralframe.refpos": "SPECSYS",
329 "coordsys.spectralframe.redshift": "REST_Z",
330 "coordsys.spectralframe.name": "SPECNAME",
331 "coordsys.redshiftframe.name": "ZNAME",
332 "coordsys.redshiftframe.refpos": "SPECSYSZ",
333 "curation.publisher": "VOPUB",
334 "curation.reference": "VOREF",
335 "curation.publisherid": "VOPUBID",
336 "curation.version": "VOVER",
337 "curation.contactname": "CONTACT",
338 "curation.contactemail": "EMAIL",
339 "curation.rights": "VORIGHTS",
340 "curation.date": "VODATE",
341 "curation.publisherdid": "DS_IDPUB",
342 "target.name": "OBJECT",
343 "target.description": "OBJDESC",
344 "target.class": "SRCCLASS",
345 "target.spectralclass": "SPECTYPE",
346 "target.redshift": "REDSHIFT",
347 "target.varampl": "TARGVAR",
348 "dataid.title": "TITLE",
349 "dataid.creator": "AUTHOR",
350 "dataid.datasetid": "DS_IDENT",
351 "dataid.creatordid": "CR_IDENT",
352 "dataid.date": "DATE",
353 "dataid.version": "VERSION",
354 "dataid.instrument": "INSTRUME",
355 "dataid.creationtype": "CRETYPE",
356 "dataid.logo": "VOLOGO",
357
358 "dataid.collection": "COLLECT1",
359 "dataid.contributor": "CONTRIB1",
360 "dataid.datasource": "DSSOURCE",
361 "dataid.bandpass": "SPECBAND",
362 "derived.snr": "DER_SNR",
363 "derived.redshift.value": "DER_Z",
364 "derived.redshift.staterror": "DER_ZERR",
365 "derived.redshift.confidence": "DER_ZCNF",
366 "derived.varampl": "DER_VAR",
367 "timesi": "TIMESDIM",
368 "spectralsi": "SPECSDIM",
369 "fluxsi": "FLUXSDIM",
370 "char.fluxaxis.name": None,
371 "char.fluxaxis.unit": None,
372 "char.fluxaxis.ucd": None,
373 "char.spectralaxis.name": None,
374 "char.spectralaxis.unit": None,
375 "char.spectralaxis.ucd": None,
376 "char.timeaxis.name": None,
377 "char.timeaxis.ucd": None,
378 "char.spatialaxis.name": None,
379 "char.spatialaxis.unit": None,
380 "char.fluxaxis.accuracy.staterror": "STAT_ERR",
381 "char.fluxaxis.accuracy.syserror": "SYS_ERR",
382 "char.timeaxis.accuracy.staterror": "TIME_ERR",
383 "char.timeaxis.accuracy.syserror": "TIME_SYE",
384 "char.timeaxis.resolution": "TIME_RES",
385 "char.fluxaxis.calibration": "FLUX_CAL",
386 "char.spectralaxis.calibration": "SPEC_CAL",
387 "char.spectralaxis.coverage.location.value": "SPEC_VAL",
388 "char.spectralaxis.coverage.bounds.extent": "SPEC_BW",
389 "char.spectralaxis.coverage.bounds.start":
390 _make_limits_func("spec:spectrum.data.spectralaxis.value", "TDMIN"),
391 "char.spectralaxis.coverage.bounds.stop":
392 _make_limits_func("spec:spectrum.data.spectralaxis.value", "TDMAX"),
393 "char.spectralaxis.samplingprecision.": None,
394 "samplingprecisionrefval.fillfactor": "SPEC_FIL",
395 "char.spectralaxis.samplingprecision.SampleExtent": "SPEC BIN",
396 "char.spectralaxis.accuracy.binsize": "SPEC_BIN",
397 "char.spectralaxis.accuracy.staterror": "SPEC_ERR",
398 "char.spectralaxis.accuracy.syserror": "SPEC_SYE",
399 "char.spectralaxis.resolution": "SPEC_RES",
400 "char.spectralaxis.respower": "SPEC_RP",
401 "char.spectralaxis.coverage.support.extent": "SPECWID",
402 "char.timeaxis.unit": "TIMEUNIT",
403 "char.timeaxis.accuracy.binsize": "TIMEDEL",
404 "char.timeaxis.calibration": "TIME_CAL",
405 "char.timeaxis.coverage.location.value": "TMID",
406 "char.timeaxis.coverage.bounds.extent": "TELAPSE",
407 "char.timeaxis.coverage.bounds.start": "TSTART",
408 "char.timeaxis.coverage.bounds.stop": "TSTOP",
409 "char.timeaxis.coverage.support.extent": "EXPOSURE",
410 "char.timeaxis.samplingprecision.samplingprecisionrefval.fillfactor": "DTCOR",
411 "char.timeaxis.samplingprecision.sampleextent": "TIMEDEL",
412 "char.spatialaxis.ucd": "SKY_UCD",
413 "char.spatialaxis.accuracy.staterr": "SKY_ERR",
414 "char.spatialaxis.accuracy.syserror": "SKY_SYE",
415 "char.spatialaxis.calibration": "SKY_CAL",
416 "char.spatialaxis.resolution": "SKY_RES",
417 "char.spatialaxis.coverage.bounds.extent": "APERTURE",
418 "char.spatialaxis.coverage.support.area": "REGION",
419 "char.spatialaxis.coverage.support.extent": "AREA",
420 "char.spatialaxis.samplingprecision.samplingprecisionrefval.fillfactor":
421 "SKY_FILL",
422
423
424 "target.pos": _add_target_pos_cards,
425 "char.spatialaxis.coverage.location.value": _add_location_cards,
426 }
427
457
458
474
475
476
477
516
517
518
519
520
521
522
523
524
525
527 """returns the column containing the flux in sdmTable.
528
529 sdmTable can be in SDM1 or SDM2.
530 """
531 return sdmTable.tableDef.getByUtypes(
532 "spec:Spectrum.Data.FluxAxis.Value",
533 "spec2:Data.FluxAxis.Value")
534
535
537 """returns the column containing the spectral coordindate in sdmTable.
538
539 sdmTable can be in SDM1 or SDM2.
540 """
541 return sdmTable.tableDef.getByUtypes(
542 "spec:Spectrum.Data.SpectralAxis.Value",
543 "spec2:Data.SpectralAxis.Value")
544
545
547 """returns only those rows from sdmTable for which the spectral coordinate
548 is between low and high.
549
550 Both low and high must be given. If you actually want half-open intervals,
551 do it in interface code (low=-1 and high=1e308 should do fine).
552 """
553 spectralColumn = getSpectralColumn(sdmTable)
554
555 spectralUnit = spectralColumn.unit
556
557
558 factor = base.computeConversionFactor("m", spectralUnit)
559 low = low*factor
560 high = high*factor
561
562
563
564 spectralName = spectralColumn.name
565 sdmTable.rows=[
566 row for row in sdmTable.rows if low<=row[spectralName]<=high]
567
568 specVals = [r[spectralName] for r in sdmTable.rows]
569 if specVals:
570 specstart, specend = min(specVals)/factor, max(specVals)/factor
571 sdmTable.setParam("ssa_specext", specend-specstart)
572 sdmTable.setParam("ssa_specstart", specstart)
573 sdmTable.setParam("ssa_specend", specend)
574 sdmTable.setParam("ssa_specmid", (specstart+specend)/2)
575
576 return sdmTable
577
578
580 """returns sdmTable with a new calibration.
581
582 Currently, we can only normalize the spectrum to the maximum value.
583 """
584 newCalib = newCalib.lower()
585 if newCalib==sdmTable.getParam("ssa_fluxcalib").lower():
586 return sdmTable
587 fluxName = getFluxColumn(sdmTable).name
588
589 try:
590
591 errorName = sdmTable.tableDef.getColumnByUCD(
592 "stat.error;phot.flux;em.opt").name
593 except ValueError:
594
595 errorName = None
596
597 if newCalib=="relative":
598
599
600 normalizer = float(max(row[fluxName] for row in sdmTable.rows))
601 for row in sdmTable.rows:
602 row[fluxName] = row[fluxName]/normalizer
603
604 if errorName:
605 for row in sdmTable.rows:
606 row[errorName] = row[errorName]/normalizer
607
608 sdmTable.setParam("ssa_fluxcalib", "RELATIVE")
609 sdmTable.tableDef = sdmTable.tableDef.copy(sdmTable.tableDef.parent)
610 sdmTable.tableDef.getColumnByName("flux").unit = ""
611 return sdmTable
612
613 raise base.ValidationError("Do not know how to turn a %s spectrum"
614 " into a %s one."%(sdmTable.getParam("ssa_fluxcalib"), newCalib),
615 "FLUXCALIB")
616