"""
Products and a core turning accrefs into lists of products.
There is a substantial overlap between what's going on there and datalink
(and datalink uses some of the products mentioned here). The cutouts
and scale things here shouldn't be developed on, all this should
move towards datalink. Meanwhile, we still have siapCutoutCore and
friends that relies on the mess here, so all this is going to remain
for the foreseeable future. Just don't extend it.
The "user-visible" part are just accrefs, as modelled by the RAccref
-- they can contain instructions for cutouts or scaling, hence the additional
structure.
Using the product table and some logic in this module, such accrefs
are turned into subclasses of ProductBase.
These have mime types and know how to generate their data through their
synchronous iterData methods. They must also work as t.w resources and thus
have implement asynchronuous render(request) methods. It's a bit unfortunate
that we thus depend on t.w here, but we'd have to reimplement quite a bit of
it if we don't, and for now it doesn't seem we'll support a different framework
in the foreseeable future.
"""
#c Copyright 2008-2023, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL. See the
#c COPYING file in the source distribution.
import datetime
import gzip
import functools
import re
import os
import urllib.request, urllib.parse, urllib.error
import urllib.parse
from io import BytesIO
import numpy
from PIL import Image
from twisted.internet import defer
from twisted.internet import threads
from twisted.web import http
from twisted.web import static
from gavo import base
from gavo import rscdef
from gavo import svcs
from gavo import utils
from gavo.base import coords
from gavo.protocols import creds
from gavo.svcs import streaming
from gavo.utils import imgtools
from gavo.utils import fitstools
from gavo.utils import pyfits
# TODO: make this configurable -- globally? by service?
PREVIEW_SIZE = 200
PRODUCTS_TDID = "//products#products"
REMOTE_URL_PATTERN = re.compile("(https?|ftp)://")
MS = base.makeStruct
[docs]def makePreviewFromFITS(product):
"""returns image/jpeg bytes for a preview of a product spitting out a
2D FITS.
"""
if hasattr(product, "getFile"):
# hack to preserve no-so-well-thought out existing functionality
if product.rAccref.accref.endswith(".gz"):
inFile = gzip.GzipFile(fileobj=product.getFile())
else:
inFile = product.getFile()
with utils.fitsLock():
pixels = numpy.array([row
for row in fitstools.iterScaledRows(inFile,
destSize=PREVIEW_SIZE)])
else:
raise NotImplementedError("TODO: Fix fitstools.iterScaledRows"
" to be more accommodating to weird things")
return imgtools.jpegFromNumpyArray(pixels)
[docs]def makePreviewWithPIL(product):
"""returns image/jpeg bytes for a preview of the PIL-readable product.
"""
# TODO: Teach products to at least accept seek(0) and directly read from
# them; at least make read(None) work properly
fullsize = BytesIO(product.read(1000000000))
im = Image.open(fullsize)
scale = max(im.size)/float(PREVIEW_SIZE)
resized = im.resize((
int(im.size[0]/scale),
int(im.size[1]/scale)))
f = BytesIO()
resized.save(f, format="jpeg")
return f.getvalue()
_PIL_COMPATIBLE_MIMES = frozenset(['image/jpeg', 'image/png'])
[docs]def computePreviewFor(product):
"""returns a deferred returning image/jpeg bytes containing a
preview of product.
This only works for a select subset of products. You're usually
better off using static previews.
This will raise a DataError if it doesn't know how to make a preview.
"""
if hasattr(product, "makePreview"):
return threads.deferToThread(product.makePreview)
sourceMime = product.pr["mime"]
if sourceMime=='image/fits':
return threads.deferToThread(makePreviewFromFITS, product)
elif sourceMime in _PIL_COMPATIBLE_MIMES:
return threads.deferToThread(makePreviewWithPIL, product)
else:
raise base.DataError("Cannot make automatic preview for %s"%
sourceMime)
[docs]class PreviewCacheManager(object):
"""is a class that manages the preview cache.
It's really the class that manages it, so don't bother creating instances.
The normal operation is that you pass the product you want a preview to
getPreviewFor. If a cached preview already exists, you get back its content
(the mime type must be taken from the products table).
If the file does not exist yet, some internal magic tries to come up with
a preview and determines whether it should be cached, in which case it does
so provided a preview has been generated successfully.
A cache file is touched when it is used, so you can clean up rarely used
cache files by deleting all files in the preview cache older than some
limit.
"""
cachePath = base.getConfig("web", "previewCache")
[docs] @classmethod
def getCacheName(cls, accref):
"""returns the full path a preview for accref is be stored under.
"""
return os.path.join(cls.cachePath, rscdef.getFlatName(accref))
[docs] @classmethod
def getCachedPreviewPath(cls, accref):
"""returns the path to a cached preview if it exists, None otherwise.
"""
cacheName = cls.getCacheName(accref)
if os.path.exists(cacheName):
return cacheName
return None
[docs] @classmethod
def saveToCache(self, data, cacheName):
try:
with open(cacheName, "wb") as f:
f.write(data)
except IOError: # caching failed, don't care
pass
return data
[docs] @classmethod
def getPreviewFor(cls, product):
"""returns a deferred firing the data for a preview.
This will raise a DataError if it doesn't know how to make the
preview.
"""
if not product.rAccref.previewIsCacheable():
return computePreviewFor(product)
accref = product.rAccref.accref
cacheName = cls.getCacheName(accref)
if os.path.exists(cacheName):
# Cache hit
try:
os.utime(cacheName, None)
except os.error:
pass # don't fail just because we can't touch
with open(cacheName, "rb") as f:
return defer.succeed(f.read())
else:
# Cache miss
return computePreviewFor(product
).addCallback(cls.saveToCache, cacheName)
[docs]class ProductBase(object):
"""A base class for products returned by the product core.
See the module docstring for the big picture.
The constructor arguments of RAccrefs depend on what they are.
The common interface is the the class method
fromRAccref(rAccref, authGroups=None).
It returns None if the RAccref is not for a product of the
respective sort, the product otherwise.
authGroups is a set of groups authorised for the user when
controlling access to embargoed products. This is the main
reason you should never hand out products yourself but always
expose the to the user through the product core.
The actual constructor requires a RAccref, which is exposed as the
rAccref attribute. Do not use the productsRow attribute from rAccref,
though, as constructors may want to manipulate the content of the
product row (e.g., in NonExistingProduct). Access the product
row as self.pr in product classes.
In addition to those, all Products have a name attribute,
which must be something suitable as a file name; the default
constructor calls a _makeName method to come up with one, and
you should simply override it.
The iterData method has to yield reasonably-sized chunks of
data (self.chunkSize should be a good choice). It must be
synchronous.
Products usually are used as t.w resources. Therefore, they
must have a render method. This must be asynchronuous,
i.e., it should not block for extended periods of time.
Products also work as rudimentary files via read and close
methods; by default, these are implemented on top of iterData.
Clients must never mix calls to the file interface and to
iterData. Derived classes that are based on actual files should
set up optimized read and close methods using the setupRealFile
class method (look for the getFile method on the instance to see
if there's a real file). Again, the assumption is made there that clients
use either iterData or read, but never both.
If a product knows how to (quickly) generate a preview for itself,
it can define a makePreview() method. This must return content
for a mime type conventional for that kind of product (which is laid
down in the products table).
"""
chunkSize = 2**16
_curIterator = None
def __init__(self, rAccref):
# If you change things here, change NonExistingProduct's constructor
# as well.
self.rAccref = rAccref
self.pr = self.rAccref.productsRow
self._makeName()
def _makeName(self):
self.name = "invalid product"
def __str__(self):
return "<%s %s (%s)>"%(self.__class__.__name__,
self.name,
self.pr["mime"])
def __repr__(self):
return str(self)
def __eq__(self, other):
return (isinstance(other, self.__class__)
and self.rAccref==other.rAccref)
def __ne__(self, other):
return not self==other
[docs] @classmethod
def fromRAccref(self, accref, authGroups=None):
return None # ProductBase is not responsible for anything.
[docs] @classmethod
def setupRealFile(cls, openMethod):
"""changes cls such that read and close work an an actual file-like
object rather than the inefficient iterData.
openMethod has to be an instance method of the class returning
an opened input file.
"""
cls._openedInputFile = None
def getFileMethod(self):
return openMethod(self)
def readMethod(self, size=None):
if self._openedInputFile is None:
self._openedInputFile = openMethod(self)
return self._openedInputFile.read(size)
def closeMethod(self):
if self._openedInputFile is not None:
self._openedInputFile.close()
self._openedInputFile = None
cls.read = readMethod
cls.close = closeMethod
cls.getFile = getFileMethod
[docs] def iterData(self):
raise NotImplementedError("Internal error: %s products do not"
" implement iterData"%self.__class__.__name__)
[docs] def render(self, request):
raise NotImplementedError("Internal error: %s products cannot be"
" rendered."%self.__class__.__name__)
[docs] def read(self, size=None):
if self._curIterator is None:
self._curIterator = self.iterData()
self._readBuf, self._curSize = [], 0
while size is None or self._curSize<size:
try:
chunk = next(self._curIterator)
except StopIteration:
break
self._readBuf.append(chunk)
self._curSize += len(chunk)
content = b"".join(self._readBuf)
if size is None:
self._readBuf, self._curSize = [], 0
result = content
else:
result = content[:size]
self._readBuf = [content[size:]]
self._curSize = len(self._readBuf[0])
return result
[docs] def close(self):
for _ in self.iterData():
pass
self._curIterator = None
[docs]class FileProduct(ProductBase):
"""A product corresponding to a local file.
As long as the accessPath in the RAccref's productsRow corresponds
to a real file and no params are in the RAccref, this will return
a product.
"""
forSaving = True
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if set(rAccref.params)-set(["preview"]): # not a plain file
return None
if os.path.exists(rAccref.localpath):
return cls(rAccref)
def _makeName(self):
self.name = os.path.basename(self.rAccref.localpath)
def _openUnderlyingFile(self):
return open(self.rAccref.localpath, "rb")
[docs] def iterData(self):
with self._openUnderlyingFile() as f:
data = f.read(self.chunkSize)
if not data:
return
yield data
[docs] def render(self, request):
if self.forSaving:
request.setHeader("content-disposition", 'attachment; filename="%s"'%
str(self.name))
if request.setLastModified(os.path.getmtime(self.rAccref.localpath)
)==http.CACHED:
request.finish()
return
res = static.File(self.rAccref.localpath)
# we set the type manually to avoid having different mime types
# by our and t.w's estimate. This forces us to clamp encoding
# to None now. I *guess* we should do something about .gz and .bz2
res.type = str(self.pr["mime"])
res.encoding = None
return res.render(request)
FileProduct.setupRealFile(FileProduct._openUnderlyingFile)
[docs]class StaticPreview(FileProduct):
"""A product that's a cached or pre-computed preview.
"""
forSaving = False
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if not rAccref.params.get("preview"):
return None
# no static previews on cutouts
if rAccref.params.get("sra"):
return None
previewPath = rAccref.productsRow["preview"]
localName = None
if previewPath is None:
return None
elif previewPath=="AUTO":
localName = PreviewCacheManager.getCachedPreviewPath(rAccref.accref)
else:
# remote URLs can't happen here as RemotePreview is checked
# before us.
localName = os.path.join(base.getConfig("inputsDir"), previewPath)
if localName is None:
return None
elif os.path.exists(localName):
rAccref.productsRow["accessPath"] = localName
rAccref.productsRow["mime"] = rAccref.productsRow["preview_mime"
] or "image/jpeg"
return cls(rAccref)
[docs]class RemoteProduct(ProductBase):
"""A class for products at remote sites, given by their URL.
"""
def _makeName(self):
self.name = urllib.parse.urlparse(self.pr["accessPath"]
).path.split("/")[-1] or "file"
def __str__(self):
return "<Remote %s at %s>"%(self.pr["mime"], self.pr["accessPath"])
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if REMOTE_URL_PATTERN.match(rAccref.productsRow["accessPath"]):
return cls(rAccref)
[docs] @classmethod
def fromURL(cls, url):
class rAccref:
productsRow = {"accessPath": url}
return cls(rAccref)
[docs] def iterData(self):
f = urllib.request.urlopen(self.pr["accessPath"])
while True:
data = f.read(self.chunkSize)
if not data:
break
yield data
[docs] def render(self, request):
raise svcs.WebRedirect(self.pr["accessPath"])
[docs]class RemotePreview(RemoteProduct):
"""A preview that's on a remote server.
"""
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if not rAccref.params.get("preview"):
return None
# no static previews on cutouts
if rAccref.params.get("sra"):
return None
if REMOTE_URL_PATTERN.match(rAccref.productsRow["preview"] or ""):
rAccref.productsRow["accessPath"] = rAccref.productsRow["preview"]
rAccref.productsRow["mime"] = rAccref.productsRow["preview_mime"]
return cls(rAccref)
[docs]class UnauthorizedProduct(FileProduct):
"""A local file that is not delivered to the current client.
iterData returns the data for the benefit of preview making.
However, there is a render method, so the product renderer will
not use it; it will, instead, raise an Authenticate exception.
"""
forSaving = False
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
dbRow = rAccref.productsRow
if (dbRow["embargo"] is None
or dbRow["embargo"]<datetime.date.today()):
return None
if authGroups is None or dbRow["owner"] not in authGroups:
return cls(rAccref)
def __str__(self):
return "<Protected product %s, access denied>"%self.name
def __eq__(self, other):
return self.__class__==other.__class__
[docs] def render(self, request):
raise svcs.Authenticate()
[docs]class NonExistingProduct(ProductBase):
"""A local file that went away.
iterData here raises an IOError, render an UnknownURI.
These should normally yield 404s.
We don't immediately raise some error here as archive generation
shouldn't fail just because a single part of it is missing.
"""
def __init__(self, rAccref):
# as rAccref.productsRow is bad here, don't call the base constructor
self.rAccref = rAccref
self.pr = {
'accessPath': None, 'accref': None,
'embargo': None, 'owner': None,
'mime': 'text/html', 'sourceTable': None,
'datalink': None, 'preview': None}
def __str__(self):
return "<Non-existing product %s>"%self.rAccref.accref
def __eq__(self, other):
return self.__class__==other.__class__
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
try:
rAccref.productsRow
except base.NotFoundError:
return cls(rAccref)
def _makeName(self):
self.name = "missing.html"
[docs] def iterData(self):
raise IOError("%s does not exist"%self.rAccref.accref)
[docs] def render(self, request):
raise svcs.UnknownURI("No dataset with accref %s known here."%
self.rAccref.accref)
[docs]class InvalidProduct(NonExistingProduct):
"""An invalid file.
This is returned by getProductForRAccref if all else fails. This
usually happens when a file known to the products table is deleted,
but it could also be an attempt to use unsupported combinations
of files and parameters.
Since any situation leading here is a bit weird, we probably
should be doing something else but just return a 404. Hm...
This class always returns an instance from fromRAccref; this means
any resolution chain ends with it. But it shouldn't be in
PRODUCT_CLASSES in the first place since the fallback is
hardcoded into getProductForRAccref.
"""
def __str__(self):
return "<Invalid product %s>"%self.rAccref
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
return cls(rAccref)
def _makeName(self):
self.name = "invalid.html"
[docs] def iterData(self):
raise IOError("%s is invalid"%self.rAccref)
[docs]class CutoutProduct(ProductBase):
"""A class representing cutouts from FITS files.
This only works for local FITS files with two axes. For everything
else, use datalink.
We assume the cutouts are smallish -- they are, right now, not
streamed, but accumulated in memory.
"""
def _makeName(self):
self.name = "<cutout-"+os.path.basename(self.pr["accessPath"])
def __str__(self):
return "<cutout-%s %s>"%(self.name, self.rAccref.params)
_myKeys = ["ra", "dec", "sra", "sdec"]
_myKeySet = frozenset(_myKeys)
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if (len(set(rAccref.params.keys())&cls._myKeySet)==4
and rAccref.productsRow["mime"]=="image/fits"):
return cls(rAccref)
def _getCutoutHDU(self):
ra, dec, sra, sdec = [self.rAccref.params[k] for k in self._myKeys]
with utils.fitsLock():
hdus = pyfits.open(self.rAccref.localpath,
do_not_scale_image_data=True,
memmap=True)
try:
skyWCS = coords.getWCS(hdus[0].header)
pixelFootprint = numpy.asarray(
numpy.round(skyWCS.wcs_world2pix([
(ra-sra/2., dec-sdec/2.),
(ra+sra/2., dec+sdec/2.)], 1)), numpy.int32)
res = fitstools.cutoutFITS(hdus[0],
(skyWCS.longAxis, min(pixelFootprint[:,0]), max(pixelFootprint[:,0])),
(skyWCS.latAxis, min(pixelFootprint[:,1]), max(pixelFootprint[:,1])))
finally:
hdus.close()
return res
[docs] def iterData(self):
# we need to write into a BytesIO (rather than to request) because
# pyfits wants to seek
res = self._getCutoutHDU()
bytes = BytesIO()
res.writeto(bytes)
bytes.seek(0)
while True:
res = bytes.read(self.chunkSize)
if not res:
break
yield res
def _writeStuffTo(self, destF):
for chunk in self.iterData():
destF.write(chunk)
[docs] def render(self, request):
request.setHeader("content-type", "image/fits")
return streaming.streamOut(self._writeStuffTo, request)
[docs] def makePreview(self):
img = imgtools.scaleNumpyArray(self._getCutoutHDU().data, PREVIEW_SIZE)
return imgtools.jpegFromNumpyArray(img)
[docs]class ScaledFITSProduct(ProductBase):
"""A class representing a scaled FITS file.
Right now, this only works for local FITS files. Still, the
class is constructed with a full rAccref.
"""
def __init__(self, rAccref):
ProductBase.__init__(self, rAccref)
self.scale = rAccref.params["scale"]
self.baseAccref = rAccref.accref
def __str__(self):
return "<%s scaled by %s>"%(self.name, self.scale)
[docs] @classmethod
def fromRAccref(cls, rAccref, authGroups=None):
if ("scale" in rAccref.params
and rAccref.productsRow["mime"]=="image/fits"):
return cls(rAccref)
def _makeName(self):
self.name = "scaled-"+os.path.basename(self.pr["accref"])
[docs] def iterData(self):
scale = int(self.scale)
if scale<2:
scale = 2
for stuff in fitstools.iterScaledBytes(
self.rAccref.localpath,
scale,
extraCards={"FULLURL": str(makeProductLink(self.baseAccref))}):
yield stuff
def _writeStuffTo(self, destF):
for chunk in self.iterData():
destF.write(chunk)
[docs] def render(self, request):
request.setHeader("content-type", "image/fits")
return streaming.streamOut(self._writeStuffTo, request)
# The following list is checked by getProductForRAccref in sequence.
# Each product is asked in turn, and the first that matches wins.
# So, ORDER IS ALL-IMPORTANT here.
PRODUCT_CLASSES = [
RemotePreview,
StaticPreview,
NonExistingProduct,
UnauthorizedProduct,
RemoteProduct,
CutoutProduct,
ScaledFITSProduct,
FileProduct,
]
[docs]def getProductForRAccref(rAccref, authGroups=None):
"""returns a product for a RAccref.
This tries, in sequence, to make a product using each element
of PRODUCT_CLASSES' fromRAccref method. If nothing succeeds,
it will return an InvalidProduct.
If rAccref is a string, the function makes a real RAccref through
RAccref's fromString method from it.
"""
if not isinstance(rAccref, RAccref):
rAccref = RAccref.fromString(rAccref)
for prodClass in PRODUCT_CLASSES:
res = prodClass.fromRAccref(rAccref, authGroups)
if res is not None:
return res
return InvalidProduct.fromRAccref(rAccref, authGroups)
[docs]class ProductCore(svcs.DBCore):
"""A core retrieving paths and/or data from the product table.
You will not usually mention this core in your RDs. It is mainly
used internally to serve /getproduct queries.
It is instantiated from within //products.rd and relies on
tables within that RD.
The input data consists of accref; you can use the string form
of RAccrefs, and if you renderer wants, it can pass in ready-made
RAccrefs. You can pass accrefs in through both an accref
param and table rows.
The accref param is the normal way if you just want to retrieve a single
image, the table case is for building tar files and such. There is one core
instance in //products for each case.
The core returns a list of instances of a subclass of ProductBase above.
This core and its supporting machinery handles all the fancy product
functionality (user authorization, cutouts, ...).
"""
name_ = "productCore"
def _getRAccrefs(self, inputTable):
"""returns a list of RAccref requested within inputTable.
"""
keys = []
args = inputTable.args
if args["accref"]:
keys.extend(RAccref.fromString(a) for a in args["accref"])
if args.get("pattern"):
try:
tablepat, filepat = args["pattern"].split("#")
except ValueError:
raise base.ValidationError(
"Must be of the form tablepattern#filepattern", "pattern")
with base.getTableConn() as conn:
for row in conn.queryToDicts(
"SELECT accref FROM dc.products"
" WHERE"
" accref LIKE %(filepat)s"
" AND sourceTable LIKE %(tablepat)s",
{"filepat": filepat, "tablepat": tablepat}):
keys.append(RAccref.fromString(row["accref"]))
return keys
def _getGroups(self, user, password):
if user is None:
return set()
else:
return creds.getGroupsForUser(user, password)
[docs] def run(self, service, inputTable, queryMeta):
"""returns a list of {"source": product} dicts for products matching
the inputTable.
"""
authGroups = self._getGroups(queryMeta["user"], queryMeta["password"])
return [getProductForRAccref(r, authGroups)
for r in self._getRAccrefs(inputTable)]
[docs]class RAccref(object):
"""A product key including possible modifiers.
The product key is in the accref attribute.
The modifiers come in the params dictionary. It contains (typed)
values, the possible keys of which are given in _buildKeys. The
values in passed in the inputDict constructor argument are parsed,
anything not in _buildKeys is discarded.
In principle, these modifiers are just the query part of a URL,
and they generally come from the arguments of a web request. However,
we don't want to carry around all request args, just those meant
for product generation.
One major reason for having this class is serialization into URL-parts.
Basically, stringifying a RAccref yields something that can be pasted
to <server root>/getproduct to yield the product URL. For the
path part, this means just percent-escaping blanks, plusses and percents
in the file name. The parameters are urlencoded and appended with
a question mark. This representation is be parsed by the fromString
function.
RAccrefs have a (read only) property productsRow attribute -- that's
a dictionary containing the row for accres from //products#products
if that exists. If it doesn't, accessing the property will raise
an NotFoundError.
"""
_buildKeys = dict((
("dm", str), # data model, VOTable generation
("ra", float), # cutouts
("dec", float), # cutouts
("sra", float), # cutouts
("sdec", float),# cutouts
("scale", int), # FITS scaling
("preview", base.parseBooleanLiteral), # return a preview?
))
def __init__(self, accref, inputDict={}):
self.accref = accref
self.params = self._parseInputDict(inputDict)
[docs] @classmethod
def fromPathAndArgs(cls, path, args):
"""returns a rich accref from a path and a parse_qs-dictionary args.
(it's mainly a helper for fromRequest and fromString).
"""
inputDict = {}
for key, value in args.items():
if len(value)>0:
inputDict[key] = value[-1]
# Save old URLs: if no (real) path was passed, try to get it
# from key. Remove this ca. 2014, together with
# RaccrefTest.(testPathFromKey|testKeyMandatory)
if not path.strip("/").strip():
if "key" in inputDict:
path = inputDict["key"]
else:
raise base.ValidationError(
"Must give key when constructing RAccref",
"accref")
return cls(path, inputDict)
[docs] @classmethod
def fromRequest(cls, path, request):
"""returns a rich accref from a t.w request.
Basically, it raises an error if there's no key at all, it will return
a (string) accref if no processing is desired, and it will return
a RAccref if any processing is requested.
"""
return cls.fromPathAndArgs(path, request.strargs)
[docs] @classmethod
def fromString(cls, keyString):
"""returns a fat product key from a string representation.
As a convenience, if keyString already is a RAccref,
it is returned unchanged.
"""
if isinstance(keyString, RAccref):
return keyString
qSep = keyString.rfind("?")
if qSep!=-1:
return cls.fromPathAndArgs(
unquoteProductKey(keyString[:qSep]),
urllib.parse.parse_qs(keyString[qSep+1:]))
return cls(unquoteProductKey(keyString))
@property
def productsRow(self):
"""returns the row in dc.products corresponding to this RAccref's
accref, or raises a NotFoundError.
"""
try:
return self._productsRowCache
except AttributeError:
res = base.resolveCrossId(PRODUCTS_TDID).doSimpleQuery(
fragments="accref=%(accref)s", params={"accref": self.accref})
if not res:
raise base.NotFoundError(self.accref, "accref", "product table",
hint="Product URLs may disappear, though in general they should"
" not. If you have an IVOID (pubDID) for the file you are trying to"
" locate, you may still find it by querying the ivoa.obscore table"
" using TAP and ADQL.")
self._productsRowCache = res[0]
# make sure whatever can end up being written to something
# file-like
for key in ["mime", "accessPath", "accref"]:
self._productsRowCache[key] = str(self._productsRowCache[key])
return self._productsRowCache
def __str__(self):
# See the class docstring on quoting considerations.
res = quoteProductKey(self.accref)
args = urllib.parse.urlencode(dict(
(k,str(v)) for k, v in sorted(self.params.items())))
if args:
res = res+"?"+args
return res
def __repr__(self):
return str(self)
def __eq__(self, other):
return (isinstance(other, RAccref)
and self.accref==other.accref
and self.params==other.params)
def __ne__(self, other):
return not self.__eq__(other)
def _parseInputDict(self, inputDict):
res = {}
for key, val in inputDict.items():
if val is not None and key in self._buildKeys:
try:
res[key] = self._buildKeys[key](val)
except (ValueError, TypeError):
raise base.ValidationError(
"Invalid value for constructor argument to %s:"
" %s=%r"%(self.__class__.__name__, key, val), "accref")
return res
@property
def localpath(self):
try:
return self._localpathCache
except AttributeError:
self._localpathCache = os.path.join(base.getConfig("inputsDir"),
self.productsRow["accessPath"])
return self._localpathCache
[docs] def previewIsCacheable(self):
"""returns True if the a preview generated for this rAccref
is representative for all representative rAccrefs.
Basically, scaled versions all have the same preview, cutouts do not.
"""
if "ra" in self.params:
return False
return True
[docs]def unquoteProductKey(key):
"""reverses quoteProductKey.
"""
return urllib.parse.unquote(key)
[docs]def getProductColumns(colSeq):
"""returns the columns within colSeq that contain product links of some
sort.
"""
return [col for col in colSeq if col.displayHint.get("type")=="product"]
[docs]@utils.document
def quoteProductKey(key):
"""returns key as getproduct URL-part.
If ``key`` is a string, it is quoted as a naked accref so it's usable
as the path part of an URL. If it's an ``RAccref``, it is just stringified.
The result is something that can be used after getproduct in URLs
in any case.
"""
if isinstance(key, RAccref):
return str(key)
return urllib.parse.quote(key)
rscdef.addProcDefObject("quoteProductKey", quoteProductKey)
[docs]@utils.document
def makeProductLink(key, withHost=True, useHost=None):
"""returns the URL at which a product can be retrieved.
key can be an accref string or an RAccref.
Note that this is using the preferred host as the basic URL. If
you are running dual-protocol http/https and you ingest results
of this function into the database, it is advisable to cut off
the scheme part of the URI (e.g., ``split(":", 1)[-1]``). In
data products served, DaCHS will then put in the scheme used
for the query.
DaCHS (almost always) also allows full http URIs as accrefs. These
will be returned unchanged.
"""
if hasattr(key, "startswith") and (
key.startswith("http://") or key.startswith("https://")):
return key
url = base.makeSitePath("/getproduct/%s"%RAccref.fromString(key))
if withHost:
if useHost is None:
useHost = base.getCurrentServerURL()
url = urllib.parse.urljoin(useHost, url)
return url
rscdef.addProcDefObject("makeProductLink", makeProductLink)
def _productMapperFactory(colDesc):
"""A value mapper factory for product links.
Within the DC, any column called accref, with a display hint of
type=product, a UCD of VOX:Image_AccessReference, or a utype
of Access.Reference may contain a key into the product table.
Here, we map those to links to the get renderer unless they look
like a URL to begin with.
"""
if not (
colDesc["utype"]=="ssa:Access.Reference"
or colDesc["ucd"]=="VOX:Image_AccessReference"
or colDesc["displayHint"].get("type")=="product"):
return
return functools.partial(
formatProductLink, useHost=base.getCurrentServerURL())
utils.registerDefaultMF(_productMapperFactory)