"""
DC administration interface.
"""
#c Copyright 2008-2023, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL. See the
#c COPYING file in the source distribution.
import pathlib
import re
import sys
from gavo import base
from gavo import utils
from gavo import rscdef
from gavo import rscdesc #noflake: for cache registration
from gavo import svcs
from gavo.protocols import creds
from gavo.protocols import uws
from gavo.utils import Arg, exposedFunction, makeCLIParser
[docs]@exposedFunction([
Arg("user", help="the user name"),
Arg("password", help="a password for the user"),
Arg("remarks", help="optional remarks",
default="", nargs='?')],
help="add a user/password pair and a matching group to the DC server")
def adduser(querier, args):
try:
creds.addUser(querier.connection, args.user, args.password, args.remarks)
except base.IntegrityError:
raise base.ui.logOldExc(base.ReportableError("User %s already exists."
" Use 'changeuser' command to edit."%args.user))
[docs]@exposedFunction([
Arg("user", help="the user name to remove")],
help="remove a user from the DC server")
def deluser(querier, args):
rowsAffected = creds.delUser(querier.connection, args.user)
if not rowsAffected:
sys.stderr.write("Warning: No rows deleted while deleting user %s\n"%
args.user)
[docs]@exposedFunction([
Arg("user", help="the user name"),
Arg("password", help="a password for the user"),
Arg("remarks", help="optional remarks",
default="", nargs='?')],
help="change remarks and/or password for a DC user")
def changeuser(querier, args):
creds.changeUser(querier.connection,
args.user, args.password, args.remarks)
[docs]@exposedFunction([
Arg("user", help="a user name"),
Arg("group", help="the group to add the user to")],
help="add a user to a group")
def addtogroup(querier, args):
try:
creds.addToGroup(querier.connection, args.user, args.group)
except Exception:
raise base.ui.logOldExc(base.ReportableError(
"User %s does not exist."%args.user))
[docs]@exposedFunction([
Arg("user", help="a user name"),
Arg("group", help="the group to remove the user from")],
help="remove a user from a group")
def delfromgroup(querier, args):
if not creds.removeFromGroup(querier.connection, args.user, args.group):
sys.stderr.write("Warning: No rows deleted while deleting user"
" %s from group %s\n"%(args.user, args.group))
[docs]@exposedFunction(help="list users known to the DC")
def listusers(querier, args):
data = list(querier.connection.query("SELECT username, groupname, remarks"
" FROM dc.users NATURAL JOIN dc.groups ORDER BY username"))
curUser = None
for user, group, remark in data:
if user!=curUser:
print("\n%s (%s) --"%(user, remark), end=' ')
curUser = user
print(group, end=' ')
print()
[docs]@exposedFunction([
Arg("-f", help="also remove all jobs in ERROR and ABORTED states (only use"
" if you are sure what you are doing).", action="store_true",
dest="includeFailed"),
Arg("-p", help="also remove all jobs in PENDING states (only use"
" if you are sure what you are doing).", action="store_true",
dest="includeForgotten"),
Arg("--all", help="remove all jobs (this is extremely unfriendly."
" Don't use this on public UWSes)", action="store_true",
dest="includeAll"),
Arg("--nuke-completed", help="also remove COMPLETEd jobs (this is"
" unfriendly. Don't do this on public UWSes).", action="store_true",
dest="includeCompleted"),],
help="remove expired UWS jobs")
def cleantap(querier, args):
from gavo.protocols import tap
tap.WORKER_SYSTEM.cleanupJobsTable(includeFailed=args.includeFailed,
includeCompleted=args.includeCompleted,
includeAll=args.includeAll,
includeForgotten=args.includeForgotten)
[docs]@exposedFunction([
Arg("jobId", help="id of the job to abort"),
Arg("helpMsg", help="A helpful message to add to the abort message")],
help="manually abort a TAP job and send some message to a user")
def tapabort(querier, args):
from gavo.protocols import tap
tap.WORKER_SYSTEM.changeToPhase(args.jobId, uws.ERROR,
"Job aborted by an administrator, probably because the query\n"
" should be written differently to be less of a resource hog.\n"
" Here's what the administrator had to say:\n\n"+args.helpMsg+
"\n\nIf you have further questions, just send a mail to "+
base.getMetaText(base.caches.getRD("//tap").getById("run"),
"contact.email"))
[docs]@exposedFunction([Arg(help="rd#table-id of the table containing the"
" products that should get cached previews", dest="tableId"),
Arg("-w", type=str,
help="width to compute the preview for", dest="width", default="200"),],
help="Precompute previews for the product interface columns in a table.")
def cacheprev(querier, args):
from gavo.protocols.products import PreviewCacheManager, getProductForRAccref
from twisted.internet import reactor
td = base.resolveId(None, args.tableId)
rows = querier.queryToDicts(
td.getSimpleQuery(["accref", "mime"]))
def runNext(token):
try:
row = next(rows)
res = PreviewCacheManager.getPreviewFor(
getProductForRAccref(row["accref"]))
if getattr(res, "result", None): # don't add a callback on a
# fired deferred or you'll exhaust the stack
reactor.callLater(0.1, runNext, "continue")
else:
res.addCallback(runNext)
return res
except StopIteration:
pass
except:
import traceback
traceback.print_exc()
reactor.stop()
return ""
reactor.callLater(0, runNext, "startup")
reactor.run()
[docs]@exposedFunction([Arg(help="rd#table-id of the table to look at",
dest="tableId")],
help="Make suggestions for UCDs of columns not having one (based"
" on their descriptions; this uses a GAVO web service).")
def suggestucds(querier, args):
from gavo import api
from gavo import votable
apiURL = "http://dc.zah.uni-heidelberg.de/ucds/ui/ui/api"
def getMatches(description):
res = utils.urlopenRemote(apiURL, data={"description": description})
data, metadata = votable.load(res)
if metadata:
return list(metadata.iterDicts(data))
else:
return []
td = api.getReferencedElement(args.tableId, forceType=api.TableDef)
for col in td:
if (not col.ucd or col.ucd=="XXX") and col.description:
try:
res = [(row["score"], row["ucd"], row["is_valid"])
for row in getMatches(col.description)]
res.sort()
res.reverse()
print(col.name)
for score, ucd, is_valid in res:
if is_valid:
print(" ", ucd)
else:
print(f" (invalid: {ucd})")
except IOError:
# remote failure, guess it's "no matches" (TODO: distinguish)
pass
[docs]@exposedFunction([Arg(help="rd#table-id of the table of interest",
dest="tableId")],
help="Show the statements to create the indices on a table.")
def indexStatements(querier, args):
import re
td = rscdef.getReferencedElement(args.tableId, forceType=rscdef.TableDef)
for ind in td.indices:
print("\n".join(re.sub(r"\s+", " ", s) for s in ind.iterCode()))
[docs]@exposedFunction([Arg(help="rd#exec-id of the execute element to run (note:"
" the title won't work, you have to give the thing an id to use adm exec).",
dest="execId")],
help="Execute the contents of an RD execute element. You must"
" give that element an explicit id in order to make this work.")
def execute(querier, args):
from gavo.user import logui
logui.LoggingUI(base.ui)
execEl = rscdef.getReferencedElement(args.execId, rscdef.Execute)
execEl.callable().join()
[docs]@exposedFunction([Arg(help="Package resource path"
" (like '/inputs/__system__/scs.rd); for system RDs, the special"
" //rd-id syntax is supported.",
dest="path")],
help="Dump the source of a distribution file; this is useful when you want"
" to override them and you are running DaCHS from a zipped egg")
def dumpDF(querier, args):
import pkg_resources
if args.path.startswith("//"):
args.path = "inputs/__system__"+args.path[1:]+".rd"
try:
with pkg_resources.resource_stream('gavo', "resources/"+args.path) as f:
sys.stdout.buffer.write(f.read())
except FileNotFoundError:
raise base.ReportableError("No such distribution file: %s\n"%args.path)
[docs]@exposedFunction([Arg(help="XML file", dest="path", nargs='+')],
help="Validate a file against built-in VO schemas and with built-in"
" schema validator.")
def xsdValidate(querier, args):
from gavo.helpers import testtricks
rtval = 0
for path in args.path:
print(path, end=" -- ")
try:
with open(path, "rb") as f:
msgs = testtricks.getXSDErrors(f.read())
except Exception as ex:
msgs = [str(ex)]
if not msgs:
print("valid")
else:
print(msgs)
rtval = 1
return rtval
[docs]@exposedFunction([Arg(help="IVOID to mark as deleted", dest="ivoid")],
help="Add a registry entry for a deleted record with IVOID. This"
" should only be necessary if you accidentally manually removed"
" records from your dc.resources table.")
def makedelrec(querier, args):
from gavo import registry
authority, path = registry.parseIdentifier(args.ivoid)
if authority not in registry.getManagedAuthorities():
raise base.ReportableError("You can only declare ivo ids from your"
" own authority as deleted.")
registry.makeDeletedRecord(args.ivoid, querier.connection)
[docs]@exposedFunction([], help="Update the TAP_SCHEMA metadata for all"
" RDs mentioned in TAP_SCHEMA.")
def updateTAPSchema(querier, args):
from gavo.protocols import tap
for rdId, in querier.connection.query(
"select sourcerd from TAP_SCHEMA.tables"):
try:
rd = base.caches.getRD(rdId)
tap.publishToTAP(rd, querier.connection)
except base.NotFoundError as msg:
base.ui.notifyWarning("Stale records in TAP_SCHEMA: %s"%msg)
[docs]@exposedFunction([Arg(help="Password to hash", dest="pw")],
help="Hash a password (typically for [web]adminpasswd)")
def hashPasswd(querier, args):
from gavo.protocols import creds
print(creds.hashPassword(args.pw))
def _getConstantPrefix(path):
"""returns the segments of path without a wildcard.
"""
segs = []
for seg in path.split("/"):
if "*" in seg or "?" in seg:
break
segs.append(seg)
return "/".join(segs)
[docs]@exposedFunction([Arg(help="Id of a data element importing what you want to"
" turn into a HiPS", dest="dataId"),
Arg(help="Minimal Order to generate (0 is full sky, 4 is an area about"
" 4 degrees in diameter)", dest="minOrder")],
help="Write a Hipsgen parameter file to stdout")
def hipsgen(querier, args):
data = rscdef.getReferencedElement(args.dataId,
forceType=rscdef.DataDescriptor)
for pat in data.sources.patterns:
print("in={}".format(_getConstantPrefix(pat)))
print("minOrder={}".format(args.minOrder))
for m in data.rd.iterMeta("creator.name"):
print("creator: {}".format(m.getContent("text")))
break
# now find some publication we can piggyback on; we'll take the
# first we find.
from gavo.registry import publication
for dest, rec in publication.RDRscRecGrammar(None).parse(data.rd):
if dest=="resources":
print("id={}".format(rec["ivoid"]))
break
print("status=public clonable")
print("title={}".format(base.getMetaText(data, "title")))
print("out=hips")
def _normalizeWhitespace(s):
return re.sub("\s+", " ", s)
def _getBoundsFromIntervals(intervals):
return (min((i[0] for i in intervals), default=None),
max((i[1] for i in intervals), default=None))
def _getHipsFillers(svc):
"""returns a dictionary of HiPS propertes keys to values fillable
from svcs.
"""
res = {}
for metaKey, hipsKey in [
("description", "obs_description"),
("creator.name", "hips_creator"),
("source", "bib_reference"),
("rights", "obs_copyright"),
("coverage.waveband", "obs_regime"),
("rights.rightsURI", "obs_copyright_url"),]:
for m in svc.rd.iterMeta(metaKey):
res[hipsKey] = _normalizeWhitespace(m.getContent(
"text", macroPackage=svc.rd))
break
res["hips_service_url"] = svc.getURL("hips")
minTime, maxTime = _getBoundsFromIntervals(svc.rd.coverage.temporal)
minE, maxE = _getBoundsFromIntervals(svc.rd.coverage.spectral)
if minE:
maxLambda = base.PLANCK_H*base.LIGHT_C/minE
if maxE:
minLambda = base.PLANCK_H*base.LIGHT_C/maxE
for hipsKey, val in [
("t_min", minTime),
("t_max", maxTime),
("em_min", minLambda),
("em_max", maxLambda)]:
if val is not None:
res[hipsKey] = val
return res
def _editProp(propLn, fillers):
"""edits a hips properties template line with fillers if appropriate.
What we edit is lines commented out with keys present in fillers.
Unedited lines are returned stripped but otherwise unchanged.
"""
propLn = propLn.strip()
mat = re.match("(#?)(\w+)\s+=\s+(.*)", propLn)
if not mat:
raise base.ReportableError(f"Invalid properties line: {propLn}")
isTemplate, key, value = mat.groups()
if isTemplate and key in fillers:
propLn = "{:21s}= {}".format(key, fillers[key])
return propLn
[docs]@exposedFunction([Arg(help="Reference to the HiPS-serving service",
dest="svcId"),
Arg("-d", "--hipsdir", help="Directory the HiPS is stored in",
type=pathlib.Path, dest="hipsDir", default="hips")],
help="Fill out the templated elements in hipsDir's properties file.")
def hipsfill(querier, args):
svc = rscdef.getReferencedElement(args.svcId,
forceType=svcs.Service)
fillers = _getHipsFillers(svc)
propFile = args.hipsDir/"properties"
newLines = []
with open(propFile, "r", encoding="utf-8") as f:
for ln in f:
newLines.append(_editProp(ln, fillers))
newContent = "\n".join(newLines).encode("utf-8")+b"\n"
with open(propFile, "wb") as f:
f.write(newContent)
[docs]def main():
with base.AdhocQuerier(lambda: base.getDBConnection("feed")) as querier:
args = makeCLIParser(globals()).parse_args()
retval = args.subAction(querier, args) or 0
sys.exit(retval)