1 """
2 Initial setup for the file system hierarchy.
3
4 This module is supposed to create as much of the DaCHS file system environment
5 as possible. Take care to give sensible error messages -- much can go wrong
6 here, and it's nice if the user has a way to figure out what's wrong.
7 """
8
9
10
11
12
13
14
15 import datetime
16 import os
17 import sys
18 import textwrap
19 import warnings
20
21 import psycopg2
22
23 from gavo import base
24 from gavo import utils
25
26
28 sys.stderr.write("*** Error: %s\n\n"%msg)
29 if hint is not None:
30 sys.stderr.write(textwrap.fill(hint)+"\n")
31 sys.exit(1)
32
33
35 return "\n".join(s.strip() for s in s.split("\n"))+"\n"
36
37
39 rootDir = base.getConfig("rootDir")
40 if os.path.isdir(rootDir):
41 return
42 try:
43 os.makedirs(rootDir)
44 except os.error:
45 bailOut("Cannot create root directory %s."%rootDir,
46 "This usually means that the current user has insufficient privileges"
47 " to write to the parent directory. To fix this, either have rootDir"
48 " somewhere you can write to (edit /etc/gavorc) or create the directory"
49 " as root and grant it to your user id.")
50
51
53 if not os.path.isdir(path):
54 try:
55 os.makedirs(path)
56 except os.error as err:
57 bailOut("Could not create directory %s (%s)"%(
58 path, err))
59 except Exception as msg:
60 bailOut("Could not create directory %s (%s)"%(
61 path, msg))
62 if setGroupTo is not None:
63 stats = os.stat(path)
64 if stats.st_mode&0060!=060 or stats.st_gid!=setGroupTo:
65 try:
66 os.chown(path, -1, setGroupTo)
67 if makeWritable:
68 os.chmod(path, stats.st_mode | 0060)
69 except Exception as msg:
70 bailOut("Cannot set %s to group ownership %s, group writable"%(
71 path, setGroupTo),
72 hint="Certain directories must be writable by multiple user ids."
73 " They must therefore belong to the group %s and be group"
74 " writeable. The attempt to make sure that's so just failed"
75 " with the error message %s."
76 " Either grant the directory in question to yourself, or"
77 " fix permissions manually. If you own the directory and"
78 " sill see permission errors, try 'newgrp %s'"%(
79 base.getConfig("group"), msg, base.getConfig("group")))
80
81
82 _GAVO_WRITABLE_DIRS = set([
83 "stateDir",
84 "cacheDir",
85 "logDir",
86 "tempDir",
87 "uwsWD",])
88
89
93
94
127
128
130 destPath = os.path.join(base.getConfig("configDir"), "matplotlibrc")
131 if os.path.exists(destPath):
132 return
133 with open(destPath, "w") as f:
134 f.write("backend: Agg\n")
135
136
138 makeDirVerbose(os.path.join(base.getConfig("webDir"), "nv_static"),
139 groupId, False)
140
141
143 """returns a random string that may be suitable as a database password.
144
145 The entropy of the generated passwords should be close to 160 bits, so
146 the passwords themselves would probably not be a major issue. Of course,
147 within DaCHS they are stored in the file system in clear text...
148 """
149 return os.urandom(20).encode("base64")
150
151
153 """writes profiles with made-up passwords to DaCHS' config dir.
154
155 This will mess everything up when the users already exist. We
156 should probably provide an option to drop standard users.
157
158 userPrefix is mainly for the test infrastructure.
159 """
160 profilePath = base.getConfig("configDir")
161 dsnContent = ["database = %s"%(dsn.parsed["dbname"])]
162 if "host" in dsn.parsed:
163 dsnContent.append("host = %s"%dsn.parsed["host"])
164 else:
165 dsnContent.append("host = localhost")
166 if "port" in dsn.parsed:
167 dsnContent.append("port = %s"%dsn.parsed["port"])
168 else:
169 dsnContent.append("port = 5432")
170
171 for fName, content in [
172 ("dsn", "\n".join(dsnContent)+"\n"),
173 ("feed", "include dsn\nuser = %sgavoadmin\npassword = %s\n"%(
174 userPrefix, _genPW())),
175 ("trustedquery", "include dsn\nuser = %sgavo\npassword = %s\n"%(
176 userPrefix, _genPW())),
177 ("untrustedquery", "include dsn\nuser = %suntrusted\npassword = %s\n"%(
178 userPrefix, _genPW())),]:
179 destPath = os.path.join(profilePath, fName)
180 if not os.path.exists(destPath):
181 with open(destPath, "w") as f:
182 f.write(content)
183
184
186 """creates the directories required by DaCHS.
187
188 userPrefix is for use of the test infrastructure.
189 """
190 makeRoot()
191 grpId = base.getGroupId()
192 for configKey in ["configDir", "inputsDir", "cacheDir", "logDir",
193 "tempDir", "webDir", "stateDir"]:
194 makeDirForConfig(configKey, grpId)
195 makeDirVerbose(os.path.join(base.getConfig("inputsDir"), "__system"),
196 grpId, False)
197 makeDefaultMeta()
198 makeMatplotlibCfg()
199 makeProfiles(dsn, userPrefix)
200 prepareWeb(grpId)
201
202
203
204
205
206
208 """a psycopg-style DSN, both parsed and unparsed.
209 """
211 self.full = dsn
212 self._parse()
213 self._validate()
214
215 _knownKeys = set(["dbname", "user", "password", "host", "port", "sslmode"])
216
218 for key in self.parsed:
219 if key not in self._knownKeys:
220 sys.stderr.write("Unknown DSN key %s will get lost in profiles."%(
221 key))
222
229
230
232 """returns the result of running query with args through conn.
233
234 No transaction management is being done here.
235 """
236 cursor = conn.cursor()
237 cursor.execute(query, args)
238 return list(cursor)
239
240
242 return _execDB(conn,
243 "SELECT rolname FROM pg_roles WHERE rolname=%(rolname)s",
244 {"rolname": roleName})
245
246
248 cursor = conn.cursor()
249 try:
250 verb = "CREATE"
251 if _roleExists(conn, profile.user):
252 verb = "ALTER"
253 cursor.execute(
254 "%s ROLE %s PASSWORD %%(password)s %s LOGIN"%(
255 verb, profile.user, privileges), {
256 "password": profile.password,})
257 conn.commit()
258 except:
259 warnings.warn("Could not create role %s (see db server log)"%
260 profile.user)
261 conn.rollback()
262
263
265 """creates the roles for the DaCHS profiles admin, trustedquery
266 and untrustedquery.
267 """
268 from gavo.base import config
269
270 conn = psycopg2.connect(dsn.full)
271 for profileName, privileges in [
272 ("admin", "CREATEROLE"),
273 ("trustedquery", ""),
274 ("untrustedquery", "")]:
275 _createRoleFromProfile(conn,
276 config.getDBProfile(profileName),
277 privileges)
278
279 adminProfile = config.getDBProfile("admin")
280 cursor = conn.cursor()
281 cursor.execute("GRANT ALL ON DATABASE %s TO %s"%(dsn.parsed["dbname"],
282 adminProfile.user))
283 conn.commit()
284
285
287 """returns the path where a local postgres server would store its
288 contrib scripts.
289
290 This is probably Debian specific. It's used by the the extension
291 script upload.
292 """
293 from gavo.base import sqlsupport
294 version = sqlsupport.parseBannerString(
295 _execDB(conn, "SELECT version()")[0][0])
296 name = "/usr/share/postgresql/%s/contrib"%version
297 if os.path.isdir(name):
298 return name
299 name = "/usr/share/postgresql/contrib"
300
301 return name
302
303
305 """tries to execute the sql script in scriptPath within conn.
306
307 sourceName is some user-targeted indicator what package the script
308 comes from, procName the name of a procedure left by the script
309 so we don't run the script again when it's already run.
310 """
311 if not os.path.exists(scriptPath):
312 warnings.warn("SQL script file for %s not found. There are many"
313 " reasons why that may be ok, but unless you know what you are"
314 " doing, you probably should install the corresponding postgres"
315 " extension."%scriptPath)
316 from gavo.rscdef import scripting
317
318 cursor = conn.cursor()
319 if _execDB(conn, "SELECT * FROM pg_proc WHERE proname=%(procName)s",
320 {"procName": procName}):
321
322 return
323
324 try:
325 for statement in scripting.getSQLScriptGrammar().parseString(
326 open(scriptPath).read()):
327 cursor.execute(statement)
328 except:
329 conn.rollback()
330 warnings.warn("SQL script file %s failed. Try running manually"
331 " using psql. While it hasn't run, the %s extension is not"
332 " available."%(scriptPath, sourceName))
333 else:
334 conn.commit()
335
336
338 """tries to create the extension extName.
339
340 This is for new-style extensions (e.g., pgsphere starting from 1.1.1.7)
341 that don't have a load script any more.
342
343 It returns True if the extension was found (and has created it as a
344 side effect).
345 """
346 res = _execDB(conn, "SELECT default_version, installed_version"
347 " FROM pg_available_extensions"
348 " WHERE name=%(name)s", {"name": extName})
349 if not res:
350
351 return False
352
353 if res[0][1] is not None:
354
355
356
357 return True
358
359 cursor = conn.cursor()
360 cursor.execute("CREATE EXTENSION "+extName)
361 cursor.close()
362 return True
363
364
366 """executes some commands that need to be executed with superuser
367 privileges.
368 """
369
370 conn = psycopg2.connect(dsn.full)
371 for statement in [
372 "CREATE OR REPLACE LANGUAGE plpgsql"]:
373 cursor = conn.cursor()
374 try:
375 cursor.execute(statement)
376 except psycopg2.DatabaseError as msg:
377 warnings.warn("SQL statement '%s' failed (%s); continuing."%(
378 statement, msg))
379 conn.rollback()
380 else:
381 conn.commit()
382
383
385 """loads definitions of pgsphere, q3c and similar into the DB.
386
387 This only works for local installations, and the script location
388 is more or less hardcoded (Debian and SuSE work, at least).
389 """
390 conn = psycopg2.connect(dsn.full)
391 scriptPath = _getServerScriptPath(conn)
392 for extScript, pkgName, procName, extName in [
393 ("pg_sphere.sql", "pgSphere", "spoint_in", "pg_sphere"),
394 ("q3c.sql", "q3c", "q3c_ang2ipix", "q3c")]:
395
396 if not _loadPgExtension(conn, extName):
397 _readDBScript(conn,
398 os.path.join(scriptPath, extScript),
399 pkgName,
400 procName)
401 conn.commit()
402
403
417
418
420 """creates users and tables expected by DaCHS in the database described
421 by the DSN dsn.
422
423 Connecting with dsn must give you superuser privileges.
424 """
425 _createRoles(dsn)
426 _doLocalSetup(dsn)
427 _readDBScripts(dsn)
428 _importBasicResources()
429
430
432 import argparse
433 parser = argparse.ArgumentParser(description="Create or update DaCHS'"
434 " file system and database environment.")
435 parser.add_argument("-d", "--dsn", help="DSN to use to connect to"
436 " the future DaCHS database. The DSN must let DaCHS connect"
437 " to the DB as an administrator. dbname, host, and port"
438 " get copied to the profile, if given. The DSN looks roughly like"
439 ' "host=foo.bar user=admin password=secret". If you followed the'
440 " installation instructions, you don't need this option.",
441 action="store", type=str, dest="dsn", default="gavo")
442 parser.add_argument("--nodb", help="Inhibit initialization of the"
443 " database (you may want to use this when refreshing the file system"
444 " hierarchy)", action="store_false", dest="initDB")
445 return parser.parse_args()
446
447
449 """initializes the DaCHS environment (where that's not already done).
450 """
451 opts = parseCommandLine()
452 dsn = DSN(opts.dsn)
453 createFSHierarchy(dsn)
454 if opts.initDB:
455 initDB(dsn)
456