1 """
2 Stuff dealing with the upgrade of the database schema.
3
4 From software version 0.8.2 on, there is a dc.metastore table with a key
5 schemaversion. Each change in the central schema increases the value
6 (interpreted as an integer) by one, and this module will contain a
7 corresponding upgrader.
8
9 An upgrader inherits form the Upgrader class. See there for more details.
10
11 This module contains the current schemaversion expected by the software; gavo
12 upgrade does everything required to bring the what's in the database in sync
13 with the code (or so I hope).
14 """
15
16
17
18
19
20
21
22 from __future__ import print_function
23
24 import sys
25
26 from psycopg2 import extensions
27
28 from gavo import base
29 from gavo import rsc
30 from gavo import rscdesc
31 from gavo import utils
35 """A sentinel used by iterStatements.
36 """
37
38
39 CURRENT_SCHEMAVERSION = 20
49
56
59 """re-ingests TAP_SCHEMA metadata for all RDs already mentioned
60 in TAP_SCHEMA.
61 """
62 from gavo.protocols import tap
63
64 toDo = [r[0] for r in
65 connection.query(
66 "SELECT DISTINCT sourceRD FROM dc.tablemeta")]
67
68 for rdId in toDo:
69 try:
70 tap.publishToTAP(
71 base.caches.getRD(rdId, doQueries=False), connection)
72 except Exception as msg:
73 base.ui.notifyWarning("RD %s couldn't be loaded or ingested to"
74 " TAP_SCHEMA (%s). Fix and run dachs imp -m on it to have"
75 " up-to-date metadata in TAP_SCHEMA"%(rdId, msg))
76 continue
77
80 """a string with an annotation.
81
82 This is (optionally) used to hold SQL statements here; the annotation
83 is shown to the user instead of the naked statement when present.
84 """
85 - def __new__(cls, content, annotation):
89
92 """outputs msg to stdout without lf "immediately".
93 """
94 sys.stdout.write(msg)
95 sys.stdout.flush()
96
99 """returns the schemaversion given in the database.
100
101 This will return -1 if no schemaversion is declared.
102 """
103 try:
104 return int(base.getDBMeta("schemaversion"))
105 except (KeyError, base.DBError):
106 return -1
107
110 """prints SQL to bring [db]managedExtensions up to date.
111 """
112 statements = []
113
114 for extName in base.getConfig("db", "managedextensions"):
115 res = list(conn.query(
116 "SELECT default_version, installed_version"
117 " FROM pg_available_extensions"
118 " WHERE name=%(extName)s", locals()))
119
120 if res:
121 def_version, ins_version = res[0]
122 if ins_version is None:
123
124
125 if extName in ['q3c', 'pg_sphere']:
126 statements.append("CREATE EXTENSION %s FROM unpackaged;"%extName)
127 else:
128 statements.append("CREATE EXTENSION %s;"%extName)
129
130 elif ins_version!=def_version:
131 statements.append("ALTER EXTENSION %s UPDATE TO %s;"%(
132 extName,
133 extensions.adapt(def_version).getquoted()))
134
135
136
137 if statements:
138 print(("\n".join(statements)+"\n"))
139 return 0
140
141 else:
142 return 1
143
146 """A specification to upgrade from some schema version to another schema
147 version.
148
149 Upgraders live as uninstanciated classes. Their version attribute gives the
150 version their instructions update *from*; their destination version
151 therefore is version+1.
152
153 Each upgrader has attributes named u_<seqno>_<something>. These can
154 be either strings, which are then directly executed in the database,
155 or class methods, which will be called with a connection argument. You
156 must not commit this connection. You must not swallow exceptions
157 that have left the connection unready (i.e., require a rollback).
158
159 Note that if you run rsc.makeData, you MUST pass both
160 connection=connection and runCommit=False in order to avoid messing
161 people's lives up.
162
163 The individual upgrader classmethods will be run in the sequence
164 given by the sequence number.
165
166 The updaters should have 1-line docstrings explaining what they do.
167
168 The update of the schemaversion is done automatically, you don't
169 need to worry about it.
170 """
171 version = None
172
173 @classmethod
178
179 @classmethod
181 """returns strings and classmethods that, in all, perform the necessary
182 upgrade.
183 """
184 for cmdAttr in (s for s in sorted(dir(cls)) if s.startswith("u_")):
185 yield getattr(cls, cmdAttr)
186 yield cls.updateSchemaversion
187
190 """This is executed when there's no schema version defined in the database.
191
192 The assumption is that the database reflects the status of 0.8, so
193 it adds the author column in dc.services if necessary (which it's
194 not if the software has been updated to 0.8.1).
195 """
196 version = -1
197
198 @classmethod
200 """add an author column to dc.services if necessary"""
201 if "authors" in list(connection.queryToDicts(
202 "SELECT * FROM dc.resources LIMIT 1"))[0]:
203 return
204 connection.query("alter table dc.resources add column authors")
205 for sourceRD, resId in connection.query("select sourcrd, resid"
206 " from dc.resources"):
207 try:
208 res = base.getRD(sourceRD).getById(resId)
209 authors = "; ".join(m.getContent("text")
210 for m in res.iterMeta("creator.name", propagate=True))
211 except:
212
213 pass
214 else:
215 connection.query("update dc.resources set authors=%(authors)s"
216 " where resid=%(resId)s and sourcerd=%(sourceRD)s",
217 locals())
218
219 @classmethod
224
227 version = 0
228
229 @classmethod
234
237 version = 1
238
239 @classmethod
252
253
254 @classmethod
270
273 version = 2
274
275 @classmethod
280
281 @classmethod
291
294 version = 3
295
296 @classmethod
301
304 version = 4
305
306 @classmethod
315
333
340
345
348 version = 8
349 u_010_chuckADQLPrefix = AnnotatedString("UPDATE TAP_SCHEMA.columns"
350 " SET datatype=substring(datatype from 6)"
351 " WHERE datatype LIKE 'adql:%%'",
352 "Remove adql: prefix in TAP_SCHEMA.columns.datatype")
353 u_020_setSize1OnAtoms = AnnotatedString("UPDATE tap_schema.columns"
354 " SET \"size\"=1 WHERE NOT datatype LIKE '%%(*)'",
355 "Set size=1 in TAP_SCHEMA.columns for atomic types")
356 u_030_removeArrayMarkInText = AnnotatedString("UPDATE tap_schema.columns"
357 " SET datatype=replace(datatype, '(*)', '') WHERE datatype LIKE '%%(*)'",
358 "Turn VARCHAR(*) into simple VARCHAR (size=NULL already set for those)")
359
362 version = 9
363
364 @classmethod
369
370 @classmethod
376
379 version = 10
380
381 @classmethod
383 """inform about tables with non-trivial mixins."""
384
385
386
387
388
389 print("\n!! Important: column sequences"
390 " of tables with some mixins have changed.")
391 print("!! If this affects you, below commands are shown that will re-import")
392 print("!! the affected tables. Some services on top of these tables may")
393 print("!! be *broken* until these commands have run.")
394 print("!! Sorry for this inconvenience; we hope it won't happen again.\n")
395
396 from gavo import registry
397 for rdId in registry.findAllRDs():
398 if rdId.startswith("__system"):
399 continue
400
401 try:
402 rd = base.caches.getRD(rdId)
403 except:
404
405 continue
406
407 ids = set()
408
409 for td in rd.tables:
410 try:
411 td.getColumnByName("accref") and td.getColumnByName("embargo")
412 except base.NotFoundError:
413 continue
414 else:
415
416 if not rsc.TableForDef(td, connection=connection, create=False
417 ).exists():
418 continue
419
420
421
422 for dd in rd.dds:
423 for make in dd.makes:
424 if make.table==td:
425 ids.add(dd.id)
426 if ids:
427 print("dachs imp '%s' %s"%(rd.sourceId,
428 " ".join("'%s'"%id for id in ids)))
429
430 sys.stderr.write("\nEnd of scan of mixin-affected tables...")
431
434 version = 11
435
436 @classmethod
445
457
460 version = 13
461
462 @classmethod
464 """Adding column_index column to TAP_SCHEMA.columns"
465 """
466 dbCols = getColumnNamesFor("//tap#columns", connection)
467 if not "column_index" in dbCols:
468 connection.execute("ALTER TABLE TAP_SCHEMA.columns"
469 " ADD COLUMN column_index SMALLINT")
470 if not "arraysize" in dbCols:
471 connection.execute("ALTER TABLE TAP_SCHEMA.columns"
472 " ADD COLUMN arraysize TEXT")
473 if not "xtype" in dbCols:
474 connection.execute("ALTER TABLE TAP_SCHEMA.columns"
475 " ADD COLUMN xtype TEXT")
476
479 version = 14
480
481 @classmethod
493
496 version = 15
497
498 u_010_delete_obscore_1_0_model = AnnotatedString(
499
500
501
502
503 "DELETE FROM tap_schema.supportedmodels"
504 " WHERE dmivorn='ivo://ivoa.net/std/ObsCore/v1.0'",
505 "Deleting obscore 1.0 model support declaration")
506
507 @classmethod
509 """adding schema_index to tap_schema.schemas"""
510
511
512 if not "schema_index" in getColumnNamesFor("//tap#schemas", connection):
513 connection.execute("ALTER TABLE tap_schema.schemas ADD COLUMN"
514 " schema_index INTEGER")
515
516 @classmethod
518 """Adding 1.1 columns to TAP_SCHEMA.tables.
519 """
520 if not "table_index" in getColumnNamesFor("//tap#tables", connection):
521 connection.execute("ALTER TABLE TAP_SCHEMA.tables"
522 " ADD COLUMN table_index SMALLINT")
523
524 @classmethod
526 """Adding 1.1 columns to TAP_SCHEMA.columns.
527 """
528 dbCols = getColumnNamesFor("//tap#columns", connection)
529 if not "arraysize" in dbCols:
530 connection.execute("ALTER TABLE TAP_SCHEMA.columns"
531 " ADD COLUMN arraysize TEXT")
532 if not "xtype" in dbCols:
533 connection.execute("ALTER TABLE TAP_SCHEMA.columns"
534 " ADD COLUMN xtype TEXT")
535
536 @classmethod
538 """Filling new TAP_SCHEMA columns"""
539 _updateTAP_SCHEMA(connection)
540
541 @classmethod
566
569 version = 16
570
571 @classmethod
577
578 @classmethod
585
586 @classmethod
594
597 version = 17
598
599 @classmethod
601 """recreating a possibly damaged TAP_SCHEMA"""
602 _updateTAP_SCHEMA(connection)
603
606 version = 18
607
608 @classmethod
610 """adding schema_index to tap_schema.schemas"""
611 if not "schema_index" in getColumnNamesFor("//tap#schemas", connection):
612 connection.execute("ALTER TABLE tap_schema.schemas ADD COLUMN"
613 " schema_index INTEGER")
614
615 @classmethod
617 """re-importing TAP_SCHEMA to update foreign key declarations"""
618 _updateTAP_SCHEMA(connection)
619
622 version = 19
623
624 @classmethod
631
632 @classmethod
638
642 """yields all upgraders from startVersion to endVersion in sequence.
643 """
644 toRun = []
645 for upgrader in utils.iterDerivedClasses(Upgrader,
646 upgraders or globals().values()):
647 if startVersion<=upgrader.version<endVersion:
648 toRun.append(upgrader)
649 toRun.sort(key=lambda upgrader:upgrader.version)
650 for upgrader in toRun:
651 for statement in upgrader.iterStatements():
652 yield statement
653 yield _COMMIT
654
655
656 -def upgrade(forceDBVersion=None):
657 """runs all updates necessary to bring a database to the
658 CURRENT_SCHEMAVERSION.
659
660 Unless catastrophic things go on, each upgrade is a transaction
661 of its own; the first failed transaction stops the upgrade at the
662 version last successfully upgraded to.
663 """
664 if forceDBVersion is None:
665 startVersion = getDBSchemaVersion()
666 else:
667 startVersion = forceDBVersion
668
669 with base.getWritableAdminConn() as conn:
670 for statement in iterStatements(startVersion, CURRENT_SCHEMAVERSION):
671 if statement is _COMMIT:
672 conn.commit()
673
674 elif callable(statement):
675 if statement.__doc__:
676 showProgress("> %s..."%statement.__doc__)
677
678
679 statement(conn)
680
681 else:
682 showProgress("> "+getattr(statement, "annotation",
683 "executing %s"%utils.makeEllipsis(statement, 60))+"... ")
684 conn.execute(statement)
685 showProgress(" ok\n")
686
689 import argparse
690 parser = argparse.ArgumentParser()
691 parser.add_argument("--force-dbversion", help="assume this as the"
692 " database's schema version. If you don't develop DaCHS, you"
693 " almost certainly should stay clear of this flag", type=int,
694 dest="forceDBVersion", default=None)
695 parser.add_argument("-e", "--get-extension-script",
696 help="Dump a script to update DaCHS-managed extensions (will"
697 " print nothing if no extensions need updating). This will return"
698 " 0 if material was written, 1 otherwise.",
699 dest="dumpExtScript", action="store_true")
700 return parser.parse_args()
701
704 args = parseCommandLine()
705 if args.dumpExtScript:
706 with base.getTableConn() as conn:
707 sys.exit(dumpExtensionUpdater(conn))
708 else:
709 upgrade(args.forceDBVersion)
710