source: grass/tags/release_20170810_grass_7_2_2RC1/lib/python/temporal/core.py

Last change on this file was 69812, checked in by neteler, 8 years ago

Numerous typos fixed (identified with tools/fix_typos.sh) (trunk, r69811)

  • Property svn:eol-style set to native
  • Property svn:mime-type set to text/x-python
File size: 49.5 KB
Line 
1"""
2This module provides the functionality to create the temporal
3SQL database and to establish a connection to the database.
4
5Usage:
6
7.. code-block:: python
8
9 >>> import grass.temporal as tgis
10 >>> # Create the temporal database
11 >>> tgis.init()
12 >>> # Establish a database connection
13 >>> dbif, connected = tgis.init_dbif(None)
14 >>> dbif.connect()
15 >>> # Execute a SQL statement
16 >>> dbif.execute_transaction("SELECT datetime(0, 'unixepoch', 'localtime');")
17 >>> # Mogrify an SQL statement
18 >>> dbif.mogrify_sql_statement(["SELECT name from raster_base where name = ?",
19 ... ("precipitation",)])
20 "SELECT name from raster_base where name = 'precipitation'"
21 >>> dbif.close()
22
23
24(C) 2011-2014 by the GRASS Development Team
25This program is free software under the GNU General Public
26License (>=v2). Read the file COPYING that comes with GRASS
27for details.
28
29:author: Soeren Gebbert
30"""
31#import traceback
32import os
33# i18N
34import gettext
35gettext.install('grasslibs', os.path.join(os.getenv("GISBASE"), 'locale'))
36
37try:
38 from builtins import long
39except ImportError:
40 # python3
41 long = int
42
43import grass.script as gscript
44from datetime import datetime
45from .c_libraries_interface import *
46from grass.pygrass import messages
47# Import all supported database backends
48# Ignore import errors since they are checked later
49try:
50 import sqlite3
51except ImportError:
52 pass
53# Postgresql is optional, existence is checked when needed
54try:
55 import psycopg2
56 import psycopg2.extras
57except:
58 pass
59
60import atexit
61
62###############################################################################
63
64
65def profile_function(func):
66 """Profiling function provided by the temporal framework"""
67 do_profiling = os.getenv("GRASS_TGIS_PROFILE")
68
69 if do_profiling is "True" or do_profiling is "1":
70 import cProfile, pstats
71 try:
72 import StringIO as io
73 except ImportError:
74 import io
75 pr = cProfile.Profile()
76 pr.enable()
77 func()
78 pr.disable()
79 s = io.StringIO()
80 sortby = 'cumulative'
81 ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
82 ps.print_stats()
83 print(s.getvalue())
84 else:
85 func()
86
87# Global variable that defines the backend
88# of the temporal GIS
89# It can either be "sqlite" or "pg"
90tgis_backend = None
91
92
93def get_tgis_backend():
94 """Return the temporal GIS backend as string
95
96 :returns: either "sqlite" or "pg"
97 """
98 global tgis_backend
99 return tgis_backend
100
101# Global variable that defines the database string
102# of the temporal GIS
103tgis_database = None
104
105
106def get_tgis_database():
107 """Return the temporal database string specified with t.connect
108 """
109 global tgis_database
110 return tgis_database
111
112# The version of the temporal framework
113# this value must be an integer larger than 0
114# Increase this value in case of backward incompatible changes in the TGIS API
115tgis_version = 2
116# The version of the temporal database since framework and database version
117# can differ this value must be an integer larger than 0
118# Increase this value in case of backward incompatible changes
119# temporal database SQL layout
120tgis_db_version = 2
121
122# We need to know the parameter style of the database backend
123tgis_dbmi_paramstyle = None
124
125
126def get_tgis_dbmi_paramstyle():
127 """Return the temporal database backend parameter style
128
129 :returns: "qmark" or ""
130 """
131 global tgis_dbmi_paramstyle
132 return tgis_dbmi_paramstyle
133
134# We need to access the current mapset quite often in the framework, so we make
135# a global variable that will be initiated when init() is called
136current_mapset = None
137current_location = None
138current_gisdbase = None
139
140###############################################################################
141
142
143def get_current_mapset():
144 """Return the current mapset
145
146 This is the fastest way to receive the current mapset.
147 The current mapset is set by init() and stored in a global variable.
148 This function provides access to this global variable.
149 """
150 global current_mapset
151 return current_mapset
152
153###############################################################################
154
155
156def get_current_location():
157 """Return the current location
158
159 This is the fastest way to receive the current location.
160 The current location is set by init() and stored in a global variable.
161 This function provides access to this global variable.
162 """
163 global current_location
164 return current_location
165
166###############################################################################
167
168
169def get_current_gisdbase():
170 """Return the current gis database (gisdbase)
171
172 This is the fastest way to receive the current gisdbase.
173 The current gisdbase is set by init() and stored in a global variable.
174 This function provides access to this global variable.
175 """
176 global current_gisdbase
177 return current_gisdbase
178
179###############################################################################
180
181# If this global variable is set True, then maps can only be registered in
182# space time datasets with the same mapset. In addition, only maps in the
183# current mapset can be inserted, updated or deleted from the temporal database.
184# Overwrite this global variable by: g.gisenv set="TGIS_DISABLE_MAPSET_CHECK=True"
185# ATTENTION: Be aware to face corrupted temporal database in case this global
186# variable is set to False. This feature is highly
187# experimental and violates the grass permission guidance.
188enable_mapset_check = True
189# If this global variable is set True, the timestamps of maps will be written
190# as textfiles for each map that will be inserted or updated in the temporal
191# database using the C-library timestamp interface.
192# Overwrite this global variable by: g.gisenv set="TGIS_DISABLE_TIMESTAMP_WRITE=True"
193# ATTENTION: Be aware to face corrupted temporal database in case this global
194# variable is set to False. This feature is highly
195# experimental and violates the grass permission guidance.
196enable_timestamp_write = True
197
198
199def get_enable_mapset_check():
200 """Return True if the mapsets should be checked while insert, update,
201 delete requests and space time dataset registration.
202
203 If this global variable is set True, then maps can only be registered
204 in space time datasets with the same mapset. In addition, only maps in
205 the current mapset can be inserted, updated or deleted from the temporal
206 database.
207 Overwrite this global variable by: g.gisenv set="TGIS_DISABLE_MAPSET_CHECK=True"
208
209 ..warning::
210
211 Be aware to face corrupted temporal database in case this
212 global variable is set to False. This feature is highly
213 experimental and violates the grass permission guidance.
214
215 """
216 global enable_mapset_check
217 return enable_mapset_check
218
219
220def get_enable_timestamp_write():
221 """Return True if the map timestamps should be written to the spatial
222 database metadata as well.
223
224 If this global variable is set True, the timestamps of maps will be
225 written as textfiles for each map that will be inserted or updated in
226 the temporal database using the C-library timestamp interface.
227 Overwrite this global variable by: g.gisenv set="TGIS_DISABLE_TIMESTAMP_WRITE=True"
228
229 ..warning::
230
231 Be aware that C-libraries can not access timestamp information if
232 they are not written as spatial database metadata, hence modules
233 that make use of timestamps using the C-library interface will not
234 work with maps that were created without writing the timestamps.
235 """
236 global enable_timestamp_write
237 return enable_timestamp_write
238
239###############################################################################
240
241# The global variable that stores the PyGRASS Messenger object that
242# provides a fast and exit safe interface to the C-library message functions
243message_interface = None
244
245
246def _init_tgis_message_interface(raise_on_error=False):
247 """Initiate the global message interface
248
249 :param raise_on_error: If True raise a FatalError exception in case of
250 a fatal error, call sys.exit(1) otherwise
251 """
252 global message_interface
253 if message_interface is None:
254 message_interface = messages.get_msgr(raise_on_error=raise_on_error)
255
256
257def get_tgis_message_interface():
258 """Return the temporal GIS message interface which is of type
259 grass.pygrass.message.Messenger()
260
261 Use this message interface to print messages to stdout using the
262 GRASS C-library messaging system.
263 """
264 global message_interface
265 return message_interface
266
267###############################################################################
268
269# The global variable that stores the C-library interface object that
270# provides a fast and exit safe interface to the C-library libgis,
271# libraster, libraster3d and libvector functions
272c_library_interface = None
273
274
275def _init_tgis_c_library_interface():
276 """Set the global C-library interface variable that
277 provides a fast and exit safe interface to the C-library libgis,
278 libraster, libraster3d and libvector functions
279 """
280 global c_library_interface
281 if c_library_interface is None:
282 c_library_interface = CLibrariesInterface()
283
284
285def get_tgis_c_library_interface():
286 """Return the C-library interface that
287 provides a fast and exit safe interface to the C-library libgis,
288 libraster, libraster3d and libvector functions
289 """
290 global c_library_interface
291 return c_library_interface
292
293###############################################################################
294
295# Set this variable True to raise a FatalError exception
296# in case a fatal error occurs using the messenger interface
297raise_on_error = False
298
299
300def set_raise_on_error(raise_exp=True):
301 """Define behavior on fatal error, invoked using the tgis messenger
302 interface (msgr.fatal())
303
304 The messenger interface will be restarted using the new error policy
305
306 :param raise_exp: True to raise a FatalError exception instead of calling
307 sys.exit(1) when using the tgis messenger interface
308
309 .. code-block:: python
310
311 >>> import grass.temporal as tgis
312 >>> tgis.init()
313 >>> ignore = tgis.set_raise_on_error(False)
314 >>> msgr = tgis.get_tgis_message_interface()
315 >>> tgis.get_raise_on_error()
316 False
317 >>> msgr.fatal("Ohh no no no!")
318 Traceback (most recent call last):
319 File "__init__.py", line 239, in fatal
320 sys.exit(1)
321 SystemExit: 1
322
323 >>> tgis.set_raise_on_error(True)
324 False
325 >>> msgr.fatal("Ohh no no no!")
326 Traceback (most recent call last):
327 File "__init__.py", line 241, in fatal
328 raise FatalError(message)
329 FatalError: Ohh no no no!
330
331 :returns: current status
332 """
333 global raise_on_error
334 tmp_raise = raise_on_error
335 raise_on_error = raise_exp
336
337 global message_interface
338 if message_interface:
339 message_interface.set_raise_on_error(raise_on_error)
340 else:
341 _init_tgis_message_interface(raise_on_error)
342
343 return tmp_raise
344
345
346def get_raise_on_error():
347 """Return True if a FatalError exception is raised instead of calling
348 sys.exit(1) in case a fatal error was invoked with msgr.fatal()
349 """
350 global raise_on_error
351 return raise_on_error
352
353
354###############################################################################
355
356
357def get_tgis_version():
358 """Get the version number of the temporal framework
359 :returns: The version number of the temporal framework as string
360 """
361 global tgis_version
362 return tgis_version
363
364###############################################################################
365
366
367def get_tgis_db_version():
368 """Get the version number of the temporal framework
369 :returns: The version number of the temporal framework as string
370 """
371 global tgis_db_version
372 return tgis_db_version
373
374###############################################################################
375
376
377def get_tgis_metadata(dbif=None):
378 """Return the tgis metadata table as a list of rows (dicts) or None if not
379 present
380
381 :param dbif: The database interface to be used
382 :returns: The selected rows with key/value columns or None
383 """
384
385 dbif, connected = init_dbif(dbif)
386
387 # Select metadata if the table is present
388 try:
389 statement = "SELECT * FROM tgis_metadata;\n"
390 dbif.execute(statement)
391 rows = dbif.fetchall()
392 except:
393 rows = None
394
395 if connected:
396 dbif.close()
397
398 return rows
399
400###############################################################################
401
402# The temporal database string set with t.connect
403# with substituted GRASS variables gisdbase, location and mapset
404tgis_database_string = None
405
406
407def get_tgis_database_string():
408 """Return the preprocessed temporal database string
409
410 This string is the temporal database string set with t.connect
411 that was processed to substitue location, gisdbase and mapset
412 variables.
413 """
414 global tgis_database_string
415 return tgis_database_string
416
417###############################################################################
418
419
420def get_sql_template_path():
421 base = os.getenv("GISBASE")
422 base_etc = os.path.join(base, "etc")
423 return os.path.join(base_etc, "sql")
424
425###############################################################################
426
427
428def stop_subprocesses():
429 """Stop the messenger and C-interface subprocesses
430 that are started by tgis.init()
431 """
432 global message_interface
433 global c_library_interface
434 if message_interface:
435 message_interface.stop()
436 if c_library_interface:
437 c_library_interface.stop()
438
439# We register this function to be called at exit
440atexit.register(stop_subprocesses)
441
442
443def get_available_temporal_mapsets():
444 """Return a list of of mapset names with temporal database driver and names
445 that are accessible from the current mapset.
446
447 :returns: A dictionary, mapset names are keys, the tuple (driver,
448 database) are the values
449 """
450 global c_library_interface
451 global message_interface
452
453 mapsets = c_library_interface.available_mapsets()
454
455 tgis_mapsets = {}
456
457 for mapset in mapsets:
458 driver = c_library_interface.get_driver_name(mapset)
459 database = c_library_interface.get_database_name(mapset)
460
461 message_interface.debug(1, "get_available_temporal_mapsets: "\
462 "\n mapset %s\n driver %s\n database %s"%(mapset,
463 driver, database))
464
465 if driver and database:
466 # Check if the temporal sqlite database exists
467 # We need to set non-existing databases in case the mapset is the current mapset
468 # to create it
469 if (driver == "sqlite" and os.path.exists(database)) or mapset == get_current_mapset() :
470 tgis_mapsets[mapset] = (driver, database)
471
472 # We need to warn if the connection is defined but the database does not
473 # exists
474 if driver == "sqlite" and not os.path.exists(database):
475 message_interface.warning("Temporal database connection defined as:\n" + \
476 database + "\nBut database file does not exist.")
477
478 return tgis_mapsets
479
480###############################################################################
481
482
483def init(raise_fatal_error=False):
484 """This function set the correct database backend from GRASS environmental
485 variables and creates the grass temporal database structure for raster,
486 vector and raster3d maps as well as for the space-time datasets strds,
487 str3ds and stvds in case it does not exist.
488
489 Several global variables are initiated and the messenger and C-library
490 interface subprocesses are spawned.
491
492 Re-run this function in case the following GRASS variables change while
493 the process runs:
494
495 - MAPSET
496 - LOCATION_NAME
497 - GISDBASE
498 - TGIS_DISABLE_MAPSET_CHECK
499 - TGIS_DISABLE_TIMESTAMP_WRITE
500
501 Re-run this function if the following t.connect variables change while
502 the process runs:
503
504 - temporal GIS driver (set by t.connect driver=)
505 - temporal GIS database (set by t.connect database=)
506
507 The following environmental variables are checked:
508
509 - GRASS_TGIS_PROFILE (True, False, 1, 0)
510 - GRASS_TGIS_RAISE_ON_ERROR (True, False, 1, 0)
511
512 ..warning::
513
514 This functions must be called before any spatio-temporal processing
515 can be started
516
517 :param raise_fatal_error: Set this True to assure that the init()
518 function does not kill a persistent process
519 like the GUI. If set True a
520 grass.pygrass.messages.FatalError
521 exception will be raised in case a fatal
522 error occurs in the init process, otherwise
523 sys.exit(1) will be called.
524 """
525 # We need to set the correct database backend and several global variables
526 # from the GRASS mapset specific environment variables of g.gisenv and t.connect
527 global tgis_backend
528 global tgis_database
529 global tgis_database_string
530 global tgis_dbmi_paramstyle
531 global raise_on_error
532 global enable_mapset_check
533 global enable_timestamp_write
534 global current_mapset
535 global current_location
536 global current_gisdbase
537
538 raise_on_error = raise_fatal_error
539
540 # We must run t.connect at first to create the temporal database and to
541 # get the environmental variables
542 gscript.run_command("t.connect", flags="c")
543 grassenv = gscript.gisenv()
544
545 # Set the global variable for faster access
546 current_mapset = grassenv["MAPSET"]
547 current_location = grassenv["LOCATION_NAME"]
548 current_gisdbase = grassenv["GISDBASE"]
549
550 # Check environment variable GRASS_TGIS_RAISE_ON_ERROR
551 if os.getenv("GRASS_TGIS_RAISE_ON_ERROR") == "True" or \
552 os.getenv("GRASS_TGIS_RAISE_ON_ERROR") == "1":
553 raise_on_error = True
554
555 # Check if the script library raises on error,
556 # if so we do the same
557 if gscript.get_raise_on_error() is True:
558 raise_on_error = True
559
560 # Start the GRASS message interface server
561 _init_tgis_message_interface(raise_on_error)
562 # Start the C-library interface server
563 _init_tgis_c_library_interface()
564 msgr = get_tgis_message_interface()
565 msgr.debug(1, "Initiate the temporal database")
566 #"\n traceback:%s"%(str(" \n".join(traceback.format_stack()))))
567
568 ciface = get_tgis_c_library_interface()
569 driver_string = ciface.get_driver_name()
570 database_string = ciface.get_database_name()
571
572 # Set the mapset check and the timestamp write
573 if "TGIS_DISABLE_MAPSET_CHECK" in grassenv:
574 if grassenv["TGIS_DISABLE_MAPSET_CHECK"] == "True" or \
575 grassenv["TGIS_DISABLE_MAPSET_CHECK"] == "1":
576 enable_mapset_check = False
577 msgr.warning("TGIS_DISABLE_MAPSET_CHECK is True")
578
579 if "TGIS_DISABLE_TIMESTAMP_WRITE" in grassenv:
580 if grassenv["TGIS_DISABLE_TIMESTAMP_WRITE"] == "True" or \
581 grassenv["TGIS_DISABLE_TIMESTAMP_WRITE"] == "1":
582 enable_timestamp_write = False
583 msgr.warning("TGIS_DISABLE_TIMESTAMP_WRITE is True")
584
585 if driver_string is not None and driver_string is not "":
586 if driver_string == "sqlite":
587 tgis_backend = driver_string
588 try:
589 import sqlite3
590 except ImportError:
591 msgr.error("Unable to locate the sqlite SQL Python interface"
592 " module sqlite3.")
593 raise
594 dbmi = sqlite3
595 elif driver_string == "pg":
596 tgis_backend = driver_string
597 try:
598 import psycopg2
599 except ImportError:
600 msgr.error("Unable to locate the Postgresql SQL Python "
601 "interface module psycopg2.")
602 raise
603 dbmi = psycopg2
604 else:
605 msgr.fatal(_("Unable to initialize the temporal DBMI interface. "
606 "Please use t.connect to specify the driver and the"
607 " database string"))
608 else:
609 # Set the default sqlite3 connection in case nothing was defined
610 gscript.run_command("t.connect", flags="d")
611 driver_string = ciface.get_driver_name()
612 database_string = ciface.get_database_name()
613 tgis_backend = driver_string
614 dbmi = sqlite3
615
616 tgis_database_string = database_string
617 # Set the parameter style
618 tgis_dbmi_paramstyle = dbmi.paramstyle
619
620 # We do not know if the database already exists
621 db_exists = False
622 dbif = SQLDatabaseInterfaceConnection()
623
624 # Check if the database already exists
625 if tgis_backend == "sqlite":
626 # Check path of the sqlite database
627 if os.path.exists(tgis_database_string):
628 dbif.connect()
629 # Check for raster_base table
630 dbif.execute("SELECT name FROM sqlite_master WHERE type='table' "
631 "AND name='raster_base';")
632 name = dbif.fetchone()
633 if name and name[0] == "raster_base":
634 db_exists = True
635 dbif.close()
636 elif tgis_backend == "pg":
637 # Connect to database
638 dbif.connect()
639 # Check for raster_base table
640 dbif.execute("SELECT EXISTS(SELECT * FROM information_schema.tables "
641 "WHERE table_name=%s)", ('raster_base',))
642 if dbif.fetchone()[0]:
643 db_exists = True
644
645 backup_howto = "The format of your actual temporal database is not " \
646 "supported any more.\nSolution: You need to export it by " \
647 "restoring the GRASS GIS version used for creating this DB"\
648 ". From there, create a backup of your temporal database "\
649 "to avoid the loss of your temporal data.\nNotes: Use " \
650 "t.rast.export and t.vect.export to make a backup of your" \
651 " existing space time datasets.To safe the timestamps of" \
652 " your existing maps and space time datasets, use " \
653 "t.rast.list, t.vect.list and t.rast3d.list. "\
654 "You can register the existing time stamped maps easily if"\
655 " you export columns=id,start_time,end_time into text "\
656 "files and use t.register to register them again in new" \
657 " created space time datasets (t.create). After the backup"\
658 " remove the existing temporal database, a new one will be"\
659 " created automatically.\n"
660
661 if db_exists is True:
662 # Check the version of the temporal database
663 dbif.close()
664 dbif.connect()
665 metadata = get_tgis_metadata(dbif)
666 dbif.close()
667 if metadata is None:
668 msgr.fatal(_("Unable to receive temporal database metadata.\n"
669 "Current temporal database info:%(info)s") % (
670 {"info": get_database_info_string()}))
671 for entry in metadata:
672 if "tgis_version" in entry and entry[1] != str(get_tgis_version()):
673 msgr.fatal(_("Unsupported temporal database: version mismatch."
674 "\n %(backup)s Supported temporal API version is:"
675 " %(api)i.\nPlease update your GRASS GIS "
676 "installation.\nCurrent temporal database info:"
677 "%(info)s") % ({"backup": backup_howto,
678 "api": get_tgis_version(),
679 "info": get_database_info_string()}))
680 if "tgis_db_version" in entry and entry[1] != str(get_tgis_db_version()):
681 msgr.fatal(_("Unsupported temporal database: version mismatch."
682 "\n %(backup)sSupported temporal database version"
683 " is: %(tdb)i\nCurrent temporal database info:"
684 "%(info)s") % ({"backup": backup_howto,
685 "tdb": get_tgis_version(),
686 "info": get_database_info_string()}))
687 return
688
689 create_temporal_database(dbif)
690
691###############################################################################
692
693
694def get_database_info_string():
695 dbif = SQLDatabaseInterfaceConnection()
696
697 info = "\nDBMI interface:..... " + str(dbif.get_dbmi().__name__)
698 info += "\nTemporal database:.. " + str(get_tgis_database_string())
699 return info
700
701###############################################################################
702
703
704def create_temporal_database(dbif):
705 """This function will create the temporal database
706
707 It will create all tables and triggers that are needed to run
708 the temporal GIS
709
710 :param dbif: The database interface to be used
711 """
712 global tgis_backend
713 global tgis_version
714 global tgis_db_version
715 global tgis_database_string
716
717 template_path = get_sql_template_path()
718 msgr = get_tgis_message_interface()
719
720 # Read all SQL scripts and templates
721 map_tables_template_sql = open(os.path.join(
722 template_path, "map_tables_template.sql"), 'r').read()
723 raster_metadata_sql = open(os.path.join(
724 get_sql_template_path(), "raster_metadata_table.sql"), 'r').read()
725 raster3d_metadata_sql = open(os.path.join(template_path,
726 "raster3d_metadata_table.sql"),
727 'r').read()
728 vector_metadata_sql = open(os.path.join(template_path,
729 "vector_metadata_table.sql"),
730 'r').read()
731 raster_views_sql = open(os.path.join(template_path, "raster_views.sql"),
732 'r').read()
733 raster3d_views_sql = open(os.path.join(template_path,
734 "raster3d_views.sql"), 'r').read()
735 vector_views_sql = open(os.path.join(template_path, "vector_views.sql"),
736 'r').read()
737
738 stds_tables_template_sql = open(os.path.join(template_path,
739 "stds_tables_template.sql"),
740 'r').read()
741 strds_metadata_sql = open(os.path.join(template_path,
742 "strds_metadata_table.sql"),
743 'r').read()
744 str3ds_metadata_sql = open(os.path.join(template_path,
745 "str3ds_metadata_table.sql"),
746 'r').read()
747 stvds_metadata_sql = open(os.path.join(template_path,
748 "stvds_metadata_table.sql"),
749 'r').read()
750 strds_views_sql = open(os.path.join(template_path, "strds_views.sql"),
751 'r').read()
752 str3ds_views_sql = open(os.path.join(template_path, "str3ds_views.sql"),
753 'r').read()
754 stvds_views_sql = open(os.path.join(template_path, "stvds_views.sql"),
755 'r').read()
756
757 # Create the raster, raster3d and vector tables SQL statements
758 raster_tables_sql = map_tables_template_sql.replace("GRASS_MAP", "raster")
759 vector_tables_sql = map_tables_template_sql.replace("GRASS_MAP", "vector")
760 raster3d_tables_sql = map_tables_template_sql.replace(
761 "GRASS_MAP", "raster3d")
762
763 # Create the space-time raster, raster3d and vector dataset tables
764 # SQL statements
765 strds_tables_sql = stds_tables_template_sql.replace("STDS", "strds")
766 stvds_tables_sql = stds_tables_template_sql.replace("STDS", "stvds")
767 str3ds_tables_sql = stds_tables_template_sql.replace("STDS", "str3ds")
768
769 msgr.message(_("Creating temporal database: %s" % (str(tgis_database_string))))
770
771 if tgis_backend == "sqlite":
772 # We need to create the sqlite3 database path if it does not exist
773 tgis_dir = os.path.dirname(tgis_database_string)
774 if not os.path.exists(tgis_dir):
775 try:
776 os.makedirs(tgis_dir)
777 except Exception as e:
778 msgr.fatal(_("Unable to create SQLite temporal database\n"
779 "Exception: %s\nPlease use t.connect to set a "
780 "read- and writable temporal database path" % (e)))
781
782 # Set up the trigger that takes care of
783 # the correct deletion of entries across the different tables
784 delete_trigger_sql = open(os.path.join(template_path,
785 "sqlite3_delete_trigger.sql"),
786 'r').read()
787 indexes_sql = open(os.path.join(template_path, "sqlite3_indexes.sql"),
788 'r').read()
789 else:
790 # Set up the trigger that takes care of
791 # the correct deletion of entries across the different tables
792 delete_trigger_sql = open(os.path.join(template_path,
793 "postgresql_delete_trigger.sql"),
794 'r').read()
795 indexes_sql = open(os.path.join(template_path,
796 "postgresql_indexes.sql"), 'r').read()
797
798 # Connect now to the database
799 if dbif.connected is not True:
800 dbif.connect()
801
802 # Execute the SQL statements for sqlite
803 # Create the global tables for the native grass datatypes
804 dbif.execute_transaction(raster_tables_sql)
805 dbif.execute_transaction(raster_metadata_sql)
806 dbif.execute_transaction(raster_views_sql)
807 dbif.execute_transaction(vector_tables_sql)
808 dbif.execute_transaction(vector_metadata_sql)
809 dbif.execute_transaction(vector_views_sql)
810 dbif.execute_transaction(raster3d_tables_sql)
811 dbif.execute_transaction(raster3d_metadata_sql)
812 dbif.execute_transaction(raster3d_views_sql)
813 # Create the tables for the new space-time datatypes
814 dbif.execute_transaction(strds_tables_sql)
815 dbif.execute_transaction(strds_metadata_sql)
816 dbif.execute_transaction(strds_views_sql)
817 dbif.execute_transaction(stvds_tables_sql)
818 dbif.execute_transaction(stvds_metadata_sql)
819 dbif.execute_transaction(stvds_views_sql)
820 dbif.execute_transaction(str3ds_tables_sql)
821 dbif.execute_transaction(str3ds_metadata_sql)
822 dbif.execute_transaction(str3ds_views_sql)
823
824 # The delete trigger
825 dbif.execute_transaction(delete_trigger_sql)
826 # The indexes
827 dbif.execute_transaction(indexes_sql)
828
829 # Create the tgis metadata table to store the database
830 # initial configuration
831 # The metadata table content
832 metadata = {}
833 metadata["tgis_version"] = tgis_version
834 metadata["tgis_db_version"] = tgis_db_version
835 metadata["creation_time"] = datetime.today()
836 _create_tgis_metadata_table(metadata, dbif)
837
838 dbif.close()
839
840###############################################################################
841
842
843def _create_tgis_metadata_table(content, dbif=None):
844 """!Create the temporal gis metadata table which stores all metadata
845 information about the temporal database.
846
847 :param content: The dictionary that stores the key:value metadata
848 that should be stored in the metadata table
849 :param dbif: The database interface to be used
850 """
851 dbif, connected = init_dbif(dbif)
852 statement = "CREATE TABLE tgis_metadata (key VARCHAR NOT NULL, value VARCHAR);\n";
853 dbif.execute_transaction(statement)
854
855 for key in content.keys():
856 statement = "INSERT INTO tgis_metadata (key, value) VALUES " + \
857 "(\'%s\' , \'%s\');\n" % (str(key), str(content[key]))
858 dbif.execute_transaction(statement)
859
860 if connected:
861 dbif.close()
862
863###############################################################################
864
865
866class SQLDatabaseInterfaceConnection(object):
867 def __init__(self):
868 self.tgis_mapsets = get_available_temporal_mapsets()
869 self.current_mapset = get_current_mapset()
870 self.connections = {}
871 self.connected = False
872
873 self.unique_connections = {}
874
875 for mapset in self.tgis_mapsets.keys():
876 driver, dbstring = self.tgis_mapsets[mapset]
877
878 if dbstring not in self.unique_connections.keys():
879 self.unique_connections[dbstring] = DBConnection(backend=driver,
880 dbstring=dbstring)
881
882 self.connections[mapset] = self.unique_connections[dbstring]
883
884 self.msgr = get_tgis_message_interface()
885
886 def get_dbmi(self, mapset=None):
887 if mapset is None:
888 mapset = self.current_mapset
889 return self.connections[mapset].dbmi
890
891 def rollback(self, mapset=None):
892 """
893 Roll back the last transaction. This must be called
894 in case a new query should be performed after a db error.
895
896 This is only relevant for postgresql database.
897 """
898 if mapset is None:
899 mapset = self.current_mapset
900
901 def connect(self):
902 """Connect to the DBMI to execute SQL statements
903
904 Supported backends are sqlite3 and postgresql
905 """
906 for mapset in self.tgis_mapsets.keys():
907 driver, dbstring = self.tgis_mapsets[mapset]
908 conn = self.connections[mapset]
909 if conn.is_connected() is False:
910 conn.connect(dbstring)
911
912 self.connected = True
913
914 def is_connected(self):
915 return self.connected
916
917 def close(self):
918 """Close the DBMI connection
919
920 There may be several temporal databases in a location, hence
921 close all temporal databases that have been opened.
922 """
923 for key in self.unique_connections.keys():
924 self.unique_connections[key].close()
925
926 self.connected = False
927
928 def mogrify_sql_statement(self, content, mapset=None):
929 """Return the SQL statement and arguments as executable SQL string
930
931 :param content: The content as tuple with two entries, the first
932 entry is the SQL statement with DBMI specific
933 place holder (?), the second entry is the argument
934 list that should substitute the place holder.
935 :param mapset: The mapset of the abstract dataset or temporal
936 database location, if None the current mapset
937 will be used
938 """
939 if mapset is None:
940 mapset = self.current_mapset
941
942 if mapset not in self.tgis_mapsets.keys():
943 self.msgr.fatal(_("Unable to mogrify sql statement. " +
944 self._create_mapset_error_message(mapset)))
945
946 return self.connections[mapset].mogrify_sql_statement(content)
947
948 def check_table(self, table_name, mapset=None):
949 """Check if a table exists in the temporal database
950
951 :param table_name: The name of the table to be checked for existence
952 :param mapset: The mapset of the abstract dataset or temporal
953 database location, if None the current mapset
954 will be used
955 :returns: True if the table exists, False otherwise
956
957 TODO:
958 There may be several temporal databases in a location, hence
959 the mapset is used to query the correct temporal database.
960 """
961 if mapset is None:
962 mapset = self.current_mapset
963
964 if mapset not in self.tgis_mapsets.keys():
965 self.msgr.fatal(_("Unable to check table. " +
966 self._create_mapset_error_message(mapset)))
967
968 return self.connections[mapset].check_table(table_name)
969
970 def execute(self, statement, args=None, mapset=None):
971 """
972
973 :param mapset: The mapset of the abstract dataset or temporal
974 database location, if None the current mapset
975 will be used
976 """
977 if mapset is None:
978 mapset = self.current_mapset
979
980 if mapset not in self.tgis_mapsets.keys():
981 self.msgr.fatal(_("Unable to execute sql statement. " +
982 self._create_mapset_error_message(mapset)))
983
984 return self.connections[mapset].execute(statement, args)
985
986 def fetchone(self, mapset=None):
987 if mapset is None:
988 mapset = self.current_mapset
989
990 if mapset not in self.tgis_mapsets.keys():
991 self.msgr.fatal(_("Unable to fetch one. " +
992 self._create_mapset_error_message(mapset)))
993
994 return self.connections[mapset].fetchone()
995
996 def fetchall(self, mapset=None):
997 if mapset is None:
998 mapset = self.current_mapset
999
1000 if mapset not in self.tgis_mapsets.keys():
1001 self.msgr.fatal(_("Unable to fetch all. " +
1002 self._create_mapset_error_message(mapset)))
1003
1004 return self.connections[mapset].fetchall()
1005
1006 def execute_transaction(self, statement, mapset=None):
1007 """Execute a transactional SQL statement
1008
1009 The BEGIN and END TRANSACTION statements will be added automatically
1010 to the sql statement
1011
1012 :param statement: The executable SQL statement or SQL script
1013 """
1014 if mapset is None:
1015 mapset = self.current_mapset
1016
1017 if mapset not in self.tgis_mapsets.keys():
1018 self.msgr.fatal(_("Unable to execute transaction. " +
1019 self._create_mapset_error_message(mapset)))
1020
1021 return self.connections[mapset].execute_transaction(statement)
1022
1023 def _create_mapset_error_message(self, mapset):
1024
1025 return("You have no permission to "
1026 "access mapset <%(mapset)s>, or "
1027 "mapset <%(mapset)s> has no temporal database. "
1028 "Accessible mapsets are: <%(mapsets)s>" % \
1029 {"mapset": mapset,
1030 "mapsets":','.join(self.tgis_mapsets.keys())})
1031
1032###############################################################################
1033
1034
1035class DBConnection(object):
1036 """This class represents the database interface connection
1037 and provides access to the chosen backend modules.
1038
1039 The following DBMS are supported:
1040
1041 - sqlite via the sqlite3 standard library
1042 - postgresql via psycopg2
1043 """
1044
1045 def __init__(self, backend=None, dbstring=None):
1046 """ Constructor of a database connection
1047
1048 param backend:The database backend sqlite or pg
1049 param dbstring: The database connection string
1050 """
1051 self.connected = False
1052 if backend is None:
1053 global tgis_backend
1054 if tgis_backend == "sqlite":
1055 self.dbmi = sqlite3
1056 else:
1057 self.dbmi = psycopg2
1058 else:
1059 if backend == "sqlite":
1060 self.dbmi = sqlite3
1061 else:
1062 self.dbmi = psycopg2
1063
1064 if dbstring is None:
1065 global tgis_database_string
1066 self.dbstring = tgis_database_string
1067
1068 self.dbstring = dbstring
1069
1070 self.msgr = get_tgis_message_interface()
1071 self.msgr.debug(1, "DBConnection constructor:"\
1072 "\n backend: %s"\
1073 "\n dbstring: %s"%(backend, self.dbstring))
1074 #"\n traceback:%s"%(backend, self.dbstring,
1075 #str(" \n".join(traceback.format_stack()))))
1076
1077 def __del__(self):
1078 if self.connected is True:
1079 self.close()
1080
1081 def is_connected(self):
1082 return self.connected
1083
1084 def rollback(self):
1085 """
1086 Roll back the last transaction. This must be called
1087 in case a new query should be performed after a db error.
1088
1089 This is only relevant for postgresql database.
1090 """
1091 if self.dbmi.__name__ == "psycopg2":
1092 if self.connected:
1093 self.connection.rollback()
1094
1095 def connect(self, dbstring=None):
1096 """Connect to the DBMI to execute SQL statements
1097
1098 Supported backends are sqlite3 and postgresql
1099
1100 param dbstring: The database connection string
1101 """
1102 # Connection in the current mapset
1103 if dbstring is None:
1104 dbstring = self.dbstring
1105 try:
1106 if self.dbmi.__name__ == "sqlite3":
1107 self.connection = self.dbmi.connect(dbstring,
1108 detect_types=self.dbmi.PARSE_DECLTYPES | self.dbmi.PARSE_COLNAMES)
1109 self.connection.row_factory = self.dbmi.Row
1110 self.connection.isolation_level = None
1111 self.cursor = self.connection.cursor()
1112 self.cursor.execute("PRAGMA synchronous = OFF")
1113 self.cursor.execute("PRAGMA journal_mode = MEMORY")
1114 elif self.dbmi.__name__ == "psycopg2":
1115 self.connection = self.dbmi.connect(dbstring)
1116 #self.connection.set_isolation_level(dbmi.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
1117 self.cursor = self.connection.cursor(
1118 cursor_factory=self.dbmi.extras.DictCursor)
1119 self.connected = True
1120 except Exception as e:
1121 self.msgr.fatal(_("Unable to connect to %(db)s database: "
1122 "%(string)s\nException: \"%(ex)s\"\nPlease use"
1123 " t.connect to set a read- and writable "
1124 "temporal database backend") % (
1125 {"db": self.dbmi.__name__,
1126 "string": tgis_database_string, "ex": e, }))
1127
1128 def close(self):
1129 """Close the DBMI connection
1130 TODO:
1131 There may be several temporal databases in a location, hence
1132 close all temporal databases that have been opened. Use a dictionary
1133 to manage different connections.
1134 """
1135 self.connection.commit()
1136 self.cursor.close()
1137 self.connected = False
1138
1139 def mogrify_sql_statement(self, content):
1140 """Return the SQL statement and arguments as executable SQL string
1141
1142 TODO:
1143 Use the mapset argument to identify the correct database driver
1144
1145 :param content: The content as tuple with two entries, the first
1146 entry is the SQL statement with DBMI specific
1147 place holder (?), the second entry is the argument
1148 list that should substitute the place holder.
1149 :param mapset: The mapset of the abstract dataset or temporal
1150 database location, if None the current mapset
1151 will be used
1152
1153 Usage:
1154
1155 .. code-block:: python
1156
1157 >>> init()
1158 >>> dbif = SQLDatabaseInterfaceConnection()
1159 >>> dbif.mogrify_sql_statement(["SELECT ctime FROM raster_base WHERE id = ?",
1160 ... ["soil@PERMANENT",]])
1161 "SELECT ctime FROM raster_base WHERE id = 'soil@PERMANENT'"
1162
1163 """
1164 sql = content[0]
1165 args = content[1]
1166
1167 if self.dbmi.__name__ == "psycopg2":
1168 if len(args) == 0:
1169 return sql
1170 else:
1171 if self.connected:
1172 try:
1173 return self.cursor.mogrify(sql, args)
1174 except Exception as exc:
1175 print(sql, args)
1176 raise exc
1177 else:
1178 self.connect()
1179 statement = self.cursor.mogrify(sql, args)
1180 self.close()
1181 return statement
1182
1183 elif self.dbmi.__name__ == "sqlite3":
1184 if len(args) == 0:
1185 return sql
1186 else:
1187 # Unfortunately as sqlite does not support
1188 # the transformation of sql strings and qmarked or
1189 # named arguments we must make our hands dirty
1190 # and do it by ourself. :(
1191 # Doors are open for SQL injection because of the
1192 # limited python sqlite3 implementation!!!
1193 pos = 0
1194 count = 0
1195 maxcount = 100
1196 statement = sql
1197
1198 while count < maxcount:
1199 pos = statement.find("?", pos + 1)
1200 if pos == -1:
1201 break
1202
1203 if args[count] is None:
1204 statement = "%sNULL%s" % (statement[0:pos],
1205 statement[pos + 1:])
1206 elif isinstance(args[count], (int, long)):
1207 statement = "%s%d%s" % (statement[0:pos], args[count],
1208 statement[pos + 1:])
1209 elif isinstance(args[count], float):
1210 statement = "%s%f%s" % (statement[0:pos], args[count],
1211 statement[pos + 1:])
1212 else:
1213 # Default is a string, this works for datetime
1214 # objects too
1215 statement = "%s\'%s\'%s" % (statement[0:pos],
1216 str(args[count]),
1217 statement[pos + 1:])
1218 count += 1
1219
1220 return statement
1221
1222 def check_table(self, table_name):
1223 """Check if a table exists in the temporal database
1224
1225 :param table_name: The name of the table to be checked for existence
1226 :param mapset: The mapset of the abstract dataset or temporal
1227 database location, if None the current mapset
1228 will be used
1229 :returns: True if the table exists, False otherwise
1230
1231 TODO:
1232 There may be several temporal databases in a location, hence
1233 the mapset is used to query the correct temporal database.
1234 """
1235 table_exists = False
1236 connected = False
1237 if not self.connected:
1238 self.connect()
1239 connected = True
1240
1241 # Check if the database already exists
1242 if self.dbmi.__name__ == "sqlite3":
1243
1244 self.cursor.execute("SELECT name FROM sqlite_master WHERE "
1245 "type='table' AND name='%s';" % table_name)
1246 name = self.cursor.fetchone()
1247 if name and name[0] == table_name:
1248 table_exists = True
1249 else:
1250 # Check for raster_base table
1251 self.cursor.execute("SELECT EXISTS(SELECT * FROM information_schema.tables "
1252 "WHERE table_name=%s)", ('%s' % table_name,))
1253 if self.cursor.fetchone()[0]:
1254 table_exists = True
1255
1256 if connected:
1257 self.close()
1258
1259 return table_exists
1260
1261 def execute(self, statement, args=None):
1262 """Execute a SQL statement
1263
1264 :param statement: The executable SQL statement or SQL script
1265 """
1266 connected = False
1267 if not self.connected:
1268 self.connect()
1269 connected = True
1270 try:
1271 if args:
1272 self.cursor.execute(statement, args)
1273 else:
1274 self.cursor.execute(statement)
1275 except:
1276 if connected:
1277 self.close()
1278 self.msgr.error(_("Unable to execute :\n %(sql)s" %
1279 {"sql": statement}))
1280 raise
1281
1282 if connected:
1283 self.close()
1284
1285 def fetchone(self):
1286 if self.connected:
1287 return self.cursor.fetchone()
1288 return None
1289
1290 def fetchall(self):
1291 if self.connected:
1292 return self.cursor.fetchall()
1293 return None
1294
1295 def execute_transaction(self, statement, mapset=None):
1296 """Execute a transactional SQL statement
1297
1298 The BEGIN and END TRANSACTION statements will be added automatically
1299 to the sql statement
1300
1301 :param statement: The executable SQL statement or SQL script
1302 """
1303 connected = False
1304 if not self.connected:
1305 self.connect()
1306 connected = True
1307
1308 sql_script = ""
1309 sql_script += "BEGIN TRANSACTION;\n"
1310 sql_script += statement
1311 sql_script += "END TRANSACTION;"
1312
1313 try:
1314 if self.dbmi.__name__ == "sqlite3":
1315 self.cursor.executescript(statement)
1316 else:
1317 self.cursor.execute(statement)
1318 self.connection.commit()
1319 except:
1320 if connected:
1321 self.close()
1322 self.msgr.error(_("Unable to execute transaction:\n %(sql)s" %
1323 {"sql": statement}))
1324 raise
1325
1326 if connected:
1327 self.close()
1328
1329###############################################################################
1330
1331
1332def init_dbif(dbif):
1333 """This method checks if the database interface connection exists,
1334 if not a new one will be created, connected and True will be returned.
1335 If the database interface exists but is connected, the connection will
1336 be established.
1337
1338 :returns: the tuple (dbif, True|False)
1339
1340 Usage code sample:
1341
1342 .. code-block:: python
1343
1344 dbif, connect = tgis.init_dbif(None)
1345
1346 sql = dbif.mogrify_sql_statement(["SELECT * FROM raster_base WHERE ? = ?"],
1347 ["id", "soil@PERMANENT"])
1348 dbif.execute_transaction(sql)
1349
1350 if connect:
1351 dbif.close()
1352
1353 """
1354 if dbif is None:
1355 dbif = SQLDatabaseInterfaceConnection()
1356 dbif.connect()
1357 return dbif, True
1358 elif dbif.is_connected() is False:
1359 dbif.connect()
1360 return dbif, True
1361
1362 return dbif, False
1363
1364###############################################################################
1365
1366if __name__ == "__main__":
1367 import doctest
1368 doctest.testmod()
Note: See TracBrowser for help on using the repository browser.