source: grass/trunk/gui/wxpython/core/gcmd.py

Last change on this file was 74480, checked in by annakrat, 5 years ago

pythonlib: cleanup decoding/encoding in scripting library
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id
  • Property svn:mime-type set to text/x-python
File size: 23.8 KB
Line 
1"""
2@package core.gcmd
3
4@brief wxGUI command interface
5
6Classes:
7 - gcmd::GError
8 - gcmd::GWarning
9 - gcmd::GMessage
10 - gcmd::GException
11 - gcmd::Popen (from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440554)
12 - gcmd::Command
13 - gcmd::CommandThread
14
15Functions:
16 - RunCommand
17 - GetDefaultEncoding
18
19(C) 2007-2008, 2010-2011 by the GRASS Development Team
20
21This program is free software under the GNU General Public License
22(>=v2). Read the file COPYING that comes with GRASS for details.
23
24@author Jachym Cepicky
25@author Martin Landa <landa.martin gmail.com>
26"""
27
28from __future__ import print_function
29
30import os
31import sys
32import time
33import errno
34import signal
35import traceback
36import locale
37import subprocess
38from threading import Thread
39import wx
40
41is_mswindows = sys.platform == 'win32'
42if is_mswindows:
43 from win32file import ReadFile, WriteFile
44 from win32pipe import PeekNamedPipe
45 import msvcrt
46else:
47 import select
48 import fcntl
49
50from core.debug import Debug
51from core.globalvar import SCT_EXT
52
53from grass.script import core as grass
54from grass.script.utils import decode, encode
55
56if sys.version_info.major == 2:
57 bytes = str
58
59
60def DecodeString(string):
61 """Decode string using system encoding
62
63 :param string: string to be decoded
64
65 :return: decoded string
66 """
67 if not string:
68 return string
69
70 if _enc and isinstance(string, bytes):
71 Debug.msg(5, "DecodeString(): enc=%s" % _enc)
72 return string.decode(_enc)
73 return string
74
75
76def EncodeString(string):
77 """Return encoded string using system locales
78
79 :param string: string to be encoded
80
81 :return: encoded string
82 """
83 if not string:
84 return string
85 if _enc:
86 Debug.msg(5, "EncodeString(): enc=%s" % _enc)
87 return string.encode(_enc)
88 return string
89
90
91class GError:
92
93 def __init__(self, message, parent=None, caption=None, showTraceback=True):
94 """Show error message window
95
96 :param message: error message
97 :param parent: centre window on parent if given
98 :param caption: window caption (if not given "Error")
99 :param showTraceback: True to show also Python traceback
100 """
101 if not caption:
102 caption = _('Error')
103 style = wx.OK | wx.ICON_ERROR | wx.CENTRE
104 exc_type, exc_value, exc_traceback = sys.exc_info()
105 if exc_traceback:
106 exception = traceback.format_exc()
107 reason = exception.splitlines()[-1].split(':', 1)[-1].strip()
108
109 if Debug.GetLevel() > 0 and exc_traceback:
110 sys.stderr.write(exception)
111
112 if showTraceback and exc_traceback:
113 wx.MessageBox(parent=parent,
114 message=message + '\n\n%s: %s\n\n%s' %
115 (_('Reason'),
116 reason, exception),
117 caption=caption,
118 style=style)
119 else:
120 wx.MessageBox(parent=parent,
121 message=message,
122 caption=caption,
123 style=style)
124
125
126class GWarning:
127
128 def __init__(self, message, parent=None):
129 caption = _('Warning')
130 style = wx.OK | wx.ICON_WARNING | wx.CENTRE
131 wx.MessageBox(parent=parent,
132 message=message,
133 caption=caption,
134 style=style)
135
136
137class GMessage:
138
139 def __init__(self, message, parent=None):
140 caption = _('Message')
141 style = wx.OK | wx.ICON_INFORMATION | wx.CENTRE
142 wx.MessageBox(parent=parent,
143 message=message,
144 caption=caption,
145 style=style)
146
147
148class GException(Exception):
149
150 def __init__(self, value=''):
151 self.value = value
152
153 def __str__(self):
154 return self.value
155
156 def __unicode__(self):
157 return self.value
158
159
160class Popen(subprocess.Popen):
161 """Subclass subprocess.Popen"""
162
163 def __init__(self, args, **kwargs):
164 if is_mswindows:
165 # encoding not needed for Python3
166 # args = list(map(EncodeString, args))
167
168 # The Windows shell (cmd.exe) requires some special characters to
169 # be escaped by preceding them with 3 carets (^^^). cmd.exe /?
170 # mentions <space> and &()[]{}^=;!'+,`~. A quick test revealed that
171 # only ^|&<> need to be escaped. A single quote can be escaped by
172 # enclosing it with double quotes and vice versa.
173 for i in range(2, len(args)):
174 # "^" must be the first character in the list to avoid double
175 # escaping.
176 for c in ("^", "|", "&", "<", ">"):
177 if c in args[i]:
178 if "=" in args[i]:
179 a = args[i].split("=")
180 k = a[0] + "="
181 v = "=".join(a[1:len(a)])
182 else:
183 k = ""
184 v = args[i]
185
186 # If there are spaces, the argument was already
187 # esscaped with double quotes, so don't escape it
188 # again.
189 if c in v and not " " in v:
190 # Here, we escape each ^ in ^^^ with ^^ and a
191 # <special character> with ^ + <special character>,
192 # so we need 7 carets.
193
194 v = v.replace(c, "^^^^^^^" + c)
195 args[i] = k + v
196
197 subprocess.Popen.__init__(self, args, **kwargs)
198
199 def recv(self, maxsize=None):
200 return self._recv('stdout', maxsize)
201
202 def recv_err(self, maxsize=None):
203 return self._recv('stderr', maxsize)
204
205 def send_recv(self, input='', maxsize=None):
206 return self.send(input), self.recv(maxsize), self.recv_err(maxsize)
207
208 def get_conn_maxsize(self, which, maxsize):
209 if maxsize is None:
210 maxsize = 1024
211 elif maxsize < 1:
212 maxsize = 1
213 return getattr(self, which), maxsize
214
215 def _close(self, which):
216 getattr(self, which).close()
217 setattr(self, which, None)
218
219 def kill(self):
220 """Try to kill running process"""
221 if is_mswindows:
222 import win32api
223 handle = win32api.OpenProcess(1, 0, self.pid)
224 return (0 != win32api.TerminateProcess(handle, 0))
225 else:
226 try:
227 os.kill(-self.pid, signal.SIGTERM) # kill whole group
228 except OSError:
229 pass
230
231 if sys.platform == 'win32':
232 def send(self, input):
233 if not self.stdin:
234 return None
235
236 import pywintypes
237 try:
238 x = msvcrt.get_osfhandle(self.stdin.fileno())
239 (errCode, written) = WriteFile(x, input)
240 except ValueError:
241 return self._close('stdin')
242 except (pywintypes.error, Exception) as why:
243 if why.winerror in (109, errno.ESHUTDOWN):
244 return self._close('stdin')
245 raise
246
247 return written
248
249 def _recv(self, which, maxsize):
250 conn, maxsize = self.get_conn_maxsize(which, maxsize)
251 if conn is None:
252 return None
253
254 import pywintypes
255 try:
256 x = msvcrt.get_osfhandle(conn.fileno())
257 (read, nAvail, nMessage) = PeekNamedPipe(x, 0)
258 if maxsize < nAvail:
259 nAvail = maxsize
260 if nAvail > 0:
261 (errCode, read) = ReadFile(x, nAvail, None)
262 except ValueError:
263 return self._close(which)
264 except (pywintypes.error, Exception) as why:
265 if why.winerror in (109, errno.ESHUTDOWN):
266 return self._close(which)
267 raise
268
269 if self.universal_newlines:
270 read = self._translate_newlines(read)
271 return read
272
273 else:
274 def send(self, input):
275 if not self.stdin:
276 return None
277
278 if not select.select([], [self.stdin], [], 0)[1]:
279 return 0
280
281 try:
282 written = os.write(self.stdin.fileno(), input)
283 except OSError as why:
284 if why[0] == errno.EPIPE: # broken pipe
285 return self._close('stdin')
286 raise
287
288 return written
289
290 def _recv(self, which, maxsize):
291 conn, maxsize = self.get_conn_maxsize(which, maxsize)
292 if conn is None:
293 return None
294
295 flags = fcntl.fcntl(conn, fcntl.F_GETFL)
296 if not conn.closed:
297 fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK)
298
299 try:
300 if not select.select([conn], [], [], 0)[0]:
301 return ''
302
303 r = conn.read(maxsize)
304
305 if not r:
306 return self._close(which)
307
308 if self.universal_newlines:
309 r = self._translate_newlines(r)
310 return r
311 finally:
312 if not conn.closed:
313 fcntl.fcntl(conn, fcntl.F_SETFL, flags)
314
315message = "Other end disconnected!"
316
317
318def recv_some(p, t=.1, e=1, tr=5, stderr=0):
319 if tr < 1:
320 tr = 1
321 x = time.time() + t
322 y = []
323 r = ''
324 pr = p.recv
325 if stderr:
326 pr = p.recv_err
327 while time.time() < x or r:
328 r = pr()
329 if r is None:
330 if e:
331 raise Exception(message)
332 else:
333 break
334 elif r:
335 y.append(decode(r))
336 else:
337 time.sleep(max((x - time.time()) / tr, 0))
338 return ''.join(y)
339
340
341def send_all(p, data):
342 while len(data):
343 sent = p.send(data)
344 if sent is None:
345 raise Exception(message)
346 data = buffer(data, sent)
347
348
349class Command:
350 """Run command in separate thread. Used for commands launched
351 on the background.
352
353 If stdout/err is redirected, write() method is required for the
354 given classes.
355
356 cmd = Command(cmd=['d.rast', 'elevation.dem'], verbose=3, wait=True)
357
358 if cmd.returncode == None:
359 print 'RUNNING?'
360 elif cmd.returncode == 0:
361 print 'SUCCESS'
362 else:
363 print 'FAILURE (%d)' % cmd.returncode
364 """
365
366 def __init__(self, cmd, stdin=None,
367 verbose=None, wait=True, rerr=False,
368 stdout=None, stderr=None):
369 """
370 :param cmd: command given as list
371 :param stdin: standard input stream
372 :param verbose: verbose level [0, 3] (--q, --v)
373 :param wait: wait for child execution terminated
374 :param rerr: error handling (when GException raised).
375 True for redirection to stderr, False for GUI
376 dialog, None for no operation (quiet mode)
377 :param stdout: redirect standard output or None
378 :param stderr: redirect standard error output or None
379 """
380 Debug.msg(1, "gcmd.Command(): %s" % ' '.join(cmd))
381 self.cmd = cmd
382 self.stderr = stderr
383
384 #
385 # set verbosity level
386 #
387 verbose_orig = None
388 if ('--q' not in self.cmd and '--quiet' not in self.cmd) and \
389 ('--v' not in self.cmd and '--verbose' not in self.cmd):
390 if verbose is not None:
391 if verbose == 0:
392 self.cmd.append('--quiet')
393 elif verbose == 3:
394 self.cmd.append('--verbose')
395 else:
396 verbose_orig = os.getenv("GRASS_VERBOSE")
397 os.environ["GRASS_VERBOSE"] = str(verbose)
398
399 #
400 # create command thread
401 #
402 self.cmdThread = CommandThread(cmd, stdin,
403 stdout, stderr)
404 self.cmdThread.start()
405
406 if wait:
407 self.cmdThread.join()
408 if self.cmdThread.module:
409 self.cmdThread.module.wait()
410 self.returncode = self.cmdThread.module.returncode
411 else:
412 self.returncode = 1
413 else:
414 self.cmdThread.join(0.5)
415 self.returncode = None
416
417 if self.returncode is not None:
418 Debug.msg(
419 3, "Command(): cmd='%s', wait=%s, returncode=%d, alive=%s" %
420 (' '.join(cmd), wait, self.returncode, self.cmdThread.isAlive()))
421 if rerr is not None and self.returncode != 0:
422 if rerr is False: # GUI dialog
423 raise GException("%s '%s'%s%s%s %s%s" %
424 (_("Execution failed:"),
425 ' '.join(self.cmd),
426 os.linesep, os.linesep,
427 _("Details:"),
428 os.linesep,
429 _("Error: ") + self.__GetError()))
430 elif rerr == sys.stderr: # redirect message to sys
431 stderr.write("Execution failed: '%s'" %
432 (' '.join(self.cmd)))
433 stderr.write(
434 "%sDetails:%s%s" %
435 (os.linesep,
436 _("Error: ") +
437 self.__GetError(),
438 os.linesep))
439 else:
440 pass # nop
441 else:
442 Debug.msg(
443 3, "Command(): cmd='%s', wait=%s, returncode=?, alive=%s" %
444 (' '.join(cmd), wait, self.cmdThread.isAlive()))
445
446 if verbose_orig:
447 os.environ["GRASS_VERBOSE"] = verbose_orig
448 elif "GRASS_VERBOSE" in os.environ:
449 del os.environ["GRASS_VERBOSE"]
450
451 def __ReadOutput(self, stream):
452 """Read stream and return list of lines
453
454 :param stream: stream to be read
455 """
456 lineList = []
457
458 if stream is None:
459 return lineList
460
461 while True:
462 line = stream.readline()
463 if not line:
464 break
465 line = line.replace('%s' % os.linesep, '').strip()
466 lineList.append(line)
467
468 return lineList
469
470 def __ReadErrOutput(self):
471 """Read standard error output and return list of lines"""
472 return self.__ReadOutput(self.cmdThread.module.stderr)
473
474 def __ProcessStdErr(self):
475 """
476 Read messages/warnings/errors from stderr
477
478 :return: list of (type, message)
479 """
480 if self.stderr is None:
481 lines = self.__ReadErrOutput()
482 else:
483 lines = self.cmdThread.error.strip('%s' % os.linesep). \
484 split('%s' % os.linesep)
485
486 msg = []
487
488 type = None
489 content = ""
490 for line in lines:
491 if len(line) == 0:
492 continue
493 if 'GRASS_' in line: # error or warning
494 if 'GRASS_INFO_WARNING' in line: # warning
495 type = "WARNING"
496 elif 'GRASS_INFO_ERROR' in line: # error
497 type = "ERROR"
498 elif 'GRASS_INFO_END': # end of message
499 msg.append((type, content))
500 type = None
501 content = ""
502
503 if type:
504 content += line.split(':', 1)[1].strip()
505 else: # stderr
506 msg.append((None, line.strip()))
507
508 return msg
509
510 def __GetError(self):
511 """Get error message or ''"""
512 if not self.cmdThread.module:
513 return _("Unable to exectute command: '%s'") % ' '.join(self.cmd)
514
515 for type, msg in self.__ProcessStdErr():
516 if type == 'ERROR':
517 if _enc:
518 return unicode(msg, _enc)
519 return msg
520
521 return ''
522
523
524class CommandThread(Thread):
525 """Create separate thread for command. Used for commands launched
526 on the background."""
527
528 def __init__(self, cmd, env=None, stdin=None,
529 stdout=sys.stdout, stderr=sys.stderr):
530 """
531 :param cmd: command (given as list)
532 :param env: environmental variables
533 :param stdin: standard input stream
534 :param stdout: redirect standard output or None
535 :param stderr: redirect standard error output or None
536 """
537 Thread.__init__(self)
538
539 self.cmd = cmd
540 self.stdin = stdin
541 self.stdout = stdout
542 self.stderr = stderr
543 self.env = env
544
545 self.module = None
546 self.error = ''
547
548 self._want_abort = False
549 self.aborted = False
550
551 self.setDaemon(True)
552
553 # set message formatting
554 self.message_format = os.getenv("GRASS_MESSAGE_FORMAT")
555 os.environ["GRASS_MESSAGE_FORMAT"] = "gui"
556
557 def __del__(self):
558 if self.message_format:
559 os.environ["GRASS_MESSAGE_FORMAT"] = self.message_format
560 else:
561 del os.environ["GRASS_MESSAGE_FORMAT"]
562
563 def run(self):
564 """Run command"""
565 if len(self.cmd) == 0:
566 return
567
568 Debug.msg(1, "gcmd.CommandThread(): %s" % ' '.join(self.cmd))
569
570 self.startTime = time.time()
571
572 # TODO: replace ugly hack below
573 # this cannot be replaced it can be only improved
574 # also unifying this with 3 other places in code would be nice
575 # changing from one chdir to get_real_command function
576 args = self.cmd
577 if sys.platform == 'win32':
578 if os.path.splitext(args[0])[1] == SCT_EXT:
579 args[0] = args[0][:-3]
580 # using Python executable to run the module if it is a script
581 # expecting at least module name at first position
582 # cannot use make_command for this now because it is used in GUI
583 # The same code is in grass.script.core already twice.
584 args[0] = grass.get_real_command(args[0])
585 if args[0].endswith('.py'):
586 args.insert(0, sys.executable)
587
588 try:
589 self.module = Popen(args,
590 stdin=subprocess.PIPE,
591 stdout=subprocess.PIPE,
592 stderr=subprocess.PIPE,
593 shell=sys.platform == "win32",
594 env=self.env)
595
596 except OSError as e:
597 self.error = str(e)
598 print(e, file=sys.stderr)
599 return 1
600
601 if self.stdin: # read stdin if requested ...
602 self.module.stdin.write(self.stdin)
603 self.module.stdin.close()
604
605 # redirect standard outputs...
606 self._redirect_stream()
607
608 def _redirect_stream(self):
609 """Redirect stream"""
610 if self.stdout:
611 # make module stdout/stderr non-blocking
612 out_fileno = self.module.stdout.fileno()
613 if not is_mswindows:
614 flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL)
615 fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK)
616
617 if self.stderr:
618 # make module stdout/stderr non-blocking
619 out_fileno = self.module.stderr.fileno()
620 if not is_mswindows:
621 flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL)
622 fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK)
623
624 # wait for the process to end, sucking in stuff until it does end
625 while self.module.poll() is None:
626 if self._want_abort: # abort running process
627 self.module.terminate()
628 self.aborted = True
629 return
630 if self.stdout:
631 line = recv_some(self.module, e=0, stderr=0)
632 self.stdout.write(line)
633 if self.stderr:
634 line = recv_some(self.module, e=0, stderr=1)
635 self.stderr.write(line)
636 if len(line) > 0:
637 self.error = line
638
639 # get the last output
640 if self.stdout:
641 line = recv_some(self.module, e=0, stderr=0)
642 self.stdout.write(line)
643 if self.stderr:
644 line = recv_some(self.module, e=0, stderr=1)
645 self.stderr.write(line)
646 if len(line) > 0:
647 self.error = line
648
649 def abort(self):
650 """Abort running process, used by main thread to signal an abort"""
651 self._want_abort = True
652
653
654def _formatMsg(text):
655 """Format error messages for dialogs
656 """
657 message = ''
658 for line in text.splitlines():
659 if len(line) == 0:
660 continue
661 elif 'GRASS_INFO_MESSAGE' in line:
662 message += line.split(':', 1)[1].strip() + '\n'
663 elif 'GRASS_INFO_WARNING' in line:
664 message += line.split(':', 1)[1].strip() + '\n'
665 elif 'GRASS_INFO_ERROR' in line:
666 message += line.split(':', 1)[1].strip() + '\n'
667 elif 'GRASS_INFO_END' in line:
668 return message
669 else:
670 message += line.strip() + '\n'
671
672 return message
673
674
675def RunCommand(prog, flags="", overwrite=False, quiet=False,
676 verbose=False, parent=None, read=False,
677 parse=None, stdin=None, getErrorMsg=False, **kwargs):
678 """Run GRASS command
679
680 :param prog: program to run
681 :param flags: flags given as a string
682 :param overwrite, quiet, verbose: flags
683 :param parent: parent window for error messages
684 :param read: fetch stdout
685 :param parse: fn to parse stdout (e.g. grass.parse_key_val) or None
686 :param stdin: stdin or None
687 :param getErrorMsg: get error messages on failure
688 :param kwargs: program parameters
689
690 :return: returncode (read == False and getErrorMsg == False)
691 :return: returncode, messages (read == False and getErrorMsg == True)
692 :return: stdout (read == True and getErrorMsg == False)
693 :return: returncode, stdout, messages (read == True and getErrorMsg == True)
694 :return: stdout, stderr
695 """
696 cmdString = ' '.join(grass.make_command(prog, flags, overwrite,
697 quiet, verbose, **kwargs))
698
699 Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
700
701 kwargs['stderr'] = subprocess.PIPE
702
703 if read:
704 kwargs['stdout'] = subprocess.PIPE
705
706 if stdin:
707 kwargs['stdin'] = subprocess.PIPE
708
709 if parent:
710 messageFormat = os.getenv('GRASS_MESSAGE_FORMAT', 'gui')
711 os.environ['GRASS_MESSAGE_FORMAT'] = 'standard'
712
713 start = time.time()
714
715 ps = grass.start_command(prog, flags, overwrite, quiet, verbose, **kwargs)
716
717 if stdin:
718 ps.stdin.write(encode(stdin))
719 ps.stdin.close()
720 ps.stdin = None
721
722 stdout, stderr = list(map(DecodeString, ps.communicate()))
723
724 if parent: # restore previous settings
725 os.environ['GRASS_MESSAGE_FORMAT'] = messageFormat
726
727 ret = ps.returncode
728 Debug.msg(1, "gcmd.RunCommand(): get return code %d (%.6f sec)" %
729 (ret, (time.time() - start)))
730
731 if ret != 0:
732 if stderr:
733 Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
734 else:
735 Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
736
737 if parent:
738 GError(parent=parent,
739 caption=_("Error in %s") % prog,
740 message=stderr)
741
742 if not read:
743 if not getErrorMsg:
744 return ret
745 else:
746 return ret, _formatMsg(stderr)
747
748 if stdout:
749 Debug.msg(3, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
750 else:
751 Debug.msg(3, "gcmd.RunCommand(): return stdout = None")
752
753 if parse:
754 stdout = parse(stdout)
755
756 if not getErrorMsg:
757 return stdout
758
759 if read and getErrorMsg:
760 return ret, stdout, _formatMsg(stderr)
761
762 return stdout, _formatMsg(stderr)
763
764
765def GetDefaultEncoding(forceUTF8=False):
766 """Get default system encoding
767
768 :param bool forceUTF8: force 'UTF-8' if encoding is not defined
769
770 :return: system encoding (can be None)
771 """
772 enc = locale.getdefaultlocale()[1]
773 if forceUTF8 and (enc is None or enc == 'UTF8'):
774 return 'UTF-8'
775
776 Debug.msg(1, "GetSystemEncoding(): %s" % enc)
777 return enc
778
779_enc = GetDefaultEncoding() # define as global variable
Note: See TracBrowser for help on using the repository browser.