Context Navigation

source: grass/trunk/lib/python/gunittest/case.py@ 61459

Last change on this file since 61459 was 61459, checked in by wenzeslaus, 10 years ago
gunittest: possibility to automatically test non-empty module output (implemented for runModule and used in assertModuleKeyValue, motivation: zero return code but empty output by r3.univar in case of region diff from map region)
Property svn:eol-style set to `native` Property svn:mime-type set to `text/x-python`
File size: 49.3 KB

Line
1	# -- coding: utf-8 --
2
3	"""!@package grass.gunittest.case
4
5	@brief GRASS Python testing framework test case
6
7	Copyright (C) 2014 by the GRASS Development Team
8	This program is free software under the GNU General Public
9	License (>=v2). Read the file COPYING that comes with GRASS
10	for details.
11
12	@author Vaclav Petras
13	"""
14
15	import os
16	import subprocess
17	import StringIO
18
19	import unittest
20	from unittest.util import safe_repr
21
22	from grass.pygrass.modules import Module
23	from grass.exceptions import CalledModuleError
24
25	from .gmodules import call_module, SimpleModule
26	from .checkers import (check_text_ellipsis,
27	text_to_keyvalue, keyvalue_equals, diff_keyvalue,
28	file_md5, files_equal_md5)
29
30
31	class TestCase(unittest.TestCase):
32	# we dissable R0904 for all TestCase classes because their purpose is to
33	# provide a lot of assert methods
34	# pylint: disable=R0904
35	"""
36
37	Always use keyword arguments for all parameters other than first two. For
38	the first two, it is recommended to use keyword arguments but not required.
39	"""
40	longMessage = True # to get both standard and custom message
41	maxDiff = None # we can afford long diffs
42	_temp_region = None # to control the temporary region
43	html_reports = False # output additional HTML files with failure details
44
45	def __init__(self, methodName):
46	super(TestCase, self).__init__(methodName)
47	self.grass_modules = []
48	self.supplementary_files = []
49
50	def _formatMessage(self, msg, standardMsg):
51	"""Honor the longMessage attribute when generating failure messages.
52
53	If longMessage is False this means:
54
55	* Use only an explicit message if it is provided
56	* Otherwise use the standard message for the assert
57
58	If longMessage is True:
59
60	* Use the standard message
61	* If an explicit message is provided, return string with both messages
62
63	Based on Python unittest _formatMessage, formatting changed.
64	"""
65	if not self.longMessage:
66	return msg or standardMsg
67	if msg is None:
68	return standardMsg
69	try:
70	# don't switch to '{}' formatting in Python 2.X
71	# it changes the way unicode input is handled
72	return '%s \n%s' % (msg, standardMsg)
73	except UnicodeDecodeError:
74	return '%s \n%s' % (safe_repr(msg), safe_repr(standardMsg))
75
76	@classmethod
77	def use_temp_region(cls):
78	"""Use temporary region instead of the standard one for this process.
79
80	If you use this method, you have to call it in `setUpClass()`
81	and call `del_temp_region()` in `tearDownClass()`. By this you
82	ensure that each test method will have its own region and will
83	not influence other classes.
84
85	::
86
87	@classmethod
88	def setUpClass(self):
89	self.use_temp_region()
90
91	@classmethod
92	def tearDownClass(self):
93	self.del_temp_region()
94
95	You can also call the methods in `setUp()` and `tearDown()` if
96	you are using them.
97
98	Copies the current region to a temporary region with
99	``g.region save=``, then sets ``WIND_OVERRIDE`` to refer
100	to that region.
101	"""
102	# we use just the class name since we rely on the invokation system
103	# where each test file is separate process and nothing runs
104	# in parallel inside
105	name = "tmp.%s" % (cls.__name__)
106	call_module("g.region", save=name, overwrite=True)
107	os.environ['WIND_OVERRIDE'] = name
108	cls._temp_region = name
109
110	@classmethod
111	def del_temp_region(cls):
112	"""Remove the temporary region.
113
114	Unsets ``WIND_OVERRIDE`` and removes any region named by it.
115	"""
116	assert cls._temp_region
117	name = os.environ.pop('WIND_OVERRIDE')
118	if name != cls._temp_region:
119	# be strict about usage of region
120	raise RuntimeError("Inconsistent use of"
121	" TestCase.use_temp_region, WIND_OVERRIDE"
122	" or temporary region in general\n"
123	"Region to which should be now deleted ({n})"
124	" by TestCase class"
125	"does not corresond to currently set"
126	" WIND_OVERRIDE ({c})",
127	n=cls._temp_region, c=name)
128	call_module("g.remove", quiet=True, region=name)
129	# TODO: we don't know if user calls this
130	# so perhaps some decorator which would use with statemet
131	# but we have zero chance of infuencing another test class
132	# since we use class-specific name for temporary region
133
134	def assertLooksLike(self, actual, reference, msg=None):
135	"""Test that ``actual`` text is the same as ``referece`` with ellipses.
136
137	See :func:`check_text_ellipsis` for details of behavior.
138	"""
139	self.assertTrue(isinstance(actual, basestring), (
140	'actual argument is not a string'))
141	self.assertTrue(isinstance(reference, basestring), (
142	'reference argument is not a string'))
143	if not check_text_ellipsis(actual=actual, reference=reference):
144	# TODO: add support for multiline (first line general, others with details)
145	standardMsg = '"%s" does not correspond with "%s"' % (actual,
146	reference)
147	self.fail(self._formatMessage(msg, standardMsg))
148
149	# TODO: decide if precision is mandatory
150	# (note that we don't need precision for strings and usually for integers)
151	# TODO: auto-determine precision based on the map type
152	# TODO: we can have also more general function without the subset reference
153	# TODO: change name to Module
154	def assertModuleKeyValue(self, module, reference, sep,
155	precision, msg=None, **parameters):
156	"""Test that output of a module is the same as provided subset.
157
158	::
159
160	self.assertModuleKeyValue('r.info', map='elevation', flags='gr',
161	reference=dict(min=55.58, max=156.33),
162	precision=0.01, sep='=')
163
164	::
165
166	module = SimpleModule('r.info', map='elevation', flags='gr')
167	self.assertModuleKeyValue(module,
168	reference=dict(min=55.58, max=156.33),
169	precision=0.01, sep='=')
170
171	The output of the module should be key-value pairs (shell script style)
172	which is typically obtained using ``-g`` flag.
173	"""
174	if isinstance(reference, basestring):
175	reference = text_to_keyvalue(reference, sep=sep, skip_empty=True)
176	module = _module_from_parameters(module, **parameters)
177	self.runModule(module, expecting_stdout=True)
178	raster_univar = text_to_keyvalue(module.outputs.stdout,
179	sep=sep, skip_empty=True)
180	if not keyvalue_equals(dict_a=reference, dict_b=raster_univar,
181	a_is_subset=True, precision=precision):
182	unused, missing, mismatch = diff_keyvalue(dict_a=reference,
183	dict_b=raster_univar,
184	a_is_subset=True,
185	precision=precision)
186	# TODO: add region vs map extent and res check in case of error
187	if missing:
188	raise ValueError("%s output does not contain"
189	" the following keys"
190	" provided in reference"
191	": %s\n" % (module, ", ".join(missing)))
192	if mismatch:
193	stdMsg = "%s difference:\n" % module
194	stdMsg += "mismatch values"
195	stdMsg += "(key, reference, actual): %s\n" % mismatch
196	stdMsg += 'command: %s %s' % (module, parameters)
197	else:
198	# we can probably remove this once we have more tests
199	# of keyvalue_equals and diff_keyvalue against each other
200	raise RuntimeError("keyvalue_equals() showed difference but"
201	" diff_keyvalue() did not. This can be"
202	" a bug in one of them or in the caller"
203	" (assertModuleKeyValue())")
204	self.fail(self._formatMessage(msg, stdMsg))
205
206	def assertRasterFitsUnivar(self, raster, reference,
207	precision=None, msg=None):
208	r"""Test that raster map has the values obtained by r.univar module.
209
210	The function does not require all values from r.univar.
211	Only the provided values are tested.
212	Typical example is checking minimum, maximum and number of NULL cells
213	in the map::
214
215	values = 'null_cells=0\nmin=55.5787925720215\nmax=156.329864501953'
216	self.assertRasterFitsUnivar(map='elevation', reference=values)
217
218	Use keyword arguments syntax for all function parameters.
219
220	Does not -e (extended statistics) flag, use `assertModuleKeyValue()`
221	for the full interface of arbitrary module.
222	"""
223	self.assertModuleKeyValue(module='r.univar',
224	map=raster,
225	separator='=',
226	flags='g',
227	reference=reference, msg=msg, sep='=',
228	precision=precision)
229
230	def assertRasterFitsInfo(self, raster, reference,
231	precision=None, msg=None):
232	r"""Test that raster map has the values obtained by r.univar module.
233
234	The function does not require all values from r.univar.
235	Only the provided values are tested.
236	Typical example is checking minimum, maximum and type of the map::
237
238	minmax = 'min=0\nmax=1451\ndatatype=FCELL'
239	self.assertRasterFitsInfo(map='elevation', reference=values)
240
241	Use keyword arguments syntax for all function parameters.
242
243	This function supports values obtained -r (range) and
244	-e (extended metadata) flags.
245	"""
246	self.assertModuleKeyValue(module='r.info',
247	map=raster, flags='gre',
248	reference=reference, msg=msg, sep='=',
249	precision=precision)
250
251	def assertRaster3dFitsUnivar(self, raster, reference,
252	precision=None, msg=None):
253	r"""Test that 3D raster map has the values obtained by r3.univar module.
254
255	The function does not require all values from r3.univar.
256	Only the provided values are tested.
257
258	Use keyword arguments syntax for all function parameters.
259
260	Function does not use -e (extended statistics) flag,
261	use `assertModuleKeyValue()` for the full interface of arbitrary
262	module.
263	"""
264	self.assertModuleKeyValue(module='r3.univar',
265	map=raster,
266	separator='=',
267	flags='g',
268	reference=reference, msg=msg, sep='=',
269	precision=precision)
270
271	def assertRaster3dFitsInfo(self, raster, reference,
272	precision=None, msg=None):
273	r"""Test that raster map has the values obtained by r3.info module.
274
275	The function does not require all values from r3.info.
276	Only the provided values are tested.
277
278	Use keyword arguments syntax for all function parameters.
279
280	This function supports values obtained by -g (info) and -r (range).
281	"""
282	self.assertModuleKeyValue(module='r3.info',
283	map=raster, flags='gr',
284	reference=reference, msg=msg, sep='=',
285	precision=precision)
286
287	def assertVectorFitsTopoInfo(self, vector, reference, msg=None):
288	r"""Test that raster map has the values obtained by ``v.info`` module.
289
290	This function uses ``-t`` flag of ``v.info`` module to get topology
291	info, so the reference dictionary should contain appropriate set or
292	subset of values (only the provided values are tested).
293
294	A example of checking number of points::
295
296	topology = dict(points=10938, primitives=10938)
297	self.assertVectorFitsTopoInfo(map='bridges', reference=topology)
298
299	Note that here we are checking also the number of primitives to prove
300	that there are no other features besides points.
301
302	No precision is applied (no difference is required). So, this function
303	is not suitable for testing items which are floating point number
304	(no such items are currently in topological information).
305
306	Use keyword arguments syntax for all function parameters.
307	"""
308	self.assertModuleKeyValue(module='v.info',
309	map=vector, flags='t',
310	reference=reference, msg=msg, sep='=',
311	precision=0)
312
313	def assertVectorFitsRegionInfo(self, vector, reference,
314	precision, msg=None):
315	r"""Test that raster map has the values obtained by ``v.info`` module.
316
317	This function uses ``-g`` flag of ``v.info`` module to get topology
318	info, so the reference dictionary should contain appropriate set or
319	subset of values (only the provided values are tested).
320
321	Use keyword arguments syntax for all function parameters.
322	"""
323	self.assertModuleKeyValue(module='v.info',
324	map=vector, flags='g',
325	reference=reference, msg=msg, sep='=',
326	precision=precision)
327
328	def assertVectorFitsExtendedInfo(self, vector, reference, msg=None):
329	r"""Test that raster map has the values obtained by ``v.info`` module.
330
331	This function uses ``-e`` flag of ``v.info`` module to get topology
332	info, so the reference dictionary should contain appropriate set or
333	subset of values (only the provided values are tested).
334
335	The most useful items for testing (considering circumstances of test
336	invocation) are name, title, level and num_dblinks. (When testing
337	storing of ``v.info -e`` metadata, the selection might be different.)
338
339	No precision is applied (no difference is required). So, this function
340	is not suitable for testing items which are floating point number.
341
342	Use keyword arguments syntax for all function parameters.
343	"""
344	self.assertModuleKeyValue(module='v.info',
345	map=vector, flags='e',
346	reference=reference, msg=msg, sep='=',
347	precision=0)
348
349	def assertVectorInfoEqualsVectorInfo(self, actual, reference, precision,
350	msg=None):
351	"""Test that two vectors are equal according to ``v.info -tg``.
352
353	This function does not test geometry itself just the region of the
354	vector map and number of features.
355	"""
356	module = SimpleModule('v.info', flags='t', map=reference)
357	self.runModule(module)
358	ref_topo = text_to_keyvalue(module.outputs.stdout, sep='=')
359	module = SimpleModule('v.info', flags='g', map=reference)
360	self.runModule(module)
361	ref_info = text_to_keyvalue(module.outputs.stdout, sep='=')
362	self.assertVectorFitsTopoInfo(vector=actual, reference=ref_topo,
363	msg=msg)
364	self.assertVectorFitsRegionInfo(vector=actual, reference=ref_info,
365	precision=precision, msg=msg)
366
367	def assertVectorFitsUnivar(self, map, column, reference, msg=None,
368	layer=None, type=None, where=None,
369	precision=None):
370	r"""Test that vector map has the values obtained by v.univar module.
371
372	The function does not require all values from v.univar.
373	Only the provided values are tested.
374	Typical example is checking minimum and maximum of a column::
375
376	minmax = 'min=0\nmax=1451'
377	self.assertVectorFitsUnivar(map='bridges', column='WIDTH',
378	reference=minmax)
379
380	Use keyword arguments syntax for all function parameters.
381
382	Does not support -d (geometry distances) flag, -e (extended statistics)
383	flag and few other, use `assertModuleKeyValue` for the full interface
384	of arbitrary module.
385	"""
386	parameters = dict(map=map, column=column, flags='g')
387	if layer:
388	parameters.update(layer=layer)
389	if type:
390	parameters.update(type=type)
391	if where:
392	parameters.update(where=where)
393	self.assertModuleKeyValue(module='v.univar',
394	reference=reference, msg=msg, sep='=',
395	precision=precision,
396	**parameters)
397
398	# TODO: use precision?
399	# TODO: write a test for this method with r.in.ascii
400	def assertRasterMinMax(self, map, refmin, refmax, msg=None):
401	"""Test that raster map minimum and maximum are within limits.
402
403	Map minimum and maximum is tested against expression::
404
405	refmin <= actualmin and refmax >= actualmax
406
407	Use keyword arguments syntax for all function parameters.
408
409	To check that more statistics have certain values use
410	`assertRasterFitsUnivar()` or `assertRasterFitsInfo()`
411	"""
412	stdout = call_module('r.info', map=map, flags='r')
413	actual = text_to_keyvalue(stdout, sep='=')
414	if refmin > actual['min']:
415	stdmsg = ('The actual minimum ({a}) is smaller than the reference'
416	' one ({r}) for raster map {m}'
417	' (with maximum {o})'.format(
418	a=actual['min'], r=refmin, m=map, o=actual['max']))
419	self.fail(self._formatMessage(msg, stdmsg))
420	if refmax < actual['max']:
421	stdmsg = ('The actual maximum ({a}) is greater than the reference'
422	' one ({r}) for raster map {m}'
423	' (with minimum {o})'.format(
424	a=actual['max'], r=refmax, m=map, o=actual['min']))
425	self.fail(self._formatMessage(msg, stdmsg))
426
427	# TODO: use precision?
428	# TODO: write a test for this method with r.in.ascii
429	# TODO: almost the same as 2D version
430	def assertRaster3dMinMax(self, map, refmin, refmax, msg=None):
431	"""Test that 3D raster map minimum and maximum are within limits.
432
433	Map minimum and maximum is tested against expression::
434
435	refmin <= actualmin and refmax >= actualmax
436
437	Use keyword arguments syntax for all function parameters.
438
439	To check that more statistics have certain values use
440	`assertRaster3DFitsUnivar()` or `assertRaster3DFitsInfo()`
441	"""
442	stdout = call_module('r3.info', map=map, flags='r')
443	actual = text_to_keyvalue(stdout, sep='=')
444	if refmin > actual['min']:
445	stdmsg = ('The actual minimum ({a}) is smaller than the reference'
446	' one ({r}) for 3D raster map {m}'
447	' (with maximum {o})'.format(
448	a=actual['min'], r=refmin, m=map, o=actual['max']))
449	self.fail(self._formatMessage(msg, stdmsg))
450	if refmax < actual['max']:
451	stdmsg = ('The actual maximum ({a}) is greater than the reference'
452	' one ({r}) for 3D raster map {m}'
453	' (with minimum {o})'.format(
454	a=actual['max'], r=refmax, m=map, o=actual['min']))
455	self.fail(self._formatMessage(msg, stdmsg))
456
457	def assertFileExists(self, filename, msg=None,
458	skip_size_check=False, skip_access_check=False):
459	"""Test the existence of a file.
460
461	.. note:
462	By default this also checks if the file size is greater than 0
463	since we rarely want a file to be empty. And it also checks
464	if the file is access for reading.
465	"""
466	if not os.path.isfile(filename):
467	stdmsg = 'File %s does not exist' % filename
468	self.fail(self._formatMessage(msg, stdmsg))
469	if not skip_size_check and not os.path.getsize(filename):
470	stdmsg = 'File %s is empty' % filename
471	self.fail(self._formatMessage(msg, stdmsg))
472	if not skip_access_check and not os.access(filename, os.R_OK):
473	stdmsg = 'File %s is not accessible for reading' % filename
474	self.fail(self._formatMessage(msg, stdmsg))
475
476	def assertFileMd5(self, filename, md5, msg=None):
477	"""Test that file MD5 sum is equal to the provided sum.
478
479	The typical workflow is that you create a file in a way you
480	trust (that you obtain the right file). Then you compute MD5
481	sum of the file. And provide the sum in a test as a string::
482
483	self.assertFileMd5('result.txt', md5='807bba4ffa...')
484
485	Use `file_md5()` function from this package::
486
487	file_md5('original_result.txt')
488
489	Or in command line, use ``md5sum`` command if available:
490
491	.. code-block:: sh
492	md5sum some_file.txt
493
494	Finaly, you can use Python ``hashlib`` to obtain MD5::
495
496	import hashlib
497	hasher = hashlib.md5()
498	# expecting the file to fit into memory
499	hasher.update(open('original_result.txt', 'rb').read())
500	hasher.hexdigest()
501	"""
502	self.assertFileExists(filename, msg=msg)
503	if not file_md5(filename) == md5:
504	standardMsg = 'File %s does not have the right MD5 sum' % filename
505	self.fail(self._formatMessage(msg, standardMsg))
506
507	def assertFilesEqualMd5(self, filename, reference, msg=None):
508	"""Test that files are the same using MD5 sum.
509
510	This functions requires you to provide a file to test and
511	a reference file. For both, MD5 sum will be computed and compared with
512	each other.
513	"""
514	self.assertFileExists(filename, msg=msg)
515	# nothing for ref, missing ref_filename is an error not a test failure
516	if not files_equal_md5(filename, reference):
517	stdmsg = 'Files %s and %s don\'t have the same MD5 sums' % (filename,
518	reference)
519	self.fail(self._formatMessage(msg, stdmsg))
520
521	def _compute_difference_raster(self, first, second, name_part):
522	"""Compute difference of two rasters (first - second)
523
524	The name of the new raster is a long name designed to be as unique as
525	possible and contains names of two input rasters.
526
527	:param first: raster to subtract from
528	:param second: raster used as decrement
529	:param name_part: a unique string to be used in the difference name
530
531	:returns: name of a new raster
532	"""
533	diff = ('tmp_' + self.id() + '_compute_difference_raster_'
534	+ name_part + '_' + first + '_minus_' + second)
535	call_module('r.mapcalc',
536	stdin='"{d}" = "{f}" - "{s}"'.format(d=diff,
537	f=first,
538	s=second))
539	return diff
540
541	# TODO: name of map generation is repeted three times
542	# TODO: this method is almost the same as the one for 2D
543	def _compute_difference_raster3d(self, first, second, name_part):
544	"""Compute difference of two rasters (first - second)
545
546	The name of the new raster is a long name designed to be as unique as
547	possible and contains names of two input rasters.
548
549	:param first: raster to subtract from
550	:param second: raster used as decrement
551	:param name_part: a unique string to be used in the difference name
552
553	:returns: name of a new raster
554	"""
555	diff = ('tmp_' + self.id() + '_compute_difference_raster_'
556	+ name_part + '_' + first + '_minus_' + second)
557	call_module('r3.mapcalc',
558	stdin='"{d}" = "{f}" - "{s}"'.format(d=diff,
559	f=first,
560	s=second))
561	return diff
562
563	def _compute_vector_xor(self, ainput, alayer, binput, blayer, name_part):
564	"""Compute symmetric difference (xor) of two vectors
565
566	:returns: name of a new vector
567	"""
568	diff = ('tmp_' + self.id() + '_compute_difference_vector_'
569	+ name_part + '_' + ainput + '_' + alayer
570	+ '_minus_' + binput + '_' + blayer)
571	call_module('v.overlay', operator='xor', ainput=ainput, binput=binput,
572	alayer=alayer, blayer=blayer,
573	output=diff, atype='area', btype='area', olayer='')
574	# trying to avoid long reports full of categories by olayer=''
575	# olayer Output layer for new category, ainput and binput
576	# If 0 or not given, the category is not written
577	return diff
578
579	# TODO: -z and 3D support
580	def _import_ascii_vector(self, filename, name_part):
581	"""Import a vector stored in GRASS vector ASCII format.
582
583	:returns: name of a new vector
584	"""
585	import hashlib
586	# hash is the easiest way how to get a valied vector name
587	# TODO: introduce some function which will make file valid
588	hasher = hashlib.md5()
589	hasher.update(filename)
590	namehash = hasher.hexdigest()
591	vector = ('tmp_' + self.id().replace('.', '_')
592	+ '_import_ascii_vector_'
593	+ name_part + '_' + namehash)
594	call_module('v.in.ascii', input=filename,
595	output=vector, format='standard')
596	return vector
597
598	# TODO: -z and 3D support
599	def _export_ascii_vector(self, vector, name_part, digits):
600	"""Import a vector stored in GRASS vector ASCII format.
601
602	:returns: name of a new vector
603	"""
604	# TODO: perhaps we can afford just simple file name
605	filename = ('tmp_' + self.id() + '_export_ascii_vector_'
606	+ name_part + '_' + vector)
607	call_module('v.out.ascii', input=vector,
608	output=filename, format='standard', layer='-1', dp=digits)
609	return filename
610
611	def assertRastersNoDifference(self, actual, reference,
612	precision, statistics=None, msg=None):
613	"""Test that `actual` raster is not different from `reference` raster
614
615	Method behaves in the same way as `assertRasterFitsUnivar()`
616	but works on difference ``reference - actual``.
617	If statistics is not given ``dict(min=-precision, max=precision)``
618	is used.
619	"""
620	if statistics is None or sorted(statistics.keys()) == ['max', 'min']:
621	if statistics is None:
622	statistics = dict(min=-precision, max=precision)
623	diff = self._compute_difference_raster(reference, actual,
624	'assertRastersNoDifference')
625	try:
626	self.assertModuleKeyValue('r.info', map=diff, flags='r',
627	sep='=', precision=precision,
628	reference=statistics, msg=msg)
629	finally:
630	call_module('g.remove', rast=diff)
631	else:
632	# general case
633	# TODO: we are using r.info min max and r.univar min max interchangably
634	# but they might be different if region is different from map
635	# not considered as an huge issue since we expect the tested maps
636	# to match with region, however a documentation should containe a notice
637	self.assertRastersDifference(actual=actual, reference=reference,
638	statistics=statistics,
639	precision=precision, msg=msg)
640
641	def assertRastersDifference(self, actual, reference,
642	statistics, precision, msg=None):
643	"""Test statistical values of difference of reference and actual rasters
644
645	For cases when you are interested in no or minimal difference,
646	use `assertRastersNoDifference()` instead.
647
648	This method should not be used to test r.mapcalc or r.univar.
649	"""
650	diff = self._compute_difference_raster(reference, actual,
651	'assertRastersDifference')
652	try:
653	self.assertRasterFitsUnivar(raster=diff, reference=statistics,
654	precision=precision, msg=msg)
655	finally:
656	call_module('g.remove', rast=diff)
657
658	def assertRasters3dNoDifference(self, actual, reference,
659	precision, statistics=None, msg=None):
660	"""Test that `actual` raster is not different from `reference` raster
661
662	Method behaves in the same way as `assertRasterFitsUnivar()`
663	but works on difference ``reference - actual``.
664	If statistics is not given ``dict(min=-precision, max=precision)``
665	is used.
666	"""
667	if statistics is None or sorted(statistics.keys()) == ['max', 'min']:
668	if statistics is None:
669	statistics = dict(min=-precision, max=precision)
670	diff = self._compute_difference_raster3d(reference, actual,
671	'assertRasters3dNoDifference')
672	try:
673	self.assertModuleKeyValue('r3.info', map=diff, flags='r',
674	sep='=', precision=precision,
675	reference=statistics, msg=msg)
676	finally:
677	call_module('g.remove', rast3d=diff)
678	else:
679	# general case
680	# TODO: we are using r.info min max and r.univar min max interchangably
681	# but they might be different if region is different from map
682	# not considered as an huge issue since we expect the tested maps
683	# to match with region, however a documentation should contain a notice
684	self.assertRasters3dDifference(actual=actual, reference=reference,
685	statistics=statistics,
686	precision=precision, msg=msg)
687
688	def assertRasters3dDifference(self, actual, reference,
689	statistics, precision, msg=None):
690	"""Test statistical values of difference of reference and actual rasters
691
692	For cases when you are interested in no or minimal difference,
693	use `assertRastersNoDifference()` instead.
694
695	This method should not be used to test r3.mapcalc or r3.univar.
696	"""
697	diff = self._compute_difference_raster3d(reference, actual,
698	'assertRasters3dDifference')
699	try:
700	self.assertRaster3dFitsUnivar(raster=diff, reference=statistics,
701	precision=precision, msg=msg)
702	finally:
703	call_module('g.remove', rast3d=diff)
704
705	# TODO: this works only in 2D
706	# TODO: write tests
707	def assertVectorIsVectorBuffered(self, actual, reference, precision, msg=None):
708	"""
709
710	This method should not be used to test v.buffer, v.overlay or v.select.
711	"""
712	# TODO: if msg is None: add info specific to this function
713	layer = '-1'
714	self.assertVectorInfoEqualsVectorInfo(actual=actual,
715	reference=reference,
716	precision=precision, msg=msg)
717	remove = []
718	buffered = reference + '_buffered' # TODO: more unique name
719	intersection = reference + '_intersection' # TODO: more unique name
720	self.runModule('v.buffer', input=reference, layer=layer,
721	output=buffered, distance=precision)
722	remove.append(buffered)
723	try:
724	self.runModule('v.overlay', operator='and', ainput=actual,
725	binput=reference,
726	alayer=layer, blayer=layer,
727	output=intersection, atype='area', btype='area',
728	olayer='')
729	remove.append(intersection)
730	# TODO: this would use some refactoring
731	# perhaps different functions or more low level functions would
732	# be more appropriate
733	module = SimpleModule('v.info', flags='t', map=reference)
734	self.runModule(module)
735	ref_topo = text_to_keyvalue(module.outputs.stdout, sep='=')
736	self.assertVectorFitsTopoInfo(vector=intersection,
737	reference=ref_topo,
738	msg=msg)
739	module = SimpleModule('v.info', flags='g', map=reference)
740	self.runModule(module)
741	ref_info = text_to_keyvalue(module.outputs.stdout, sep='=')
742	self.assertVectorFitsRegionInfo(vector=intersection,
743	reference=ref_info,
744	msg=msg, precision=precision)
745	finally:
746	call_module('g.remove', vect=remove)
747
748	# TODO: write tests
749	def assertVectorsNoAreaDifference(self, actual, reference, precision,
750	layer=1, msg=None):
751	"""Test statistical values of difference of reference and actual rasters
752
753	Works only for areas.
754
755	Use keyword arguments syntax for all function parameters.
756
757	This method should not be used to test v.overlay or v.select.
758	"""
759	diff = self._compute_xor_vectors(ainput=reference, binput=actual,
760	alayer=layer, blayer=layer,
761	name_part='assertVectorsNoDifference')
762	try:
763	module = SimpleModule('v.to.db', map=diff,
764	flags='pc', separator='=')
765	self.runModule(module)
766	# the output of v.to.db -pc sep== should look like:
767	# ...
768	# 43=98606087.5818323
769	# 44=727592.902311112
770	# total area=2219442027.22035
771	total_area = module.outputs.stdout.splitlines()[-1].split('=')[-1]
772	if total_area > precision:
773	stdmsg = ("Area of difference of vectors <{va}> and <{vr}>"
774	" should be 0"
775	" in the given precision ({p}) not {a}").format(
776	va=actual, vr=reference, p=precision, a=total_area)
777	self.fail(self._formatMessage(msg, stdmsg))
778	finally:
779	call_module('g.remove', vect=diff)
780
781	# TODO: here we have to have significant digits which is not consistent
782	# TODO: documentation for all new asserts
783	# TODO: same can be created for raster and 3D raster
784	def assertVectorEqualsVector(self, actual, reference, digits, precision, msg=None):
785	"""Test that two vectors are equal.
786
787	.. note:
788	This test should not be used to test ``v.in.ascii`` and
789	``v.out.ascii`` modules.
790
791	.. warning:
792	ASCII files for vectors are loaded into memory, so this
793	function works well only for "not too big" vector maps.
794	"""
795	# both vectors to ascii
796	# text diff of two ascii files
797	# may also do other comparisons on vectors themselves (asserts)
798	self.assertVectorInfoEqualsVectorInfo(actual=actual, reference=reference, precision=precision, msg=msg)
799	factual = self._export_ascii_vector(vector=actual,
800	name_part='assertVectorEqualsVector_actual',
801	digits=digits)
802	freference = self._export_ascii_vector(vector=reference,
803	name_part='assertVectorEqualsVector_reference',
804	digits=digits)
805	self.assertVectorAsciiEqualsVectorAscii(actual=factual,
806	reference=freference,
807	remove_files=True,
808	msg=msg)
809
810	def assertVectorEqualsAscii(self, actual, reference, digits, precision, msg=None):
811	"""Test that vector is equal to the vector stored in GRASS ASCII file.
812
813	.. note:
814	This test should not be used to test ``v.in.ascii`` and
815	``v.out.ascii`` modules.
816
817	.. warning:
818	ASCII files for vectors are loaded into memory, so this
819	function works well only for "not too big" vector maps.
820	"""
821	# vector to ascii
822	# text diff of two ascii files
823	# it may actually import the file and do other asserts
824	factual = self._export_ascii_vector(vector=actual,
825	name_part='assertVectorEqualsAscii_actual',
826	digits=digits)
827	vreference = None
828	try:
829	vreference = self._import_ascii_vector(filename=reference,
830	name_part='assertVectorEqualsAscii_reference')
831	self.assertVectorInfoEqualsVectorInfo(actual=actual,
832	reference=vreference,
833	precision=precision, msg=msg)
834	self.assertVectorAsciiEqualsVectorAscii(actual=factual,
835	reference=reference,
836	remove_files=False,
837	msg=msg)
838	finally:
839	# TODO: manage using cleanup settings
840	# we rely on fail method to either raise or return (soon)
841	os.remove(factual)
842	if vreference:
843	self.runModule('g.remove', vect=vreference)
844
845	# TODO: we expect v.out.ascii to give the same order all the time, is that OK?
846	def assertVectorAsciiEqualsVectorAscii(self, actual, reference,
847	remove_files=False, msg=None):
848	"""Test that two GRASS ASCII vector files are equal.
849
850	.. note:
851	This test should not be used to test ``v.in.ascii`` and
852	``v.out.ascii`` modules.
853
854	.. warning:
855	ASCII files for vectors are loaded into memory, so this
856	function works well only for "not too big" vector maps.
857	"""
858	import difflib
859	# 'U' taken from difflib documentation
860	fromlines = open(actual, 'U').readlines()
861	tolines = open(reference, 'U').readlines()
862	context_lines = 3 # number of context lines
863	# TODO: filenames are set to "actual" and "reference", isn't it too general?
864	# it is even more useful if map names or file names are some generated
865	# with hash or some other unreadable things
866	# other styles of diffs are available too
867	# but unified is a good choice if you are used to svn or git
868	# workaround for missing -h (do not print header) flag in v.out.ascii
869	num_lines_of_header = 10
870	diff = difflib.unified_diff(fromlines[num_lines_of_header:],
871	tolines[num_lines_of_header:],
872	'reference', 'actual', n=context_lines)
873	# TODO: this should be solved according to cleanup policy
874	# but the parameter should be kept if it is an existing file
875	# or using this method by itself
876	if remove_files:
877	os.remove(actual)
878	os.remove(reference)
879	stdmsg = ("There is a difference between vectors when compared as"
880	" ASCII files.\n")
881
882	output = StringIO.StringIO()
883	# TODO: there is a diff size constant which we can use
884	# we are setting it unlimited but we can just set it large
885	maxlines = 100
886	i = 0
887	for line in diff:
888	if i >= maxlines:
889	break
890	output.write(line)
891	i += 1
892	stdmsg += output.getvalue()
893	output.close()
894	# it seems that there is not better way of asking whether there was
895	# a difference (always a iterator object is returned)
896	if i > 0:
897	# do HTML diff only if there is not too many lines
898	# TODO: this might be tough to do with some more sophisticated way of reports
899	if self.html_reports and i < maxlines:
900	# TODO: this might be here and somehow stored as file or done in reporter again if right information is stored
901	# i.e., files not deleted or the whole strings passed
902	# alternative is make_table() which is the same but creates just a table not a whole document
903	# TODO: all HTML files might be collected by the main reporter
904	# TODO: standardize the format of name of HTML file
905	# for one test id there is only one possible file of this name
906	htmldiff_file_name = self.id() + '_ascii_diff' + '.html'
907	self.supplementary_files.append(htmldiff_file_name)
908	htmldiff = difflib.HtmlDiff().make_file(fromlines, tolines,
909	'reference', 'actual',
910	context=True,
911	numlines=context_lines)
912	htmldiff_file = open(htmldiff_file_name, 'w')
913	for line in htmldiff:
914	htmldiff_file.write(line)
915	htmldiff_file.close()
916
917	self.fail(self._formatMessage(msg, stdmsg))
918
919	@classmethod
920	def runModule(cls, module, expecting_stdout=False, **kwargs):
921	"""Run PyGRASS module.
922
923	Runs the module and raises an exception if the module ends with
924	non-zero return code. Usually, this is the same as testing the
925	return code and raising exception but by using this method,
926	you give testing framework more control over the execution,
927	error handling and storing of output.
928
929	In terms of testing framework, this function causes a common error,
930	not a test failure.
931
932	:raises CalledModuleError: if the module failed
933	"""
934	module = _module_from_parameters(module, **kwargs)
935	_check_module_run_parameters(module)
936	try:
937	module.run()
938	except CalledModuleError:
939	# here exception raised by run() with finish_=True would be
940	# almost enough but we want some additional info to be included
941	# in the test report
942	errors = module.outputs.stderr
943	# provide diagnostic at least in English locale
944	# TODO: standardized error code would be handy here
945	import re
946	if re.search('Raster map.*not found', errors, flags=re.DOTALL):
947	errors += "\nSee available raster maps:\n"
948	errors += call_module('g.list', type='rast')
949	if re.search('Vector map.*not found', errors, flags=re.DOTALL):
950	errors += "\nSee available vector maps:\n"
951	errors += call_module('g.list', type='vect')
952	# TODO: message format, parameters
953	raise CalledModuleError(module.popen.returncode, module.name,
954	module.get_python(),
955	errors=errors)
956	# TODO: use this also in assert and apply when appropriate
957	if expecting_stdout and not module.outputs.stdout.strip():
958
959	if module.outputs.stderr:
960	errors = " The errors are:\n" + module.outputs.stderr
961	else:
962	errors = " There were no error messages."
963	if module.outputs.stdout:
964	# this is not appropriate for translation but we don't want
965	# and don't need testing to be translated
966	got = "only whitespace."
967	else:
968	got = "nothing."
969	raise RuntimeError("Module call " + module.get_python() +
970	" ended successfully but we were expecting"
971	" output and got " + got + errors)
972	# TODO: we can also comapre time to some expected but that's tricky
973	# maybe we should measure time but the real benchmarks with stdin/stdout
974	# should be done by some other function
975	# TODO: this should be the function used for valgrind or profiling or debug
976	# TODO: it asserts the rc but it does much more, so testModule?
977	# TODO: do we need special function for testing module failures or just add parameter returncode=0?
978	# TODO: consider not allowing to call this method more than once
979	# the original idea was to run this method just once for test method
980	# but for "integration" tests (script-like tests with more than one module)
981	# it would be better to be able to use this multiple times
982	# TODO: enable merging streams?
983	def assertModule(self, module, msg=None, **kwargs):
984	"""Run PyGRASS module in controlled way and assert non-zero return code.
985
986	You should use this method to invoke module you are testing.
987	By using this method, you give testing framework more control over
988	the execution, error handling and storing of output.
989
990	It will not print module stdout and stderr, instead it will always
991	store them for further examination. Streams are stored separately.
992
993	This method is not suitable for testing error states of the module.
994	If you want to test behavior which involves non-zero return codes
995	and examine stderr in test, use `assertModuleFail()` method.
996
997	Runs the module and causes test failure if module ends with
998	non-zero return code.
999	"""
1000	module = _module_from_parameters(module, **kwargs)
1001	_check_module_run_parameters(module)
1002	try:
1003	module.run()
1004	self.grass_modules.append(module.name)
1005	except CalledModuleError:
1006	print module.outputs.stdout
1007	print module.outputs.stderr
1008	# TODO: message format
1009	# TODO: stderr?
1010	stdmsg = ('Running <{m.name}> module ended'
1011	' with non-zero return code ({m.popen.returncode})\n'
1012	'Called: {code}\n'
1013	'See the folowing errors:\n'
1014	'{errors}'.format(
1015	m=module, code=module.get_python(),
1016	errors=module.outputs.stderr
1017	))
1018	self.fail(self._formatMessage(msg, stdmsg))
1019	print module.outputs.stdout
1020	print module.outputs.stderr
1021	# log these to final report
1022	# TODO: always or only if the calling test method failed?
1023	# in any case, this must be done before self.fail()
1024	# module.outputs['stdout'].value
1025	# module.outputs['stderr'].value
1026
1027	# TODO: should we merge stderr to stdout in this case?
1028	def assertModuleFail(self, module, msg=None, **kwargs):
1029	"""Test that module fails with a non-zero return code.
1030
1031	Works like `assertModule()` but expects module to fail.
1032	"""
1033	module = _module_from_parameters(module, **kwargs)
1034	_check_module_run_parameters(module)
1035	# note that we cannot use finally because we do not leave except
1036	try:
1037	module.run()
1038	self.grass_modules.append(module.name)
1039	except CalledModuleError:
1040	print module.outputs.stdout
1041	print module.outputs.stderr
1042	else:
1043	print module.outputs.stdout
1044	print module.outputs.stderr
1045	stdmsg = ('Running <%s> ended with zero (successful) return code'
1046	' when expecting module to fail' % module.get_python())
1047	self.fail(self._formatMessage(msg, stdmsg))
1048
1049
1050	# TODO: add tests and documentation to methods which are using this function
1051	# some test and documentation add to assertModuleKeyValue
1052	def _module_from_parameters(module, **kwargs):
1053	if kwargs:
1054	if not isinstance(module, basestring):
1055	raise ValueError('module can be only string or PyGRASS Module')
1056	if isinstance(module, Module):
1057	raise ValueError('module can be only string if other'
1058	' parameters are given')
1059	# allow to pass all parameters in one dictionary called parameters
1060	if kwargs.keys() == ['parameters']:
1061	kwargs = kwargs['parameters']
1062	module = SimpleModule(module, **kwargs)
1063	return module
1064
1065
1066	def _check_module_run_parameters(module):
1067	# in this case module already run and we would start it again
1068	if module.run_:
1069	raise ValueError('Do not run the module manually, set run_=False')
1070	if not module.finish_:
1071	raise ValueError('This function will always finish module run,'
1072	' set finish_=None or finish_=True.')
1073	# we expect most of the usages with stdout=PIPE
1074	# TODO: in any case capture PIPE always?
1075	if module.stdout_ is None:
1076	module.stdout_ = subprocess.PIPE
1077	elif module.stdout_ != subprocess.PIPE:
1078	raise ValueError('stdout_ can be only PIPE or None')
1079	if module.stderr_ is None:
1080	module.stderr_ = subprocess.PIPE
1081	elif module.stderr_ != subprocess.PIPE:
1082	raise ValueError('stderr_ can be only PIPE or None')
1083	# because we want to capture it

Note: See TracBrowser for help on using the repository browser.

Download in other formats: