| 1 | #!/usr/bin/env python
|
|---|
| 2 |
|
|---|
| 3 | ############################################################################
|
|---|
| 4 | #
|
|---|
| 5 | # MODULE: g.extension
|
|---|
| 6 | # AUTHOR(S): Markus Neteler (original shell script)
|
|---|
| 7 | # Martin Landa <landa.martin gmail com> (Pythonized & upgraded for GRASS 7)
|
|---|
| 8 | # Vaclav Petras <wenzeslaus gmail com> (support for general sources)
|
|---|
| 9 | # PURPOSE: Tool to download and install extensions into local installation
|
|---|
| 10 | #
|
|---|
| 11 | # COPYRIGHT: (C) 2009-2019 by Markus Neteler, and the GRASS Development Team
|
|---|
| 12 | #
|
|---|
| 13 | # This program is free software under the GNU General
|
|---|
| 14 | # Public License (>=v2). Read the file COPYING that
|
|---|
| 15 | # comes with GRASS for details.
|
|---|
| 16 | #
|
|---|
| 17 | # TODO: - add sudo support where needed (i.e. check first permission to write into
|
|---|
| 18 | # $GISBASE directory)
|
|---|
| 19 | # - fix toolbox support in install_private_extension_xml()
|
|---|
| 20 | #############################################################################
|
|---|
| 21 |
|
|---|
| 22 | #%module
|
|---|
| 23 | #% label: Maintains GRASS Addons extensions in local GRASS installation.
|
|---|
| 24 | #% description: Downloads and installs extensions from GRASS Addons repository or other source into the local GRASS installation or removes installed extensions.
|
|---|
| 25 | #% keyword: general
|
|---|
| 26 | #% keyword: installation
|
|---|
| 27 | #% keyword: extensions
|
|---|
| 28 | #% keyword: addons
|
|---|
| 29 | #% keyword: download
|
|---|
| 30 | #%end
|
|---|
| 31 |
|
|---|
| 32 | #%option
|
|---|
| 33 | #% key: extension
|
|---|
| 34 | #% type: string
|
|---|
| 35 | #% key_desc: name
|
|---|
| 36 | #% label: Name of extension to install or remove
|
|---|
| 37 | #% description: Name of toolbox (set of extensions) when -t flag is given
|
|---|
| 38 | #% required: yes
|
|---|
| 39 | #%end
|
|---|
| 40 | #%option
|
|---|
| 41 | #% key: operation
|
|---|
| 42 | #% type: string
|
|---|
| 43 | #% description: Operation to be performed
|
|---|
| 44 | #% required: yes
|
|---|
| 45 | #% options: add,remove
|
|---|
| 46 | #% answer: add
|
|---|
| 47 | #%end
|
|---|
| 48 | #%option
|
|---|
| 49 | #% key: url
|
|---|
| 50 | #% type: string
|
|---|
| 51 | #% key_desc: url
|
|---|
| 52 | #% label: URL or directory to get the extension from (supported only on Linux and Mac)
|
|---|
| 53 | #% description: The official repository is used by default. User can specify a ZIP file, directory or a repository on common hosting services. If not identified, Subversion repository is assumed. See manual for all options.
|
|---|
| 54 | #%end
|
|---|
| 55 | #%option
|
|---|
| 56 | #% key: prefix
|
|---|
| 57 | #% type: string
|
|---|
| 58 | #% key_desc: path
|
|---|
| 59 | #% description: Prefix where to install extension (ignored when flag -s is given)
|
|---|
| 60 | #% answer: $GRASS_ADDON_BASE
|
|---|
| 61 | #% required: no
|
|---|
| 62 | #%end
|
|---|
| 63 | #%option
|
|---|
| 64 | #% key: proxy
|
|---|
| 65 | #% type: string
|
|---|
| 66 | #% key_desc: proxy
|
|---|
| 67 | #% description: Set the proxy with: "http=<value>,ftp=<value>"
|
|---|
| 68 | #% required: no
|
|---|
| 69 | #% multiple: yes
|
|---|
| 70 | #%end
|
|---|
| 71 |
|
|---|
| 72 | #%flag
|
|---|
| 73 | #% key: l
|
|---|
| 74 | #% description: List available extensions in the official GRASS GIS Addons repository
|
|---|
| 75 | #% guisection: Print
|
|---|
| 76 | #% suppress_required: yes
|
|---|
| 77 | #%end
|
|---|
| 78 | #%flag
|
|---|
| 79 | #% key: c
|
|---|
| 80 | #% description: List available extensions in the official GRASS GIS Addons repository including module description
|
|---|
| 81 | #% guisection: Print
|
|---|
| 82 | #% suppress_required: yes
|
|---|
| 83 | #%end
|
|---|
| 84 | #%flag
|
|---|
| 85 | #% key: g
|
|---|
| 86 | #% description: List available extensions in the official GRASS GIS Addons repository (shell script style)
|
|---|
| 87 | #% guisection: Print
|
|---|
| 88 | #% suppress_required: yes
|
|---|
| 89 | #%end
|
|---|
| 90 | #%flag
|
|---|
| 91 | #% key: a
|
|---|
| 92 | #% description: List locally installed extensions
|
|---|
| 93 | #% guisection: Print
|
|---|
| 94 | #% suppress_required: yes
|
|---|
| 95 | #%end
|
|---|
| 96 | #%flag
|
|---|
| 97 | #% key: s
|
|---|
| 98 | #% description: Install system-wide (may need system administrator rights)
|
|---|
| 99 | #% guisection: Install
|
|---|
| 100 | #%end
|
|---|
| 101 | #%flag
|
|---|
| 102 | #% key: d
|
|---|
| 103 | #% description: Download source code and exit
|
|---|
| 104 | #% guisection: Install
|
|---|
| 105 | #%end
|
|---|
| 106 | #%flag
|
|---|
| 107 | #% key: i
|
|---|
| 108 | #% description: Do not install new extension, just compile it
|
|---|
| 109 | #% guisection: Install
|
|---|
| 110 | #%end
|
|---|
| 111 | #%flag
|
|---|
| 112 | #% key: f
|
|---|
| 113 | #% description: Force removal when uninstalling extension (operation=remove)
|
|---|
| 114 | #% guisection: Remove
|
|---|
| 115 | #%end
|
|---|
| 116 | #%flag
|
|---|
| 117 | #% key: t
|
|---|
| 118 | #% description: Operate on toolboxes instead of single modules (experimental)
|
|---|
| 119 | #% suppress_required: yes
|
|---|
| 120 | #%end
|
|---|
| 121 |
|
|---|
| 122 | #%rules
|
|---|
| 123 | #% required: extension, -l, -c, -g, -a
|
|---|
| 124 | #% exclusive: extension, -l, -c, -g
|
|---|
| 125 | #% exclusive: extension, -l, -c, -a
|
|---|
| 126 | #%end
|
|---|
| 127 |
|
|---|
| 128 | # TODO: solve addon-extension(-module) confusion
|
|---|
| 129 |
|
|---|
| 130 |
|
|---|
| 131 | from __future__ import print_function
|
|---|
| 132 | import os
|
|---|
| 133 | import sys
|
|---|
| 134 | import re
|
|---|
| 135 | import atexit
|
|---|
| 136 | import shutil
|
|---|
| 137 | import zipfile
|
|---|
| 138 | import tempfile
|
|---|
| 139 | import xml.etree.ElementTree as etree
|
|---|
| 140 | from distutils.dir_util import copy_tree
|
|---|
| 141 |
|
|---|
| 142 | from six.moves.urllib.request import urlopen, urlretrieve, ProxyHandler, build_opener, install_opener
|
|---|
| 143 | from six.moves.urllib.error import HTTPError, URLError
|
|---|
| 144 |
|
|---|
| 145 | # Get the XML parsing exceptions to catch. The behavior changed with Python 2.7
|
|---|
| 146 | # and ElementTree 1.3.
|
|---|
| 147 | from xml.parsers import expat # TODO: works for any Python?
|
|---|
| 148 | if hasattr(etree, 'ParseError'):
|
|---|
| 149 | ETREE_EXCEPTIONS = (etree.ParseError, expat.ExpatError)
|
|---|
| 150 | else:
|
|---|
| 151 | ETREE_EXCEPTIONS = (expat.ExpatError)
|
|---|
| 152 |
|
|---|
| 153 | import grass.script as gscript
|
|---|
| 154 | from grass.script.utils import try_rmdir
|
|---|
| 155 | from grass.script import core as grass
|
|---|
| 156 | from grass.script import task as gtask
|
|---|
| 157 |
|
|---|
| 158 | # temp dir
|
|---|
| 159 | REMOVE_TMPDIR = True
|
|---|
| 160 | PROXIES = {}
|
|---|
| 161 |
|
|---|
| 162 |
|
|---|
| 163 | def etree_fromfile(filename):
|
|---|
| 164 | """Create XML element tree from a given file name"""
|
|---|
| 165 | with open(filename, 'r') as file_:
|
|---|
| 166 | return etree.fromstring(file_.read())
|
|---|
| 167 |
|
|---|
| 168 |
|
|---|
| 169 | def etree_fromurl(url):
|
|---|
| 170 | """Create XML element tree from a given URL"""
|
|---|
| 171 | file_ = urlopen(url)
|
|---|
| 172 | return etree.fromstring(file_.read())
|
|---|
| 173 |
|
|---|
| 174 |
|
|---|
| 175 | def check_progs():
|
|---|
| 176 | """Check if the necessary programs are available"""
|
|---|
| 177 | # TODO: we need svn for the Subversion repo downloads
|
|---|
| 178 | # also git would be tested once supported
|
|---|
| 179 | for prog in ('make', 'gcc'):
|
|---|
| 180 | if not grass.find_program(prog, '--help'):
|
|---|
| 181 | grass.fatal(_("'%s' required. Please install '%s' first.")
|
|---|
| 182 | % (prog, prog))
|
|---|
| 183 |
|
|---|
| 184 | # expand prefix to class name
|
|---|
| 185 |
|
|---|
| 186 |
|
|---|
| 187 | def expand_module_class_name(class_letters):
|
|---|
| 188 | """Convert module class (family) letter or letters to class (family) name
|
|---|
| 189 |
|
|---|
| 190 | The letter or letters are used in module names, e.g. r.slope.aspect.
|
|---|
| 191 | The names are used in directories in Addons but also in the source code.
|
|---|
| 192 |
|
|---|
| 193 | >>> expand_module_class_name('r')
|
|---|
| 194 | 'raster'
|
|---|
| 195 | >>> expand_module_class_name('v')
|
|---|
| 196 | 'vector'
|
|---|
| 197 | """
|
|---|
| 198 | name = {
|
|---|
| 199 | 'd': 'display',
|
|---|
| 200 | 'db': 'database',
|
|---|
| 201 | 'g': 'general',
|
|---|
| 202 | 'i': 'imagery',
|
|---|
| 203 | 'm': 'misc',
|
|---|
| 204 | 'ps': 'postscript',
|
|---|
| 205 | 'p': 'paint',
|
|---|
| 206 | 'r': 'raster',
|
|---|
| 207 | 'r3': 'raster3d',
|
|---|
| 208 | 's': 'sites',
|
|---|
| 209 | 't': 'temporal',
|
|---|
| 210 | 'v': 'vector',
|
|---|
| 211 | 'wx': 'gui/wxpython'
|
|---|
| 212 | }
|
|---|
| 213 |
|
|---|
| 214 | return name.get(class_letters, class_letters)
|
|---|
| 215 |
|
|---|
| 216 |
|
|---|
| 217 | def get_module_class_name(module_name):
|
|---|
| 218 | """Return class (family) name for a module
|
|---|
| 219 |
|
|---|
| 220 | The names are used in directories in Addons but also in the source code.
|
|---|
| 221 |
|
|---|
| 222 | >>> get_module_class_name('r.slope.aspect')
|
|---|
| 223 | 'raster'
|
|---|
| 224 | >>> get_module_class_name('v.to.rast')
|
|---|
| 225 | 'vector'
|
|---|
| 226 | """
|
|---|
| 227 | classchar = module_name.split('.', 1)[0]
|
|---|
| 228 | return expand_module_class_name(classchar)
|
|---|
| 229 |
|
|---|
| 230 |
|
|---|
| 231 | def get_installed_extensions(force=False):
|
|---|
| 232 | """Get list of installed extensions or toolboxes (if -t is set)"""
|
|---|
| 233 | if flags['t']:
|
|---|
| 234 | return get_installed_toolboxes(force)
|
|---|
| 235 |
|
|---|
| 236 | return get_installed_modules(force)
|
|---|
| 237 |
|
|---|
| 238 |
|
|---|
| 239 | def list_installed_extensions(toolboxes=False):
|
|---|
| 240 | """List installed extensions"""
|
|---|
| 241 | elist = get_installed_extensions()
|
|---|
| 242 | if elist:
|
|---|
| 243 | if toolboxes:
|
|---|
| 244 | grass.message(_("List of installed extensions (toolboxes):"))
|
|---|
| 245 | else:
|
|---|
| 246 | grass.message(_("List of installed extensions (modules):"))
|
|---|
| 247 | sys.stdout.write('\n'.join(elist))
|
|---|
| 248 | sys.stdout.write('\n')
|
|---|
| 249 | else:
|
|---|
| 250 | if toolboxes:
|
|---|
| 251 | grass.info(_("No extension (toolbox) installed"))
|
|---|
| 252 | else:
|
|---|
| 253 | grass.info(_("No extension (module) installed"))
|
|---|
| 254 |
|
|---|
| 255 |
|
|---|
| 256 | def get_installed_toolboxes(force=False):
|
|---|
| 257 | """Get list of installed toolboxes
|
|---|
| 258 |
|
|---|
| 259 | Writes toolboxes file if it does not exist.
|
|---|
| 260 | Creates a new toolboxes file if it is not possible
|
|---|
| 261 | to read the current one.
|
|---|
| 262 | """
|
|---|
| 263 | xml_file = os.path.join(options['prefix'], 'toolboxes.xml')
|
|---|
| 264 | if not os.path.exists(xml_file):
|
|---|
| 265 | write_xml_toolboxes(xml_file)
|
|---|
| 266 | # read XML file
|
|---|
| 267 | try:
|
|---|
| 268 | tree = etree_fromfile(xml_file)
|
|---|
| 269 | except ETREE_EXCEPTIONS + (OSError, IOError):
|
|---|
| 270 | os.remove(xml_file)
|
|---|
| 271 | write_xml_toolboxes(xml_file)
|
|---|
| 272 | return []
|
|---|
| 273 | ret = list()
|
|---|
| 274 | for tnode in tree.findall('toolbox'):
|
|---|
| 275 | ret.append(tnode.get('code'))
|
|---|
| 276 | return ret
|
|---|
| 277 |
|
|---|
| 278 |
|
|---|
| 279 | def get_installed_modules(force=False):
|
|---|
| 280 | """Get list of installed modules.
|
|---|
| 281 |
|
|---|
| 282 | Writes modules file if it does not exist and *force* is set to ``True``.
|
|---|
| 283 | Creates a new modules file if it is not possible
|
|---|
| 284 | to read the current one.
|
|---|
| 285 | """
|
|---|
| 286 | xml_file = os.path.join(options['prefix'], 'modules.xml')
|
|---|
| 287 | if not os.path.exists(xml_file):
|
|---|
| 288 | if force:
|
|---|
| 289 | write_xml_modules(xml_file)
|
|---|
| 290 | else:
|
|---|
| 291 | grass.debug("No addons metadata file available", 1)
|
|---|
| 292 | return []
|
|---|
| 293 | # read XML file
|
|---|
| 294 | try:
|
|---|
| 295 | tree = etree_fromfile(xml_file)
|
|---|
| 296 | except ETREE_EXCEPTIONS + (OSError, IOError):
|
|---|
| 297 | os.remove(xml_file)
|
|---|
| 298 | write_xml_modules(xml_file)
|
|---|
| 299 | return []
|
|---|
| 300 | ret = list()
|
|---|
| 301 | for tnode in tree.findall('task'):
|
|---|
| 302 | if flags['g']:
|
|---|
| 303 | desc, keyw = get_optional_params(tnode)
|
|---|
| 304 | ret.append('name={0}'.format(tnode.get('name').strip()))
|
|---|
| 305 | ret.append('description={0}'.format(desc))
|
|---|
| 306 | ret.append('keywords={0}'.format(keyw))
|
|---|
| 307 | ret.append('executables={0}'.format(','.join(
|
|---|
| 308 | get_module_executables(tnode))
|
|---|
| 309 | ))
|
|---|
| 310 | else:
|
|---|
| 311 | ret.append(tnode.get('name').strip())
|
|---|
| 312 |
|
|---|
| 313 | return ret
|
|---|
| 314 |
|
|---|
| 315 | # list extensions (read XML file from grass.osgeo.org/addons)
|
|---|
| 316 |
|
|---|
| 317 |
|
|---|
| 318 | def list_available_extensions(url):
|
|---|
| 319 | """List available extensions/modules or toolboxes (if -t is given)
|
|---|
| 320 |
|
|---|
| 321 | For toolboxes it lists also all modules.
|
|---|
| 322 | """
|
|---|
| 323 | gscript.debug("list_available_extensions(url={0})".format(url))
|
|---|
| 324 | if flags['t']:
|
|---|
| 325 | grass.message(_("List of available extensions (toolboxes):"))
|
|---|
| 326 | tlist = get_available_toolboxes(url)
|
|---|
| 327 | for toolbox_code, toolbox_data in tlist.items():
|
|---|
| 328 | if flags['g']:
|
|---|
| 329 | print('toolbox_name=' + toolbox_data['name'])
|
|---|
| 330 | print('toolbox_code=' + toolbox_code)
|
|---|
| 331 | else:
|
|---|
| 332 | print('%s (%s)' % (toolbox_data['name'], toolbox_code))
|
|---|
| 333 | if flags['c'] or flags['g']:
|
|---|
| 334 | list_available_modules(url, toolbox_data['modules'])
|
|---|
| 335 | else:
|
|---|
| 336 | if toolbox_data['modules']:
|
|---|
| 337 | print(os.linesep.join(['* ' + x for x in toolbox_data['modules']]))
|
|---|
| 338 | else:
|
|---|
| 339 | grass.message(_("List of available extensions (modules):"))
|
|---|
| 340 | list_available_modules(url)
|
|---|
| 341 |
|
|---|
| 342 |
|
|---|
| 343 | def get_available_toolboxes(url):
|
|---|
| 344 | """Return toolboxes available in the repository"""
|
|---|
| 345 | tdict = dict()
|
|---|
| 346 | url = url + "toolboxes.xml"
|
|---|
| 347 | try:
|
|---|
| 348 | tree = etree_fromurl(url)
|
|---|
| 349 | for tnode in tree.findall('toolbox'):
|
|---|
| 350 | mlist = list()
|
|---|
| 351 | clist = list()
|
|---|
| 352 | tdict[tnode.get('code')] = {'name': tnode.get('name'),
|
|---|
| 353 | 'correlate': clist,
|
|---|
| 354 | 'modules': mlist}
|
|---|
| 355 |
|
|---|
| 356 | for cnode in tnode.findall('correlate'):
|
|---|
| 357 | clist.append(cnode.get('name'))
|
|---|
| 358 |
|
|---|
| 359 | for mnode in tnode.findall('task'):
|
|---|
| 360 | mlist.append(mnode.get('name'))
|
|---|
| 361 | except (HTTPError, IOError, OSError):
|
|---|
| 362 | grass.fatal(_("Unable to fetch addons metadata file"))
|
|---|
| 363 |
|
|---|
| 364 | return tdict
|
|---|
| 365 |
|
|---|
| 366 |
|
|---|
| 367 | def get_toolbox_modules(url, name):
|
|---|
| 368 | """Get modules inside a toolbox in toolbox file at given URL
|
|---|
| 369 |
|
|---|
| 370 | :param url: URL of the directory (file name will be attached)
|
|---|
| 371 | :param name: toolbox name
|
|---|
| 372 | """
|
|---|
| 373 | tlist = list()
|
|---|
| 374 |
|
|---|
| 375 | url = url + "toolboxes.xml"
|
|---|
| 376 |
|
|---|
| 377 | try:
|
|---|
| 378 | tree = etree_fromurl(url)
|
|---|
| 379 | for tnode in tree.findall('toolbox'):
|
|---|
| 380 | if name == tnode.get('code'):
|
|---|
| 381 | for mnode in tnode.findall('task'):
|
|---|
| 382 | tlist.append(mnode.get('name'))
|
|---|
| 383 | break
|
|---|
| 384 | except (HTTPError, IOError, OSError):
|
|---|
| 385 | grass.fatal(_("Unable to fetch addons metadata file"))
|
|---|
| 386 |
|
|---|
| 387 | return tlist
|
|---|
| 388 |
|
|---|
| 389 |
|
|---|
| 390 | def get_module_files(mnode):
|
|---|
| 391 | """Return list of module files
|
|---|
| 392 |
|
|---|
| 393 | :param mnode: XML node for a module
|
|---|
| 394 | """
|
|---|
| 395 | flist = []
|
|---|
| 396 | for file_node in mnode.find('binary').findall('file'):
|
|---|
| 397 | filepath = file_node.text
|
|---|
| 398 | flist.append(filepath)
|
|---|
| 399 |
|
|---|
| 400 | return flist
|
|---|
| 401 |
|
|---|
| 402 |
|
|---|
| 403 | def get_module_executables(mnode):
|
|---|
| 404 | """Return list of module executables
|
|---|
| 405 |
|
|---|
| 406 | :param mnode: XML node for a module
|
|---|
| 407 | """
|
|---|
| 408 | flist = []
|
|---|
| 409 | for filepath in get_module_files(mnode):
|
|---|
| 410 | if filepath.startswith(options['prefix'] + os.path.sep + 'bin') or \
|
|---|
| 411 | (sys.platform != 'win32' and
|
|---|
| 412 | filepath.startswith(options['prefix'] + os.path.sep + 'scripts')):
|
|---|
| 413 | filename = os.path.basename(filepath)
|
|---|
| 414 | if sys.platform == 'win32':
|
|---|
| 415 | filename = os.path.splitext(filename)[0]
|
|---|
| 416 | flist.append(filename)
|
|---|
| 417 |
|
|---|
| 418 | return flist
|
|---|
| 419 |
|
|---|
| 420 |
|
|---|
| 421 | def get_optional_params(mnode):
|
|---|
| 422 | """Return description and keywords as a tuple
|
|---|
| 423 |
|
|---|
| 424 | :param mnode: XML node for a module
|
|---|
| 425 | """
|
|---|
| 426 | try:
|
|---|
| 427 | desc = mnode.find('description').text
|
|---|
| 428 | except AttributeError:
|
|---|
| 429 | desc = ''
|
|---|
| 430 | if desc is None:
|
|---|
| 431 | desc = ''
|
|---|
| 432 | try:
|
|---|
| 433 | keyw = mnode.find('keywords').text
|
|---|
| 434 | except AttributeError:
|
|---|
| 435 | keyw = ''
|
|---|
| 436 | if keyw is None:
|
|---|
| 437 | keyw = ''
|
|---|
| 438 |
|
|---|
| 439 | return desc, keyw
|
|---|
| 440 |
|
|---|
| 441 |
|
|---|
| 442 | def list_available_modules(url, mlist=None):
|
|---|
| 443 | """List modules available in the repository
|
|---|
| 444 |
|
|---|
| 445 | Tries to use XML metadata file first. Fallbacks to HTML page with a list.
|
|---|
| 446 |
|
|---|
| 447 | :param url: URL of the directory (file name will be attached)
|
|---|
| 448 | :param mlist: list only modules in this list
|
|---|
| 449 | """
|
|---|
| 450 | file_url = url + "modules.xml"
|
|---|
| 451 | grass.debug("url=%s" % file_url, 1)
|
|---|
| 452 | try:
|
|---|
| 453 | tree = etree_fromurl(file_url)
|
|---|
| 454 | except ETREE_EXCEPTIONS:
|
|---|
| 455 | grass.warning(_("Unable to parse '%s'. Trying to scan"
|
|---|
| 456 | " SVN repository (may take some time)...") % file_url)
|
|---|
| 457 | list_available_extensions_svn(url)
|
|---|
| 458 | return
|
|---|
| 459 | except (HTTPError, URLError, IOError, OSError):
|
|---|
| 460 | list_available_extensions_svn(url)
|
|---|
| 461 | return
|
|---|
| 462 |
|
|---|
| 463 | for mnode in tree.findall('task'):
|
|---|
| 464 | name = mnode.get('name').strip()
|
|---|
| 465 | if mlist and name not in mlist:
|
|---|
| 466 | continue
|
|---|
| 467 | if flags['c'] or flags['g']:
|
|---|
| 468 | desc, keyw = get_optional_params(mnode)
|
|---|
| 469 |
|
|---|
| 470 | if flags['g']:
|
|---|
| 471 | print('name=' + name)
|
|---|
| 472 | print('description=' + desc)
|
|---|
| 473 | print('keywords=' + keyw)
|
|---|
| 474 | elif flags['c']:
|
|---|
| 475 | if mlist:
|
|---|
| 476 | print('*', end='')
|
|---|
| 477 | print(name + ' - ' + desc)
|
|---|
| 478 | else:
|
|---|
| 479 | print(name)
|
|---|
| 480 |
|
|---|
| 481 |
|
|---|
| 482 | # TODO: this is now broken/dead code, SVN is basically not used
|
|---|
| 483 | # fallback for Trac should parse Trac HTML page
|
|---|
| 484 | # this might be useful for potential SVN repos or anything
|
|---|
| 485 | # which would list the extensions/addons as list
|
|---|
| 486 | # TODO: fail when nothing is accessible
|
|---|
| 487 | def list_available_extensions_svn(url):
|
|---|
| 488 | """List available extensions from HTML given by URL
|
|---|
| 489 |
|
|---|
| 490 | Filename is generated based on the module class/family.
|
|---|
| 491 | This works well for the structure which is in grass-addons repository.
|
|---|
| 492 |
|
|---|
| 493 | ``<li><a href=...`` is parsed to find module names.
|
|---|
| 494 | This works well for HTML page generated by Subversion.
|
|---|
| 495 |
|
|---|
| 496 | :param url: a directory URL (filename will be attached)
|
|---|
| 497 | """
|
|---|
| 498 | gscript.debug("list_available_extensions_svn(url=%s)" % url, 2)
|
|---|
| 499 | grass.message(_('Fetching list of extensions from'
|
|---|
| 500 | ' GRASS-Addons SVN repository (be patient)...'))
|
|---|
| 501 | pattern = re.compile(r'(<li><a href=".+">)(.+)(</a></li>)', re.IGNORECASE)
|
|---|
| 502 |
|
|---|
| 503 | if flags['c']:
|
|---|
| 504 | grass.warning(
|
|---|
| 505 | _("Flag 'c' ignored, addons metadata file not available"))
|
|---|
| 506 | if flags['g']:
|
|---|
| 507 | grass.warning(
|
|---|
| 508 | _("Flag 'g' ignored, addons metadata file not available"))
|
|---|
| 509 |
|
|---|
| 510 | prefixes = ['d', 'db', 'g', 'i', 'm', 'ps',
|
|---|
| 511 | 'p', 'r', 'r3', 's', 't', 'v']
|
|---|
| 512 | for prefix in prefixes:
|
|---|
| 513 | modclass = expand_module_class_name(prefix)
|
|---|
| 514 | grass.verbose(_("Checking for '%s' modules...") % modclass)
|
|---|
| 515 |
|
|---|
| 516 | # construct a full URL of a file
|
|---|
| 517 | file_url = '%s/%s' % (url, modclass)
|
|---|
| 518 | grass.debug("url = %s" % file_url, debug=2)
|
|---|
| 519 | try:
|
|---|
| 520 | file_ = urlopen(url)
|
|---|
| 521 | except (HTTPError, IOError, OSError):
|
|---|
| 522 | grass.debug(_("Unable to fetch '%s'") % file_url, debug=1)
|
|---|
| 523 | continue
|
|---|
| 524 |
|
|---|
| 525 | for line in file_.readlines():
|
|---|
| 526 | # list extensions
|
|---|
| 527 | sline = pattern.search(line)
|
|---|
| 528 | if not sline:
|
|---|
| 529 | continue
|
|---|
| 530 | name = sline.group(2).rstrip('/')
|
|---|
| 531 | if name.split('.', 1)[0] == prefix:
|
|---|
| 532 | print(name)
|
|---|
| 533 |
|
|---|
| 534 | # get_wxgui_extensions(url)
|
|---|
| 535 |
|
|---|
| 536 |
|
|---|
| 537 | # TODO: this is a dead code, not clear why not used, but seems not needed
|
|---|
| 538 | def get_wxgui_extensions(url):
|
|---|
| 539 | """Return list of extensions/addons in wxGUI directory at given URL
|
|---|
| 540 |
|
|---|
| 541 | :param url: a directory URL (filename will be attached)
|
|---|
| 542 | """
|
|---|
| 543 | mlist = list()
|
|---|
| 544 | grass.debug('Fetching list of wxGUI extensions from '
|
|---|
| 545 | 'GRASS-Addons SVN repository (be patient)...')
|
|---|
| 546 | pattern = re.compile(r'(<li><a href=".+">)(.+)(</a></li>)', re.IGNORECASE)
|
|---|
| 547 | grass.verbose(_("Checking for '%s' modules...") % 'gui/wxpython')
|
|---|
| 548 |
|
|---|
| 549 | # construct a full URL of a file
|
|---|
| 550 | url = '%s/%s' % (url, 'gui/wxpython')
|
|---|
| 551 | grass.debug("url = %s" % url, debug=2)
|
|---|
| 552 | file_ = urlopen(url)
|
|---|
| 553 | if not file_:
|
|---|
| 554 | grass.warning(_("Unable to fetch '%s'") % url)
|
|---|
| 555 | return
|
|---|
| 556 |
|
|---|
| 557 | for line in file.readlines():
|
|---|
| 558 | # list extensions
|
|---|
| 559 | sline = pattern.search(line)
|
|---|
| 560 | if not sline:
|
|---|
| 561 | continue
|
|---|
| 562 | name = sline.group(2).rstrip('/')
|
|---|
| 563 | if name not in ('..', 'Makefile'):
|
|---|
| 564 | mlist.append(name)
|
|---|
| 565 |
|
|---|
| 566 | return mlist
|
|---|
| 567 |
|
|---|
| 568 |
|
|---|
| 569 | def cleanup():
|
|---|
| 570 | """Cleanup after the downloads and copilation"""
|
|---|
| 571 | if REMOVE_TMPDIR:
|
|---|
| 572 | try_rmdir(TMPDIR)
|
|---|
| 573 | else:
|
|---|
| 574 | grass.message("\n%s\n" % _("Path to the source code:"))
|
|---|
| 575 | sys.stderr.write('%s\n' % os.path.join(TMPDIR, options['extension']))
|
|---|
| 576 |
|
|---|
| 577 |
|
|---|
| 578 | def write_xml_modules(name, tree=None):
|
|---|
| 579 | """Write element tree as a modules matadata file
|
|---|
| 580 |
|
|---|
| 581 | If the *tree* is not given, an empty file is created.
|
|---|
| 582 |
|
|---|
| 583 | :param name: file name
|
|---|
| 584 | :param tree: XML element tree
|
|---|
| 585 | """
|
|---|
| 586 | file_ = open(name, 'w')
|
|---|
| 587 | file_.write('<?xml version="1.0" encoding="UTF-8"?>\n')
|
|---|
| 588 | file_.write('<!DOCTYPE task SYSTEM "grass-addons.dtd">\n')
|
|---|
| 589 | file_.write('<addons version="%s">\n' % version[0])
|
|---|
| 590 |
|
|---|
| 591 | libgis_revison = grass.version()['libgis_revision']
|
|---|
| 592 | if tree is not None:
|
|---|
| 593 | for tnode in tree.findall('task'):
|
|---|
| 594 | indent = 4
|
|---|
| 595 | file_.write('%s<task name="%s">\n' %
|
|---|
| 596 | (' ' * indent, tnode.get('name')))
|
|---|
| 597 | indent += 4
|
|---|
| 598 | file_.write('%s<description>%s</description>\n' %
|
|---|
| 599 | (' ' * indent, tnode.find('description').text))
|
|---|
| 600 | file_.write('%s<keywords>%s</keywords>\n' %
|
|---|
| 601 | (' ' * indent, tnode.find('keywords').text))
|
|---|
| 602 | bnode = tnode.find('binary')
|
|---|
| 603 | if bnode is not None:
|
|---|
| 604 | file_.write('%s<binary>\n' % (' ' * indent))
|
|---|
| 605 | indent += 4
|
|---|
| 606 | for fnode in bnode.findall('file'):
|
|---|
| 607 | file_.write('%s<file>%s</file>\n' %
|
|---|
| 608 | (' ' * indent, os.path.join(options['prefix'],
|
|---|
| 609 | fnode.text)))
|
|---|
| 610 | indent -= 4
|
|---|
| 611 | file_.write('%s</binary>\n' % (' ' * indent))
|
|---|
| 612 | file_.write('%s<libgis revision="%s" />\n' %
|
|---|
| 613 | (' ' * indent, libgis_revison))
|
|---|
| 614 | indent -= 4
|
|---|
| 615 | file_.write('%s</task>\n' % (' ' * indent))
|
|---|
| 616 |
|
|---|
| 617 | file_.write('</addons>\n')
|
|---|
| 618 | file_.close()
|
|---|
| 619 |
|
|---|
| 620 |
|
|---|
| 621 | def write_xml_toolboxes(name, tree=None):
|
|---|
| 622 | """Write element tree as a toolboxes matadata file
|
|---|
| 623 |
|
|---|
| 624 | If the *tree* is not given, an empty file is created.
|
|---|
| 625 |
|
|---|
| 626 | :param name: file name
|
|---|
| 627 | :param tree: XML element tree
|
|---|
| 628 | """
|
|---|
| 629 | file_ = open(name, 'w')
|
|---|
| 630 | file_.write('<?xml version="1.0" encoding="UTF-8"?>\n')
|
|---|
| 631 | file_.write('<!DOCTYPE toolbox SYSTEM "grass-addons.dtd">\n')
|
|---|
| 632 | file_.write('<addons version="%s">\n' % version[0])
|
|---|
| 633 | if tree is not None:
|
|---|
| 634 | for tnode in tree.findall('toolbox'):
|
|---|
| 635 | indent = 4
|
|---|
| 636 | file_.write('%s<toolbox name="%s" code="%s">\n' %
|
|---|
| 637 | (' ' * indent, tnode.get('name'), tnode.get('code')))
|
|---|
| 638 | indent += 4
|
|---|
| 639 | for cnode in tnode.findall('correlate'):
|
|---|
| 640 | file_.write('%s<correlate code="%s" />\n' %
|
|---|
| 641 | (' ' * indent, tnode.get('code')))
|
|---|
| 642 | for mnode in tnode.findall('task'):
|
|---|
| 643 | file_.write('%s<task name="%s" />\n' %
|
|---|
| 644 | (' ' * indent, mnode.get('name')))
|
|---|
| 645 | indent -= 4
|
|---|
| 646 | file_.write('%s</toolbox>\n' % (' ' * indent))
|
|---|
| 647 |
|
|---|
| 648 | file_.write('</addons>\n')
|
|---|
| 649 | file_.close()
|
|---|
| 650 |
|
|---|
| 651 |
|
|---|
| 652 | def install_extension(source, url, xmlurl):
|
|---|
| 653 | """Install extension (e.g. one module) or a toolbox (list of modules)"""
|
|---|
| 654 | gisbase = os.getenv('GISBASE')
|
|---|
| 655 | if not gisbase:
|
|---|
| 656 | grass.fatal(_('$GISBASE not defined'))
|
|---|
| 657 |
|
|---|
| 658 | if options['extension'] in get_installed_extensions(force=True):
|
|---|
| 659 | grass.warning(_("Extension <%s> already installed. Re-installing...") %
|
|---|
| 660 | options['extension'])
|
|---|
| 661 |
|
|---|
| 662 | if flags['t']:
|
|---|
| 663 | grass.message(_("Installing toolbox <%s>...") % options['extension'])
|
|---|
| 664 | mlist = get_toolbox_modules(xmlurl, options['extension'])
|
|---|
| 665 | else:
|
|---|
| 666 | mlist = [options['extension']]
|
|---|
| 667 | if not mlist:
|
|---|
| 668 | grass.warning(_("Nothing to install"))
|
|---|
| 669 | return
|
|---|
| 670 |
|
|---|
| 671 | ret = 0
|
|---|
| 672 | for module in mlist:
|
|---|
| 673 | if sys.platform == "win32":
|
|---|
| 674 | ret += install_extension_win(module)
|
|---|
| 675 | else:
|
|---|
| 676 | ret1, installed_modules, tmp_dir = install_extension_std_platforms(module,
|
|---|
| 677 | source=source, url=url)
|
|---|
| 678 | ret += ret1
|
|---|
| 679 | if len(mlist) > 1:
|
|---|
| 680 | print('-' * 60)
|
|---|
| 681 |
|
|---|
| 682 | if flags['d']:
|
|---|
| 683 | return
|
|---|
| 684 |
|
|---|
| 685 | if ret != 0:
|
|---|
| 686 | grass.warning(_('Installation failed, sorry.'
|
|---|
| 687 | ' Please check above error messages.'))
|
|---|
| 688 | else:
|
|---|
| 689 | # for now it is reasonable to assume that only official source
|
|---|
| 690 | # will provide the metadata file
|
|---|
| 691 | if source == 'official' and len(installed_modules) <= len(mlist):
|
|---|
| 692 | grass.message(_("Updating addons metadata file..."))
|
|---|
| 693 | blist = install_extension_xml(xmlurl, mlist)
|
|---|
| 694 | if source == 'official' and len(installed_modules) > len(mlist):
|
|---|
| 695 | grass.message(_("Updating addons metadata file..."))
|
|---|
| 696 | blist = install_private_extension_xml(tmp_dir, installed_modules)
|
|---|
| 697 | else:
|
|---|
| 698 | grass.message(_("Updating private addons metadata file..."))
|
|---|
| 699 | if len(installed_modules) > 1:
|
|---|
| 700 | blist = install_private_extension_xml(tmp_dir, installed_modules)
|
|---|
| 701 | else:
|
|---|
| 702 | blist = install_private_extension_xml(tmp_dir, mlist)
|
|---|
| 703 |
|
|---|
| 704 | # the blist was used here, but it seems that it is the same as mlist
|
|---|
| 705 | for module in mlist:
|
|---|
| 706 | update_manual_page(module)
|
|---|
| 707 |
|
|---|
| 708 | grass.message(_("Installation of <%s> successfully finished") %
|
|---|
| 709 | options['extension'])
|
|---|
| 710 |
|
|---|
| 711 | if not os.getenv('GRASS_ADDON_BASE'):
|
|---|
| 712 | grass.warning(_('This add-on module will not function until'
|
|---|
| 713 | ' you set the GRASS_ADDON_BASE environment'
|
|---|
| 714 | ' variable (see "g.manual variables")'))
|
|---|
| 715 |
|
|---|
| 716 |
|
|---|
| 717 | def get_toolboxes_metadata(url):
|
|---|
| 718 | """Return metadata for all toolboxes from given URL
|
|---|
| 719 |
|
|---|
| 720 | :param url: URL of a modules matadata file
|
|---|
| 721 | :param mlist: list of modules to get metadata for
|
|---|
| 722 | :returns: tuple where first item is dictionary with module names as keys
|
|---|
| 723 | and dictionary with dest, keyw, files keys as value, the second item
|
|---|
| 724 | is list of 'binary' files (installation files)
|
|---|
| 725 | """
|
|---|
| 726 | data = dict()
|
|---|
| 727 | try:
|
|---|
| 728 | tree = etree_fromurl(url)
|
|---|
| 729 | for tnode in tree.findall('toolbox'):
|
|---|
| 730 | clist = list()
|
|---|
| 731 | for cnode in tnode.findall('correlate'):
|
|---|
| 732 | clist.append(cnode.get('code'))
|
|---|
| 733 |
|
|---|
| 734 | mlist = list()
|
|---|
| 735 | for mnode in tnode.findall('task'):
|
|---|
| 736 | mlist.append(mnode.get('name'))
|
|---|
| 737 |
|
|---|
| 738 | code = tnode.get('code')
|
|---|
| 739 | data[code] = {
|
|---|
| 740 | 'name': tnode.get('name'),
|
|---|
| 741 | 'correlate': clist,
|
|---|
| 742 | 'modules': mlist,
|
|---|
| 743 | }
|
|---|
| 744 | except (HTTPError, IOError, OSError):
|
|---|
| 745 | grass.error(_("Unable to read addons metadata file "
|
|---|
| 746 | "from the remote server"))
|
|---|
| 747 | return data
|
|---|
| 748 |
|
|---|
| 749 |
|
|---|
| 750 | def install_toolbox_xml(url, name):
|
|---|
| 751 | """Update local toolboxes metadata file"""
|
|---|
| 752 | # read metadata from remote server (toolboxes)
|
|---|
| 753 | url = url + "toolboxes.xml"
|
|---|
| 754 | data = get_toolboxes_metadata(url)
|
|---|
| 755 | if not data:
|
|---|
| 756 | grass.warning(_("No addons metadata available"))
|
|---|
| 757 | return
|
|---|
| 758 | if name not in data:
|
|---|
| 759 | grass.warning(_("No addons metadata available for <%s>") % name)
|
|---|
| 760 | return
|
|---|
| 761 |
|
|---|
| 762 | xml_file = os.path.join(options['prefix'], 'toolboxes.xml')
|
|---|
| 763 | # create an empty file if not exists
|
|---|
| 764 | if not os.path.exists(xml_file):
|
|---|
| 765 | write_xml_modules(xml_file)
|
|---|
| 766 |
|
|---|
| 767 | # read XML file
|
|---|
| 768 | with open(xml_file, 'r') as xml:
|
|---|
| 769 | tree = etree.fromstring(xml.read())
|
|---|
| 770 |
|
|---|
| 771 | # update tree
|
|---|
| 772 | tnode = None
|
|---|
| 773 | for node in tree.findall('toolbox'):
|
|---|
| 774 | if node.get('code') == name:
|
|---|
| 775 | tnode = node
|
|---|
| 776 | break
|
|---|
| 777 |
|
|---|
| 778 | tdata = data[name]
|
|---|
| 779 | if tnode is not None:
|
|---|
| 780 | # update existing node
|
|---|
| 781 | for cnode in tnode.findall('correlate'):
|
|---|
| 782 | tnode.remove(cnode)
|
|---|
| 783 | for mnode in tnode.findall('task'):
|
|---|
| 784 | tnode.remove(mnode)
|
|---|
| 785 | else:
|
|---|
| 786 | # create new node for task
|
|---|
| 787 | tnode = etree.Element(
|
|---|
| 788 | 'toolbox', attrib={'name': tdata['name'], 'code': name})
|
|---|
| 789 | tree.append(tnode)
|
|---|
| 790 |
|
|---|
| 791 | for cname in tdata['correlate']:
|
|---|
| 792 | cnode = etree.Element('correlate', attrib={'code': cname})
|
|---|
| 793 | tnode.append(cnode)
|
|---|
| 794 | for tname in tdata['modules']:
|
|---|
| 795 | mnode = etree.Element('task', attrib={'name': tname})
|
|---|
| 796 | tnode.append(mnode)
|
|---|
| 797 |
|
|---|
| 798 | write_xml_toolboxes(xml_file, tree)
|
|---|
| 799 |
|
|---|
| 800 |
|
|---|
| 801 | def get_addons_metadata(url, mlist):
|
|---|
| 802 | """Return metadata for list of modules from given URL
|
|---|
| 803 |
|
|---|
| 804 | :param url: URL of a modules matadata file
|
|---|
| 805 | :param mlist: list of modules to get metadata for
|
|---|
| 806 | :returns: tuple where first item is dictionary with module names as keys
|
|---|
| 807 | and dictionary with dest, keyw, files keys as value, the second item
|
|---|
| 808 | is list of 'binary' files (installation files)
|
|---|
| 809 | """
|
|---|
| 810 | data = {}
|
|---|
| 811 | bin_list = []
|
|---|
| 812 | try:
|
|---|
| 813 | tree = etree_fromurl(url)
|
|---|
| 814 | except (HTTPError, URLError, IOError, OSError) as error:
|
|---|
| 815 | grass.error(_("Unable to read addons metadata file"
|
|---|
| 816 | " from the remote server: {0}").format(error))
|
|---|
| 817 | return data, bin_list
|
|---|
| 818 | except ETREE_EXCEPTIONS as error:
|
|---|
| 819 | grass.warning(_("Unable to parse '%s': {0}").format(error) % url)
|
|---|
| 820 | return data, bin_list
|
|---|
| 821 | for mnode in tree.findall('task'):
|
|---|
| 822 | name = mnode.get('name')
|
|---|
| 823 | if name not in mlist:
|
|---|
| 824 | continue
|
|---|
| 825 | file_list = list()
|
|---|
| 826 | bnode = mnode.find('binary')
|
|---|
| 827 | windows = sys.platform == 'win32'
|
|---|
| 828 | if bnode is not None:
|
|---|
| 829 | for fnode in bnode.findall('file'):
|
|---|
| 830 | path = fnode.text.split('/')
|
|---|
| 831 | if path[0] == 'bin':
|
|---|
| 832 | bin_list.append(path[-1])
|
|---|
| 833 | if windows:
|
|---|
| 834 | path[-1] += '.exe'
|
|---|
| 835 | elif path[0] == 'scripts':
|
|---|
| 836 | bin_list.append(path[-1])
|
|---|
| 837 | if windows:
|
|---|
| 838 | path[-1] += '.py'
|
|---|
| 839 | file_list.append(os.path.sep.join(path))
|
|---|
| 840 | desc, keyw = get_optional_params(mnode)
|
|---|
| 841 | data[name] = {
|
|---|
| 842 | 'desc': desc,
|
|---|
| 843 | 'keyw': keyw,
|
|---|
| 844 | 'files': file_list,
|
|---|
| 845 | }
|
|---|
| 846 | return data, bin_list
|
|---|
| 847 |
|
|---|
| 848 |
|
|---|
| 849 | def install_extension_xml(url, mlist):
|
|---|
| 850 | """Update XML files with metadata about installed modules and toolbox
|
|---|
| 851 |
|
|---|
| 852 | Uses the remote/repository XML files for modules to obtain the metadata.
|
|---|
| 853 |
|
|---|
| 854 | :returns: list of executables (usable for ``update_manual_page()``)
|
|---|
| 855 | """
|
|---|
| 856 | if len(mlist) > 1:
|
|---|
| 857 | # read metadata from remote server (toolboxes)
|
|---|
| 858 | install_toolbox_xml(url, options['extension'])
|
|---|
| 859 |
|
|---|
| 860 | # read metadata from remote server (modules)
|
|---|
| 861 | url = url + "modules.xml"
|
|---|
| 862 | data, bin_list = get_addons_metadata(url, mlist)
|
|---|
| 863 | if not data:
|
|---|
| 864 | grass.warning(_("No addons metadata available."
|
|---|
| 865 | " Addons metadata file not updated."))
|
|---|
| 866 | return []
|
|---|
| 867 |
|
|---|
| 868 | xml_file = os.path.join(options['prefix'], 'modules.xml')
|
|---|
| 869 | # create an empty file if not exists
|
|---|
| 870 | if not os.path.exists(xml_file):
|
|---|
| 871 | write_xml_modules(xml_file)
|
|---|
| 872 |
|
|---|
| 873 | # read XML file
|
|---|
| 874 | tree = etree_fromfile(xml_file)
|
|---|
| 875 |
|
|---|
| 876 | # update tree
|
|---|
| 877 | for name in mlist:
|
|---|
| 878 | tnode = None
|
|---|
| 879 | for node in tree.findall('task'):
|
|---|
| 880 | if node.get('name') == name:
|
|---|
| 881 | tnode = node
|
|---|
| 882 | break
|
|---|
| 883 |
|
|---|
| 884 | if name not in data:
|
|---|
| 885 | grass.warning(_("No addons metadata found for <%s>") % name)
|
|---|
| 886 | continue
|
|---|
| 887 |
|
|---|
| 888 | ndata = data[name]
|
|---|
| 889 | if tnode is not None:
|
|---|
| 890 | # update existing node
|
|---|
| 891 | dnode = tnode.find('description')
|
|---|
| 892 | if dnode is not None:
|
|---|
| 893 | dnode.text = ndata['desc']
|
|---|
| 894 | knode = tnode.find('keywords')
|
|---|
| 895 | if knode is not None:
|
|---|
| 896 | knode.text = ndata['keyw']
|
|---|
| 897 | bnode = tnode.find('binary')
|
|---|
| 898 | if bnode is not None:
|
|---|
| 899 | tnode.remove(bnode)
|
|---|
| 900 | bnode = etree.Element('binary')
|
|---|
| 901 | for file_name in ndata['files']:
|
|---|
| 902 | fnode = etree.Element('file')
|
|---|
| 903 | fnode.text = file_name
|
|---|
| 904 | bnode.append(fnode)
|
|---|
| 905 | tnode.append(bnode)
|
|---|
| 906 | else:
|
|---|
| 907 | # create new node for task
|
|---|
| 908 | tnode = etree.Element('task', attrib={'name': name})
|
|---|
| 909 | dnode = etree.Element('description')
|
|---|
| 910 | dnode.text = ndata['desc']
|
|---|
| 911 | tnode.append(dnode)
|
|---|
| 912 | knode = etree.Element('keywords')
|
|---|
| 913 | knode.text = ndata['keyw']
|
|---|
| 914 | tnode.append(knode)
|
|---|
| 915 | bnode = etree.Element('binary')
|
|---|
| 916 | for file_name in ndata['files']:
|
|---|
| 917 | fnode = etree.Element('file')
|
|---|
| 918 | fnode.text = file_name
|
|---|
| 919 | bnode.append(fnode)
|
|---|
| 920 | tnode.append(bnode)
|
|---|
| 921 | tree.append(tnode)
|
|---|
| 922 |
|
|---|
| 923 | write_xml_modules(xml_file, tree)
|
|---|
| 924 |
|
|---|
| 925 | return bin_list
|
|---|
| 926 |
|
|---|
| 927 |
|
|---|
| 928 | def install_private_extension_xml(url, mlist):
|
|---|
| 929 | """Update XML files with metadata about installed modules and toolbox
|
|---|
| 930 | of an private addon
|
|---|
| 931 |
|
|---|
| 932 | """
|
|---|
| 933 | # TODO toolbox
|
|---|
| 934 | # if len(mlist) > 1:
|
|---|
| 935 | # # read metadata from remote server (toolboxes)
|
|---|
| 936 | # install_toolbox_xml(url, options['extension'])
|
|---|
| 937 |
|
|---|
| 938 | xml_file = os.path.join(options['prefix'], 'modules.xml')
|
|---|
| 939 | # create an empty file if not exists
|
|---|
| 940 | if not os.path.exists(xml_file):
|
|---|
| 941 | write_xml_modules(xml_file)
|
|---|
| 942 |
|
|---|
| 943 | # read XML file
|
|---|
| 944 | tree = etree_fromfile(xml_file)
|
|---|
| 945 |
|
|---|
| 946 | # update tree
|
|---|
| 947 | for name in mlist:
|
|---|
| 948 |
|
|---|
| 949 | try:
|
|---|
| 950 | desc = gtask.parse_interface(name).description
|
|---|
| 951 | # mname = gtask.parse_interface(name).name
|
|---|
| 952 | keywords = gtask.parse_interface(name).keywords
|
|---|
| 953 | except Exception as e:
|
|---|
| 954 | grass.warning(_("No addons metadata available."
|
|---|
| 955 | " Addons metadata file not updated."))
|
|---|
| 956 | return []
|
|---|
| 957 |
|
|---|
| 958 | tnode = None
|
|---|
| 959 | for node in tree.findall('task'):
|
|---|
| 960 | if node.get('name') == name:
|
|---|
| 961 | tnode = node
|
|---|
| 962 | break
|
|---|
| 963 |
|
|---|
| 964 | # create new node for task
|
|---|
| 965 | tnode = etree.Element('task', attrib={'name': name})
|
|---|
| 966 | dnode = etree.Element('description')
|
|---|
| 967 | dnode.text = desc
|
|---|
| 968 | tnode.append(dnode)
|
|---|
| 969 | knode = etree.Element('keywords')
|
|---|
| 970 | knode.text = (',').join(keywords)
|
|---|
| 971 | tnode.append(knode)
|
|---|
| 972 |
|
|---|
| 973 | # create binary
|
|---|
| 974 | bnode = etree.Element('binary')
|
|---|
| 975 | list_of_binary_files = []
|
|---|
| 976 | for file_name in os.listdir(url):
|
|---|
| 977 | file_type = os.path.splitext(file_name)[-1]
|
|---|
| 978 | file_n = os.path.splitext(file_name)[0]
|
|---|
| 979 | html_path = os.path.join(options['prefix'], 'docs', 'html')
|
|---|
| 980 | c_path = os.path.join(options['prefix'], 'bin')
|
|---|
| 981 | py_path = os.path.join(options['prefix'], 'scripts')
|
|---|
| 982 | # html or image file
|
|---|
| 983 | if file_type in ['.html', '.jpg', '.png'] \
|
|---|
| 984 | and file_n in os.listdir(html_path):
|
|---|
| 985 | list_of_binary_files.append(os.path.join(html_path, file_name))
|
|---|
| 986 | # c file
|
|---|
| 987 | elif file_type in ['.c'] and file_name in os.listdir(c_path):
|
|---|
| 988 | list_of_binary_files.append(os.path.join(c_path, file_n))
|
|---|
| 989 | # python file
|
|---|
| 990 | elif file_type in ['.py'] and file_name in os.listdir(py_path):
|
|---|
| 991 | list_of_binary_files.append(os.path.join(py_path, file_n))
|
|---|
| 992 | # man file
|
|---|
| 993 | man_path = os.path.join(options['prefix'], 'docs', 'man', 'man1')
|
|---|
| 994 | if name + '.1' in os.listdir(man_path):
|
|---|
| 995 | list_of_binary_files.append(os.path.join(man_path, name + '.1'))
|
|---|
| 996 | # add binaries to xml file
|
|---|
| 997 | for binary_file_name in list_of_binary_files:
|
|---|
| 998 | fnode = etree.Element('file')
|
|---|
| 999 | fnode.text = binary_file_name
|
|---|
| 1000 | bnode.append(fnode)
|
|---|
| 1001 | tnode.append(bnode)
|
|---|
| 1002 | tree.append(tnode)
|
|---|
| 1003 |
|
|---|
| 1004 | write_xml_modules(xml_file, tree)
|
|---|
| 1005 |
|
|---|
| 1006 | return mlist
|
|---|
| 1007 |
|
|---|
| 1008 |
|
|---|
| 1009 | def install_extension_win(name):
|
|---|
| 1010 | """Install extension on MS Windows"""
|
|---|
| 1011 | grass.message(_("Downloading precompiled GRASS Addons <%s>...") %
|
|---|
| 1012 | options['extension'])
|
|---|
| 1013 |
|
|---|
| 1014 | # build base URL
|
|---|
| 1015 | if build_platform == 'x86_64':
|
|---|
| 1016 | platform = build_platform
|
|---|
| 1017 | else:
|
|---|
| 1018 | platform = 'x86'
|
|---|
| 1019 | base_url = "http://wingrass.fsv.cvut.cz/" \
|
|---|
| 1020 | "grass%(major)s%(minor)s/%(platform)s/addons/" \
|
|---|
| 1021 | "grass-%(major)s.%(minor)s.%(patch)s" % \
|
|---|
| 1022 | {'platform': platform,
|
|---|
| 1023 | 'major': version[0], 'minor': version[1],
|
|---|
| 1024 | 'patch': version[2]}
|
|---|
| 1025 |
|
|---|
| 1026 | # resolve ZIP URL
|
|---|
| 1027 | source, url = resolve_source_code(url='{0}/{1}.zip'.format(base_url, name))
|
|---|
| 1028 |
|
|---|
| 1029 | # to hide non-error messages from subprocesses
|
|---|
| 1030 | if grass.verbosity() <= 2:
|
|---|
| 1031 | outdev = open(os.devnull, 'w')
|
|---|
| 1032 | else:
|
|---|
| 1033 | outdev = sys.stdout
|
|---|
| 1034 |
|
|---|
| 1035 | # download Addons ZIP file
|
|---|
| 1036 | os.chdir(TMPDIR) # this is just to not leave something behind
|
|---|
| 1037 | srcdir = os.path.join(TMPDIR, name)
|
|---|
| 1038 | download_source_code(source=source, url=url, name=name,
|
|---|
| 1039 | outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
|
|---|
| 1040 |
|
|---|
| 1041 | # copy Addons copy tree to destination directory
|
|---|
| 1042 | move_extracted_files(extract_dir=srcdir, target_dir=options['prefix'],
|
|---|
| 1043 | files=os.listdir(srcdir))
|
|---|
| 1044 |
|
|---|
| 1045 | return 0
|
|---|
| 1046 |
|
|---|
| 1047 |
|
|---|
| 1048 | def download_source_code_svn(url, name, outdev, directory=None):
|
|---|
| 1049 | """Download source code from a Subversion reporsitory
|
|---|
| 1050 |
|
|---|
| 1051 | .. note:
|
|---|
| 1052 | Stdout is passed to to *outdev* while stderr is will be just printed.
|
|---|
| 1053 |
|
|---|
| 1054 | :param url: URL of the repository
|
|---|
| 1055 | (module class/family and name are attached)
|
|---|
| 1056 | :param name: module name
|
|---|
| 1057 | :param outdev: output divide for the standard output of the svn command
|
|---|
| 1058 | :param directory: directory where the source code will be downloaded
|
|---|
| 1059 | (default is the current directory with name attached)
|
|---|
| 1060 |
|
|---|
| 1061 | :returns: full path to the directory with the source code
|
|---|
| 1062 | (useful when you not specify directory, if *directory* is specified
|
|---|
| 1063 | the return value is equal to it)
|
|---|
| 1064 | """
|
|---|
| 1065 | if not directory:
|
|---|
| 1066 | directory = os.path.join(os.getcwd, name)
|
|---|
| 1067 | classchar = name.split('.', 1)[0]
|
|---|
| 1068 | moduleclass = expand_module_class_name(classchar)
|
|---|
| 1069 | url = url + '/' + moduleclass + '/' + name
|
|---|
| 1070 | if grass.call(['svn', 'checkout',
|
|---|
| 1071 | url, directory], stdout=outdev) != 0:
|
|---|
| 1072 | grass.fatal(_("GRASS Addons <%s> not found") % name)
|
|---|
| 1073 | return directory
|
|---|
| 1074 |
|
|---|
| 1075 |
|
|---|
| 1076 | def move_extracted_files(extract_dir, target_dir, files):
|
|---|
| 1077 | """Fix state of extracted file by moving them to different diretcory
|
|---|
| 1078 |
|
|---|
| 1079 | When extracting, it is not clear what will be the root directory
|
|---|
| 1080 | or if there will be one at all. So this function moves the files to
|
|---|
| 1081 | a different directory in the way that if there was one directory extracted,
|
|---|
| 1082 | the contained files are moved.
|
|---|
| 1083 | """
|
|---|
| 1084 | gscript.debug("move_extracted_files({0})".format(locals()))
|
|---|
| 1085 | if len(files) == 1:
|
|---|
| 1086 | shutil.copytree(os.path.join(extract_dir, files[0]), target_dir)
|
|---|
| 1087 | else:
|
|---|
| 1088 | if not os.path.exists(target_dir):
|
|---|
| 1089 | os.mkdir(target_dir)
|
|---|
| 1090 | for file_name in files:
|
|---|
| 1091 | actual_file = os.path.join(extract_dir, file_name)
|
|---|
| 1092 | if os.path.isdir(actual_file):
|
|---|
| 1093 | # shutil.copytree() replaced by copy_tree() because
|
|---|
| 1094 | # shutil's copytree() fails when subdirectory exists
|
|---|
| 1095 | copy_tree(actual_file,
|
|---|
| 1096 | os.path.join(target_dir, file_name))
|
|---|
| 1097 | else:
|
|---|
| 1098 | shutil.copy(actual_file, os.path.join(target_dir, file_name))
|
|---|
| 1099 |
|
|---|
| 1100 |
|
|---|
| 1101 | # Original copyright and license of the original version of the CRLF function
|
|---|
| 1102 | # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
|
|---|
| 1103 | # Python Software Foundation; All Rights Reserved
|
|---|
| 1104 | # Python Software Foundation License Version 2
|
|---|
| 1105 | # http://svn.python.org/projects/python/trunk/Tools/scripts/crlf.py
|
|---|
| 1106 | def fix_newlines(directory):
|
|---|
| 1107 | """Replace CRLF with LF in all files in the directory
|
|---|
| 1108 |
|
|---|
| 1109 | Binary files are ignored. Recurses into subdirectories.
|
|---|
| 1110 | """
|
|---|
| 1111 | # skip binary files
|
|---|
| 1112 | # see https://stackoverflow.com/a/7392391
|
|---|
| 1113 | textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
|
|---|
| 1114 | is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
|
|---|
| 1115 |
|
|---|
| 1116 | for root, unused, files in os.walk(directory):
|
|---|
| 1117 | for name in files:
|
|---|
| 1118 | filename = os.path.join(root, name)
|
|---|
| 1119 | if is_binary_string(open(filename, 'rb').read(1024)):
|
|---|
| 1120 | continue # ignore binary files
|
|---|
| 1121 |
|
|---|
| 1122 | # read content of text file
|
|---|
| 1123 | with open(filename, 'rb') as fd:
|
|---|
| 1124 | data = fd.read()
|
|---|
| 1125 |
|
|---|
| 1126 | # we don't expect there would be CRLF file by
|
|---|
| 1127 | # purpose if we want to allow CRLF files we would
|
|---|
| 1128 | # have to whitelite .py etc
|
|---|
| 1129 | newdata = data.replace(b'\r\n', b'\n')
|
|---|
| 1130 | if newdata != data:
|
|---|
| 1131 | with open(filename, 'wb') as newfile:
|
|---|
| 1132 | newfile.write(newdata)
|
|---|
| 1133 |
|
|---|
| 1134 | def extract_zip(name, directory, tmpdir):
|
|---|
| 1135 | """Extract a ZIP file into a directory"""
|
|---|
| 1136 | gscript.debug("extract_zip(name={name}, directory={directory},"
|
|---|
| 1137 | " tmpdir={tmpdir})".format(name=name, directory=directory,
|
|---|
| 1138 | tmpdir=tmpdir), 3)
|
|---|
| 1139 | try:
|
|---|
| 1140 | zip_file = zipfile.ZipFile(name, mode='r')
|
|---|
| 1141 | file_list = zip_file.namelist()
|
|---|
| 1142 | # we suppose we can write to parent of the given dir
|
|---|
| 1143 | # (supposing a tmp dir)
|
|---|
| 1144 | extract_dir = os.path.join(tmpdir, 'extract_dir')
|
|---|
| 1145 | os.mkdir(extract_dir)
|
|---|
| 1146 | for subfile in file_list:
|
|---|
| 1147 | # this should be safe in Python 2.7.4
|
|---|
| 1148 | zip_file.extract(subfile, extract_dir)
|
|---|
| 1149 | files = os.listdir(extract_dir)
|
|---|
| 1150 | move_extracted_files(extract_dir=extract_dir,
|
|---|
| 1151 | target_dir=directory, files=files)
|
|---|
| 1152 | except zipfile.BadZipfile as error:
|
|---|
| 1153 | gscript.fatal(_("ZIP file is unreadable: {0}").format(error))
|
|---|
| 1154 |
|
|---|
| 1155 |
|
|---|
| 1156 | # TODO: solve the other related formats
|
|---|
| 1157 | def extract_tar(name, directory, tmpdir):
|
|---|
| 1158 | """Extract a TAR or a similar file into a directory"""
|
|---|
| 1159 | gscript.debug("extract_tar(name={name}, directory={directory},"
|
|---|
| 1160 | " tmpdir={tmpdir})".format(name=name, directory=directory,
|
|---|
| 1161 | tmpdir=tmpdir), 3)
|
|---|
| 1162 | try:
|
|---|
| 1163 | import tarfile # we don't need it anywhere else
|
|---|
| 1164 | tar = tarfile.open(name)
|
|---|
| 1165 | extract_dir = os.path.join(tmpdir, 'extract_dir')
|
|---|
| 1166 | os.mkdir(extract_dir)
|
|---|
| 1167 | tar.extractall(path=extract_dir)
|
|---|
| 1168 | files = os.listdir(extract_dir)
|
|---|
| 1169 | move_extracted_files(extract_dir=extract_dir,
|
|---|
| 1170 | target_dir=directory, files=files)
|
|---|
| 1171 | except tarfile.TarError as error:
|
|---|
| 1172 | gscript.fatal(_("Archive file is unreadable: {0}").format(error))
|
|---|
| 1173 |
|
|---|
| 1174 | extract_tar.supported_formats = ['tar.gz', 'gz', 'bz2', 'tar', 'gzip', 'targz']
|
|---|
| 1175 |
|
|---|
| 1176 |
|
|---|
| 1177 | def download_source_code(source, url, name, outdev,
|
|---|
| 1178 | directory=None, tmpdir=None):
|
|---|
| 1179 | """Get source code to a local directory for compilation"""
|
|---|
| 1180 | gscript.verbose("Downloading source code for <{name}> from <{url}>"
|
|---|
| 1181 | " which is identified as '{source}' type of source..."
|
|---|
| 1182 | .format(source=source, url=url, name=name))
|
|---|
| 1183 | if source == 'svn':
|
|---|
| 1184 | download_source_code_svn(url, name, outdev, directory)
|
|---|
| 1185 | elif source in ['remote_zip', 'official']:
|
|---|
| 1186 | # we expect that the module.zip file is not by chance in the archive
|
|---|
| 1187 | zip_name = os.path.join(tmpdir, 'extension.zip')
|
|---|
| 1188 | try:
|
|---|
| 1189 | response = urlopen(url)
|
|---|
| 1190 | except URLError:
|
|---|
| 1191 | grass.fatal(_("Extension <%s> not found") % name)
|
|---|
| 1192 | with open(zip_name, 'wb') as out_file:
|
|---|
| 1193 | shutil.copyfileobj(response, out_file)
|
|---|
| 1194 | extract_zip(name=zip_name, directory=directory, tmpdir=tmpdir)
|
|---|
| 1195 | fix_newlines(directory)
|
|---|
| 1196 | elif source.startswith('remote_') and \
|
|---|
| 1197 | source.split('_')[1] in extract_tar.supported_formats:
|
|---|
| 1198 | # we expect that the module.tar.gz file is not by chance in the archive
|
|---|
| 1199 | archive_name = os.path.join(tmpdir,
|
|---|
| 1200 | 'extension.' + source.split('_')[1])
|
|---|
| 1201 | urlretrieve(url, archive_name)
|
|---|
| 1202 | extract_tar(name=archive_name, directory=directory, tmpdir=tmpdir)
|
|---|
| 1203 | fix_newlines(directory)
|
|---|
| 1204 | elif source == 'zip':
|
|---|
| 1205 | extract_zip(name=url, directory=directory, tmpdir=tmpdir)
|
|---|
| 1206 | fix_newlines(directory)
|
|---|
| 1207 | elif source in extract_tar.supported_formats:
|
|---|
| 1208 | extract_tar(name=url, directory=directory, tmpdir=tmpdir)
|
|---|
| 1209 | fix_newlines(directory)
|
|---|
| 1210 | elif source == 'dir':
|
|---|
| 1211 | shutil.copytree(url, directory)
|
|---|
| 1212 | fix_newlines(directory)
|
|---|
| 1213 | else:
|
|---|
| 1214 | # probably programmer error
|
|---|
| 1215 | grass.fatal(_("Unknown extension (addon) source type '{0}'."
|
|---|
| 1216 | " Please report this to the grass-user mailing list.")
|
|---|
| 1217 | .format(source))
|
|---|
| 1218 | assert os.path.isdir(directory)
|
|---|
| 1219 |
|
|---|
| 1220 |
|
|---|
| 1221 | def install_extension_std_platforms(name, source, url):
|
|---|
| 1222 | """Install extension on standard platforms"""
|
|---|
| 1223 | gisbase = os.getenv('GISBASE')
|
|---|
| 1224 | source_url = "https://trac.osgeo.org/grass/browser/grass-addons/grass7/"
|
|---|
| 1225 |
|
|---|
| 1226 | if source == 'official':
|
|---|
| 1227 | gscript.message(_("Fetching <%s> from "
|
|---|
| 1228 | "GRASS GIS Addons repository (be patient)...") % name)
|
|---|
| 1229 | else:
|
|---|
| 1230 | gscript.message(_("Fetching <{name}> from "
|
|---|
| 1231 | "<{url}> (be patient)...").format(name=name, url=url))
|
|---|
| 1232 |
|
|---|
| 1233 | # to hide non-error messages from subprocesses
|
|---|
| 1234 | if grass.verbosity() <= 2:
|
|---|
| 1235 | outdev = open(os.devnull, 'w')
|
|---|
| 1236 | else:
|
|---|
| 1237 | outdev = sys.stdout
|
|---|
| 1238 |
|
|---|
| 1239 | os.chdir(TMPDIR) # this is just to not leave something behind
|
|---|
| 1240 | srcdir = os.path.join(TMPDIR, name)
|
|---|
| 1241 | download_source_code(source=source, url=url, name=name,
|
|---|
| 1242 | outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
|
|---|
| 1243 | os.chdir(srcdir)
|
|---|
| 1244 |
|
|---|
| 1245 | dirs = {
|
|---|
| 1246 | 'bin': os.path.join(TMPDIR, name, 'bin'),
|
|---|
| 1247 | 'docs': os.path.join(TMPDIR, name, 'docs'),
|
|---|
| 1248 | 'html': os.path.join(TMPDIR, name, 'docs', 'html'),
|
|---|
| 1249 | 'rest': os.path.join(TMPDIR, name, 'docs', 'rest'),
|
|---|
| 1250 | 'man': os.path.join(TMPDIR, name, 'docs', 'man'),
|
|---|
| 1251 | 'script': os.path.join(TMPDIR, name, 'scripts'),
|
|---|
| 1252 | # TODO: handle locales also for addons
|
|---|
| 1253 | # 'string' : os.path.join(TMPDIR, name, 'locale'),
|
|---|
| 1254 | 'string': os.path.join(TMPDIR, name),
|
|---|
| 1255 | 'etc': os.path.join(TMPDIR, name, 'etc'),
|
|---|
| 1256 | }
|
|---|
| 1257 |
|
|---|
| 1258 | make_cmd = [
|
|---|
| 1259 | 'make',
|
|---|
| 1260 | 'MODULE_TOPDIR=%s' % gisbase.replace(' ', r'\ '),
|
|---|
| 1261 | 'RUN_GISRC=%s' % os.environ['GISRC'],
|
|---|
| 1262 | 'BIN=%s' % dirs['bin'],
|
|---|
| 1263 | 'HTMLDIR=%s' % dirs['html'],
|
|---|
| 1264 | 'RESTDIR=%s' % dirs['rest'],
|
|---|
| 1265 | 'MANBASEDIR=%s' % dirs['man'],
|
|---|
| 1266 | 'SCRIPTDIR=%s' % dirs['script'],
|
|---|
| 1267 | 'STRINGDIR=%s' % dirs['string'],
|
|---|
| 1268 | 'ETC=%s' % os.path.join(dirs['etc']),
|
|---|
| 1269 | 'SOURCE_URL=%s' % source_url
|
|---|
| 1270 | ]
|
|---|
| 1271 |
|
|---|
| 1272 | install_cmd = [
|
|---|
| 1273 | 'make',
|
|---|
| 1274 | 'MODULE_TOPDIR=%s' % gisbase,
|
|---|
| 1275 | 'ARCH_DISTDIR=%s' % os.path.join(TMPDIR, name),
|
|---|
| 1276 | 'INST_DIR=%s' % options['prefix'],
|
|---|
| 1277 | 'install'
|
|---|
| 1278 | ]
|
|---|
| 1279 |
|
|---|
| 1280 | if flags['d']:
|
|---|
| 1281 | grass.message("\n%s\n" % _("To compile run:"))
|
|---|
| 1282 | sys.stderr.write(' '.join(make_cmd) + '\n')
|
|---|
| 1283 | grass.message("\n%s\n" % _("To install run:"))
|
|---|
| 1284 | sys.stderr.write(' '.join(install_cmd) + '\n')
|
|---|
| 1285 | return 0
|
|---|
| 1286 |
|
|---|
| 1287 | os.chdir(os.path.join(TMPDIR, name))
|
|---|
| 1288 |
|
|---|
| 1289 | grass.message(_("Compiling..."))
|
|---|
| 1290 | if not os.path.exists(os.path.join(gisbase, 'include',
|
|---|
| 1291 | 'Make', 'Module.make')):
|
|---|
| 1292 | grass.fatal(_("Please install GRASS development package"))
|
|---|
| 1293 |
|
|---|
| 1294 | if 0 != grass.call(make_cmd,
|
|---|
| 1295 | stdout=outdev):
|
|---|
| 1296 | grass.fatal(_('Compilation failed, sorry.'
|
|---|
| 1297 | ' Please check above error messages.'))
|
|---|
| 1298 |
|
|---|
| 1299 | if flags['i']:
|
|---|
| 1300 | return 0
|
|---|
| 1301 |
|
|---|
| 1302 | grass.message(_("Installing..."))
|
|---|
| 1303 |
|
|---|
| 1304 |
|
|---|
| 1305 | with open(os.path.join(TMPDIR, name, 'Makefile')) as f:
|
|---|
| 1306 | datafile = f.readlines()
|
|---|
| 1307 |
|
|---|
| 1308 | makefile_part = ""
|
|---|
| 1309 | next_line = False
|
|---|
| 1310 | for line in datafile:
|
|---|
| 1311 | if 'SUBDIRS' in line or next_line:
|
|---|
| 1312 | makefile_part += line
|
|---|
| 1313 | if (line.strip()).endswith('\\'):
|
|---|
| 1314 | next_line = True
|
|---|
| 1315 | else:
|
|---|
| 1316 | next_line = False
|
|---|
| 1317 |
|
|---|
| 1318 | modules = makefile_part.replace('SUBDIRS', '').replace('=', '').replace('\\', '').strip().split('\n')
|
|---|
| 1319 | c_path = os.path.join(options['prefix'], 'bin')
|
|---|
| 1320 | py_path = os.path.join(options['prefix'], 'scripts')
|
|---|
| 1321 |
|
|---|
| 1322 | all_modules = os.listdir(c_path)
|
|---|
| 1323 | all_modules.extend(os.listdir(py_path))
|
|---|
| 1324 | module_list = [x.strip() for x in modules if x.strip() in all_modules]
|
|---|
| 1325 |
|
|---|
| 1326 | return grass.call(install_cmd, stdout=outdev), module_list, os.path.join(TMPDIR, name)
|
|---|
| 1327 |
|
|---|
| 1328 |
|
|---|
| 1329 | def remove_extension(force=False):
|
|---|
| 1330 | """Remove existing extension (module or toolbox if -t is given)"""
|
|---|
| 1331 | if flags['t']:
|
|---|
| 1332 | mlist = get_toolbox_modules(options['prefix'], options['extension'])
|
|---|
| 1333 | else:
|
|---|
| 1334 | mlist = [options['extension']]
|
|---|
| 1335 |
|
|---|
| 1336 | if force:
|
|---|
| 1337 | grass.verbose(_("List of removed files:"))
|
|---|
| 1338 | else:
|
|---|
| 1339 | grass.info(_("Files to be removed:"))
|
|---|
| 1340 |
|
|---|
| 1341 | remove_modules(mlist, force)
|
|---|
| 1342 |
|
|---|
| 1343 | if force:
|
|---|
| 1344 | grass.message(_("Updating addons metadata file..."))
|
|---|
| 1345 | remove_extension_xml(mlist)
|
|---|
| 1346 | grass.message(_("Extension <%s> successfully uninstalled.") %
|
|---|
| 1347 | options['extension'])
|
|---|
| 1348 | else:
|
|---|
| 1349 | grass.warning(_("Extension <%s> not removed. "
|
|---|
| 1350 | "Re-run '%s' with '-f' flag to force removal")
|
|---|
| 1351 | % (options['extension'], 'g.extension'))
|
|---|
| 1352 |
|
|---|
| 1353 | # remove existing extension(s) (reading XML file)
|
|---|
| 1354 |
|
|---|
| 1355 |
|
|---|
| 1356 | def remove_modules(mlist, force=False):
|
|---|
| 1357 | """Remove extensions/modules specified in a list
|
|---|
| 1358 |
|
|---|
| 1359 | Collects the file names from the file with metadata and fallbacks
|
|---|
| 1360 | to standard layout of files on prefix path on error.
|
|---|
| 1361 | """
|
|---|
| 1362 | # try to read XML metadata file first
|
|---|
| 1363 | xml_file = os.path.join(options['prefix'], 'modules.xml')
|
|---|
| 1364 | installed = get_installed_modules()
|
|---|
| 1365 |
|
|---|
| 1366 | if os.path.exists(xml_file):
|
|---|
| 1367 | tree = etree_fromfile(xml_file)
|
|---|
| 1368 | else:
|
|---|
| 1369 | tree = None
|
|---|
| 1370 |
|
|---|
| 1371 | for name in mlist:
|
|---|
| 1372 | if name not in installed:
|
|---|
| 1373 | # try even if module does not seem to be available,
|
|---|
| 1374 | # as the user may be trying to get rid of left over cruft
|
|---|
| 1375 | grass.warning(_("Extension <%s> not found") % name)
|
|---|
| 1376 |
|
|---|
| 1377 | if tree is not None:
|
|---|
| 1378 | flist = []
|
|---|
| 1379 | for task in tree.findall('task'):
|
|---|
| 1380 | if name == task.get('name') and \
|
|---|
| 1381 | task.find('binary') is not None:
|
|---|
| 1382 | flist = get_module_files(task)
|
|---|
| 1383 | break
|
|---|
| 1384 |
|
|---|
| 1385 | if flist:
|
|---|
| 1386 | removed = False
|
|---|
| 1387 | err = list()
|
|---|
| 1388 | for fpath in flist:
|
|---|
| 1389 | grass.verbose(fpath)
|
|---|
| 1390 | if force:
|
|---|
| 1391 | try:
|
|---|
| 1392 | os.remove(fpath)
|
|---|
| 1393 | removed = True
|
|---|
| 1394 | except OSError:
|
|---|
| 1395 | msg = "Unable to remove file '%s'"
|
|---|
| 1396 | err.append((_(msg) % fpath))
|
|---|
| 1397 | if force and not removed:
|
|---|
| 1398 | grass.fatal(_("Extension <%s> not found") % name)
|
|---|
| 1399 |
|
|---|
| 1400 | if err:
|
|---|
| 1401 | for error_line in err:
|
|---|
| 1402 | grass.error(error_line)
|
|---|
| 1403 | else:
|
|---|
| 1404 | remove_extension_std(name, force)
|
|---|
| 1405 | else:
|
|---|
| 1406 | remove_extension_std(name, force)
|
|---|
| 1407 |
|
|---|
| 1408 | # remove module libraries directories under GRASS_ADDONS/etc/{name}/*
|
|---|
| 1409 | libpath = os.path.join(options['prefix'], 'etc', name)
|
|---|
| 1410 | if os.path.isdir(libpath):
|
|---|
| 1411 | grass.verbose(libpath)
|
|---|
| 1412 | if force:
|
|---|
| 1413 | shutil.rmtree(libpath)
|
|---|
| 1414 |
|
|---|
| 1415 |
|
|---|
| 1416 | def remove_extension_std(name, force=False):
|
|---|
| 1417 | """Remove extension/module expecting the standard layout"""
|
|---|
| 1418 | for fpath in [os.path.join(options['prefix'], 'bin', name),
|
|---|
| 1419 | os.path.join(options['prefix'], 'scripts', name),
|
|---|
| 1420 | os.path.join(
|
|---|
| 1421 | options['prefix'], 'docs', 'html', name + '.html'),
|
|---|
| 1422 | os.path.join(
|
|---|
| 1423 | options['prefix'], 'docs', 'rest', name + '.txt'),
|
|---|
| 1424 | os.path.join(options['prefix'], 'docs', 'man', 'man1',
|
|---|
| 1425 | name + '.1')]:
|
|---|
| 1426 | if os.path.isfile(fpath):
|
|---|
| 1427 | grass.verbose(fpath)
|
|---|
| 1428 | if force:
|
|---|
| 1429 | os.remove(fpath)
|
|---|
| 1430 |
|
|---|
| 1431 | # remove module libraries under GRASS_ADDONS/etc/{name}/*
|
|---|
| 1432 | libpath = os.path.join(options['prefix'], 'etc', name)
|
|---|
| 1433 | if os.path.isdir(libpath):
|
|---|
| 1434 | grass.verbose(libpath)
|
|---|
| 1435 | if force:
|
|---|
| 1436 | shutil.rmtree(libpath)
|
|---|
| 1437 |
|
|---|
| 1438 |
|
|---|
| 1439 | def remove_from_toolbox_xml(name):
|
|---|
| 1440 | """Update local meta-file when removing existing toolbox"""
|
|---|
| 1441 | xml_file = os.path.join(options['prefix'], 'toolboxes.xml')
|
|---|
| 1442 | if not os.path.exists(xml_file):
|
|---|
| 1443 | return
|
|---|
| 1444 | # read XML file
|
|---|
| 1445 | tree = etree_fromfile(xml_file)
|
|---|
| 1446 | for node in tree.findall('toolbox'):
|
|---|
| 1447 | if node.get('code') != name:
|
|---|
| 1448 | continue
|
|---|
| 1449 | tree.remove(node)
|
|---|
| 1450 |
|
|---|
| 1451 | write_xml_toolboxes(xml_file, tree)
|
|---|
| 1452 |
|
|---|
| 1453 |
|
|---|
| 1454 | def remove_extension_xml(modules):
|
|---|
| 1455 | """Update local meta-file when removing existing extension"""
|
|---|
| 1456 | if len(modules) > 1:
|
|---|
| 1457 | # update also toolboxes metadata
|
|---|
| 1458 | remove_from_toolbox_xml(options['extension'])
|
|---|
| 1459 | xml_file = os.path.join(options['prefix'], 'modules.xml')
|
|---|
| 1460 | if not os.path.exists(xml_file):
|
|---|
| 1461 | return
|
|---|
| 1462 | # read XML file
|
|---|
| 1463 | tree = etree_fromfile(xml_file)
|
|---|
| 1464 | for name in modules:
|
|---|
| 1465 | for node in tree.findall('task'):
|
|---|
| 1466 | if node.get('name') != name:
|
|---|
| 1467 | continue
|
|---|
| 1468 | tree.remove(node)
|
|---|
| 1469 | write_xml_modules(xml_file, tree)
|
|---|
| 1470 |
|
|---|
| 1471 | # check links in CSS
|
|---|
| 1472 |
|
|---|
| 1473 |
|
|---|
| 1474 | def check_style_files(fil):
|
|---|
| 1475 | """Ensures that a specified HTML documentation support file exists
|
|---|
| 1476 |
|
|---|
| 1477 | If the file, e.g. a CSS file does not exist, the file is copied from
|
|---|
| 1478 | the distribution.
|
|---|
| 1479 | """
|
|---|
| 1480 | dist_file = os.path.join(os.getenv('GISBASE'), 'docs', 'html', fil)
|
|---|
| 1481 | addons_file = os.path.join(options['prefix'], 'docs', 'html', fil)
|
|---|
| 1482 |
|
|---|
| 1483 | if os.path.isfile(addons_file):
|
|---|
| 1484 | return
|
|---|
| 1485 |
|
|---|
| 1486 | try:
|
|---|
| 1487 | shutil.copyfile(dist_file, addons_file)
|
|---|
| 1488 | except OSError as error:
|
|---|
| 1489 | grass.fatal(_("Unable to create '%s': %s") % (addons_file, error))
|
|---|
| 1490 |
|
|---|
| 1491 |
|
|---|
| 1492 | def create_dir(path):
|
|---|
| 1493 | """Creates the specified directory (with all dirs in between)
|
|---|
| 1494 |
|
|---|
| 1495 | NOOP for existing directory.
|
|---|
| 1496 | """
|
|---|
| 1497 | if os.path.isdir(path):
|
|---|
| 1498 | return
|
|---|
| 1499 |
|
|---|
| 1500 | try:
|
|---|
| 1501 | os.makedirs(path)
|
|---|
| 1502 | except OSError as error:
|
|---|
| 1503 | grass.fatal(_("Unable to create '%s': %s") % (path, error))
|
|---|
| 1504 |
|
|---|
| 1505 | grass.debug("'%s' created" % path)
|
|---|
| 1506 |
|
|---|
| 1507 |
|
|---|
| 1508 | def check_dirs():
|
|---|
| 1509 | """Ensure that the necessary directories in prefix path exist"""
|
|---|
| 1510 | create_dir(os.path.join(options['prefix'], 'bin'))
|
|---|
| 1511 | create_dir(os.path.join(options['prefix'], 'docs', 'html'))
|
|---|
| 1512 | create_dir(os.path.join(options['prefix'], 'docs', 'rest'))
|
|---|
| 1513 | check_style_files('grass_logo.png')
|
|---|
| 1514 | check_style_files('grassdocs.css')
|
|---|
| 1515 | create_dir(os.path.join(options['prefix'], 'etc'))
|
|---|
| 1516 | create_dir(os.path.join(options['prefix'], 'docs', 'man', 'man1'))
|
|---|
| 1517 | create_dir(os.path.join(options['prefix'], 'scripts'))
|
|---|
| 1518 |
|
|---|
| 1519 | # fix file URI in manual page
|
|---|
| 1520 |
|
|---|
| 1521 |
|
|---|
| 1522 | def update_manual_page(module):
|
|---|
| 1523 | """Fix manual page for addons which are at different directory then rest"""
|
|---|
| 1524 | if module.split('.', 1)[0] == 'wx':
|
|---|
| 1525 | return # skip for GUI modules
|
|---|
| 1526 |
|
|---|
| 1527 | grass.verbose(_("Manual page for <%s> updated") % module)
|
|---|
| 1528 | # read original html file
|
|---|
| 1529 | htmlfile = os.path.join(
|
|---|
| 1530 | options['prefix'], 'docs', 'html', module + '.html')
|
|---|
| 1531 | try:
|
|---|
| 1532 | oldfile = open(htmlfile)
|
|---|
| 1533 | shtml = oldfile.read()
|
|---|
| 1534 | except IOError as error:
|
|---|
| 1535 | gscript.fatal(_("Unable to read manual page: %s") % error)
|
|---|
| 1536 | else:
|
|---|
| 1537 | oldfile.close()
|
|---|
| 1538 |
|
|---|
| 1539 | pos = []
|
|---|
| 1540 |
|
|---|
| 1541 | # fix logo URL
|
|---|
| 1542 | pattern = r'''<a href="([^"]+)"><img src="grass_logo.png"'''
|
|---|
| 1543 | for match in re.finditer(pattern, shtml):
|
|---|
| 1544 | pos.append(match.start(1))
|
|---|
| 1545 |
|
|---|
| 1546 | # find URIs
|
|---|
| 1547 | pattern = r'''<a href="([^"]+)">([^>]+)</a>'''
|
|---|
| 1548 | addons = get_installed_extensions(force=True)
|
|---|
| 1549 | for match in re.finditer(pattern, shtml):
|
|---|
| 1550 | if match.group(1)[:4] == 'http':
|
|---|
| 1551 | continue
|
|---|
| 1552 | if match.group(1).replace('.html', '') in addons:
|
|---|
| 1553 | continue
|
|---|
| 1554 | pos.append(match.start(1))
|
|---|
| 1555 |
|
|---|
| 1556 | if not pos:
|
|---|
| 1557 | return # no match
|
|---|
| 1558 |
|
|---|
| 1559 | # replace file URIs
|
|---|
| 1560 | prefix = 'file://' + '/'.join([os.getenv('GISBASE'), 'docs', 'html'])
|
|---|
| 1561 | ohtml = shtml[:pos[0]]
|
|---|
| 1562 | for i in range(1, len(pos)):
|
|---|
| 1563 | ohtml += prefix + '/' + shtml[pos[i - 1]:pos[i]]
|
|---|
| 1564 | ohtml += prefix + '/' + shtml[pos[-1]:]
|
|---|
| 1565 |
|
|---|
| 1566 | # write updated html file
|
|---|
| 1567 | try:
|
|---|
| 1568 | newfile = open(htmlfile, 'w')
|
|---|
| 1569 | newfile.write(ohtml)
|
|---|
| 1570 | except IOError as error:
|
|---|
| 1571 | gscript.fatal(_("Unable for write manual page: %s") % error)
|
|---|
| 1572 | else:
|
|---|
| 1573 | newfile.close()
|
|---|
| 1574 |
|
|---|
| 1575 |
|
|---|
| 1576 | def resolve_install_prefix(path, to_system):
|
|---|
| 1577 | """Determine and check the path for installation"""
|
|---|
| 1578 | if to_system:
|
|---|
| 1579 | path = os.environ['GISBASE']
|
|---|
| 1580 | if path == '$GRASS_ADDON_BASE':
|
|---|
| 1581 | if not os.getenv('GRASS_ADDON_BASE'):
|
|---|
| 1582 | grass.warning(_("GRASS_ADDON_BASE is not defined, "
|
|---|
| 1583 | "installing to ~/.grass%s/addons") % version[0])
|
|---|
| 1584 | path = os.path.join(
|
|---|
| 1585 | os.environ['HOME'], '.grass%s' % version[0], 'addons')
|
|---|
| 1586 | else:
|
|---|
| 1587 | path = os.environ['GRASS_ADDON_BASE']
|
|---|
| 1588 | if os.path.exists(path) and \
|
|---|
| 1589 | not os.access(path, os.W_OK):
|
|---|
| 1590 | grass.fatal(_("You don't have permission to install extension to <{0}>."
|
|---|
| 1591 | " Try to run {1} with administrator rights"
|
|---|
| 1592 | " (su or sudo).")
|
|---|
| 1593 | .format(path, 'g.extension'))
|
|---|
| 1594 | # ensure dir sep at the end for cases where path is used as URL and pasted
|
|---|
| 1595 | # together with file names
|
|---|
| 1596 | if not path.endswith(os.path.sep):
|
|---|
| 1597 | path = path + os.path.sep
|
|---|
| 1598 | return os.path.abspath(path) # make likes absolute paths
|
|---|
| 1599 |
|
|---|
| 1600 |
|
|---|
| 1601 | def resolve_xmlurl_prefix(url, source=None):
|
|---|
| 1602 | """Determine and check the URL where the XML metadata files are stored
|
|---|
| 1603 |
|
|---|
| 1604 | It ensures that there is a single slash at the end of URL, so we can attach
|
|---|
| 1605 | file name easily:
|
|---|
| 1606 |
|
|---|
| 1607 | >>> resolve_xmlurl_prefix('https://grass.osgeo.org/addons')
|
|---|
| 1608 | 'https://grass.osgeo.org/addons/'
|
|---|
| 1609 | >>> resolve_xmlurl_prefix('https://grass.osgeo.org/addons/')
|
|---|
| 1610 | 'https://grass.osgeo.org/addons/'
|
|---|
| 1611 | """
|
|---|
| 1612 | gscript.debug("resolve_xmlurl_prefix(url={0}, source={1})".format(url, source))
|
|---|
| 1613 | if source == 'official':
|
|---|
| 1614 | # use pregenerated modules XML file
|
|---|
| 1615 | url = 'https://grass.osgeo.org/addons/grass%s/' % version[0]
|
|---|
| 1616 | # else try to get modules XMl from SVN repository (provided URL)
|
|---|
| 1617 | # the exact action depends on subsequent code (somewhere)
|
|---|
| 1618 |
|
|---|
| 1619 | if not url.endswith('/'):
|
|---|
| 1620 | url = url + '/'
|
|---|
| 1621 | return url
|
|---|
| 1622 |
|
|---|
| 1623 |
|
|---|
| 1624 | KNOWN_HOST_SERVICES_INFO = {
|
|---|
| 1625 | 'OSGeo Trac': {
|
|---|
| 1626 | 'domain': 'trac.osgeo.org',
|
|---|
| 1627 | 'ignored_suffixes': ['format=zip'],
|
|---|
| 1628 | 'possible_starts': ['', 'https://', 'http://'],
|
|---|
| 1629 | 'url_start': 'https://',
|
|---|
| 1630 | 'url_end': '?format=zip',
|
|---|
| 1631 | },
|
|---|
| 1632 | 'GitHub': {
|
|---|
| 1633 | 'domain': 'github.com',
|
|---|
| 1634 | 'ignored_suffixes': ['.zip', '.tar.gz'],
|
|---|
| 1635 | 'possible_starts': ['', 'https://', 'http://'],
|
|---|
| 1636 | 'url_start': 'https://',
|
|---|
| 1637 | 'url_end': '/archive/master.zip',
|
|---|
| 1638 | },
|
|---|
| 1639 | 'GitLab': {
|
|---|
| 1640 | 'domain': 'gitlab.com',
|
|---|
| 1641 | 'ignored_suffixes': ['.zip', '.tar.gz', '.tar.bz2', '.tar'],
|
|---|
| 1642 | 'possible_starts': ['', 'https://', 'http://'],
|
|---|
| 1643 | 'url_start': 'https://',
|
|---|
| 1644 | 'url_end': '/repository/archive.zip',
|
|---|
| 1645 | },
|
|---|
| 1646 | 'Bitbucket': {
|
|---|
| 1647 | 'domain': 'bitbucket.org',
|
|---|
| 1648 | 'ignored_suffixes': ['.zip', '.tar.gz', '.gz', '.bz2'],
|
|---|
| 1649 | 'possible_starts': ['', 'https://', 'http://'],
|
|---|
| 1650 | 'url_start': 'https://',
|
|---|
| 1651 | 'url_end': '/get/master.zip',
|
|---|
| 1652 | },
|
|---|
| 1653 | }
|
|---|
| 1654 |
|
|---|
| 1655 | # TODO: support ZIP URLs which don't end with zip
|
|---|
| 1656 | # https://gitlab.com/user/reponame/repository/archive.zip?ref=b%C3%A9po
|
|---|
| 1657 |
|
|---|
| 1658 |
|
|---|
| 1659 | def resolve_known_host_service(url):
|
|---|
| 1660 | """Determine source type and full URL for known hosting service
|
|---|
| 1661 |
|
|---|
| 1662 | If the service is not determined from the provided URL, tuple with
|
|---|
| 1663 | is two ``None`` values is returned.
|
|---|
| 1664 | """
|
|---|
| 1665 | match = None
|
|---|
| 1666 | actual_start = None
|
|---|
| 1667 | for key, value in KNOWN_HOST_SERVICES_INFO.items():
|
|---|
| 1668 | for start in value['possible_starts']:
|
|---|
| 1669 | if url.startswith(start + value['domain']):
|
|---|
| 1670 | match = value
|
|---|
| 1671 | actual_start = start
|
|---|
| 1672 | gscript.verbose(_("Identified {0} as known hosting service")
|
|---|
| 1673 | .format(key))
|
|---|
| 1674 | for suffix in value['ignored_suffixes']:
|
|---|
| 1675 | if url.endswith(suffix):
|
|---|
| 1676 | gscript.verbose(
|
|---|
| 1677 | _("Not using {service} as known hosting service"
|
|---|
| 1678 | " because the URL ends with '{suffix}'")
|
|---|
| 1679 | .format(service=key, suffix=suffix))
|
|---|
| 1680 | return None, None
|
|---|
| 1681 | if match:
|
|---|
| 1682 | if not actual_start:
|
|---|
| 1683 | actual_start = match['url_start']
|
|---|
| 1684 | else:
|
|---|
| 1685 | actual_start = ''
|
|---|
| 1686 | url = '{prefix}{base}{suffix}'.format(prefix=actual_start,
|
|---|
| 1687 | base=url.rstrip('/'),
|
|---|
| 1688 | suffix=match['url_end'])
|
|---|
| 1689 | gscript.verbose(_("Will use the following URL for download: {0}")
|
|---|
| 1690 | .format(url))
|
|---|
| 1691 | return 'remote_zip', url
|
|---|
| 1692 | else:
|
|---|
| 1693 | return None, None
|
|---|
| 1694 |
|
|---|
| 1695 |
|
|---|
| 1696 | # TODO: add also option to enforce the source type
|
|---|
| 1697 | def resolve_source_code(url=None, name=None):
|
|---|
| 1698 | """Return type and URL or path of the source code
|
|---|
| 1699 |
|
|---|
| 1700 | Local paths are not presented as URLs to be usable in standard functions.
|
|---|
| 1701 | Path is identified as local path if the directory of file exists which
|
|---|
| 1702 | has the unfortunate consequence that the not existing files are evaluated
|
|---|
| 1703 | as remote URLs. When path is not evaluated, Subversion is assumed for
|
|---|
| 1704 | backwards compatibility. When GitHub repository is specified, ZIP file
|
|---|
| 1705 | link is returned. The ZIP is for master branch, not the default one because
|
|---|
| 1706 | GitHub does not provide the default branch in the URL (July 2015).
|
|---|
| 1707 |
|
|---|
| 1708 | :returns: tuple with type of source and full URL or path
|
|---|
| 1709 |
|
|---|
| 1710 | Official repository:
|
|---|
| 1711 |
|
|---|
| 1712 | >>> resolve_source_code(name='g.example') # doctest: +SKIP
|
|---|
| 1713 | ('official', 'https://trac.osgeo.org/.../general/g.example')
|
|---|
| 1714 |
|
|---|
| 1715 | Subversion:
|
|---|
| 1716 |
|
|---|
| 1717 | >>> resolve_source_code('https://svn.osgeo.org/grass/grass-addons/grass7')
|
|---|
| 1718 | ('svn', 'https://svn.osgeo.org/grass/grass-addons/grass7')
|
|---|
| 1719 |
|
|---|
| 1720 | ZIP files online:
|
|---|
| 1721 |
|
|---|
| 1722 | >>> resolve_source_code('https://trac.osgeo.org/.../r.modis?format=zip') # doctest: +SKIP
|
|---|
| 1723 | ('remote_zip', 'https://trac.osgeo.org/.../r.modis?format=zip')
|
|---|
| 1724 |
|
|---|
| 1725 | Local directories and ZIP files:
|
|---|
| 1726 |
|
|---|
| 1727 | >>> resolve_source_code(os.path.expanduser("~")) # doctest: +ELLIPSIS
|
|---|
| 1728 | ('dir', '...')
|
|---|
| 1729 | >>> resolve_source_code('/local/directory/downloaded.zip') # doctest: +SKIP
|
|---|
| 1730 | ('zip', '/local/directory/downloaded.zip')
|
|---|
| 1731 |
|
|---|
| 1732 | OSGeo Trac:
|
|---|
| 1733 |
|
|---|
| 1734 | >>> resolve_source_code('trac.osgeo.org/.../r.agent.aco') # doctest: +SKIP
|
|---|
| 1735 | ('remote_zip', 'https://trac.osgeo.org/.../r.agent.aco?format=zip')
|
|---|
| 1736 | >>> resolve_source_code('https://trac.osgeo.org/.../r.agent.aco') # doctest: +SKIP
|
|---|
| 1737 | ('remote_zip', 'https://trac.osgeo.org/.../r.agent.aco?format=zip')
|
|---|
| 1738 |
|
|---|
| 1739 | GitHub:
|
|---|
| 1740 |
|
|---|
| 1741 | >>> resolve_source_code('github.com/user/g.example') # doctest: +SKIP
|
|---|
| 1742 | ('remote_zip', 'https://github.com/user/g.example/archive/master.zip')
|
|---|
| 1743 | >>> resolve_source_code('github.com/user/g.example/') # doctest: +SKIP
|
|---|
| 1744 | ('remote_zip', 'https://github.com/user/g.example/archive/master.zip')
|
|---|
| 1745 | >>> resolve_source_code('https://github.com/user/g.example') # doctest: +SKIP
|
|---|
| 1746 | ('remote_zip', 'https://github.com/user/g.example/archive/master.zip')
|
|---|
| 1747 | >>> resolve_source_code('https://github.com/user/g.example/') # doctest: +SKIP
|
|---|
| 1748 | ('remote_zip', 'https://github.com/user/g.example/archive/master.zip')
|
|---|
| 1749 |
|
|---|
| 1750 | GitLab:
|
|---|
| 1751 |
|
|---|
| 1752 | >>> resolve_source_code('gitlab.com/JoeUser/GrassModule') # doctest: +SKIP
|
|---|
| 1753 | ('remote_zip', 'https://gitlab.com/JoeUser/GrassModule/repository/archive.zip')
|
|---|
| 1754 | >>> resolve_source_code('https://gitlab.com/JoeUser/GrassModule') # doctest: +SKIP
|
|---|
| 1755 | ('remote_zip', 'https://gitlab.com/JoeUser/GrassModule/repository/archive.zip')
|
|---|
| 1756 |
|
|---|
| 1757 | Bitbucket:
|
|---|
| 1758 |
|
|---|
| 1759 | >>> resolve_source_code('bitbucket.org/joe-user/grass-module') # doctest: +SKIP
|
|---|
| 1760 | ('remote_zip', 'https://bitbucket.org/joe-user/grass-module/get/default.zip')
|
|---|
| 1761 | >>> resolve_source_code('https://bitbucket.org/joe-user/grass-module') # doctest: +SKIP
|
|---|
| 1762 | ('remote_zip', 'https://bitbucket.org/joe-user/grass-module/get/default.zip')
|
|---|
| 1763 | """
|
|---|
| 1764 | if not url and name:
|
|---|
| 1765 | module_class = get_module_class_name(name)
|
|---|
| 1766 | trac_url = 'https://trac.osgeo.org/grass/browser/grass-addons/' \
|
|---|
| 1767 | 'grass{version}/{module_class}/{module_name}?format=zip' \
|
|---|
| 1768 | .format(version=version[0],
|
|---|
| 1769 | module_class=module_class, module_name=name)
|
|---|
| 1770 | return 'official', trac_url
|
|---|
| 1771 |
|
|---|
| 1772 | # Check if URL can be found
|
|---|
| 1773 | # Catch corner case if local URL is given starting with file://
|
|---|
| 1774 | url = url[6:] if url.startswith('file://') else url
|
|---|
| 1775 | if not os.path.exists(url):
|
|---|
| 1776 | url_validated = False
|
|---|
| 1777 | if url.startswith('http'):
|
|---|
| 1778 | try:
|
|---|
| 1779 | open_url = urlopen(url)
|
|---|
| 1780 | open_url.close()
|
|---|
| 1781 | url_validated = True
|
|---|
| 1782 | except:
|
|---|
| 1783 | pass
|
|---|
| 1784 | else:
|
|---|
| 1785 | try:
|
|---|
| 1786 | open_url = urlopen('http://' + url)
|
|---|
| 1787 | open_url.close()
|
|---|
| 1788 | url_validated = True
|
|---|
| 1789 | except:
|
|---|
| 1790 | pass
|
|---|
| 1791 | try:
|
|---|
| 1792 | open_url = urlopen('https://' + url)
|
|---|
| 1793 | open_url.close()
|
|---|
| 1794 | url_validated = True
|
|---|
| 1795 | except:
|
|---|
| 1796 | pass
|
|---|
| 1797 |
|
|---|
| 1798 | if not url_validated:
|
|---|
| 1799 | grass.fatal(_('Cannot open URL: {}'.format(url)))
|
|---|
| 1800 |
|
|---|
| 1801 |
|
|---|
| 1802 | # Handle local URLs
|
|---|
| 1803 | if os.path.isdir(url):
|
|---|
| 1804 | return 'dir', os.path.abspath(url)
|
|---|
| 1805 | elif os.path.exists(url):
|
|---|
| 1806 | if url.endswith('.zip'):
|
|---|
| 1807 | return 'zip', os.path.abspath(url)
|
|---|
| 1808 | for suffix in extract_tar.supported_formats:
|
|---|
| 1809 | if url.endswith('.' + suffix):
|
|---|
| 1810 | return suffix, os.path.abspath(url)
|
|---|
| 1811 | # Handle remote URLs
|
|---|
| 1812 | else:
|
|---|
| 1813 | source, resolved_url = resolve_known_host_service(url)
|
|---|
| 1814 | if source:
|
|---|
| 1815 | return source, resolved_url
|
|---|
| 1816 | # we allow URL to end with =zip or ?zip and not only .zip
|
|---|
| 1817 | # unfortunately format=zip&version=89612 would require something else
|
|---|
| 1818 | # special option to force the source type would solve it
|
|---|
| 1819 | if url.endswith('zip'):
|
|---|
| 1820 | return 'remote_zip', url
|
|---|
| 1821 | for suffix in extract_tar.supported_formats:
|
|---|
| 1822 | if url.endswith(suffix):
|
|---|
| 1823 | return 'remote_' + suffix, url
|
|---|
| 1824 | # fallback to the classic behavior
|
|---|
| 1825 | return 'svn', url
|
|---|
| 1826 |
|
|---|
| 1827 |
|
|---|
| 1828 | def main():
|
|---|
| 1829 | # check dependencies
|
|---|
| 1830 | if not flags['a'] and sys.platform != "win32":
|
|---|
| 1831 | check_progs()
|
|---|
| 1832 |
|
|---|
| 1833 | original_url = options['url']
|
|---|
| 1834 |
|
|---|
| 1835 | # manage proxies
|
|---|
| 1836 | global PROXIES
|
|---|
| 1837 | if options['proxy']:
|
|---|
| 1838 | PROXIES = {}
|
|---|
| 1839 | for ptype, purl in (p.split('=') for p in options['proxy'].split(',')):
|
|---|
| 1840 | PROXIES[ptype] = purl
|
|---|
| 1841 | proxy = ProxyHandler(PROXIES)
|
|---|
| 1842 | opener = build_opener(proxy)
|
|---|
| 1843 | install_opener(opener)
|
|---|
| 1844 |
|
|---|
| 1845 | # define path
|
|---|
| 1846 | options['prefix'] = resolve_install_prefix(path=options['prefix'],
|
|---|
| 1847 | to_system=flags['s'])
|
|---|
| 1848 |
|
|---|
| 1849 | # list available extensions
|
|---|
| 1850 | if flags['l'] or flags['c'] or (flags['g'] and not flags['a']):
|
|---|
| 1851 | # using dummy module, we don't need any module URL now,
|
|---|
| 1852 | # but will work only as long as the function does not check
|
|---|
| 1853 | # if the URL is actually valid or something
|
|---|
| 1854 | source, url = resolve_source_code(name='dummy',
|
|---|
| 1855 | url=original_url)
|
|---|
| 1856 | xmlurl = resolve_xmlurl_prefix(original_url, source=source)
|
|---|
| 1857 | list_available_extensions(xmlurl)
|
|---|
| 1858 | return 0
|
|---|
| 1859 | elif flags['a']:
|
|---|
| 1860 | list_installed_extensions(toolboxes=flags['t'])
|
|---|
| 1861 | return 0
|
|---|
| 1862 |
|
|---|
| 1863 | if flags['d']:
|
|---|
| 1864 | if options['operation'] != 'add':
|
|---|
| 1865 | grass.warning(_("Flag 'd' is relevant only to"
|
|---|
| 1866 | " 'operation=add'. Ignoring this flag."))
|
|---|
| 1867 | else:
|
|---|
| 1868 | global REMOVE_TMPDIR
|
|---|
| 1869 | REMOVE_TMPDIR = False
|
|---|
| 1870 |
|
|---|
| 1871 | if options['operation'] == 'add':
|
|---|
| 1872 | check_dirs()
|
|---|
| 1873 | source, url = resolve_source_code(name=options['extension'],
|
|---|
| 1874 | url=original_url)
|
|---|
| 1875 | xmlurl = resolve_xmlurl_prefix(original_url, source=source)
|
|---|
| 1876 | install_extension(source=source, url=url, xmlurl=xmlurl)
|
|---|
| 1877 | else: # remove
|
|---|
| 1878 | remove_extension(force=flags['f'])
|
|---|
| 1879 |
|
|---|
| 1880 | return 0
|
|---|
| 1881 |
|
|---|
| 1882 |
|
|---|
| 1883 | if __name__ == "__main__":
|
|---|
| 1884 | if len(sys.argv) == 2 and sys.argv[1] == '--doctest':
|
|---|
| 1885 | import doctest
|
|---|
| 1886 | sys.exit(doctest.testmod().failed)
|
|---|
| 1887 | options, flags = grass.parser()
|
|---|
| 1888 | global TMPDIR
|
|---|
| 1889 | TMPDIR = tempfile.mkdtemp()
|
|---|
| 1890 | atexit.register(cleanup)
|
|---|
| 1891 |
|
|---|
| 1892 | grass_version = grass.version()
|
|---|
| 1893 | version = grass_version['version'].split('.')
|
|---|
| 1894 | build_platform = grass_version['build_platform'].split('-', 1)[0]
|
|---|
| 1895 |
|
|---|
| 1896 | sys.exit(main())
|
|---|
