run_tests.py 16.9 KB
Newer Older
Damien Naudet's avatar
Damien Naudet committed
1 2 3 4
#!/usr/bin/env python
# coding: utf-8
# /*##########################################################################
#
5
# Copyright (c) 2015-2017 European Synchrotron Radiation Facility
Damien Naudet's avatar
Damien Naudet committed
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
"""Run the tests of the project.

This script expects a suite function in <project_package>.test,
which returns a unittest.TestSuite.

Test coverage dependencies: coverage, lxml.
"""

__authors__ = ["Jérôme Kieffer", "Thomas Vincent"]
35
__date__ = "02/03/2018"
Damien Naudet's avatar
Damien Naudet committed
36 37 38 39 40 41 42 43 44
__license__ = "MIT"

import distutils.util
import logging
import os
import subprocess
import sys
import time
import unittest
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
import collections
from argparse import ArgumentParser


class StreamHandlerUnittestReady(logging.StreamHandler):
    """The unittest class TestResult redefine sys.stdout/err to capture
    stdout/err from tests and to display them only when a test fail.
    This class allow to use unittest stdout-capture by using the last sys.stdout
    and not a cached one.
    """

    def emit(self, record):
        """
        :type record: logging.LogRecord
        """
        self.stream = sys.stderr
        super(StreamHandlerUnittestReady, self).emit(record)

    def flush(self):
        pass

Damien Naudet's avatar
Damien Naudet committed
66

67 68 69 70 71 72 73 74 75 76 77 78
def createBasicHandler():
    """Create the handler using the basic configuration"""
    hdlr = StreamHandlerUnittestReady()
    fs = logging.BASIC_FORMAT
    dfs = None
    fmt = logging.Formatter(fs, dfs)
    hdlr.setFormatter(fmt)
    return hdlr


# Use an handler compatible with unittests, else use_buffer is not working
logging.root.addHandler(createBasicHandler())
79 80

# Capture all default warnings
81
logging.captureWarnings(True)
82 83
import warnings
warnings.simplefilter('default')
Damien Naudet's avatar
Damien Naudet committed
84 85

logger = logging.getLogger("run_tests")
86
logger.setLevel(logging.WARNING)
Damien Naudet's avatar
Damien Naudet committed
87

88
logger.info("Python %s %s", sys.version, tuple.__itemsize__ * 8)
Damien Naudet's avatar
Damien Naudet committed
89 90 91 92 93 94 95 96


try:
    import resource
except ImportError:
    resource = None
    logger.warning("resource module missing")

97 98 99
try:
    import importlib
    importer = importlib.import_module
100 101 102 103 104 105 106 107 108
except ImportError:
    def importer(name):
        module = __import__(name)
        # returns the leaf module, instead of the root module
        subnames = name.split(".")
        subnames.pop(0)
        for subname in subnames:
            module = getattr(module, subname)
            return module
109 110


Damien Naudet's avatar
Damien Naudet committed
111 112 113
try:
    import numpy
except Exception as error:
114
    logger.warning("Numpy missing: %s", error)
Damien Naudet's avatar
Damien Naudet committed
115
else:
116
    logger.info("Numpy %s", numpy.version.version)
Damien Naudet's avatar
Damien Naudet committed
117 118 119 120 121


try:
    import h5py
except Exception as error:
122
    logger.warning("h5py missing: %s", error)
Damien Naudet's avatar
Damien Naudet committed
123
else:
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
    logger.info("h5py %s", h5py.version.version)


def get_project_name(root_dir):
    """Retrieve project name by running python setup.py --name in root_dir.

    :param str root_dir: Directory where to run the command.
    :return: The name of the project stored in root_dir
    """
    logger.debug("Getting project name in %s", root_dir)
    p = subprocess.Popen([sys.executable, "setup.py", "--name"],
                         shell=False, cwd=root_dir, stdout=subprocess.PIPE)
    name, _stderr_data = p.communicate()
    logger.debug("subprocess ended with rc= %s", p.returncode)
    return name.split()[-1].decode('ascii')


141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
class TextTestResultWithSkipList(unittest.TextTestResult):
    """Override default TextTestResult to display list of skipped tests at the
    end
    """

    def printErrors(self):
        unittest.TextTestResult.printErrors(self)
        # Print skipped tests at the end
        self.printGroupedList("SKIPPED", self.skipped)

    def printGroupedList(self, flavour, errors):
        grouped = collections.OrderedDict()

        for test, err in errors:
            if err in grouped:
                grouped[err] = grouped[err] + [test]
            else:
                grouped[err] = [test]

        for err, tests in grouped.items():
            self.stream.writeln(self.separator1)
            for test in tests:
                self.stream.writeln("%s: %s" % (flavour, self.getDescription(test)))
            self.stream.writeln(self.separator2)
            self.stream.writeln("%s" % err)
166

Damien Naudet's avatar
Damien Naudet committed
167

168 169 170 171 172 173 174
class ProfileTextTestResult(unittest.TextTestRunner.resultclass):

    def __init__(self, *arg, **kwarg):
        unittest.TextTestRunner.resultclass.__init__(self, *arg, **kwarg)
        self.logger = logging.getLogger("memProf")
        self.logger.setLevel(min(logging.INFO, logging.root.level))
        self.logger.handlers.append(logging.FileHandler("profile.log"))
Damien Naudet's avatar
Damien Naudet committed
175 176

    def startTest(self, test):
177
        unittest.TextTestRunner.resultclass.startTest(self, test)
Damien Naudet's avatar
Damien Naudet committed
178
        if resource:
179 180
            self.__mem_start = \
                resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
Damien Naudet's avatar
Damien Naudet committed
181 182 183
        self.__time_start = time.time()

    def stopTest(self, test):
184 185 186 187 188 189
        unittest.TextTestRunner.resultclass.stopTest(self, test)
        # see issue 311. For other platform, get size of ru_maxrss in "man getrusage"
        if sys.platform == "darwin":
            ratio = 1e-6
        else:
            ratio = 1e-3
Damien Naudet's avatar
Damien Naudet committed
190
        if resource:
191 192
            memusage = (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss -
                        self.__mem_start) * ratio
Damien Naudet's avatar
Damien Naudet committed
193 194
        else:
            memusage = 0
195
        self.logger.info("Time: %.3fs \t RAM: %.3f Mb\t%s",
196 197
                         time.time() - self.__time_start,
                         memusage, test.id())
Damien Naudet's avatar
Damien Naudet committed
198 199


200
def report_rst(cov, package, version="0.0.0", base=""):
Damien Naudet's avatar
Damien Naudet committed
201 202 203 204 205
    """
    Generate a report of test coverage in RST (for Sphinx inclusion)

    :param cov: test coverage instance
    :param str package: Name of the package
206
    :param str base: base directory of modules to include in the report
Damien Naudet's avatar
Damien Naudet committed
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
    :return: RST string
    """
    import tempfile
    fd, fn = tempfile.mkstemp(suffix=".xml")
    os.close(fd)
    cov.xml_report(outfile=fn)

    from lxml import etree
    xml = etree.parse(fn)
    classes = xml.xpath("//class")

    line0 = "Test coverage report for %s" % package
    res = [line0, "=" * len(line0), ""]
    res.append("Measured on *%s* version %s, %s" %
               (package, version, time.strftime("%d/%m/%Y")))
    res += ["",
            ".. csv-table:: Test suite coverage",
            '   :header: "Name", "Stmts", "Exec", "Cover"',
            '   :widths: 35, 8, 8, 8',
            '']
    tot_sum_lines = 0
    tot_sum_hits = 0

    for cl in classes:
        name = cl.get("name")
232 233 234
        fname = cl.get("filename")
        if not os.path.abspath(fname).startswith(base):
            continue
Damien Naudet's avatar
Damien Naudet committed
235 236 237 238 239 240 241 242
        lines = cl.find("lines").getchildren()
        hits = [int(i.get("hits")) for i in lines]

        sum_hits = sum(hits)
        sum_lines = len(lines)

        cover = 100.0 * sum_hits / sum_lines if sum_lines else 0

243 244 245
        if base:
            name = os.path.relpath(fname, base)

Damien Naudet's avatar
Damien Naudet committed
246 247 248 249 250 251 252 253 254 255 256 257
        res.append('   "%s", "%s", "%s", "%.1f %%"' %
                   (name, sum_lines, sum_hits, cover))
        tot_sum_lines += sum_lines
        tot_sum_hits += sum_hits
    res.append("")
    res.append('   "%s total", "%s", "%s", "%.1f %%"' %
               (package, tot_sum_lines, tot_sum_hits,
                100.0 * tot_sum_hits / tot_sum_lines if tot_sum_lines else 0))
    res.append("")
    return os.linesep.join(res)


258 259 260 261 262 263 264 265 266 267 268 269 270 271
def is_debug_python():
    """Returns true if the Python interpreter is in debug mode."""
    try:
        import sysconfig
    except ImportError:  # pragma nocover
        # Python < 2.7
        import distutils.sysconfig as sysconfig

    if sysconfig.get_config_var("Py_DEBUG"):
        return True

    return hasattr(sys, "gettotalrefcount")


Damien Naudet's avatar
Damien Naudet committed
272 273 274 275 276 277 278 279 280 281 282 283
def build_project(name, root_dir):
    """Run python setup.py build for the project.

    Build directory can be modified by environment variables.

    :param str name: Name of the project.
    :param str root_dir: Root directory of the project
    :return: The path to the directory were build was performed
    """
    platform = distutils.util.get_platform()
    architecture = "lib.%s-%i.%i" % (platform,
                                     sys.version_info[0], sys.version_info[1])
284 285
    if is_debug_python():
        architecture += "-pydebug"
Damien Naudet's avatar
Damien Naudet committed
286 287 288 289 290 291 292 293 294

    if os.environ.get("PYBUILD_NAME") == name:
        # we are in the debian packaging way
        home = os.environ.get("PYTHONPATH", "").split(os.pathsep)[-1]
    elif os.environ.get("BUILDPYTHONPATH"):
        home = os.path.abspath(os.environ.get("BUILDPYTHONPATH", ""))
    else:
        home = os.path.join(root_dir, "build", architecture)

295
    logger.warning("Building %s to %s", name, home)
Damien Naudet's avatar
Damien Naudet committed
296 297
    p = subprocess.Popen([sys.executable, "setup.py", "build"],
                         shell=False, cwd=root_dir)
298
    logger.debug("subprocess ended with rc= %s", p.wait())
299 300 301 302 303 304

    if os.path.isdir(home):
        return home
    alt_home = os.path.join(os.path.dirname(home), "lib")
    if os.path.isdir(alt_home):
        return alt_home
Damien Naudet's avatar
Damien Naudet committed
305 306


307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
def import_project_module(project_name, project_dir):
    """Import project module, from the system of from the project directory"""
    # Prevent importing from source directory
    if (os.path.dirname(os.path.abspath(__file__)) == os.path.abspath(sys.path[0])):
        removed_from_sys_path = sys.path.pop(0)
        logger.info("Patched sys.path, removed: '%s'", removed_from_sys_path)

    if "--installed" in sys.argv:
        try:
            module = importer(project_name)
        except ImportError:
            raise ImportError(
                "%s not installed: Cannot run tests on installed version" %
                PROJECT_NAME)
    else:  # Use built source
        build_dir = build_project(project_name, project_dir)
323 324
        if build_dir is None:
            logging.error("Built project is not available !!! investigate")
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
        sys.path.insert(0, build_dir)
        logger.warning("Patched sys.path, added: '%s'", build_dir)
        module = importer(project_name)
    return module


def get_test_options(project_module):
    """Returns the test options if available, else None"""
    module_name = project_module.__name__ + '.test.utils'
    logger.info('Import %s', module_name)
    try:
        test_utils = importer(module_name)
    except ImportError:
        logger.warning("No module named '%s'. No test options available.", module_name)
        return None

    test_options = getattr(test_utils, "test_options", None)
    return test_options


345 346 347 348
if __name__ == "__main__":  # Needed for multiprocessing support on Windows
    PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
    PROJECT_NAME = get_project_name(PROJECT_DIR)
    logger.info("Project name: %s", PROJECT_NAME)
349

350 351 352
    project_module = import_project_module(PROJECT_NAME, PROJECT_DIR)
    PROJECT_VERSION = getattr(project_module, 'version', '')
    PROJECT_PATH = project_module.__path__[0]
353 354


355 356
    test_options = get_test_options(project_module)
    """Contains extra configuration for the tests."""
357

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452

    epilog = """Environment variables:
    WITH_QT_TEST=False to disable graphical tests
    SILX_OPENCL=False to disable OpenCL tests
    SILX_TEST_LOW_MEM=True to disable tests taking large amount of memory
    GPU=False to disable the use of a GPU with OpenCL test
    WITH_GL_TEST=False to disable tests using OpenGL
    """
    parser = ArgumentParser(description='Run the tests.',
                            epilog=epilog)

    parser.add_argument("--installed",
                        action="store_true", dest="installed", default=False,
                        help=("Test the installed version instead of" +
                              "building from the source"))
    parser.add_argument("-c", "--coverage", dest="coverage",
                        action="store_true", default=False,
                        help=("Report code coverage" +
                              "(requires 'coverage' and 'lxml' module)"))
    parser.add_argument("-m", "--memprofile", dest="memprofile",
                        action="store_true", default=False,
                        help="Report memory profiling")
    parser.add_argument("-v", "--verbose", default=0,
                        action="count", dest="verbose",
                        help="Increase verbosity. Option -v prints additional " +
                             "INFO messages. Use -vv for full verbosity, " +
                             "including debug messages and test help strings.")
    parser.add_argument("--qt-binding", dest="qt_binding", default=None,
                        help="Force using a Qt binding, from 'PyQt4', 'PyQt5', or 'PySide'")
    if test_options is not None:
        test_options.add_parser_argument(parser)

    default_test_name = "%s.test.suite" % PROJECT_NAME
    parser.add_argument("test_name", nargs='*',
                        default=(default_test_name,),
                        help="Test names to run (Default: %s)" % default_test_name)
    options = parser.parse_args()
    sys.argv = [sys.argv[0]]


    test_verbosity = 1
    use_buffer = True
    if options.verbose == 1:
        logging.root.setLevel(logging.INFO)
        logger.info("Set log level: INFO")
        test_verbosity = 2
        use_buffer = False
    elif options.verbose > 1:
        logging.root.setLevel(logging.DEBUG)
        logger.info("Set log level: DEBUG")
        test_verbosity = 2
        use_buffer = False

    if options.coverage:
        logger.info("Running test-coverage")
        import coverage
        omits = ["*test*", "*third_party*", "*/setup.py",
                 # temporary test modules (silx.math.fit.test.test_fitmanager)
                 "*customfun.py", ]
        try:
            cov = coverage.Coverage(omit=omits)
        except AttributeError:
            cov = coverage.coverage(omit=omits)
        cov.start()

    if options.qt_binding:
        binding = options.qt_binding.lower()
        if binding == "pyqt4":
            logger.info("Force using PyQt4")
            if sys.version < "3.0.0":
                try:
                    import sip
                    sip.setapi("QString", 2)
                    sip.setapi("QVariant", 2)
                except Exception:
                    logger.warning("Cannot set sip API")
            import PyQt4.QtCore  # noqa
        elif binding == "pyqt5":
            logger.info("Force using PyQt5")
            import PyQt5.QtCore  # noqa
        elif binding == "pyside":
            logger.info("Force using PySide")
            import PySide.QtCore  # noqa
        elif binding == "pyside2":
            logger.info("Force using PySide2")
            import PySide2.QtCore  # noqa
        else:
            raise ValueError("Qt binding '%s' is unknown" % options.qt_binding)

    # Run the tests
    runnerArgs = {}
    runnerArgs["verbosity"] = test_verbosity
    runnerArgs["buffer"] = use_buffer
    if options.memprofile:
        runnerArgs["resultclass"] = ProfileTextTestResult
453
    else:
454 455
        runnerArgs["resultclass"] = TextTestResultWithSkipList
    runner = unittest.TextTestRunner(**runnerArgs)
Damien Naudet's avatar
Damien Naudet committed
456

457 458
    logger.warning("Test %s %s from %s",
                   PROJECT_NAME, PROJECT_VERSION, PROJECT_PATH)
459

460 461 462 463
    test_module_name = PROJECT_NAME + '.test'
    logger.info('Import %s', test_module_name)
    test_module = importer(test_module_name)
    test_suite = unittest.TestSuite()
464

465 466 467 468 469
    if test_options is not None:
        # Configure the test options according to the command lines and the the environment
        test_options.configure(options)
    else:
        logger.warning("No test options available.")
470 471


472 473 474 475 476 477 478 479
    if not options.test_name:
        # Do not use test loader to avoid cryptic exception
        # when an error occur during import
        project_test_suite = getattr(test_module, 'suite')
        test_suite.addTest(project_test_suite())
    else:
        test_suite.addTest(
            unittest.defaultTestLoader.loadTestsFromNames(options.test_name))
Damien Naudet's avatar
Damien Naudet committed
480

481 482
    # Display the result when using CTRL-C
    unittest.installHandler()
Damien Naudet's avatar
Damien Naudet committed
483

484
    result = runner.run(test_suite)
485

486 487 488 489
    if result.wasSuccessful():
        exit_status = 0
    else:
        exit_status = 1
Damien Naudet's avatar
Damien Naudet committed
490 491


492 493 494 495 496
    if options.coverage:
        cov.stop()
        cov.save()
        with open("coverage.rst", "w") as fn:
            fn.write(report_rst(cov, PROJECT_NAME, PROJECT_VERSION, PROJECT_PATH))
Damien Naudet's avatar
Damien Naudet committed
497

498
    sys.exit(exit_status)