From f22ebaf7f14925bd5d1230072dce2026d6aaf0e8 Mon Sep 17 00:00:00 2001 From: Alexander Afanasyev Date: Thu, 17 Dec 2015 21:23:05 -0800 Subject: [PATCH] tests+ci: Enable dual XML and HRF output of unit test results This commit also makes unit tests to run against a debug build, to ensure assertions are properly evaluated (based on suggestion in I5f4419ea48d4eb333e9d107115ef3df4123f76e5). Change-Id: I1bbf14f75aab155ed80a36fc4006f7a5d13f1289 Refs: #2252, #2805 --- .jenkins.d/01-ndn-cxx.sh | 2 +- .jenkins.d/10-build.sh | 17 +-- .jenkins.d/20-tests.sh | 31 ++-- .jenkins.d/30-coverage.sh | 5 +- .jenkins.d/README.md | 3 + .waf-tools/coverage.py | 8 +- tests/boost-multi-log-formatter.hpp | 214 ++++++++++++++++++++++++++++ tests/main.cpp | 81 ++++++++++- tests/other/wscript | 24 +++- tests/wscript | 73 ++++------ 10 files changed, 379 insertions(+), 79 deletions(-) create mode 100644 tests/boost-multi-log-formatter.hpp diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh index d22bfb68..2bc46fdd 100755 --- a/.jenkins.d/01-ndn-cxx.sh +++ b/.jenkins.d/01-ndn-cxx.sh @@ -5,7 +5,7 @@ set -e JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "$JDIR"/util.sh -pushd /tmp >/dev/null +pushd ${CACHE_DIR:-/tmp} >/dev/null INSTALLED_VERSION=$((cd ndn-cxx && git rev-parse HEAD) 2>/dev/null || echo NONE) diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh index 1c4fdf28..09fa916d 100755 --- a/.jenkins.d/10-build.sh +++ b/.jenkins.d/10-build.sh @@ -6,30 +6,31 @@ git submodule init git submodule sync git submodule update -COVERAGE=$( python -c "print '--with-coverage' if 'code-coverage' in '$JOB_NAME' else ''" ) - # Cleanup sudo ./waf -j1 --color=yes distclean -# Configure/build in debug mode -./waf -j1 --color=yes configure --with-tests --debug +# Configure/build in optimized mode with tests and precompiled headers +./waf -j1 --color=yes configure --with-tests ./waf -j1 --color=yes build # Cleanup sudo ./waf -j1 --color=yes distclean -# Configure/build in optimized mode without tests with precompiled headers +# Configure/build in optimized mode without tests and with precompiled headers ./waf -j1 --color=yes configure ./waf -j1 --color=yes build # Cleanup sudo ./waf -j1 --color=yes distclean -# Configure/build in optimized mode -./waf -j1 --color=yes configure --with-tests --without-pch $COVERAGE +# Configure/build in debug mode +if [[ "$JOB_NAME" == *"code-coverage" ]]; then + COVERAGE="--with-coverage" +fi +./waf -j1 --color=yes configure --debug --with-tests --without-pch $COVERAGE ./waf -j1 --color=yes build -# (tests will be run against optimized version) +# (tests will be run against debug version) # Install sudo ./waf -j1 --color=yes install diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh index b4bf79aa..dcd2b9d5 100755 --- a/.jenkins.d/20-tests.sh +++ b/.jenkins.d/20-tests.sh @@ -2,19 +2,19 @@ set -x set -e +JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source "$JDIR"/util.sh + # Prepare environment rm -Rf ~/.ndnx ~/.ndn -echo $NODE_LABELS -IS_OSX=$( python -c "print 'yes' if 'OSX' in '$NODE_LABELS'.strip().split(' ') else 'no'" ) -IS_LINUX=$( python -c "print 'yes' if 'Linux' in '$NODE_LABELS'.strip().split(' ') else 'no'" ) - -if [[ $IS_OSX == "yes" ]]; then +if has OSX $NODE_LABELS; then security unlock-keychain -p "named-data" sudo chgrp admin /dev/bpf* sudo chmod g+rw /dev/bpf* fi -if [[ $IS_LINUX = "yes" ]]; then + +if has Linux $NODE_LABELS; then sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-core || true sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-daemon || true sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-rib || true @@ -24,11 +24,18 @@ ndnsec-keygen "/tmp/jenkins/$NODE_NAME" | ndnsec-install-cert - # Run unit tests # Core -./build/unit-tests-core -l test_suite -sudo ./build/unit-tests-core -t TestPrivilegeHelper -l test_suite +if [[ -n $XUNIT ]]; then + ./build/unit-tests-core -l all --log_format2=XML --log_sink2=build/xunit-core-report.xml + sudo ./build/unit-tests-core -t TestPrivilegeHelper -l all --log_format2=XML --log_sink2=build/xunit-core-sudo-report.xml -# Daemon -./build/unit-tests-daemon -l test_suite + ./build/unit-tests-daemon -l all --log_format2=XML --log_sink2=build/xunit-daemon-report.xml -# RIB -./build/unit-tests-rib -l test_suite + ./build/unit-tests-rib -l all --log_format2=XML --log_sink2=build/xunit-rib-report.xml +else + ./build/unit-tests-core -l test_suite + sudo ./build/unit-tests-core -t TestPrivilegeHelper -l test_suite + + ./build/unit-tests-daemon -l test_suite + + ./build/unit-tests-rib -l test_suite +fi diff --git a/.jenkins.d/30-coverage.sh b/.jenkins.d/30-coverage.sh index e4628847..a00f8f13 100755 --- a/.jenkins.d/30-coverage.sh +++ b/.jenkins.d/30-coverage.sh @@ -2,9 +2,10 @@ set -x set -e -IS_COVR=$( python -c "print 'yes' if 'code-coverage' in '$JOB_NAME' else 'no'" ) +JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source "$JDIR"/util.sh -if [[ $IS_COVR == "yes" ]]; then +if [[ "$JOB_NAME" == *"code-coverage" ]]; then BASE="`pwd | sed -e 's|/|\\\/|g'`\\" (cd build && gcovr -x -f $BASE/core -f $BASE/daemon -f $BASE/rib -r ../ -o coverage.xml ./) fi diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md index 956ae186..3bacba8c 100644 --- a/.jenkins.d/README.md +++ b/.jenkins.d/README.md @@ -28,3 +28,6 @@ Environment Variables Used in Build Scripts * empty: default build process * `code-coverage` (Linux OS is assumed): build process with code coverage analysis + +- `CACHE_DIR`: the variable defines a path to folder containing cached files from previous builds, + e.g., a compiled version of ndn-cxx library. If not set, `/tmp` is used. diff --git a/.waf-tools/coverage.py b/.waf-tools/coverage.py index 0a3db656..ce928838 100644 --- a/.waf-tools/coverage.py +++ b/.waf-tools/coverage.py @@ -1,10 +1,6 @@ # -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- -# -# Copyright (c) 2014, Regents of the University of California -# -# GPL 3.0 license, see the COPYING.md file for more information -from waflib import TaskGen +from waflib import TaskGen, Logs def options(opt): opt.add_option('--with-coverage', action='store_true', default=False, dest='with_coverage', @@ -12,6 +8,8 @@ def options(opt): def configure(conf): if conf.options.with_coverage: + if not conf.options.debug: + conf.fatal("Code coverage flags require debug mode compilation (add --debug)") conf.check_cxx(cxxflags=['-fprofile-arcs', '-ftest-coverage', '-fPIC'], linkflags=['-fprofile-arcs'], uselib_store='GCOV', mandatory=True) diff --git a/tests/boost-multi-log-formatter.hpp b/tests/boost-multi-log-formatter.hpp new file mode 100644 index 00000000..ae37416b --- /dev/null +++ b/tests/boost-multi-log-formatter.hpp @@ -0,0 +1,214 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/** + * Copyright (c) 2015 Regents of the University of California. + * + * Based on work by Martin Ba (http://stackoverflow.com/a/26718189) + * + * This file is distributed under the Boost Software License, Version 1.0. + * (See http://www.boost.org/LICENSE_1_0.txt) + */ + +#ifndef NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP +#define NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP + +#include + +#if BOOST_VERSION >= 105900 +#include +#else +#include +#endif // BOOST_VERSION >= 105900 + +#include +#include +#include + +namespace boost { +namespace unit_test { +namespace output { + +/** + * @brief Log formatter for Boost.Test that outputs the logging to multiple formatters + * + * The log formatter is designed to output to one or multiple formatters at the same time. For + * example, one HRF formatter can output to the standard output, while XML formatter output to + * the file. + * + * Usage: + * + * // Call in init_unit_test_suite: (this will override the --log_format parameter) + * auto formatter = new boost::unit_test::output::multi_log_formatter; // same as already configured logger + * + * // Prepare and add additional logger(s) + * formatter.add(std::make_shared(), + * std::make_shared("out.xml")); + * + * boost::unit_test::unit_test_log.set_formatter(formatter); + * + * @note Calling `boost::unit_test::unit_test_log.set_stream(...)` will change the stream for + * the original logger. + */ +class multi_log_formatter : public unit_test_log_formatter +{ +public: + /** + * @brief Create instance of the logger, based on the configured logger instance + */ + multi_log_formatter() + { + auto format = +#if BOOST_VERSION > 105900 + runtime_config::get(runtime_config::LOG_FORMAT); +#else + runtime_config::log_format(); +#endif // BOOST_VERSION > 105900 + + switch (format) { + default: +#if BOOST_VERSION >= 105900 + case OF_CLF: +#else + case CLF: +#endif // BOOST_VERSION >= 105900 + m_loggers.push_back({std::make_shared(), nullptr}); + break; +#if BOOST_VERSION >= 105900 + case OF_XML: +#else + case XML: +#endif // BOOST_VERSION >= 105900 + m_loggers.push_back({std::make_shared(), nullptr}); + break; + } + } + + void + add(std::shared_ptr formatter, std::shared_ptr os) + { + m_loggers.push_back({formatter, os}); + } + + // Formatter interface + void + log_start(std::ostream& os, counter_t test_cases_amount) + { + for (auto& l : m_loggers) + l.logger->log_start(l.os == nullptr ? os : *l.os, test_cases_amount); + } + + void + log_finish(std::ostream& os) + { + for (auto& l : m_loggers) + l.logger->log_finish(l.os == nullptr ? os : *l.os); + } + + void + log_build_info(std::ostream& os) + { + for (auto& l : m_loggers) + l.logger->log_build_info(l.os == nullptr ? os : *l.os); + } + + void + test_unit_start(std::ostream& os, const test_unit& tu) + { + for (auto& l : m_loggers) + l.logger->test_unit_start(l.os == nullptr ? os : *l.os, tu); + } + + void + test_unit_finish(std::ostream& os, const test_unit& tu, unsigned long elapsed) + { + for (auto& l : m_loggers) + l.logger->test_unit_finish(l.os == nullptr ? os : *l.os, tu, elapsed); + } + + void + test_unit_skipped(std::ostream& os, const test_unit& tu) + { + for (auto& l : m_loggers) + l.logger->test_unit_skipped(l.os == nullptr ? os : *l.os, tu); + } + +#if BOOST_VERSION >= 105900 + void + log_exception_start(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex) + { + for (auto& l : m_loggers) + l.logger->log_exception_start(l.os == nullptr ? os : *l.os, lcd, ex); + } + + void + log_exception_finish(std::ostream& os) + { + for (auto& l : m_loggers) + l.logger->log_exception_finish(l.os == nullptr ? os : *l.os); + } +#else + void + log_exception(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex) + { + for (auto& l : m_loggers) + l.logger->log_exception(l.os == nullptr ? os : *l.os, lcd, ex); + } +#endif // BOOST_VERSION >= 105900 + + void + log_entry_start(std::ostream& os, const log_entry_data& entry_data, log_entry_types let) + { + for (auto& l : m_loggers) + l.logger->log_entry_start(l.os == nullptr ? os : *l.os, entry_data, let); + } + + void + log_entry_value(std::ostream& os, const_string value) + { + for (auto& l : m_loggers) + l.logger->log_entry_value(l.os == nullptr ? os : *l.os, value); + } + + void + log_entry_finish(std::ostream& os) + { + for (auto& l : m_loggers) + l.logger->log_entry_finish(l.os == nullptr ? os : *l.os); + } + +#if BOOST_VERSION >= 105900 + void + entry_context_start(std::ostream& os, log_level level) + { + for (auto& l : m_loggers) + l.logger->entry_context_start(l.os == nullptr ? os : *l.os, level); + } + + void + log_entry_context(std::ostream& os, const_string value) + { + for (auto& l : m_loggers) + l.logger->log_entry_context(l.os == nullptr ? os : *l.os, value); + } + + void + entry_context_finish(std::ostream& os) + { + for (auto& l : m_loggers) + l.logger->entry_context_finish(l.os == nullptr ? os : *l.os); + } +#endif // BOOST_VERSION >= 105900 + +private: + struct LoggerInfo + { + std::shared_ptr logger; + std::shared_ptr os; + }; + std::vector m_loggers; +}; + +} // namespace output +} // namespace unit_test +} // namespace boost + +#endif // NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP diff --git a/tests/main.cpp b/tests/main.cpp index 27336346..49c3e9cb 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -22,7 +22,84 @@ * NFD, e.g., in COPYING.md file. If not, see . **/ -#define BOOST_TEST_MAIN 1 -#define BOOST_TEST_DYN_LINK 1 +#define BOOST_TEST_NO_MAIN +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_ALTERNATIVE_INIT_API #include "boost-test.hpp" +#include "boost-multi-log-formatter.hpp" + +#include +#include +#include + +#include +#include + +static bool +init_tests() +{ + init_unit_test(); + + namespace po = boost::program_options; + namespace ut = boost::unit_test; + + po::options_description extraOptions; + std::string logger; + std::string outputFile = "-"; + extraOptions.add_options() + ("log_format2", po::value(&logger), "Type of second log formatter: HRF or XML") + ("log_sink2", po::value(&outputFile)->default_value(outputFile), "Second log sink, - for stdout") + ; + po::variables_map vm; + try { + po::store(po::command_line_parser(ut::framework::master_test_suite().argc, + ut::framework::master_test_suite().argv) + .options(extraOptions) + .run(), + vm); + po::notify(vm); + } + catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << "\n" + << extraOptions << std::endl; + return false; + } + + if (vm.count("log_format2") == 0) { + // second logger is not configured + return true; + } + + std::shared_ptr formatter; + if (logger == "XML") { + formatter = std::make_shared(); + } + else if (logger == "HRF") { + formatter = std::make_shared(); + } + else { + std::cerr << "ERROR: only HRF or XML log formatter can be specified" << std::endl; + return false; + } + + std::shared_ptr output; + if (outputFile == "-") { + output = std::shared_ptr(&std::cout, std::bind([]{})); + } + else { + output = std::make_shared(outputFile.c_str()); + } + + auto multiFormatter = new ut::output::multi_log_formatter; + multiFormatter->add(formatter, output); + ut::unit_test_log.set_formatter(multiFormatter); + + return true; +} + +int +main(int argc, char* argv[]) +{ + return ::boost::unit_test::unit_test_main(&init_tests, argc, argv); +} diff --git a/tests/other/wscript b/tests/other/wscript index 51f8b6be..074e72ce 100644 --- a/tests/other/wscript +++ b/tests/other/wscript @@ -27,8 +27,22 @@ NFD, e.g., in COPYING.md file. If not, see . top = '../..' def build(bld): - bld.program(target="../../cs-benchmark", - source="cs-benchmark.cpp", - use='daemon-objects unit-tests-main', - install_path=None, - ) + for module, name in {"cs-benchmark": "CS Benchmark"}.iteritems(): + # main() + bld(target='unit-tests-%s-main' % module, + name='unit-tests-%s-main' % module, + features='cxx', + use='BOOST', + source='../main.cpp', + defines=['BOOST_TEST_MODULE=%s' % name] + ) + + # unit-tests-%module + bld.program( + target='../../%s' % module, + features='cxx cxxprogram', + source=bld.path.ant_glob(['%s*.cpp' % module]), + use='daemon-objects unit-tests-base unit-tests-%s-main' % module, + includes='.', + install_path=None, + ) diff --git a/tests/wscript b/tests/wscript index 7cfe4d26..823d25ea 100644 --- a/tests/wscript +++ b/tests/wscript @@ -28,14 +28,6 @@ top = '..' def build(bld): # Unit tests if bld.env['WITH_TESTS']: - # main() - unit_test_main = bld( - target='unit-tests-main', - name='unit-tests-main', - features='cxx', - use='core-objects', - source='main.cpp', - ) # common test modules unit_test_base = bld( @@ -47,46 +39,39 @@ def build(bld): headers='../common.hpp boost-test.hpp', ) - # core tests - unit_tests_core = bld.program( - target='../unit-tests-core', - features='cxx cxxprogram', - source=bld.path.ant_glob(['core/**/*.cpp']), - use='core-objects unit-tests-base unit-tests-main', - includes='.', - install_path=None, - ) + for module, name in {"core": "NFD Core Tests", + "daemon": "NFD Daemon Tests", + "rib": "NFD RIB Tests"}.iteritems(): + # main() + bld(target='unit-tests-%s-main' % module, + name='unit-tests-%s-main' % module, + features='cxx', + use='BOOST', + source='main.cpp', + defines=['BOOST_TEST_MODULE=%s' % name] + ) - # NFD tests - unit_tests_nfd = bld.program( - target='../unit-tests-daemon', - features='cxx cxxprogram', - source=bld.path.ant_glob(['daemon/**/*.cpp'], - excl=['daemon/face/ethernet*.cpp', - 'daemon/face/unix*.cpp', - 'daemon/face/websocket*.cpp']), - use='daemon-objects unit-tests-base unit-tests-main', - includes='.', - install_path=None, - ) + # unit-tests-%module + unit_tests = bld.program( + target='../unit-tests-%s' % module, + features='cxx cxxprogram', + source=bld.path.ant_glob(['%s/**/*.cpp' % module], + excl=['%s/**/ethernet*.cpp' % module, + '%s/**/unix*.cpp' % module, + '%s/**/websocket*.cpp' % module]), + use='%s-objects unit-tests-base unit-tests-%s-main' % (module, module), + includes='.', + install_path=None, + ) - if bld.env['HAVE_LIBPCAP']: - unit_tests_nfd.source += bld.path.ant_glob('daemon/face/ethernet*.cpp') + if bld.env['HAVE_LIBPCAP']: + unit_tests.source += bld.path.ant_glob('%s/**/ethernet*.cpp' % module) - if bld.env['HAVE_UNIX_SOCKETS']: - unit_tests_nfd.source += bld.path.ant_glob('daemon/face/unix*.cpp') + if bld.env['HAVE_UNIX_SOCKETS']: + unit_tests.source += bld.path.ant_glob('%s/**/unix*.cpp' % module) - if bld.env['HAVE_WEBSOCKET']: - unit_tests_nfd.source += bld.path.ant_glob('daemon/face/websocket*.cpp') - - unit_tests_rib = bld.program( - target='../unit-tests-rib', - features='cxx cxxprogram', - source=bld.path.ant_glob(['rib/**/*.cpp']), - use='rib-objects unit-tests-base unit-tests-main', - includes='.', - install_path=None, - ) + if bld.env['HAVE_WEBSOCKET']: + unit_tests.source += bld.path.ant_glob('%s/**/websocket*.cpp' % module) # Other tests (e.g., stress tests that can be enabled even if unit tests are disabled) if bld.env['WITH_TESTS'] or bld.env['WITH_OTHER_TESTS']: