From bd3642f31d4939feb41c5ac02e6569b1af322875 Mon Sep 17 00:00:00 2001 From: Davide Pesavento Date: Fri, 12 Aug 2022 18:19:31 -0400 Subject: [PATCH] ci: auto-detect the OS instead of relying on NODE_LABELS And remove WAF_JOBS: waf already defaults to the number of available CPUs, which is appropriate in most cases, and we can always use the built-in JOBS variable to override it where necessary. Change-Id: Id132b9ef8467a563694222ccc64a676cdc076e17 --- .github/workflows/ci.yml | 5 --- .github/workflows/docs.yml | 5 --- .jenkins | 32 +++++++++++++----- .jenkins.d/00-deps.sh | 67 ++++++++++++++++++-------------------- .jenkins.d/01-ndn-cxx.sh | 16 ++++----- .jenkins.d/10-build.sh | 15 ++++----- .jenkins.d/20-tests.sh | 11 ++++--- .jenkins.d/30-coverage.sh | 2 +- .jenkins.d/README.md | 40 ++++++++++++++--------- .jenkins.d/util.sh | 39 ---------------------- manpages/conf.py | 44 +++++-------------------- 11 files changed, 112 insertions(+), 164 deletions(-) delete mode 100644 .jenkins.d/util.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d9a6a..15841e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,6 @@ jobs: runs-on: ${{ matrix.os }} env: CXX: ${{ matrix.compiler }} - NODE_LABELS: Linux Ubuntu - WAF_JOBS: 2 steps: - name: Install C++ compiler run: | @@ -52,9 +50,6 @@ jobs: - xcode: '13.2' os: macos-11 runs-on: ${{ matrix.os }} - env: - NODE_LABELS: OSX - WAF_JOBS: 3 steps: - name: Set up Xcode uses: maxim-lobanov/setup-xcode@v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a5b4c30..37b8c1c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,16 +18,11 @@ jobs: runs-on: ${{ matrix.os }} env: JOB_NAME: Docs - WAF_JOBS: 3 steps: - name: Checkout uses: actions/checkout@v3 - name: Install dependencies run: | - case ${RUNNER_OS} in - Linux) export NODE_LABELS="Linux Ubuntu" ;; - macOS) export NODE_LABELS="OSX" ;; - esac find .jenkins.d/ -type f -name '[1-9]*.sh' -exec chmod -x '{}' + ./.jenkins - name: Build documentation diff --git a/.jenkins b/.jenkins index a9abf6d..ee16e29 100755 --- a/.jenkins +++ b/.jenkins @@ -1,14 +1,30 @@ #!/usr/bin/env bash -set -e -source .jenkins.d/util.sh +set -eo pipefail + +case $(uname) in + Linux) + if [[ -e /etc/os-release ]]; then + source /etc/os-release + else + source /usr/lib/os-release + fi + export ID VERSION_ID + export ID_LIKE="${ID} ${ID_LIKE} linux" + export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" + ;; + Darwin) + # Emulate a subset of os-release(5) + export ID=macos + export VERSION_ID=$(sw_vers -productVersion) + if [[ -x /opt/homebrew/bin/brew ]]; then + eval "$(/opt/homebrew/bin/brew shellenv)" + elif [[ -x /usr/local/bin/brew ]]; then + eval "$(/usr/local/bin/brew shellenv)" + fi + ;; +esac -if has Linux $NODE_LABELS; then - export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" -elif [[ -x /opt/homebrew/bin/brew ]]; then - eval "$(/opt/homebrew/bin/brew shellenv)" -fi export CACHE_DIR=${CACHE_DIR:-/tmp} -export WAF_JOBS=${WAF_JOBS:-1} [[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes for file in .jenkins.d/*; do diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh index 5a9c2f3..8d84e1b 100755 --- a/.jenkins.d/00-deps.sh +++ b/.jenkins.d/00-deps.sh @@ -1,47 +1,44 @@ #!/usr/bin/env bash -set -ex +set -eo pipefail -if has OSX $NODE_LABELS; then - FORMULAE=(boost openssl pkg-config) +APT_PKGS=(build-essential pkg-config python3-minimal + libboost-all-dev libssl-dev libsqlite3-dev + libpcap-dev) +FORMULAE=(boost openssl pkg-config) +PIP_PKGS=() +case $JOB_NAME in + *code-coverage) + APT_PKGS+=(lcov python3-pip) + PIP_PKGS+=('gcovr~=5.2') + ;; + *Docs) + APT_PKGS+=(python3-pip) + PIP_PKGS+=(sphinx) + ;; +esac +set -x + +if [[ $ID == macos ]]; then if [[ -n $GITHUB_ACTIONS ]]; then - # GitHub Actions runners have a large number of pre-installed - # Homebrew packages. Don't waste time upgrading all of them. - brew list --versions "${FORMULAE[@]}" || brew update - for FORMULA in "${FORMULAE[@]}"; do - brew list --versions "$FORMULA" || brew install "$FORMULA" - done - # Ensure /usr/local/opt/openssl exists - brew reinstall openssl - else - brew update - brew upgrade - brew install "${FORMULAE[@]}" - brew cleanup + export HOMEBREW_NO_INSTALL_UPGRADE=1 + fi + brew update + brew install --formula "${FORMULAE[@]}" + + if (( ${#PIP_PKGS[@]} )); then + pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" fi - if [[ $JOB_NAME == *"Docs" ]]; then - pip3 install --upgrade --upgrade-strategy=eager sphinx - fi - -elif has Ubuntu $NODE_LABELS; then +elif [[ $ID_LIKE == *debian* ]]; then sudo apt-get -qq update - sudo apt-get -qy install build-essential pkg-config python3-minimal \ - libboost-all-dev libssl-dev libsqlite3-dev \ - libpcap-dev + sudo apt-get -qy install "${APT_PKGS[@]}" - case $JOB_NAME in - *code-coverage) - sudo apt-get -qy install lcov python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager 'gcovr~=5.1' - ;; - *Docs) - sudo apt-get -qy install python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager sphinx - ;; - esac + if (( ${#PIP_PKGS[@]} )); then + pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" + fi -elif has CentOS $NODE_LABELS; then +elif [[ $ID_LIKE == *fedora* ]]; then sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \ boost-devel openssl-devel sqlite-devel \ libpcap-devel diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh index 5da6d35..91ebefd 100755 --- a/.jenkins.d/01-ndn-cxx.sh +++ b/.jenkins.d/01-ndn-cxx.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -set -ex +set -exo pipefail pushd "$CACHE_DIR" >/dev/null INSTALLED_VERSION= -if has OSX $NODE_LABELS; then - BOOST=$(brew ls --versions boost) +if [[ $ID == macos ]]; then + BOOST=$(brew list --formula --versions boost) OLD_BOOST=$(cat boost.txt || :) if [[ $OLD_BOOST != $BOOST ]]; then echo "$BOOST" > boost.txt @@ -35,16 +35,16 @@ sudo rm -f /usr/local/lib{,64}/pkgconfig/libndn-cxx.pc pushd ndn-cxx >/dev/null -./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain -./waf --color=yes build -j$WAF_JOBS -sudo_preserve_env PATH -- ./waf --color=yes install +./waf --color=yes configure --without-osx-keychain +./waf --color=yes build +sudo ./waf --color=yes install popd >/dev/null popd >/dev/null -if has CentOS $NODE_LABELS; then +if [[ $ID_LIKE == *fedora* ]]; then sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64 fi -if has Linux $NODE_LABELS; then +if [[ $ID_LIKE == *linux* ]]; then sudo ldconfig fi diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh index 2a60f0d..199ed20 100755 --- a/.jenkins.d/10-build.sh +++ b/.jenkins.d/10-build.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -set -ex - -git submodule sync -git submodule update --init +set -eo pipefail if [[ -z $DISABLE_ASAN ]]; then ASAN="--with-sanitizer=address" @@ -11,17 +8,19 @@ if [[ $JOB_NAME == *"code-coverage" ]]; then COVERAGE="--with-coverage" fi +set -x + if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then # Build in release mode with tests ./waf --color=yes configure --with-tests - ./waf --color=yes build -j$WAF_JOBS + ./waf --color=yes build # Cleanup ./waf --color=yes distclean # Build in release mode without tests ./waf --color=yes configure - ./waf --color=yes build -j$WAF_JOBS + ./waf --color=yes build # Cleanup ./waf --color=yes distclean @@ -29,9 +28,9 @@ fi # Build in debug mode with tests ./waf --color=yes configure --debug --with-tests $ASAN $COVERAGE -./waf --color=yes build -j$WAF_JOBS +./waf --color=yes build # (tests will be run against the debug version) # Install -sudo_preserve_env PATH -- ./waf --color=yes install +sudo ./waf --color=yes install diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh index 81c00b1..ab14284 100755 --- a/.jenkins.d/20-tests.sh +++ b/.jenkins.d/20-tests.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -set -ex - -# Prepare environment -rm -rf ~/.ndn +set -eo pipefail # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags ASAN_OPTIONS="color=always" @@ -14,10 +11,16 @@ ASAN_OPTIONS+=":detect_invalid_pointer_pairs=2" ASAN_OPTIONS+=":strip_path_prefix=${PWD}/" export ASAN_OPTIONS +# https://www.boost.org/doc/libs/release/libs/test/doc/html/boost_test/runtime_config/summary.html export BOOST_TEST_BUILD_INFO=1 export BOOST_TEST_COLOR_OUTPUT=1 export BOOST_TEST_DETECT_MEMORY_LEAK=0 export BOOST_TEST_LOGGER=HRF,test_suite,stdout:XML,all,build/xunit-log.xml +set -x + +# Prepare environment +rm -rf ~/.ndn + # Run unit tests ./build/unit-tests diff --git a/.jenkins.d/30-coverage.sh b/.jenkins.d/30-coverage.sh index 34e082a..aae00f5 100755 --- a/.jenkins.d/30-coverage.sh +++ b/.jenkins.d/30-coverage.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -ex +set -exo pipefail if [[ $JOB_NAME == *"code-coverage" ]]; then # Generate an XML report (Cobertura format) and a detailed HTML report using gcovr diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md index e8dbf37..385af34 100644 --- a/.jenkins.d/README.md +++ b/.jenkins.d/README.md @@ -1,28 +1,36 @@ -# CONTINUOUS INTEGRATION SCRIPTS +# Continuous Integration Scripts -## Environment Variables Used in Build Scripts +## Environment Variables -- `NODE_LABELS`: space-separated list of platform properties. The included values are used by - the build scripts to select the proper behavior for different operating systems and versions. +- `ID`: lower-case string that identifies the operating system, for example: `ID=ubuntu`, + `ID=centos`. See [os-release(5)] for more information. On macOS, where `os-release` is + not available, we emulate it by setting `ID=macos`. - The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`. +- `ID_LIKE`: space-separated list of operating system identifiers that are closely related + to the running OS. See [os-release(5)] for more information. The listed values are used + by the CI scripts to select the proper behavior for different platforms and OS flavors. - Example values: + Examples: - - `[OS_TYPE]`: `Linux`, `OSX` - - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS` - - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15` + - On CentOS, `ID_LIKE="centos rhel fedora linux"` + - On Ubuntu, `ID_LIKE="ubuntu debian linux"` -- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type, - the build scripts can perform different tasks. +- `VERSION_ID`: identifies the operating system version, excluding any release code names. + See [os-release(5)] for more information. Examples: `VERSION_ID=42`, `VERSION_ID=22.04`. - Possible values: +- `JOB_NAME`: defines the type of the current CI job. Depending on the job type, the CI + scripts can perform different tasks. + + Supported values: - empty: default build task - - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed) + - `code-coverage`: debug build with tests and code coverage analysis - `limited-build`: only a single debug build with tests -- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version - of ndn-cxx. If not set, `/tmp` is used. +- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled + version of ndn-cxx. If not set, `/tmp` is used. -- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1. +- `DISABLE_ASAN`: disable building with AddressSanitizer. This is automatically set for + the `code-coverage` job type. + +[os-release(5)]: https://www.freedesktop.org/software/systemd/man/os-release.html diff --git a/.jenkins.d/util.sh b/.jenkins.d/util.sh deleted file mode 100644 index 8077a74..0000000 --- a/.jenkins.d/util.sh +++ /dev/null @@ -1,39 +0,0 @@ -has() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local p=$1 - shift - local i ret=1 - for i in "$@"; do - if [[ "${i}" == "${p}" ]]; then - ret=0 - break - fi - done - - set ${saved_xtrace} - return ${ret} -} -export -f has - -sudo_preserve_env() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local vars=() - while [[ $# -gt 0 ]]; do - local arg=$1 - shift - case ${arg} in - --) break ;; - *) vars+=("${arg}=${!arg}") ;; - esac - done - - set ${saved_xtrace} - sudo env "${vars[@]}" "$@" -} -export -f sudo_preserve_env diff --git a/manpages/conf.py b/manpages/conf.py index 7c3ea95..79e603b 100644 --- a/manpages/conf.py +++ b/manpages/conf.py @@ -1,25 +1,14 @@ # Configuration file for the Sphinx documentation builder. # -# This file only contains a selection of the most common options. For a full -# list see the documentation: +# For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - # -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = u'NDN Essential Tools' -copyright = u'Copyright © 2014-2022 Named Data Networking Project.' -author = u'Named Data Networking Project' +project = 'NDN Essential Tools' +copyright = 'Copyright © 2014-2022 Named Data Networking Project.' +author = 'Named Data Networking Project' # The short X.Y version. #version = '' @@ -35,30 +24,18 @@ today_fmt = '%Y-%m-%d' # -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# If your documentation needs a minimal Sphinx version, state it here. -# -needs_sphinx = '1.3' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. +needs_sphinx = '4.0' extensions = [ ] -# The master toctree document. -master_doc = 'index' - -# Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for manual page output ------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-manual-page-output # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). @@ -71,6 +48,3 @@ man_pages = [ ('ndndump', 'ndndump', 'traffic analysis tool', [], 8), ('ndn-dissect', 'ndn-dissect', 'NDN packet format inspector', [], 1), ] - -# If true, show URL addresses after external links. -#man_show_urls = True