From f5006be472ce1d8adc8f6675f218f76513e5f1fa Mon Sep 17 00:00:00 2001 From: Davide Pesavento Date: Fri, 19 Aug 2022 17:36:36 -0400 Subject: [PATCH] build+ci: support CentOS Stream 9 and macOS/arm64 This commit also syncs the CI config and scripts with ndn-tools Change-Id: I54707a7119c5e91138eaf55ad72af8fa98125028 --- .github/workflows/ci.yml | 35 +++++++------- .jenkins | 30 +++++++++--- .jenkins.d/00-deps.sh | 70 ++++++++++++++-------------- .jenkins.d/01-ndn-cxx.sh | 21 ++++----- .jenkins.d/10-build.sh | 11 ++--- .jenkins.d/README.md | 40 +++++++++------- .jenkins.d/util.sh | 39 ---------------- .waf-tools/default-compiler-flags.py | 5 +- README.md | 14 ++++-- wscript | 5 ++ 10 files changed, 127 insertions(+), 143 deletions(-) delete mode 100644 .jenkins.d/util.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d9a6a..3495a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,22 +13,24 @@ permissions: jobs: linux: name: ${{ matrix.compiler }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - compiler: [g++-8, g++-9, g++-10, g++-11, - clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12] os: [ubuntu-20.04] + compiler: [g++-7, g++-8, g++-9, g++-10, + clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12] include: - - compiler: g++-7 - os: ubuntu-18.04 - - compiler: clang++-6.0 - os: ubuntu-18.04 - runs-on: ${{ matrix.os }} + - os: ubuntu-22.04 + compiler: g++-11 + - os: ubuntu-22.04 + compiler: g++-12 + - os: ubuntu-22.04 + compiler: clang++-13 + - os: ubuntu-22.04 + compiler: clang++-14 env: CXX: ${{ matrix.compiler }} - NODE_LABELS: Linux Ubuntu - WAF_JOBS: 2 steps: - name: Install C++ compiler run: | @@ -41,20 +43,15 @@ jobs: macos: name: Xcode ${{ matrix.xcode }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - xcode: ['11.3', '11.7', '12.4'] - os: [macos-10.15] + os: [macos-11] + xcode: ['12.4', '12.5', '13.2'] include: - - xcode: '12.5' - os: macos-11 - - xcode: '13.2' - os: macos-11 - runs-on: ${{ matrix.os }} - env: - NODE_LABELS: OSX - WAF_JOBS: 3 + - os: macos-12 + xcode: '13.4' steps: - name: Set up Xcode uses: maxim-lobanov/setup-xcode@v1 diff --git a/.jenkins b/.jenkins index da10d69..ee16e29 100755 --- a/.jenkins +++ b/.jenkins @@ -1,12 +1,30 @@ #!/usr/bin/env bash -set -e -source .jenkins.d/util.sh +set -eo pipefail + +case $(uname) in + Linux) + if [[ -e /etc/os-release ]]; then + source /etc/os-release + else + source /usr/lib/os-release + fi + export ID VERSION_ID + export ID_LIKE="${ID} ${ID_LIKE} linux" + export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" + ;; + Darwin) + # Emulate a subset of os-release(5) + export ID=macos + export VERSION_ID=$(sw_vers -productVersion) + if [[ -x /opt/homebrew/bin/brew ]]; then + eval "$(/opt/homebrew/bin/brew shellenv)" + elif [[ -x /usr/local/bin/brew ]]; then + eval "$(/usr/local/bin/brew shellenv)" + fi + ;; +esac -if has Linux $NODE_LABELS; then - export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" -fi export CACHE_DIR=${CACHE_DIR:-/tmp} -export WAF_JOBS=${WAF_JOBS:-1} [[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes for file in .jenkins.d/*; do diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh index 95dfe37..25e3b77 100755 --- a/.jenkins.d/00-deps.sh +++ b/.jenkins.d/00-deps.sh @@ -1,45 +1,43 @@ #!/usr/bin/env bash -set -ex +set -eo pipefail -if has OSX $NODE_LABELS; then - FORMULAE=(boost openssl pkg-config) - if [[ $JOB_NAME == *"Docs" ]]; then - FORMULAE+=(doxygen graphviz) - fi +APT_PKGS=(build-essential pkg-config python3-minimal + libboost-all-dev libssl-dev libsqlite3-dev) +FORMULAE=(boost openssl pkg-config) +PIP_PKGS=() +case $JOB_NAME in + *code-coverage) + APT_PKGS+=(lcov python3-pip) + PIP_PKGS+=('gcovr~=5.2') + ;; + *Docs) + APT_PKGS+=(python3-pip) + PIP_PKGS+=(sphinx) + ;; +esac +set -x + +if [[ $ID == macos ]]; then if [[ -n $GITHUB_ACTIONS ]]; then - # GitHub Actions runners have a large number of pre-installed - # Homebrew packages. Don't waste time upgrading all of them. - brew list --versions "${FORMULAE[@]}" || brew update - for FORMULA in "${FORMULAE[@]}"; do - brew list --versions "$FORMULA" || brew install "$FORMULA" - done - # Ensure /usr/local/opt/openssl exists - brew reinstall openssl - else - brew update - brew upgrade - brew install "${FORMULAE[@]}" - brew cleanup + export HOMEBREW_NO_INSTALL_UPGRADE=1 + fi + brew update + brew install --formula "${FORMULAE[@]}" + + if (( ${#PIP_PKGS[@]} )); then + pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" fi - if [[ $JOB_NAME == *"Docs" ]]; then - pip3 install --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink - fi - -elif has Ubuntu $NODE_LABELS; then +elif [[ $ID_LIKE == *debian* ]]; then sudo apt-get -qq update - sudo apt-get -qy install build-essential pkg-config python3-minimal \ - libboost-all-dev libssl-dev libsqlite3-dev + sudo apt-get -qy install "${APT_PKGS[@]}" - case $JOB_NAME in - *code-coverage) - sudo apt-get -qy install lcov python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager 'gcovr~=5.0' - ;; - *Docs) - sudo apt-get -qy install doxygen graphviz python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink - ;; - esac + if (( ${#PIP_PKGS[@]} )); then + pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" + fi + +elif [[ $ID_LIKE == *fedora* ]]; then + sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \ + boost-devel openssl-devel sqlite-devel fi diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh index 4e0e154..91ebefd 100755 --- a/.jenkins.d/01-ndn-cxx.sh +++ b/.jenkins.d/01-ndn-cxx.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -set -ex +set -exo pipefail pushd "$CACHE_DIR" >/dev/null INSTALLED_VERSION= -if has OSX $NODE_LABELS; then - BOOST=$(brew ls --versions boost) +if [[ $ID == macos ]]; then + BOOST=$(brew list --formula --versions boost) OLD_BOOST=$(cat boost.txt || :) if [[ $OLD_BOOST != $BOOST ]]; then echo "$BOOST" > boost.txt @@ -35,21 +35,16 @@ sudo rm -f /usr/local/lib{,64}/pkgconfig/libndn-cxx.pc pushd ndn-cxx >/dev/null -if has CentOS-8 $NODE_LABELS; then - # https://bugzilla.redhat.com/show_bug.cgi?id=1721553 - PCH="--without-pch" -fi - -./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain $PCH -./waf --color=yes build -j$WAF_JOBS -sudo_preserve_env PATH -- ./waf --color=yes install +./waf --color=yes configure --without-osx-keychain +./waf --color=yes build +sudo ./waf --color=yes install popd >/dev/null popd >/dev/null -if has CentOS-8 $NODE_LABELS; then +if [[ $ID_LIKE == *fedora* ]]; then sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64 fi -if has Linux $NODE_LABELS; then +if [[ $ID_LIKE == *linux* ]]; then sudo ldconfig fi diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh index 7dd67cc..77e08bd 100755 --- a/.jenkins.d/10-build.sh +++ b/.jenkins.d/10-build.sh @@ -1,19 +1,16 @@ #!/usr/bin/env bash -set -ex - -git submodule sync -git submodule update --init +set -exo pipefail # Build in debug mode ./waf --color=yes configure --debug -./waf --color=yes build -j$WAF_JOBS +./waf --color=yes build # Cleanup ./waf --color=yes distclean # Build in release mode ./waf --color=yes configure -./waf --color=yes build -j$WAF_JOBS +./waf --color=yes build # Install -sudo_preserve_env PATH -- ./waf --color=yes install +sudo ./waf --color=yes install diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md index e8dbf37..385af34 100644 --- a/.jenkins.d/README.md +++ b/.jenkins.d/README.md @@ -1,28 +1,36 @@ -# CONTINUOUS INTEGRATION SCRIPTS +# Continuous Integration Scripts -## Environment Variables Used in Build Scripts +## Environment Variables -- `NODE_LABELS`: space-separated list of platform properties. The included values are used by - the build scripts to select the proper behavior for different operating systems and versions. +- `ID`: lower-case string that identifies the operating system, for example: `ID=ubuntu`, + `ID=centos`. See [os-release(5)] for more information. On macOS, where `os-release` is + not available, we emulate it by setting `ID=macos`. - The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`. +- `ID_LIKE`: space-separated list of operating system identifiers that are closely related + to the running OS. See [os-release(5)] for more information. The listed values are used + by the CI scripts to select the proper behavior for different platforms and OS flavors. - Example values: + Examples: - - `[OS_TYPE]`: `Linux`, `OSX` - - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS` - - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15` + - On CentOS, `ID_LIKE="centos rhel fedora linux"` + - On Ubuntu, `ID_LIKE="ubuntu debian linux"` -- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type, - the build scripts can perform different tasks. +- `VERSION_ID`: identifies the operating system version, excluding any release code names. + See [os-release(5)] for more information. Examples: `VERSION_ID=42`, `VERSION_ID=22.04`. - Possible values: +- `JOB_NAME`: defines the type of the current CI job. Depending on the job type, the CI + scripts can perform different tasks. + + Supported values: - empty: default build task - - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed) + - `code-coverage`: debug build with tests and code coverage analysis - `limited-build`: only a single debug build with tests -- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version - of ndn-cxx. If not set, `/tmp` is used. +- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled + version of ndn-cxx. If not set, `/tmp` is used. -- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1. +- `DISABLE_ASAN`: disable building with AddressSanitizer. This is automatically set for + the `code-coverage` job type. + +[os-release(5)]: https://www.freedesktop.org/software/systemd/man/os-release.html diff --git a/.jenkins.d/util.sh b/.jenkins.d/util.sh deleted file mode 100644 index 8077a74..0000000 --- a/.jenkins.d/util.sh +++ /dev/null @@ -1,39 +0,0 @@ -has() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local p=$1 - shift - local i ret=1 - for i in "$@"; do - if [[ "${i}" == "${p}" ]]; then - ret=0 - break - fi - done - - set ${saved_xtrace} - return ${ret} -} -export -f has - -sudo_preserve_env() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local vars=() - while [[ $# -gt 0 ]]; do - local arg=$1 - shift - case ${arg} in - --) break ;; - *) vars+=("${arg}=${!arg}") ;; - esac - done - - set ${saved_xtrace} - sudo env "${vars[@]}" "$@" -} -export -f sudo_preserve_env diff --git a/.waf-tools/default-compiler-flags.py b/.waf-tools/default-compiler-flags.py index 7c6d282..3ba2dc3 100644 --- a/.waf-tools/default-compiler-flags.py +++ b/.waf-tools/default-compiler-flags.py @@ -33,7 +33,7 @@ def configure(conf): 'The minimum supported clang version is 6.0.') conf.flags = ClangFlags() else: - warnmsg = '%s compiler is unsupported' % cxx + warnmsg = f'{cxx} compiler is unsupported' conf.flags = CompilerFlags() if errmsg: @@ -200,7 +200,8 @@ class ClangFlags(GccBasicFlags): flags = super(ClangFlags, self).getGeneralFlags(conf) if Utils.unversioned_sys_platform() == 'darwin': # Bug #4296 - flags['CXXFLAGS'] += [['-isystem', '/usr/local/include'], # for Homebrew + brewdir = '/opt/homebrew' if platform.machine() == 'arm64' else '/usr/local' + flags['CXXFLAGS'] += [['-isystem', f'{brewdir}/include'], # for Homebrew ['-isystem', '/opt/local/include']] # for MacPorts elif Utils.unversioned_sys_platform() == 'freebsd': # Bug #4790 diff --git a/README.md b/README.md index 24829bc..6112e61 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Compiling and running ndn-traffic-generator requires the following dependencies: ## Compilation & Installation -``` +```shell ./waf configure ./waf sudo ./waf install @@ -26,7 +26,7 @@ sudo ./waf install ## Command Line Options -#### ndn-traffic-server +### `ndn-traffic-server` Usage: ndn-traffic-server [options] Respond to Interests as per provided Traffic_Configuration_File. @@ -38,7 +38,7 @@ sudo ./waf install -d [ --delay ] arg (=0) wait this amount of milliseconds before responding to each Interest -q [ --quiet ] turn off logging of Interest reception/Data generation -#### ndn-traffic-client +### `ndn-traffic-client` Usage: ndn-traffic-client [options] Generate Interest traffic as per provided Traffic_Configuration_File. @@ -62,7 +62,9 @@ sudo ./waf install Start the traffic server: - ndn-traffic-server ndn-traffic-server.conf +```shell +ndn-traffic-server ndn-traffic-server.conf +``` #### ON MACHINE #2 @@ -70,4 +72,6 @@ Start the traffic server: Start the traffic client: - ndn-traffic-client ndn-traffic-client.conf +```shell +ndn-traffic-client ndn-traffic-client.conf +``` diff --git a/wscript b/wscript index ad201d1..3b1a38d 100644 --- a/wscript +++ b/wscript @@ -15,6 +15,11 @@ def configure(conf): conf.load(['compiler_cxx', 'gnu_dirs', 'default-compiler-flags']) + # Prefer pkgconf if it's installed, because it gives more correct results + # on Fedora/CentOS/RHEL/etc. See https://bugzilla.redhat.com/show_bug.cgi?id=1953348 + # Store the result in env.PKGCONFIG, which is the variable used inside check_cfg() + conf.find_program(['pkgconf', 'pkg-config'], var='PKGCONFIG') + pkg_config_path = os.environ.get('PKG_CONFIG_PATH', f'{conf.env.LIBDIR}/pkgconfig') conf.check_cfg(package='libndn-cxx', args=['libndn-cxx >= 0.8.0', '--cflags', '--libs'], uselib_store='NDN_CXX', pkg_config_path=pkg_config_path)