From 15a7442759a0bdb1e5089735cc15fe6a11f3b67c Mon Sep 17 00:00:00 2001 From: Davide Pesavento Date: Fri, 19 Aug 2022 15:48:45 -0400 Subject: [PATCH] build+ci: support macOS on arm64 This commit also syncs the CI config and scripts with ndn-cxx Change-Id: I9866345393e2354e65a0082c047e846b2f8641f5 --- .github/workflows/ci.yml | 35 +++++++-------- .github/workflows/docs.yml | 9 +--- .jenkins | 30 ++++++++++--- .jenkins.d/00-deps.sh | 67 +++++++++++++--------------- .jenkins.d/01-ndn-cxx.sh | 16 +++---- .jenkins.d/09-cleanup.sh | 2 +- .jenkins.d/10-build.sh | 19 ++++---- .jenkins.d/20-tests.sh | 11 +++-- .jenkins.d/40-headers-check.sh | 11 ++--- .jenkins.d/README.md | 40 ++++++++++------- .jenkins.d/util.sh | 39 ---------------- .waf-tools/boost.py | 4 +- .waf-tools/default-compiler-flags.py | 5 ++- README.md | 2 +- 14 files changed, 132 insertions(+), 158 deletions(-) delete mode 100644 .jenkins.d/util.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d9a6a..3495a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,22 +13,24 @@ permissions: jobs: linux: name: ${{ matrix.compiler }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - compiler: [g++-8, g++-9, g++-10, g++-11, - clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12] os: [ubuntu-20.04] + compiler: [g++-7, g++-8, g++-9, g++-10, + clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12] include: - - compiler: g++-7 - os: ubuntu-18.04 - - compiler: clang++-6.0 - os: ubuntu-18.04 - runs-on: ${{ matrix.os }} + - os: ubuntu-22.04 + compiler: g++-11 + - os: ubuntu-22.04 + compiler: g++-12 + - os: ubuntu-22.04 + compiler: clang++-13 + - os: ubuntu-22.04 + compiler: clang++-14 env: CXX: ${{ matrix.compiler }} - NODE_LABELS: Linux Ubuntu - WAF_JOBS: 2 steps: - name: Install C++ compiler run: | @@ -41,20 +43,15 @@ jobs: macos: name: Xcode ${{ matrix.xcode }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - xcode: ['11.3', '11.7', '12.4'] - os: [macos-10.15] + os: [macos-11] + xcode: ['12.4', '12.5', '13.2'] include: - - xcode: '12.5' - os: macos-11 - - xcode: '13.2' - os: macos-11 - runs-on: ${{ matrix.os }} - env: - NODE_LABELS: OSX - WAF_JOBS: 3 + - os: macos-12 + xcode: '13.4' steps: - name: Set up Xcode uses: maxim-lobanov/setup-xcode@v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3fb1749..d5fcfb0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,23 +11,18 @@ permissions: jobs: build: + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [macos-11, ubuntu-20.04] - runs-on: ${{ matrix.os }} + os: [macos-12, ubuntu-20.04] env: JOB_NAME: Docs - WAF_JOBS: 3 steps: - name: Checkout uses: actions/checkout@v3 - name: Install dependencies run: | - case ${RUNNER_OS} in - Linux) export NODE_LABELS="Linux Ubuntu" ;; - macOS) export NODE_LABELS="OSX" ;; - esac find .jenkins.d/ -type f -name '[1-9]*.sh' -exec chmod -x '{}' + ./.jenkins - name: Build documentation diff --git a/.jenkins b/.jenkins index da10d69..ee16e29 100755 --- a/.jenkins +++ b/.jenkins @@ -1,12 +1,30 @@ #!/usr/bin/env bash -set -e -source .jenkins.d/util.sh +set -eo pipefail + +case $(uname) in + Linux) + if [[ -e /etc/os-release ]]; then + source /etc/os-release + else + source /usr/lib/os-release + fi + export ID VERSION_ID + export ID_LIKE="${ID} ${ID_LIKE} linux" + export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" + ;; + Darwin) + # Emulate a subset of os-release(5) + export ID=macos + export VERSION_ID=$(sw_vers -productVersion) + if [[ -x /opt/homebrew/bin/brew ]]; then + eval "$(/opt/homebrew/bin/brew shellenv)" + elif [[ -x /usr/local/bin/brew ]]; then + eval "$(/usr/local/bin/brew shellenv)" + fi + ;; +esac -if has Linux $NODE_LABELS; then - export PATH="${HOME}/.local/bin${PATH:+:}${PATH}" -fi export CACHE_DIR=${CACHE_DIR:-/tmp} -export WAF_JOBS=${WAF_JOBS:-1} [[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes for file in .jenkins.d/*; do diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh index d3b16f9..12b5de3 100755 --- a/.jenkins.d/00-deps.sh +++ b/.jenkins.d/00-deps.sh @@ -1,49 +1,44 @@ #!/usr/bin/env bash -set -ex +set -eo pipefail -if has OSX $NODE_LABELS; then - FORMULAE=(boost openssl pkg-config) - if [[ $JOB_NAME == *"Docs" ]]; then +APT_PKGS=(build-essential pkg-config python3-minimal + libboost-all-dev libssl-dev libsqlite3-dev) +FORMULAE=(boost openssl pkg-config) +PIP_PKGS=() +case $JOB_NAME in + *code-coverage) + APT_PKGS+=(lcov python3-pip) + PIP_PKGS+=('gcovr~=5.2') + ;; + *Docs) + APT_PKGS+=(doxygen graphviz python3-pip) FORMULAE+=(doxygen graphviz) - fi + PIP_PKGS+=(sphinx sphinxcontrib-doxylink) + ;; +esac +set -x + +if [[ $ID == macos ]]; then if [[ -n $GITHUB_ACTIONS ]]; then - # GitHub Actions runners have a large number of pre-installed - # Homebrew packages. Don't waste time upgrading all of them. - brew list --versions "${FORMULAE[@]}" || brew update - for FORMULA in "${FORMULAE[@]}"; do - brew list --versions "$FORMULA" || brew install "$FORMULA" - done - # Ensure /usr/local/opt/openssl exists - brew reinstall openssl - else - brew update - brew upgrade - brew install "${FORMULAE[@]}" - brew cleanup + export HOMEBREW_NO_INSTALL_UPGRADE=1 + fi + brew update + brew install --formula "${FORMULAE[@]}" + + if (( ${#PIP_PKGS[@]} )); then + pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" fi - if [[ $JOB_NAME == *"Docs" ]]; then - pip3 install --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink - fi - -elif has Ubuntu $NODE_LABELS; then +elif [[ $ID_LIKE == *debian* ]]; then sudo apt-get -qq update - sudo apt-get -qy install build-essential pkg-config python3-minimal \ - libboost-all-dev libssl-dev libsqlite3-dev + sudo apt-get -qy install "${APT_PKGS[@]}" - case $JOB_NAME in - *code-coverage) - sudo apt-get -qy install lcov python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager 'gcovr~=5.1' - ;; - *Docs) - sudo apt-get -qy install doxygen graphviz python3-pip - pip3 install --user --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink - ;; - esac + if (( ${#PIP_PKGS[@]} )); then + pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" + fi -elif has CentOS $NODE_LABELS; then +elif [[ $ID_LIKE == *fedora* ]]; then sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \ boost-devel openssl-devel sqlite-devel fi diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh index 5da6d35..91ebefd 100755 --- a/.jenkins.d/01-ndn-cxx.sh +++ b/.jenkins.d/01-ndn-cxx.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -set -ex +set -exo pipefail pushd "$CACHE_DIR" >/dev/null INSTALLED_VERSION= -if has OSX $NODE_LABELS; then - BOOST=$(brew ls --versions boost) +if [[ $ID == macos ]]; then + BOOST=$(brew list --formula --versions boost) OLD_BOOST=$(cat boost.txt || :) if [[ $OLD_BOOST != $BOOST ]]; then echo "$BOOST" > boost.txt @@ -35,16 +35,16 @@ sudo rm -f /usr/local/lib{,64}/pkgconfig/libndn-cxx.pc pushd ndn-cxx >/dev/null -./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain -./waf --color=yes build -j$WAF_JOBS -sudo_preserve_env PATH -- ./waf --color=yes install +./waf --color=yes configure --without-osx-keychain +./waf --color=yes build +sudo ./waf --color=yes install popd >/dev/null popd >/dev/null -if has CentOS $NODE_LABELS; then +if [[ $ID_LIKE == *fedora* ]]; then sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64 fi -if has Linux $NODE_LABELS; then +if [[ $ID_LIKE == *linux* ]]; then sudo ldconfig fi diff --git a/.jenkins.d/09-cleanup.sh b/.jenkins.d/09-cleanup.sh index fe007ef..531949c 100755 --- a/.jenkins.d/09-cleanup.sh +++ b/.jenkins.d/09-cleanup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -ex +set -exo pipefail PROJ=ChronoSync diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh index 25892f5..2180a93 100755 --- a/.jenkins.d/10-build.sh +++ b/.jenkins.d/10-build.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -set -ex - -git submodule sync -git submodule update --init +set -eo pipefail if [[ -z $DISABLE_ASAN ]]; then ASAN="--with-sanitizer=address" @@ -11,17 +8,19 @@ if [[ $JOB_NAME == *"code-coverage" ]]; then COVERAGE="--with-coverage" fi +set -x + if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then # Build in release mode with tests ./waf --color=yes configure --with-tests - ./waf --color=yes build -j$WAF_JOBS + ./waf --color=yes build # Cleanup ./waf --color=yes distclean # Build in release mode without tests ./waf --color=yes configure - ./waf --color=yes build -j$WAF_JOBS + ./waf --color=yes build # Cleanup ./waf --color=yes distclean @@ -29,16 +28,16 @@ fi # Build in debug mode with tests ./waf --color=yes configure --debug --with-tests $ASAN $COVERAGE -./waf --color=yes build -j$WAF_JOBS +./waf --color=yes build # (tests will be run against the debug version) # Install -sudo_preserve_env PATH -- ./waf --color=yes install +sudo ./waf --color=yes install -if has CentOS $NODE_LABELS; then +if [[ $ID_LIKE == *fedora* ]]; then sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64 fi -if has Linux $NODE_LABELS; then +if [[ $ID_LIKE == *linux* ]]; then sudo ldconfig fi diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh index 81c00b1..ab14284 100755 --- a/.jenkins.d/20-tests.sh +++ b/.jenkins.d/20-tests.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -set -ex - -# Prepare environment -rm -rf ~/.ndn +set -eo pipefail # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags ASAN_OPTIONS="color=always" @@ -14,10 +11,16 @@ ASAN_OPTIONS+=":detect_invalid_pointer_pairs=2" ASAN_OPTIONS+=":strip_path_prefix=${PWD}/" export ASAN_OPTIONS +# https://www.boost.org/doc/libs/release/libs/test/doc/html/boost_test/runtime_config/summary.html export BOOST_TEST_BUILD_INFO=1 export BOOST_TEST_COLOR_OUTPUT=1 export BOOST_TEST_DETECT_MEMORY_LEAK=0 export BOOST_TEST_LOGGER=HRF,test_suite,stdout:XML,all,build/xunit-log.xml +set -x + +# Prepare environment +rm -rf ~/.ndn + # Run unit tests ./build/unit-tests diff --git a/.jenkins.d/40-headers-check.sh b/.jenkins.d/40-headers-check.sh index f4bfb8a..88c0003 100755 --- a/.jenkins.d/40-headers-check.sh +++ b/.jenkins.d/40-headers-check.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash - +set -eo pipefail # It's intentional not to use `set -x`, because this script explicitly prints useful information # and should not run in trace mode. -# It's intentional not to use `set -e`, because this script wants to check all headers -# (similar to running all test cases), instead of failing at the first error. PROJ=ChronoSync PCFILE=ChronoSync @@ -13,7 +11,7 @@ if [[ -n $DISABLE_HEADERS_CHECK ]]; then exit 0 fi -if has CentOS $NODE_LABELS; then +if [[ $ID_LIKE == *fedora* ]]; then export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig fi @@ -28,9 +26,8 @@ NCHECKED=0 NERRORS=0 while IFS= read -r -d '' H; do echo "Checking header ${H#${INCLUDEDIR}/}" - "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H" - [[ $? -eq 0 ]] || ((NERRORS++)) - ((NCHECKED++)) + "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H" || : $((NERRORS++)) + : $((NCHECKED++)) done < <(find "$INCLUDEDIR" -name '*.hpp' -type f -print0 2>/dev/null) if [[ $NCHECKED -eq 0 ]]; then diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md index e8dbf37..385af34 100644 --- a/.jenkins.d/README.md +++ b/.jenkins.d/README.md @@ -1,28 +1,36 @@ -# CONTINUOUS INTEGRATION SCRIPTS +# Continuous Integration Scripts -## Environment Variables Used in Build Scripts +## Environment Variables -- `NODE_LABELS`: space-separated list of platform properties. The included values are used by - the build scripts to select the proper behavior for different operating systems and versions. +- `ID`: lower-case string that identifies the operating system, for example: `ID=ubuntu`, + `ID=centos`. See [os-release(5)] for more information. On macOS, where `os-release` is + not available, we emulate it by setting `ID=macos`. - The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`. +- `ID_LIKE`: space-separated list of operating system identifiers that are closely related + to the running OS. See [os-release(5)] for more information. The listed values are used + by the CI scripts to select the proper behavior for different platforms and OS flavors. - Example values: + Examples: - - `[OS_TYPE]`: `Linux`, `OSX` - - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS` - - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15` + - On CentOS, `ID_LIKE="centos rhel fedora linux"` + - On Ubuntu, `ID_LIKE="ubuntu debian linux"` -- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type, - the build scripts can perform different tasks. +- `VERSION_ID`: identifies the operating system version, excluding any release code names. + See [os-release(5)] for more information. Examples: `VERSION_ID=42`, `VERSION_ID=22.04`. - Possible values: +- `JOB_NAME`: defines the type of the current CI job. Depending on the job type, the CI + scripts can perform different tasks. + + Supported values: - empty: default build task - - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed) + - `code-coverage`: debug build with tests and code coverage analysis - `limited-build`: only a single debug build with tests -- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version - of ndn-cxx. If not set, `/tmp` is used. +- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled + version of ndn-cxx. If not set, `/tmp` is used. -- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1. +- `DISABLE_ASAN`: disable building with AddressSanitizer. This is automatically set for + the `code-coverage` job type. + +[os-release(5)]: https://www.freedesktop.org/software/systemd/man/os-release.html diff --git a/.jenkins.d/util.sh b/.jenkins.d/util.sh deleted file mode 100644 index 8077a74..0000000 --- a/.jenkins.d/util.sh +++ /dev/null @@ -1,39 +0,0 @@ -has() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local p=$1 - shift - local i ret=1 - for i in "$@"; do - if [[ "${i}" == "${p}" ]]; then - ret=0 - break - fi - done - - set ${saved_xtrace} - return ${ret} -} -export -f has - -sudo_preserve_env() { - local saved_xtrace - [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x - set +x - - local vars=() - while [[ $# -gt 0 ]]; do - local arg=$1 - shift - case ${arg} in - --) break ;; - *) vars+=("${arg}=${!arg}") ;; - esac - done - - set ${saved_xtrace} - sudo env "${vars[@]}" "$@" -} -export -f sudo_preserve_env diff --git a/.waf-tools/boost.py b/.waf-tools/boost.py index 4b2ede5..a6cdabe 100644 --- a/.waf-tools/boost.py +++ b/.waf-tools/boost.py @@ -54,8 +54,8 @@ from waflib import Utils, Logs, Errors from waflib.Configure import conf from waflib.TaskGen import feature, after_method -BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/local/lib', '/sw/lib', '/lib'] -BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/local/include', '/sw/include'] +BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/homebrew/lib', '/opt/local/lib', '/sw/lib', '/lib'] +BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/homebrew/include', '/opt/local/include', '/sw/include'] BOOST_VERSION_FILE = 'boost/version.hpp' BOOST_VERSION_CODE = ''' diff --git a/.waf-tools/default-compiler-flags.py b/.waf-tools/default-compiler-flags.py index 7c6d282..3ba2dc3 100644 --- a/.waf-tools/default-compiler-flags.py +++ b/.waf-tools/default-compiler-flags.py @@ -33,7 +33,7 @@ def configure(conf): 'The minimum supported clang version is 6.0.') conf.flags = ClangFlags() else: - warnmsg = '%s compiler is unsupported' % cxx + warnmsg = f'{cxx} compiler is unsupported' conf.flags = CompilerFlags() if errmsg: @@ -200,7 +200,8 @@ class ClangFlags(GccBasicFlags): flags = super(ClangFlags, self).getGeneralFlags(conf) if Utils.unversioned_sys_platform() == 'darwin': # Bug #4296 - flags['CXXFLAGS'] += [['-isystem', '/usr/local/include'], # for Homebrew + brewdir = '/opt/homebrew' if platform.machine() == 'arm64' else '/usr/local' + flags['CXXFLAGS'] += [['-isystem', f'{brewdir}/include'], # for Homebrew ['-isystem', '/opt/local/include']] # for MacPorts elif Utils.unversioned_sys_platform() == 'freebsd': # Bug #4790 diff --git a/README.md b/README.md index 66e8fb9..5ada05c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Language](https://img.shields.io/badge/C%2B%2B-17-blue) ![Latest version](https://img.shields.io/github/v/tag/named-data/ChronoSync?label=Latest%20version) -> DEPRECATION NOTICE: ChronoSync's design is outdated. We recommend using more recent sync protocols, such as [PSync](https://named-data.net/doc/PSync/current/index.html) or [StateVectorSync](https://named-data.github.io/StateVectorSync/). +> DEPRECATION NOTICE: ChronoSync's design is outdated. We recommend using more recent sync protocols, such as [PSync](https://named-data.net/doc/PSync/current/) or [StateVectorSync](https://named-data.github.io/StateVectorSync/). In supporting many distributed applications, such as group text messaging, file sharing, and joint editing, a basic requirement is the efficient and robust synchronization of