build+ci: support macOS on arm64

This commit also syncs the CI config and scripts with ndn-cxx

Change-Id: I9866345393e2354e65a0082c047e846b2f8641f5

.github/workflows/ci.yml

@@ -13,22 +13,24 @@ permissions:
 jobs:
   linux:
     name: ${{ matrix.compiler }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        compiler: [g++-8, g++-9, g++-10, g++-11,
-                   clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12]
         os: [ubuntu-20.04]
+        compiler: [g++-7, g++-8, g++-9, g++-10,
+                   clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12]
         include:
-          - compiler: g++-7
-            os: ubuntu-18.04
-          - compiler: clang++-6.0
-            os: ubuntu-18.04
-    runs-on: ${{ matrix.os }}
+          - os: ubuntu-22.04
+            compiler: g++-11
+          - os: ubuntu-22.04
+            compiler: g++-12
+          - os: ubuntu-22.04
+            compiler: clang++-13
+          - os: ubuntu-22.04
+            compiler: clang++-14
     env:
       CXX: ${{ matrix.compiler }}
-      NODE_LABELS: Linux Ubuntu
-      WAF_JOBS: 2
     steps:
       - name: Install C++ compiler
         run: |
@@ -41,20 +43,15 @@ jobs:
 
   macos:
     name: Xcode ${{ matrix.xcode }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        xcode: ['11.3', '11.7', '12.4']
-        os: [macos-10.15]
+        os: [macos-11]
+        xcode: ['12.4', '12.5', '13.2']
         include:
-          - xcode: '12.5'
-            os: macos-11
-          - xcode: '13.2'
-            os: macos-11
-    runs-on: ${{ matrix.os }}
-    env:
-      NODE_LABELS: OSX
-      WAF_JOBS: 3
+          - os: macos-12
+            xcode: '13.4'
     steps:
       - name: Set up Xcode
         uses: maxim-lobanov/setup-xcode@v1

.github/workflows/docs.yml

@@ -11,23 +11,18 @@ permissions:
 
 jobs:
   build:
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-11, ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+        os: [macos-12, ubuntu-20.04]
     env:
       JOB_NAME: Docs
-      WAF_JOBS: 3
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Install dependencies
         run: |
-          case ${RUNNER_OS} in
-            Linux) export NODE_LABELS="Linux Ubuntu" ;;
-            macOS) export NODE_LABELS="OSX" ;;
-          esac
           find .jenkins.d/ -type f -name '[1-9]*.sh' -exec chmod -x '{}' +
           ./.jenkins
       - name: Build documentation

.jenkins

@@ -1,12 +1,30 @@
 #!/usr/bin/env bash
-set -e
-source .jenkins.d/util.sh
+set -eo pipefail
+
+case $(uname) in
+    Linux)
+        if [[ -e /etc/os-release ]]; then
+            source /etc/os-release
+        else
+            source /usr/lib/os-release
+        fi
+        export ID VERSION_ID
+        export ID_LIKE="${ID} ${ID_LIKE} linux"
+        export PATH="${HOME}/.local/bin${PATH:+:}${PATH}"
+        ;;
+    Darwin)
+        # Emulate a subset of os-release(5)
+        export ID=macos
+        export VERSION_ID=$(sw_vers -productVersion)
+        if [[ -x /opt/homebrew/bin/brew ]]; then
+            eval "$(/opt/homebrew/bin/brew shellenv)"
+        elif [[ -x /usr/local/bin/brew ]]; then
+            eval "$(/usr/local/bin/brew shellenv)"
+        fi
+        ;;
+esac
 
-if has Linux $NODE_LABELS; then
-    export PATH="${HOME}/.local/bin${PATH:+:}${PATH}"
-fi
 export CACHE_DIR=${CACHE_DIR:-/tmp}
-export WAF_JOBS=${WAF_JOBS:-1}
 [[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes
 
 for file in .jenkins.d/*; do

.jenkins.d/00-deps.sh

@@ -1,49 +1,44 @@
 #!/usr/bin/env bash
-set -ex
+set -eo pipefail
 
-if has OSX $NODE_LABELS; then
-    FORMULAE=(boost openssl pkg-config)
-    if [[ $JOB_NAME == *"Docs" ]]; then
+APT_PKGS=(build-essential pkg-config python3-minimal
+          libboost-all-dev libssl-dev libsqlite3-dev)
+FORMULAE=(boost openssl pkg-config)
+PIP_PKGS=()
+case $JOB_NAME in
+    *code-coverage)
+        APT_PKGS+=(lcov python3-pip)
+        PIP_PKGS+=('gcovr~=5.2')
+        ;;
+    *Docs)
+        APT_PKGS+=(doxygen graphviz python3-pip)
         FORMULAE+=(doxygen graphviz)
-    fi
+        PIP_PKGS+=(sphinx sphinxcontrib-doxylink)
+        ;;
+esac
 
+set -x
+
+if [[ $ID == macos ]]; then
     if [[ -n $GITHUB_ACTIONS ]]; then
-        # GitHub Actions runners have a large number of pre-installed
-        # Homebrew packages. Don't waste time upgrading all of them.
-        brew list --versions "${FORMULAE[@]}" || brew update
-        for FORMULA in "${FORMULAE[@]}"; do
-            brew list --versions "$FORMULA" || brew install "$FORMULA"
-        done
-        # Ensure /usr/local/opt/openssl exists
-        brew reinstall openssl
-    else
-        brew update
-        brew upgrade
-        brew install "${FORMULAE[@]}"
-        brew cleanup
+        export HOMEBREW_NO_INSTALL_UPGRADE=1
+    fi
+    brew update
+    brew install --formula "${FORMULAE[@]}"
+
+    if (( ${#PIP_PKGS[@]} )); then
+        pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
     fi
 
-    if [[ $JOB_NAME == *"Docs" ]]; then
-        pip3 install --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink
-    fi
-
-elif has Ubuntu $NODE_LABELS; then
+elif [[ $ID_LIKE == *debian* ]]; then
     sudo apt-get -qq update
-    sudo apt-get -qy install build-essential pkg-config python3-minimal \
-                             libboost-all-dev libssl-dev libsqlite3-dev
+    sudo apt-get -qy install "${APT_PKGS[@]}"
 
-    case $JOB_NAME in
-        *code-coverage)
-            sudo apt-get -qy install lcov python3-pip
-            pip3 install --user --upgrade --upgrade-strategy=eager 'gcovr~=5.1'
-            ;;
-        *Docs)
-            sudo apt-get -qy install doxygen graphviz python3-pip
-            pip3 install --user --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink
-            ;;
-    esac
+    if (( ${#PIP_PKGS[@]} )); then
+        pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
+    fi
 
-elif has CentOS $NODE_LABELS; then
+elif [[ $ID_LIKE == *fedora* ]]; then
     sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \
                         boost-devel openssl-devel sqlite-devel
 fi

.jenkins.d/01-ndn-cxx.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
-set -ex
+set -exo pipefail
 
 pushd "$CACHE_DIR" >/dev/null
 
 INSTALLED_VERSION=
-if has OSX $NODE_LABELS; then
-    BOOST=$(brew ls --versions boost)
+if [[ $ID == macos ]]; then
+    BOOST=$(brew list --formula --versions boost)
     OLD_BOOST=$(cat boost.txt || :)
     if [[ $OLD_BOOST != $BOOST ]]; then
         echo "$BOOST" > boost.txt
@@ -35,16 +35,16 @@ sudo rm -f /usr/local/lib{,64}/pkgconfig/libndn-cxx.pc
 
 pushd ndn-cxx >/dev/null
 
-./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain
-./waf --color=yes build -j$WAF_JOBS
-sudo_preserve_env PATH -- ./waf --color=yes install
+./waf --color=yes configure --without-osx-keychain
+./waf --color=yes build
+sudo ./waf --color=yes install
 
 popd >/dev/null
 popd >/dev/null
 
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
     sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64
 fi
-if has Linux $NODE_LABELS; then
+if [[ $ID_LIKE == *linux* ]]; then
     sudo ldconfig
 fi

.jenkins.d/09-cleanup.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -ex
+set -exo pipefail
 
 PROJ=ChronoSync
 

.jenkins.d/10-build.sh

@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
-set -ex
-
-git submodule sync
-git submodule update --init
+set -eo pipefail
 
 if [[ -z $DISABLE_ASAN ]]; then
     ASAN="--with-sanitizer=address"
@@ -11,17 +8,19 @@ if [[ $JOB_NAME == *"code-coverage" ]]; then
     COVERAGE="--with-coverage"
 fi
 
+set -x
+
 if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then
     # Build in release mode with tests
     ./waf --color=yes configure --with-tests
-    ./waf --color=yes build -j$WAF_JOBS
+    ./waf --color=yes build
 
     # Cleanup
     ./waf --color=yes distclean
 
     # Build in release mode without tests
     ./waf --color=yes configure
-    ./waf --color=yes build -j$WAF_JOBS
+    ./waf --color=yes build
 
     # Cleanup
     ./waf --color=yes distclean
@@ -29,16 +28,16 @@ fi
 
 # Build in debug mode with tests
 ./waf --color=yes configure --debug --with-tests $ASAN $COVERAGE
-./waf --color=yes build -j$WAF_JOBS
+./waf --color=yes build
 
 # (tests will be run against the debug version)
 
 # Install
-sudo_preserve_env PATH -- ./waf --color=yes install
+sudo ./waf --color=yes install
 
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
     sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64
 fi
-if has Linux $NODE_LABELS; then
+if [[ $ID_LIKE == *linux* ]]; then
     sudo ldconfig
 fi

.jenkins.d/20-tests.sh

@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
-set -ex
-
-# Prepare environment
-rm -rf ~/.ndn
+set -eo pipefail
 
 # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags
 ASAN_OPTIONS="color=always"
@@ -14,10 +11,16 @@ ASAN_OPTIONS+=":detect_invalid_pointer_pairs=2"
 ASAN_OPTIONS+=":strip_path_prefix=${PWD}/"
 export ASAN_OPTIONS
 
+# https://www.boost.org/doc/libs/release/libs/test/doc/html/boost_test/runtime_config/summary.html
 export BOOST_TEST_BUILD_INFO=1
 export BOOST_TEST_COLOR_OUTPUT=1
 export BOOST_TEST_DETECT_MEMORY_LEAK=0
 export BOOST_TEST_LOGGER=HRF,test_suite,stdout:XML,all,build/xunit-log.xml
 
+set -x
+
+# Prepare environment
+rm -rf ~/.ndn
+
 # Run unit tests
 ./build/unit-tests

.jenkins.d/40-headers-check.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
-
+set -eo pipefail
 # It's intentional not to use `set -x`, because this script explicitly prints useful information
 # and should not run in trace mode.
-# It's intentional not to use `set -e`, because this script wants to check all headers
-# (similar to running all test cases), instead of failing at the first error.
 
 PROJ=ChronoSync
 PCFILE=ChronoSync
@@ -13,7 +11,7 @@ if [[ -n $DISABLE_HEADERS_CHECK ]]; then
   exit 0
 fi
 
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
   export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig
 fi
 
@@ -28,9 +26,8 @@ NCHECKED=0
 NERRORS=0
 while IFS= read -r -d '' H; do
   echo "Checking header ${H#${INCLUDEDIR}/}"
-  "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H"
-  [[ $? -eq 0 ]] || ((NERRORS++))
-  ((NCHECKED++))
+  "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H" || : $((NERRORS++))
+  : $((NCHECKED++))
 done < <(find "$INCLUDEDIR" -name '*.hpp' -type f -print0 2>/dev/null)
 
 if [[ $NCHECKED -eq 0 ]]; then

.jenkins.d/README.md

@@ -1,28 +1,36 @@
-# CONTINUOUS INTEGRATION SCRIPTS
+# Continuous Integration Scripts
 
-## Environment Variables Used in Build Scripts
+## Environment Variables
 
-- `NODE_LABELS`: space-separated list of platform properties. The included values are used by
-  the build scripts to select the proper behavior for different operating systems and versions.
+- `ID`: lower-case string that identifies the operating system, for example: `ID=ubuntu`,
+  `ID=centos`. See [os-release(5)] for more information. On macOS, where `os-release` is
+  not available, we emulate it by setting `ID=macos`.
 
-  The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`.
+- `ID_LIKE`: space-separated list of operating system identifiers that are closely related
+  to the running OS. See [os-release(5)] for more information. The listed values are used
+  by the CI scripts to select the proper behavior for different platforms and OS flavors.
 
-  Example values:
+  Examples:
 
-  - `[OS_TYPE]`: `Linux`, `OSX`
-  - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS`
-  - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15`
+  - On CentOS, `ID_LIKE="centos rhel fedora linux"`
+  - On Ubuntu, `ID_LIKE="ubuntu debian linux"`
 
-- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type,
-  the build scripts can perform different tasks.
+- `VERSION_ID`: identifies the operating system version, excluding any release code names.
+  See [os-release(5)] for more information. Examples: `VERSION_ID=42`, `VERSION_ID=22.04`.
 
-  Possible values:
+- `JOB_NAME`: defines the type of the current CI job. Depending on the job type, the CI
+  scripts can perform different tasks.
+
+  Supported values:
 
   - empty: default build task
-  - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed)
+  - `code-coverage`: debug build with tests and code coverage analysis
   - `limited-build`: only a single debug build with tests
 
-- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version
-  of ndn-cxx. If not set, `/tmp` is used.
+- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled
+  version of ndn-cxx. If not set, `/tmp` is used.
 
-- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1.
+- `DISABLE_ASAN`: disable building with AddressSanitizer. This is automatically set for
+  the `code-coverage` job type.
+
+[os-release(5)]: https://www.freedesktop.org/software/systemd/man/os-release.html

.jenkins.d/util.sh

@@ -1,39 +0,0 @@
-has() {
-    local saved_xtrace
-    [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x
-    set +x
-
-    local p=$1
-    shift
-    local i ret=1
-    for i in "$@"; do
-        if [[ "${i}" == "${p}" ]]; then
-            ret=0
-            break
-        fi
-    done
-
-    set ${saved_xtrace}
-    return ${ret}
-}
-export -f has
-
-sudo_preserve_env() {
-    local saved_xtrace
-    [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x
-    set +x
-
-    local vars=()
-    while [[ $# -gt 0 ]]; do
-        local arg=$1
-        shift
-        case ${arg} in
-            --) break ;;
-            *)  vars+=("${arg}=${!arg}") ;;
-        esac
-    done
-
-    set ${saved_xtrace}
-    sudo env "${vars[@]}" "$@"
-}
-export -f sudo_preserve_env

.waf-tools/boost.py

@@ -54,8 +54,8 @@ from waflib import Utils, Logs, Errors
 from waflib.Configure import conf
 from waflib.TaskGen import feature, after_method
 
-BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/local/lib', '/sw/lib', '/lib']
-BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/local/include', '/sw/include']
+BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/homebrew/lib', '/opt/local/lib', '/sw/lib', '/lib']
+BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/homebrew/include', '/opt/local/include', '/sw/include']
 
 BOOST_VERSION_FILE = 'boost/version.hpp'
 BOOST_VERSION_CODE = '''

.waf-tools/default-compiler-flags.py

@@ -33,7 +33,7 @@ def configure(conf):
                       'The minimum supported clang version is 6.0.')
         conf.flags = ClangFlags()
     else:
-        warnmsg = '%s compiler is unsupported' % cxx
+        warnmsg = f'{cxx} compiler is unsupported'
         conf.flags = CompilerFlags()
 
     if errmsg:
@@ -200,7 +200,8 @@ class ClangFlags(GccBasicFlags):
         flags = super(ClangFlags, self).getGeneralFlags(conf)
         if Utils.unversioned_sys_platform() == 'darwin':
             # Bug #4296
-            flags['CXXFLAGS'] += [['-isystem', '/usr/local/include'], # for Homebrew
+            brewdir = '/opt/homebrew' if platform.machine() == 'arm64' else '/usr/local'
+            flags['CXXFLAGS'] += [['-isystem', f'{brewdir}/include'], # for Homebrew
                                   ['-isystem', '/opt/local/include']] # for MacPorts
         elif Utils.unversioned_sys_platform() == 'freebsd':
             # Bug #4790

README.md

@@ -5,7 +5,7 @@
 ![Language](https://img.shields.io/badge/C%2B%2B-17-blue)
 ![Latest version](https://img.shields.io/github/v/tag/named-data/ChronoSync?label=Latest%20version)
 
-> DEPRECATION NOTICE: ChronoSync's design is outdated. We recommend using more recent sync protocols, such as [PSync](https://named-data.net/doc/PSync/current/index.html) or [StateVectorSync](https://named-data.github.io/StateVectorSync/).
+> DEPRECATION NOTICE: ChronoSync's design is outdated. We recommend using more recent sync protocols, such as [PSync](https://named-data.net/doc/PSync/current/) or [StateVectorSync](https://named-data.github.io/StateVectorSync/).
 
 In supporting many distributed applications, such as group text messaging, file sharing,
 and joint editing, a basic requirement is the efficient and robust synchronization of