nDPI/.github/workflows/build.yml
mark andrews 5d5b46e514 Add proj and sln for compile of dynamic x64 lib under Visual Studio 2019.
* add CI support via MSBuild

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
2022-09-21 18:03:22 +02:00

539 lines
22 KiB
YAML

name: Build
on:
push:
branches:
- dev
pull_request:
branches:
- dev
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
coverage:
name: Coverage (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install rrdtool librrd-dev
sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
sudo apt-get install wdiff colordiff
- name: Configure
run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre --with-maxminddb --enable-tls-sigs
- name: Build
run: make all
- name: Test
run: |
make check VERBOSE=1
- name: Generate Coverage Report
run: |
mkdir -vp coverage_report/ndpi_coverage_report
lcov --directory . --capture --output-file lcov.info
genhtml -o coverage_report/ndpi_coverage_report lcov.info
- uses: actions/upload-artifact@v2
with:
name: ndpi-coverage-report
path: coverage_report
retention-days: 7
python-bindings:
name: Python Bindings (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
python-version: "3.9"
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
- name: Build nDPI library
run: |
./autogen.sh
make
sudo make install
- name: Generate Python bindings
run: |
pip install --upgrade pip
pip install -r python/requirements.txt
cd python
python setup.py install
cd ..
- name: Test Python Bindings
run: |
cd python
python tests.py
documentation:
name: Documentation (ubuntu-latest)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
python-version: "3.9"
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison doxygen
- name: Configure nDPI library
run: |
./autogen.sh --with-only-libndpi --enable-option-checking=fatal
- name: Generate Documentation
run: |
pip install --upgrade pip
pip install -r doc/requirements.txt
make doc
mkdir -vp doc/_build/ndpi-documentation-upload/ndpi-documentation
mv -v doc/_build/html doc/_build/ndpi-documentation-upload/ndpi-documentation/html
- uses: actions/upload-artifact@v2
with:
name: ndpi-documentation
path: doc/_build/ndpi-documentation-upload
retention-days: 7
performance:
name: Performance (ubuntu-latest)
runs-on: ubuntu-latest
env:
GO111MODULE: on
CFLAGS: -Werror
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libpcap-dev
sudo apt-get install golang-1.16-go libgoogle-perftools-dev graphviz
go install github.com/google/pprof@latest
sudo ln -s ${HOME}/go/bin/pprof /usr/bin/pprof
pprof -h
- name: Configure nDPI library
run: |
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs
- name: Build nDPI library
run: |
make
- name: Performance Profiling
run: |
./tests/do.sh
mkdir ndpi-performance-upload
mv -v tests/result/cpu_profile.png ndpi-performance-upload/cpu_profile.png
mv -v tests/result/heap_profile.png ndpi-performance-upload/heap_profile.png
- uses: actions/upload-artifact@v2
with:
name: ndpi-performance
path: ndpi-performance-upload
retention-days: 7
test-scripts:
name: Test Utils (ubuntu-latest)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install python3-netaddr git
- name: Run Scripts
run: |
echo 'Running ./utils/bitcoinnodes.sh'
./utils/bitcoinnodes.sh >/dev/null
echo 'Running ./utils/get_routes_by_asn.sh AS714'
./utils/get_routes_by_asn.sh AS714 >/dev/null
echo 'Running ./utils/update_every_lists.sh'
./utils/update_every_lists.sh
echo 'Checking for changes in the git tree..'
git update-index --refresh || echo "::warning file=utils/update_every_lists.sh::Please re-run utils/update_every_lists.sh and commit any changes."
git diff-index --quiet HEAD -- || true
test:
name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }}
runs-on: ${{ matrix.os }}
env:
CC: ${{ matrix.compiler }}
CFLAGS: -Werror
strategy:
fail-fast: true
matrix:
os: ["ubuntu-latest", "ubuntu-22.04", "macOS-latest", "macos-12", "windows-latest"]
arch: ["x86_64"]
gcrypt: ["--with-local-libgcrypt", ""]
compiler: ["cc"]
pcre: [""]
maxminddb: [""]
msan: [""]
nBPF: [""]
include:
- compiler: "gcc-7"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "gcc-11"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "clang-7"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "clang-14"
os: ubuntu-22.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-thread-sanitizer"
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: "nBPF"
- compiler: "clang-14"
os: ubuntu-22.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-memory-sanitizer"
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "cc"
os: macOS-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on macos
nBPF: ""
- compiler: "cc"
os: macos-12
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on macos
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "arm64"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on arm64
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "armhf"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
- compiler: "cc"
os: ubuntu-latest
arch: "s390x"
gcrypt: ""
pcre: "--with-pcre"
maxminddb: "--with-maxminddb"
msan: ""
nBPF: ""
steps:
- name: Setup multiarch/qemu-user-static
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.arch, 'x86_64')
run: |
docker run --rm --privileged multiarch/qemu-user-static:register --reset
- uses: actions/checkout@v2
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Ubuntu Prerequisites
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64')
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install gcc-mingw-w64 libc6-dev
sudo apt-get install doxygen python3-sphinx python3-sphinx-rtd-theme python3-breathe python3-pip
sudo apt-get install rrdtool librrd-dev
- name: Install Ubuntu Prerequisites (libgcrypt)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
sudo apt-get install libgcrypt20-dev
- name: Install Ubuntu Prerequisites (libpcre)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
run: |
sudo apt-get install libpcre3-dev
- name: Install Ubuntu Prerequisites (maxminddb)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb')
run: |
sudo apt-get install libmaxminddb-dev
- name: Install Ubuntu Prerequisites (nBPF)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.nBPF, 'nBPF')
run: |
git clone https://github.com/ntop/PF_RING.git ../PF_RING
cd ../PF_RING/userland/nbpf
./configure
make
cd -
- name: Setup Ubuntu specified compiler (gcc)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'gcc')
run: |
sudo apt-get install ${{ matrix.compiler }}
- name: Setup Ubuntu specified compiler (clang)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'clang')
run: |
sudo apt-get install ${{ matrix.compiler }}
- name: Install Windows msys2 prerequisites
if: startsWith(matrix.os, 'windows')
uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
update: true
install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap
- name: Setup Windows msys2 prerequisites (libgcrypt)
if: startsWith(matrix.os, 'windows') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
msys2 -c 'git clone --branch libgpg-error-1.42 https://github.com/gpg/libgpg-error'
cd libgpg-error
msys2 -c './autogen.sh'
msys2 -c './configure -enable-maintainer-mode --enable-static --enable-shared --with-pic --disable-doc --disable-nls'
msys2 -c 'make'
msys2 -c 'make install'
msys2 -c 'cd ..'
msys2 -c 'rm -rf libgpg-error/'
msys2 -c 'git clone --branch libgcrypt-1.8.8 https://github.com/gpg/libgcrypt'
cd libgcrypt
msys2 -c './autogen.sh'
msys2 -c './configure -enable-maintainer-mode --enable-static --enable-shared --with-pic --disable-doc'
msys2 -c 'make'
msys2 -c 'make install'
msys2 -c 'cd ..'
msys2 -c 'rm -rf libgcrypt/'
env:
CC:
CFLAGS:
- name: Installing MacOS prerequisites
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64')
run: |
brew install coreutils wdiff colordiff autoconf automake libtool pkg-config gettext json-c rrdtool
- name: Install MacOS Prerequisites (libgcrypt)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
brew install libgcrypt
- name: Install MacOS Prerequisites (libpcre)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
run: |
brew install pcre
- name: Install MacOS Prerequisites (maxminddb)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb')
run: |
brew install libmaxminddb
- name: Configure nDPI on Ubuntu
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
run: |
./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs
- name: Configure nDPI on MacOS
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
run: |
./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs
- name: Configure nDPI on Windows msys2
if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
run: |
msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}'
- name: Build nDPI on Windows msys2
if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
run: |
msys2 -c 'make all'
msys2 -c 'ldd ./example/ndpiReader.exe'
- name: Configure nDPI with specified GCC version on Ubuntu
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'gcc')
run: |
./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs
- name: Configure nDPI with specified CLANG on Ubuntu
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'clang')
run: |
./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs
- name: Build nDPI
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
make all
make -C example ndpiSimpleIntegration
make -C rrdtool
- name: Print nDPI long help
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
./example/ndpiReader -H
- name: Install nDPI
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
make install DESTDIR=$(realpath _install)
ls -alhHR _install
- name: Test nDPI [DIFF]
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
./tests/do.sh
- name: Test nDPI [UNIT]
#Some issues with masan + json-c. Disable the test as workaround
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer')
run: |
./tests/do-unit.sh
- name: Test nDPI [DGA]
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
./tests/do-dga.sh
- name: Test nDPI [DIFF] (runs only on windows jobs)
if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
run: |
msys2 -c './tests/do.sh'
- name: Test nDPI [UNIT] (runs only on windows jobs)
if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
run: |
msys2 -c './tests/do-unit.sh'
- name: Test nDPI [DGA] (runs only on windows jobs)
if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
run: |
msys2 -c './tests/do-dga.sh'
- name: Generate/Verify tarball
if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
run: |
make dist
./utils/verify_dist_tarball.sh
- name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.nBPF, 'nBPF')
run: |
make distclean
./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32
make all
env:
CC:
- name: Display qemu specified architecture (arm64 - little endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64')
uses: docker://multiarch/ubuntu-core:arm64-bionic
with:
args: >
bash -c
"uname -a &&
lscpu | grep Endian
"
- name: Configure, compile and test using qemu for the specified architecture (arm64 - little endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64')
uses: docker://multiarch/ubuntu-core:arm64-bionic
with:
args: >
bash -c
"apt-get -y update &&
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
"
- name: Display qemu specified architecture (armhf - little endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf')
uses: docker://multiarch/ubuntu-core:armhf-bionic
with:
args: >
bash -c
"uname -a &&
lscpu | grep Endian
"
- name: Configure, compile and test using qemu for the specified architecture (armhf - little endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf')
uses: docker://multiarch/ubuntu-core:armhf-bionic
with:
args: >
bash -c
"apt-get -y update &&
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
"
- name: Display qemu specified architecture (s390x - big endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x')
uses: docker://multiarch/ubuntu-core:s390x-bionic
with:
args: >
bash -c
"uname -a &&
lscpu | grep Endian
"
- name: Configure and compile using qemu for the specified architecture (s390x - big endian)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x')
uses: docker://multiarch/ubuntu-core:s390x-bionic
with:
args: >
bash -c
"apt-get -y update &&
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
"