vrr/nfstream
2
0
Fork 0
mirror of https://github.com/nfstream/nfstream.git synced 2026-05-19 16:28:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Simplify building process.

Browse source
This commit is contained in:
Zied Aouini 2022-08-17 18:58:09 +02:00
parent 2b16a64243
commit a782cd7e35
14 changed files with 52 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/build_publish_manylinux_aarch64.yml vendored
View file

@ -32,7 +32,6 @@ jobs:
python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 pp37-pypy37_pp73 pp38-pypy38_pp73 pp39-pypy39_pp73'
build-requirements: 'cffi>=1.15.0 psutil>=5.8.0 numpy>=1.19.5 pandas>=1.1.5,<6.3 dpkt>=1.9.7'
system-packages: 'autoconf automake libtool pkg-config gettext libjson-c-dev libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison python3-pip'
pre-build-command: 'python3 prepare.py'
- name: Upload generated wheel
uses: actions/upload-artifact@v3

1
.github/workflows/build_publish_manylinux_x86_64.yml vendored
View file

@ -23,7 +23,6 @@ jobs:
python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 pp37-pypy37_pp73 pp38-pypy38_pp73 pp39-pypy39_pp73'
build-requirements: 'cffi>=1.15.0 psutil>=5.8.0 numpy>=1.19.5 pandas>=1.1.5,<6.3 dpkt>=1.9.7'
system-packages: 'autoconf automake libtool pkg-config gettext libjson-c-dev libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison python3-pip'
pre-build-command: 'python3 prepare.py'
- name: Upload generated wheel
uses: actions/upload-artifact@v3

5
.github/workflows/build_test_aarch64.yml vendored
View file

@ -32,12 +32,9 @@ jobs:
${{ matrix.python-version }} -m pip install --upgrade pip
${{ matrix.python-version }} -m pip install -r dev_requirements.txt
- name: Prepare
run: |
${{ matrix.python-version }} prepare.py
- name: Build
run: |
${{ matrix.python-version }} setup.py build
${{ matrix.python-version }} -m pip install .
- name: Test and generate coverage report

1
.github/workflows/build_test_armhf.yml vendored
View file

@ -38,6 +38,7 @@ jobs:
- name: Build
run: |
${{ matrix.python-version }} setup.py build
${{ matrix.python-version }} -m pip install .
- name: Test and generate coverage report

5
.github/workflows/build_test_linux.yml vendored
View file

@ -59,12 +59,9 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r dev_requirements.txt
- name: Prepare
run: |
python prepare.py
- name: Build
run: |
python setup.py build
python -m pip install .
- name: Test

5
.github/workflows/build_test_publish_macos.yml vendored
View file

@ -59,12 +59,9 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r dev_requirements.txt
- name: Prepare
run: |
python prepare.py
- name: Build
run: |
python setup.py build
python -m pip install .
python -m pip wheel --no-deps --wheel-dir=dist .

9
.github/workflows/build_test_publish_windows.yml vendored
View file

@ -50,17 +50,12 @@ jobs:
wget --user ${{ secrets.NPCAP_OEM_USERNAME }} --password ${{ secrets.NPCAP_OEM_PASSWORD }} https://npcap.org/oem/dist/npcap-1.60-oem.exe
Start-Process npcap-1.60-oem.exe -ArgumentList "/loopback_support=yes /winpcap_mode=yes /dot11_support=yes /S" -wait
- name: Prepare
- name: Build
env:
MSYSTEM: MINGW64
MSYS2_PATH: D:/a/_temp/msys64
run: |
python prepare.py
- name: Build
env:
MSYS2_PATH: D:/a/_temp/msys64
run: |
python setup.py build
python -m pip install .
python -m pip wheel --no-deps --wheel-dir=dist .

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -36,7 +36,7 @@ jobs:
sudo apt-get install libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison
python3 -m pip install --upgrade pip
python3 -m pip install -r dev_requirements.txt
python3 prepare.py
python3 setup.py build
python3 -m pip install .
- name: Perform CodeQL Analysis

2
dev_requirements.txt
View file

@ -10,4 +10,4 @@ setuptools>=57.4.0
codecov>=2.1.12
termcolor>=1.1.0
tqdm>=4.63.0
auditwheel>=5.1.2
cibuildwheel==2.9.0

5
nfstream/engine/scripts/build_aarch64.sh → nfstream/engine/scripts/build.sh
View file

@ -1,5 +1,5 @@
#-----------------------------------------------------------------------------------------------------------------------
# build_aarch64.sh
# build.sh
# Copyright (C) 2019-22 - NFStream Developers
# This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
# NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
@ -32,6 +32,7 @@ build_libndpi() {
echo "Compiling libndpi"
echo "---------------------------------------------------------------------------------------------------------------"
cd nDPI
gcc --version
./autogen.sh
CFLAGS="-I/tmp/nfstream_build/usr/local/include"
LDFLAGS="-L/tmp/nfstream_build/usr/local/lib"
@ -57,4 +58,4 @@ gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream
gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
cd ../..
cd ../..

63
nfstream/engine/scripts/build_linux.sh
View file

@ -1,63 +0,0 @@
#-----------------------------------------------------------------------------------------------------------------------
# build_linux.sh
# Copyright (C) 2019-22 - NFStream Developers
# This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
# NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
# version.
# NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public License along with NFStream.
# If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------------------------------------------------------
build_libpcap() {
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Compiling libpcap (fanout)"
echo "---------------------------------------------------------------------------------------------------------------"
cd libpcap
git config --global --add safe.directory $(realpath .) &&
./configure --enable-ipv6 --disable-universal --enable-dbus=no --without-libnl --disable-rdma
make
make DESTDIR=/tmp/nfstream_build install
make clean
cd ..
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
}
build_libndpi() {
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Compiling libndpi"
echo "---------------------------------------------------------------------------------------------------------------"
cd nDPI
git config --global --add safe.directory $(realpath .) &&
gcc --version
./autogen.sh
CFLAGS="-I/tmp/nfstream_build/usr/local/include"
LDFLAGS="-L/tmp/nfstream_build/usr/local/lib"
CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} ./configure && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} make
make DESTDIR=/tmp/nfstream_build install
make clean
cd ..
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
}
rm -rf /tmp/nfstream_build
cd nfstream/engine/dependencies
build_libpcap
build_libndpi
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Prepare engine_cc"
echo "---------------------------------------------------------------------------------------------------------------"
cd ..
gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions.h
gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions_packed.h
gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
cd ../..

60
nfstream/engine/scripts/build_macos.sh
View file

@ -1,60 +0,0 @@
#-----------------------------------------------------------------------------------------------------------------------
# build_macos.sh
# Copyright (C) 2019-22 - NFStream Developers
# This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
# NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
# version.
# NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public License along with NFStream.
# If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------------------------------------------------------
build_libpcap() {
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Compiling libpcap (fanout)"
echo "---------------------------------------------------------------------------------------------------------------"
cd libpcap
./configure --enable-ipv6 --disable-universal --enable-dbus=no --without-libnl --disable-rdma
make
make DESTDIR=/tmp/nfstream_build install
make clean
cd ..
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
}
build_libndpi() {
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Compiling libndpi"
echo "---------------------------------------------------------------------------------------------------------------"
cd nDPI
./autogen.sh
CFLAGS="-I/tmp/nfstream_build/usr/local/include"
LDFLAGS="-L/tmp/nfstream_build/usr/local/lib"
CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} ./configure && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} make
make DESTDIR=/tmp/nfstream_build install
make clean
cd ..
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
}
rm -rf /tmp/nfstream_build
cd nfstream/engine/dependencies
build_libpcap
build_libndpi
echo ""
echo "---------------------------------------------------------------------------------------------------------------"
echo "Prepare engine_cc"
echo "---------------------------------------------------------------------------------------------------------------"
cd ..
gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions.h
gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions_packed.h
gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c
echo "---------------------------------------------------------------------------------------------------------------"
echo ""
cd ../..

51
prepare.py
View file

@ -1,51 +0,0 @@
"""
------------------------------------------------------------------------------------------------------------------------
prepare.py
Copyright (C) 2019-22 - NFStream Developers
This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
import platform
import subprocess
import pathlib
import sys
import os
BUILD_SCRIPT_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")
.joinpath("scripts").joinpath("build"))
# Patched path as it is passed to msys2 bash
ENGINE_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")).replace("\\", "/")
def prepare_lib_engine_requirements():
if os.name != 'posix': # Windows case
os.environ["MSYSTEM"] = "MINGW64"
msys = os.getenv("MSYS2_PATH")
if msys is None:
os.environ["MSYS2_PATH"] = "C:/msys64"
msys = os.getenv("MSYS2_PATH")
build_script_command = r"""'{}'""".format(str(BUILD_SCRIPT_PATH) + "_windows.sh")
subprocess.check_call(["{msys}/usr/bin/bash".format(msys=msys).replace("/", "\\"),
"-l",
build_script_command, ENGINE_PATH],
shell=True)
else:
if sys.platform == 'darwin':
subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_macos.sh"], shell=True)
elif "aarch" in platform.machine():
subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_aarch64.sh"], shell=True)
else:
subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_linux.sh"], shell=True)
if __name__ == "__main__":
prepare_lib_engine_requirements()

73
setup.py
View file

@ -13,67 +13,76 @@ If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from setuptools.command.build_py import build_py
from setuptools import setup
import subprocess
import platform
import pathlib
import sys
import os
import platform
from setuptools import setup
THIS_DIRECTORY = str(pathlib.Path(__file__).parent.resolve())
BUILD_SCRIPT_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")
.joinpath("scripts").joinpath("build"))
# Patched path as it is passed to msys2 bash
ENGINE_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")).replace("\\", "/")
if (not sys.version_info[0] == 3) and (not sys.version_info[1] >= 6):
sys.exit("Sorry, nfstream requires Python3.6+ versions.")
THIS_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(THIS_DIRECTORY, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
LONG_DESCRIPTION = f.read()
install_requires = ['cffi>=1.15.0',
INSTALL_REQUIRES = ['cffi>=1.15.0',
'psutil>=5.8.0',
'dpkt>=1.9.7',
'numpy>=1.19.5']
# This is mandatory to fix pandas issues with PyPy
if platform.python_implementation() == 'PyPy':
install_requires.append("pandas<=1.2.5")
if platform.python_implementation() == 'PyPy': # This is mandatory to fix pandas issues with PyPy
INSTALL_REQUIRES.append("pandas<=1.2.5")
else:
install_requires.append("pandas>=1.1.5")
try:
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
class bdist_wheel(_bdist_wheel):
def get_tag(self):
tag = _bdist_wheel.get_tag(self)
pypi_compliant_tag = list(tag)
if 'linux' == pypi_compliant_tag[2][0:5]:
pypi_compliant_tag[2] = pypi_compliant_tag[2].replace("linux", "manylinux1")
if pypi_compliant_tag[2] == "manylinux1_aarch64":
pypi_compliant_tag[2] = "manylinux2014_aarch64"
pypi_compliant_tag = tuple(pypi_compliant_tag)
return pypi_compliant_tag
INSTALL_REQUIRES.append("pandas>=1.1.5")
except ImportError:
print('Warning: cannot import "wheel" package to build platform-specific wheel')
print('Install the "wheel" package to fix this warning')
bdist_wheel = None
class BuildPyCommand(build_py):
""" Custom build command to compile lib_engine dependencies."""
def run(self):
if not self.dry_run:
if os.name != 'posix': # Windows case
os.environ["MSYSTEM"] = "MINGW64"
msys = os.getenv("MSYS2_PATH")
if msys is None:
os.environ["MSYS2_PATH"] = "C:/msys64"
msys = os.getenv("MSYS2_PATH")
build_script_command = r"""'{}'""".format(str(BUILD_SCRIPT_PATH) + "_windows.sh")
subprocess.check_call(["{msys}/usr/bin/bash".format(msys=msys).replace("/", "\\"),
"-l",
build_script_command, ENGINE_PATH],
shell=True)
else: # Linux, MacOS
subprocess.check_call([str(BUILD_SCRIPT_PATH) + ".sh"], shell=True)
build_py.run(self)
cmdclass = {'bdist_wheel': bdist_wheel} if bdist_wheel is not None else dict()
setup(
cmdclass={
"build_py": BuildPyCommand
},
name="nfstream",
version='6.5.2',
url='https://www.nfstream.org/',
license='LGPLv3',
description="A Flexible Network Data Analysis Framework",
long_description=long_description,
long_description=LONG_DESCRIPTION,
long_description_content_type='text/markdown',
author='Zied Aouini',
author_email='aouinizied@gmail.com',
packages=['nfstream', 'nfstream.plugins', 'nfstream.engine'],
setup_requires=["cffi>=1.15.0"],
cffi_modules=["nfstream/engine/engine_build.py:ffi_builder"],
install_requires=install_requires,
install_requires=INSTALL_REQUIRES,
include_package_data=True,
platforms=["Linux", "Mac OS-X", "Windows", "Unix"],
classifiers=[