From a782cd7e35008a2cb36dbe42f0ba4039084ecf69 Mon Sep 17 00:00:00 2001 From: Zied Aouini Date: Wed, 17 Aug 2022 18:58:09 +0200 Subject: [PATCH] Simplify building process. --- .../build_publish_manylinux_aarch64.yml | 1 - .../build_publish_manylinux_x86_64.yml | 1 - .github/workflows/build_test_aarch64.yml | 5 +- .github/workflows/build_test_armhf.yml | 1 + .github/workflows/build_test_linux.yml | 5 +- .../workflows/build_test_publish_macos.yml | 5 +- .../workflows/build_test_publish_windows.yml | 9 +-- .github/workflows/codeql-analysis.yml | 2 +- dev_requirements.txt | 2 +- .../scripts/{build_aarch64.sh => build.sh} | 5 +- nfstream/engine/scripts/build_linux.sh | 63 ---------------- nfstream/engine/scripts/build_macos.sh | 60 --------------- prepare.py | 51 ------------- setup.py | 73 +++++++++++-------- 14 files changed, 52 insertions(+), 231 deletions(-) rename nfstream/engine/scripts/{build_aarch64.sh => build.sh} (98%) delete mode 100755 nfstream/engine/scripts/build_linux.sh delete mode 100755 nfstream/engine/scripts/build_macos.sh delete mode 100644 prepare.py diff --git a/.github/workflows/build_publish_manylinux_aarch64.yml b/.github/workflows/build_publish_manylinux_aarch64.yml index 26d9c1b..3253991 100644 --- a/.github/workflows/build_publish_manylinux_aarch64.yml +++ b/.github/workflows/build_publish_manylinux_aarch64.yml @@ -32,7 +32,6 @@ jobs: python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 pp37-pypy37_pp73 pp38-pypy38_pp73 pp39-pypy39_pp73' build-requirements: 'cffi>=1.15.0 psutil>=5.8.0 numpy>=1.19.5 pandas>=1.1.5,<6.3 dpkt>=1.9.7' system-packages: 'autoconf automake libtool pkg-config gettext libjson-c-dev libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison python3-pip' - pre-build-command: 'python3 prepare.py' - name: Upload generated wheel uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build_publish_manylinux_x86_64.yml b/.github/workflows/build_publish_manylinux_x86_64.yml index af9eb28..8687bdf 100644 --- a/.github/workflows/build_publish_manylinux_x86_64.yml +++ b/.github/workflows/build_publish_manylinux_x86_64.yml @@ -23,7 +23,6 @@ jobs: python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 pp37-pypy37_pp73 pp38-pypy38_pp73 pp39-pypy39_pp73' build-requirements: 'cffi>=1.15.0 psutil>=5.8.0 numpy>=1.19.5 pandas>=1.1.5,<6.3 dpkt>=1.9.7' system-packages: 'autoconf automake libtool pkg-config gettext libjson-c-dev libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison python3-pip' - pre-build-command: 'python3 prepare.py' - name: Upload generated wheel uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build_test_aarch64.yml b/.github/workflows/build_test_aarch64.yml index d441273..fb361f4 100644 --- a/.github/workflows/build_test_aarch64.yml +++ b/.github/workflows/build_test_aarch64.yml @@ -32,12 +32,9 @@ jobs: ${{ matrix.python-version }} -m pip install --upgrade pip ${{ matrix.python-version }} -m pip install -r dev_requirements.txt - - name: Prepare - run: | - ${{ matrix.python-version }} prepare.py - - name: Build run: | + ${{ matrix.python-version }} setup.py build ${{ matrix.python-version }} -m pip install . - name: Test and generate coverage report diff --git a/.github/workflows/build_test_armhf.yml b/.github/workflows/build_test_armhf.yml index a66b699..73f347a 100644 --- a/.github/workflows/build_test_armhf.yml +++ b/.github/workflows/build_test_armhf.yml @@ -38,6 +38,7 @@ jobs: - name: Build run: | + ${{ matrix.python-version }} setup.py build ${{ matrix.python-version }} -m pip install . - name: Test and generate coverage report diff --git a/.github/workflows/build_test_linux.yml b/.github/workflows/build_test_linux.yml index c97d5b3..2357f15 100644 --- a/.github/workflows/build_test_linux.yml +++ b/.github/workflows/build_test_linux.yml @@ -59,12 +59,9 @@ jobs: python -m pip install --upgrade pip python -m pip install -r dev_requirements.txt - - name: Prepare - run: | - python prepare.py - - name: Build run: | + python setup.py build python -m pip install . - name: Test diff --git a/.github/workflows/build_test_publish_macos.yml b/.github/workflows/build_test_publish_macos.yml index 4187b57..ca89179 100644 --- a/.github/workflows/build_test_publish_macos.yml +++ b/.github/workflows/build_test_publish_macos.yml @@ -59,12 +59,9 @@ jobs: python -m pip install --upgrade pip python -m pip install -r dev_requirements.txt - - name: Prepare - run: | - python prepare.py - - name: Build run: | + python setup.py build python -m pip install . python -m pip wheel --no-deps --wheel-dir=dist . diff --git a/.github/workflows/build_test_publish_windows.yml b/.github/workflows/build_test_publish_windows.yml index e8b9acc..7714587 100644 --- a/.github/workflows/build_test_publish_windows.yml +++ b/.github/workflows/build_test_publish_windows.yml @@ -50,17 +50,12 @@ jobs: wget --user ${{ secrets.NPCAP_OEM_USERNAME }} --password ${{ secrets.NPCAP_OEM_PASSWORD }} https://npcap.org/oem/dist/npcap-1.60-oem.exe Start-Process npcap-1.60-oem.exe -ArgumentList "/loopback_support=yes /winpcap_mode=yes /dot11_support=yes /S" -wait - - name: Prepare + - name: Build env: MSYSTEM: MINGW64 MSYS2_PATH: D:/a/_temp/msys64 run: | - python prepare.py - - - name: Build - env: - MSYS2_PATH: D:/a/_temp/msys64 - run: | + python setup.py build python -m pip install . python -m pip wheel --no-deps --wheel-dir=dist . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 74d86c9..90ddb65 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,7 +36,7 @@ jobs: sudo apt-get install libusb-1.0-0-dev libdbus-glib-1-dev libbluetooth-dev libnl-genl-3-dev flex bison python3 -m pip install --upgrade pip python3 -m pip install -r dev_requirements.txt - python3 prepare.py + python3 setup.py build python3 -m pip install . - name: Perform CodeQL Analysis diff --git a/dev_requirements.txt b/dev_requirements.txt index 43e5e0b..4232829 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -10,4 +10,4 @@ setuptools>=57.4.0 codecov>=2.1.12 termcolor>=1.1.0 tqdm>=4.63.0 -auditwheel>=5.1.2 \ No newline at end of file +cibuildwheel==2.9.0 \ No newline at end of file diff --git a/nfstream/engine/scripts/build_aarch64.sh b/nfstream/engine/scripts/build.sh similarity index 98% rename from nfstream/engine/scripts/build_aarch64.sh rename to nfstream/engine/scripts/build.sh index 7790e0d..f6d37ed 100755 --- a/nfstream/engine/scripts/build_aarch64.sh +++ b/nfstream/engine/scripts/build.sh @@ -1,5 +1,5 @@ #----------------------------------------------------------------------------------------------------------------------- -# build_aarch64.sh +# build.sh # Copyright (C) 2019-22 - NFStream Developers # This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/). # NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -32,6 +32,7 @@ build_libndpi() { echo "Compiling libndpi" echo "---------------------------------------------------------------------------------------------------------------" cd nDPI + gcc --version ./autogen.sh CFLAGS="-I/tmp/nfstream_build/usr/local/include" LDFLAGS="-L/tmp/nfstream_build/usr/local/lib" @@ -57,4 +58,4 @@ gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c echo "---------------------------------------------------------------------------------------------------------------" echo "" -cd ../.. +cd ../.. \ No newline at end of file diff --git a/nfstream/engine/scripts/build_linux.sh b/nfstream/engine/scripts/build_linux.sh deleted file mode 100755 index 2cac559..0000000 --- a/nfstream/engine/scripts/build_linux.sh +++ /dev/null @@ -1,63 +0,0 @@ -#----------------------------------------------------------------------------------------------------------------------- -# build_linux.sh -# Copyright (C) 2019-22 - NFStream Developers -# This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/). -# NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later -# version. -# NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# You should have received a copy of the GNU Lesser General Public License along with NFStream. -# If not, see . -# ---------------------------------------------------------------------------------------------------------------------- - -build_libpcap() { - echo "" - echo "---------------------------------------------------------------------------------------------------------------" - echo "Compiling libpcap (fanout)" - echo "---------------------------------------------------------------------------------------------------------------" - cd libpcap - git config --global --add safe.directory $(realpath .) && - ./configure --enable-ipv6 --disable-universal --enable-dbus=no --without-libnl --disable-rdma - make - make DESTDIR=/tmp/nfstream_build install - make clean - cd .. - echo "---------------------------------------------------------------------------------------------------------------" - echo "" - } - -build_libndpi() { - echo "" - echo "---------------------------------------------------------------------------------------------------------------" - echo "Compiling libndpi" - echo "---------------------------------------------------------------------------------------------------------------" - cd nDPI - git config --global --add safe.directory $(realpath .) && - gcc --version - ./autogen.sh - CFLAGS="-I/tmp/nfstream_build/usr/local/include" - LDFLAGS="-L/tmp/nfstream_build/usr/local/lib" - CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} ./configure && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} make - make DESTDIR=/tmp/nfstream_build install - make clean - cd .. - echo "---------------------------------------------------------------------------------------------------------------" - echo "" - } - -rm -rf /tmp/nfstream_build -cd nfstream/engine/dependencies -build_libpcap -build_libndpi -echo "" -echo "---------------------------------------------------------------------------------------------------------------" -echo "Prepare engine_cc" -echo "---------------------------------------------------------------------------------------------------------------" -cd .. -gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions.h -gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions_packed.h -gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c -echo "---------------------------------------------------------------------------------------------------------------" -echo "" -cd ../.. \ No newline at end of file diff --git a/nfstream/engine/scripts/build_macos.sh b/nfstream/engine/scripts/build_macos.sh deleted file mode 100755 index f6b67e9..0000000 --- a/nfstream/engine/scripts/build_macos.sh +++ /dev/null @@ -1,60 +0,0 @@ -#----------------------------------------------------------------------------------------------------------------------- -# build_macos.sh -# Copyright (C) 2019-22 - NFStream Developers -# This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/). -# NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later -# version. -# NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# You should have received a copy of the GNU Lesser General Public License along with NFStream. -# If not, see . -# ---------------------------------------------------------------------------------------------------------------------- - -build_libpcap() { - echo "" - echo "---------------------------------------------------------------------------------------------------------------" - echo "Compiling libpcap (fanout)" - echo "---------------------------------------------------------------------------------------------------------------" - cd libpcap - ./configure --enable-ipv6 --disable-universal --enable-dbus=no --without-libnl --disable-rdma - make - make DESTDIR=/tmp/nfstream_build install - make clean - cd .. - echo "---------------------------------------------------------------------------------------------------------------" - echo "" - } - -build_libndpi() { - echo "" - echo "---------------------------------------------------------------------------------------------------------------" - echo "Compiling libndpi" - echo "---------------------------------------------------------------------------------------------------------------" - cd nDPI - ./autogen.sh - CFLAGS="-I/tmp/nfstream_build/usr/local/include" - LDFLAGS="-L/tmp/nfstream_build/usr/local/lib" - CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} ./configure && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} make - make DESTDIR=/tmp/nfstream_build install - make clean - cd .. - echo "---------------------------------------------------------------------------------------------------------------" - echo "" - } - -rm -rf /tmp/nfstream_build -cd nfstream/engine/dependencies -build_libpcap -build_libndpi -echo "" -echo "---------------------------------------------------------------------------------------------------------------" -echo "Prepare engine_cc" -echo "---------------------------------------------------------------------------------------------------------------" -cd .. -gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions.h -gcc -DNDPI_LIB_COMPILATION -DNDPI_CFFI_PREPROCESSING -E -x c -P -C /tmp/nfstream_build/usr/include/ndpi/ndpi_typedefs.h > /tmp/nfstream_build/ndpi_cdefinitions_packed.h -gcc -E -x c -P -C lib_engine.c > /tmp/nfstream_build/lib_engine_cdefinitions.c -echo "---------------------------------------------------------------------------------------------------------------" -echo "" -cd ../.. \ No newline at end of file diff --git a/prepare.py b/prepare.py deleted file mode 100644 index 8439588..0000000 --- a/prepare.py +++ /dev/null @@ -1,51 +0,0 @@ -""" ------------------------------------------------------------------------------------------------------------------------- -prepare.py -Copyright (C) 2019-22 - NFStream Developers -This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/). -NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later -version. -NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -You should have received a copy of the GNU Lesser General Public License along with NFStream. -If not, see . ------------------------------------------------------------------------------------------------------------------------- -""" - -import platform -import subprocess -import pathlib -import sys -import os - -BUILD_SCRIPT_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine") - .joinpath("scripts").joinpath("build")) - -# Patched path as it is passed to msys2 bash -ENGINE_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")).replace("\\", "/") - - -def prepare_lib_engine_requirements(): - if os.name != 'posix': # Windows case - os.environ["MSYSTEM"] = "MINGW64" - msys = os.getenv("MSYS2_PATH") - if msys is None: - os.environ["MSYS2_PATH"] = "C:/msys64" - msys = os.getenv("MSYS2_PATH") - build_script_command = r"""'{}'""".format(str(BUILD_SCRIPT_PATH) + "_windows.sh") - subprocess.check_call(["{msys}/usr/bin/bash".format(msys=msys).replace("/", "\\"), - "-l", - build_script_command, ENGINE_PATH], - shell=True) - else: - if sys.platform == 'darwin': - subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_macos.sh"], shell=True) - elif "aarch" in platform.machine(): - subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_aarch64.sh"], shell=True) - else: - subprocess.check_call([str(BUILD_SCRIPT_PATH) + "_linux.sh"], shell=True) - - -if __name__ == "__main__": - prepare_lib_engine_requirements() \ No newline at end of file diff --git a/setup.py b/setup.py index 1e2c913..dda2249 100644 --- a/setup.py +++ b/setup.py @@ -13,67 +13,76 @@ If not, see . ------------------------------------------------------------------------------------------------------------------------ """ +from setuptools.command.build_py import build_py +from setuptools import setup +import subprocess +import platform +import pathlib import sys import os -import platform -from setuptools import setup + +THIS_DIRECTORY = str(pathlib.Path(__file__).parent.resolve()) + +BUILD_SCRIPT_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine") + .joinpath("scripts").joinpath("build")) + +# Patched path as it is passed to msys2 bash +ENGINE_PATH = str(pathlib.Path(__file__).parent.resolve().joinpath("nfstream").joinpath("engine")).replace("\\", "/") if (not sys.version_info[0] == 3) and (not sys.version_info[1] >= 6): sys.exit("Sorry, nfstream requires Python3.6+ versions.") -THIS_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) - - with open(os.path.join(THIS_DIRECTORY, 'README.md'), encoding='utf-8') as f: - long_description = f.read() + LONG_DESCRIPTION = f.read() -install_requires = ['cffi>=1.15.0', +INSTALL_REQUIRES = ['cffi>=1.15.0', 'psutil>=5.8.0', 'dpkt>=1.9.7', 'numpy>=1.19.5'] -# This is mandatory to fix pandas issues with PyPy -if platform.python_implementation() == 'PyPy': - install_requires.append("pandas<=1.2.5") +if platform.python_implementation() == 'PyPy': # This is mandatory to fix pandas issues with PyPy + INSTALL_REQUIRES.append("pandas<=1.2.5") else: - install_requires.append("pandas>=1.1.5") - -try: - from wheel.bdist_wheel import bdist_wheel as _bdist_wheel - - class bdist_wheel(_bdist_wheel): - def get_tag(self): - tag = _bdist_wheel.get_tag(self) - pypi_compliant_tag = list(tag) - if 'linux' == pypi_compliant_tag[2][0:5]: - pypi_compliant_tag[2] = pypi_compliant_tag[2].replace("linux", "manylinux1") - if pypi_compliant_tag[2] == "manylinux1_aarch64": - pypi_compliant_tag[2] = "manylinux2014_aarch64" - pypi_compliant_tag = tuple(pypi_compliant_tag) - return pypi_compliant_tag + INSTALL_REQUIRES.append("pandas>=1.1.5") -except ImportError: - print('Warning: cannot import "wheel" package to build platform-specific wheel') - print('Install the "wheel" package to fix this warning') - bdist_wheel = None +class BuildPyCommand(build_py): + """ Custom build command to compile lib_engine dependencies.""" + def run(self): + if not self.dry_run: + if os.name != 'posix': # Windows case + os.environ["MSYSTEM"] = "MINGW64" + msys = os.getenv("MSYS2_PATH") + if msys is None: + os.environ["MSYS2_PATH"] = "C:/msys64" + msys = os.getenv("MSYS2_PATH") + build_script_command = r"""'{}'""".format(str(BUILD_SCRIPT_PATH) + "_windows.sh") + subprocess.check_call(["{msys}/usr/bin/bash".format(msys=msys).replace("/", "\\"), + "-l", + build_script_command, ENGINE_PATH], + shell=True) + else: # Linux, MacOS + subprocess.check_call([str(BUILD_SCRIPT_PATH) + ".sh"], shell=True) + build_py.run(self) -cmdclass = {'bdist_wheel': bdist_wheel} if bdist_wheel is not None else dict() setup( + cmdclass={ + "build_py": BuildPyCommand + }, name="nfstream", version='6.5.2', url='https://www.nfstream.org/', license='LGPLv3', description="A Flexible Network Data Analysis Framework", - long_description=long_description, + long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', author='Zied Aouini', author_email='aouinizied@gmail.com', packages=['nfstream', 'nfstream.plugins', 'nfstream.engine'], setup_requires=["cffi>=1.15.0"], cffi_modules=["nfstream/engine/engine_build.py:ffi_builder"], - install_requires=install_requires, + install_requires=INSTALL_REQUIRES, include_package_data=True, platforms=["Linux", "Mac OS-X", "Windows", "Unix"], classifiers=[