From f7fdc40c7244ffc2de8615174fcd83ecd511a6ab Mon Sep 17 00:00:00 2001 From: Simone Mainardi Date: Mon, 6 Aug 2018 17:54:25 +0200 Subject: [PATCH] Major nEdge code cleanup --- configure.seed | 54 +- include/Flow.h | 22 +- include/NetworkInterface.h | 2 + include/ntop_includes.h | 4 +- scripts/lua/if_stats.lua | 996 +---------------------- scripts/lua/inc/footer.lua | 7 - scripts/lua/modules/host_pools_utils.lua | 4 +- scripts/lua/network_load.lua | 21 - src/Flow.cpp | 40 +- src/FlowGrouper.cpp | 4 +- src/HostPools.cpp | 2 +- src/LuaEngine.cpp | 50 +- src/NetworkInterface.cpp | 30 +- src/main.cpp | 2 +- 14 files changed, 99 insertions(+), 1139 deletions(-) diff --git a/configure.seed b/configure.seed index 65b6e33202..d9f5765770 100644 --- a/configure.seed +++ b/configure.seed @@ -21,6 +21,24 @@ NEDGE=0 if test "${with_nedge+set}" = set; then AC_DEFINE(HAVE_NEDGE, 1, [nedge support is present]) NEDGE=1 + + AC_CHECK_HEADERS([netinet/in.h libnetfilter_queue/libnetfilter_queue.h]) + AC_CHECK_LIB([netfilter_queue], [nfq_create_queue]) + AC_CHECK_LIB([nfnetlink], [nfnl_rcvbufsiz]) + AC_CHECK_LIB([netfilter_conntrack], [nfct_filter_dump_create], nfct=true) + AC_CHECK_FUNC([nfq_set_verdict2], [AC_DEFINE([HAVE_NFQ_SET_VERDICT2],[1],[Define if nfq_set_verdict2 exists in netfilter_queue.])]) + + if test "x$ac_cv_lib_netfilter_queue_nfq_create_queue" != xyes; then + AC_MSG_RESULT(Cannot build nedge without Netfilter, exiting.) + exit 1 + fi + + if test x$nfct = x; then + AC_MSG_RESULT(Cannot build nedge without Conntrack, exiting.) + exit 1 + fi + + LDFLAGS="${LDFLAGS} -lnetfilter_queue -lnfnetlink -lnetfilter_conntrack" fi # @@ -136,7 +154,7 @@ if test -z "$NDPI_INC" || test -z "$NDPI_LIB" ; then : echo "Could not find nDPI" echo "Please do cd ..; git clone https://github.com/ntop/nDPI.git; cd nDPI; ./autogen.sh; make; cd ../ntopng" echo "and try again" - exit + exit 1 fi dnl finish: nDPI handling @@ -279,7 +297,7 @@ if test "${with_nedge+set}" != set; then echo "Using PF_RING installed in ${PF_RING_HOME}" else echo "Found PF_RING but the userland libs have not been compiled" - exit + exit 1 fi else if test -d "/opt/pfring"; then @@ -297,12 +315,12 @@ if test "${with_nedge+set}" != set; then if test x$pcap = x then echo "Found /usr/local/lib/libpfring.a but missing header files." - exit + exit 1 fi else echo "Found /usr/local/lib/libpfring.a but missing libpcap.a in the same directory" echo "Please make sure you have installed your PF_RING libraries and try again" - exit + exit 1 fi fi fi @@ -316,18 +334,11 @@ else if test x$pcap = x then echo "Please install libpcap(-dev) (http://tcpdump.org)" - exit + exit 1 fi fi fi -AC_CHECK_LIB([netfilter_conntrack], [nfct_filter_dump_create], nfct=true) -if test x$nfct != x -then - AC_DEFINE([HAVE_CONNTRACK],[1],[Support for Linux conntrack]) - LDFLAGS="${LDFLAGS} -lnetfilter_conntrack" -fi - if test "${with_nedge+set}" != set; then PFRING_GIT_RELEASE="" if test -d $HOME/PF_RING; then @@ -340,16 +351,6 @@ if test "${with_nedge+set}" != set; then fi fi -AC_CHECK_HEADERS([netinet/in.h libnetfilter_queue/libnetfilter_queue.h]) -AC_CHECK_LIB([netfilter_queue], [nfq_create_queue]) -AC_CHECK_LIB([nfnetlink], [nfnl_rcvbufsiz]) -AC_CHECK_FUNC([nfq_set_verdict2], [AC_DEFINE([HAVE_NFQ_SET_VERDICT2],[1],[Define if nfq_set_verdict2 exists in netfilter_queue.])]) - -if test "x$ac_cv_lib_netfilter_queue_nfq_create_queue" = xyes; then - AC_DEFINE_UNQUOTED(HAVE_NETFILTER, 1, [NFQ is present]) - LDFLAGS="${LDFLAGS} -lnetfilter_queue -lnfnetlink" -fi - AC_CHECK_LIB([cap], [cap_get_proc], cap=true) if test x$cap != x then @@ -382,7 +383,7 @@ AC_CHECK_LIB([dl], [dlopen], [LDFLAGS="${LDFLAGS} -ldl"]) AC_CHECK_LIB([curl], [curl_easy_perform], [LDFLAGS="${LDFLAGS} -lcurl"]) if test ${ac_cv_lib_curl_curl_easy_perform} = "no"; then echo "Please install libcurl(-dev) (http://curl.haxx.se/)" - exit + exit 1 else dnl> libcurl has CURLOPT_MAIL_RCPT only starting at version 7.20.0 dnl> https://github.com/curl/curl/blob/curl-7_20_0/TODO-RELEASE @@ -435,10 +436,9 @@ if test ".${ac_cv_lib_maxminddb_MMDB_lookup_sockaddr}" = ".yes" && AC_DEFINE_UNQUOTED(HAVE_MAXMINDDB, 1, [MaxMind DB support]) else AC_MSG_RESULT(Please install libmaxminddb-dev (https://github.com/maxmind/libmaxminddb) and try again) - exit + exit 1 fi - AC_CHECK_LIB([sqlite3], [sqlite3_open], sqlite=true) if test x$sqlite = xtrue then @@ -446,7 +446,7 @@ then SQLITE_LIB="-lsqlite3" else echo "SQLite 3.x missing (libsqlite3-dev): please install it and try again" - exit + exit 1 fi if test "${with_nedge+set}" != set; then @@ -476,7 +476,7 @@ if test -n "$MYSQLTOOL"; then AC_CHECK_LIB([wrap], [main]) else echo "MySQL libraries not found. Please install them as specified in README.compilation" - exit 0 + exit 1 fi fi diff --git a/include/Flow.h b/include/Flow.h index 31ee41a1e3..0e1621974a 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -172,7 +172,7 @@ class Flow : public GenericHashEntry { u_int32_t last_dump; } last_db_dump; -#ifdef NTOPNG_PRO +#ifdef HAVE_NEDGE struct { struct { TrafficShaper *ingress, *egress; @@ -220,7 +220,6 @@ class Flow : public GenericHashEntry { char* printTCPflags(u_int8_t flags, char *buf, u_int buf_len); inline bool isProto(u_int16_t p ) { return((ndpi_get_lower_proto(ndpiDetectedProtocol) == p) ? true : false); } #ifdef NTOPNG_PRO - bool updateDirectionShapers(bool src2dst_direction, TrafficShaper **ingress_shaper, TrafficShaper **egress_shaper); void update_pools_stats(const struct timeval *tv, u_int64_t diff_sent_packets, u_int64_t diff_sent_bytes, u_int64_t diff_rcvd_packets, u_int64_t diff_rcvd_bytes); @@ -265,7 +264,7 @@ class Flow : public GenericHashEntry { inline u_int8_t getTcpFlags() { return(src2dst_tcp_flags | dst2src_tcp_flags); }; inline u_int8_t getTcpFlagsCli2Srv() { return(src2dst_tcp_flags); }; inline u_int8_t getTcpFlagsSrv2Cli() { return(dst2src_tcp_flags); }; -#ifdef NTOPNG_PRO +#ifdef HAVE_NEDGE bool checkPassVerdict(const struct tm *now); bool isPassVerdict(); #endif @@ -417,13 +416,10 @@ class Flow : public GenericHashEntry { FlowSSLEncryptionStatus getSSLEncryptionStatus(); void setDumpFlowTraffic(bool what) { dump_flow_traffic = what; } bool getDumpFlowTraffic(void) { return dump_flow_traffic; } -#ifdef NTOPNG_PRO -#ifndef HAVE_NEDGE + +#if defined(NTOPNG_PRO) && !defined(HAVE_NEDGE) inline void updateProfile() { trafficProfile = iface->getFlowProfile(this); } inline char* get_profile_name() { return(trafficProfile ? trafficProfile->getName() : (char*)"");} -#endif - void updateFlowShapers(bool first_update=false); - void recheckQuota(const struct tm *now); #endif inline float getFlowRTT() { return(rttSec); } /* http://bradhedlund.com/2008/12/19/how-to-calculate-tcp-throughput-for-long-distance-links/ */ @@ -464,12 +460,8 @@ class Flow : public GenericHashEntry { bool isNetfilterIdleFlow(); #endif -#ifdef NTOPNG_PRO - -#ifdef HAVE_NETFILTER +#ifdef HAVE_NEDGE void setPacketsBytes(time_t now, u_int32_t s2d_pkts, u_int32_t d2s_pkts, u_int64_t s2d_bytes, u_int64_t d2s_bytes); -#endif - void getFlowShapers(bool src2dst_direction, TrafficShaper **shaper_ingress, TrafficShaper **shaper_egress) { if(src2dst_direction) { *shaper_ingress = flowShaperIds.cli2srv.ingress, @@ -479,7 +471,9 @@ class Flow : public GenericHashEntry { *shaper_egress = flowShaperIds.srv2cli.egress; } } - + bool updateDirectionShapers(bool src2dst_direction, TrafficShaper **ingress_shaper, TrafficShaper **egress_shaper); + void updateFlowShapers(bool first_update=false); + void recheckQuota(const struct tm *now); inline u_int8_t getFlowRoutingTableId() { return(routing_table_id); } inline void setIngress2EgressDirection(bool _ingress2egress) { ingress2egress_direction = _ingress2egress; } inline bool isIngress2EgressDirection() { return(ingress2egress_direction); } diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 28c71a9556..ad2e63259c 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -519,8 +519,10 @@ class NetworkInterface : public Checkpointable { bool deregisterLiveCapture(struct ntopngLuaContext * const luactx); #ifdef NTOPNG_PRO +#ifdef HAVE_NEDGE void updateHostsL7Policy(u_int16_t host_pool_id); void updateFlowsL7Policy(); +#endif void resetPoolsStats(u_int16_t pool_filter); inline void luaHostPoolsStats(lua_State *vm) { if (host_pools) host_pools->luaStats(vm); }; inline void luaHostPoolsVolatileMembers(lua_State *vm) { if (host_pools) host_pools->luaVolatileMembers(vm); }; diff --git a/include/ntop_includes.h b/include/ntop_includes.h index 3b3127638e..22944bd985 100644 --- a/include/ntop_includes.h +++ b/include/ntop_includes.h @@ -116,7 +116,7 @@ extern "C" { #include "pfring.h" #include "pfring_zc.h" #endif -#ifdef HAVE_NETFILTER +#ifdef HAVE_NEDGE #include #include /* for NF_ACCEPT */ #include @@ -267,7 +267,7 @@ using namespace std; #include "FrequentStringItems.h" #include "FrequentNumericItems.h" #include "FrequentTrafficItems.h" -#if defined(HAVE_NETFILTER) && defined(HAVE_NEDGE) +#ifdef HAVE_NEDGE #include "NetfilterInterface.h" #endif #endif diff --git a/scripts/lua/if_stats.lua b/scripts/lua/if_stats.lua index 5f6774bfd4..8ee25d900c 100644 --- a/scripts/lua/if_stats.lua +++ b/scripts/lua/if_stats.lua @@ -161,9 +161,7 @@ end ntop.dumpFile(dirs.installdir .. "/httpdocs/inc/header.inc") print("") -if page == "filtering" then - print('') -end + dofile(dirs.installdir .. "/scripts/lua/inc/menu.lua") print(msg) @@ -308,14 +306,6 @@ if(hasSnmpDevices(ifstats.id) and is_packet_interface and false --[[disabled: no end end -if(ifstats.inline and isAdministrator() and not have_nedge) then - if(page == "filtering") then - print("
  • "..i18n('traffic_policy').."
    • ") - else - print("
  • "..i18n('traffic_policy').."
    • ") - end -end - local ifname_clean = "iface_"..tostring(ifid) print [[ @@ -633,29 +623,6 @@ print("\n") print("") end - if((ifstats["bridge.device_a"] ~= nil) and (ifstats["bridge.device_b"] ~= nil)) then - print(""..i18n("if_stats_overview.bridged_traffic").."\n") - print(""..i18n("interface")..""..i18n("if_stats_overview.ingress_packets")..""..i18n("if_stats_overview.egress_packets")..""..i18n("if_stats_overview.shaped_filtered_packets")..""..i18n("if_stats_overview.send_error")..""..i18n("if_stats_overview.buffer_full").."\n") - print("".. ifstats["bridge.device_a"] .. "".. formatPackets(ifstats["bridge.a_to_b.in_pkts"]) .." ") - print("".. formatPackets(ifstats["bridge.a_to_b.out_pkts"]) .." ") - print("".. formatPackets(ifstats["bridge.a_to_b.filtered_pkts"]) .."") - - print("".. formatPackets(ifstats["bridge.a_to_b.num_pkts_send_error"]) .."") - print("".. formatPackets(ifstats["bridge.a_to_b.num_pkts_send_buffer_full"]) .."") - - print("\n") - - print("".. ifstats["bridge.device_b"] .. "".. formatPackets(ifstats["bridge.b_to_a.in_pkts"]) .." ") - print(""..formatPackets( ifstats["bridge.b_to_a.out_pkts"]) .." ") - - print("".. formatPackets(ifstats["bridge.b_to_a.filtered_pkts"]) .."") - - print("".. formatPackets(ifstats["bridge.b_to_a.num_pkts_send_error"]) .."") - print("".. formatPackets(ifstats["bridge.b_to_a.num_pkts_send_buffer_full"]) .."") - - print("\n") - end - print [[ ]] print(i18n("if_stats_overview.note")..":

    "..i18n("if_stats_overview.note_packets")) print[[ ]] @@ -1461,892 +1428,6 @@ elseif(page == "pools") then end dofile(dirs.installdir .. "/scripts/lua/admin/host_pools.lua") -elseif(page == "filtering") then - if not isAdministrator() then - error() - end - - -- ==================================== - - -- possibly decode parameters pairs - local _POST = paramsPairsDecode(_POST) - - local perPageProtos - if tonumber(tablePreferences("protocolShapers")) == nil then - perPageProtos = "10" - else - perPageProtos = tablePreferences("protocolShapers") - end - - -- TODO refactor view_network logic - - local selected_pool_id = _GET["pool"] or _POST["target_pool"] - local selected_pool = nil - - local available_pools = host_pools_utils.getPoolsList(ifId) - - for _, pool in ipairs(available_pools) do - if pool.id == selected_pool_id then - selected_pool = pool - end - end - - if selected_pool == nil then - selected_pool = available_pools[2] or available_pools[1] - end - - local SHAPERS_MAX_RATE_KPBS = 100*1000*1000 -- 100 Gbit/s - - if(_POST["add_shapers"] ~= nil) then - local num_added = 0 - local last_added = nil - for shaper,mrate in pairs(_POST) do - local sp = split(shaper, "shaper_") - if #sp == 2 then - local shaper_id = tonumber(sp[2]) - local max_rate = tonumber(mrate) - --~ tprint(shaper_id.." "..max_rate) - - if(max_rate > SHAPERS_MAX_RATE_KPBS) then max_rate = -1 end - if(max_rate < -1) then max_rate = -1 end - - shaper_utils.setShaperMaxRate(ifid, shaper_id, max_rate) - num_added = num_added + 1 - last_added = shaper_id - end - end - - if num_added == 1 then - print("") - end - - interface.reloadShapers() - end - - if(_POST["delete_shaper"] ~= nil) then - local shaper_id = _POST["delete_shaper"] - - shaper_utils.deleteShaper(ifid, shaper_id) - end - - if(_POST["target_pool"] ~= nil) then - local target_pool = _POST["target_pool"] - - if (_POST["del_l7_proto"] ~= nil) then - local protocol_id = _POST["del_l7_proto"] - shaper_utils.deleteProtocol(ifid, target_pool, protocol_id) - elseif (_POST["delete_all_policies"] ~= nil) then - shaper_utils.flushPoolRules(ifid, target_pool) - else - -- first remove the rules which have changed protocol - local rules_to_delete = {} - for option,value in pairs(_POST) do - local sp = split(option, "oldrule_") - if #sp == 2 then - -- mark the rule as to be deleted - rules_to_delete[sp[2]] = true - end - end - - shaper_utils.get_shapers_from_parameters(function(proto_id) - -- A new rule will be set for the protocol, no need to delete it - rules_to_delete[proto_id] = nil - end) - - for proto in pairs(rules_to_delete) do - shaper_utils.deleteProtocol(ifid, target_pool, proto) - end - - -- set protocols policy for the pool - shaper_utils.get_shapers_from_parameters(function(proto_id, ingress_shaper, egress_shaper, traffic_quota, time_quota) - if proto_id == "default" then - -- This is not the default protocol quota but the overall quota - shaper_utils.setCrossApplicationQuotas(ifid, target_pool, traffic_quota, time_quota) - traffic_quota = shaper_utils.NO_QUOTA - time_quota = shaper_utils.NO_QUOTA - end - - shaper_utils.setProtocolShapers(ifid, target_pool, proto_id, ingress_shaper, egress_shaper, traffic_quota, time_quota) - end) - end - - interface.reloadL7Rules(tonumber(selected_pool.id)) - end - print [[ -

    -
    ]] - - --- ****************************************** - -local shapers = shaper_utils.getSortedShapers(ifid) - -function print_shapers(shapers, curshaper_id, terminator) - terminator = terminator or "\n" - if(curshaper_id == "") then curshaper_id = "0" else curshaper_id = tostring(curshaper_id) end - - for _,shaper in ipairs(shapers) do - print(""..terminator) - end -end - --- ****************************************** - --- Create delete dialogs - -print( - template.gen("modal_confirm_dialog.html", { - dialog={ - id = "delete_policy_dialog", - action = "deleteShapedProtocol(delete_protocol_id)", - title = i18n("shaping.delete_policy"), - message = i18n("shaping.confirm_delete_policy") .. ' ' .. i18n("shaping.policy_from_pool") .. " \"" .. selected_pool.name .. "\"?", - confirm = i18n("delete"), - } - }) -) - -print( - template.gen("modal_confirm_dialog.html", { - dialog={ - id = "delete_shaper_dialog", - action = "deleteShaper(delete_shaper_id)", - title = i18n("shaping.delete_shaper"), - message = i18n("shaping.confirm_delete_shaper") .. ' ?', - confirm = i18n("delete"), - } - }) -) - -print( - template.gen("modal_confirm_dialog.html", { - dialog={ - id = "deleteAllPolicies", - action = "deleteAllPoliciesConfig()", - title = i18n("shaping.delete_all_policies"), - message = i18n("shaping.confirm_delete_all_policies", {pool_name = selected_pool.name}).."?", - confirm = i18n("delete"), - } - }) -) - --- ****************************************** - --- ==== Manage policies tab ==== - -print [[

    - -
    - - - -
    - -
    - - - -
    - -]] print(i18n("host_pools.pool")..":") print[[ ') - -if selected_pool.id ~= host_pools_utils.DEFAULT_POOL_ID then - print(' ') - - if ntop.getCache("ntopng.prefs.host_pools_rrd_creation") == "1" and ts_utils.exists("host_pool:traffic", {ifid=ifid, pool=selected_pool.id}) then - print("  ") - end -end - -print[[
    - - -]] - -local protos = interface.getnDPIProtocols() -local protos_in_use = shaper_utils.getPoolProtoShapers(ifid, selected_pool.id, true --[[ do not aggregate into categories ]]) -local protocol_categories = shaper_utils.getCategoriesWithProtocols() -local uses_per_member_quota = selected_pool.enforce_quotas_per_pool_member - --- families of protocols which are currently used by at least one protocol -local categories_in_use = {} -for k,v in pairs(protos_in_use) do - local proto_id = tonumber(v.protoId) - - -- can be null for default - if proto_id ~= nil then - local category_id = tostring(interface.getnDPIProtoCategory(proto_id).id) - if not categories_in_use[category_id] then - categories_in_use[category_id] = 1 - else - categories_in_use[category_id] = categories_in_use[category_id] + 1 - end - end -end - -local policy_target_type = ntop.getPref("ntopng.prefs.bridging_policy_target_type") -if isEmptyString(policy_target_type) then policy_target_type = "per_category" end - -function print_ndpi_families(categories, protos, categories_disabled, protos_disabled, terminator) - local protos_excluded = {GRE=1, BGP=1, IGMP=1, IPP=1, IP_in_IP=1, OSPF=1, PPTP=1, SCTP=1, TFTP=1} - - local show_groups = (policy_target_type == "both") - if show_groups then print('') end - - if (policy_target_type == "per_category") or (policy_target_type == "both") then - for k,category in pairsByKeys(categories, asc_insensitive) do - print(''..terminator) - end - end - - if show_groups then print('') end - - if show_groups then print('') end - - if (policy_target_type == "per_protocol") or (policy_target_type == "both") then - for protoName,protoId in pairsByKeys(protos, asc_insensitive) do - if not protos_excluded[protoName] then - -- find protocol category - for _,category in pairs(categories) do - for _,catProto in pairs(category.protos) do - if catProto == protoId then - print('"..terminator) - break - end - end - end - end - end - end - - if show_groups then print('') else print(' ') end -end - -local split_shaping_directions = (ntop.getPref("ntopng.prefs.split_shaping_directions") == "1") - - print [[
    - - -
    -]] - -if selected_pool.id ~= host_pools_utils.DEFAULT_POOL_ID then -print([[]]..i18n("shaping.notes")..[[:]]) - -print([[ -
      -
    • ]]..i18n("shaping.note_drop_core")..[[
      • -
    • ]]..i18n("shaping.note_quota_unlimited")..[[
      • -
    • ]]..i18n("shaping.see_quotas_here", {url=ntop.getHttpPrefix().."/lua/pool_details.lua?page=quotas&pool="..selected_pool.id})..[[
      • -
    • ]]..i18n("shaping.note_target_type", {url=ntop.getHttpPrefix().."/lua/admin/prefs.lua?tab=bridging"})..[[
      • -
    • ]]..i18n("shaping.note_traffic_categories", {url=ntop.getHttpPrefix().."/lua/admin/edit_ndpi_applications.lua"})..[[
      • ]]) - -print[[ - -
    -
    - - -]] -end - -local rate_buttons = shaper_utils.buttons("rate") -local traffic_buttons = shaper_utils.buttons("traffic") -local time_buttons = shaper_utils.buttons("time") - -print(" - -
    -]] - --- ****************************************** - --- ==== Bandwidth Manager tab ==== - -print[[ -
    -
    - - -
    -
    - - -
    - -
    - - -
    - - ]] - -print [[
    - ]] print(i18n('shaping.notes')) print[[ -
      -
    • ]] print(i18n('shaping.shaper0_message')) print[[
      • -
    • ]] print(i18n('shaping.shapers_in_use_message')) print[[
      • -
    • ]] print(i18n('shaping.set_max_rate_to')) print[[
        -
      • -1 ]] print(i18n('shaping.for_no_shaping')) print[[
        • -
      • 0 ]] print(i18n('shaping.for_dropping_all_traffic')) print[[
        • -
      • -
    -
    - - -]] - elseif page == "traffic_report" then dofile(dirs.installdir .. "/pro/scripts/lua/enterprise/traffic_report.lua") end @@ -2355,25 +1436,6 @@ print("