diff --git a/include/LocklessList.h b/include/LocklessList.h deleted file mode 100644 index 23e5fee3df..0000000000 --- a/include/LocklessList.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * - * (C) 2014-18 - ntop.org - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - */ - -#ifndef _LOCKLESS_LIST_H_ -#define _LOCKLESS_LIST_H_ - -#include "ntop_includes.h" - -/* - * Wait-Free Single-Producer Single-Consumer list - * Lock is used in the Multi-Producer case only - */ - -typedef struct lockless_list_item { - struct lockless_list_item *next; - void *value; -} lockless_list_item_t; - -class LocklessList { - private: - - Mutex *m; - struct { - lockless_list_item_t *head; /* producer */ - u_char __cacheline_padding_0[64-sizeof(lockless_list_item_t *)]; - lockless_list_item_t *tail; /* consumer */ - u_char __cacheline_padding_1[64-sizeof(lockless_list_item_t *)]; - } l; - - - public: - - LocklessList(bool multi_producer) { - lockless_list_item_t *i = (lockless_list_item_t *) calloc(1, sizeof(lockless_list_item_t)); - if (i == NULL) throw 1; - - i->next = NULL; - l.tail = l.head = i; - - if (multi_producer) - m = new Mutex(); - else - m = NULL; - } - - ~LocklessList() { - lockless_list_item_t *i; - - /* Note: all the items should be removed from the list before deleting it, - * as the list itself is generic and it cannot delete i->'value' */ - while (dequeue(&i)) - free(i); - - free(l.tail); - - if(m) delete m; - } - - inline void enqueue(lockless_list_item_t *i) { - if(m) m->lock(__FILE__, __LINE__); - - i->next = NULL; - - //gcc_mb(); - - l.head->next = i; - l.head = i; - - if(m) m->unlock(__FILE__, __LINE__); - } - - inline int dequeue(lockless_list_item_t **i) { - if (l.tail->next != NULL) { - - //gcc_mb(); - - l.tail->value = l.tail->next->value; - l.tail->next->value = NULL; /* useless - safety */ - (*i) = l.tail; - l.tail = l.tail->next; - (*i)->next = NULL; /* useless - safety */ - return 1; - } - - return 0; - } - - inline int empty() { - return l.tail->next == NULL; - } - - /* Iterator with remove operation while iterating */ - - inline lockless_list_item_t *getFirst(void **saveptr) { - lockless_list_item_t **pprev = &l.tail; - (*saveptr) = pprev; - return l.tail->next; - } - - inline lockless_list_item_t *getNext(lockless_list_item_t *i, void **saveptr) { - lockless_list_item_t **pprev = (lockless_list_item_t **) (*saveptr); - pprev = &((*pprev)->next); - (*saveptr) = pprev; - return i->next; - } - - /* Removes an item (item->value should be deleted by the caller) - * Returns the next item, if any */ - inline lockless_list_item_t * remove(lockless_list_item_t *i, void **saveptr) { - lockless_list_item_t **pprev = (lockless_list_item_t **) (*saveptr); - lockless_list_item_t *tmp; - i->value = (*pprev)->value; - tmp = (*pprev); - (*pprev) = i; - free(tmp); - return i->next; - } - -}; - -#endif /* _LOCKLESS_LIST_H_ */ - diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index d2feb42e97..ba5a26e9cb 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -85,9 +85,11 @@ class NetworkInterface : public Checkpointable { MDNS *mdns; /* Live Capture */ - LocklessList *live_captures; + Mutex active_captures_lock; + u_int8_t num_active_captures; + struct ntopngLuaContext *live_captures[MAX_NUM_PCAP_CAPTURES]; static bool matchLiveCapture(struct ntopngLuaContext * const luactx, Flow * const f); - int deliverLiveCapture(const struct pcap_pkthdr * const h, const u_char * const packet, Flow * const f); + void deliverLiveCapture(const struct pcap_pkthdr * const h, const u_char * const packet, Flow * const f); string ip_addresses; int id; @@ -512,7 +514,9 @@ class NetworkInterface : public Checkpointable { PacketDumper *getPacketDumper(void) { return pkt_dumper; } PacketDumperTuntap *getPacketDumperTap(void) { return pkt_dumper_tap; } - int registerLiveCapture(struct ntopngLuaContext * const luactx); + bool registerLiveCapture(struct ntopngLuaContext * const luactx); + bool deregisterLiveCapture(struct ntopngLuaContext * const luactx); + #ifdef NTOPNG_PRO void updateHostsL7Policy(u_int16_t host_pool_id); void updateFlowsL7Policy(); diff --git a/include/ntop_defines.h b/include/ntop_defines.h index b44b855007..467b26ff6b 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -878,6 +878,7 @@ inline struct ntopngLuaContext* getUserdata(lua_State *vm) { #define MIN_TIME_SPAWN_THREAD_POOL 10 /* sec */ #define MAX_NDPI_IDLE_TIME_BEFORE_GUESS 5 /* sec */ +#define MAX_NUM_PCAP_CAPTURES 4 #define ALERT_ACTION_ENGAGE "engage" #define ALERT_ACTION_RELEASE "release" diff --git a/include/ntop_includes.h b/include/ntop_includes.h index ed22b35ddb..911ff26e7e 100644 --- a/include/ntop_includes.h +++ b/include/ntop_includes.h @@ -164,7 +164,6 @@ using namespace std; #include "ntop_defines.h" #include "Mutex.h" #include "RwLock.h" -#include "LocklessList.h" #include "MDNS.h" #include "AddressTree.h" #include "VlanAddressTree.h" diff --git a/include/ntop_typedefs.h b/include/ntop_typedefs.h index c54ee39d01..0eb6b3e14b 100644 --- a/include/ntop_typedefs.h +++ b/include/ntop_typedefs.h @@ -458,11 +458,9 @@ struct ntopngLuaContext { /* Live capture written to mongoose socket */ struct { - /* Filters */ - struct { - IpAddress ip; - u_int16_t vlan_id; - } filters; + u_int32_t capture_until, capture_max_pkts, num_captured_packets; + void *matching_host; + /* Status */ bool pcaphdr_sent; bool done; diff --git a/scripts/lua/host_details.lua b/scripts/lua/host_details.lua index 11200415d7..41046e06cb 100644 --- a/scripts/lua/host_details.lua +++ b/scripts/lua/host_details.lua @@ -592,7 +592,11 @@ end end if(host["json"] ~= nil) then - print(""..i18n("download").." JSON / pcap\n") + print(""..i18n("download").." JSON ") + + if(isAdministrator()) then + print("/ pcap\n") + end end if(host["ssdp"] ~= nil) then diff --git a/scripts/lua/if_stats.lua b/scripts/lua/if_stats.lua index e7402064a9..fad25640aa 100644 --- a/scripts/lua/if_stats.lua +++ b/scripts/lua/if_stats.lua @@ -593,8 +593,10 @@ print("\n") print(" ") print("") end - + if (isAdministrator() and ifstats.isView == false and ifstats.isDynamic == false) then + print(""..i18n("download").." pcap\n") + print(""..i18n("if_stats_overview.reset_counters").."") print("") diff --git a/scripts/lua/live_traffic.lua b/scripts/lua/live_traffic.lua index 6ca7a653d4..f0b28764d6 100644 --- a/scripts/lua/live_traffic.lua +++ b/scripts/lua/live_traffic.lua @@ -20,17 +20,23 @@ local function send_error(error_type) print(json.encode({error = msg})) end -local host = _GET["host"] -if isEmptyString(host) then - send_error("not_found") +interface.select(ifname) + +local granted = true -- interface.requestLiveTraffic(host) + +if not granted then + send_error("not_granted") else - local granted = true -- interface.requestLiveTraffic(host) - - if not granted then - send_error("not_granted") - else - sendHTTPContentTypeHeader('application/vnd.tcpdump.pcap', 'attachment; filename="'..host..'_live.pcap"') - - interface.liveCapture(host) + local host = _GET["host"] + local fname = ifname + + if(host ~= nil) then + fname = fname .. "_"..host end + + fname = fname .."_live.pcap" + + sendHTTPContentTypeHeader('application/vnd.tcpdump.pcap', 'attachment; filename="'..fname..'"') + + interface.liveCapture(host) end diff --git a/src/LuaEngine.cpp b/src/LuaEngine.cpp index 7c23706d47..97a5d99538 100644 --- a/src/LuaEngine.cpp +++ b/src/LuaEngine.cpp @@ -130,6 +130,9 @@ LuaEngine::~LuaEngine() { pthread_join(ctx->pkt_capture.captureThreadLoop, NULL); } + if(ctx->iface != NULL) + ctx->iface->deregisterLiveCapture(ctx); + free(ctx); } @@ -3565,18 +3568,10 @@ static int ntop_interface_live_capture(lua_State* vm) { NetworkInterface *ntop_interface = getCurrentInterface(vm); struct ntopngLuaContext *c; - char *key, buf[64]; - u_int16_t vlan_id = 0; - ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - - if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK) { - lua_pushnil(vm); - return(CONST_LUA_ERROR); - } - get_host_vlan_info((char*)lua_tostring(vm, 1), &key, &vlan_id, buf, sizeof(buf)); - + if(!Utils::isUserAdministrator(vm)) return(CONST_LUA_ERROR); + #ifdef DONT_USE_LUAJIT lua_getglobal(vm, "userdata"); c = (struct ntopngLuaContext*)lua_touserdata(vm, lua_gettop(vm)); @@ -3584,19 +3579,36 @@ static int ntop_interface_live_capture(lua_State* vm) { c = (struct ntopngLuaContext*)(G(vm)->userdata); #endif - if(c) { - c->live_capture.done = c->live_capture.pcaphdr_sent = false, - c->live_capture.filters.ip.set(key), c->live_capture.filters.vlan_id = vlan_id; - ntop_interface->registerLiveCapture(c); + if((!ntop_interface) || (!c)) + return(CONST_LUA_ERROR); + + if(lua_type(vm, 1) == LUA_TSTRING) /* Optional */ { + Host *h; + char *key, host_ip[64]; + u_int16_t vlan_id = 0; - while(!c->live_capture.done) { - ntop->getTrace()->traceEvent(TRACE_INFO, "Capturing...."); - sleep(1); - } + get_host_vlan_info((char*)lua_tostring(vm, 1), &key, &vlan_id, host_ip, sizeof(host_ip)); - ntop->getTrace()->traceEvent(TRACE_INFO, "Done."); + if((!ntop_interface) || ((h = ntop_interface->findHostByIP(get_allowed_nets(vm), host_ip, vlan_id)) == NULL)) + return(CONST_LUA_ERROR); + else { + c->live_capture.matching_host = h; + } } + c->live_capture.capture_until = time(NULL)+60; /* 1 min max */ + c->live_capture.num_captured_packets = 100000; /* No more than 100k packets */ + c->live_capture.done = c->live_capture.pcaphdr_sent = false; + + ntop_interface->registerLiveCapture(c); + + while(!c->live_capture.done) { + ntop->getTrace()->traceEvent(TRACE_INFO, "Capturing...."); + sleep(1); + } + + ntop->getTrace()->traceEvent(TRACE_INFO, "Done."); + lua_pushnil(vm); return(CONST_LUA_OK); } @@ -7504,9 +7516,9 @@ static const luaL_Reg ntop_interface_reg[] = { { "liveCapture", ntop_interface_live_capture }, /* Packet Capture */ - { "captureToPcap", ntop_capture_to_pcap }, - { "isCaptureRunning", ntop_is_capture_running }, - { "stopRunningCapture", ntop_stop_running_capture }, + { "captureToPcap", ntop_capture_to_pcap }, + { "isCaptureRunning", ntop_is_capture_running }, + { "stopRunningCapture", ntop_stop_running_capture }, /* Alert Generation */ { "getCachedNumAlerts", ntop_interface_get_cached_num_alerts }, diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 7772058fd1..563ce36750 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -125,8 +125,8 @@ NetworkInterface::NetworkInterface(const char *name, ntop->getTrace()->traceEvent(TRACE_WARNING, "Unable to read IPv4 address of %s: %s", ifname, pcap_error_buffer); } else { - - + + try { discovery = new NetworkDiscovery(this); } catch (...) { @@ -311,8 +311,9 @@ void NetworkInterface::init() { gettimeofday(&last_frequent_reset, NULL); frequentMacs = new FrequentTrafficItems(5); frequentProtocols = new FrequentTrafficItems(5); - live_captures = new LocklessList(true); - + num_active_captures = 0; + memset(live_captures, 0, sizeof(live_captures)); + db = NULL; #ifdef NTOPNG_PRO aggregated_flows_hash = NULL, flow_interfaces_stats = NULL; @@ -688,7 +689,6 @@ NetworkInterface::~NetworkInterface() { if(networkStats) delete []networkStats; if(pkt_dumper) delete pkt_dumper; if(pkt_dumper_tap) delete pkt_dumper_tap; - if(live_captures) delete live_captures; if(interfaceStats) delete interfaceStats; if(flowHashing) { @@ -722,7 +722,7 @@ NetworkInterface::~NetworkInterface() { #endif if(hide_from_top) delete(hide_from_top); if(hide_from_top_shadow) delete(hide_from_top_shadow); - if(tsExporter) delete tsExporter; + if(tsExporter) delete tsExporter; } /* **************************************************** */ @@ -1943,7 +1943,8 @@ bool NetworkInterface::processPacket(u_int32_t bridge_iface_idx, } /* Live packet dump to mongoose */ - deliverLiveCapture(h, packet, flow); + if(num_active_captures > 0) + deliverLiveCapture(h, packet, flow); incStats(ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, flow->get_detected_protocol().app_protocol, @@ -1998,10 +1999,10 @@ bool NetworkInterface::dissectPacket(u_int32_t bridge_iface_idx, #if 0 static u_int n = 0; - + ntop->getTrace()->traceEvent(TRACE_NORMAL, "%u %s", ++n, ingressPacket ? "RX" : "TX"); #endif - + /* Netfilter interfaces don't report MAC addresses on packets */ if(getIfType() == interface_type_NETFILTER) rawsize += sizeof(struct ndpi_ethhdr); @@ -2589,7 +2590,7 @@ void NetworkInterface::findFlowHosts(u_int16_t vlanId, (*dst) = new LocalHost(this, dst_mac, vlanId, _dst_ip); else (*dst) = new RemoteHost(this, dst_mac, vlanId, _dst_ip); - + if(!hosts_hash->add(*dst)) { // ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many hosts in interface %s", ifname); delete *dst; @@ -4879,7 +4880,7 @@ void NetworkInterface::getnDPIProtocols(lua_State *vm, ndpi_protocol_category_t int i; u_int num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(ndpi_struct); ndpi_proto_defaults_t* proto_defaults = ndpi_get_proto_defaults(ndpi_struct); - + lua_newtable(vm); for(i=0; i<(int)num_supported_protocols; i++) { @@ -6445,7 +6446,7 @@ void NetworkInterface::checkMacIPAssociation(bool triggerEvent, u_char *_mac, u_ return; u_int64_t mac = Utils::mac2int(_mac); - + if((ipv4 != 0) && (mac != 0)) { std::map::iterator it; @@ -6463,7 +6464,7 @@ void NetworkInterface::checkMacIPAssociation(bool triggerEvent, u_char *_mac, u_ Utils::formatMac(tmp, oldmac, sizeof(oldmac)); Utils::formatMac(_mac, newmac, sizeof(newmac)); ipa = Utils::intoaV4(ntohl(ipv4), ipbuf, sizeof(ipbuf)); - + ntop->getTrace()->traceEvent(TRACE_INFO, "IP %s: modified MAC association %s -> %s", ipa, oldmac, newmac); @@ -6538,85 +6539,112 @@ void NetworkInterface::finishInitialization() { /* *************************************** */ -int NetworkInterface::registerLiveCapture(struct ntopngLuaContext * const luactx) { - lockless_list_item_t *i; +bool NetworkInterface::registerLiveCapture(struct ntopngLuaContext * const luactx) { + bool ret = false; - i = (lockless_list_item_t *) malloc(sizeof(lockless_list_item_t)); + active_captures_lock.lock(__FILE__, __LINE__); - if(luactx && i) { - i->value = luactx; - live_captures->enqueue(i); + if(num_active_captures < MAX_NUM_PCAP_CAPTURES) { + for(int i=0; iget_cli_host()) cli_ip = f->get_cli_host()->get_ip(); - if(f && f->get_srv_host()) srv_ip = f->get_srv_host()->get_ip(); - - if(f->get_vlan_id() == luactx->live_capture.filters.vlan_id - && ((cli_ip && cli_ip->equal(&luactx->live_capture.filters.ip)) - || (srv_ip && srv_ip->equal(&luactx->live_capture.filters.ip)))) - return true; + if((luactx->live_capture.matching_host == NULL) + || (luactx->live_capture.matching_host == f->get_cli_host()) + || (luactx->live_capture.matching_host == f->get_srv_host())) + return(true); return false; } /* *************************************** */ -int NetworkInterface::deliverLiveCapture(const struct pcap_pkthdr * const h, const u_char * const packet, Flow * const f) { - struct pcap_file_header pcaphdr; - struct pcap_disk_pkthdr pkthdr; - struct ntopngLuaContext *c; - lockless_list_item_t *i; - void *saveptr; - int res; - bool done; +void NetworkInterface::deliverLiveCapture(const struct pcap_pkthdr * const h, + const u_char * const packet, Flow * const f) { + u_int num_live_captures = num_active_captures; /* Cache because of (*) */ - i = live_captures->getFirst(&saveptr); - while(i) { - done = false; + for(int i=0, num_found = 0; (ivalue - && (c = (struct ntopngLuaContext *)i->value) - && c->conn - && matchLiveCapture(c, f)) { + num_found++; - if(!c->live_capture.pcaphdr_sent) { - Utils::init_pcap_header(&pcaphdr, this); - if((res = mg_write(c->conn, &pcaphdr, sizeof(pcaphdr))) < (int)sizeof(pcaphdr)) - done = true; - c->live_capture.pcaphdr_sent = true; + if(c->live_capture.capture_until < h->ts.tv_sec) { + http_client_disconnected = true; + mg_close_connection(c->conn); + } + + if((!http_client_disconnected) && c->conn && matchLiveCapture(c, f)) { + if(!c->live_capture.pcaphdr_sent) { + struct pcap_file_header pcaphdr; + int res; + + Utils::init_pcap_header(&pcaphdr, this); + + if((res = mg_write(c->conn, &pcaphdr, sizeof(pcaphdr))) < (int)sizeof(pcaphdr)) + http_client_disconnected = true; + + c->live_capture.pcaphdr_sent = true; + } + + if(!http_client_disconnected) { + int res; + struct pcap_disk_pkthdr pkthdr; /* Cannot use h as the format on disk differs */ + + pkthdr.ts.tv_sec = h->ts.tv_sec, pkthdr.ts.tv_usec = h->ts.tv_usec, + pkthdr.caplen = h->caplen, pkthdr.len = h->len; + + if( + ((res = mg_write(c->conn, &pkthdr, sizeof(pkthdr))) < (int)sizeof(pkthdr)) + || ((res = mg_write(c->conn, packet, h->caplen)) < (int)h->caplen) + ) + http_client_disconnected = true; + else { + c->live_capture.num_captured_packets++; + + if((c->live_capture.capture_max_pkts != 0) + && (c->live_capture.num_captured_packets == c->live_capture.capture_max_pkts)) + http_client_disconnected = true; + } + } } - if(!done) { - /* Cannot use h as the format on disk differs */ - pkthdr.ts.tv_sec = h->ts.tv_sec, pkthdr.ts.tv_usec = h->ts.tv_usec, - pkthdr.caplen = h->caplen, pkthdr.len = h->len; - - if((res = mg_write(c->conn, &pkthdr, sizeof(pkthdr))) < (int)sizeof(pkthdr)) - done = true; - else if((res = mg_write(c->conn, packet, h->caplen)) < (int)h->caplen) - done = true; - } - - if(done) { - i = live_captures->remove(i, &saveptr); - c->live_capture.done = true; - } + if(http_client_disconnected) + deregisterLiveCapture(c); /* (*) */ } - - if(!done) - i = live_captures->getNext(i, &saveptr); } - - return 0; }