diff --git a/include/TimelineExtract.h b/include/TimelineExtract.h index 428985f914..53bca093a8 100644 --- a/include/TimelineExtract.h +++ b/include/TimelineExtract.h @@ -44,6 +44,10 @@ class TimelineExtract { char *bpf_filter; } extraction; +#ifdef HAVE_PF_RING + pfring *openTimeline(NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter); +#endif + public: TimelineExtract(); ~TimelineExtract(); @@ -55,7 +59,8 @@ class TimelineExtract { inline bool isRunning() { return running; }; void stop(); /* sync */ - bool extract(u_int32_t id, NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter); + bool extractToDisk(u_int32_t id, NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter); + bool extractLive(struct mg_connection *conn, NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter); /* async */ void runExtractionJob(u_int32_t id, NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter); void stopExtractionJob(u_int32_t id); diff --git a/scripts/lua/live_traffic_extraction.lua b/scripts/lua/live_traffic_extraction.lua new file mode 100644 index 0000000000..2552d3e580 --- /dev/null +++ b/scripts/lua/live_traffic_extraction.lua @@ -0,0 +1,41 @@ +-- +-- (C) 2013-18 - ntop.org +-- + +dirs = ntop.getDirs() +package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path +require "lua_utils" +local json = require("dkjson") +local recording_utils = require "recording_utils" + +local function send_error(msg) + sendHTTPContentTypeHeader('application/json') + local res = {} + res.error = msg + print(json.encode(res)) +end + +if not recording_utils.isAvailable() then + send_error(i18n("traffic_recording.not_granted")) +else + if _GET["epoch_begin"] == nil or _GET["epoch_end"] == nil then + send_error(i18n("traffic_recording.missing_parameters")) + else + interface.select(ifname) + + local filter = _GET["bpf_filter"] + local time_from = tonumber(_GET["epoch_begin"]) + local time_to = tonumber(_GET["epoch_end"]) + + if filter == nil then + filter = "" + end + + local fname = time_from.."-"..time_to..".pcap" + sendHTTPContentTypeHeader('application/vnd.tcpdump.pcap', 'attachment; filename="'..fname..'"') + + interface.runLiveExtraction(time_from, time_to, filter) + + end +end + diff --git a/src/LuaEngine.cpp b/src/LuaEngine.cpp index fa14617696..fc4a0ca192 100644 --- a/src/LuaEngine.cpp +++ b/src/LuaEngine.cpp @@ -5618,6 +5618,50 @@ static int ntop_get_extraction_status(lua_State *vm) { /* ****************************************** */ +// ***API*** +static int ntop_run_live_extraction(lua_State *vm) { + struct ntopngLuaContext *c; + NetworkInterface *iface = getCurrentInterface(vm); + TimelineExtract timeline; + time_t time_from, time_to; + char *filter; + + ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); + + if(!Utils::isUserAdministrator(vm)) + return(CONST_LUA_ERROR); + +#ifdef DONT_USE_LUAJIT + lua_getglobal(vm, "userdata"); + c = (struct ntopngLuaContext*)lua_touserdata(vm, lua_gettop(vm)); +#else + c = (struct ntopngLuaContext*)(G(vm)->userdata); +#endif + + if (!c) + return(CONST_LUA_ERROR); + + if(!iface) return(CONST_LUA_ERROR); + + if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER) != CONST_LUA_OK) + return(CONST_LUA_PARAM_ERROR); + if (ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER) != CONST_LUA_OK) + return(CONST_LUA_PARAM_ERROR); + if (ntop_lua_check(vm, __FUNCTION__, 3, LUA_TSTRING) != CONST_LUA_OK) + return(CONST_LUA_PARAM_ERROR); + + time_from = lua_tointeger(vm, 1); + time_to = lua_tointeger(vm, 2); + if ((filter = (char *) lua_tostring(vm, 3)) == NULL) return(CONST_LUA_PARAM_ERROR); + + timeline.extractLive(c->conn, iface, time_from, time_to, filter); + + lua_pushnil(vm); + return(CONST_LUA_OK); +} + +/* ****************************************** */ + static int ntop_interface_exec_sql_query(lua_State *vm) { NetworkInterface *ntop_interface = getCurrentInterface(vm); bool limit_rows = true; // honour the limit by default @@ -7846,6 +7890,9 @@ static const luaL_Reg ntop_interface_reg[] = { { "isCaptureRunning", ntop_is_capture_running }, { "stopRunningCapture", ntop_stop_running_capture }, + /* Live Extraction */ + { "runLiveExtraction", ntop_run_live_extraction }, + /* Alert Generation */ { "getCachedNumAlerts", ntop_interface_get_cached_num_alerts }, { "queryAlertsRaw", ntop_interface_query_alerts_raw }, @@ -8056,8 +8103,8 @@ static const luaL_Reg ntop_reg[] = { { "reloadDeviceProtocols", ntop_reload_device_protocols }, /* Traffic Recording/Extraction */ - { "runExtraction", ntop_run_extraction }, - { "stopExtraction", ntop_stop_extraction }, + { "runExtraction", ntop_run_extraction }, + { "stopExtraction", ntop_stop_extraction }, { "isExtractionRunning", ntop_is_extraction_running }, { "getExtractionStatus", ntop_get_extraction_status }, diff --git a/src/TimelineExtract.cpp b/src/TimelineExtract.cpp index 942b2238e7..b8d6315aec 100644 --- a/src/TimelineExtract.cpp +++ b/src/TimelineExtract.cpp @@ -38,33 +38,22 @@ TimelineExtract::~TimelineExtract() { /* ********************************************* */ -bool TimelineExtract::extract(u_int32_t id, NetworkInterface *iface, - time_t from, time_t to, const char *bpf_filter) { - bool completed = false; #ifdef HAVE_PF_RING +pfring *TimelineExtract::openTimeline(NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter) { char timeline_path[MAX_PATH]; - char out_path[MAX_PATH]; char from_buff[24], to_buff[24]; - PacketDumper *dumper; - pfring *handle; - u_char *packet = NULL; - struct pfring_pkthdr header = { 0 }; - struct pcap_pkthdr *h; - struct tm *time_info; + pfring *handle = NULL; char *filter; + struct tm *time_info; int rc; - - shutdown = false; - stats.packets = stats.bytes = 0; - status_code = 1; - snprintf(out_path, sizeof(out_path), "%s/%u/extr_pcap/%u", ntop->getPrefs()->get_pcap_dir(), iface->get_id(), id); + snprintf(timeline_path, sizeof(timeline_path), "timeline:%s/%d/timeline", ntop->getPrefs()->get_pcap_dir(), iface->get_id()); - dumper = new PacketDumper(iface, out_path); + handle = pfring_open(timeline_path, 16384, 0); - if (dumper == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to initialize packet dumper"); - status_code = 2; + if (handle == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to open %s", timeline_path); + status_code = 4; /* Unable to open timeline */ goto error; } @@ -72,8 +61,8 @@ bool TimelineExtract::extract(u_int32_t id, NetworkInterface *iface, if (filter == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory"); - status_code = 3; - goto delete_dumper; + status_code = 3; /* Memory allocation failure */ + goto close_pfring; } filter[0] = '\0'; @@ -89,54 +78,88 @@ bool TimelineExtract::extract(u_int32_t id, NetworkInterface *iface, if (bpf_filter && strlen(bpf_filter) > 0) sprintf(&filter[strlen(filter)], " and %s", bpf_filter); - snprintf(timeline_path, sizeof(timeline_path), "timeline:%s/%d/timeline", ntop->getPrefs()->get_pcap_dir(), iface->get_id()); - - ntop->getTrace()->traceEvent(TRACE_INFO, "Running extraction #%u from %s matching %s to %s", - id, timeline_path, filter, out_path); - - handle = pfring_open(timeline_path, 16384, 0); - - if (handle == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to open %s", timeline_path); - status_code = 4; - goto free_filter; - } + ntop->getTrace()->traceEvent(TRACE_INFO, "Running extraction from '%s' matching filter '%s'", + timeline_path, filter); rc = pfring_set_bpf_filter(handle, filter); + free(filter); + if (rc != 0) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to set filter '%s' (%d)", filter, rc); - status_code = 5; + status_code = 5; /* Unable to set filter */ goto close_pfring; } if (pfring_enable_ring(handle) != 0) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to start extraction on %s", timeline_path); - status_code = 6; + status_code = 6; /* Unable to open timeline */ goto close_pfring; } + return handle; + +close_pfring: + pfring_close(handle); + +error: + return NULL; +} +#endif + +/* ********************************************* */ + +bool TimelineExtract::extractToDisk(u_int32_t id, NetworkInterface *iface, + time_t from, time_t to, const char *bpf_filter) { + bool completed = false; +#ifdef HAVE_PF_RING + char out_path[MAX_PATH]; + PacketDumper *dumper; + pfring *handle; + u_char *packet = NULL; + struct pfring_pkthdr header = { 0 }; + struct pcap_pkthdr *h; + + shutdown = false; + stats.packets = stats.bytes = 0; + status_code = 1; /* default: unexpected error */ + + snprintf(out_path, sizeof(out_path), "%s/%u/extr_pcap/%u", ntop->getPrefs()->get_pcap_dir(), iface->get_id(), id); + + dumper = new PacketDumper(iface, out_path); + + if (dumper == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to initialize packet dumper"); + status_code = 2; /* Unable to initialize dumper */ + goto error; + } + + handle = openTimeline(iface, from, to, bpf_filter); + + if (handle == NULL) + goto delete_dumper; + + ntop->getTrace()->traceEvent(TRACE_INFO, "Dumping traffic to '%s'", out_path); + while (!shutdown && !ntop->getGlobals()->isShutdown() && - (rc = pfring_recv(handle, &packet, 0, &header, 0)) > 0) { + pfring_recv(handle, &packet, 0, &header, 0) > 0) { h = (struct pcap_pkthdr *) &header; dumper->dumpPacket(h, packet); stats.packets++; stats.bytes += sizeof(struct pcap_disk_pkthdr) + h->caplen; } - status_code = 0; + status_code = 0; /* Successfully completed */ completed = true; - close_pfring: pfring_close(handle); - free_filter: - free(filter); - delete_dumper: delete dumper; error: +#else + status_code = 7; /* No PF_RING support */ #endif ntop->getTrace()->traceEvent(TRACE_INFO, "Extraction #%u %s", @@ -147,10 +170,65 @@ bool TimelineExtract::extract(u_int32_t id, NetworkInterface *iface, /* ********************************************* */ +bool TimelineExtract::extractLive(struct mg_connection *conn, NetworkInterface *iface, time_t from, time_t to, const char *bpf_filter) { + bool completed = false; +#ifdef HAVE_PF_RING + pfring *handle; + u_char *packet = NULL; + struct pfring_pkthdr h = { 0 }; + struct pcap_file_header pcaphdr; + struct pcap_disk_pkthdr pkthdr; + bool http_client_disconnected = false; + + stats.packets = stats.bytes = 0; + + ntop->getTrace()->traceEvent(TRACE_INFO, "Running live extraction"); + + Utils::init_pcap_header(&pcaphdr, iface); + + if (mg_write_async(conn, &pcaphdr, sizeof(pcaphdr)) < (int) sizeof(pcaphdr)) + http_client_disconnected = true; + + handle = openTimeline(iface, from, to, bpf_filter); + + if (handle == NULL) + goto error; + + while (!http_client_disconnected && + !ntop->getGlobals()->isShutdown() && + pfring_recv(handle, &packet, 0, &h, 0) > 0) { + + pkthdr.ts.tv_sec = h.ts.tv_sec; + pkthdr.ts.tv_usec = h.ts.tv_usec, + pkthdr.caplen = h.caplen; + pkthdr.len = h.len; + + if (mg_write_async(conn, &pkthdr, sizeof(pkthdr)) < (int) sizeof(pkthdr) || + mg_write_async(conn, packet, h.caplen) < (int) h.caplen) + http_client_disconnected = true; + + usleep(100); /* FIXX it seems that sendint too fast with mg_write_async breaks the connection */ + + stats.packets++; + stats.bytes += sizeof(struct pcap_disk_pkthdr) + h.caplen; + } + + completed = true; + pfring_close(handle); + + error: +#endif + ntop->getTrace()->traceEvent(TRACE_INFO, "Live extraction %s %s", + completed ? "completed" : "failed", http_client_disconnected ? "(disconnected)" : ""); + return completed; +} + +/* ********************************************* */ + static void *extractionThread(void *ptr) { TimelineExtract *extr = (TimelineExtract *) ptr; - extr->extract( + extr->extractToDisk( extr->getID(), extr->getNetworkInterface(), extr->getFrom(),