From 9d31ee1a62a7d18a38e1b938a65a2f272c6a3442 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 14:02:23 -0500 Subject: [PATCH 1/6] Add json_file output format (uses json-c) --- INSTALL | 6 + src/Makefile | 6 + src/output_modules/module_json.c | 236 ++++++++++++++++++++++++++++ src/output_modules/output_modules.c | 6 + 4 files changed, 254 insertions(+) create mode 100644 src/output_modules/module_json.c diff --git a/INSTALL b/INSTALL index 6b3167a..ff003b8 100644 --- a/INSTALL +++ b/INSTALL @@ -24,3 +24,9 @@ Redis support is not enabled by default. If you are want to use ZMap with Redis, you will first need to install Hiredis. Then, rebuild ZMap with the command "make REDIS=true". +JSON support is not enabled by default. If you are want to use ZMap +with JSON output, you will first need to install json-c. Then, rebuild +ZMap with the command "make JSON=true". + +Installing json-c requires git and autotools to be available. For more +information on how to install json-c, please see http://github.com/json-c/json-c diff --git a/src/Makefile b/src/Makefile index 72eb0d5..1f95d98 100644 --- a/src/Makefile +++ b/src/Makefile @@ -43,6 +43,12 @@ ifeq ($(REDIS), true) CFLAGS+=-DREDIS endif +ifeq ($(JSON), true) + LDLIBS += $(shell pkg-config --libs json-c) + CFLAGS += $(shell pkg-config --cflags json-c) -DJSON + objects+=module_json.o +endif + all: $(TARGETS) $(TARGETS): diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c new file mode 100644 index 0000000..c9be429 --- /dev/null +++ b/src/output_modules/module_json.c @@ -0,0 +1,236 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../lib/logger.h" + +#include "output_modules.h" + +static FILE *file = NULL; +#define UNUSED __attribute__((unused)) + + +//#define json_object_object_add(a,b,c) { fprintf(stderr, "adding %s\n", b); json_object_object_add(a,b,c); } + +int json_output_file_init(struct state_conf *conf) +{ + json_object *obj = json_object_new_object(); + assert(conf); + + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + perror("Couldn't open output file"); + exit(EXIT_FAILURE); + } + } + + // Create a header json object to describe this output file + json_object_object_add(obj, "type", json_object_new_string("header")); + json_object_object_add(obj, "log_level", json_object_new_int(conf->log_level)); + json_object_object_add(obj, "target_port", json_object_new_int(conf->target_port)); + json_object_object_add(obj, "source_port_first", json_object_new_int(conf->source_port_first)); + json_object_object_add(obj, "source_port_last", json_object_new_int(conf->source_port_last)); + json_object_object_add(obj, "max_targets", json_object_new_int(conf->max_targets)); + json_object_object_add(obj, "max_runtime", json_object_new_int(conf->max_runtime)); + json_object_object_add(obj, "max_results", json_object_new_int(conf->max_results)); + if (conf->iface) json_object_object_add(obj, "iface", json_object_new_string(conf->iface)); + json_object_object_add(obj, "rate", json_object_new_int(conf->rate)); + + json_object_object_add(obj, "bandwidth", json_object_new_int64(conf->bandwidth)); + json_object_object_add(obj, "cooldown_secs", json_object_new_int(conf->cooldown_secs)); + json_object_object_add(obj, "senders", json_object_new_int(conf->senders)); + json_object_object_add(obj, "use_seed", json_object_new_int(conf->use_seed)); + json_object_object_add(obj, "seed", json_object_new_int(conf->seed)); + json_object_object_add(obj, "generator", json_object_new_int(conf->generator)); + + json_object_object_add(obj, "packet_streams", json_object_new_int(conf->packet_streams)); + + //json_object_object_add(obj, "probe_module", json_object_new_string(conf->probe_module->name)); + //json_object_object_add(obj, "output_module", json_object_new_string(conf->output_module->name)); + + if (conf->probe_args) json_object_object_add(obj, "probe_args", json_object_new_string(conf->probe_args)); + if (conf->output_args) json_object_object_add(obj, "output_args", json_object_new_string(conf->output_args)); + + // macaddr_t gw_mac[IFHWADDRLEN]; + //json_object_object_add(obj, "gw_mac", json_object_new_string(conf->gw_mac)); + + json_object_object_add(obj, "source_ip_first", json_object_new_string(conf->source_ip_first)); + json_object_object_add(obj, "source_ip_last", json_object_new_string(conf->source_ip_last)); + json_object_object_add(obj, "output_filename", json_object_new_string(conf->output_filename)); + if (conf->blacklist_filename) json_object_object_add(obj, "blacklist_filename", json_object_new_string(conf->blacklist_filename)); + if (conf->whitelist_filename) json_object_object_add(obj, "whitelist_filename", json_object_new_string(conf->whitelist_filename)); + json_object_object_add(obj, "dryrun", json_object_new_int(conf->dryrun)); + json_object_object_add(obj, "summary", json_object_new_int(conf->summary)); + json_object_object_add(obj, "quiet", json_object_new_int(conf->quiet)); + json_object_object_add(obj, "recv_ready", json_object_new_int(conf->recv_ready)); + + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + } + return EXIT_SUCCESS; +} + +static void json_output_file_store_tv(json_object *obj, struct timeval *tv) +{ + char time_string[40]; + struct tm *ptm = localtime(&tv->tv_sec); + long milliseconds = tv->tv_usec / 1000; + + strftime(time_string, sizeof (time_string), "%Y-%m-%d %H:%M:%S", ptm); + + json_object_object_add(obj, "t", json_object_new_string(time_string)); + json_object_object_add(obj, "ts", json_object_new_int(tv->tv_sec)); + json_object_object_add(obj, "tm", json_object_new_int(milliseconds)); +} + +static void json_output_file_store_data(json_object *obj, const u_char *packet, size_t buflen) +{ + unsigned int i; + char *buf; + + buf = malloc((buflen*2)+1); + buf[buflen*2] = 0; + + for (i=0; iihl*4)) + return EXIT_FAILURE; + + if (buflen < (sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct tcphdr)) && ip_hdr->protocol == IPPROTO_TCP) + return EXIT_FAILURE; + + if (buflen < (sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct udphdr)) && ip_hdr->protocol == IPPROTO_UDP) + return EXIT_FAILURE; + + if (buflen < (sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct icmphdr)) && ip_hdr->protocol == IPPROTO_ICMP) + return EXIT_FAILURE; + + if (!file) + return EXIT_SUCCESS; + + json_object_object_add(obj, "type", json_object_new_string("result")); + json_object_object_add(obj, "response-type", json_object_new_string(response_type)); + + addr.s_addr = saddr; + json_object_object_add(obj, "saddr", json_object_new_string(inet_ntoa(addr))); + + addr.s_addr = daddr; + json_object_object_add(obj, "daddr", json_object_new_string(inet_ntoa(addr))); + + switch(ip_hdr->protocol){ + case IPPROTO_ICMP: + icmp = (struct icmphdr *)((char *)ip_hdr + ip_hdr->ihl * 4); + json_object_object_add(obj, "proto", json_object_new_string("icmp")); + json_object_object_add(obj, "icmp_type", json_object_new_int(icmp->type)); + json_object_object_add(obj, "icmp_code", json_object_new_int(icmp->code)); + break; + + case IPPROTO_IGMP: + json_object_object_add(obj, "protocol", json_object_new_string("igmp")); + break; + + case IPPROTO_TCP: + tcp = (struct tcphdr *)((char *)ip_hdr + ip_hdr->ihl * 4); + json_object_object_add(obj, "proto", json_object_new_string("tcp")); + json_object_object_add(obj, "sport", json_object_new_int(ntohs(tcp->source))); + json_object_object_add(obj, "dport", json_object_new_int(ntohs(tcp->dest))); + // Print these as 64-bit bits to keep the printed value unsigned + json_object_object_add(obj, "seq", json_object_new_int64(ntohl(tcp->seq))); + json_object_object_add(obj, "ack", json_object_new_int64(ntohl(tcp->ack_seq))); + data_offset = sizeof(struct ethhdr) + ip_hdr->ihl*4 + tcp->doff*4; + break; + + case IPPROTO_UDP: + udp = (struct udphdr *)((char *)ip_hdr + ip_hdr->ihl * 4); + json_object_object_add(obj, "proto", json_object_new_string("udp")); + json_object_object_add(obj, "sport", json_object_new_int(ntohs(udp->source))); + json_object_object_add(obj, "dport", json_object_new_int(ntohs(udp->dest))); + data_offset = sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct udphdr); + break; + + default: + memset(tbuff, 0, sizeof(tbuff)); + snprintf(tbuff, sizeof(tbuff)-1, "proto-%d", ip_hdr->protocol); + json_object_object_add(obj, "proto", json_object_new_string(tbuff)); + } + + json_object_object_add(obj, "in_cooldown", json_object_new_int(in_cooldown)); + json_object_object_add(obj, "is_repeat", json_object_new_int(is_repeat)); + + gettimeofday(&t, NULL); + json_output_file_store_tv(obj, &t); + + if ((ip_hdr->protocol == IPPROTO_UDP || ip_hdr->protocol == IPPROTO_ICMP) && buflen - data_offset > 0) + json_output_file_store_data(obj, packet + data_offset, buflen - data_offset); + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + fflush(file); + + return EXIT_SUCCESS; +} + +int json_output_file_close(UNUSED struct state_conf* c, + UNUSED struct state_send* s, UNUSED struct state_recv* r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +output_module_t module_json_file = { + .name = "json_file", + .init = &json_output_file_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &json_output_file_close, + .success_ip = &json_output_file_ip, + .other_ip = &json_output_file_ip +}; + diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c index 438902c..e2a4a7d 100644 --- a/src/output_modules/output_modules.c +++ b/src/output_modules/output_modules.c @@ -21,6 +21,9 @@ extern output_module_t module_redis; extern output_module_t module_ssldbfeed; #endif +#ifdef JSON +extern output_module_t module_json_file; +#endif output_module_t* output_modules[] = { &module_simple_file, @@ -28,6 +31,9 @@ output_module_t* output_modules[] = { #ifdef REDIS &module_redis, &module_ssldbfeed, +#endif +#ifdef JSON + &module_json_file, #endif // ADD YOUR MODULE HERE }; From 36564d8bb563ffade87421d0dbf85a2e03a4c127 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 14:44:29 -0500 Subject: [PATCH 2/6] Add module names and gw mac address to output --- src/output_modules/module_json.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c index c9be429..202cca1 100644 --- a/src/output_modules/module_json.c +++ b/src/output_modules/module_json.c @@ -25,6 +25,7 @@ #include "../../lib/logger.h" #include "output_modules.h" +#include "../probe_modules/probe_modules.h" static FILE *file = NULL; #define UNUSED __attribute__((unused)) @@ -34,6 +35,9 @@ static FILE *file = NULL; int json_output_file_init(struct state_conf *conf) { + int i; + char mac_buf[ (IFHWADDRLEN * 2) + (IFHWADDRLEN - 1) + 1 ]; + char *p; json_object *obj = json_object_new_object(); assert(conf); @@ -65,17 +69,27 @@ int json_output_file_init(struct state_conf *conf) json_object_object_add(obj, "use_seed", json_object_new_int(conf->use_seed)); json_object_object_add(obj, "seed", json_object_new_int(conf->seed)); json_object_object_add(obj, "generator", json_object_new_int(conf->generator)); - json_object_object_add(obj, "packet_streams", json_object_new_int(conf->packet_streams)); - - //json_object_object_add(obj, "probe_module", json_object_new_string(conf->probe_module->name)); - //json_object_object_add(obj, "output_module", json_object_new_string(conf->output_module->name)); - + json_object_object_add(obj, "probe_module", json_object_new_string(((probe_module_t *)conf->probe_module)->name)); + json_object_object_add(obj, "output_module", json_object_new_string(((output_module_t *)conf->output_module)->name)); + if (conf->probe_args) json_object_object_add(obj, "probe_args", json_object_new_string(conf->probe_args)); if (conf->output_args) json_object_object_add(obj, "output_args", json_object_new_string(conf->output_args)); - // macaddr_t gw_mac[IFHWADDRLEN]; - //json_object_object_add(obj, "gw_mac", json_object_new_string(conf->gw_mac)); + if (conf->gw_mac) { + memset(mac_buf, 0, sizeof(mac_buf)); + p = mac_buf; + for(i=0; i < IFHWADDRLEN; i++) { + if (i == IFHWADDRLEN-1) { + snprintf(p, 3, "%.2x", conf->gw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", conf->gw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "gw_mac", json_object_new_string(mac_buf)); + } json_object_object_add(obj, "source_ip_first", json_object_new_string(conf->source_ip_first)); json_object_object_add(obj, "source_ip_last", json_object_new_string(conf->source_ip_last)); From c6f21fdeca82fb0bf176474e156eb140df2a9abd Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 14:46:55 -0500 Subject: [PATCH 3/6] Small typo fix to a comment --- src/output_modules/module_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c index 202cca1..f46def6 100644 --- a/src/output_modules/module_json.c +++ b/src/output_modules/module_json.c @@ -192,7 +192,7 @@ int json_output_file_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, json_object_object_add(obj, "proto", json_object_new_string("tcp")); json_object_object_add(obj, "sport", json_object_new_int(ntohs(tcp->source))); json_object_object_add(obj, "dport", json_object_new_int(ntohs(tcp->dest))); - // Print these as 64-bit bits to keep the printed value unsigned + // Print these as 64-bit values to keep the text output unsigned json_object_object_add(obj, "seq", json_object_new_int64(ntohl(tcp->seq))); json_object_object_add(obj, "ack", json_object_new_int64(ntohl(tcp->ack_seq))); data_offset = sizeof(struct ethhdr) + ip_hdr->ihl*4 + tcp->doff*4; From 7455793d2da8c651fb506988697dc7aa1589a660 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 15:08:32 -0500 Subject: [PATCH 4/6] Fix the data offset for icmp_echoscan --- src/output_modules/module_json.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c index f46def6..73ae1e6 100644 --- a/src/output_modules/module_json.c +++ b/src/output_modules/module_json.c @@ -181,6 +181,7 @@ int json_output_file_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, json_object_object_add(obj, "proto", json_object_new_string("icmp")); json_object_object_add(obj, "icmp_type", json_object_new_int(icmp->type)); json_object_object_add(obj, "icmp_code", json_object_new_int(icmp->code)); + data_offset = sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct icmphdr); break; case IPPROTO_IGMP: From 60011ca520b2632204b59b6dd7cc37a05dae7fab Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 15:10:50 -0500 Subject: [PATCH 5/6] Clarify order of operations for compare --- src/output_modules/module_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c index 73ae1e6..b8ccd51 100644 --- a/src/output_modules/module_json.c +++ b/src/output_modules/module_json.c @@ -219,7 +219,7 @@ int json_output_file_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, gettimeofday(&t, NULL); json_output_file_store_tv(obj, &t); - if ((ip_hdr->protocol == IPPROTO_UDP || ip_hdr->protocol == IPPROTO_ICMP) && buflen - data_offset > 0) + if ((ip_hdr->protocol == IPPROTO_UDP || ip_hdr->protocol == IPPROTO_ICMP) && (buflen - data_offset) > 0) json_output_file_store_data(obj, packet + data_offset, buflen - data_offset); fprintf(file, "%s\n", json_object_to_json_string(obj)); From c455d99298300db86a150d03fc6e0665b8f9ff02 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 17 Aug 2013 15:16:35 -0500 Subject: [PATCH 6/6] Clean up snprintf and buffer size (less sloppy). --- src/output_modules/module_json.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c index b8ccd51..d4481e8 100644 --- a/src/output_modules/module_json.c +++ b/src/output_modules/module_json.c @@ -143,7 +143,7 @@ int json_output_file_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; int data_offset = 0; struct timeval t; - char tbuff[16]; + char tbuff[10]; struct tcphdr *tcp; struct udphdr *udp; struct icmphdr *icmp; @@ -208,8 +208,7 @@ int json_output_file_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, break; default: - memset(tbuff, 0, sizeof(tbuff)); - snprintf(tbuff, sizeof(tbuff)-1, "proto-%d", ip_hdr->protocol); + snprintf(tbuff, sizeof(tbuff), "proto-%d", ip_hdr->protocol); json_object_object_add(obj, "proto", json_object_new_string(tbuff)); }