diff --git a/INSTALL b/INSTALL index 6d05d16..442f292 100644 --- a/INSTALL +++ b/INSTALL @@ -30,3 +30,9 @@ Redis support is not enabled by default. If you are want to use ZMap with Redis, you will first need to install Hiredis. Then, rebuild ZMap with the command "make REDIS=true". +JSON support is not enabled by default. If you are want to use ZMap +with JSON output, you will first need to install json-c. Then, rebuild +ZMap with the command "make JSON=true". + +Installing json-c requires git and autotools to be available. For more +information on how to install json-c, please see http://github.com/json-c/json-c diff --git a/src/Makefile b/src/Makefile index 6f9f98f..cf95b68 100644 --- a/src/Makefile +++ b/src/Makefile @@ -41,6 +41,12 @@ ifeq ($(REDIS), true) CFLAGS+=-DREDIS endif +ifeq ($(JSON), true) + LDLIBS += $(shell pkg-config --libs json-c) + CFLAGS += $(shell pkg-config --cflags json-c) -DJSON + objects+=module_json.o +endif + all: $(TARGETS) $(TARGETS): diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c new file mode 100644 index 0000000..b598078 --- /dev/null +++ b/src/output_modules/module_json.c @@ -0,0 +1,190 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../lib/logger.h" + +#include "output_modules.h" +#include "../probe_modules/probe_modules.h" + +static FILE *file = NULL; +#define UNUSED __attribute__((unused)) + +int json_output_file_init(struct state_conf *conf) +{ + int i; + char mac_buf[ (IFHWADDRLEN * 2) + (IFHWADDRLEN - 1) + 1 ]; + char *p; + json_object *obj = json_object_new_object(); + assert(conf); + + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + perror("Couldn't open output file"); + exit(EXIT_FAILURE); + } + } + + // Create a header json object to describe this output file + json_object_object_add(obj, "type", json_object_new_string("header")); + json_object_object_add(obj, "log_level", json_object_new_int(conf->log_level)); + json_object_object_add(obj, "target_port", + json_object_new_int(conf->target_port)); + json_object_object_add(obj, "source_port_first", + json_object_new_int(conf->source_port_first)); + json_object_object_add(obj, "source_port_last", + json_object_new_int(conf->source_port_last)); + json_object_object_add(obj, "max_targets", json_object_new_int(conf->max_targets)); + json_object_object_add(obj, "max_runtime", json_object_new_int(conf->max_runtime)); + json_object_object_add(obj, "max_results", json_object_new_int(conf->max_results)); + if (conf->iface) { + json_object_object_add(obj, "iface", json_object_new_string(conf->iface)); + } + json_object_object_add(obj, "rate", json_object_new_int(conf->rate)); + + json_object_object_add(obj, "bandwidth", json_object_new_int64(conf->bandwidth)); + json_object_object_add(obj, "cooldown_secs", json_object_new_int(conf->cooldown_secs)); + json_object_object_add(obj, "senders", json_object_new_int(conf->senders)); + json_object_object_add(obj, "use_seed", json_object_new_int(conf->use_seed)); + json_object_object_add(obj, "seed", json_object_new_int(conf->seed)); + json_object_object_add(obj, "generator", json_object_new_int(conf->generator)); + json_object_object_add(obj, "packet_streams", + json_object_new_int(conf->packet_streams)); + json_object_object_add(obj, "probe_module", + json_object_new_string(((probe_module_t *)conf->probe_module)->name)); + json_object_object_add(obj, "output_module", + json_object_new_string(((output_module_t *)conf->output_module)->name)); + + if (conf->probe_args) { + json_object_object_add(obj, "probe_args", + json_object_new_string(conf->probe_args)); + } + if (conf->output_args) { + json_object_object_add(obj, "output_args", + json_object_new_string(conf->output_args)); + } + + if (conf->gw_mac) { + memset(mac_buf, 0, sizeof(mac_buf)); + p = mac_buf; + for(i=0; i < IFHWADDRLEN; i++) { + if (i == IFHWADDRLEN-1) { + snprintf(p, 3, "%.2x", conf->gw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", conf->gw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "gw_mac", json_object_new_string(mac_buf)); + } + + json_object_object_add(obj, "source_ip_first", + json_object_new_string(conf->source_ip_first)); + json_object_object_add(obj, "source_ip_last", + json_object_new_string(conf->source_ip_last)); + json_object_object_add(obj, "output_filename", + json_object_new_string(conf->output_filename)); + if (conf->blacklist_filename) json_object_object_add(obj, + "blacklist_filename", + json_object_new_string(conf->blacklist_filename)); + if (conf->whitelist_filename) json_object_object_add(obj, "whitelist_filename", json_object_new_string(conf->whitelist_filename)); + json_object_object_add(obj, "dryrun", json_object_new_int(conf->dryrun)); + json_object_object_add(obj, "summary", json_object_new_int(conf->summary)); + json_object_object_add(obj, "quiet", json_object_new_int(conf->quiet)); + json_object_object_add(obj, "recv_ready", json_object_new_int(conf->recv_ready)); + + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + } + return EXIT_SUCCESS; +} + +static void json_output_file_store_data(json_object *obj, const u_char *packet, size_t buflen) +{ + unsigned int i; + char *buf; + + buf = malloc((buflen*2)+1); + buf[buflen*2] = 0; + + for (i=0; ilen; i++) { + field_t *f = &(fs->fields[i]); + if (f->type == FS_STRING) { + json_object_object_add(obj, f->name, + json_object_new_string((char*) f->value)); + } else if (f->type == FS_UINT64) { + json_object_object_add(obj, f->name, + json_object_new_int((int) f->value)); + } else if (f->type == FS_BINARY) { + json_output_file_store_data(obj, + (const u_char*) f->value, f->len); + } else { + log_fatal("csv", "received unknown output type"); + } + } + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + fflush(file); + // free memory + json_object_put(obj); + return EXIT_SUCCESS; +} + +int json_output_file_close(UNUSED struct state_conf* c, + UNUSED struct state_send* s, UNUSED struct state_recv* r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +output_module_t module_json_file = { + .name = "json", + .init = &json_output_file_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &json_output_file_close, + .success_ip = &json_output_file_ip, +}; + diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c index 6f52a7e..5eca37d 100644 --- a/src/output_modules/output_modules.c +++ b/src/output_modules/output_modules.c @@ -13,16 +13,22 @@ #include "output_modules.h" extern output_module_t module_csv_file; -// ADD YOUR MODULE HERE #ifdef REDIS extern output_module_t module_redis; #endif +#ifdef JSON +extern output_module_t module_json_file; +#endif + output_module_t* output_modules[] = { &module_csv_file #ifdef REDIS //&module_redis, +#endif +#ifdef JSON + &module_json_file, #endif // ADD YOUR MODULE HERE }; diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c index d9aa94f..c05072f 100644 --- a/src/probe_modules/probe_modules.c +++ b/src/probe_modules/probe_modules.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -83,10 +84,13 @@ void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) log_fatal("recv", "unable to allocate memory for " "timestamp string in fieldset."); } - time_t now = time(0); - strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S%z", - localtime(&now)); + struct timeval t; + gettimeofday(&t, NULL); + struct tm *ptm = localtime(&t.tv_sec); + strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S%z", ptm); fs_add_string(fs, "timestamp-str", timestr, 1); + fs_add_uint64(fs, "timestamp-ts", (uint64_t) t.tv_sec); + fs_add_uint64(fs, "timestamp-us", (uint64_t) t.tv_usec); } fielddef_t ip_fields[] = { @@ -99,5 +103,8 @@ fielddef_t ip_fields[] = { fielddef_t sys_fields[] = { {.name="repeat", .type="int", .desc="Is response a repeat response from host"}, {.name="cooldown", .type="int", .desc="Was response received during the cooldown period"}, - {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."} + {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."}, + {.name="timestamp-ts", .type="int", .desc="timestamp of when response arrived in seconds since Epoch"}, + {.name="timestamp-us", .type="int", .desc="timestamp of when response arrive in microseconds since Epoch"} }; + diff --git a/src/zmap.c b/src/zmap.c index 598c9d1..bb060c7 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -427,7 +427,7 @@ int main(int argc, char *argv[]) gen_fielddef_set(fds, zconf.probe_module->fields, zconf.probe_module->numfields); gen_fielddef_set(fds, (fielddef_t*) &(sys_fields), - 3); + 5); if (args.list_output_fields_given) { for (int i = 0; i < fds->len; i++) { printf("%s (%s): %s\n", fds->fielddefs[i].name,