From b70a2835b9952bda9d8971a68312237fcfdc54bb Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Sun, 18 Aug 2013 23:36:57 -0400 Subject: [PATCH 01/18] abstracting out port validation because shared between udp and tcp. --- src/probe_modules/module_tcp_synscan.c | 30 ++++++++------------ src/probe_modules/module_udp.c | 38 ++++++++------------------ src/probe_modules/packet.h | 21 ++++++++++++++ 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c index 44bce1b..ce59bcc 100644 --- a/src/probe_modules/module_tcp_synscan.c +++ b/src/probe_modules/module_tcp_synscan.c @@ -26,7 +26,13 @@ #include "packet.h" probe_module_t module_tcp_synscan; -uint32_t num_ports = 1; +static uint32_t num_ports; + +int synscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} int synscan_init_perthread(void* buf, macaddr_t *src, macaddr_t *gw, port_h_t dst_port) @@ -39,7 +45,6 @@ int synscan_init_perthread(void* buf, macaddr_t *src, make_ip_header(ip_header, IPPROTO_TCP, len); struct tcphdr *tcp_header = (struct tcphdr*)(&ip_header[1]); make_tcp_header(tcp_header, dst_port); - num_ports = zconf.source_port_last - zconf.source_port_first + 1; return EXIT_SUCCESS; } @@ -49,15 +54,13 @@ int synscan_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, struct ethhdr *eth_header = (struct ethhdr *)buf; struct iphdr *ip_header = (struct iphdr*)(ð_header[1]); struct tcphdr *tcp_header = (struct tcphdr*)(&ip_header[1]); - uint16_t src_port = zconf.source_port_first - + ((validation[1] + probe_num) % num_ports); uint32_t tcp_seq = validation[0]; - ip_header->saddr = src_ip; ip_header->daddr = dst_ip; - tcp_header->source = htons(src_port); + tcp_header->source = htons(get_src_port(num_ports, + probe_num, validation)); tcp_header->seq = tcp_seq; tcp_header->check = 0; tcp_header->check = tcp_checksum(sizeof(struct tcphdr), @@ -121,19 +124,7 @@ response_type_t* synscan_classify_packet(const u_char *packet, uint32_t len) return &(module_tcp_synscan.responses[0]); } } -// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise -static inline int check_dst_port(uint16_t port, uint32_t *validation) -{ - if (port > zconf.source_port_last - || port < zconf.source_port_first) { - return EXIT_FAILURE; - } - int32_t to_validate = port - zconf.source_port_first; - int32_t min = validation[1] % num_ports; - int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports; - return (((max - min) % num_ports) >= ((to_validate - min) % num_ports)); -} int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, __attribute__((unused))uint32_t *src_ip, uint32_t *validation) @@ -156,7 +147,7 @@ int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, } // validate destination port - if (!check_dst_port(ntohs(dport), validation)) { + if (!check_dst_port(ntohs(dport), num_ports, validation)) { return 0; } @@ -185,6 +176,7 @@ probe_module_t module_tcp_synscan = { .pcap_filter = "tcp && tcp[13] & 4 != 0 || tcp[13] == 18", .pcap_snaplen = 96, .port_args = 1, + .global_initialize = &synscan_global_initialize, .thread_initialize = &synscan_init_perthread, .make_packet = &synscan_make_packet, .print_packet = &synscan_print_packet, diff --git a/src/probe_modules/module_udp.c b/src/probe_modules/module_udp.c index 9c27026..be02adb 100644 --- a/src/probe_modules/module_udp.c +++ b/src/probe_modules/module_udp.c @@ -34,25 +34,26 @@ int udp_send_msg_len = 0; const char *udp_send_msg_default = "GET / HTTP/1.1\r\nHost: www\r\n\r\n"; -static int num_ports = 1; +static int num_ports; probe_module_t module_udp; - -int udp_global_initialize(struct state_conf * zconf) { +int udp_global_initialize(struct state_conf *conf) { char *args, *c; int i; unsigned int n; FILE *inp; + num_ports = conf->source_port_last - conf->source_port_first + 1; + udp_send_msg = strdup(udp_send_msg_default); udp_send_msg_len = strlen(udp_send_msg); - if (! (zconf->probe_args && strlen(zconf->probe_args) > 0)) + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) return(0); - args = strdup(zconf->probe_args); + args = strdup(conf->probe_args); if (! args) exit(1); c = strchr(args, ':'); @@ -154,23 +155,23 @@ int udp_init_perthread(void* buf, macaddr_t *src, memcpy(payload, udp_send_msg, udp_send_msg_len); - num_ports = zconf.source_port_last - zconf.source_port_first + 1; - return EXIT_SUCCESS; } + + + int udp_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint32_t *validation, int probe_num) { struct ethhdr *eth_header = (struct ethhdr *)buf; struct iphdr *ip_header = (struct iphdr*)(ð_header[1]); struct udphdr *udp_header = (struct udphdr*)(&ip_header[1]); - uint16_t src_port = zconf.source_port_first - + ((validation[1] + probe_num) % num_ports); ip_header->saddr = src_ip; ip_header->daddr = dst_ip; - udp_header->source = src_port; + udp_header->source = get_src_port(num_ports, probe_num, + validation); ip_header->check = 0; ip_header->check = ip_checksum((unsigned short *) ip_header); @@ -218,7 +219,6 @@ void udp_print_packet(FILE *fp, void* packet) fprintf(fp, "------------------------------------------------------\n"); } - response_type_t* udp_classify_packet(const u_char *packet, uint32_t len) { (void)len; @@ -232,20 +232,6 @@ response_type_t* udp_classify_packet(const u_char *packet, uint32_t len) } } -// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise -static inline int check_dst_port(uint16_t port, uint32_t *validation) -{ - if (port > zconf.source_port_last - || port < zconf.source_port_first) { - return EXIT_FAILURE; - } - int32_t to_validate = port - zconf.source_port_first; - int32_t min = validation[1] % num_ports; - int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports; - - return (((max - min) % num_ports) >= ((to_validate - min) % num_ports)); -} - int udp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, __attribute__((unused))uint32_t *src_ip, uint32_t *validation) { @@ -290,7 +276,7 @@ int udp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, if (dport != zconf.target_port) { return 0; } - if (!check_dst_port(sport, validation)) { + if (!check_dst_port(sport, num_ports, validation)) { return 0; } return 1; diff --git a/src/probe_modules/packet.h b/src/probe_modules/packet.h index 20f50c2..2acb3b0 100644 --- a/src/probe_modules/packet.h +++ b/src/probe_modules/packet.h @@ -79,4 +79,25 @@ static __attribute__((unused)) uint16_t tcp_checksum(unsigned short len_tcp, return (unsigned short) (~sum); } +// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise +static __attribute__((unused)) inline int check_dst_port(uint16_t port, + int num_ports, uint32_t *validation) +{ + if (port > zconf.source_port_last + || port < zconf.source_port_first) { + return -1; + } + int32_t to_validate = port - zconf.source_port_first; + int32_t min = validation[1] % num_ports; + int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports; + + return (((max - min) % num_ports) >= ((to_validate - min) % num_ports)); +} + +static __attribute__((unused)) inline uint16_t get_src_port(int num_ports, + int probe_num, uint32_t *validation) +{ + return zconf.source_port_first + ((validation[1] + probe_num) % num_ports); +} + #endif From a0288adec8f717fe147e0f8cbaa95f5c82c51fd4 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 19 Aug 2013 03:02:52 -0400 Subject: [PATCH 02/18] beginning the implementation of new output / probe module interface --- src/fieldset.c | 129 ++++++++++++++++++++++ src/fieldset.h | 0 src/output_modules/module_extended_file.c | 1 + src/probe_modules/module_icmp_echo.c | 93 ++++++++-------- src/probe_modules/module_tcp_synscan.c | 78 +++++++++---- src/zopt.c | 96 +++++++++++----- src/zopt.ggo | 31 +++--- src/zopt.h | 6 + 8 files changed, 322 insertions(+), 112 deletions(-) create mode 100644 src/fieldset.c create mode 100644 src/fieldset.h diff --git a/src/fieldset.c b/src/fieldset.c new file mode 100644 index 0000000..b83303d --- /dev/null +++ b/src/fieldset.c @@ -0,0 +1,129 @@ +#include "fieldset.h" + +#include +#include +#include +#include + +#include "../lib/logger.h" + +// maximum number of records that can be stored in a fieldset +#define MAX_FIELDS 128 + +// types of data that can be stored in a field +#define FS_STRING 0 +#define FS_UINT64 1 +#define FS_BINARY 2 + +// definition of a field that's provided by a probe module +// these are used so that users can ask at the command-line +// what fields are available for consumption +typedef struct field_def { + const char *name; + const char *type; + const char *desc; +} field_def_t; + +// the internal field type used by fieldset +typedef struct field { + const char *name; + int type; + int free_; + size_t len; + void *value; +} field_t; + +// data structure that is populated by the probe module +// and translated into the data structure that's passed +// to the output module +typedef struct fieldset { + int len; + field_t fields[MAX_FIELDS]; +} fieldset_t; + +// we pass a different fieldset to an output module than +// the probe module generates for us because a user may +// only want certain fields and will expect them in a certain +// order. We generate a translated fieldset that contains +// only the fields we want to export to the output module. +// a translation specifies how to efficiently convert the fs +// povided by the probe module to the fs for the output module. +typedef struct translation { + int len; + int translation[MAX_FIELDS]; +} translation_t; + +int fs_split_string(int *len, char**results) +{ + +} + +fieldset_t *fs_new_fieldset(void) +{ + fieldset_t *f = malloc(sizeof(fieldset_t)); + if (!f) { + log_fatal("fieldset", "unable to allocate new fieldset"); + } + memset(f, 0, sizeof(fieldset_t)); + return f; +} + +static inline void fs_add_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, void *value) +{ + if (fs->len + 1 >= MAX_FIELDS) { + log_fatal("fieldset", "out of room in fieldset"); + } + field_t *f = &(fs->fields[fs->len]); + fs->len++; + f->type = type; + f->name = name; + f->len = len; + f->value = value; + f->free_ = free_; +} + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + fs_add_word(fs, name, FS_STRING, free_, strlen(value), (void*) value); +} + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value) +{ + fs_add_word(fs, name, FS_STRING, 0, sizeof(uint64_t), (void*) value); +} + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, + void *value, int free_) +{ + fs_add_word(fs, name, FS_BINARY, free_, len, value); +} + +void fs_free(fieldset_t *fs) +{ + for (int i=0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (f->free_) { + free(f->value); + } + } + free(fs); +} + +translation_t *fs_generate_fieldset_translation() +{ + +} + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) +{ + fieldset_t *retv = fs_new_fieldset(); + if (!retv) { + log_fatal("fieldset", "unable to allocate space for translated field set"); + } + for (int i=0; i < t->len; i++) { + int o = t->translation[i]; + memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t)); + } + retv->len = t->len; +} diff --git a/src/fieldset.h b/src/fieldset.h new file mode 100644 index 0000000..e69de29 diff --git a/src/output_modules/module_extended_file.c b/src/output_modules/module_extended_file.c index 477bd67..5e035bc 100644 --- a/src/output_modules/module_extended_file.c +++ b/src/output_modules/module_extended_file.c @@ -24,6 +24,7 @@ static FILE *file = NULL; #define UNUSED __attribute__((unused)) + int extendedfile_init(struct state_conf *conf) { assert(conf); diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c index 4c06763..be12eb3 100644 --- a/src/probe_modules/module_icmp_echo.c +++ b/src/probe_modules/module_icmp_echo.c @@ -24,6 +24,7 @@ #include #include "probe_modules.h" +#include "../fieldset.h" #include "packet.h" #include "validate.h" @@ -111,29 +112,10 @@ void icmp_echo_print_packet(FILE *fp, void* packet) fprintf(fp, "------------------------------------------------------\n"); } -response_type_t* icmp_echo_classify_packet(const u_char *packet, uint32_t len) -{ - (void)len; - struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; - struct icmp *icmp_hdr = (struct icmp*)((char *)ip_hdr - + sizeof(struct iphdr)); - switch (icmp_hdr->icmp_type) { - case ICMP_ECHOREPLY: - return &(module_icmp_echo.responses[0]); - case ICMP_UNREACH: - return &(module_icmp_echo.responses[1]); - case ICMP_SOURCEQUENCH: - return &(module_icmp_echo.responses[2]); - case ICMP_REDIRECT: - return &(module_icmp_echo.responses[3]); - case ICMP_TIMXCEED: - return &(module_icmp_echo.responses[4]); - default: - return &(module_icmp_echo.responses[5]); - } -} -int icmp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src_ip, uint32_t *validation) + +int icmp_validate_packet(const struct iphdr *ip_hdr, + uint32_t len, uint32_t *src_ip, uint32_t *validation) { if (ip_hdr->protocol != IPPROTO_ICMP) { return 0; @@ -175,32 +157,38 @@ int icmp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src return 1; } -static response_type_t responses[] = { - { - .name = "echoreply", - .is_success = 1 - }, - { - .name = "unreach", - .is_success = 0 - }, - { - .name = "sourcequench", - .is_success = 0 - }, - { - .name = "redirect", - .is_success = 0 - }, - { - .name = "timxceed", - .is_success = 0 - }, - { - .name = "other", - .is_success = 0 +void icmp_echo_process_packet(const u_char *packet, + __attribute__((unused)) uint32_t len, fieldset *fs) +{ + struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; + struct icmp *icmp_hdr = (struct icmp*)((char *)ip_hdr + + sizeof(struct iphdr)); + + fs_add_uint64(fs, "type", ntohs(icmp_hdr->type)); + fs_add_uint64(fs, "code", ntohs(icmp_hdr->code)); + fs_add_uint64(fs, "icmp-id", ntohs(icmp_hdr->un.echo.id)); + fs_add_uint64(fs, "seq", ntohs(icmp_hdr->un.echo.sequence)); + switch (icmp_hdr->icmp_type) { + case ICMP_ECHOREPLY: + fs_add_string(fs, "classification", "echoreply", 0); + fs_add_uint64(fs, "success", 1); + case ICMP_UNREACH: + fs_add_string(fs, "classification", "unreach", 0); + fs_add_uint64(fs, "success", 0); + case ICMP_SOURCEQUENCH: + fs_add_string(fs, "classification", "sourcequench", 0); + fs_add_uint64(fs, "success", 0); + case ICMP_REDIRECT: + fs_add_string(fs, "classification", "redirect", 0); + fs_add_uint64(fs, "success", 0); + case ICMP_TIMXCEED: + fs_add_string(fs, "classification", "timxceed", 0); + fs_add_uint64(fs, "success", 0); + default: + fs_add_string(fs, "classification", "other", 0); + fs_add_uint64(fs, "success", 0); } -}; +} probe_module_t module_icmp_echo = { .name = "icmp_echoscan", @@ -211,9 +199,16 @@ probe_module_t module_icmp_echo = { .thread_initialize = &icmp_echo_init_perthread, .make_packet = &icmp_echo_make_packet, .print_packet = &icmp_echo_print_packet, - .classify_packet = &icmp_echo_classify_packet, + .process_response = &icmp_echo_process_response, .validate_packet = &icmp_validate_packet, .close = NULL, - .responses = responses + .fields = { + {.name "type", .type="int", .desc="icmp message type"}, + {.name "code", .type="int", .desc="icmp message sub type code"}, + {.name "icmp-id", .type="int", .desc="icmp id number"}, + {.name "seq", .type="int", .desc="icmp sequence number"}, + {.name="classification", .type="string", .desc="probe module classification"}, + {.name="success", .type="int", .desc="did probe module classify response as success"} + } }; diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c index ce59bcc..24da7c2 100644 --- a/src/probe_modules/module_tcp_synscan.c +++ b/src/probe_modules/module_tcp_synscan.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include +#include "../fieldset.h" #include "probe_modules.h" #include "packet.h" @@ -112,18 +114,6 @@ void synscan_print_packet(FILE *fp, void* packet) fprintf(fp, "------------------------------------------------------\n"); } -response_type_t* synscan_classify_packet(const u_char *packet, uint32_t len) -{ - (void)len; - struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; - struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr - + (sizeof(struct iphdr))); - if (tcp->rst) { // RST packet - return &(module_tcp_synscan.responses[1]); - } else { // SYNACK packet - return &(module_tcp_synscan.responses[0]); - } -} int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, @@ -159,16 +149,52 @@ int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, return 1; } -static response_type_t responses[] = { - { - .is_success = 1, - .name = "synack" - }, - { - .is_success = 0, - .name = "rst" +void fs_add_sys_fields(fieldset_t *fs) +{ + +} + +char *make_ip_str(uint32_t ip) +{ + struct in_addr t; + t.saddr = ip; + const char *temp = inet_ntoa(t); + char *retv = malloc(strlen(temp)+1); + assert (retv); + strcpy(retv, temp); + return retv; +} + +void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) +{ + fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1); + fs_add_string(fs, "daddr", make_ip_str(ip->daddr), 1); + fs_add_uint64(fs, "ipid", ntohl(ip->id); + fs_add_uint64(fs, "ttl", ntohl(ip->ttl); +} + +void synscan_process_packet(const u_char *packet, + __attribute__((unused)) uint32_t len, fieldset_t *fs) +{ + struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; + struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr + + (sizeof(struct iphdr))); + + fs_add_uint64(fs, "sport", (uint64_t) ntohs(tcp->source)); + fs_add_uint64(fs, "dport", (uint64_t) ntohs(tcp->dest)); + fs_add_uint64(fs, "seqnum", (uint64_t) ntohs(tcp->seq)); + fs_add_uint64(fs, "acknum", (uint64_t) ntohl(tcp->ack_seq)); + fs_add_uint64(fs, "window", (uint64_t) ntohs(tcp->window)); + + if (tcp->rst) { // RST packet + fs_add_string(fs, "classification", "rst", 0); + fs_add_uint64(fs, "success", 0); + } else { // SYNACK packet + fs_add_string(fs, "classification", "synack", 0); + fs_add_uint64(fs, "success", 1); } -}; + return +} probe_module_t module_tcp_synscan = { .name = "tcp_synscan", @@ -180,9 +206,15 @@ probe_module_t module_tcp_synscan = { .thread_initialize = &synscan_init_perthread, .make_packet = &synscan_make_packet, .print_packet = &synscan_print_packet, - .classify_packet = &synscan_classify_packet, + .process_packet = &synscan_process_packet, .validate_packet = &synscan_validate_packet, .close = NULL, - .responses = responses, + .fields = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + } }; diff --git a/src/zopt.c b/src/zopt.c index 0441c76..e733f6e 100644 --- a/src/zopt.c +++ b/src/zopt.c @@ -37,6 +37,7 @@ const char *gengetopt_args_info_help[] = { " -o, --output-file=name Output file", " -b, --blacklist-file=path File of subnets to exclude, in CIDR notation, \n e.g. 192.168.0.0/16", " -w, --whitelist-file=path File of subnets to constrain scan to, in CIDR \n notation, e.g. 192.168.0.0/16", + " -f, --output-fields=fields Fields that should be output in result set", "\nScan options:", " -n, --max-targets=n Cap number of targets to probe (as a number or \n a percentage of the address space)", " -N, --max-results=n Cap number of results to return", @@ -60,6 +61,7 @@ const char *gengetopt_args_info_help[] = { " --output-args=args Arguments to pass to output module", " --list-output-modules List available output modules", " --list-probe-modules List available probe modules", + " --list-output-fields List all fields that can be output by selected \n probe module", "\nAdditional options:", " -C, --config=filename Read a configuration file, which can specify \n any of these options \n (default=`/etc/zmap/zmap.conf')", " -q, --quiet Do not print status updates", @@ -120,6 +122,7 @@ void clear_given (struct gengetopt_args_info *args_info) args_info->output_file_given = 0 ; args_info->blacklist_file_given = 0 ; args_info->whitelist_file_given = 0 ; + args_info->output_fields_given = 0 ; args_info->max_targets_given = 0 ; args_info->max_results_given = 0 ; args_info->max_runtime_given = 0 ; @@ -140,6 +143,7 @@ void clear_given (struct gengetopt_args_info *args_info) args_info->output_args_given = 0 ; args_info->list_output_modules_given = 0 ; args_info->list_probe_modules_given = 0 ; + args_info->list_output_fields_given = 0 ; args_info->config_given = 0 ; args_info->quiet_given = 0 ; args_info->summary_given = 0 ; @@ -159,6 +163,8 @@ void clear_args (struct gengetopt_args_info *args_info) args_info->blacklist_file_orig = NULL; args_info->whitelist_file_arg = NULL; args_info->whitelist_file_orig = NULL; + args_info->output_fields_arg = NULL; + args_info->output_fields_orig = NULL; args_info->max_targets_arg = NULL; args_info->max_targets_orig = NULL; args_info->max_results_orig = NULL; @@ -205,32 +211,34 @@ void init_args_info(struct gengetopt_args_info *args_info) args_info->output_file_help = gengetopt_args_info_help[2] ; args_info->blacklist_file_help = gengetopt_args_info_help[3] ; args_info->whitelist_file_help = gengetopt_args_info_help[4] ; - args_info->max_targets_help = gengetopt_args_info_help[6] ; - args_info->max_results_help = gengetopt_args_info_help[7] ; - args_info->max_runtime_help = gengetopt_args_info_help[8] ; - args_info->rate_help = gengetopt_args_info_help[9] ; - args_info->bandwidth_help = gengetopt_args_info_help[10] ; - args_info->cooldown_time_help = gengetopt_args_info_help[11] ; - args_info->seed_help = gengetopt_args_info_help[12] ; - args_info->sender_threads_help = gengetopt_args_info_help[13] ; - args_info->probes_help = gengetopt_args_info_help[14] ; - args_info->dryrun_help = gengetopt_args_info_help[15] ; - args_info->source_port_help = gengetopt_args_info_help[17] ; - args_info->source_ip_help = gengetopt_args_info_help[18] ; - args_info->gateway_mac_help = gengetopt_args_info_help[19] ; - args_info->interface_help = gengetopt_args_info_help[20] ; - args_info->probe_module_help = gengetopt_args_info_help[22] ; - args_info->output_module_help = gengetopt_args_info_help[23] ; - args_info->probe_args_help = gengetopt_args_info_help[24] ; - args_info->output_args_help = gengetopt_args_info_help[25] ; - args_info->list_output_modules_help = gengetopt_args_info_help[26] ; - args_info->list_probe_modules_help = gengetopt_args_info_help[27] ; - args_info->config_help = gengetopt_args_info_help[29] ; - args_info->quiet_help = gengetopt_args_info_help[30] ; - args_info->summary_help = gengetopt_args_info_help[31] ; - args_info->verbosity_help = gengetopt_args_info_help[32] ; - args_info->help_help = gengetopt_args_info_help[33] ; - args_info->version_help = gengetopt_args_info_help[34] ; + args_info->output_fields_help = gengetopt_args_info_help[5] ; + args_info->max_targets_help = gengetopt_args_info_help[7] ; + args_info->max_results_help = gengetopt_args_info_help[8] ; + args_info->max_runtime_help = gengetopt_args_info_help[9] ; + args_info->rate_help = gengetopt_args_info_help[10] ; + args_info->bandwidth_help = gengetopt_args_info_help[11] ; + args_info->cooldown_time_help = gengetopt_args_info_help[12] ; + args_info->seed_help = gengetopt_args_info_help[13] ; + args_info->sender_threads_help = gengetopt_args_info_help[14] ; + args_info->probes_help = gengetopt_args_info_help[15] ; + args_info->dryrun_help = gengetopt_args_info_help[16] ; + args_info->source_port_help = gengetopt_args_info_help[18] ; + args_info->source_ip_help = gengetopt_args_info_help[19] ; + args_info->gateway_mac_help = gengetopt_args_info_help[20] ; + args_info->interface_help = gengetopt_args_info_help[21] ; + args_info->probe_module_help = gengetopt_args_info_help[23] ; + args_info->output_module_help = gengetopt_args_info_help[24] ; + args_info->probe_args_help = gengetopt_args_info_help[25] ; + args_info->output_args_help = gengetopt_args_info_help[26] ; + args_info->list_output_modules_help = gengetopt_args_info_help[27] ; + args_info->list_probe_modules_help = gengetopt_args_info_help[28] ; + args_info->list_output_fields_help = gengetopt_args_info_help[29] ; + args_info->config_help = gengetopt_args_info_help[31] ; + args_info->quiet_help = gengetopt_args_info_help[32] ; + args_info->summary_help = gengetopt_args_info_help[33] ; + args_info->verbosity_help = gengetopt_args_info_help[34] ; + args_info->help_help = gengetopt_args_info_help[35] ; + args_info->version_help = gengetopt_args_info_help[36] ; } @@ -318,6 +326,8 @@ cmdline_parser_release (struct gengetopt_args_info *args_info) free_string_field (&(args_info->blacklist_file_orig)); free_string_field (&(args_info->whitelist_file_arg)); free_string_field (&(args_info->whitelist_file_orig)); + free_string_field (&(args_info->output_fields_arg)); + free_string_field (&(args_info->output_fields_orig)); free_string_field (&(args_info->max_targets_arg)); free_string_field (&(args_info->max_targets_orig)); free_string_field (&(args_info->max_results_orig)); @@ -386,6 +396,8 @@ cmdline_parser_dump(FILE *outfile, struct gengetopt_args_info *args_info) write_into_file(outfile, "blacklist-file", args_info->blacklist_file_orig, 0); if (args_info->whitelist_file_given) write_into_file(outfile, "whitelist-file", args_info->whitelist_file_orig, 0); + if (args_info->output_fields_given) + write_into_file(outfile, "output-fields", args_info->output_fields_orig, 0); if (args_info->max_targets_given) write_into_file(outfile, "max-targets", args_info->max_targets_orig, 0); if (args_info->max_results_given) @@ -426,6 +438,8 @@ cmdline_parser_dump(FILE *outfile, struct gengetopt_args_info *args_info) write_into_file(outfile, "list-output-modules", 0, 0 ); if (args_info->list_probe_modules_given) write_into_file(outfile, "list-probe-modules", 0, 0 ); + if (args_info->list_output_fields_given) + write_into_file(outfile, "list-output-fields", 0, 0 ); if (args_info->config_given) write_into_file(outfile, "config", args_info->config_orig, 0); if (args_info->quiet_given) @@ -692,6 +706,7 @@ cmdline_parser_internal ( { "output-file", 1, NULL, 'o' }, { "blacklist-file", 1, NULL, 'b' }, { "whitelist-file", 1, NULL, 'w' }, + { "output-fields", 1, NULL, 'f' }, { "max-targets", 1, NULL, 'n' }, { "max-results", 1, NULL, 'N' }, { "max-runtime", 1, NULL, 't' }, @@ -712,6 +727,7 @@ cmdline_parser_internal ( { "output-args", 1, NULL, 0 }, { "list-output-modules", 0, NULL, 0 }, { "list-probe-modules", 0, NULL, 0 }, + { "list-output-fields", 0, NULL, 0 }, { "config", 1, NULL, 'C' }, { "quiet", 0, NULL, 'q' }, { "summary", 0, NULL, 'g' }, @@ -721,7 +737,7 @@ cmdline_parser_internal ( { 0, 0, 0, 0 } }; - c = getopt_long (argc, argv, "p:o:b:w:n:N:t:r:B:c:e:T:P:ds:S:G:i:M:O:C:qgv:hV", long_options, &option_index); + c = getopt_long (argc, argv, "p:o:b:w:f:n:N:t:r:B:c:e:T:P:ds:S:G:i:M:O:C:qgv:hV", long_options, &option_index); if (c == -1) break; /* Exit from `while (1)' loop. */ @@ -774,6 +790,18 @@ cmdline_parser_internal ( additional_error)) goto failure; + break; + case 'f': /* Fields that should be output in result set. */ + + + if (update_arg( (void *)&(args_info->output_fields_arg), + &(args_info->output_fields_orig), &(args_info->output_fields_given), + &(local_args_info.output_fields_given), optarg, 0, 0, ARG_STRING, + check_ambiguity, override, 0, 0, + "output-fields", 'f', + additional_error)) + goto failure; + break; case 'n': /* Cap number of targets to probe (as a number or a percentage of the address space). */ @@ -1096,6 +1124,20 @@ cmdline_parser_internal ( additional_error)) goto failure; + } + /* List all fields that can be output by selected probe module. */ + else if (strcmp (long_options[option_index].name, "list-output-fields") == 0) + { + + + if (update_arg( 0 , + 0 , &(args_info->list_output_fields_given), + &(local_args_info.list_output_fields_given), optarg, 0, 0, ARG_NO, + check_ambiguity, override, 0, 0, + "list-output-fields", '-', + additional_error)) + goto failure; + } break; diff --git a/src/zopt.ggo b/src/zopt.ggo index b52a4e1..0446738 100644 --- a/src/zopt.ggo +++ b/src/zopt.ggo @@ -22,6 +22,9 @@ option "blacklist-file" b "File of subnets to exclude, in CIDR notation, e.g. 1 option "whitelist-file" w "File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16" typestr="path" optional string +option "output-fields" f "Fields that should be output in result set" + typestr="fields" + optional string section "Scan options" @@ -51,11 +54,11 @@ option "sender-threads" T "Threads used to send packets" typestr="n" default="1" optional int -option "probes" P "Number of probes to send to each IP" +option "probes" P "Number of probes to send to each IP" typestr="n" default="1" optional int -option "dryrun" d "Don't actually send packets" +option "dryrun" d "Don't actually send packets" optional section "Network options" @@ -63,13 +66,13 @@ section "Network options" option "source-port" s "Source port(s) for scan packets" typestr="port|range" optional string -option "source-ip" S "Source address(es) for scan packets" +option "source-ip" S "Source address(es) for scan packets" typestr="ip|range" optional string option "gateway-mac" G "Specify gateway MAC address" typestr="addr" optional string -option "interface" i "Specify network interface to use" +option "interface" i "Specify network interface to use" typestr="name" optional string @@ -83,34 +86,36 @@ option "output-module" O "Select output module" typestr="name" default="simple_file" optional string -option "probe-args" - "Arguments to pass to probe module" +option "probe-args" - "Arguments to pass to probe module" typestr="args" optional string option "output-args" - "Arguments to pass to output module" typestr="args" optional string -option "list-output-modules" - "List available output modules" +option "list-output-modules" - "List available output modules" optional -option "list-probe-modules" - "List available probe modules" +option "list-probe-modules" - "List available probe modules" optional +option "list-output-fields" - "List all fields that can be output by selected probe module" + optional section "Additional options" -option "config" C "Read a configuration file, which can specify any of these options" +option "config" C "Read a configuration file, which can specify any of these options" typestr="filename" default="/etc/zmap/zmap.conf" optional string -option "quiet" q "Do not print status updates" +option "quiet" q "Do not print status updates" optional -option "summary" g "Print configuration and summary at end of scan" +option "summary" g "Print configuration and summary at end of scan" optional -option "verbosity" v "Level of log detail (0-5)" +option "verbosity" v "Level of log detail (0-5)" typestr="n" default="3" optional int -option "help" h "Print help and exit" +option "help" h "Print help and exit" optional -option "version" V "Print version and exit" +option "version" V "Print version and exit" optional text "\nExamples:\n\ diff --git a/src/zopt.h b/src/zopt.h index 30f30ee..c1fe727 100644 --- a/src/zopt.h +++ b/src/zopt.h @@ -49,6 +49,9 @@ struct gengetopt_args_info char * whitelist_file_arg; /**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16. */ char * whitelist_file_orig; /**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 original value given at command line. */ const char *whitelist_file_help; /**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 help description. */ + char * output_fields_arg; /**< @brief Fields that should be output in result set. */ + char * output_fields_orig; /**< @brief Fields that should be output in result set original value given at command line. */ + const char *output_fields_help; /**< @brief Fields that should be output in result set help description. */ char * max_targets_arg; /**< @brief Cap number of targets to probe (as a number or a percentage of the address space). */ char * max_targets_orig; /**< @brief Cap number of targets to probe (as a number or a percentage of the address space) original value given at command line. */ const char *max_targets_help; /**< @brief Cap number of targets to probe (as a number or a percentage of the address space) help description. */ @@ -103,6 +106,7 @@ struct gengetopt_args_info const char *output_args_help; /**< @brief Arguments to pass to output module help description. */ const char *list_output_modules_help; /**< @brief List available output modules help description. */ const char *list_probe_modules_help; /**< @brief List available probe modules help description. */ + const char *list_output_fields_help; /**< @brief List all fields that can be output by selected probe module help description. */ char * config_arg; /**< @brief Read a configuration file, which can specify any of these options (default='/etc/zmap/zmap.conf'). */ char * config_orig; /**< @brief Read a configuration file, which can specify any of these options original value given at command line. */ const char *config_help; /**< @brief Read a configuration file, which can specify any of these options help description. */ @@ -118,6 +122,7 @@ struct gengetopt_args_info unsigned int output_file_given ; /**< @brief Whether output-file was given. */ unsigned int blacklist_file_given ; /**< @brief Whether blacklist-file was given. */ unsigned int whitelist_file_given ; /**< @brief Whether whitelist-file was given. */ + unsigned int output_fields_given ; /**< @brief Whether output-fields was given. */ unsigned int max_targets_given ; /**< @brief Whether max-targets was given. */ unsigned int max_results_given ; /**< @brief Whether max-results was given. */ unsigned int max_runtime_given ; /**< @brief Whether max-runtime was given. */ @@ -138,6 +143,7 @@ struct gengetopt_args_info unsigned int output_args_given ; /**< @brief Whether output-args was given. */ unsigned int list_output_modules_given ; /**< @brief Whether list-output-modules was given. */ unsigned int list_probe_modules_given ; /**< @brief Whether list-probe-modules was given. */ + unsigned int list_output_fields_given ; /**< @brief Whether list-output-fields was given. */ unsigned int config_given ; /**< @brief Whether config was given. */ unsigned int quiet_given ; /**< @brief Whether quiet was given. */ unsigned int summary_given ; /**< @brief Whether summary was given. */ From 952f6cefa16218b695ca6bc3d4f9ad1b1cefbaae Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 19 Aug 2013 04:42:25 -0400 Subject: [PATCH 03/18] building fieldset and probe modules --- src/Makefile | 2 +- src/fieldset.c | 53 ++---------------- src/fieldset.h | 77 ++++++++++++++++++++++++++ src/probe_modules/module_icmp_echo.c | 42 +++++++------- src/probe_modules/module_tcp_synscan.c | 45 ++++----------- src/probe_modules/probe_modules.c | 35 ++++++++++++ src/probe_modules/probe_modules.h | 11 ++-- 7 files changed, 159 insertions(+), 106 deletions(-) diff --git a/src/Makefile b/src/Makefile index 72eb0d5..eeffc0b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -35,7 +35,7 @@ LDFLAGS+=$(EXTRALDFLAGS) modules=module_tcp_synscan.o module_icmp_echo.o module_udp.o #ADD YOUR MODULE HERE -objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt.o zmap.o random.o output_modules.o module_simple_file.o module_extended_file.o packet.o probe_modules.o ${modules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o +objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt.o zmap.o random.o output_modules.o module_simple_file.o module_extended_file.o packet.o probe_modules.o ${modules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o fieldset.o ifeq ($(REDIS), true) LDLIBS+=-lhiredis diff --git a/src/fieldset.c b/src/fieldset.c index b83303d..7c0de51 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -7,55 +7,12 @@ #include "../lib/logger.h" -// maximum number of records that can be stored in a fieldset -#define MAX_FIELDS 128 - -// types of data that can be stored in a field -#define FS_STRING 0 -#define FS_UINT64 1 -#define FS_BINARY 2 - -// definition of a field that's provided by a probe module -// these are used so that users can ask at the command-line -// what fields are available for consumption -typedef struct field_def { - const char *name; - const char *type; - const char *desc; -} field_def_t; - -// the internal field type used by fieldset -typedef struct field { - const char *name; - int type; - int free_; - size_t len; - void *value; -} field_t; - -// data structure that is populated by the probe module -// and translated into the data structure that's passed -// to the output module -typedef struct fieldset { - int len; - field_t fields[MAX_FIELDS]; -} fieldset_t; - -// we pass a different fieldset to an output module than -// the probe module generates for us because a user may -// only want certain fields and will expect them in a certain -// order. We generate a translated fieldset that contains -// only the fields we want to export to the output module. -// a translation specifies how to efficiently convert the fs -// povided by the probe module to the fs for the output module. -typedef struct translation { - int len; - int translation[MAX_FIELDS]; -} translation_t; int fs_split_string(int *len, char**results) { - + (void)len; + (void)results; + return 0; } fieldset_t *fs_new_fieldset(void) @@ -112,7 +69,7 @@ void fs_free(fieldset_t *fs) translation_t *fs_generate_fieldset_translation() { - + return NULL; } fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) @@ -126,4 +83,6 @@ fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t)); } retv->len = t->len; + return retv; } + diff --git a/src/fieldset.h b/src/fieldset.h index e69de29..5c47990 100644 --- a/src/fieldset.h +++ b/src/fieldset.h @@ -0,0 +1,77 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include + +#ifndef _FIELDSET_H +#define _FIELDSET_H + +// maximum number of records that can be stored in a fieldset +#define MAX_FIELDS 128 + +// types of data that can be stored in a field +#define FS_STRING 0 +#define FS_UINT64 1 +#define FS_BINARY 2 + +// definition of a field that's provided by a probe module +// these are used so that users can ask at the command-line +// what fields are available for consumption +typedef struct field_def { + const char *name; + const char *type; + const char *desc; +} fielddef_t; + +// the internal field type used by fieldset +typedef struct field { + const char *name; + int type; + int free_; + size_t len; + void *value; +} field_t; + +// data structure that is populated by the probe module +// and translated into the data structure that's passed +// to the output module +typedef struct fieldset { + int len; + field_t fields[MAX_FIELDS]; +} fieldset_t; + +// we pass a different fieldset to an output module than +// the probe module generates for us because a user may +// only want certain fields and will expect them in a certain +// order. We generate a translated fieldset that contains +// only the fields we want to export to the output module. +// a translation specifies how to efficiently convert the fs +// povided by the probe module to the fs for the output module. +typedef struct translation { + int len; + int translation[MAX_FIELDS]; +} translation_t; + + +fieldset_t *fs_new_fieldset(void); + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, + void *value, int free_); + +void fs_free(fieldset_t *fs); + +translation_t *fs_generate_fieldset_translation(); + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); + +#endif diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c index be12eb3..50907ad 100644 --- a/src/probe_modules/module_icmp_echo.c +++ b/src/probe_modules/module_icmp_echo.c @@ -158,38 +158,46 @@ int icmp_validate_packet(const struct iphdr *ip_hdr, } void icmp_echo_process_packet(const u_char *packet, - __attribute__((unused)) uint32_t len, fieldset *fs) + __attribute__((unused)) uint32_t len, fieldset_t *fs) { struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; - struct icmp *icmp_hdr = (struct icmp*)((char *)ip_hdr - + sizeof(struct iphdr)); - + struct icmphdr *icmp_hdr = (struct icmphdr*)((char *)ip_hdr + 4 *ip_hdr->ihl); fs_add_uint64(fs, "type", ntohs(icmp_hdr->type)); fs_add_uint64(fs, "code", ntohs(icmp_hdr->code)); fs_add_uint64(fs, "icmp-id", ntohs(icmp_hdr->un.echo.id)); fs_add_uint64(fs, "seq", ntohs(icmp_hdr->un.echo.sequence)); - switch (icmp_hdr->icmp_type) { + switch (icmp_hdr->type) { case ICMP_ECHOREPLY: - fs_add_string(fs, "classification", "echoreply", 0); + fs_add_string(fs, "classification", (char*) "echoreply", 0); fs_add_uint64(fs, "success", 1); case ICMP_UNREACH: - fs_add_string(fs, "classification", "unreach", 0); + fs_add_string(fs, "classification", (char*) "unreach", 0); fs_add_uint64(fs, "success", 0); case ICMP_SOURCEQUENCH: - fs_add_string(fs, "classification", "sourcequench", 0); + fs_add_string(fs, "classification", (char*) "sourcequench", 0); fs_add_uint64(fs, "success", 0); case ICMP_REDIRECT: - fs_add_string(fs, "classification", "redirect", 0); + fs_add_string(fs, "classification", (char*) "redirect", 0); fs_add_uint64(fs, "success", 0); case ICMP_TIMXCEED: - fs_add_string(fs, "classification", "timxceed", 0); + fs_add_string(fs, "classification", (char*) "timxceed", 0); fs_add_uint64(fs, "success", 0); default: - fs_add_string(fs, "classification", "other", 0); + fs_add_string(fs, "classification", (char*) "other", 0); fs_add_uint64(fs, "success", 0); } } +fielddef_t fields[] = { + {.name="type", .type="int", .desc="icmp message type"}, + {.name="code", .type="int", .desc="icmp message sub type code"}, + {.name="icmp-id", .type="int", .desc="icmp id number"}, + {.name="seq", .type="int", .desc="icmp sequence number"}, + {.name="classification", .type="string", .desc="probe module classification"}, + {.name="success", .type="int", .desc="did probe module classify response as success"} +}; + + probe_module_t module_icmp_echo = { .name = "icmp_echoscan", .packet_length = 62, @@ -199,16 +207,8 @@ probe_module_t module_icmp_echo = { .thread_initialize = &icmp_echo_init_perthread, .make_packet = &icmp_echo_make_packet, .print_packet = &icmp_echo_print_packet, - .process_response = &icmp_echo_process_response, + .process_packet = &icmp_echo_process_packet, .validate_packet = &icmp_validate_packet, .close = NULL, - .fields = { - {.name "type", .type="int", .desc="icmp message type"}, - {.name "code", .type="int", .desc="icmp message sub type code"}, - {.name "icmp-id", .type="int", .desc="icmp id number"}, - {.name "seq", .type="int", .desc="icmp sequence number"}, - {.name="classification", .type="string", .desc="probe module classification"}, - {.name="success", .type="int", .desc="did probe module classify response as success"} - } -}; + .fields = fields}; diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c index 24da7c2..448a678 100644 --- a/src/probe_modules/module_tcp_synscan.c +++ b/src/probe_modules/module_tcp_synscan.c @@ -149,29 +149,7 @@ int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, return 1; } -void fs_add_sys_fields(fieldset_t *fs) -{ -} - -char *make_ip_str(uint32_t ip) -{ - struct in_addr t; - t.saddr = ip; - const char *temp = inet_ntoa(t); - char *retv = malloc(strlen(temp)+1); - assert (retv); - strcpy(retv, temp); - return retv; -} - -void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) -{ - fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1); - fs_add_string(fs, "daddr", make_ip_str(ip->daddr), 1); - fs_add_uint64(fs, "ipid", ntohl(ip->id); - fs_add_uint64(fs, "ttl", ntohl(ip->ttl); -} void synscan_process_packet(const u_char *packet, __attribute__((unused)) uint32_t len, fieldset_t *fs) @@ -187,15 +165,23 @@ void synscan_process_packet(const u_char *packet, fs_add_uint64(fs, "window", (uint64_t) ntohs(tcp->window)); if (tcp->rst) { // RST packet - fs_add_string(fs, "classification", "rst", 0); + fs_add_string(fs, "classification", (char*) "rst", 0); fs_add_uint64(fs, "success", 0); } else { // SYNACK packet - fs_add_string(fs, "classification", "synack", 0); + fs_add_string(fs, "classification", (char*) "synack", 0); fs_add_uint64(fs, "success", 1); } - return } +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, +}; + + probe_module_t module_tcp_synscan = { .name = "tcp_synscan", .packet_length = 54, @@ -209,12 +195,5 @@ probe_module_t module_tcp_synscan = { .process_packet = &synscan_process_packet, .validate_packet = &synscan_validate_packet, .close = NULL, - .fields = { - {.name = "sport", .type = "int", .desc = "TCP source port"}, - {.name = "dport", .type = "int", .desc = "TCP destination port"}, - {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, - {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, - {.name = "window", .type = "int", .desc = "TCP window"}, - } -}; + .fields = fields}; diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c index f67e831..de3c618 100644 --- a/src/probe_modules/probe_modules.c +++ b/src/probe_modules/probe_modules.c @@ -8,7 +8,15 @@ #include #include +#include +#include +#include +#include +#include +#include + +#include "../fieldset.h" #include "probe_modules.h" extern probe_module_t module_tcp_synscan; @@ -40,3 +48,30 @@ void print_probe_modules(void) printf("%s\n", probe_modules[i]->name); } } + +void print_probe_module_fields(probe_module_t *p) +{ + for (int i=0; i < (int) (sizeof(p->fields)/sizeof(p->fields[0])); i++) { + + } +} + +char *make_ip_str(uint32_t ip) +{ + struct in_addr t; + t.s_addr = ip; + const char *temp = inet_ntoa(t); + char *retv = malloc(strlen(temp)+1); + assert (retv); + strcpy(retv, temp); + return retv; +} + +void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) +{ + fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1); + fs_add_string(fs, "daddr", make_ip_str(ip->daddr), 1); + fs_add_uint64(fs, "ipid", ntohl(ip->id)); + fs_add_uint64(fs, "ttl", ntohl(ip->ttl)); +} + diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h index 18f8438..613c1e2 100644 --- a/src/probe_modules/probe_modules.h +++ b/src/probe_modules/probe_modules.h @@ -1,4 +1,5 @@ #include "../state.h" +#include "../fieldset.h" #ifndef HEADER_PROBE_MODULES_H #define HEADER_PROBE_MODULES_H @@ -14,9 +15,10 @@ typedef int (*probe_make_packet_cb)(void* packetbuf, ipaddr_n_t src_ip, ipaddr_n uint32_t *validation, int probe_num); typedef void (*probe_print_packet_cb)(FILE *, void* packetbuf); typedef int (*probe_close_cb)(struct state_conf*, struct state_send*, struct state_recv*); -typedef response_type_t* (*probe_classify_packet_cb)(const u_char* packetbuf, uint32_t len); typedef int (*probe_validate_packet_cb)(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src_ip, uint32_t *validation); +typedef void (*probe_classify_packet_cb)(const u_char* packetbuf, uint32_t len, fieldset_t*); + typedef struct probe_module { const char *name; size_t packet_length; @@ -27,19 +29,20 @@ typedef struct probe_module { // source and target port numbers? uint8_t port_args; - response_type_t *responses; - probe_global_init_cb global_initialize; probe_thread_init_cb thread_initialize; probe_make_packet_cb make_packet; probe_print_packet_cb print_packet; probe_validate_packet_cb validate_packet; - probe_classify_packet_cb classify_packet; + probe_classify_packet_cb process_packet; probe_close_cb close; + fielddef_t *fields; + } probe_module_t; probe_module_t* get_probe_module_by_name(const char*); + void print_probe_modules(void); #endif // HEADER_PROBE_MODULES_H From 582d381b3080950a0aefd2501974cd352a4f4c23 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 19 Aug 2013 10:53:48 -0400 Subject: [PATCH 04/18] adding output-fields to global conf --- src/fieldset.c | 5 +++++ src/state.c | 3 +++ src/state.h | 3 +++ 3 files changed, 11 insertions(+) diff --git a/src/fieldset.c b/src/fieldset.c index 7c0de51..327afe1 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -15,6 +15,11 @@ int fs_split_string(int *len, char**results) return 0; } +void combine_definitions() +{ + +} + fieldset_t *fs_new_fieldset(void) { fieldset_t *f = malloc(sizeof(fieldset_t)); diff --git a/src/state.c b/src/state.c index a1caa8a..9b8a583 100644 --- a/src/state.c +++ b/src/state.c @@ -38,6 +38,9 @@ struct state_conf zconf = { .gw_mac_set = 0, .source_ip_first = NULL, .source_ip_last = NULL, + .raw_output_fields = NULL, + .output_fields = NULL, + .output_fields_len = 0, .dryrun = 0, .quiet = 0, .summary = 0, diff --git a/src/state.h b/src/state.h index f1c967e..7730b99 100644 --- a/src/state.h +++ b/src/state.h @@ -71,6 +71,9 @@ struct state_conf { char *output_filename; char *blacklist_filename; char *whitelist_filename; + char *raw_output_fields; + char **output_fields; + int output_fields_len; int dryrun; int summary; int quiet; From b4f8698cb04f135a45cb9f38443b8c131e281168 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 19 Aug 2013 18:10:55 -0400 Subject: [PATCH 05/18] further progress --- src/fieldset.c | 18 ++++++++++++++++-- src/probe_modules/probe_modules.c | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/fieldset.c b/src/fieldset.c index 327afe1..5c105cb 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -7,10 +7,24 @@ #include "../lib/logger.h" - -int fs_split_string(int *len, char**results) +int fs_split_string(char* in, int *len, char**results) { + + char** fields = calloc(MAX_FIELDS, sizeof(char*)); + memset(fields, 0, sizeof(fields)); + int retvlen = 0; + char *currloc = in; + // parse csv into a set of strings + while (1) { + size_t len = strcspn(currloc, ", "); + char *new = malloc(len+1); + + + } + (void)len; + *results = fields; + *len = retvlen; (void)results; return 0; } diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c index de3c618..f87ebeb 100644 --- a/src/probe_modules/probe_modules.c +++ b/src/probe_modules/probe_modules.c @@ -67,6 +67,18 @@ char *make_ip_str(uint32_t ip) return retv; } +fielddef_t ip_fields[] = { + {.name="saddr", .type="string", .desc="source IP address of response"}, + {.name="daddr", .type="string", .desc="destination IP address of response"}, + {.name="ipid", .type="int", .desc="IP identification number of response"}, + {.name="ttl", .type="int", .desc="time-to-live of response packet"} +} + +fielddef_t sys_fields[] = { + {.name="repeat", .type="int", .desc="Is response a repeat response from host"}, + {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."} +} + void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) { fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1); @@ -75,3 +87,9 @@ void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) fs_add_uint64(fs, "ttl", ntohl(ip->ttl)); } +void fs_add_system_fields(fieldset_t *fs) +{ + + +} + From 99312020bad2d0da260473b7601a8e357d18ec48 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Tue, 20 Aug 2013 09:45:52 -0400 Subject: [PATCH 06/18] parse CLI provided output columns --- src/zmap.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/zmap.c b/src/zmap.c index 1d44079..79a0f7c 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -39,6 +39,35 @@ pthread_mutex_t cpu_affinity_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; +// splits comma delimited string into char*[]. Does not handle +// escaping or complicated setups: designed to process a set +// of fields that the user wants output +static void fs_split_string(char* in, int *len, char***results) +{ + char** fields = calloc(MAX_FIELDS, sizeof(char*)); + memset(fields, 0, sizeof(fields)); + int retvlen = 0; + char *currloc = in; + // parse csv into a set of strings + while (1) { + size_t len = strcspn(currloc, ", "); + if (len == 0) { + currloc++; + } else { + char *new = malloc(len+1); + strncpy(new, currloc, len); + new[len] = '\0'; + fields[retvlen++] = new; + } + currloc += len; + if (currloc == in + (size_t) strlen(in)) { + break; + } + } + *results = fields; + *len = retvlen; +} + static void set_cpu(void) { pthread_mutex_lock(&cpu_affinity_mutex); @@ -368,6 +397,25 @@ int main(int argc, char *argv[]) CMDLINE_PARSER_PACKAGE, args.probe_module_arg); exit(EXIT_FAILURE); } + // we need to generate a master fielddef list + + // process the list of output fields. + if (args.output_fields_given) { + zconf.raw_output_fields = zopt.output_fields; + fs_split_string(zopt.output_fields, &(zconf.output_fields), + &(zconf.output_fields_len)); + } else { + zconf.output_fields = {"saddr"}; + zconf.num_output_fields = 1; + } + // check the list of requested fields to see if they are available + // with the selected probe module. + for (int i=0; i < zonf.num_output_fields; i++) { + fs_check + + } + + if (zconf.probe_module->port_args) { if (args.source_port_given) { char *dash = strchr(args.source_port_arg, '-'); From c6fc67288a63d143851171d871ae06fa90baa54f Mon Sep 17 00:00:00 2001 From: Jim Cheetham Date: Tue, 20 Aug 2013 21:53:37 -0400 Subject: [PATCH 07/18] Install man files under $(PREFIX) --- src/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index ff65bab..3610c73 100644 --- a/src/Makefile +++ b/src/Makefile @@ -7,7 +7,7 @@ PREFIX=/usr/local INSTALL=install INSTALLDATA=install -m 644 -mandir=/usr/share/man/man1/ +mandir=$(PREFIX)/man/man1/ bindir=$(PREFIX)/sbin # Hardening and warnings for building with gcc @@ -55,6 +55,7 @@ zopt.c zopt.h: zopt.ggo install: zmap $(INSTALL) zmap $(bindir)/zmap test -d /etc/zmap || (mkdir /etc/zmap && $(INSTALLDATA) ../conf/* /etc/zmap/) + test -d $(mandir) || mkdir -p $(mandir) $(INSTALLDATA) ./zmap.1 $(mandir) @echo "\n**************\nSuccess! ZMap is installed. Try running (as root):\nzmap -p 80 -N 10 -B 1M -o -\n**************" From 4ec783f74892eed3c02426f0dddfd2b6a0922dda Mon Sep 17 00:00:00 2001 From: Christopher Meng Date: Thu, 22 Aug 2013 13:40:19 +0800 Subject: [PATCH 08/18] $(EXTRA_CFLAGS) FOR RPM %{optflags} We RPM packaging system need to use optflags for secure building, and of course will override the flags you've defined. However the current one is not good, we should leave it to the end of the flags queue. [rpmaker@fab SPECS]$ rpm -E %{optflags} -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index ff65bab..45d7b37 100644 --- a/src/Makefile +++ b/src/Makefile @@ -23,7 +23,7 @@ GCCWARNINGS = -Wall -Wformat=2 -Wno-format-nonliteral\ GCCHARDENING=-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1 LDHARDENING=-z relro -z now -EXTRACFLAGS=-std=gnu99 -g -O2 $(EXTRA_CFLAGS) $(GCCHARDENING) $(GCCWARNINGS) -Werror +EXTRACFLAGS=-std=gnu99 -g -O2 $(GCCHARDENING) $(GCCWARNINGS) $(EXTRA_CFLAGS) -Werror EXTRALDFLAGS= $(LDHARDENING) CFLAGS+=$(INCLUDE) $(EXTRACFLAGS) From ea39b3bac47c5e44abc46f9d7e2acf59d042199c Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 26 Aug 2013 07:59:41 -0400 Subject: [PATCH 09/18] further progress --- src/fieldset.c | 50 +++++++-------- src/fieldset.h | 14 +++++ src/probe_modules/module_icmp_echo.c | 29 +-------- src/probe_modules/module_tcp_synscan.c | 51 ++++------------ src/probe_modules/module_udp.c | 41 +++---------- src/probe_modules/packet.c | 36 ++++++++++- src/probe_modules/probe_modules.c | 6 +- src/probe_modules/probe_modules.h | 4 +- src/recv.c | 29 +++++---- src/state.h | 20 ++++-- src/zmap.c | 84 +++++++++++++++++--------- 11 files changed, 187 insertions(+), 177 deletions(-) diff --git a/src/fieldset.c b/src/fieldset.c index 5c105cb..25752fa 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -7,31 +7,14 @@ #include "../lib/logger.h" -int fs_split_string(char* in, int *len, char**results) +void gen_fielddef_set(fielddefset_t *fds, fieldset_t fs[], int len) { - - char** fields = calloc(MAX_FIELDS, sizeof(char*)); - memset(fields, 0, sizeof(fields)); - int retvlen = 0; - char *currloc = in; - // parse csv into a set of strings - while (1) { - size_t len = strcspn(currloc, ", "); - char *new = malloc(len+1); - - + if (fds->len + len > MAX_FIELDS) { + log_fatal("fieldset", "out of room in field def set"); } - - (void)len; - *results = fields; - *len = retvlen; - (void)results; - return 0; -} - -void combine_definitions() -{ - + fielddef_t *open = &(fds->fielddefs[fds->len]); + memcpy(open, fs, len*sizeof(fielddef_t)); + fds->len += len; } fieldset_t *fs_new_fieldset(void) @@ -75,8 +58,27 @@ void fs_add_binary(fieldset_t *fs, const char *name, size_t len, fs_add_word(fs, name, FS_BINARY, free_, len, value); } -void fs_free(fieldset_t *fs) +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index) { + return (uint64_t) fs->fields[i].value; +} + +char* fs_get_string_by_index(fieldset_t *fs, int index) +{ + return (char*) fs->fields[i].value; +} + +int fds_get_index_by_name(fielddefset_t *fds, char *name) +{ + for (int i=0; i < fds->len; i++) { + if (!strcmp(fds->fielddefs[i].name, name)) { + return i; + } + } + return -1; +} + +void fs_free(fieldset_t *fs) for (int i=0; i < fs->len; i++) { field_t *f = &(fs->fields[i]); if (f->free_) { diff --git a/src/fieldset.h b/src/fieldset.h index 5c47990..d733dee 100644 --- a/src/fieldset.h +++ b/src/fieldset.h @@ -8,6 +8,7 @@ #include #include +#include "types.h" #ifndef _FIELDSET_H #define _FIELDSET_H @@ -29,6 +30,11 @@ typedef struct field_def { const char *desc; } fielddef_t; +typedef struct fielddef_set { + fielddef_t fielddefs[MAX_FIELDS]; + int len; +} fielddefset_t; + // the internal field type used by fieldset typedef struct field { const char *name; @@ -61,6 +67,14 @@ typedef struct translation { fieldset_t *fs_new_fieldset(void); +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); + +char* fs_get_string_by_index(fieldset_t *fs, int index); + +int fds_get_index_by_name(fielddefset_t *fds, char *name); + +void gen_fielddef_set(fielddefset_t *fds, fieldset_t fs[], int len); + void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c index 50907ad..cb6e9a4 100644 --- a/src/probe_modules/module_icmp_echo.c +++ b/src/probe_modules/module_icmp_echo.c @@ -82,33 +82,8 @@ void icmp_echo_print_packet(FILE *fp, void* packet) ntohs(icmp_header->icmp_cksum), ntohs(icmp_header->icmp_id), ntohs(icmp_header->icmp_seq)); - struct in_addr *s = (struct in_addr *) &(iph->saddr); - struct in_addr *d = (struct in_addr *) &(iph->daddr); - char srcip[20]; - char dstip[20]; - // inet_ntoa is a const char * so we if just call it in - // fprintf, you'll get back wrong results since we're - // calling it twice. - strncpy(srcip, inet_ntoa(*s), 19); - strncpy(dstip, inet_ntoa(*d), 19); - fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n", - srcip, - dstip, - ntohl(iph->check)); - fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " - "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", - (int) ((unsigned char *) ethh->h_source)[0], - (int) ((unsigned char *) ethh->h_source)[1], - (int) ((unsigned char *) ethh->h_source)[2], - (int) ((unsigned char *) ethh->h_source)[3], - (int) ((unsigned char *) ethh->h_source)[4], - (int) ((unsigned char *) ethh->h_source)[5], - (int) ((unsigned char *) ethh->h_dest)[0], - (int) ((unsigned char *) ethh->h_dest)[1], - (int) ((unsigned char *) ethh->h_dest)[2], - (int) ((unsigned char *) ethh->h_dest)[3], - (int) ((unsigned char *) ethh->h_dest)[4], - (int) ((unsigned char *) ethh->h_dest)[5]); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); fprintf(fp, "------------------------------------------------------\n"); } diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c index 448a678..a8f2e23 100644 --- a/src/probe_modules/module_tcp_synscan.c +++ b/src/probe_modules/module_tcp_synscan.c @@ -84,45 +84,18 @@ void synscan_print_packet(FILE *fp, void* packet) ntohs(tcph->dest), ntohl(tcph->seq), ntohl(tcph->check)); - struct in_addr *s = (struct in_addr *) &(iph->saddr); - struct in_addr *d = (struct in_addr *) &(iph->daddr); - char srcip[20]; - char dstip[20]; - // inet_ntoa is a const char * so we if just call it in - // fprintf, you'll get back wrong results since we're - // calling it twice. - strncpy(srcip, inet_ntoa(*s), 19); - strncpy(dstip, inet_ntoa(*d), 19); - fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n", - srcip, - dstip, - ntohl(iph->check)); - fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " - "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", - (int) ((unsigned char *) ethh->h_source)[0], - (int) ((unsigned char *) ethh->h_source)[1], - (int) ((unsigned char *) ethh->h_source)[2], - (int) ((unsigned char *) ethh->h_source)[3], - (int) ((unsigned char *) ethh->h_source)[4], - (int) ((unsigned char *) ethh->h_source)[5], - (int) ((unsigned char *) ethh->h_dest)[0], - (int) ((unsigned char *) ethh->h_dest)[1], - (int) ((unsigned char *) ethh->h_dest)[2], - (int) ((unsigned char *) ethh->h_dest)[3], - (int) ((unsigned char *) ethh->h_dest)[4], - (int) ((unsigned char *) ethh->h_dest)[5]); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); fprintf(fp, "------------------------------------------------------\n"); } - - int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, - __attribute__((unused))uint32_t *src_ip, uint32_t *validation) + __attribute__((unused))uint32_t *src_ip, + uint32_t *validation) { if (ip_hdr->protocol != IPPROTO_TCP) { return 0; } - if ((4*ip_hdr->ihl + sizeof(struct tcphdr)) > len) { // buffer not large enough to contain expected tcp header return 0; @@ -130,27 +103,21 @@ int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr + 4*ip_hdr->ihl); uint16_t sport = tcp->source; uint16_t dport = tcp->dest; - // validate source port if (ntohs(sport) != zconf.target_port) { return 0; } - // validate destination port if (!check_dst_port(ntohs(dport), num_ports, validation)) { return 0; } - // validate tcp acknowledgement number if (htonl(tcp->ack_seq) != htonl(validation[0])+1) { return 0; } - return 1; } - - void synscan_process_packet(const u_char *packet, __attribute__((unused)) uint32_t len, fieldset_t *fs) { @@ -174,13 +141,18 @@ void synscan_process_packet(const u_char *packet, } static fielddef_t fields[] = { - {.name = "sport", .type = "int", .desc = "TCP source port"}, - {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, {.name = "window", .type = "int", .desc = "TCP window"}, }; +const char *help = + "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response."; probe_module_t module_tcp_synscan = { .name = "tcp_synscan", @@ -195,5 +167,6 @@ probe_module_t module_tcp_synscan = { .process_packet = &synscan_process_packet, .validate_packet = &synscan_validate_packet, .close = NULL, + .helptext = help, .fields = fields}; diff --git a/src/probe_modules/module_udp.c b/src/probe_modules/module_udp.c index be02adb..e048b64 100644 --- a/src/probe_modules/module_udp.c +++ b/src/probe_modules/module_udp.c @@ -109,14 +109,17 @@ int udp_global_initialize(struct state_conf *conf) { udp_send_msg[i] = (n & 0xff); } } else { - log_fatal("udp", "unknown UDP probe specification (expected file:/path, text:STRING, or hex:01020304)"); + log_fatal("udp", "unknown UDP probe specification " + "(expected file:/path, text:STRING, " + "or hex:01020304)"); free(udp_send_msg); free(args); exit(1); } if (udp_send_msg_len > MAX_UDP_PAYLOAD_LEN) { - log_warn("udp", "warning: reducing UDP payload to %d bytes (from %d) to fit on the wire\n", + log_warn("udp", "warning: reducing UDP payload to %d " + "bytes (from %d) to fit on the wire\n", MAX_UDP_PAYLOAD_LEN, udp_send_msg_len); udp_send_msg_len = MAX_UDP_PAYLOAD_LEN; } @@ -132,7 +135,6 @@ int udp_global_cleanup(__attribute__((unused)) struct state_conf *zconf, return(0); } - int udp_init_perthread(void* buf, macaddr_t *src, macaddr_t *gw, __attribute__((unused)) port_h_t dst_port) { @@ -158,9 +160,6 @@ int udp_init_perthread(void* buf, macaddr_t *src, return EXIT_SUCCESS; } - - - int udp_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint32_t *validation, int probe_num) { @@ -188,34 +187,8 @@ void udp_print_packet(FILE *fp, void* packet) ntohs(udph->source), ntohs(udph->dest), ntohl(udph->check)); - //ip_header = (struct iphdr*)(ð_header[1]) - struct in_addr *s = (struct in_addr *) &(iph->saddr); - struct in_addr *d = (struct in_addr *) &(iph->daddr); - char srcip[20]; - char dstip[20]; - // inet_ntoa is a const char * so we if just call it in - // fprintf, you'll get back wrong results since we're - // calling it twice. - strncpy(srcip, inet_ntoa(*s), 19); - strncpy(dstip, inet_ntoa(*d), 19); - fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n", - srcip, - dstip, - ntohl(iph->check)); - fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " - "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", - (int) ((unsigned char *) ethh->h_source)[0], - (int) ((unsigned char *) ethh->h_source)[1], - (int) ((unsigned char *) ethh->h_source)[2], - (int) ((unsigned char *) ethh->h_source)[3], - (int) ((unsigned char *) ethh->h_source)[4], - (int) ((unsigned char *) ethh->h_source)[5], - (int) ((unsigned char *) ethh->h_dest)[0], - (int) ((unsigned char *) ethh->h_dest)[1], - (int) ((unsigned char *) ethh->h_dest)[2], - (int) ((unsigned char *) ethh->h_dest)[3], - (int) ((unsigned char *) ethh->h_dest)[4], - (int) ((unsigned char *) ethh->h_dest)[5]); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); fprintf(fp, "------------------------------------------------------\n"); } diff --git a/src/probe_modules/packet.c b/src/probe_modules/packet.c index 7e2ee42..0ff752a 100644 --- a/src/probe_modules/packet.c +++ b/src/probe_modules/packet.c @@ -6,7 +6,6 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ - #include "packet.h" #include @@ -36,6 +35,41 @@ void print_macaddr(struct ifreq* i) (int) ((unsigned char *) &i->ifr_hwaddr.sa_data)[5]); } +void fprintf_ip_header(FILE *fp, struct iphdr *iph) +{ + struct in_addr *s = (struct in_addr *) &(iph->saddr); + struct in_addr *d = (struct in_addr *) &(iph->daddr); + char srcip[20]; + char dstip[20]; + // inet_ntoa is a const char * so we if just call it in + // fprintf, you'll get back wrong results since we're + // calling it twice. + strncpy(srcip, inet_ntoa(*s), 19); + strncpy(dstip, inet_ntoa(*d), 19); + fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n", + srcip, + dstip, + ntohl(iph->check)); +} + +void fprintf_eth_header(FILE *fp, struct ethhdr *ethh) +{ + fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " + "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", + (int) ((unsigned char *) ethh->h_source)[0], + (int) ((unsigned char *) ethh->h_source)[1], + (int) ((unsigned char *) ethh->h_source)[2], + (int) ((unsigned char *) ethh->h_source)[3], + (int) ((unsigned char *) ethh->h_source)[4], + (int) ((unsigned char *) ethh->h_source)[5], + (int) ((unsigned char *) ethh->h_dest)[0], + (int) ((unsigned char *) ethh->h_dest)[1], + (int) ((unsigned char *) ethh->h_dest)[2], + (int) ((unsigned char *) ethh->h_dest)[3], + (int) ((unsigned char *) ethh->h_dest)[4], + (int) ((unsigned char *) ethh->h_dest)[5]); +} + void make_eth_header(struct ethhdr *ethh, macaddr_t *src, macaddr_t *dst) { memcpy(ethh->h_source, src, ETH_ALEN); diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c index f87ebeb..27a3238 100644 --- a/src/probe_modules/probe_modules.c +++ b/src/probe_modules/probe_modules.c @@ -31,7 +31,6 @@ probe_module_t* probe_modules[] = { // ADD YOUR MODULE HERE }; - probe_module_t* get_probe_module_by_name(const char* name) { for (int i=0; i < (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); i++) { @@ -76,6 +75,8 @@ fielddef_t ip_fields[] = { fielddef_t sys_fields[] = { {.name="repeat", .type="int", .desc="Is response a repeat response from host"}, + {.name="cooldown", .type="int", .desc="Was response received during the cooldown period"}, + {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."} } @@ -87,9 +88,8 @@ void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) fs_add_uint64(fs, "ttl", ntohl(ip->ttl)); } -void fs_add_system_fields(fieldset_t *fs) +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) { - } diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h index 613c1e2..b077af7 100644 --- a/src/probe_modules/probe_modules.h +++ b/src/probe_modules/probe_modules.h @@ -37,12 +37,14 @@ typedef struct probe_module { probe_classify_packet_cb process_packet; probe_close_cb close; fielddef_t *fields; - + const char *helptext; } probe_module_t; probe_module_t* get_probe_module_by_name(const char*); +void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip); +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown); void print_probe_modules(void); #endif // HEADER_PROBE_MODULES_H diff --git a/src/recv.c b/src/recv.c index 1afa9ee..76370cd 100644 --- a/src/recv.c +++ b/src/recv.c @@ -27,11 +27,13 @@ #include #include #include +#include #include "../lib/logger.h" #include "state.h" #include "validate.h" +#include "fieldset.h" #include "probe_modules/probe_modules.h" #include "output_modules/output_modules.h" @@ -92,9 +94,16 @@ void packet_cb(u_char __attribute__((__unused__)) *user, } int is_repeat = check_ip(src_ip); - response_type_t *r = zconf.probe_module->classify_packet(bytes, buflen); - if (r->is_success) { + fieldset_t *fs = fs_new_fieldset(); + fs_add_ip_fields(fs, ip_hdr); + zconf.probe_module->process_packet(bytes, buflen, fs); + fs_add_system_fields(fs, is_repeat, zsend.complete); + int success_index = zconf.fsconf.success_index; + assert(success_index < fs->len); + int is_success = fs_get_uint64_byindex(success_index); + + if (is_success) { zrecv.success_total++; if (!is_repeat) { zrecv.success_unique++; @@ -106,19 +115,15 @@ void packet_cb(u_char __attribute__((__unused__)) *user, zrecv.cooldown_unique++; } } - if (zconf.output_module && zconf.output_module->success_ip) { - zconf.output_module->success_ip( - ip_hdr->saddr, ip_hdr->daddr, - r->name, is_repeat, zsend.complete, bytes, buflen); - } + } else { zrecv.failure_total++; - if (zconf.output_module && zconf.output_module->other_ip) { - zconf.output_module->other_ip( - ip_hdr->saddr, ip_hdr->daddr, - r->name, is_repeat, zsend.complete, bytes, buflen); - } } + //if (zconf.output_module && zconf.output_module->process_record) { + // //zconf.output_module->success_ip( + // // ip_hdr->saddr, ip_hdr->daddr, + // // r->name, is_repeat, zsend.complete, bytes, buflen); + //} if (zconf.output_module && zconf.output_module->update && !(zrecv.success_unique % zconf.output_module->update_interval)) { diff --git a/src/state.h b/src/state.h index 7730b99..efba852 100644 --- a/src/state.h +++ b/src/state.h @@ -13,20 +13,26 @@ #include #include +#include "types.h" +#include "fieldset.h" + #ifndef _STATE_H #define _STATE_H #define MAX_PACKET_SIZE 4096 -typedef uint32_t ipaddr_n_t; // IPv4 address network order -typedef uint32_t ipaddr_h_t; // IPv4 address host order -typedef uint16_t port_n_t; // port network order -typedef uint16_t port_h_t; // port host order -typedef unsigned char macaddr_t; - struct probe_module; struct output_module; +struct fieldset_conf { + fielddefset_t defs; + int *translation; + int translation_len; + int success_index; + int classification_index; +}; + + // global configuration struct state_conf { int log_level; @@ -73,6 +79,7 @@ struct state_conf { char *whitelist_filename; char *raw_output_fields; char **output_fields; + struct fieldset_conf fs_conf; int output_fields_len; int dryrun; int summary; @@ -81,6 +88,7 @@ struct state_conf { }; extern struct state_conf zconf; + // global sender stats struct state_send { double start; diff --git a/src/zmap.c b/src/zmap.c index 79a0f7c..3766d33 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -367,6 +367,59 @@ int main(int argc, char *argv[]) if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) { exit(EXIT_FAILURE); } + // parse the provided probe and output module s.t. that we can support + // other command-line helpers (e.g. probe help) + zconf.output_module = get_output_module_by_name(args.output_module_arg); + if (!zconf.output_module) { + fprintf(stderr, "%s: specified output module (%s) does not exist\n", + CMDLINE_PARSER_PACKAGE, args.output_module_arg); + exit(EXIT_FAILURE); + } + zconf.probe_module = get_probe_module_by_name(args.probe_module_arg); + if (!zconf.probe_module) { + fprintf(stderr, "%s: specified probe module (%s) does not exist\n", + CMDLINE_PARSER_PACKAGE, args.probe_module_arg); + exit(EXIT_FAILURE); + } + + // now that we know the probe module, let's find what it supports + memset(zconf.fs_conf, 0, sizeof(struct fieldset_conf)); + // the set of fields made available to a user is constructed + // of IP header fields + probe module fields + system fields + fielddefset_t *fds = &(zconf.fs_conf.defs); + gen_fieldset_def(fds, &(ip_fields), + sizeof(ip_fields)/sizeof(fielddef_t)); + gen_fieldset_def(fds, &(zconf.probe_module->fields), + sizeof(zconf.probe_module->fields)/sizeof(fielddef_t)); + gen_fieldset_def(fds, &(sys_fields), + sizeof(sys_fields)/sizeof(fielddef_t)); + // find the fields we need for the framework + zconf.fs_conf.success_index = + fs_get_index_by_name(fds, "success"); + if (zconf.fs_conf.success_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required success field."); + } + zconf.fs_conf.classification_index = + fs_get_index_by_name(fds, "classification"); + if (zconf.fs_conf.classification_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required packet classification field."); + } + // process the list of requested output fields. + if (args.output_fields_given) { + zconf.raw_output_fields = args.output_fields_arg; + } else { + zconf.raw_output_fields = (char*) "saddr"; + } + fs_split_string(args.output_fields_arg, &(zconf.output_fields_len), + &(zconf.output_fields)); + for (int i=0; i < zconf.output_fields_len; i++) { + log_trace("zmap", "requested output field (%i): %s", + zconf.output_fields[i]); + } + // generate a translation that can be used to convert output + // from a probe module to the input for an output module SET_IF_GIVEN(zconf.output_filename, output_file); SET_IF_GIVEN(zconf.blacklist_filename, blacklist_file); @@ -385,36 +438,7 @@ int main(int argc, char *argv[]) zconf.senders = args.sender_threads_arg; zconf.log_level = args.verbosity_arg; - zconf.output_module = get_output_module_by_name(args.output_module_arg); - if (!zconf.output_module) { - fprintf(stderr, "%s: specified output module (%s) does not exist\n", - CMDLINE_PARSER_PACKAGE, args.output_module_arg); - exit(EXIT_FAILURE); - } - zconf.probe_module = get_probe_module_by_name(args.probe_module_arg); - if (!zconf.probe_module) { - fprintf(stderr, "%s: specified probe module (%s) does not exist\n", - CMDLINE_PARSER_PACKAGE, args.probe_module_arg); - exit(EXIT_FAILURE); - } - // we need to generate a master fielddef list - - // process the list of output fields. - if (args.output_fields_given) { - zconf.raw_output_fields = zopt.output_fields; - fs_split_string(zopt.output_fields, &(zconf.output_fields), - &(zconf.output_fields_len)); - } else { - zconf.output_fields = {"saddr"}; - zconf.num_output_fields = 1; - } - // check the list of requested fields to see if they are available - // with the selected probe module. - for (int i=0; i < zonf.num_output_fields; i++) { - fs_check - - } - + if (zconf.probe_module->port_args) { if (args.source_port_given) { From 211307e3085dcb152d0b17bbcca43123ea6c508b Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 26 Aug 2013 07:59:54 -0400 Subject: [PATCH 10/18] adding types --- src/types.h | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/types.h diff --git a/src/types.h b/src/types.h new file mode 100644 index 0000000..f039b7b --- /dev/null +++ b/src/types.h @@ -0,0 +1,11 @@ +#include + +#ifndef TYPES_H_ +#define TYPES_H_ +typedef uint32_t ipaddr_n_t; // IPv4 address network order +typedef uint32_t ipaddr_h_t; // IPv4 address host order +typedef uint16_t port_n_t; // port network order +typedef uint16_t port_h_t; // port host order +typedef unsigned char macaddr_t; + +#endif From 0d3ad52ee04162fb7e68b02ed1ac81c790d1133b Mon Sep 17 00:00:00 2001 From: zakird Date: Mon, 26 Aug 2013 12:26:01 -0400 Subject: [PATCH 11/18] fixing header guards close issue #48 --- src/aesrand.h | 4 ++-- src/cyclic.h | 4 ++-- src/get_gateway.h | 4 ++-- src/monitor.h | 4 ++-- src/output_modules/output_modules.h | 4 ++-- src/probe_modules/packet.h | 4 ++-- src/probe_modules/probe_modules.h | 4 ++-- src/recv.h | 4 ++-- src/send.h | 4 ++-- src/state.h | 4 ++-- src/validate.h | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/aesrand.h b/src/aesrand.h index 2d3dedb..352c14a 100644 --- a/src/aesrand.h +++ b/src/aesrand.h @@ -8,8 +8,8 @@ #include -#ifndef _AESRAND_H -#define _AESRAND_H +#ifndef AESRAND_H +#define AESRAND_H void aesrand_init(uint32_t seed); diff --git a/src/cyclic.h b/src/cyclic.h index f7bb7a2..f7f7387 100644 --- a/src/cyclic.h +++ b/src/cyclic.h @@ -8,8 +8,8 @@ #include -#ifndef _CYCLIC_H -#define _CYCLIC_H +#ifndef CYCLIC_H +#define CYCLIC_H int cyclic_init(uint32_t, uint32_t); diff --git a/src/get_gateway.h b/src/get_gateway.h index 398e3e1..ba5098b 100644 --- a/src/get_gateway.h +++ b/src/get_gateway.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef _GET_GATEWAY_H -#define _GET_GATEWAY_H +#ifndef GET_GATEWAY_H +#define GET_GATEWAY_H #include diff --git a/src/monitor.h b/src/monitor.h index c8bd9fb..9dcc673 100644 --- a/src/monitor.h +++ b/src/monitor.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef _MONITOR_H -#define _MONITOR_H +#ifndef MONITOR_H +#define MONITOR_H void monitor_run(); diff --git a/src/output_modules/output_modules.h b/src/output_modules/output_modules.h index bfda3f4..9e51927 100644 --- a/src/output_modules/output_modules.h +++ b/src/output_modules/output_modules.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef HEADER_OUTPUT_MODULES_H -#define HEADER_OUTPUT_MODULES_H +#ifndef OUTPUT_MODULES_H +#define OUTPUT_MODULES_H #include "../state.h" diff --git a/src/probe_modules/packet.h b/src/probe_modules/packet.h index 20f50c2..cca30b6 100644 --- a/src/probe_modules/packet.h +++ b/src/probe_modules/packet.h @@ -6,8 +6,8 @@ #include #include -#ifndef HEADER_ZMAP_PACKET_H -#define HEADER_ZMAP_PACKET_H +#ifndef PACKET_H +#define PACKET_H #define MAX_PACKET_SIZE 4096 diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h index 18f8438..ff23611 100644 --- a/src/probe_modules/probe_modules.h +++ b/src/probe_modules/probe_modules.h @@ -1,7 +1,7 @@ #include "../state.h" -#ifndef HEADER_PROBE_MODULES_H -#define HEADER_PROBE_MODULES_H +#ifndef PROBE_MODULES_H +#define PROBE_MODULES_H typedef struct probe_response_type { const uint8_t is_success; diff --git a/src/recv.h b/src/recv.h index 8befea5..3c6f055 100644 --- a/src/recv.h +++ b/src/recv.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef _RECV_H -#define _RECV_H +#ifndef RECV_H +#define RECV_H int recv_update_pcap_stats(void); int recv_run(pthread_mutex_t *recv_ready_mutex); diff --git a/src/send.h b/src/send.h index 5116e45..1b96120 100644 --- a/src/send.h +++ b/src/send.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef _SEND_H -#define _SEND_H +#ifndef SEND_H +#define SEND_H int send_init(void); int send_run(void); diff --git a/src/state.h b/src/state.h index f1c967e..8c14288 100644 --- a/src/state.h +++ b/src/state.h @@ -13,8 +13,8 @@ #include #include -#ifndef _STATE_H -#define _STATE_H +#ifndef STATE_H +#define STATE_H #define MAX_PACKET_SIZE 4096 diff --git a/src/validate.h b/src/validate.h index e1b0230..4eb6362 100644 --- a/src/validate.h +++ b/src/validate.h @@ -6,8 +6,8 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -#ifndef _VALIDATE_H -#define _VALIDATE_H +#ifndef VALIDATE_H +#define VALIDATE_H #define VALIDATE_BYTES 16 From 0d125d18e383c6bf983fd39678c661cf11fdc618 Mon Sep 17 00:00:00 2001 From: zakird Date: Mon, 26 Aug 2013 14:30:02 -0400 Subject: [PATCH 12/18] fixing .h protectors in lib/* --- lib/blacklist.h | 4 ++-- lib/constraint.h | 4 ++-- lib/logger.h | 4 ++-- lib/random.h | 4 ++-- lib/redis.h | 4 ++-- lib/rijndael-alg-fst.h | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/blacklist.h b/lib/blacklist.h index 5604171..80a44fc 100644 --- a/lib/blacklist.h +++ b/lib/blacklist.h @@ -1,7 +1,7 @@ #include -#ifndef _BLACKLIST_H -#define _BLACKLIST_H +#ifndef BLACKLIST_H +#define BLACKLIST_H int blacklist_is_allowed(uint32_t s_addr); void blacklist_prefix(char *ip, int prefix_len); diff --git a/lib/constraint.h b/lib/constraint.h index 3e914a3..2170f8f 100644 --- a/lib/constraint.h +++ b/lib/constraint.h @@ -1,5 +1,5 @@ -#ifndef _CONSTRAINT_H -#define _CONSTRAINT_H +#ifndef CONSTRAINT_H +#define CONSTRAINT_H typedef struct _constraint constraint_t; typedef int value_t; diff --git a/lib/logger.h b/lib/logger.h index 8cf2a9f..a961ac3 100644 --- a/lib/logger.h +++ b/lib/logger.h @@ -1,8 +1,8 @@ #include #include -#ifndef _LOGGER_H -#define _LOGGER_H +#ifndef LOGGER_H +#define LOGGER_H enum LogLevel { LOG_FATAL, LOG_ERROR, LOG_WARN, LOG_INFO, LOG_DEBUG, LOG_TRACE, NUM_LOGLEVELS }; diff --git a/lib/random.h b/lib/random.h index d83b14d..b05a5d6 100644 --- a/lib/random.h +++ b/lib/random.h @@ -1,8 +1,8 @@ #include #include -#ifndef _RANDOM_H -#define _RANDOM_H +#ifndef RANDOM_H +#define RANDOM_H int random_bytes(void *dst, size_t n); diff --git a/lib/redis.h b/lib/redis.h index 51a1a7f..229136c 100644 --- a/lib/redis.h +++ b/lib/redis.h @@ -2,8 +2,8 @@ #include #include -#ifndef _REDIS_ZHELPERS_H -#define _REDIS_ZHELPERS_H +#ifndef REDIS_ZHELPERS_H +#define REDIS_ZHELPERS_H int redis_init(void); diff --git a/lib/rijndael-alg-fst.h b/lib/rijndael-alg-fst.h index 2c3396e..4a8bd5a 100644 --- a/lib/rijndael-alg-fst.h +++ b/lib/rijndael-alg-fst.h @@ -23,8 +23,8 @@ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __RIJNDAEL_ALG_FST_H -#define __RIJNDAEL_ALG_FST_H +#ifndef RIJNDAEL_ALG_FST_H +#define RIJNDAEL_ALG_FST_H #define MAXKC (256/32) #define MAXKB (256/8) @@ -44,4 +44,4 @@ void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], in void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], int rounds); #endif /* INTERMEDIATE_VALUE_KAT */ -#endif /* __RIJNDAEL_ALG_FST_H */ +#endif /* RIJNDAEL_ALG_FST_H */ From 67aa6f0ab05fc26d34657635740d137c2b0aa153 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Tue, 27 Aug 2013 12:03:24 -0400 Subject: [PATCH 13/18] adding generic csv output module --- lib/logger.c | 1 + src/Makefile | 5 +- src/fieldset.c | 7 +- src/fieldset.h | 13 ++-- src/output_modules/module_csv.c | 94 +++++++++++++++++++++++++ src/output_modules/module_simple_file.c | 1 - src/output_modules/output_modules.c | 14 ++-- src/output_modules/output_modules.h | 18 ++--- src/probe_modules/module_icmp_echo.c | 3 +- src/probe_modules/module_tcp_synscan.c | 17 ++--- src/probe_modules/packet.h | 2 + src/probe_modules/probe_modules.c | 58 ++++++++------- src/probe_modules/probe_modules.h | 30 +++++--- src/recv.c | 13 ++-- src/state.h | 4 +- src/zmap.c | 86 +++++++++++++--------- 16 files changed, 252 insertions(+), 114 deletions(-) create mode 100644 src/output_modules/module_csv.c diff --git a/lib/logger.c b/lib/logger.c index 8a797e7..c1ff213 100644 --- a/lib/logger.c +++ b/lib/logger.c @@ -53,6 +53,7 @@ static int LogLogVA(enum LogLevel level, const char *loggerName, if (loggerName || logMessage) { fputs("\n", log_output_stream); } + fflush(log_output_stream); } return 0; } diff --git a/src/Makefile b/src/Makefile index 6301977..1ae5e35 100644 --- a/src/Makefile +++ b/src/Makefile @@ -31,9 +31,10 @@ EXTRALDFLAGS= $(LDHARDENING) CFLAGS+=-Wall $(INCLUDE) $(EXTRACFLAGS) LDFLAGS+=$(EXTRALDFLAGS) -modules=module_tcp_synscan.o module_icmp_echo.o module_udp.o #ADD YOUR MODULE HERE +probemodules=module_tcp_synscan.o module_icmp_echo.o #module_udp.o #ADD YOUR PROBE MODULE HERE +outputmodules= module_csv.o #module_simple_file.o module_extended_file.o module_csv.o #ADD YOUR OUTPUT MODULE HERE -objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt.o zmap.o random.o output_modules.o module_simple_file.o module_extended_file.o packet.o probe_modules.o ${modules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o fieldset.o +objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt.o zmap.o random.o output_modules.o packet.o probe_modules.o ${probemodules} ${outputmodules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o fieldset.o ifeq ($(REDIS), true) LDLIBS+=-lhiredis diff --git a/src/fieldset.c b/src/fieldset.c index 25752fa..b560074 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -7,7 +7,7 @@ #include "../lib/logger.h" -void gen_fielddef_set(fielddefset_t *fds, fieldset_t fs[], int len) +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len) { if (fds->len + len > MAX_FIELDS) { log_fatal("fieldset", "out of room in field def set"); @@ -60,12 +60,12 @@ void fs_add_binary(fieldset_t *fs, const char *name, size_t len, uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index) { - return (uint64_t) fs->fields[i].value; + return (uint64_t) fs->fields[index].value; } char* fs_get_string_by_index(fieldset_t *fs, int index) { - return (char*) fs->fields[i].value; + return (char*) fs->fields[index].value; } int fds_get_index_by_name(fielddefset_t *fds, char *name) @@ -79,6 +79,7 @@ int fds_get_index_by_name(fielddefset_t *fds, char *name) } void fs_free(fieldset_t *fs) +{ for (int i=0; i < fs->len; i++) { field_t *f = &(fs->fields[i]); if (f->free_) { diff --git a/src/fieldset.h b/src/fieldset.h index d733dee..5ed2136 100644 --- a/src/fieldset.h +++ b/src/fieldset.h @@ -10,8 +10,8 @@ #include #include "types.h" -#ifndef _FIELDSET_H -#define _FIELDSET_H +#ifndef FIELDSET_H +#define FIELDSET_H // maximum number of records that can be stored in a fieldset #define MAX_FIELDS 128 @@ -67,13 +67,11 @@ typedef struct translation { fieldset_t *fs_new_fieldset(void); -uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); - char* fs_get_string_by_index(fieldset_t *fs, int index); int fds_get_index_by_name(fielddefset_t *fds, char *name); -void gen_fielddef_set(fielddefset_t *fds, fieldset_t fs[], int len); +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len); void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); @@ -82,10 +80,13 @@ void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, int free_); +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); + void fs_free(fieldset_t *fs); translation_t *fs_generate_fieldset_translation(); fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); -#endif +#endif // FIELDSET_H + diff --git a/src/output_modules/module_csv.c b/src/output_modules/module_csv.c new file mode 100644 index 0000000..4c5ccfc --- /dev/null +++ b/src/output_modules/module_csv.c @@ -0,0 +1,94 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "../../lib/logger.h" +#include "../fieldset.h" + +#include "output_modules.h" + +static FILE *file = NULL; + +int csv_init(struct state_conf *conf, fielddefset_t *fds) +{ + assert(conf); + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal("csv", "could not open output file (%s)", + conf->output_filename); + } + } + } + // add output headers + for (int i=0; i < fds->len; i++) { + if (i) { + fprintf(file, ", "); + } + fprintf(file, "%s", fds->fielddefs[i].name); + } + return EXIT_SUCCESS; +} + +int csv_close(__attribute__((unused)) struct state_conf* c, + __attribute__((unused)) struct state_send* s, + __attribute__((unused)) struct state_recv* r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +static void hex_encode(FILE *f, unsigned char* readbuf, size_t len) +{ + for(size_t i=0; i < len; i++) { + fprintf(f, "%02x", readbuf[i]); + } +} + +int csv_process(fieldset_t *fs) +{ + for (int i=0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (i) { + fprintf(file, ", "); + } + if (f->type == FS_STRING) { + fprintf(file, "%s", (char*) f->value); + } else if (f->type == FS_UINT64) { + fprintf(file, "%lu", (uint64_t) f->value); + } else if (f->type == FS_BINARY) { + hex_encode(file, (unsigned char*) f->value, f->len); + } + } + fprintf(file, "\n"); + return EXIT_SUCCESS; +} + +output_module_t module_csv_file = { + .name = "csv", + .init = &csv_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &csv_close, + .process_ip = &csv_process, +}; + diff --git a/src/output_modules/module_simple_file.c b/src/output_modules/module_simple_file.c index 2061972..beb62bf 100644 --- a/src/output_modules/module_simple_file.c +++ b/src/output_modules/module_simple_file.c @@ -6,7 +6,6 @@ * of the License at http://www.apache.org/licenses/LICENSE-2.0 */ - #include #include #include diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c index 49032a8..a9c4382 100644 --- a/src/output_modules/output_modules.c +++ b/src/output_modules/output_modules.c @@ -12,20 +12,21 @@ #include "output_modules.h" -extern output_module_t module_simple_file; -extern output_module_t module_extended_file; +//extern output_module_t module_simple_file; +//extern output_module_t module_extended_file; +extern output_module_t module_csv_file; // ADD YOUR MODULE HERE #ifdef REDIS extern output_module_t module_redis; #endif - output_module_t* output_modules[] = { - &module_simple_file, - &module_extended_file, + &module_csv_file + //&module_simple_file, + //&module_extended_file, #ifdef REDIS - &module_redis, + //&module_redis, #endif // ADD YOUR MODULE HERE }; @@ -50,3 +51,4 @@ void print_output_modules(void) printf("%s\n", output_modules[i]->name); } } + diff --git a/src/output_modules/output_modules.h b/src/output_modules/output_modules.h index bfda3f4..64acd5d 100644 --- a/src/output_modules/output_modules.h +++ b/src/output_modules/output_modules.h @@ -10,35 +10,31 @@ #define HEADER_OUTPUT_MODULES_H #include "../state.h" +#include "../fieldset.h" // called at scanner initialization -typedef int (*output_init_cb)(struct state_conf *); +typedef int (*output_init_cb)(struct state_conf *, fielddefset_t *fds); // called on packet receipt -typedef int (*output_packet_cb)(ipaddr_n_t src_ip, ipaddr_n_t dst_ip, - const char* response_type, - int is_repeat, int in_cooldown, - const u_char* packetbuf, size_t buflen); +typedef int (*output_packet_cb)(fieldset_t *fs); // called periodically during the scan -typedef int (*output_update_cb)(struct state_conf*, struct state_send*, struct state_recv*); - +typedef int (*output_update_cb)(struct state_conf*, + struct state_send*, struct state_recv*); typedef struct output_module { const char *name; unsigned update_interval; - output_init_cb init; output_update_cb start; output_update_cb update; output_update_cb close; - output_packet_cb success_ip; - output_packet_cb other_ip; - + output_packet_cb process_ip; } output_module_t; output_module_t* get_output_module_by_name(const char*); + void print_output_modules(void); #endif // HEADER_OUTPUT_MODULES_H diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c index cb6e9a4..e3d5607 100644 --- a/src/probe_modules/module_icmp_echo.c +++ b/src/probe_modules/module_icmp_echo.c @@ -185,5 +185,6 @@ probe_module_t module_icmp_echo = { .process_packet = &icmp_echo_process_packet, .validate_packet = &icmp_validate_packet, .close = NULL, - .fields = fields}; + .fields = fields, + .numfields = 6}; diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c index a8f2e23..c578c4a 100644 --- a/src/probe_modules/module_tcp_synscan.c +++ b/src/probe_modules/module_tcp_synscan.c @@ -146,14 +146,10 @@ static fielddef_t fields[] = { {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, {.name = "window", .type = "int", .desc = "TCP window"}, + {.name = "classification", .type="string", .desc = "packet classification"}, + {.name = "success", .type="int", .desc = "is response considered success"} }; -const char *help = - "Probe module that sends a TCP SYN packet to a specific " - "port. Possible classifications are: synack and rst. A " - "SYN-ACK packet is considered a success and a reset packet " - "is considered a failed response."; - probe_module_t module_tcp_synscan = { .name = "tcp_synscan", .packet_length = 54, @@ -167,6 +163,11 @@ probe_module_t module_tcp_synscan = { .process_packet = &synscan_process_packet, .validate_packet = &synscan_validate_packet, .close = NULL, - .helptext = help, - .fields = fields}; + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + + .fields = fields, + .numfields = 7}; diff --git a/src/probe_modules/packet.h b/src/probe_modules/packet.h index 2acb3b0..91fa0b6 100644 --- a/src/probe_modules/packet.h +++ b/src/probe_modules/packet.h @@ -20,6 +20,8 @@ void make_tcp_header(struct tcphdr*, port_h_t); void make_icmp_header(struct icmp *); void make_udp_header(struct udphdr *udp_header, port_h_t dest_port, uint16_t len); +void fprintf_ip_header(FILE *fp, struct iphdr *iph); +void fprintf_eth_header(FILE *fp, struct ethhdr *ethh); static inline unsigned short in_checksum(unsigned short *ip_pkt, int len) { diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c index 27a3238..d9aa94f 100644 --- a/src/probe_modules/probe_modules.c +++ b/src/probe_modules/probe_modules.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,24 +17,26 @@ #include #include +#include "../../lib/logger.h" #include "../fieldset.h" #include "probe_modules.h" extern probe_module_t module_tcp_synscan; extern probe_module_t module_icmp_echo; -extern probe_module_t module_udp; +//extern probe_module_t module_udp; // ADD YOUR MODULE HERE probe_module_t* probe_modules[] = { &module_tcp_synscan, &module_icmp_echo, - &module_udp +// &module_udp // ADD YOUR MODULE HERE }; probe_module_t* get_probe_module_by_name(const char* name) { - for (int i=0; i < (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); i++) { + int len = (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); + for (int i=0; i < len; i++) { if (!strcmp(probe_modules[i]->name, name)) { return probe_modules[i]; } @@ -43,18 +46,12 @@ probe_module_t* get_probe_module_by_name(const char* name) void print_probe_modules(void) { - for (int i=0; i < (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); i++) { + int len = (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); + for (int i=0; i < len; i++) { printf("%s\n", probe_modules[i]->name); } } -void print_probe_module_fields(probe_module_t *p) -{ - for (int i=0; i < (int) (sizeof(p->fields)/sizeof(p->fields[0])); i++) { - - } -} - char *make_ip_str(uint32_t ip) { struct in_addr t; @@ -66,20 +63,6 @@ char *make_ip_str(uint32_t ip) return retv; } -fielddef_t ip_fields[] = { - {.name="saddr", .type="string", .desc="source IP address of response"}, - {.name="daddr", .type="string", .desc="destination IP address of response"}, - {.name="ipid", .type="int", .desc="IP identification number of response"}, - {.name="ttl", .type="int", .desc="time-to-live of response packet"} -} - -fielddef_t sys_fields[] = { - {.name="repeat", .type="int", .desc="Is response a repeat response from host"}, - {.name="cooldown", .type="int", .desc="Was response received during the cooldown period"}, - - {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."} -} - void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) { fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1); @@ -88,8 +71,33 @@ void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip) fs_add_uint64(fs, "ttl", ntohl(ip->ttl)); } +#define TIMESTR_LEN 50 + void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) { + fs_add_uint64(fs, "repeat", is_repeat); + fs_add_uint64(fs, "cooldown", in_cooldown); + char *timestr = malloc(TIMESTR_LEN+1); + if (!timestr) { + log_fatal("recv", "unable to allocate memory for " + "timestamp string in fieldset."); + } + time_t now = time(0); + strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S%z", + localtime(&now)); + fs_add_string(fs, "timestamp-str", timestr, 1); } +fielddef_t ip_fields[] = { + {.name="saddr", .type="string", .desc="source IP address of response"}, + {.name="daddr", .type="string", .desc="destination IP address of response"}, + {.name="ipid", .type="int", .desc="IP identification number of response"}, + {.name="ttl", .type="int", .desc="time-to-live of response packet"} +}; + +fielddef_t sys_fields[] = { + {.name="repeat", .type="int", .desc="Is response a repeat response from host"}, + {.name="cooldown", .type="int", .desc="Was response received during the cooldown period"}, + {.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."} +}; diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h index b077af7..4f8ea5a 100644 --- a/src/probe_modules/probe_modules.h +++ b/src/probe_modules/probe_modules.h @@ -1,8 +1,8 @@ #include "../state.h" #include "../fieldset.h" -#ifndef HEADER_PROBE_MODULES_H -#define HEADER_PROBE_MODULES_H +#ifndef PROBE_MODULES_H +#define PROBE_MODULES_H typedef struct probe_response_type { const uint8_t is_success; @@ -10,14 +10,21 @@ typedef struct probe_response_type { } response_type_t; typedef int (*probe_global_init_cb)(struct state_conf *); -typedef int (*probe_thread_init_cb)(void* packetbuf, macaddr_t* src_mac, macaddr_t* gw_mac, port_n_t src_port); -typedef int (*probe_make_packet_cb)(void* packetbuf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, - uint32_t *validation, int probe_num); -typedef void (*probe_print_packet_cb)(FILE *, void* packetbuf); -typedef int (*probe_close_cb)(struct state_conf*, struct state_send*, struct state_recv*); -typedef int (*probe_validate_packet_cb)(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src_ip, uint32_t *validation); +typedef int (*probe_thread_init_cb)(void* packetbuf, macaddr_t* src_mac, + macaddr_t* gw_mac, port_n_t src_port); -typedef void (*probe_classify_packet_cb)(const u_char* packetbuf, uint32_t len, fieldset_t*); +typedef int (*probe_make_packet_cb)(void* packetbuf, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, + uint32_t *validation, int probe_num); + +typedef void (*probe_print_packet_cb)(FILE *, void* packetbuf); +typedef int (*probe_close_cb)(struct state_conf*, + struct state_send*, struct state_recv*); +typedef int (*probe_validate_packet_cb)(const struct iphdr *ip_hdr, + uint32_t len, uint32_t *src_ip, uint32_t *validation); + +typedef void (*probe_classify_packet_cb)(const u_char* packetbuf, + uint32_t len, fieldset_t*); typedef struct probe_module { const char *name; @@ -37,6 +44,7 @@ typedef struct probe_module { probe_classify_packet_cb process_packet; probe_close_cb close; fielddef_t *fields; + int numfields; const char *helptext; } probe_module_t; @@ -47,4 +55,8 @@ void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip); void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown); void print_probe_modules(void); +extern fielddef_t ip_fields[]; +extern fielddef_t sys_fields[]; + #endif // HEADER_PROBE_MODULES_H + diff --git a/src/recv.c b/src/recv.c index 76370cd..820d938 100644 --- a/src/recv.c +++ b/src/recv.c @@ -101,7 +101,7 @@ void packet_cb(u_char __attribute__((__unused__)) *user, fs_add_system_fields(fs, is_repeat, zsend.complete); int success_index = zconf.fsconf.success_index; assert(success_index < fs->len); - int is_success = fs_get_uint64_byindex(success_index); + int is_success = fs_get_uint64_by_index(fs, success_index); if (is_success) { zrecv.success_total++; @@ -119,12 +119,13 @@ void packet_cb(u_char __attribute__((__unused__)) *user, } else { zrecv.failure_total++; } - //if (zconf.output_module && zconf.output_module->process_record) { - // //zconf.output_module->success_ip( - // // ip_hdr->saddr, ip_hdr->daddr, - // // r->name, is_repeat, zsend.complete, bytes, buflen); - //} + // we need to translate the data provided by the probe module + // into a fieldset that can be used by the output module + + if (zconf.output_module && zconf.output_module->process_ip) { + zconf.output_module->process_ip(fs); + } if (zconf.output_module && zconf.output_module->update && !(zrecv.success_unique % zconf.output_module->update_interval)) { zconf.output_module->update(&zconf, &zsend, &zrecv); diff --git a/src/state.h b/src/state.h index efba852..84aee6f 100644 --- a/src/state.h +++ b/src/state.h @@ -26,13 +26,13 @@ struct output_module; struct fieldset_conf { fielddefset_t defs; + fielddefset_t outdefs; int *translation; int translation_len; int success_index; int classification_index; }; - // global configuration struct state_conf { int log_level; @@ -79,7 +79,7 @@ struct state_conf { char *whitelist_filename; char *raw_output_fields; char **output_fields; - struct fieldset_conf fs_conf; + struct fieldset_conf fsconf; int output_fields_len; int dryrun; int summary; diff --git a/src/zmap.c b/src/zmap.c index 3766d33..59684dc 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -10,18 +10,19 @@ #include #include - +#include +#include #include #include -#include #include -#include + #include #include #include #include #include + #include #include "../lib/logger.h" @@ -32,9 +33,10 @@ #include "recv.h" #include "state.h" #include "monitor.h" +#include "get_gateway.h" + #include "output_modules/output_modules.h" #include "probe_modules/probe_modules.h" -#include "get_gateway.h" pthread_mutex_t cpu_affinity_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -55,15 +57,17 @@ static void fs_split_string(char* in, int *len, char***results) currloc++; } else { char *new = malloc(len+1); + assert(new); strncpy(new, currloc, len); new[len] = '\0'; fields[retvlen++] = new; + assert(fields[retvlen-1]); } + if (len == strlen(currloc)) { + break; + } currloc += len; - if (currloc == in + (size_t) strlen(in)) { - break; - } - } + } *results = fields; *len = retvlen; } @@ -160,7 +164,6 @@ static void summary(void) static void start_zmap(void) { - log_init(stderr, zconf.log_level); log_info("zmap", "started"); // finish setting up configuration @@ -208,7 +211,7 @@ static void start_zmap(void) // initialization if (zconf.output_module && zconf.output_module->init) { - zconf.output_module->init(&zconf); + zconf.output_module->init(&zconf, &zconf.fsconf.outdefs); } if (send_init()) { exit(EXIT_FAILURE); @@ -216,7 +219,6 @@ static void start_zmap(void) if (zconf.output_module && zconf.output_module->start) { zconf.output_module->start(&zconf, &zsend, &zrecv); } - // start threads pthread_t *tsend, trecv, tmon; int r = pthread_create(&trecv, NULL, start_recv, NULL); @@ -331,12 +333,25 @@ int parse_mac(macaddr_t *out, char *in) int main(int argc, char *argv[]) { + struct gengetopt_args_info args; struct cmdline_parser_params *params; params = cmdline_parser_params_create(); params->initialize = 1; params->override = 0; params->check_required = 0; + + SET_BOOL(zconf.dryrun, dryrun); + SET_BOOL(zconf.quiet, quiet); + SET_BOOL(zconf.summary, summary); + zconf.cooldown_secs = args.cooldown_time_arg; + zconf.senders = args.sender_threads_arg; + zconf.log_level = args.verbosity_arg; + + log_init(stderr, zconf.log_level); + log_trace("zmap", "zmap main thread started"); + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { exit(EXIT_SUCCESS); } @@ -383,26 +398,35 @@ int main(int argc, char *argv[]) } // now that we know the probe module, let's find what it supports - memset(zconf.fs_conf, 0, sizeof(struct fieldset_conf)); + memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf)); // the set of fields made available to a user is constructed // of IP header fields + probe module fields + system fields - fielddefset_t *fds = &(zconf.fs_conf.defs); - gen_fieldset_def(fds, &(ip_fields), - sizeof(ip_fields)/sizeof(fielddef_t)); - gen_fieldset_def(fds, &(zconf.probe_module->fields), - sizeof(zconf.probe_module->fields)/sizeof(fielddef_t)); - gen_fieldset_def(fds, &(sys_fields), - sizeof(sys_fields)/sizeof(fielddef_t)); + fielddefset_t *fds = &(zconf.fsconf.defs); + gen_fielddef_set(fds, (fielddef_t*) &(ip_fields), + 4); + gen_fielddef_set(fds, zconf.probe_module->fields, + zconf.probe_module->numfields); + gen_fielddef_set(fds, (fielddef_t*) &(sys_fields), + 3); + if (args.list_output_fields_given) { + for (int i = 0; i < fds->len; i++) { + printf("%s (%s): %s\n", fds->fielddefs[i].name, + fds->fielddefs[i].type, + fds->fielddefs[i].desc); + } + exit(EXIT_SUCCESS); + } + // find the fields we need for the framework - zconf.fs_conf.success_index = - fs_get_index_by_name(fds, "success"); - if (zconf.fs_conf.success_index < 0) { + zconf.fsconf.success_index = + fds_get_index_by_name(fds, (char*) "success"); + if (zconf.fsconf.success_index < 0) { log_fatal("fieldset", "probe module does not supply " "required success field."); } - zconf.fs_conf.classification_index = - fs_get_index_by_name(fds, "classification"); - if (zconf.fs_conf.classification_index < 0) { + zconf.fsconf.classification_index = + fds_get_index_by_name(fds, (char*) "classification"); + if (zconf.fsconf.classification_index < 0) { log_fatal("fieldset", "probe module does not supply " "required packet classification field."); } @@ -412,10 +436,11 @@ int main(int argc, char *argv[]) } else { zconf.raw_output_fields = (char*) "saddr"; } - fs_split_string(args.output_fields_arg, &(zconf.output_fields_len), + fs_split_string(zconf.raw_output_fields, &(zconf.output_fields_len), &(zconf.output_fields)); for (int i=0; i < zconf.output_fields_len; i++) { - log_trace("zmap", "requested output field (%i): %s", + log_debug("zmap", "requested output field (%i): %s", + i, zconf.output_fields[i]); } // generate a translation that can be used to convert output @@ -431,13 +456,6 @@ int main(int argc, char *argv[]) SET_IF_GIVEN(zconf.max_results, max_results); SET_IF_GIVEN(zconf.rate, rate); SET_IF_GIVEN(zconf.packet_streams, probes); - SET_BOOL(zconf.dryrun, dryrun); - SET_BOOL(zconf.quiet, quiet); - SET_BOOL(zconf.summary, summary); - zconf.cooldown_secs = args.cooldown_time_arg; - zconf.senders = args.sender_threads_arg; - zconf.log_level = args.verbosity_arg; - if (zconf.probe_module->port_args) { From 76744331420b6eff50177a3713549c17dcffe68e Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Tue, 27 Aug 2013 23:10:35 -0400 Subject: [PATCH 14/18] working fieldsets without headers --- src/fieldset.c | 19 ++++++++++++++++--- src/fieldset.h | 3 ++- src/output_modules/module_csv.c | 19 +++++++++++-------- src/recv.c | 6 ++++-- src/state.h | 3 +-- src/zmap.c | 23 +++++++++++------------ src/zopt.c | 4 ++-- 7 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/fieldset.c b/src/fieldset.c index b560074..73db4da 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -49,7 +49,7 @@ void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_) void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value) { - fs_add_word(fs, name, FS_STRING, 0, sizeof(uint64_t), (void*) value); + fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), (void*) value); } void fs_add_binary(fieldset_t *fs, const char *name, size_t len, @@ -89,9 +89,22 @@ void fs_free(fieldset_t *fs) free(fs); } -translation_t *fs_generate_fieldset_translation() +void fs_generate_fieldset_translation(translation_t *t, + fielddefset_t *avail, char** req, int reqlen) { - return NULL; + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", "unable to allocate memory for translation"); + } + for (int i=0; i < reqlen; i++) { + int l = fds_get_index_by_name(avail, req[i]); + if (l < 0) { + log_fatal("fieldset", "specified field (%s) not " + "available in selected " + "probe module.", req[i]); + } + t->translation[t->len++] = l; + } } fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) diff --git a/src/fieldset.h b/src/fieldset.h index 5ed2136..c2a1e39 100644 --- a/src/fieldset.h +++ b/src/fieldset.h @@ -84,7 +84,8 @@ uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); void fs_free(fieldset_t *fs); -translation_t *fs_generate_fieldset_translation(); +void fs_generate_fieldset_translation(translation_t *t, + fielddefset_t *avail, char** req, int reqlen); fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); diff --git a/src/output_modules/module_csv.c b/src/output_modules/module_csv.c index 4c5ccfc..8bc5665 100644 --- a/src/output_modules/module_csv.c +++ b/src/output_modules/module_csv.c @@ -35,13 +35,14 @@ int csv_init(struct state_conf *conf, fielddefset_t *fds) } } } - // add output headers - for (int i=0; i < fds->len; i++) { - if (i) { - fprintf(file, ", "); - } - fprintf(file, "%s", fds->fielddefs[i].name); - } + //// add output headers + (void)fds; + //for (int i=0; i < fds->len; i++) { + // if (i) { + // fprintf(file, ", "); + // } + // fprintf(file, "%s", fds->fielddefs[i].name); + //} return EXIT_SUCCESS; } @@ -76,7 +77,9 @@ int csv_process(fieldset_t *fs) fprintf(file, "%lu", (uint64_t) f->value); } else if (f->type == FS_BINARY) { hex_encode(file, (unsigned char*) f->value, f->len); - } + } else { + log_fatal("csv", "received unknown output type"); + } } fprintf(file, "\n"); return EXIT_SUCCESS; diff --git a/src/recv.c b/src/recv.c index 820d938..672fdc6 100644 --- a/src/recv.c +++ b/src/recv.c @@ -122,10 +122,12 @@ void packet_cb(u_char __attribute__((__unused__)) *user, // we need to translate the data provided by the probe module // into a fieldset that can be used by the output module - + fieldset_t *o = translate_fieldset(fs, &zconf.fsconf.translation); if (zconf.output_module && zconf.output_module->process_ip) { - zconf.output_module->process_ip(fs); + zconf.output_module->process_ip(o); } + fs_free(fs); + free(o); if (zconf.output_module && zconf.output_module->update && !(zrecv.success_unique % zconf.output_module->update_interval)) { zconf.output_module->update(&zconf, &zsend, &zrecv); diff --git a/src/state.h b/src/state.h index 84aee6f..5492058 100644 --- a/src/state.h +++ b/src/state.h @@ -27,8 +27,7 @@ struct output_module; struct fieldset_conf { fielddefset_t defs; fielddefset_t outdefs; - int *translation; - int translation_len; + translation_t translation; int success_index; int classification_index; }; diff --git a/src/zmap.c b/src/zmap.c index 59684dc..bdf278a 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -44,7 +44,7 @@ pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; // splits comma delimited string into char*[]. Does not handle // escaping or complicated setups: designed to process a set // of fields that the user wants output -static void fs_split_string(char* in, int *len, char***results) +static void split_string(char* in, int *len, char***results) { char** fields = calloc(MAX_FIELDS, sizeof(char*)); memset(fields, 0, sizeof(fields)); @@ -114,7 +114,7 @@ static void *start_mon(__attribute__((unused)) void *arg) #define SI(w,x,y) printf("%s\t%s\t%i\n", w, x, y); #define SD(w,x,y) printf("%s\t%s\t%f\n", w, x, y); #define SU(w,x,y) printf("%s\t%s\t%u\n", w, x, y); -#define SLU(w,x,y) printf("%s\t%s\t%lu\n", w, x, (long unsigned int)y); +#define SLU(w,x,y) printf("%s\t%s\t%lu\n", w, x, (long unsigned int) y); #define SS(w,x,y) printf("%s\t%s\t%s\n", w, x, y); #define STRTIME_LEN 1024 @@ -333,7 +333,6 @@ int parse_mac(macaddr_t *out, char *in) int main(int argc, char *argv[]) { - struct gengetopt_args_info args; struct cmdline_parser_params *params; params = cmdline_parser_params_create(); @@ -341,17 +340,10 @@ int main(int argc, char *argv[]) params->override = 0; params->check_required = 0; - SET_BOOL(zconf.dryrun, dryrun); - SET_BOOL(zconf.quiet, quiet); - SET_BOOL(zconf.summary, summary); - zconf.cooldown_secs = args.cooldown_time_arg; - zconf.senders = args.sender_threads_arg; zconf.log_level = args.verbosity_arg; - log_init(stderr, zconf.log_level); log_trace("zmap", "zmap main thread started"); - if (cmdline_parser_ext(argc, argv, &args, params) != 0) { exit(EXIT_SUCCESS); } @@ -416,7 +408,6 @@ int main(int argc, char *argv[]) } exit(EXIT_SUCCESS); } - // find the fields we need for the framework zconf.fsconf.success_index = fds_get_index_by_name(fds, (char*) "success"); @@ -436,7 +427,7 @@ int main(int argc, char *argv[]) } else { zconf.raw_output_fields = (char*) "saddr"; } - fs_split_string(zconf.raw_output_fields, &(zconf.output_fields_len), + split_string(zconf.raw_output_fields, &(zconf.output_fields_len), &(zconf.output_fields)); for (int i=0; i < zconf.output_fields_len; i++) { log_debug("zmap", "requested output field (%i): %s", @@ -445,7 +436,15 @@ int main(int argc, char *argv[]) } // generate a translation that can be used to convert output // from a probe module to the input for an output module + fs_generate_fieldset_translation(&zconf.fsconf.translation, + &zconf.fsconf.defs, zconf.output_fields, + zconf.output_fields_len); + SET_BOOL(zconf.dryrun, dryrun); + SET_BOOL(zconf.quiet, quiet); + SET_BOOL(zconf.summary, summary); + zconf.cooldown_secs = args.cooldown_time_arg; + zconf.senders = args.sender_threads_arg; SET_IF_GIVEN(zconf.output_filename, output_file); SET_IF_GIVEN(zconf.blacklist_filename, blacklist_file); SET_IF_GIVEN(zconf.whitelist_filename, whitelist_file); diff --git a/src/zopt.c b/src/zopt.c index e733f6e..24d2cab 100644 --- a/src/zopt.c +++ b/src/zopt.c @@ -56,7 +56,7 @@ const char *gengetopt_args_info_help[] = { " -i, --interface=name Specify network interface to use", "\nAdvanced options:", " -M, --probe-module=name Select probe module (default=`tcp_synscan')", - " -O, --output-module=name Select output module (default=`simple_file')", + " -O, --output-module=name Select output module (default=`csv')", " --probe-args=args Arguments to pass to probe module", " --output-args=args Arguments to pass to output module", " --list-output-modules List available output modules", @@ -189,7 +189,7 @@ void clear_args (struct gengetopt_args_info *args_info) args_info->interface_orig = NULL; args_info->probe_module_arg = gengetopt_strdup ("tcp_synscan"); args_info->probe_module_orig = NULL; - args_info->output_module_arg = gengetopt_strdup ("simple_file"); + args_info->output_module_arg = gengetopt_strdup ("csv"); args_info->output_module_orig = NULL; args_info->probe_args_arg = NULL; args_info->probe_args_orig = NULL; From 0c83eabfd1529d535060885d7a0c6bfe96c19b19 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Thu, 29 Aug 2013 14:29:59 -0400 Subject: [PATCH 15/18] working new interface --- INSTALL | 1 + src/fieldset.c | 3 +++ src/output_modules/module_csv.c | 24 ++++++++++------- src/output_modules/output_modules.h | 2 +- src/recv.c | 26 +++++++++++++++--- src/state.c | 2 ++ src/state.h | 2 ++ src/zmap.c | 42 ++++++++++++++++++++++++----- 8 files changed, 81 insertions(+), 21 deletions(-) diff --git a/INSTALL b/INSTALL index a873db9..6d05d16 100644 --- a/INSTALL +++ b/INSTALL @@ -29,3 +29,4 @@ followed by: Redis support is not enabled by default. If you are want to use ZMap with Redis, you will first need to install Hiredis. Then, rebuild ZMap with the command "make REDIS=true". + diff --git a/src/fieldset.c b/src/fieldset.c index 73db4da..4df5e7f 100644 --- a/src/fieldset.c +++ b/src/fieldset.c @@ -80,6 +80,9 @@ int fds_get_index_by_name(fielddefset_t *fds, char *name) void fs_free(fieldset_t *fs) { + if (!fs) { + return; + } for (int i=0; i < fs->len; i++) { field_t *f = &(fs->fields[i]); if (f->free_) { diff --git a/src/output_modules/module_csv.c b/src/output_modules/module_csv.c index 8bc5665..151a0f9 100644 --- a/src/output_modules/module_csv.c +++ b/src/output_modules/module_csv.c @@ -22,7 +22,7 @@ static FILE *file = NULL; -int csv_init(struct state_conf *conf, fielddefset_t *fds) +int csv_init(struct state_conf *conf, char **fields, int fieldlens) { assert(conf); if (conf->output_filename) { @@ -34,15 +34,18 @@ int csv_init(struct state_conf *conf, fielddefset_t *fds) conf->output_filename); } } + } else { + log_warn("csv", "no output file selected. " + "no results will be provided."); + } + if (fieldlens > 1 && file) { + for (int i=0; i < fieldlens; i++) { + if (i) { + fprintf(file, ", "); + } + fprintf(file, "%s", fields[i]); + } } - //// add output headers - (void)fds; - //for (int i=0; i < fds->len; i++) { - // if (i) { - // fprintf(file, ", "); - // } - // fprintf(file, "%s", fds->fielddefs[i].name); - //} return EXIT_SUCCESS; } @@ -66,6 +69,9 @@ static void hex_encode(FILE *f, unsigned char* readbuf, size_t len) int csv_process(fieldset_t *fs) { + if (!file) { + return EXIT_SUCCESS; + } for (int i=0; i < fs->len; i++) { field_t *f = &(fs->fields[i]); if (i) { diff --git a/src/output_modules/output_modules.h b/src/output_modules/output_modules.h index 64acd5d..85a83ae 100644 --- a/src/output_modules/output_modules.h +++ b/src/output_modules/output_modules.h @@ -13,7 +13,7 @@ #include "../fieldset.h" // called at scanner initialization -typedef int (*output_init_cb)(struct state_conf *, fielddefset_t *fds); +typedef int (*output_init_cb)(struct state_conf *, char **fields, int fieldslen); // called on packet receipt typedef int (*output_packet_cb)(fieldset_t *fs); diff --git a/src/recv.c b/src/recv.c index 672fdc6..f4c9a26 100644 --- a/src/recv.c +++ b/src/recv.c @@ -119,13 +119,20 @@ void packet_cb(u_char __attribute__((__unused__)) *user, } else { zrecv.failure_total++; } - + fieldset_t *o = NULL; // we need to translate the data provided by the probe module // into a fieldset that can be used by the output module - fieldset_t *o = translate_fieldset(fs, &zconf.fsconf.translation); + if (!is_success && zconf.filter_unsuccessful) { + goto cleanup; + } + if (is_repeat && zconf.filter_duplicates) { + goto cleanup; + } + o = translate_fieldset(fs, &zconf.fsconf.translation); if (zconf.output_module && zconf.output_module->process_ip) { zconf.output_module->process_ip(o); } +cleanup: fs_free(fs); free(o); if (zconf.output_module && zconf.output_module->update @@ -158,14 +165,14 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) num_src_ports = zconf.source_port_last - zconf.source_port_first + 1; ip_seen = calloc(IP_SEEN_SIZE, sizeof(uint64_t)); if (!ip_seen) { - log_fatal("recv", "couldn't allocate address bitmap"); + log_fatal("recv", "could not allocate address bitmap"); } log_debug("recv", "using dev %s", zconf.iface); char errbuf[PCAP_ERRBUF_SIZE]; pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen, PCAP_PROMISC, PCAP_TIMEOUT, errbuf); if (pc == NULL) { - log_fatal("recv", "couldn't open device %s: %s", + log_fatal("recv", "could not open device %s: %s", zconf.iface, errbuf); } struct bpf_program bpf; @@ -176,6 +183,17 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) log_fatal("recv", "couldn't install filter"); } log_debug("recv", "receiver ready"); + if (zconf.filter_duplicates) { + log_debug("recv", "duplicate responses will be excluded from output"); + } else { + log_debug("recv", "duplicate responses will be included in output"); + } + if (zconf.filter_unsuccessful) { + log_debug("recv", "unsuccessful responses will be excluded from output"); + } else { + log_debug("recv", "unsuccessful responses will be included in output"); + } + pthread_mutex_lock(recv_ready_mutex); zconf.recv_ready = 1; pthread_mutex_unlock(recv_ready_mutex); diff --git a/src/state.c b/src/state.c index 9b8a583..62d5a95 100644 --- a/src/state.c +++ b/src/state.c @@ -44,6 +44,8 @@ struct state_conf zconf = { .dryrun = 0, .quiet = 0, .summary = 0, + .filter_duplicates = 0, + .filter_unsuccessful = 0, .recv_ready = 0, }; diff --git a/src/state.h b/src/state.h index 5492058..8846008 100644 --- a/src/state.h +++ b/src/state.h @@ -83,6 +83,8 @@ struct state_conf { int dryrun; int summary; int quiet; + int filter_duplicates; + int filter_unsuccessful; int recv_ready; }; extern struct state_conf zconf; diff --git a/src/zmap.c b/src/zmap.c index bdf278a..4560474 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -211,7 +211,8 @@ static void start_zmap(void) // initialization if (zconf.output_module && zconf.output_module->init) { - zconf.output_module->init(&zconf, &zconf.fsconf.outdefs); + zconf.output_module->init(&zconf, zconf.output_fields, + zconf.output_fields_len); } if (send_init()) { exit(EXIT_FAILURE); @@ -376,11 +377,38 @@ int main(int argc, char *argv[]) } // parse the provided probe and output module s.t. that we can support // other command-line helpers (e.g. probe help) - zconf.output_module = get_output_module_by_name(args.output_module_arg); - if (!zconf.output_module) { - fprintf(stderr, "%s: specified output module (%s) does not exist\n", - CMDLINE_PARSER_PACKAGE, args.output_module_arg); - exit(EXIT_FAILURE); + if (!args.output_module_given) { + zconf.output_module = get_output_module_by_name("csv"); + zconf.raw_output_fields = (char*) "saddr"; + zconf.filter_duplicates = 1; + zconf.filter_unsuccessful = 1; + } else if (!strcmp(args.output_module_arg, "simple_file")) { + log_warn("zmap", "the simple_file output interface has been deprecated and " + "will be removed in the future. Users should use the csv " + "output module. Newer scan options such as output-fields " + "are not supported with this output module."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.raw_output_fields = (char*) "saddr"; + zconf.filter_duplicates = 1; + zconf.filter_unsuccessful = 1; + } else if (!strcmp(args.output_module_arg, "extended_file")) { + log_warn("zmap", "the extended_file output interface has been deprecated and " + "will be removed in the future. Users should use the csv " + "output module. Newer scan options such as output-fields " + "are not supported with this output module."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.raw_output_fields = (char*) "classification, saddr, " + "daddr, sport, dport, " + "seqnum, acknum, cooldown, " + "repeat, timstamp-str"; + zconf.filter_duplicates = 0; + } else { + zconf.output_module = get_output_module_by_name(args.output_module_arg); + if (!zconf.output_module) { + fprintf(stderr, "%s: specified output module (%s) does not exist\n", + CMDLINE_PARSER_PACKAGE, args.output_module_arg); + exit(EXIT_FAILURE); + } } zconf.probe_module = get_probe_module_by_name(args.probe_module_arg); if (!zconf.probe_module) { @@ -424,7 +452,7 @@ int main(int argc, char *argv[]) // process the list of requested output fields. if (args.output_fields_given) { zconf.raw_output_fields = args.output_fields_arg; - } else { + } else if (!zconf.raw_output_fields) { zconf.raw_output_fields = (char*) "saddr"; } split_string(zconf.raw_output_fields, &(zconf.output_fields_len), From a5eb3d0a8d8e1e3d7faef3eb85411297f31adaf6 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Thu, 29 Aug 2013 14:38:51 -0400 Subject: [PATCH 16/18] dryrun no longer requires root access --- src/send.c | 12 +++++++++++- src/zmap.c | 30 ++++++++++++++++-------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/send.c b/src/send.c index e30c044..b2e3db6 100644 --- a/src/send.c +++ b/src/send.c @@ -142,6 +142,11 @@ static int get_socket(void) return sock; } +static int get_dryrun_socket(void) +{ + return socket(AF_INET, SOCK_STREAM, 0); +} + static inline ipaddr_n_t get_src_ip(ipaddr_n_t dst, int local_offset) { if (srcip_first == srcip_last) { @@ -156,7 +161,12 @@ int send_run(void) { log_debug("send", "thread started"); pthread_mutex_lock(&send_mutex); - int sock = get_socket(); + int sock; + if (zconf.dryrun) { + sock = get_dryrun_socket(); + } else { + sock = get_socket(); + } struct sockaddr_ll sockaddr; // get source interface index struct ifreq if_idx; diff --git a/src/zmap.c b/src/zmap.c index 4560474..00a6f47 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -222,30 +222,32 @@ static void start_zmap(void) } // start threads pthread_t *tsend, trecv, tmon; - int r = pthread_create(&trecv, NULL, start_recv, NULL); - if (r != 0) { - log_fatal("zmap", "unable to create recv thread"); - exit(EXIT_FAILURE); - } - for (;;) { - pthread_mutex_lock(&recv_ready_mutex); - if (zconf.recv_ready) { - break; + if (!zconf.dryrun) { + int r = pthread_create(&trecv, NULL, start_recv, NULL); + if (r != 0) { + log_fatal("zmap", "unable to create recv thread"); + exit(EXIT_FAILURE); + } + for (;;) { + pthread_mutex_lock(&recv_ready_mutex); + if (zconf.recv_ready) { + break; + } + pthread_mutex_unlock(&recv_ready_mutex); } - pthread_mutex_unlock(&recv_ready_mutex); } tsend = malloc(zconf.senders * sizeof(pthread_t)); assert(tsend); log_debug("zmap", "using %d sender threads", zconf.senders); for (int i=0; i < zconf.senders; i++) { - r = pthread_create(&tsend[i], NULL, start_send, NULL); + int r = pthread_create(&tsend[i], NULL, start_send, NULL); if (r != 0) { log_fatal("zmap", "unable to create send thread"); exit(EXIT_FAILURE); } } if (!zconf.quiet) { - r = pthread_create(&tmon, NULL, start_mon, NULL); + int r = pthread_create(&tmon, NULL, start_mon, NULL); if (r != 0) { log_fatal("zmap", "unable to create monitor thread"); exit(EXIT_FAILURE); @@ -254,14 +256,14 @@ static void start_zmap(void) // wait for completion for (int i=0; i < zconf.senders; i++) { - pthread_join(tsend[i], NULL); + int r = pthread_join(tsend[i], NULL); if (r != 0) { log_fatal("zmap", "unable to join send thread"); exit(EXIT_FAILURE); } } log_debug("zmap", "senders finished"); - pthread_join(trecv, NULL); + int r = pthread_join(trecv, NULL); if (r != 0) { log_fatal("zmap", "unable to join recv thread"); exit(EXIT_FAILURE); From 672143e9bad1e3aa1a3f331972b5251fefb53e95 Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Thu, 29 Aug 2013 14:51:26 -0400 Subject: [PATCH 17/18] bug fixes to extended_file output --- src/recv.c | 44 +++++++++++++++++++++++++------------------- src/send.c | 12 +++++++++++- src/zmap.c | 26 ++++++++++++-------------- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/recv.c b/src/recv.c index f4c9a26..8514144 100644 --- a/src/recv.c +++ b/src/recv.c @@ -168,19 +168,21 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) log_fatal("recv", "could not allocate address bitmap"); } log_debug("recv", "using dev %s", zconf.iface); - char errbuf[PCAP_ERRBUF_SIZE]; - pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen, - PCAP_PROMISC, PCAP_TIMEOUT, errbuf); - if (pc == NULL) { - log_fatal("recv", "could not open device %s: %s", - zconf.iface, errbuf); - } - struct bpf_program bpf; - if (pcap_compile(pc, &bpf, zconf.probe_module->pcap_filter, 1, 0) < 0) { - log_fatal("recv", "couldn't compile filter"); - } - if (pcap_setfilter(pc, &bpf) < 0) { - log_fatal("recv", "couldn't install filter"); + if (!zconf.dryrun) { + char errbuf[PCAP_ERRBUF_SIZE]; + pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen, + PCAP_PROMISC, PCAP_TIMEOUT, errbuf); + if (pc == NULL) { + log_fatal("recv", "could not open device %s: %s", + zconf.iface, errbuf); + } + struct bpf_program bpf; + if (pcap_compile(pc, &bpf, zconf.probe_module->pcap_filter, 1, 0) < 0) { + log_fatal("recv", "couldn't compile filter"); + } + if (pcap_setfilter(pc, &bpf) < 0) { + log_fatal("recv", "couldn't install filter"); + } } log_debug("recv", "receiver ready"); if (zconf.filter_duplicates) { @@ -202,12 +204,16 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) zconf.max_results = -1; } do { - if (pcap_dispatch(pc, 0, packet_cb, NULL) == -1) { - log_fatal("recv", "pcap_dispatch error"); - } - if (zconf.max_results && zrecv.success_unique >= zconf.max_results) { - zsend.complete = 1; - break; + if (zconf.dryrun) { + sleep(1); + } else { + if (pcap_dispatch(pc, 0, packet_cb, NULL) == -1) { + log_fatal("recv", "pcap_dispatch error"); + } + if (zconf.max_results && zrecv.success_unique >= zconf.max_results) { + zsend.complete = 1; + break; + } } } while (!(zsend.complete && (now()-zsend.finish > zconf.cooldown_secs))); zrecv.finish = now(); diff --git a/src/send.c b/src/send.c index b2e3db6..4426da8 100644 --- a/src/send.c +++ b/src/send.c @@ -144,7 +144,17 @@ static int get_socket(void) static int get_dryrun_socket(void) { - return socket(AF_INET, SOCK_STREAM, 0); + // we need a socket in order to gather details about the system + // such as source MAC address and IP address. However, because + // we don't want to require root access in order to run dryrun, + // we just create a TCP socket. + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock <= 0) { + log_fatal("send", "couldn't create socket. " + "Error: %s\n", strerror(errno)); + } + return sock; + } static inline ipaddr_n_t get_src_ip(ipaddr_n_t dst, int local_offset) diff --git a/src/zmap.c b/src/zmap.c index 00a6f47..598c9d1 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -222,19 +222,17 @@ static void start_zmap(void) } // start threads pthread_t *tsend, trecv, tmon; - if (!zconf.dryrun) { - int r = pthread_create(&trecv, NULL, start_recv, NULL); - if (r != 0) { - log_fatal("zmap", "unable to create recv thread"); - exit(EXIT_FAILURE); - } - for (;;) { - pthread_mutex_lock(&recv_ready_mutex); - if (zconf.recv_ready) { - break; - } - pthread_mutex_unlock(&recv_ready_mutex); + int r = pthread_create(&trecv, NULL, start_recv, NULL); + if (r != 0) { + log_fatal("zmap", "unable to create recv thread"); + exit(EXIT_FAILURE); + } + for (;;) { + pthread_mutex_lock(&recv_ready_mutex); + if (zconf.recv_ready) { + break; } + pthread_mutex_unlock(&recv_ready_mutex); } tsend = malloc(zconf.senders * sizeof(pthread_t)); assert(tsend); @@ -263,7 +261,7 @@ static void start_zmap(void) } } log_debug("zmap", "senders finished"); - int r = pthread_join(trecv, NULL); + r = pthread_join(trecv, NULL); if (r != 0) { log_fatal("zmap", "unable to join recv thread"); exit(EXIT_FAILURE); @@ -402,7 +400,7 @@ int main(int argc, char *argv[]) zconf.raw_output_fields = (char*) "classification, saddr, " "daddr, sport, dport, " "seqnum, acknum, cooldown, " - "repeat, timstamp-str"; + "repeat, timestamp-str"; zconf.filter_duplicates = 0; } else { zconf.output_module = get_output_module_by_name(args.output_module_arg); From c3e6e0c041ab7fd5d0380b468f2538e08d67c4bb Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Thu, 29 Aug 2013 14:56:13 -0400 Subject: [PATCH 18/18] removing simple and extended file modules --- src/output_modules/module_extended_file.c | 110 ---------------------- src/output_modules/module_extended_file.h | 20 ---- src/output_modules/module_simple_file.c | 74 --------------- src/output_modules/output_modules.c | 4 - 4 files changed, 208 deletions(-) delete mode 100644 src/output_modules/module_extended_file.c delete mode 100644 src/output_modules/module_extended_file.h delete mode 100644 src/output_modules/module_simple_file.c diff --git a/src/output_modules/module_extended_file.c b/src/output_modules/module_extended_file.c deleted file mode 100644 index 5e035bc..0000000 --- a/src/output_modules/module_extended_file.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * ZMap Copyright 2013 Regents of the University of Michigan - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "../../lib/logger.h" - -#include "output_modules.h" - -static FILE *file = NULL; -#define UNUSED __attribute__((unused)) - - -int extendedfile_init(struct state_conf *conf) -{ - assert(conf); - if (conf->output_filename) { - if (!strcmp(conf->output_filename, "-")) { - file = stdout; - } else { - if (!(file = fopen(conf->output_filename, "w"))) { - perror("Couldn't open output file"); - exit(EXIT_FAILURE); - } - } - fprintf(file, "response, saddr, daddr, sport, " - "dport, seq, ack, in_cooldown, is_repeat, timestamp\n"); - } - return EXIT_SUCCESS; -} - -static void fprint_tv(FILE *f, struct timeval *tv) -{ - char time_string[40]; - struct tm *ptm = localtime(&tv->tv_sec); - strftime(time_string, sizeof (time_string), - "%Y-%m-%d %H:%M:%S", ptm); - long milliseconds = tv->tv_usec / 1000; - fprintf(f, "%s.%03ld\n", time_string, milliseconds); -} - - -int extendedfile_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, - const char *response_type, int is_repeat, - int in_cooldown, const u_char *packet, size_t buflen) -{ - struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)]; - if (buflen < (sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct tcphdr))) - return EXIT_FAILURE; - struct tcphdr *tcp = (struct tcphdr *)((char *)ip_hdr + ip_hdr->ihl * 4); - - if (file) { - struct in_addr addr; - addr.s_addr = saddr; - // inet_ntoa returns a <> char * - fprintf(file, "%s, %s, ", - response_type, - inet_ntoa(addr)); - addr.s_addr = daddr; - fprintf(file, "%s, %u, %u, %u, %u, %i, %i,", - inet_ntoa(addr), - ntohs(tcp->source), - ntohs(tcp->dest), - ntohl(tcp->seq), - ntohl(tcp->ack_seq), - in_cooldown, - is_repeat); - struct timeval t; - gettimeofday(&t, NULL); - fprint_tv(file, &t); - fflush(file); - } - return EXIT_SUCCESS; -} - -int extendedfile_close(UNUSED struct state_conf* c, - UNUSED struct state_send* s, UNUSED struct state_recv* r) -{ - if (file) { - fflush(file); - fclose(file); - } - return EXIT_SUCCESS; -} - -output_module_t module_extended_file = { - .name = "extended_file", - .init = &extendedfile_init, - .start = NULL, - .update = NULL, - .update_interval = 0, - .close = &extendedfile_close, - .success_ip = &extendedfile_ip, - .other_ip = &extendedfile_ip -}; - diff --git a/src/output_modules/module_extended_file.h b/src/output_modules/module_extended_file.h deleted file mode 100644 index d8b6357..0000000 --- a/src/output_modules/module_extended_file.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * ZMap Copyright 2013 Regents of the University of Michigan - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - */ - -#include - -int extendedfile_init(struct state_conf *conf); - -int extendedfile_ip(ipaddr_n_t saddr, ipaddr_n_t daddr, - port_n_t sport, port_n_t dport, struct timeval* t, - const char *response_type, int is_repeat, - int in_cooldown, const u_char *packet); - -int extendedfile_close(struct state_conf* c, struct state_send* s, - struct state_recv* r); - diff --git a/src/output_modules/module_simple_file.c b/src/output_modules/module_simple_file.c deleted file mode 100644 index beb62bf..0000000 --- a/src/output_modules/module_simple_file.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * ZMap Copyright 2013 Regents of the University of Michigan - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "output_modules.h" - -#define UNUSED __attribute__((unused)) - -static FILE *file = NULL; - -int simplefile_init(struct state_conf *conf) -{ - assert(conf); - if (conf->output_filename) { - if (!strcmp(conf->output_filename, "-")) { - file = stdout; - } else { - if (!(file = fopen(conf->output_filename, "w"))) { - perror("Couldn't open output file"); - exit(EXIT_FAILURE); - } - } - } - return EXIT_SUCCESS; -} - -int simplefile_synack_newip(ipaddr_n_t saddr, UNUSED ipaddr_n_t daddr, - UNUSED const char *response_type, - int is_repeat, UNUSED int in_cooldown, UNUSED const u_char *packet, - UNUSED size_t buflen) -{ - if (file && !is_repeat) { - struct in_addr addr; - addr.s_addr = saddr; - fprintf(file, "%s\n", inet_ntoa(addr)); - } - fflush(file); - return EXIT_SUCCESS; -} - -int simplefile_close(UNUSED struct state_conf* c, - UNUSED struct state_send* s, - UNUSED struct state_recv* r) -{ - if (file) { - fflush(file); - fclose(file); - } - return EXIT_SUCCESS; -} - -output_module_t module_simple_file = { - .name = "simple_file", - .init = &simplefile_init, - .start = NULL, - .update = NULL, - .update_interval = 0, - .close = &simplefile_close, - .success_ip = &simplefile_synack_newip, - .other_ip = NULL, -}; diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c index a9c4382..6f52a7e 100644 --- a/src/output_modules/output_modules.c +++ b/src/output_modules/output_modules.c @@ -12,8 +12,6 @@ #include "output_modules.h" -//extern output_module_t module_simple_file; -//extern output_module_t module_extended_file; extern output_module_t module_csv_file; // ADD YOUR MODULE HERE @@ -23,8 +21,6 @@ extern output_module_t module_redis; output_module_t* output_modules[] = { &module_csv_file - //&module_simple_file, - //&module_extended_file, #ifdef REDIS //&module_redis, #endif