From 277f49457d1bc5aa1de8ac73ffc34dae68d8cd6f Mon Sep 17 00:00:00 2001 From: Zakir Durumeric Date: Mon, 7 Oct 2013 17:08:23 -0400 Subject: [PATCH] adding ability to scan specific CIDR blocks or addresses. --- lib/blacklist.c | 103 +++++++++++++++++++++++++++------------------ lib/blacklist.h | 7 ++- lib/pbm.c | 54 ++++++++++++++++++++++++ lib/pbm.h | 5 +++ src/CMakeLists.txt | 3 +- src/cyclic.c | 18 +++++--- src/recv.c | 26 +++--------- src/state.h | 2 + src/zmap.c | 20 ++++++--- src/zopt.ggo | 5 ++- 10 files changed, 168 insertions(+), 75 deletions(-) create mode 100644 lib/pbm.c create mode 100644 lib/pbm.h diff --git a/lib/blacklist.c b/lib/blacklist.c index a33860a..e5d5919 100644 --- a/lib/blacklist.c +++ b/lib/blacklist.c @@ -51,7 +51,38 @@ void whitelist_prefix(char *ip, int prefix_len) constraint_set(constraint, ntohl(inet_addr(ip)), prefix_len, ADDR_ALLOWED); } -static int init(char *file, const char *name, int value) +static int init_from_string(char *ip, int value) +{ + int prefix_len = 32; + char *slash = strchr(ip, '/'); + if (slash) { // split apart network and prefix length + *slash = '\0'; + char *end; + char *len = slash+1; + errno = 0; + prefix_len = strtol(len, &end, 10); + if (end == len || errno != 0 || prefix_len < 0 || prefix_len > 32) { + log_fatal("constraint", "'%s' is not a valid prefix length", len); + return -1; + } + } + struct in_addr addr; + if (inet_aton(ip, &addr) == 0) { + log_error("constraint", "'%s' is not a valid IP address", ip); + return -1; + } + constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value); + char *name; + if (value == ADDR_DISALLOWED) + name = "blacklisting"; + else + name = "whitelisting"; + log_trace(name, "%s %s/%i", + name, ip, prefix_len); +} + + +static int init_from_file(char *file, const char *name, int value) { FILE *fp; char line[1000]; @@ -70,31 +101,22 @@ static int init(char *file, const char *name, int value) if ((sscanf(line, "%32s", ip)) == EOF) { continue; } - int prefix_len = 32; - char *slash = strchr(ip, '/'); - if (slash) { // split apart network and prefix length - *slash = '\0'; - char *end; - char *len = slash+1; - errno = 0; - prefix_len = strtol(len, &end, 10); - if (end == len || errno != 0 || prefix_len < 0 || prefix_len > 32) { - log_fatal(name, "Unable to parse %s file: %s ('%s' is not a valid prefix length)", name, file, len); - } + if (init_from_string(ip, value)) { + log_fatal(name, "unable to parse %s file: %s", name, file); } - struct in_addr addr; - if (inet_aton(ip, &addr) == 0) { - log_fatal(name, "Unable to parse %s file: %s ('%s' is not a valid IP address)", name, file, ip); - } - constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value); - log_trace(name, "%sing %s/%i", - name, ip, prefix_len); } fclose(fp); return 0; } +void static init_from_array(char **cidrs, size_t len, int value) +{ + for (int i=0; i < (int) len; i++) { + init_from_string(cidrs[i], value); + } +} + uint64_t blacklist_count_allowed() { assert(constraint); @@ -107,42 +129,43 @@ uint64_t blacklist_count_not_allowed() return constraint_count_ips(constraint, ADDR_DISALLOWED); } - // Initialize address constraints from whitelist and blacklist files. // Either can be set to NULL to omit. -int blacklist_init_from_files(char *whitelist_filename, char *blacklist_filename) +int blacklist_init(char *whitelist_filename, char *blacklist_filename, + char **whitelist_entries, size_t whitelist_entries_len, + char **blacklist_entries, size_t blacklist_entries_len) { assert(!constraint); - if (whitelist_filename) { + if (whitelist_filename && whitelist_entries) { + log_warn("whitelist", "both a whitelist file and destination addresses " + "were specified. The union of these two sources " + "will be utilized."); + } + if (whitelist_filename || whitelist_entries) { // using a whitelist, so default to allowing nothing constraint = constraint_init(ADDR_DISALLOWED); log_trace("whitelist", "blacklisting 0.0.0.0/0"); - init(whitelist_filename, "whitelist", ADDR_ALLOWED); + if (whitelist_filename) { + init_from_file(whitelist_filename, "whitelist", ADDR_ALLOWED); + } + if (whitelist_entries) { + init_from_array(whitelist_entries, + whitelist_entries_len, ADDR_ALLOWED); + } } else { // no whitelist, so default to allowing everything constraint = constraint_init(ADDR_ALLOWED); } if (blacklist_filename) { - init(blacklist_filename, "blacklist", ADDR_DISALLOWED); + init_from_file(blacklist_filename, "blacklist", ADDR_DISALLOWED); + } + if (blacklist_entries) { + init_from_array(blacklist_entries, blacklist_entries_len, ADDR_DISALLOWED); } constraint_paint_value(constraint, ADDR_ALLOWED); uint64_t allowed = blacklist_count_allowed(); log_debug("blacklist", "%lu addresses allowed to be scanned (%0.0f%% of address space)", allowed, allowed*100./((long long int)1 << 32)); - - /* - // test - log_debug("blacklist", "testing started"); - uint64_t count = constraint_count_ips(constraint, ADDR_ALLOWED); - for (unsigned int i=0; i < count; i++) { - int ip = constraint_lookup_index(constraint, i, ADDR_ALLOWED); - if ((i & 0xFFFFFF) == 0) - log_info("blacklist", "%x", i & 0xFF000000); - if (constraint_lookup_ip(constraint, ip) != ADDR_ALLOWED) { - log_error("blacklist", "test failed for index %d", i); - } - } - log_debug("blacklist", "testing complete"); - */ - return 0; + return EXIT_SUCCESS; } + diff --git a/lib/blacklist.h b/lib/blacklist.h index 98d77b4..ce5f136 100644 --- a/lib/blacklist.h +++ b/lib/blacklist.h @@ -1,3 +1,4 @@ +#include #include #ifndef BLACKLIST_H @@ -7,7 +8,11 @@ uint32_t blacklist_lookup_index(uint64_t index); int blacklist_is_allowed(uint32_t s_addr); void blacklist_prefix(char *ip, int prefix_len); void whitelist_prefix(char *ip, int prefix_len); -int blacklist_init_from_files(char *whitelist, char*blacklist); +int blacklist_init(char *whitelist, char *blacklist, + char **whitelist_entries, + size_t whitelist_entries_len, + char **blacklist_entries, + size_t blacklist_entries_len); uint64_t blacklist_count_allowed(); uint64_t blacklist_count_not_allowed(); diff --git a/lib/pbm.c b/lib/pbm.c new file mode 100644 index 0000000..154a535 --- /dev/null +++ b/lib/pbm.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include + +#include "logger.h" + +#define NUM_VALUES 0xFFFFFFFF +#define PAGE_SIZE 0xFFFF +#define NUM_PAGES (NUM_VALUES / PAGE_SIZE) +#define SIZE (8*sizeof(uint64_t)) + +uint64_t** pbm_init(void) +{ + uint64_t** retv = calloc(NUM_PAGES, sizeof(void*)); + if (!retv) { + log_fatal("pbm", "unable to allocate memory for base page table"); + } + return retv; +} + +static int bm_check(uint64_t *bm, uint32_t v) +{ + return bm[v/SIZE] & (1<< (v % SIZE)); +} + +static void bm_set(uint64_t *bm, uint32_t v) +{ + bm[v/SIZE] |= (1 << (v % SIZE)); +} + +int pbm_check(uint64_t **b, uint32_t v) +{ + uint32_t top = v >> 16; + uint32_t bottom = v & PAGE_SIZE; + return b[top] && bm_check(b[top], bottom); +} + +void pbm_set(uint64_t **b, uint32_t v) +{ + uint32_t top = v >> 16; + uint32_t bottom = v & PAGE_SIZE; + if (!b[top]) { + uint64_t *bm = malloc(PAGE_SIZE); + if (!bm) { + log_fatal("bpm", "unable to allocate memory for new bitmap page"); + } + memset(bm, 0, PAGE_SIZE); + b[top] = bm; + } + bm_set(b[top], bottom); +} + diff --git a/lib/pbm.h b/lib/pbm.h new file mode 100644 index 0000000..df3b6fd --- /dev/null +++ b/lib/pbm.h @@ -0,0 +1,5 @@ +#include + +uint64_t** pbm_init(void); +int pbm_check(uint64_t **b, uint32_t v); +void pbm_set(uint64_t **b, uint32_t v); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 50bf1ee..05177a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,6 +9,7 @@ SET(LIB_SOURCES ${PROJECT_SOURCE_DIR}/lib/blacklist.c ${PROJECT_SOURCE_DIR}/lib/constraint.c ${PROJECT_SOURCE_DIR}/lib/logger.c + ${PROJECT_SOURCE_DIR}/lib/pbm.c ${PROJECT_SOURCE_DIR}/lib/random.c ${PROJECT_SOURCE_DIR}/lib/rijndael-alg-fst.c ) @@ -67,7 +68,7 @@ if (WITH_REDIS) endif() add_custom_command(OUTPUT zopt.h - COMMAND gengetopt -C --no-help --no-version -i "${CMAKE_CURRENT_SOURCE_DIR}/zopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zopt" + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_SOURCE_DIR}/zopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zopt" ) add_executable(zmap ${SOURCES}) diff --git a/src/cyclic.c b/src/cyclic.c index 7d2eeee..9f53690 100644 --- a/src/cyclic.c +++ b/src/cyclic.c @@ -148,17 +148,25 @@ static uint64_t find_primroot(const cyclic_group_t *group) int cyclic_init(uint32_t primroot_, uint32_t current_) { assert(!(!primroot_ && current_)); - // Initialize blacklist - if (blacklist_init_from_files(zconf.whitelist_filename, - zconf.blacklist_filename)) { + if (blacklist_init(zconf.whitelist_filename, zconf.blacklist_filename, + zconf.destination_cidrs, zconf.destination_cidrs_len, + NULL, 0)) { return -1; } num_addrs = blacklist_count_allowed(); + if (!num_addrs) { + log_error("blacklist", "no addresses are eligible to be scanned in the " + "current configuration. This may be because the " + "blacklist being used by ZMap (%s) prevents " + "any addresses from receiving probe packets.", + zconf.blacklist_filename + ); + exit(EXIT_FAILURE); + } - uint32_t i; const cyclic_group_t *cur_group = NULL; - for (i=0; i num_addrs) { cur_group = &groups[i]; log_debug("cyclic", "using prime %lu, known_primroot %lu", cur_group->prime, cur_group->known_primroot); diff --git a/src/recv.c b/src/recv.c index 7f15765..bb52fcd 100644 --- a/src/recv.c +++ b/src/recv.c @@ -30,6 +30,7 @@ #include #include "../lib/logger.h" +#include "../lib/pbm.h" #include "state.h" #include "validate.h" @@ -44,20 +45,7 @@ static uint32_t num_src_ports; static pcap_t *pc = NULL; // bitmap of observed IP addresses -static uint64_t *ip_seen = NULL; -static const int IP_SEEN_SIZE = 0x4000000; // == 2^32/64 - -// check if we've received a response from this address previously -static inline int check_ip(uint32_t ip) -{ - return (ip_seen[ip >> 6] >> (ip & 0x3F)) & 1; -} - -// set that we've received a response from the address -static inline void set_ip(uint32_t ip) -{ - ip_seen[ip >> 6] |= (uint64_t)1 << (ip & 0x3F); -} +static uint64_t **seen = NULL; static u_char fake_eth_hdr[65535]; @@ -95,7 +83,7 @@ void packet_cb(u_char __attribute__((__unused__)) *user, return; } - int is_repeat = check_ip(src_ip); + int is_repeat = pbm_check(seen, ntohl(src_ip)); fieldset_t *fs = fs_new_fieldset(); fs_add_ip_fields(fs, ip_hdr); @@ -121,7 +109,7 @@ void packet_cb(u_char __attribute__((__unused__)) *user, zrecv.success_total++; if (!is_repeat) { zrecv.success_unique++; - set_ip(src_ip); + pbm_set(seen, ntohl(src_ip)); } if (zsend.complete) { zrecv.cooldown_total++; @@ -176,10 +164,6 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) { log_debug("recv", "thread started"); num_src_ports = zconf.source_port_last - zconf.source_port_first + 1; - ip_seen = calloc(IP_SEEN_SIZE, sizeof(uint64_t)); - if (!ip_seen) { - log_fatal("recv", "could not allocate address bitmap"); - } log_debug("recv", "using dev %s", zconf.iface); if (!zconf.dryrun) { char errbuf[PCAP_ERRBUF_SIZE]; @@ -202,6 +186,8 @@ int recv_run(pthread_mutex_t *recv_ready_mutex) memset(fake_eth_hdr, 0, sizeof(fake_eth_hdr)); eth->h_proto = htons(ETH_P_IP); } + // initialize paged bitmap + seen = pbm_init(); log_debug("recv", "receiver ready"); if (zconf.filter_duplicates) { log_debug("recv", "duplicate responses will be excluded from output"); diff --git a/src/state.h b/src/state.h index 5bd3f75..bb4181a 100644 --- a/src/state.h +++ b/src/state.h @@ -77,6 +77,8 @@ struct state_conf { char *output_filename; char *blacklist_filename; char *whitelist_filename; + char **destination_cidrs; + int destination_cidrs_len; char *raw_output_fields; char **output_fields; struct fieldset_conf fsconf; diff --git a/src/zmap.c b/src/zmap.c index 083566d..1356489 100644 --- a/src/zmap.c +++ b/src/zmap.c @@ -180,8 +180,6 @@ static void summary(void) static void start_zmap(void) { log_info("zmap", "started"); - - // finish setting up configuration if (zconf.iface == NULL) { char errbuf[PCAP_ERRBUF_SIZE]; char *iface = pcap_lookupdev(errbuf); @@ -223,7 +221,6 @@ static void start_zmap(void) zconf.gw_mac[0], zconf.gw_mac[1], zconf.gw_mac[2], zconf.gw_mac[3], zconf.gw_mac[4], zconf.gw_mac[5]); } - // initialization if (zconf.output_module && zconf.output_module->init) { zconf.output_module->init(&zconf, zconf.output_fields, @@ -446,7 +443,6 @@ int main(int argc, char *argv[]) CMDLINE_PARSER_PACKAGE, args.probe_module_arg); exit(EXIT_FAILURE); } - // now that we know the probe module, let's find what it supports memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf)); // the set of fields made available to a user is constructed @@ -498,6 +494,7 @@ int main(int argc, char *argv[]) &zconf.fsconf.defs, zconf.output_fields, zconf.output_fields_len); + SET_BOOL(zconf.dryrun, dryrun); SET_BOOL(zconf.quiet, quiet); SET_BOOL(zconf.summary, summary); @@ -505,7 +502,6 @@ int main(int argc, char *argv[]) zconf.senders = args.sender_threads_arg; SET_IF_GIVEN(zconf.output_filename, output_file); SET_IF_GIVEN(zconf.blacklist_filename, blacklist_file); - SET_IF_GIVEN(zconf.whitelist_filename, whitelist_file); SET_IF_GIVEN(zconf.probe_args, probe_args); SET_IF_GIVEN(zconf.output_args, output_args); SET_IF_GIVEN(zconf.iface, interface); @@ -513,8 +509,20 @@ int main(int argc, char *argv[]) SET_IF_GIVEN(zconf.max_results, max_results); SET_IF_GIVEN(zconf.rate, rate); SET_IF_GIVEN(zconf.packet_streams, probes); - + // find if zmap wants any specific cidrs scanned instead + // of the entire Internet + zconf.destination_cidrs = args.inputs; + zconf.destination_cidrs_len = args.inputs_num; + if (zconf.destination_cidrs && zconf.blacklist_filename + && !strcmp(zconf.blacklist_filename, "/etc/zmap/blacklist.conf")) { + log_warn("blacklist", "ZMap is currently using the default blacklist located " + "at /etc/zmap/blacklist.conf. By default, this blacklist excludes locally " + "scoped networks (e.g. 10.0.0.0/8, 127.0.0.1/8, and 192.168.0.0/16). If you are" + " trying to scan local networks, you can change the default blacklist by " + "editing the default ZMap configuration at /etc/zmap/zmap.conf."); + } + if (zconf.probe_module->port_args) { if (args.source_port_given) { char *dash = strchr(args.source_port_arg, '-'); diff --git a/src/zopt.ggo b/src/zopt.ggo index 71f0aef..48844b6 100644 --- a/src/zopt.ggo +++ b/src/zopt.ggo @@ -122,5 +122,6 @@ option "version" V "Print version and exit" text "\nExamples:\n\ zmap -p 443 (scans the whole Internet for hosts with port 443 open)\n\ - zmap -N 5 -B 10M -p 80 -o - (find 5 HTTP servers, scanning at 10 Mb/s)" - + zmap -N 5 -B 10M -p 80 -o - (find 5 HTTP servers, scanning at 10 Mb/s)\n + zmap -p 80 10.0.0.0/8 192.168.0.0/16 -o (scan 10.0.0.0/8 and 192.168.0.0/16 on port 80)\n + zmap -p 80 192.168.1.2 192.168.1.3 (scan 192.168.1.2 and 192.168.1.3 on port 80)"