Merge remote-tracking branch 'upstream/master' into FREEBSD

Conflicts:
	src/Makefile

.gitignore

@@ -4,3 +4,14 @@
 *~
 \#*
 src/zmap
+CMakeFiles
+*.cmake
+Makefile
+CMakeCache.txt
+src/zopt.h
+src/zopt.c
+lexer.c
+lexer.h
+parser.c
+parser.h
+install_manifest.txt

AUTHORS

@@ -1,3 +1,7 @@
 Zakir Durumeric <zakird@umich.edu>
 J. Alex Halderman <jhalderm@umich.edu>
-Eric Wustrow <ewust@umich.edu>
+Eric Wustrow <ewust@umich.edu>
+
+David Adrian <davadria@umich.edu>
+HD Moore <HD_Moore@rapid7.com>
+

CMakeLists.txt

@@ -0,0 +1,70 @@
+cmake_minimum_required (VERSION 2.6)
+project (ZMAP C)
+
+option(WITH_REDIS "Build with support for Redis DB" OFF)
+option(WITH_JSON "Build with support for JSON" OFF)
+option(ENABLE_DEVELOPMENT "Enable development specific compiler and linker flags" OFF)
+option(ENABLE_HARDENING "Add hardening specific compiler and linker flags" ON)
+
+if(ENABLE_DEVELOPMENT)
+	# Hardening and warnings for building with gcc
+	# Maybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations
+	set(GCCWARNINGS 
+		"-Wall -Wformat=2 -Wno-format-nonliteral"
+		"-pedantic -fno-strict-aliasing"
+		"-Wextra"
+		"-Wfloat-equal -Wundef -Wwrite-strings -Wredundant-decls"
+		"-Wnested-externs -Wbad-function-cast -Winit-self"
+		"-Wmissing-noreturn -Wnormalized=id"
+		"-Wstack-protector"
+		"-Werror"
+		)
+
+	# Fix line breaks
+	string(REPLACE ";" " "  GCCWARNINGS "${GCCWARNINGS}")
+
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCCWARNINGS} -g")
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g")
+endif()
+
+if(ENABLE_HARDENING)
+	set(GCCHARDENING "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1")
+	set(LDHARDENING "-z relro -z now")
+
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCCHARDENING}")
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LDHARDENING}")
+endif()
+
+if(WITH_REDIS)
+	set(REDIS_LIBS hiredis)
+	add_definitions("-DREDIS")
+endif()
+
+if(WITH_JSON)
+	include(FindPkgConfig)
+	pkg_check_modules(JSON json)
+	if(JSON_FOUND)
+		include_directories(JSON_INCLUDE_DIRS)
+	else()
+		message(FATAL_ERROR "Did not find libjson")
+	endif()
+
+	add_definitions("-DJSON")
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${JSON_CFLAGS}")
+endif()
+
+# Standard FLAGS
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread")
+
+# Extra target FLAGS
+set(CMAKE_C_FLAGS_DEBUG "-O2 -g")
+set(CMAKE_C_FLAGS_RELEASE "-O2")
+
+add_subdirectory(src)
+
+# Install conf files
+FILE(GLOB CONF_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/conf" "conf/*")
+message(STATUS "${CONF_FILES}")
+configure_file(zmap_conf_install.cmake.in zmap_conf_install.cmake)
+install(SCRIPT zmap_conf_install.cmake)

INSTALL

@@ -1,38 +1,38 @@
 SYSTEM REQUIREMENTS
 
 ZMap is designed to run on GNU/Linux systems and can be built with
-most recent versions of gcc.  Currently, ZMap only supports 64-bit
+most recent versions of gcc. Currently, ZMap only supports 64-bit
-systems.  Running it requires at least 600 MB of free memory.
+systems. Running it requires at least 600 MB of free memory.
 
 BUILDING AND INSTALLING ZMAP
 
 ZMap requires GMP, a free library for arbitrary precision arithmetic,
-gengetopt, and libpcap.  These packages can be installed on
+gengetopt, and libpcap. These packages can be installed on
 Debian-based systems by running:
 
-  sudo apt-get install libgmp3-dev gengetopt libpcap-dev
+  sudo apt-get install libgmp3-dev gengetopt libpcap-dev flex byacc
 
 or on RHEL- and Fedora-based systems by running:
 
-  sudo yum install gmp gmp-devel gengetopt libpcap-devel
+  sudo yum install gmp gmp-devel gengetopt libpcap-devel flex byacc
 
 Once these prerequisites have been installed, ZMap can be installed
  by running:
 
-  cd src
+  cmake [-DWITH_REDIS=ON] [-DWITH_JSON=ON] [-DENABLE_DEVELOPMENT=ON] [-DENABLE_HARDENING=ON] ./
   make
 
 followed by:
 
   sudo make install
 
-Redis support is not enabled by default.  If you are want to use ZMap
+Redis support is not enabled by default. If you are want to use ZMap
-with Redis, you will first need to install Hiredis.  Then, rebuild
+with Redis, you will first need to install Hiredis. Then run cmake with
-ZMap with the command "make REDIS=true".
+"-DWITH_REDIS=ON".
-
-JSON support is not enabled by default.  If you are want to use ZMap
-with JSON output, you will first need to install json-c.  Then, rebuild
-ZMap with the command "make JSON=true".
 
+JSON support is not enabled by default. If you are want to use ZMap
+with JSON output, you will first need to install json-c. Then, run cmake with
+"-DWITH_JSON=ON"
+ 
 Installing json-c requires git and autotools to be available. For more
 information on how to install json-c, please see http://github.com/json-c/json-c

lib/blacklist.c

@@ -25,6 +25,10 @@
 
 static constraint_t *constraint = NULL;
 
+uint32_t blacklist_lookup_index(uint64_t index) {
+	return ntohl(constraint_lookup_index(constraint, index, ADDR_ALLOWED));
+}
+
 // check whether a single IP address is allowed to be scanned.
 //		1 => is allowed
 //		0 => is not allowed
@@ -47,7 +51,39 @@ void whitelist_prefix(char *ip, int prefix_len)
 	constraint_set(constraint, ntohl(inet_addr(ip)), prefix_len, ADDR_ALLOWED);
 }
 
-static int init(char *file, const char *name, int value)
+static int init_from_string(char *ip, int value)
+{
+	int prefix_len = 32;
+	char *slash = strchr(ip, '/');
+	if (slash) {  // split apart network and prefix length 
+		*slash = '\0';
+		char *end;
+		char *len = slash+1;
+		errno = 0;			
+		prefix_len = strtol(len, &end, 10);
+		if (end == len || errno != 0 || prefix_len < 0 || prefix_len > 32) {
+			log_fatal("constraint", "'%s' is not a valid prefix length", len);
+			return -1;
+		}
+	}
+	struct in_addr addr;
+	if (inet_aton(ip, &addr) == 0) {
+		log_error("constraint", "'%s' is not a valid IP address", ip);
+		return -1;
+	}
+	constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value);
+	const char *name;
+	if (value == ADDR_DISALLOWED)
+		name = "blacklisting";
+	else
+		name = "whitelisting";
+	log_trace(name, "%s %s/%i",
+			  name, ip, prefix_len);
+	return 0;
+}
+
+
+static int init_from_file(char *file, const char *name, int value)
 {
 	FILE *fp;
 	char line[1000];
@@ -66,31 +102,22 @@ static int init(char *file, const char *name, int value)
 		if ((sscanf(line, "%32s", ip)) == EOF) {
 			continue;
 		}
-		int prefix_len = 32;
+		if (init_from_string(ip, value)) {
-		char *slash = strchr(ip, '/');
+			log_fatal(name, "unable to parse %s file: %s", name, file);
-		if (slash) {  // split apart network and prefix length 
-			*slash = '\0';
-			char *end;
-			char *len = slash+1;
-			errno = 0;			
-			prefix_len = strtol(len, &end, 10);
-			if (end == len || errno != 0 || prefix_len < 0 || prefix_len > 32) {
-				log_fatal(name, "Unable to parse %s file: %s ('%s' is not a valid prefix length)",  name, file, len);
-			}
 		}
-		struct in_addr addr;
-		if (inet_aton(ip, &addr) == 0) {
-			log_fatal(name,  "Unable to parse %s file: %s ('%s' is not a valid IP address)",  name, file, ip);
-		}
-		constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value);
-		log_trace(name, "%sing %s/%i",
-				  name, ip, prefix_len);
 	}
 	fclose(fp);
 
 	return 0;
 }
 
+static void init_from_array(char **cidrs, size_t len, int value)
+{
+	for (int i=0; i < (int) len; i++) {
+		init_from_string(cidrs[i], value);
+	}
+}
+
 uint64_t blacklist_count_allowed()
 {
 	assert(constraint);
@@ -103,27 +130,43 @@ uint64_t blacklist_count_not_allowed()
 	return constraint_count_ips(constraint, ADDR_DISALLOWED);
 }
 
-
 // Initialize address constraints from whitelist and blacklist files.
 // Either can be set to NULL to omit.
-int blacklist_init_from_files(char *whitelist_filename, char *blacklist_filename)
+int blacklist_init(char *whitelist_filename, char *blacklist_filename,
+		char **whitelist_entries, size_t whitelist_entries_len,
+		char **blacklist_entries, size_t blacklist_entries_len)
 {
 	assert(!constraint);
-	if (whitelist_filename) {
+	if (whitelist_filename && whitelist_entries) {
+		log_warn("whitelist", "both a whitelist file and destination addresses "
+					"were specified. The union of these two sources "
+					"will be utilized.");
+	}
+	if (whitelist_filename || whitelist_entries) {
 		// using a whitelist, so default to allowing nothing
 		constraint = constraint_init(ADDR_DISALLOWED);
 		log_trace("whitelist", "blacklisting 0.0.0.0/0");
-		init(whitelist_filename, "whitelist", ADDR_ALLOWED);
+		if (whitelist_filename) {
+			init_from_file(whitelist_filename, "whitelist", ADDR_ALLOWED);
+		}
+		if (whitelist_entries) {
+			init_from_array(whitelist_entries,
+					whitelist_entries_len, ADDR_ALLOWED);
+		}
 	} else {
 		// no whitelist, so default to allowing everything
 		constraint = constraint_init(ADDR_ALLOWED);
 	}
 	if (blacklist_filename) {
-		init(blacklist_filename, "blacklist", ADDR_DISALLOWED);
+		init_from_file(blacklist_filename, "blacklist", ADDR_DISALLOWED);
 	}
-	constraint_optimize(constraint);
+	if (blacklist_entries) {
+		init_from_array(blacklist_entries, blacklist_entries_len, ADDR_DISALLOWED);
+	}
+	constraint_paint_value(constraint, ADDR_ALLOWED);
 	uint64_t allowed = blacklist_count_allowed();
 	log_debug("blacklist", "%lu addresses allowed to be scanned (%0.0f%% of address space)", 
 			  allowed, allowed*100./((long long int)1 << 32));
-	return 0;
+	return EXIT_SUCCESS;
 }
+

lib/blacklist.h

@@ -1,12 +1,18 @@
+#include <stdlib.h>
 #include <stdint.h>
 
 #ifndef BLACKLIST_H
 #define BLACKLIST_H
 
+uint32_t blacklist_lookup_index(uint64_t index);
 int blacklist_is_allowed(uint32_t s_addr);
 void blacklist_prefix(char *ip, int prefix_len);
 void whitelist_prefix(char *ip, int prefix_len);
-int blacklist_init_from_files(char *whitelist, char*blacklist);
+int blacklist_init(char *whitelist, char *blacklist,
+		char **whitelist_entries,
+		size_t whitelist_entries_len,
+		char **blacklist_entries,
+		size_t blacklist_entries_len);
 uint64_t blacklist_count_allowed();
 uint64_t blacklist_count_not_allowed();
 

lib/constraint.c

@@ -50,16 +50,19 @@ typedef struct node {
 	struct node *l;
 	struct node *r;
 	value_t value;
+	uint64_t count;
 } node_t;
 
 // As an optimization, we precompute lookups for every prefix of this
 // length:
-#define RADIX_LENGTH 16
+#define RADIX_LENGTH 20
 
 struct _constraint {
-	node_t *root;   // root node of the tree
+	node_t *root;     // root node of the tree
-	node_t **radix; // array of nodes for every RADIX_LENGTH prefix
+	uint32_t *radix;  // array of prefixes (/RADIX_LENGTH) that are painted paint_value
-	int optimized;  // is radix populated and up-to-date?
+	size_t radix_len; // number of prefixes in radix array
+	int painted;      // have we precomputed counts for each node?
+	value_t paint_value; // value for which we precomputed counts
 };
 
 // Tree operations respect the invariant that every node that isn't a
@@ -150,7 +153,7 @@ void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value)
 {
 	assert(con);
 	_set_recurse(con->root, prefix, len, value);
-	con->optimized = 0;
+	con->painted = 0;
 }
 
 // Return the value pertaining to an address, according to the tree
@@ -176,67 +179,88 @@ static int _lookup_ip(node_t *root, uint32_t address)
 
 // Return the value pertaining to an address.
 // (Note: address must be in host byte order.)
-int constraint_lookup_ip(constraint_t *con, uint32_t address)
+value_t constraint_lookup_ip(constraint_t *con, uint32_t address)
 {
 	assert(con);
-	if (con->optimized) {
+	return _lookup_ip(con->root, address);
-		// Use radix optimization
+}
-		node_t *node = con->radix[address >> (32 - RADIX_LENGTH)];
+
+// Return the nth painted IP address.
+static int _lookup_index(node_t *root, uint64_t n)
+{
+	assert(root);
+	node_t *node = root;
+	uint32_t ip = 0;
+	uint32_t mask = 0x80000000;
+	for (;;) {
 		if (IS_LEAF(node)) {
-			return node->value;
+			return ip | n;
 		}
-		return _lookup_ip(node, address << RADIX_LENGTH);
+		if (n < node->l->count) {
-	} else {
+			node = node->l;
-		// Do a full lookup using the tree
+		} else {
-		log_trace("constraint", "Unoptimized lookup");
+			n -= node->l->count;
-		return _lookup_ip(con->root, address);
+			node = node->r;
+			ip |= mask;
+		}
+		mask >>= 1;
 	}
 }
 
+// For a given value, return the IP address with zero-based index n.
+// (i.e., if there are three addresses with value 0xFF, looking up index 1
+// will return the second one).
+// Note that the tree must have been previously painted with this value.
+uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, value_t value)
+{
+	assert(con);
+	if (!con->painted || con->paint_value != value) {
+		constraint_paint_value(con, value);
+	}
+
+	uint64_t radix_idx = index / (1 << (32 - RADIX_LENGTH));
+	if (radix_idx < con->radix_len) {
+		// Radix lookup
+		uint32_t radix_offset = index % (1 << (32 - RADIX_LENGTH));	// TODO: bitwise maths
+		return con->radix[radix_idx] | radix_offset;
+	}
+
+	// Otherwise, do the "slow" lookup in tree.
+	// Note that tree counts do NOT include things in the radix,
+	// so we subtract these off here.
+	index -= con->radix_len * (1 << (32 - RADIX_LENGTH));
+	assert(index < con->root->count);
+	return _lookup_index(con->root, index);
+}
+
 // Implement count_ips by recursing on halves of the tree.  Size represents
 // the number of addresses in a prefix at the current level of the tree.
-static uint64_t _count_ips_recurse(node_t *node, value_t value, uint64_t size)
+// If paint is specified, each node will have its count set to the number of
+// leaves under it set to value.
+// If exclude_radix is specified, the number of addresses will exlcude prefixes
+// that are a /RADIX_LENGTH or larger
+static uint64_t _count_ips_recurse(node_t *node, value_t value, uint64_t size, int paint, int exclude_radix)
 {
 	assert(node);
+	uint64_t n;
 	if (IS_LEAF(node)) {
 		if (node->value == value) {
-			return size;
+			n = size;
+			// Exclude prefixes already included in the radix
+			if (exclude_radix && size >= (1 << (32 -RADIX_LENGTH))) {
+				n = 0;
+			}
 		} else {
-			return 0;
+			n = 0;
 		}
+	} else {
+		n = _count_ips_recurse(node->l, value, size >> 1, paint, exclude_radix) +
+			_count_ips_recurse(node->r, value, size >> 1, paint, exclude_radix);
 	}
-	return _count_ips_recurse(node->l, value, size >> 1) +
+	if (paint) {
-		_count_ips_recurse(node->r, value, size >> 1);
+		node->count = n;
-}
+	}
-
+	return n;
-// Return the number of addresses that have a given value.
-uint64_t constraint_count_ips(constraint_t *con, value_t value)
-{
-	assert(con);
-	return _count_ips_recurse(con->root, value, (uint64_t)1 << 32);
-}
-
-// Initialize the tree.
-// All addresses will initally have the given value.
-constraint_t* constraint_init(value_t value)
-{
-	log_trace("constraint", "Initializing");
-	constraint_t* con = malloc(sizeof(constraint_t));
-	con->root = _create_leaf(value);
-	con->radix = calloc(sizeof(node_t *), 1 << RADIX_LENGTH);
-	assert(con->radix);
-	con->optimized = 0;
-	return con;
-}
-
-// Deinitialize and free the tree.
-void constraint_free(constraint_t *con)
-{
-	assert(con);
-	log_trace("constraint", "Cleaning up");
-	_destroy_subtree(con->root);
-	free(con->radix);
-	free(con);
 }
 
 // Return a node that determines the values for the addresses with
@@ -250,8 +274,9 @@ static node_t* _lookup_node(node_t *root, uint32_t prefix, int len)
 
 	node_t *node = root;
 	uint32_t mask = 0x80000000;
+	int i;
 
-	for (int i=0; i < len; i++) {
+	for (i=0; i < len; i++) {
 		if (IS_LEAF(node)) {
 			return node;
 		}
@@ -265,21 +290,66 @@ static node_t* _lookup_node(node_t *root, uint32_t prefix, int len)
 	return node;
 }
 
-// After values have been set, precompute prefix lookups.
+// For each node, precompute the count of leaves beneath it set to value.
-void constraint_optimize(constraint_t *con)
+// Note that the tree can be painted for only one value at a time.
+void constraint_paint_value(constraint_t *con, value_t value)
 {
 	assert(con);
-	if (con->optimized) {
+	log_trace("constraint", "Painting value %lu", value);
-		return;
+
-	}
+	// Paint everything except what we will put in radix
-	log_trace("constraint", "Optimizing constraints");
+	_count_ips_recurse(con->root, value, (uint64_t)1 << 32, 1, 1);
-	for (uint32_t i=0; i < (1 << RADIX_LENGTH); i++) {
+
+	// Fill in the radix array with a list of addresses
+	uint32_t i;
+	con->radix_len = 0;
+	for (i=0; i < (1 << RADIX_LENGTH); i++) {
 		uint32_t prefix = i << (32 - RADIX_LENGTH);
-		con->radix[i] = _lookup_node(con->root, prefix, RADIX_LENGTH);
+		node_t *node = _lookup_node(con->root, prefix, RADIX_LENGTH);
+		if (IS_LEAF(node) && node->value == value) {
+			// Add this prefix to the radix
+			con->radix[con->radix_len++] = prefix;
+		}
 	}
-	con->optimized = 1;
+	log_debug("constraint", "%lu IPs in radix array, %lu IPs in tree",
+			con->radix_len * (1 << (32 - RADIX_LENGTH)), con->root->count);
+	con->painted = 1;
+	con->paint_value = value;
 }
 
+// Return the number of addresses that have a given value.
+uint64_t constraint_count_ips(constraint_t *con, value_t value)
+{
+	assert(con);
+	if (con->painted && con->paint_value == value) {
+		return con->root->count + con->radix_len * (1 << (32 - RADIX_LENGTH));
+	} else {
+		return _count_ips_recurse(con->root, value, (uint64_t)1 << 32, 0, 0);
+	}
+}
+
+// Initialize the tree.
+// All addresses will initally have the given value.
+constraint_t* constraint_init(value_t value)
+{
+	log_trace("constraint", "Initializing");
+	constraint_t* con = malloc(sizeof(constraint_t));
+	con->root = _create_leaf(value);
+	con->radix = calloc(sizeof(uint32_t), 1 << RADIX_LENGTH);
+	assert(con->radix);
+	con->painted = 0;
+	return con;
+}
+
+// Deinitialize and free the tree.
+void constraint_free(constraint_t *con)
+{
+	assert(con);
+	log_trace("constraint", "Cleaning up");
+	_destroy_subtree(con->root);
+	free(con->radix);
+	free(con);
+}
 
 /*
 int main(void)
@@ -317,68 +387,3 @@ int main(void)
 }
 */
 
-/*
-static int init(constraint_t *con, char *file, const char *name, value_t value)
-{
-	FILE *fp;
-	char line[1000];
-	int blocked = 0;
-
-	fp = fopen(file, "r");
-	if (fp == NULL) {
-		log_fatal(name, "Unable to open %s file: %s: %s", 
-				name, file, strerror(errno));
-	}
-
-	while (fgets(line, sizeof(line), fp) != NULL) {
-		char *comment = strchr(line, '#');
-		if (comment) {
-			*comment = '\0';
-		}
-		char ip[33];
-		if ((sscanf(line, "%32s", ip)) == EOF) {
-			continue;
-		}
-		int prefix_len;
-		char *slash = strchr(ip, '/');
-		if (slash == NULL) {
-			log_fatal(name,
-		"Unable to parse %s file: %s",
-		name, file);
-		}
-		// split apart network and prefix length 
-		*slash = '\0';
-		prefix_len = atoi(&slash[1]);
-		constraint_set(con, ntohl(inet_addr(ip)), prefix_len, value);
-
-		blocked++;
-	}
-	fclose(fp);
-	return 0;
-}
-
-
-
-void main()
-{
-	log_init(stderr, LOG_TRACE);
-
-	constraint_t *con = constraint_init(1);
-	init(con, "blacklist.prefixes", "blacklist", 0);
-	//constraint_optimize(con);
-
-	printf("count(0)=%lu\n", constraint_count_ips(con, 0));
-	printf("count(1)=%lu\n", constraint_count_ips(con, 1));
-
-	uint32_t i=0, count=0;
-	do {
-		if (constraint_lookup_ip(con, i))
-			count++;
-	} while (++i != 0);
-	printf("derived count(1)=%u\n", count);
-
-	constraint_free(con);
-
-}
-
- */

lib/constraint.h

@@ -1,14 +1,17 @@
 #ifndef CONSTRAINT_H
 #define CONSTRAINT_H
 
+#include <stdint.h>
+
 typedef struct _constraint constraint_t;
-typedef int value_t;
+typedef unsigned int value_t;
 
 constraint_t* constraint_init(value_t value);
 void constraint_free(constraint_t *con);
 void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value);
-void constraint_optimize(constraint_t *con);
+value_t constraint_lookup_ip(constraint_t *con, uint32_t address);
-int constraint_lookup_ip(constraint_t *con, uint32_t address);
 uint64_t constraint_count_ips(constraint_t *con, value_t value);
+uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, value_t value);
+void constraint_paint_value(constraint_t *con, value_t value);
 
 #endif //_CONSTRAINT_H

lib/pbm.c

@@ -0,0 +1,54 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "logger.h"
+
+#define NUM_VALUES 0xFFFFFFFF
+#define PAGE_SIZE 0xFFFF
+#define NUM_PAGES (NUM_VALUES / PAGE_SIZE)
+#define SIZE (8*sizeof(uint64_t))
+
+uint64_t** pbm_init(void)
+{
+	uint64_t** retv = calloc(NUM_PAGES, sizeof(void*));
+	if (!retv) {
+		log_fatal("pbm", "unable to allocate memory for base page table");
+	}
+	return retv;
+}
+
+static int bm_check(uint64_t *bm, uint32_t v)
+{
+	return bm[v/SIZE] & (1<< (v % SIZE));
+}
+
+static void bm_set(uint64_t *bm, uint32_t v) 
+{
+	bm[v/SIZE] |= (1 << (v % SIZE));
+}
+
+int pbm_check(uint64_t **b, uint32_t v)
+{
+	uint32_t top = v >> 16;
+	uint32_t bottom = v & PAGE_SIZE;
+	return b[top] && bm_check(b[top], bottom);
+}
+
+void pbm_set(uint64_t **b, uint32_t v)
+{
+	uint32_t top = v >> 16;
+	uint32_t bottom = v & PAGE_SIZE;
+	if (!b[top]) {
+		uint64_t *bm = malloc(PAGE_SIZE);
+		if (!bm) {
+			log_fatal("bpm", "unable to allocate memory for new bitmap page");
+		}
+		memset(bm, 0, PAGE_SIZE);
+		b[top] = bm;
+	}
+	bm_set(b[top], bottom);
+}
+

lib/pbm.h

@@ -0,0 +1,5 @@
+#include <stdint.h>
+
+uint64_t** pbm_init(void);
+int pbm_check(uint64_t **b, uint32_t v);
+void pbm_set(uint64_t **b, uint32_t v);

lib/redis.c

@@ -6,16 +6,18 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
  
-#include <string.h>
 #include "redis.h"
+
+#include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include "assert.h"
-#include "logger.h"
 #include <stdint.h>
+#include <assert.h>
+
 #include <hiredis/hiredis.h>
 
-#define REDIS_UNIX_PATH "/tmp/redis.sock"
+#include "logger.h"
+
 #define REDIS_TIMEOUT 2
 
 #undef  MIN
@@ -23,13 +25,74 @@
 
 static redisContext *rctx;
 
-static redisContext* redis_connect(void)
+redisconf_t *redis_parse_connstr(char *connstr)
 {
+	redisconf_t *retv = malloc(sizeof(redisconf_t));
+	if (!strncmp("tcp://", connstr, 6)) {
+		char *servername = malloc(strlen(connstr));
+		assert(servername);
+		char *list_name = malloc(strlen(connstr));
+		assert(list_name);
+		uint32_t port;
+		if (sscanf(connstr, "tcp://%[^:]:%u/%s", servername,
+						&port, list_name) != 3) {
+			log_fatal("redis", "unable to parse redis connection string. This "
+					"should be of the form tcp://server:port/list-name "
+					"for TCP connections. All fields are required.");
+		}
+		retv->type = T_TCP;
+		retv->server = servername;
+		retv->port = port;
+		retv->list_name = list_name;
+		retv->path = NULL;
+	} else if (!strncmp("local://", connstr, 8)) {
+		// looking for something along the lines of
+		// local:///tmp/redis.sock/list-name
+		char *path = malloc(strlen(connstr));
+		assert(path);
+		char *list_name = malloc(strlen(connstr));
+		assert(list_name);
+		connstr = connstr + (size_t) 8;
+		char *listname = strrchr(connstr, '/') + (size_t) 1;
+		connstr[strrchr(connstr, '/') - connstr] = '\0';
+		strcpy(path, connstr);
+		strcpy(list_name, listname);
+		retv->type = T_LOCAL;
+		retv->list_name = list_name;
+		retv->path = path;
+		retv->server = NULL;
+		retv->port = 0;
+	} else {
+		log_fatal("redis", "unable to parse connection string. does not begin with "
+			"local:// or tcp:// as expected");
+	}
+	return retv;
+}
+
+static redisContext* redis_connect(char *connstr)
+{
+	redisconf_t *c;
+	// handle old behavior where we only connected to a specific
+	// socket that we #defined.
+	if (!connstr) {
+		c = malloc(sizeof(redisconf_t));
+		assert(c);
+		c->type = T_LOCAL;
+		c->path = "/tmp/redis.sock";
+	} else {
+		c = redis_parse_connstr(connstr);
+		assert(c);
+	}
 	struct timeval timeout;
 	timeout.tv_sec = REDIS_TIMEOUT;
 	timeout.tv_usec = 0;
-	return (redisContext*) redisConnectUnixWithTimeout(REDIS_UNIX_PATH,
+	if (c->type == T_LOCAL) {
+		return (redisContext*) redisConnectUnixWithTimeout(c->path,
 			timeout);
+	} else {
+		return (redisContext*) redisConnectWithTimeout(c->server,
+			c->port, timeout);
+	}
 }
 
 static int chkerr(redisReply *reply)
@@ -47,9 +110,9 @@ static int chkerr(redisReply *reply)
 	return 0;
 }
 
-int redis_init(void)
+int redis_init(char *connstr)
 {
-	rctx = redis_connect();
+	rctx = redis_connect(connstr);
 	if (!rctx) {
 		return -1;
 	}

lib/redis.h

@@ -5,7 +5,20 @@
 #ifndef REDIS_ZHELPERS_H
 #define REDIS_ZHELPERS_H
 
-int redis_init(void);
+#define T_TCP 0
+#define T_LOCAL 1
+
+typedef struct redisconf {
+	int type;
+	char *path;
+	char *server;
+	uint16_t port;
+	char *list_name;
+} redisconf_t;
+
+redisconf_t *redis_parse_connstr(char *connstr);
+
+int redis_init(char*);
 
 int redis_close(void);
 

lib/stack.c

@@ -0,0 +1,40 @@
+#include "stack.h"
+#include "../lib/xalloc.h"
+
+#include <stdlib.h>
+
+struct stack {
+	size_t max_size;
+	size_t cur_size;
+	void** arr;
+};
+
+stack_t* alloc_stack(size_t size)
+{
+	stack_t* stack = xmalloc(sizeof(stack_t));
+	stack->arr = xcalloc(size, sizeof(void*));
+	stack->max_size = size;
+	stack->cur_size = 0;
+	return stack;
+}
+
+void free_stack(stack_t* stack)
+{
+	xfree(stack->arr);
+	xfree(stack);
+}
+
+void push(stack_t* stack, void* elt)
+{
+	if (stack->cur_size == stack->max_size) {
+		stack->max_size *= 2;
+		xrealloc(stack->arr, stack->max_size);
+	}
+	stack->arr[stack->cur_size++] = elt;
+}
+
+void* pop(stack_t* stack)
+{
+	void* res = stack->arr[--stack->cur_size];
+	return res;
+}

lib/stack.h

@@ -0,0 +1,15 @@
+#ifndef ZMAP_STACK_H
+#define ZMAP_STACK_H
+
+#include <stddef.h>
+
+struct stack;
+typedef struct stack stack_t;
+
+stack_t* alloc_stack(size_t size);
+void free_stack(stack_t* stack);
+
+void push(stack_t* stack, void* elt);
+void* pop(stack_t* stack);
+
+#endif /* ZMAP_STACK_H */

lib/xalloc.c

@@ -0,0 +1,44 @@
+#include "xalloc.h"
+#include "../lib/logger.h"
+
+#include <stdlib.h>
+
+void die() __attribute__((noreturn));
+
+void* xcalloc(size_t count, size_t size)
+{
+	void* res = calloc(count, size);
+	if (res == NULL) {
+		die();
+	}
+	return res;
+}
+
+void xfree(void *ptr)
+{
+	free(ptr);
+}
+
+void* xmalloc(size_t size)
+{
+	void* res = malloc(size);
+	if (res == NULL) {
+		die();
+	}
+	return res;
+}
+
+void* xrealloc(void *ptr, size_t size)
+{
+	void* res = realloc(ptr, size);
+	if (res == NULL) {
+		die();
+	}
+	return res;
+}
+
+void die()
+{
+	log_fatal("zmap", "Out of memory");
+}
+

lib/xalloc.h

@@ -0,0 +1,14 @@
+#ifndef ZMAP_ALLOC_H
+#define ZMAP_ALLOC_H
+
+#include <stddef.h>
+
+void* xcalloc(size_t count, size_t size);
+
+void xfree(void *ptr);
+
+void* xmalloc(size_t size);
+
+void* xrealloc(void *ptr, size_t size);
+
+#endif

src/.gitignore

@@ -0,0 +1,4 @@
+lexer.c
+lexer.h
+parser.c
+parser.h

src/CMakeLists.txt

@@ -0,0 +1,107 @@
+include_directories(
+	"${CMAKE_CURRENT_BINARY_DIR}"
+	${PROJECT_SOURCE_DIR}/lib
+	${PROJECT_SOURCE_DIR}/src
+	${PROJECT_SOURCE_DIR}/src/output_modules
+	)
+
+SET(LIB_SOURCES
+	${PROJECT_SOURCE_DIR}/lib/blacklist.c
+	${PROJECT_SOURCE_DIR}/lib/constraint.c
+	${PROJECT_SOURCE_DIR}/lib/logger.c
+	${PROJECT_SOURCE_DIR}/lib/pbm.c
+	${PROJECT_SOURCE_DIR}/lib/random.c
+	${PROJECT_SOURCE_DIR}/lib/rijndael-alg-fst.c
+	${PROJECT_SOURCE_DIR}/lib/stack.c
+	${PROJECT_SOURCE_DIR}/lib/xalloc.c
+)
+
+# ADD YOUR PROBE MODULE HERE
+SET(EXTRA_PROBE_MODULES
+
+	)
+
+# ADD YOUR OUTPUT MODULE HERE
+SET(EXTRA_OUTPUT_MODULES
+
+	)
+
+SET(OUTPUT_MODULE_SOURCES
+	output_modules/module_csv.c
+	output_modules/output_modules.c
+	)
+
+SET(PROBE_MODULE_SOURCES
+	probe_modules/module_icmp_echo.c
+	probe_modules/module_tcp_synscan.c
+	probe_modules/module_udp.c
+	probe_modules/packet.c
+	probe_modules/probe_modules.c
+	)
+
+SET(SOURCES
+	aesrand.c
+	cyclic.c
+	expression.c
+	fieldset.c
+	filter.c
+	get_gateway.c
+	monitor.c
+	recv.c
+	send.c
+	state.c
+	validate.c
+	zmap.c
+	zopt_compat.c
+	"${CMAKE_CURRENT_BINARY_DIR}/zopt.h"
+	"${CMAKE_CURRENT_BINARY_DIR}/lexer.c"
+	"${CMAKE_CURRENT_BINARY_DIR}/parser.c"
+	${EXTRA_PROBE_MODULES}
+	${EXTRA_OUTPUT_MODULES}
+	${PROBE_MODULE_SOURCES}
+	${OUTPUT_MODULE_SOURCES}
+	${LIB_SOURCES}
+	)
+
+if (WITH_JSON)
+	SET(SOURCES ${SOURCES} output_modules/module_json.c)
+endif()
+
+if (WITH_REDIS)
+	SET(SOURCES ${SOURCES} ${PROJECT_SOURCE_DIR}/lib/redis.c output_modules/module_redis.c)
+endif()
+
+add_custom_command(OUTPUT zopt.h
+	COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_SOURCE_DIR}/zopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zopt"
+	)
+
+add_custom_command(OUTPUT lexer.c
+	COMMAND flex -o "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" --header-file="${CMAKE_CURRENT_BINARY_DIR}/lexer.h" lexer.l
+	)
+
+add_custom_command(OUTPUT parser.c
+	COMMAND byacc -d -o parser.c parser.y
+	)
+
+add_executable(zmap ${SOURCES})
+
+target_link_libraries(
+	zmap 
+	pcap gmp m
+	${REDIS_LIBS}
+	${JSON_LIBRARIES}
+	)
+
+# Install binary
+install(
+	TARGETS
+	zmap
+	RUNTIME DESTINATION sbin
+)
+
+# Install Manpages
+install(
+	FILES
+	zmap.1
+	DESTINATION share/man/man1
+	)

src/Makefile

@@ -1,116 +0,0 @@
-#<<<<<<< HEAD
-CC=gcc
-#CC=clang
-
-#CFLAGS=-Wall -pedantic -Wextra -std=gnu99 -I../lib -I./ -Ioutput_modules -O2 -g
-#wbk makefile hack to get rolling with C port. Fix later lol 
-CFLAGS=-Wall -pedantic -Wextra -std=gnu99 -I../lib -I./ -Ioutput_modules \
--O2 -g -D__FREEBSD__ -D__FREEBSD_INCLUDES__ -D__FREEBSD_PTHREADS__ -DZMAP_PCAP_INJECT
-LDFLAGS=-g -pthread
-LDLIBS= -lpcap -lgmp -lm
-#=======
-#INCLUDE+=-I../lib -I./ -Ioutput_modules
-#LDFLAGS+=-pthread
-#LDLIBS+=-lpcap -lgmp -lm
-#>>>>>>> 5cd6f3294cb4f6ddf711b4c42d72893989a9dc3d
-TARGETS=zmap
-VPATH=../lib:output_modules:probe_modules
-PREFIX=/usr/local
-
-#wbk
-LIBDIR=/usr/local/lib
-
-INSTALL=install
-INSTALLDATA=install -m 644
-mandir=$(PREFIX)/man/man1/
-oldmanfile=/usr/share/man/man1/zmap.1
-bindir=$(PREFIX)/sbin
-
-# Hardening and warnings for building with gcc
-#<<<<<<< HEAD
-#M aybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations
-#-Wold-style-definition -Wswitch-enum 
-GCCWARNINGS = -Wall -fno-strict-aliasing -W -Wfloat-equal -Wundef      \
--Wpointer-arith														   \
--Wwrite-strings -Wredundant-decls -Wchar-subscripts -Wcomment          \
--Wformat=2 -Wwrite-strings -Wredundant-decls  \
--Wnested-externs -Wbad-function-cast -Winit-self         \
--Wmissing-field-initializers										   \
--Waddress -Wmissing-noreturn -Wnormalized=id    \
--Woverride-init -Wstrict-overflow=1 -Wextra             \
--Wstack-protector -Wformat -Wformat-security -Wpointer-sign -Wno-format-nonliteral -Wno-format-y2k
-GCCHARDENING=-D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1
-# clang doesn't like this
-#LDHARDENING=-z relro -z now
-
-EXTRACFLAGS=-g -O2 $(EXTRA_CFLAGS) $(GCCHARDENING) $(GCCWARNINGS) 
-#=======
-# Maybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations
-#GCCWARNINGS = -Wall -Wformat=2 -Wno-format-nonliteral\
-#-pedantic -fno-strict-aliasing \
-#-Wextra \
-#-Wfloat-equal -Wundef -Wwrite-strings -Wredundant-decls  \
-#-Wnested-externs -Wbad-function-cast -Winit-self         \
-#-Wmissing-noreturn -Wnormalized=id    \
-#-Wstack-protector
-
-#GCCHARDENING=-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1
-#LDHARDENING=-z relro -z now
-
-#EXTRACFLAGS=-std=gnu99 -g -O2 $(GCCHARDENING) $(GCCWARNINGS) $(EXTRA_CFLAGS) -Werror
-#>>>>>>> 5cd6f3294cb4f6ddf711b4c42d72893989a9dc3d
-EXTRALDFLAGS= $(LDHARDENING)
-
-CFLAGS+=$(INCLUDE) $(EXTRACFLAGS)
-LDFLAGS+=$(EXTRALDFLAGS)
-LDFLAGS+=-L$(LIBDIR)
-
-probemodules=module_tcp_synscan.o module_icmp_echo.o module_udp.o #ADD YOUR PROBE MODULE HERE
-outputmodules= module_csv.o #ADD YOUR OUTPUT MODULE HERE
-
-objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt_compat.o zmap.o random.o output_modules.o packet.o probe_modules.o ${probemodules} ${outputmodules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o fieldset.o
-redis_objects=module_redis.o redis.o
-
-ifeq ($(REDIS), true)
-	LDLIBS+=-lhiredis
-	objects+=$(redis_objects)
-	CFLAGS+=-DREDIS
-endif
-
-ifeq ($(JSON), true)
-	LDLIBS+=$(shell pkg-config --libs json-c)
-	CFLAGS+=-DJSON $(shell pkg-config --cflags json-c)
-	objects+=module_json.o
-endif
-
-all: $(TARGETS)
-
-$(TARGETS):
-	$(CC) $(CFLAGS) $(DFLAGS) $(LDFLAGS) $^ -o $@ $(LDLIBS) 
-
-zmap: $(objects) 
-
-zopt_compat.o: zopt.c
-
-zopt.c zopt.h: zopt.ggo
-	gengetopt -C --no-help --no-version -i $^ -F $*
-
-install: zmap
-	$(INSTALL) zmap $(bindir)/zmap
-	test -d /etc/zmap || (mkdir /etc/zmap && $(INSTALLDATA) ../conf/* /etc/zmap/)
-	test -f $(oldmanfile) && rm -f $(oldmanfile) && mandb -f $(oldmanfile) || /bin/true # remove old man page if it's there
-	test -d $(mandir) || mkdir -p $(mandir)
-	$(INSTALLDATA) ./zmap.1 $(mandir)
-	@echo "\n**************\nSuccess! ZMap is installed. Try running (as root):\nzmap -p 80 -N 10 -B 1M -o -\n**************"
-
-uninstall:
-	test -f $(oldmanfile) && rm -f $(oldmanfile) && mandb -f $(oldmanfile) || /bin/true # remove old man page if it's there
-	test -f $(mandir)/zmap.1 && rm -f $(mandir)/zmap.1 && mandb -f $(mandir)/zmap.1 || /bin/true # remove current man page if it's there
-	rm -f $(bindir)/zmap
-
-
-clean:
-	-rm -f $(objects) $(redis_objects) $(TARGETS)
-
-.PHONY: install clean
-

src/cyclic.c

@@ -61,28 +61,65 @@
 #include "aesrand.h"
 
 #define LSRC "cyclic"
-#define PRIME 4294967311 // 2^32 + 15
-#define KNOWN_PRIMROOT 3
 
-// distinct prime factors of 2^32 + 15
+typedef struct cyclic_group {
-static const uint64_t psub1_f[] = { 2, 3, 5, 131, 364289 }; 
+	uint64_t prime;
+	uint64_t known_primroot;
+	size_t num_prime_factors;	// number of unique prime factors of (prime-1)
+	uint64_t prime_factors[10];	// unique prime factors of (prime-1)
+} cyclic_group_t;
 
-// selected primitive root that we'll use as the generator
+// We will pick the first cyclic group from this list that is
+// larger than the number of IPs in our whitelist. E.g. for an
+// entire Internet scan, this would be cyclic32
+// Note: this list should remain ordered by size (primes) ascending.
+static cyclic_group_t groups[] = {
+{ // 2^16 + 1
+	.prime = 65537,
+	.known_primroot = 3,
+	.prime_factors = {2},
+	.num_prime_factors = 1
+},
+{ // 2^24 + 43
+	.prime = 16777259,
+	.known_primroot = 2,
+	.prime_factors = {2, 23, 103, 3541},
+	.num_prime_factors = 4
+},
+{ // 2^28 + 3
+	.prime = 268435459,
+	.known_primroot = 2,
+	.prime_factors = {2, 3, 19, 87211},
+	.num_prime_factors = 4
+},
+{ // 2^32 + 15
+	.prime = 4294967311,
+	.known_primroot = 3,
+	.prime_factors = {2, 3, 5, 131, 364289},
+	.num_prime_factors = 5
+}
+};
+
+
+// selected prime/primitive root that we'll use as the generator
+static uint64_t prime = 0;
 static uint64_t primroot = 0;
 static uint64_t current = 0;
 
+static uint64_t num_addrs = 0;
+
 #define COPRIME 1
-#define NOT_COPRIME 0 
+#define NOT_COPRIME 0
 
 // check whether two integers are coprime
-static int check_coprime(uint64_t check)
+static int check_coprime(uint64_t check, const cyclic_group_t *group)
 {
-	for (unsigned i=0; i < sizeof(psub1_f)/sizeof(psub1_f[0]); i++) {
+	for (unsigned i=0; i < group->num_prime_factors; i++) {
-		if (psub1_f[i] > check && !(psub1_f[i] % check)) {
+		if (group->prime_factors[i] > check && !(group->prime_factors[i] % check)) {
 			return NOT_COPRIME;
-		} else if (psub1_f[i] < check && !(check % psub1_f[i])) {
+		} else if (group->prime_factors[i] < check && !(check % group->prime_factors[i])) {
 			return NOT_COPRIME;
-		} else if (psub1_f[i] == check) {
+		} else if (group->prime_factors[i] == check) {
 			return NOT_COPRIME;
 		}
 	}
@@ -90,18 +127,18 @@ static int check_coprime(uint64_t check)
 }
 
 // find gen of cyclic group Z modulo PRIME
-static uint64_t find_primroot(void)
+static uint64_t find_primroot(const cyclic_group_t *group)
 {
 	// what luck, rand() returns a uint32_t!
-	uint32_t candidate = (uint32_t) aesrand_getword() & 0xFFFF;
+	uint32_t candidate = (uint32_t) aesrand_getword() & 0xFFFFFFFF;
-	while(check_coprime(candidate) != COPRIME) {
+	while(check_coprime(candidate, group) != COPRIME) {
 		++candidate;
 	}
 	// pre-modded result is gigantic so use GMP
 	mpz_t base, power, prime, primroot;
-	mpz_init_set_d(base, (double) KNOWN_PRIMROOT);
+	mpz_init_set_d(base, (double) group->known_primroot);
 	mpz_init_set_d(power, (double) candidate);
-	mpz_init_set_d(prime, (double) PRIME);
+	mpz_init_set_d(prime, (double) group->prime);
 	mpz_init(primroot);
 	mpz_powm(primroot, base, power, prime);
 	uint64_t retv = (uint64_t) mpz_get_ui(primroot);
@@ -115,6 +152,33 @@ static uint64_t find_primroot(void)
 int cyclic_init(uint32_t primroot_, uint32_t current_)
 {
 	assert(!(!primroot_ && current_));
+	// Initialize blacklist
+	if (blacklist_init(zconf.whitelist_filename, zconf.blacklist_filename,
+			zconf.destination_cidrs, zconf.destination_cidrs_len,
+			NULL, 0)) {
+		return -1;
+	}
+	num_addrs = blacklist_count_allowed();
+	if (!num_addrs) {
+		log_error("blacklist", "no addresses are eligible to be scanned in the "
+				"current configuration. This may be because the "
+				"blacklist being used by ZMap (%s) prevents "
+				"any addresses from receiving probe packets.",
+				zconf.blacklist_filename
+			);
+		exit(EXIT_FAILURE);
+	}
+
+	const cyclic_group_t *cur_group = NULL;
+	for (uint32_t i=0; i<sizeof(groups)/sizeof(groups[0]); i++) {
+		if (groups[i].prime > num_addrs) {
+			cur_group = &groups[i];
+			log_debug("cyclic", "using prime %lu, known_primroot %lu",
+					cur_group->prime, cur_group->known_primroot);
+			prime = groups[i].prime;
+			break;
+		}
+	}
 
 	if (zconf.use_seed) {
 		aesrand_init(zconf.seed+1);
@@ -123,17 +187,17 @@ int cyclic_init(uint32_t primroot_, uint32_t current_)
 	}
 	if (!primroot_) {
 		do {
-			primroot = find_primroot();
+			primroot = find_primroot(cur_group);
 		} while (primroot >= (1LL << 32));
 		log_debug(LSRC, "primitive root: %lld", primroot);
-		current = (uint32_t) aesrand_getword() & 0xFFFF;
+		current = (uint32_t) aesrand_getword() & 0xFFFFFFFF;
 		log_debug(LSRC, "starting point: %lld", current);
 	} else {
 		primroot = primroot_;
 		log_debug(LSRC, "primitive root %lld specified by caller",
 				primroot);
 		if (!current_) {
-			current = (uint32_t) aesrand_getword() & 0xFFFF;
+			current = (uint32_t) aesrand_getword() & 0xFFFFFFFF;
 			log_debug(LSRC, "no cyclic starting point, "
 					 "selected random startpoint: %lld",
 					 current);
@@ -144,11 +208,6 @@ int cyclic_init(uint32_t primroot_, uint32_t current_)
 		}
 	}
 	zconf.generator = primroot;
-	if (blacklist_init_from_files(zconf.whitelist_filename,
-							zconf.blacklist_filename)) {
-	  return -1;
-	}
-
 	// make sure current is an allowed ip
 	cyclic_get_next_ip();
 
@@ -157,7 +216,7 @@ int cyclic_init(uint32_t primroot_, uint32_t current_)
 
 uint32_t cyclic_get_curr_ip(void)
 {
-	return (uint32_t) current;
+	return (uint32_t) blacklist_lookup_index(current-1);
 }
 
 uint32_t cyclic_get_primroot(void)
@@ -169,7 +228,7 @@ static inline uint32_t cyclic_get_next_elem(void)
 {
 	do {
 		current *= primroot;
-		current %= PRIME;
+		current %= prime;
 	} while (current >= (1LL << 32));
 	return (uint32_t) current;
 }
@@ -178,11 +237,10 @@ uint32_t cyclic_get_next_ip(void)
 {
 	while (1) {
 		uint32_t candidate = cyclic_get_next_elem();
-		if (!blacklist_is_allowed(candidate)) {
+		if (candidate-1 < num_addrs) {
-			zsend.blacklisted++;
+			return blacklist_lookup_index(candidate-1);
-		} else {
-			return candidate;
 		}
+		zsend.blacklisted++;
 	}
 }
 

src/expression.c

@@ -0,0 +1,162 @@
+#include "expression.h"
+#include "fieldset.h"
+
+#include "../lib/xalloc.h"
+
+/* Static helper functions */
+
+static node_t* alloc_node();
+static int eval_gt_node(node_t *node, fieldset_t *fields);
+static int eval_lt_node(node_t *node, fieldset_t *fields);
+static int eval_eq_node(node_t *node, fieldset_t *fields);
+static int eval_lt_eq_node(node_t *node, fieldset_t *fields);
+static int eval_gt_eq_node(node_t *node, fieldset_t *fields);
+
+
+static node_t* alloc_node() 
+{
+	node_t *node = xmalloc(sizeof(node_t));
+	memset(node, 0, sizeof(node_t));
+	return node;
+}
+
+static int eval_gt_node(node_t *node, fieldset_t *fields)
+{
+	int index = node->left_child->value.field.index;
+	uint64_t expected = node->right_child->value.int_literal;
+	uint64_t actual = fs_get_uint64_by_index(fields, index);
+	return (actual > expected);
+}
+
+static int eval_lt_node(node_t *node, fieldset_t *fields)
+{
+	int index = node->left_child->value.field.index;
+	uint64_t expected = node->right_child->value.int_literal;
+	uint64_t actual = fs_get_uint64_by_index(fields, index);
+	return (actual < expected);
+}
+
+static int eval_eq_node(node_t *node, fieldset_t *fields)
+{
+	node_t *literal = node->right_child;
+	int index = node->left_child->value.field.index;
+	char *expected, *actual;
+	switch (literal->type) {
+		case STRING:
+			expected = literal->value.string_literal;
+			actual = fs_get_string_by_index(fields, index);
+			return (strcmp(expected, actual) == 0);
+			break;
+		case INT:
+			return (fs_get_uint64_by_index(fields, index) == literal->value.int_literal);
+			break;
+		default:
+			printf("wat\n");
+			break;
+	}
+	return 0;
+}
+
+static int eval_lt_eq_node(node_t *node, fieldset_t *fields)
+{
+	return !(eval_gt_node(node, fields));
+}
+
+static int eval_gt_eq_node(node_t *node, fieldset_t *fields)
+{
+	return !(eval_lt_node(node, fields));
+}
+
+
+/* Exposed functions */
+
+node_t* make_op_node(enum operation op) 
+{
+	node_t* node = alloc_node();
+	node->type = OP;
+	node->value.op = op;
+	return node;
+}
+
+node_t* make_field_node(char *fieldname) 
+{
+	node_t *node = alloc_node();
+	node->type = FIELD;
+	node->value.field.fieldname = fieldname;
+	return node;
+}
+
+node_t* make_string_node(char *literal)
+{
+	node_t *node = alloc_node();
+	node->type = STRING;
+	node->value.string_literal = literal;
+	return node;
+}
+
+node_t* make_int_node(int literal)
+{
+	node_t *node = alloc_node();
+	node->type = INT;
+	node->value.int_literal = literal;
+	return node;
+}
+
+int evaluate_expression(node_t *root, fieldset_t *fields)
+{
+	if (!root) return 1;
+	switch (root->type) { /* XXX Not sure if runs */
+	case FIELD:
+	case STRING:
+	case INT:
+		return 1;
+	case OP:
+		break;
+	}
+	switch (root->value.op) {
+	case GT:
+		return eval_gt_node(root, fields);
+	case LT:
+		return eval_lt_node(root, fields);
+	case EQ:
+		return eval_eq_node(root, fields);
+	case NEQ:
+		return (!eval_eq_node(root, fields));
+	case LT_EQ:
+		return eval_lt_eq_node(root, fields);
+	case GT_EQ:
+		return eval_gt_eq_node(root, fields);
+	case AND:
+		return (evaluate_expression(root->left_child, fields) 
+			&& evaluate_expression(root->right_child, fields));
+	case OR:
+		return (evaluate_expression(root->left_child, fields) 
+			|| evaluate_expression(root->right_child, fields));	
+	}
+	return 0;
+}
+
+void print_expression(node_t *root)
+{
+	if (!root) return;
+	printf("%s", "( ");
+	print_expression(root->left_child);
+	switch (root->type) {
+		case OP:
+			printf(" %i ", root->value.op);
+			break;
+		case FIELD:
+			printf(" (%s", root->value.field.fieldname);
+			break;
+		case STRING:
+			printf("%s) ", root->value.string_literal);
+			break;
+		case INT:
+			printf(" %llu) ", (long long unsigned) root->value.int_literal);
+			break;
+		default:
+			break;
+	}
+	print_expression(root->right_child);
+	printf("%s", " )");
+}

src/expression.h

@@ -0,0 +1,50 @@
+#ifndef ZMAP_TREE_H
+#define ZMAP_TREE_H
+
+#include "fieldset.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+enum operation {
+	GT, LT, EQ, NEQ, AND, OR, LT_EQ, GT_EQ
+};
+
+enum node_type {
+	OP, FIELD, STRING, INT
+};
+
+struct field_id {
+	int index;
+	char *fieldname;
+};
+
+union node_value {
+	struct field_id field; 
+	char *string_literal;
+	uint64_t int_literal;
+	enum operation op;
+};
+
+typedef struct node {
+	struct node *left_child;
+	struct node *right_child;
+	enum node_type type;
+	union node_value value;
+} node_t;
+
+node_t* make_op_node(enum operation op);
+
+node_t* make_field_node(char *fieldname);
+
+node_t* make_string_node(char *literal);
+
+node_t* make_int_node(int literal);
+
+int evaluate_expression(node_t *root, fieldset_t *fields);
+
+void print_expression(node_t *root);
+
+#endif /* ZMAP_TREE_H */

src/fieldset.c

@@ -32,13 +32,13 @@ fieldset_t *fs_new_fieldset(void)
 		log_fatal("fieldset", "unable to allocate new fieldset");
 	}
 	memset(f, 0, sizeof(fieldset_t));
-	return f;	
+	return f;
 }
 
 /* wbk TODO: Might have some subtle issues here on 32 bit architectures. Getting
    compiler warnings about casting void* to uint64_t	*/
 static inline void fs_add_word(fieldset_t *fs, const char *name, int type,
-		int free_, size_t len, void *value)
+		int free_, size_t len, field_val_t value)
 {
 	if (fs->len + 1 >= MAX_FIELDS) {
 		log_fatal("fieldset", "out of room in fieldset");
@@ -48,24 +48,24 @@ static inline void fs_add_word(fieldset_t *fs, const char *name, int type,
 	f->type = type;
 	f->name = name;
 	f->len = len;
-	f->value = (uint64_t) value;
+	f->value = value;
 	f->free_ = free_;
 }
 
 static void fs_modify_word(fieldset_t *fs, const char *name, int type,
-		int free_, size_t len, void *value)
+		int free_, size_t len, field_val_t value)
 {
 	int i;
 	for (i=0; i<fs->len; i++) {
 		if (!strcmp(fs->fields[i].name, name)) {
 			if (fs->fields[i].free_) {
-				free((void*)fs->fields[i].value);
+				free(fs->fields[i].value.ptr);
-				fs->fields[i].value = 0;
+				fs->fields[i].value.ptr = NULL;
 			}
 			fs->fields[i].type = type;
 			fs->fields[i].free_ = free_;
 			fs->fields[i].len = len;
-			fs->fields[i].value = (uint64_t)value;
+			fs->fields[i].value = value;
 			return;
 		}
 	}
@@ -74,55 +74,63 @@ static void fs_modify_word(fieldset_t *fs, const char *name, int type,
 
 void fs_add_null(fieldset_t *fs, const char *name)
 {
-	fs_add_word(fs, name, FS_NULL, 0, 0, NULL);
+	field_val_t val = { .ptr = NULL };
+	fs_add_word(fs, name, FS_NULL, 0, 0, val);
 }
 
 void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_)
 {
-	fs_add_word(fs, name, FS_STRING, free_, strlen(value), (void*) value);
+	field_val_t val = { .ptr = value };
+	fs_add_word(fs, name, FS_STRING, free_, strlen(value), val);
 }
 
 void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value)
 {
-	fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), (void*) value);
+	field_val_t val = { .num = value };
+	fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val);
 }
 
 void fs_add_binary(fieldset_t *fs, const char *name, size_t len,
 		void *value, int free_)
 {
-	fs_add_word(fs, name, FS_BINARY, free_, len, value);
+	field_val_t val = { .ptr = value };
+	fs_add_word(fs, name, FS_BINARY, free_, len, val);
 }
 
 // Modify
 void fs_modify_null(fieldset_t *fs, const char *name)
 {
-	fs_modify_word(fs, name, FS_NULL, 0, 0, NULL);
+	field_val_t val = { .ptr = NULL };
+	fs_modify_word(fs, name, FS_NULL, 0, 0, val);
 }
 
 void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_)
 {
-	fs_modify_word(fs, name, FS_STRING, free_, strlen(value), (void*) value);
+	field_val_t val = { .ptr = value };
+	fs_modify_word(fs, name, FS_STRING, free_, strlen(value), val);
 }
 
 void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value)
 {
-	fs_modify_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), (void*) value);
+	field_val_t val = { .num = value };
+	fs_modify_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val);
 }
 
 void fs_modify_binary(fieldset_t *fs, const char *name, size_t len,
 		void *value, int free_)
 {
-	fs_modify_word(fs, name, FS_BINARY, free_, len, value);
+	field_val_t val = { .ptr = value };
+	fs_modify_word(fs, name, FS_BINARY, free_, len, val);
 }
 
 uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index)
 {
-	return (uint64_t) fs->fields[index].value;
+	return (uint64_t) fs->fields[index].value.num;
 }
 
 char* fs_get_string_by_index(fieldset_t *fs, int index)
 {
-	return (char*) fs->fields[index].value;
+	return (char*) fs->fields[index].value.ptr;
 }
 
 int fds_get_index_by_name(fielddefset_t *fds, char *name)
@@ -143,7 +151,7 @@ void fs_free(fieldset_t *fs)
 	for (int i=0; i < fs->len; i++) {
 		field_t *f = &(fs->fields[i]);
 		if (f->free_) {
-			free((void*) f->value);
+			free(f->value.ptr);
 		}
 	}
 	free(fs);
@@ -162,7 +170,7 @@ void fs_generate_fieldset_translation(translation_t *t,
 			log_fatal("fieldset", "specified field (%s) not "
 					      "available in selected "
 					      "probe module.", req[i]);
-		} 
+		}
 		t->translation[t->len++] = l;
 	}
 }

src/fieldset.h

@@ -36,13 +36,18 @@ typedef struct fielddef_set {
 	int len;
 } fielddefset_t;
 
+typedef union field_val {
+	void	  *ptr;
+	uint64_t   num;
+} field_val_t;
+
 // the internal field type used by fieldset
 typedef struct field {
 	const char *name;
 	int type;
 	int free_;
 	size_t len;
-	uint64_t value;
+	union field_val value;
 } field_t;
 
 // data structure that is populated by the probe module

src/filter.c

@@ -0,0 +1,99 @@
+#include "filter.h"
+#include "state.h"
+#include "lexer.h"
+#include "parser.h"
+#include "expression.h"
+#include "../lib/logger.h"
+
+#include <string.h>
+
+extern int yyparse();
+
+node_t *zfilter;
+
+static int validate_node(node_t *node, fielddefset_t *fields)
+{
+	int index, found = 0;
+	if (node->type == OP) {
+		// These end up getting validated later
+		if (node->value.op == AND || node->value.op == OR) {
+			return 1;
+		}
+		// Comparison node (=, >, <, etc.)
+		// Validate that the field (left child) exists in the fieldset
+		for (index = 0; index < fields->len; index++) {
+			if (fields->fielddefs[index].name) {
+				if (strcmp(fields->fielddefs[index].name, 
+						node->left_child->value.field.fieldname) == 0) {
+					node->left_child->value.field.index = index;
+					found = 1;
+					break;
+				}
+			}
+		}
+		if (!found) {
+			fprintf(stderr, "Field '%s' does not exist\n", 
+					node->left_child->value.field.fieldname);
+			return 0;
+		}
+		// Fieldname is fine, match the type.
+		switch (node->right_child->type) {
+		case STRING:
+			if (strcmp(fields->fielddefs[index].type, "string") == 0) {
+				return 1;
+			} else {
+				fprintf(stderr, "Field '%s' is not of type 'string'\n", 
+						fields->fielddefs[index].name);
+				return 0;
+			}
+		case INT:
+			if (strcmp(fields->fielddefs[index].type, "int") == 0) {
+				return 1;
+			} else {
+				fprintf(stderr, "Field '%s' is not of type 'int'\n", 
+						fields->fielddefs[index].name);
+				return 0;
+			}
+		default:
+			return 0;
+		}
+	} else {
+		// All non-op nodes are valid
+		return 1;
+	}
+	// Didn't validate
+	return 0;
+
+}
+
+int parse_filter_string(char *filter)
+{
+	YY_BUFFER_STATE buffer_state = yy_scan_string(filter);
+	int status = yyparse();
+	yy_delete_buffer(buffer_state);
+	if (status) {
+		// Error
+		log_error("zmap", "Unable to parse filter string: '%s'", filter);
+		return 0;
+	}
+	zconf.filter.expression = zfilter;
+	return 1;
+}
+
+/*
+ * 0     Valid
+ * -1    Invalid Field Name
+ * -2    Type Mismatch
+ */
+int validate_filter(node_t *root, fielddefset_t *fields)
+{
+	int valid;
+	if (!root) {
+		return 1;
+	}
+	valid = validate_node(root, fields);
+	if (!valid) {
+		return 0;
+	}
+	return (validate_filter(root->left_child, fields) && validate_filter(root->right_child, fields));
+}

src/filter.h

@@ -0,0 +1,15 @@
+#ifndef ZMAP_FILTER_H
+#define ZMAP_FILTER_H
+
+#include "expression.h"
+#include "fieldset.h"
+
+struct output_filter {
+	node_t *expression;
+};
+
+int parse_filter_string(char *filter);
+
+int validate_filter(node_t *root, fielddefset_t *fields);
+
+#endif /* ZMAP_FILTER_H */

src/lexer.l

@@ -0,0 +1,26 @@
+%{
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#include <string.h>
+#include "parser.h"
+
+%}
+
+%option noinput
+%option nounput
+%%
+[0-9]+               yylval.int_literal = (uint64_t) atoll(yytext); return T_NUMBER;
+\n                   /* Ignore end of line */
+[ \t]+               /* Ignore whitespace */
+!=                   return T_NOT_EQ;
+>=                   return T_GT_EQ;
+"<="                 return T_LT_EQ;
+&&                   return T_AND;
+"||"                 return T_OR;
+=                    return '=';
+">"                  return '>';
+"<"                  return '<';
+"("                  return '(';
+")"                  return ')';
+[a-zA-Z][a-zA-Z0-9]+ yylval.string_literal = strdup(yytext); return T_FIELD;
+
+%%

src/output_modules/module_csv.c

@@ -14,6 +14,7 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <assert.h>
+#include <inttypes.h>
 
 #include "../../lib/logger.h"
 #include "../fieldset.h"
@@ -79,11 +80,11 @@ int csv_process(fieldset_t *fs)
 			fprintf(file, ", ");
 		}
 		if (f->type == FS_STRING) {
-			fprintf(file, "%s", (char*) f->value); 
+			fprintf(file, "%s", (char*) f->value.ptr);
 		} else if (f->type == FS_UINT64) {
-			fprintf(file, "%lu", (uint64_t) f->value); 
+			fprintf(file, "%" PRIu64, (uint64_t) f->value.num);
 		} else if (f->type == FS_BINARY) {
-			hex_encode(file, (unsigned char*) f->value, f->len);
+			hex_encode(file, (unsigned char*) f->value.ptr, f->len);
 		} else if (f->type == FS_NULL) {
 			// do nothing
 		} else {
@@ -102,5 +103,6 @@ output_module_t module_csv_file = {
 	.update_interval = 0,
 	.close = &csv_close,
 	.process_ip = &csv_process,
+	.helptext = NULL
 };
 

src/output_modules/module_json.c

@@ -149,13 +149,13 @@ int json_output_file_ip(fieldset_t *fs)
 		field_t *f = &(fs->fields[i]);
 		if (f->type == FS_STRING) {
 			json_object_object_add(obj, f->name,
-					json_object_new_string((char*) f->value));
+					json_object_new_string((char *) f->value.ptr));
 		} else if (f->type == FS_UINT64) {
 			json_object_object_add(obj, f->name,
-					json_object_new_int((int) f->value));
+					json_object_new_int((int) f->value.num));
 		} else if (f->type == FS_BINARY) {
 			json_output_file_store_data(obj,
-					(const u_char*) f->value, f->len); 
+					(const u_char*) f->value.ptr, f->len); 
 		} else if (f->type == FS_NULL) {
 			// do nothing
 		} else {
@@ -188,5 +188,6 @@ output_module_t module_json_file = {
 	.update_interval = 0,
 	.close = &json_output_file_close,
 	.process_ip = &json_output_file_ip,
+	.helptext = NULL
 };
 

src/output_modules/module_redis.c

@@ -22,55 +22,60 @@
 
 #define UNUSED __attribute__((unused))
 
-typedef struct scannable_t {
-	in_addr_t ip_address;
-	uint8_t source;
-} scannable_t;
-
-#define QUEUE_NAME "zmap_results"
 #define BUFFER_SIZE 500
-#define SOURCE_ZMAP 0
 
-static scannable_t* buffer;
+static uint32_t *buffer;
 static int buffer_fill = 0;
+static char *queue_name = NULL;
 
-int redismodule_init(UNUSED struct state_conf *conf)
+static int redismodule_init(struct state_conf *conf, char **fields, int fieldlens)
 {
-	buffer = calloc(BUFFER_SIZE, sizeof(scannable_t));
+	assert(fieldlens == 1);
+	buffer = calloc(BUFFER_SIZE, sizeof(uint32_t));
 	assert(buffer);
 	buffer_fill = 0;
-	return redis_init();
+
+	if (conf->output_args) { 
+		redisconf_t *rconf = redis_parse_connstr(conf->output_args);
+		if (rconf->type == T_TCP) {
+			log_info("redis-module", "{type: TCP, server: %s, "
+					"port: %u, list: %s}", rconf->server, 
+					rconf->port, rconf->list_name);
+		} else {
+			log_info("redis-module", "{type: LOCAL, path: %s, "
+					"list: %s}", rconf->path, rconf->list_name);
+		}
+		queue_name = rconf->list_name;
+	} else {
+		queue_name = "zmap_output";
+	}
+	return redis_init(conf->output_args);
 }
 
-int redismodule_flush(void)
+static int redismodule_flush(void)
 {
-	if (redis_lpush((char *)QUEUE_NAME, buffer,
+	if (redis_lpush((char *)queue_name, buffer,
-			buffer_fill, sizeof(scannable_t))) {
+			buffer_fill, sizeof(uint32_t))) {
 		return EXIT_FAILURE;
 	}
 	buffer_fill = 0;
 	return EXIT_SUCCESS;
 }
 
-int redismodule_newip(ipaddr_n_t saddr, UNUSED ipaddr_n_t daddr,
+static int redismodule_process(fieldset_t *fs)
-        UNUSED const char *response_type, int is_repeat,
-        UNUSED int in_cooldown, UNUSED const u_char *packet, UNUSED size_t len)
 {
-	if (!is_repeat) {
+	field_t *f = &(fs->fields[0]);
-		buffer[buffer_fill].ip_address = saddr;
+	buffer[buffer_fill] = (uint32_t) f->value.num;
-		buffer[buffer_fill].source = SOURCE_ZMAP;
+	if (++buffer_fill == BUFFER_SIZE) {
-
+		if (redismodule_flush()) {
-		if (++buffer_fill == BUFFER_SIZE) {
+			return EXIT_FAILURE;
-			if (redismodule_flush()) {
-				return EXIT_FAILURE;
-			}
 		}
 	}
 	return EXIT_SUCCESS;
 }
 
-int redismodule_close(UNUSED struct state_conf* c, 
+static int redismodule_close(UNUSED struct state_conf* c, 
-        UNUSED struct state_send* s,
+		UNUSED struct state_send* s,
 		UNUSED struct state_recv* r)
 {
 	if (redismodule_flush()) {
@@ -83,13 +88,13 @@ int redismodule_close(UNUSED struct state_conf* c,
 }
 
 output_module_t module_redis = {
-	.name = "redis",
+	.name = "redis-packed",
 	.init = &redismodule_init,
 	.start = NULL,
 	.update = NULL,
 	.update_interval = 0,
 	.close = &redismodule_close,
-	.success_ip = &redismodule_newip,
+	.process_ip = &redismodule_process,
-	.other_ip = NULL
+	.helptext = NULL
 };
 

src/output_modules/output_modules.c

@@ -6,7 +6,6 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-
 #include <stdio.h>
 #include <string.h>
 
@@ -25,7 +24,7 @@ extern output_module_t module_json_file;
 output_module_t* output_modules[] = {
 	&module_csv_file,
 #ifdef REDIS
-	//&module_redis,
+	&module_redis,
 #endif
 #ifdef JSON
 	&module_json_file
@@ -33,8 +32,6 @@ output_module_t* output_modules[] = {
 	// ADD YOUR MODULE HERE
 };
 
-
-
 output_module_t* get_output_module_by_name(const char* name)
 {
     int num_modules = (int) (sizeof(output_modules)/sizeof(output_modules[0]));

src/output_modules/output_modules.h

@@ -30,9 +30,9 @@ typedef struct output_module {
 	output_update_cb update;
 	output_update_cb close;
 	output_packet_cb process_ip;
+	const char *helptext;
 } output_module_t;
 
-
 output_module_t* get_output_module_by_name(const char*);
 
 void print_output_modules(void);

src/parser.y

@@ -0,0 +1,144 @@
+%{
+#include <stdio.h>
+#include <string.h>
+#include "expression.h"
+#include "lexer.h"
+#include "filter.h"
+
+void yyerror(const char *str)
+{
+	fprintf(stderr,"Parse error: %s\n",str);
+}
+ 
+int yywrap()
+{
+	return 1;
+}
+
+extern node_t *zfilter;
+
+%}
+
+%union {
+	int int_literal;
+	char *string_literal;
+	struct node *expr; 
+}
+
+%token '(' ')' T_AND T_OR
+%token <int_literal> T_NUMBER
+%token <string_literal> T_FIELD
+%token T_NOT_EQ T_GT_EQ '>' '<' '=' T_LT_EQ
+
+%left T_OR
+%left T_AND
+
+%type <expr> filter
+%type <expr> number_filter
+%type <expr> string_filter
+%type <expr> filter_expr
+
+
+%%
+
+expression: filter_expr
+	{
+		zfilter = $1;
+	}
+
+
+filter_expr:
+	filter_expr T_OR filter_expr
+		{
+			$$ = make_op_node(OR);
+			$$->left_child = $1;
+			$$->right_child = $3;
+		}
+	| filter_expr T_AND filter_expr
+		{
+			$$ = make_op_node(AND);
+			$$->left_child = $1;
+			$$->right_child = $3;
+		}
+	| '(' filter_expr ')'
+		{
+			$$ = $2;
+		}
+	| filter
+		{
+			$$ = $1;
+		}
+	;
+
+filter: number_filter 
+		{
+			$$ = $1;
+		}
+	| string_filter
+		{
+			$$ = $1;
+		}
+	;
+
+number_filter: T_FIELD '=' T_NUMBER
+		{
+			$$ = make_op_node(EQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}	
+	| 
+	T_FIELD '>' T_NUMBER
+		{
+			$$ = make_op_node(GT);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}
+	|
+	T_FIELD '<' T_NUMBER
+		{
+			$$ = make_op_node(LT);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}
+	|
+	T_FIELD T_NOT_EQ T_NUMBER
+		{
+			$$ = make_op_node(NEQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}
+	|
+	T_FIELD T_GT_EQ T_NUMBER
+		{
+			$$ = make_op_node(GT_EQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}
+	|
+	T_FIELD T_LT_EQ T_NUMBER
+		{
+			$$ = make_op_node(LT_EQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_int_node($3);
+		}
+	;
+
+string_filter:
+	T_FIELD '=' T_FIELD
+		{
+			$$ = make_op_node(EQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_string_node($3);
+		}
+	|
+	T_FIELD T_NOT_EQ T_FIELD
+		{
+			$$ = make_op_node(NEQ);
+			$$->left_child = make_field_node($1);
+			$$->right_child = make_string_node($3);
+		}
+	;
+
+%%
+
+

src/recv.c

@@ -35,10 +35,12 @@
 #include <assert.h>
 
 #include "../lib/logger.h"
+#include "../lib/pbm.h"
 
 #include "state.h"
 #include "validate.h"
 #include "fieldset.h"
+#include "expression.h"
 #include "probe_modules/probe_modules.h"
 #include "output_modules/output_modules.h"
 
@@ -49,20 +51,7 @@ static uint32_t num_src_ports;
 static pcap_t *pc = NULL;
 
 // bitmap of observed IP addresses
-static uint64_t *ip_seen = NULL;
+static uint64_t **seen = NULL;
-static const int IP_SEEN_SIZE = 0x4000000; // == 2^32/64
-
-// check if we've received a response from this address previously
-static inline int check_ip(uint32_t ip)
-{
-	return (ip_seen[ip >> 6] >> (ip & 0x3F)) & 1;
-}
-
-// set that we've received a response from the address
-static inline void set_ip(uint32_t ip)
-{
-	ip_seen[ip >> 6] |= (uint64_t)1 << (ip & 0x3F);
-}
 
 static u_char fake_eth_hdr[65535];
 
@@ -120,7 +109,7 @@ void packet_cb(u_char __attribute__((__unused__)) *user,
 		return;
 	}
 
-	int is_repeat = check_ip(src_ip);
+	int is_repeat = pbm_check(seen, ntohl(src_ip));
 
 	fieldset_t *fs = fs_new_fieldset();
 	fs_add_ip_fields(fs, ip_hdr);
@@ -150,7 +139,7 @@ void packet_cb(u_char __attribute__((__unused__)) *user,
 		zrecv.success_total++;
 		if (!is_repeat) {
 			zrecv.success_unique++;
-			set_ip(src_ip);
+			pbm_set(seen, ntohl(src_ip));
 		}
 		if (zsend.complete) {
 			zrecv.cooldown_total++;
@@ -170,6 +159,9 @@ void packet_cb(u_char __attribute__((__unused__)) *user,
 	if (is_repeat && zconf.filter_duplicates) {
 		goto cleanup;
 	}
+	if (!evaluate_expression(zconf.filter.expression, fs)) {
+		goto cleanup;
+	}
 	o = translate_fieldset(fs, &zconf.fsconf.translation);
 	if (zconf.output_module && zconf.output_module->process_ip) {
 		zconf.output_module->process_ip(o);
@@ -205,10 +197,6 @@ int recv_run(pthread_mutex_t *recv_ready_mutex)
 {
 	log_debug("recv", "thread started");
 	num_src_ports = zconf.source_port_last - zconf.source_port_first + 1;
-	ip_seen = calloc(IP_SEEN_SIZE, sizeof(uint64_t));
-	if (!ip_seen) {
-		log_fatal("recv", "could not allocate address bitmap");
-	}
 	log_debug("recv", "using dev %s", zconf.iface);
 	if (!zconf.dryrun) {
 		char errbuf[PCAP_ERRBUF_SIZE];
@@ -235,6 +223,8 @@ int recv_run(pthread_mutex_t *recv_ready_mutex)
 		memset(fake_eth_hdr, 0, sizeof(fake_eth_hdr));
 		eth->h_proto = htons(ETH_P_IP);
 	}
+	// initialize paged bitmap
+	seen = pbm_init();
 	log_debug("recv", "receiver ready");
 	if (zconf.filter_duplicates) {
 		log_debug("recv", "duplicate responses will be excluded from output"); 

src/state.h

@@ -25,6 +25,7 @@
 
 #include "types.h"
 #include "fieldset.h"
+#include "filter.h"
 
 #ifndef STATE_H
 #define STATE_H
@@ -87,8 +88,11 @@ struct state_conf {
 	char *output_filename;
 	char *blacklist_filename;
 	char *whitelist_filename;
+	char **destination_cidrs;
+	int destination_cidrs_len;
 	char *raw_output_fields;
 	char **output_fields;
+	struct output_filter filter;
 	struct fieldset_conf fsconf;
 	int output_fields_len;
 	int dryrun;

src/zmap.c

@@ -40,6 +40,7 @@
 #include "state.h"
 #include "monitor.h"
 #include "get_gateway.h"
+#include "filter.h"
 
 #include "output_modules/output_modules.h"
 #include "probe_modules/probe_modules.h"
@@ -199,8 +200,6 @@ static void summary(void)
 static void start_zmap(void)
 {
 	log_info("zmap", "started");
-
-	// finish setting up configuration
 	if (zconf.iface == NULL) {
 		char errbuf[PCAP_ERRBUF_SIZE];
 		char *iface = pcap_lookupdev(errbuf);
@@ -242,7 +241,6 @@ static void start_zmap(void)
 				  zconf.gw_mac[0], zconf.gw_mac[1], zconf.gw_mac[2],
 				  zconf.gw_mac[3], zconf.gw_mac[4], zconf.gw_mac[5]);
 	}
-
 	// initialization
 	if (zconf.output_module && zconf.output_module->init) {
 		zconf.output_module->init(&zconf, zconf.output_fields,
@@ -397,9 +395,69 @@ int main(int argc, char *argv[])
 	zconf.log_level = args.verbosity_arg;
 	log_init(stderr, zconf.log_level);
 	log_trace("zmap", "zmap main thread started");
-
+	// parse the provided probe and output module s.t. that we can support
+	// other command-line helpers (e.g. probe help)
+	if (!args.output_module_given) {
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "saddr";
+		zconf.filter_duplicates = 1;
+		zconf.filter_unsuccessful = 1;
+	} else if (!strcmp(args.output_module_arg, "simple_file")) {
+		log_warn("zmap", "the simple_file output interface has been deprecated and "
+                                 "will be removed in the future. Users should use the csv "
+				 "output module. Newer scan options such as output-fields "
+				 "are not supported with this output module."); 
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "saddr";
+		zconf.filter_duplicates = 1;
+		zconf.filter_unsuccessful = 1;
+	} else if (!strcmp(args.output_module_arg, "extended_file")) {
+		log_warn("zmap", "the extended_file output interface has been deprecated and "
+                                 "will be removed in the future. Users should use the csv "
+				 "output module. Newer scan options such as output-fields "
+				 "are not supported with this output module."); 
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "classification, saddr, "
+						  "daddr, sport, dport, "
+						  "seqnum, acknum, cooldown, "
+						  "repeat, timestamp-str";
+		zconf.filter_duplicates = 0;
+	} else if (!strcmp(args.output_module_arg, "redis")) {
+		log_warn("zmap", "the redis output interface has been deprecated and "
+                                 "will be removed in the future. Users should "
+				 "either use redis-packed or redis-json in the "
+				 "future.");
+		zconf.output_module = get_output_module_by_name("redis-packed");
+		zconf.raw_output_fields = (char*) "saddr";
+		zconf.filter_duplicates = 1;
+	} else {
+		zconf.output_module = get_output_module_by_name(args.output_module_arg);
+		if (!zconf.output_module) {
+		  fprintf(stderr, "%s: specified output module (%s) does not exist\n",
+			  CMDLINE_PARSER_PACKAGE, args.output_module_arg);
+		  exit(EXIT_FAILURE);
+		}
+	}
+	zconf.probe_module = get_probe_module_by_name(args.probe_module_arg);
+	if (!zconf.probe_module) {
+		fprintf(stderr, "%s: specified probe module (%s) does not exist\n",
+				CMDLINE_PARSER_PACKAGE, args.probe_module_arg);
+	  exit(EXIT_FAILURE);
+	}
 	if (args.help_given) {
 		cmdline_parser_print_help();
+		printf("\nselected probe-module (%s) help\n", zconf.probe_module->name);
+		if (zconf.probe_module->helptext) {
+			printf("%s\n", zconf.probe_module->helptext);
+		} else {
+			printf("no help text available\n");
+		}
+		printf("\nselected output-module (%s) help\n", zconf.output_module->name);
+		if (zconf.output_module->helptext) {
+			printf("%s\n", zconf.output_module->helptext);
+		} else {
+			printf("no help text available\n");
+		}
 		exit(EXIT_SUCCESS);
 	}
 	if (args.version_given) {
@@ -430,48 +488,7 @@ int main(int argc, char *argv[])
 	if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) {
 		exit(EXIT_FAILURE);
 	}
-	// parse the provided probe and output module s.t. that we can support
+	
-	// other command-line helpers (e.g. probe help)
-	if (!args.output_module_given) {
-		zconf.output_module = get_output_module_by_name("csv");
-		zconf.raw_output_fields = (char*) "saddr";
-		zconf.filter_duplicates = 1;
-		zconf.filter_unsuccessful = 1;
-	} else if (!strcmp(args.output_module_arg, "simple_file")) {
-		log_warn("zmap", "the simple_file output interface has been deprecated and "
-                                 "will be removed in the future. Users should use the csv "
-				 "output module. Newer scan options such as output-fields "
-				 "are not supported with this output module."); 
-		zconf.output_module = get_output_module_by_name("csv");
-		zconf.raw_output_fields = (char*) "saddr";
-		zconf.filter_duplicates = 1;
-		zconf.filter_unsuccessful = 1;
-	} else if (!strcmp(args.output_module_arg, "extended_file")) {
-		log_warn("zmap", "the extended_file output interface has been deprecated and "
-                                 "will be removed in the future. Users should use the csv "
-				 "output module. Newer scan options such as output-fields "
-				 "are not supported with this output module."); 
-		zconf.output_module = get_output_module_by_name("csv");
-		zconf.raw_output_fields = (char*) "classification, saddr, "
-						  "daddr, sport, dport, "
-						  "seqnum, acknum, cooldown, "
-						  "repeat, timestamp-str";
-		zconf.filter_duplicates = 0;
-	} else {
-		zconf.output_module = get_output_module_by_name(args.output_module_arg);
-		if (!zconf.output_module) {
-		  fprintf(stderr, "%s: specified output module (%s) does not exist\n",
-			  CMDLINE_PARSER_PACKAGE, args.output_module_arg);
-		  exit(EXIT_FAILURE);
-		}
-	}
-	zconf.probe_module = get_probe_module_by_name(args.probe_module_arg);
-	if (!zconf.probe_module) {
-		fprintf(stderr, "%s: specified probe module (%s) does not exist\n",
-				CMDLINE_PARSER_PACKAGE, args.probe_module_arg);
-	  exit(EXIT_FAILURE);
-	}
-
 	// now that we know the probe module, let's find what it supports
 	memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf));
 	// the set of fields made available to a user is constructed
@@ -523,6 +540,19 @@ int main(int argc, char *argv[])
 			&zconf.fsconf.defs, zconf.output_fields,
 			zconf.output_fields_len);
 
+	// Parse and validate the output filter, if any
+	if (args.output_filter_arg) {
+		// Run it through yyparse to build the expression tree
+		if (!parse_filter_string(args.output_filter_arg)) {
+			log_fatal("zmap", "Unable to parse filter expression");
+		}
+
+		// Check the fields used against the fieldset in use
+		if (!validate_filter(zconf.filter.expression, &zconf.fsconf.defs)) {
+			log_fatal("zmap", "Invalid filter");
+		}
+	}
+
 	SET_BOOL(zconf.dryrun, dryrun);
 	SET_BOOL(zconf.quiet, quiet);
 	SET_BOOL(zconf.summary, summary);
@@ -530,7 +560,6 @@ int main(int argc, char *argv[])
 	zconf.senders = args.sender_threads_arg;
 	SET_IF_GIVEN(zconf.output_filename, output_file);
 	SET_IF_GIVEN(zconf.blacklist_filename, blacklist_file);
-	SET_IF_GIVEN(zconf.whitelist_filename, whitelist_file);
 	SET_IF_GIVEN(zconf.probe_args, probe_args);
 	SET_IF_GIVEN(zconf.output_args, output_args);
 	SET_IF_GIVEN(zconf.iface, interface);
@@ -538,8 +567,21 @@ int main(int argc, char *argv[])
 	SET_IF_GIVEN(zconf.max_results, max_results);
 	SET_IF_GIVEN(zconf.rate, rate);
 	SET_IF_GIVEN(zconf.packet_streams, probes);
-	
 
+	// find if zmap wants any specific cidrs scanned instead
+	// of the entire Internet
+	zconf.destination_cidrs = args.inputs;
+	zconf.destination_cidrs_len = args.inputs_num;
+	if (zconf.destination_cidrs && zconf.blacklist_filename
+			&& !strcmp(zconf.blacklist_filename, "/etc/zmap/blacklist.conf")) {
+		log_warn("blacklist", "ZMap is currently using the default blacklist located "
+				"at /etc/zmap/blacklist.conf. By default, this blacklist excludes locally "
+				"scoped networks (e.g. 10.0.0.0/8, 127.0.0.1/8, and 192.168.0.0/16). If you are"
+				" trying to scan local networks, you can change the default blacklist by "
+				"editing the default ZMap configuration at /etc/zmap/zmap.conf.");
+	}
+	SET_IF_GIVEN(zconf.whitelist_filename, whitelist_file);
+	
 	if (zconf.probe_module->port_args) {
 		if (args.source_port_given) {
 			char *dash = strchr(args.source_port_arg, '-');

src/zopt.ggo

@@ -94,6 +94,9 @@ option "probe-args"		- "Arguments to pass to probe module"
 option "output-args"		- "Arguments to pass to output module"
 	   typestr="args"
 	   optional string
+option "output-filter"          - "Read a file containing an output filter on the first line"
+	   typestr="filename"
+	   optional string
 option "list-output-modules"	- "List available output modules"
            optional
 option "list-probe-modules"	- "List available probe modules"
@@ -122,5 +125,6 @@ option "version"   		V "Print version and exit"
 
 text "\nExamples:\n\
      zmap -p 443  (scans the whole Internet for hosts with port 443 open)\n\
-     zmap -N 5 -B 10M -p 80 -o -  (find 5 HTTP servers, scanning at 10 Mb/s)"
+     zmap -N 5 -B 10M -p 80 -o -  (find 5 HTTP servers, scanning at 10 Mb/s)\n
-
+     zmap -p 80 10.0.0.0/8 192.168.0.0/16 -o (scan 10.0.0.0/8 and 192.168.0.0/16 on port 80)\n
+     zmap -p 80 192.168.1.2 192.168.1.3 (scan 192.168.1.2 and 192.168.1.3 on port 80)"

src/zopt.h

@@ -1,306 +0,0 @@
-/** @file zopt.h
- *  @brief The header file for the command line option parser
- *  generated by GNU Gengetopt version 2.22.5
- *  http://www.gnu.org/software/gengetopt.
- *  DO NOT modify this file, since it can be overwritten
- *  @author GNU Gengetopt by Lorenzo Bettini */
-
-#ifndef ZOPT_H
-#define ZOPT_H
-
-/* If we use autoconf.  */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h> /* for FILE */
-
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-
-#ifndef CMDLINE_PARSER_PACKAGE
-/** @brief the program name (used for printing errors) */
-#define CMDLINE_PARSER_PACKAGE "zmap"
-#endif
-
-#ifndef CMDLINE_PARSER_PACKAGE_NAME
-/** @brief the complete program name (used for help and version) */
-#define CMDLINE_PARSER_PACKAGE_NAME "zmap"
-#endif
-
-#ifndef CMDLINE_PARSER_VERSION
-/** @brief the program version */
-#define CMDLINE_PARSER_VERSION "1.0.0"
-#endif
-
-/** @brief Where the command line options are stored */
-struct gengetopt_args_info
-{
-  int target_port_arg;	/**< @brief TCP port number to scan (for SYN scans).  */
-  char * target_port_orig;	/**< @brief TCP port number to scan (for SYN scans) original value given at command line.  */
-  const char *target_port_help; /**< @brief TCP port number to scan (for SYN scans) help description.  */
-  char * output_file_arg;	/**< @brief Output file.  */
-  char * output_file_orig;	/**< @brief Output file original value given at command line.  */
-  const char *output_file_help; /**< @brief Output file help description.  */
-  char * blacklist_file_arg;	/**< @brief File of subnets to exclude, in CIDR notation, e.g. 192.168.0.0/16.  */
-  char * blacklist_file_orig;	/**< @brief File of subnets to exclude, in CIDR notation, e.g. 192.168.0.0/16 original value given at command line.  */
-  const char *blacklist_file_help; /**< @brief File of subnets to exclude, in CIDR notation, e.g. 192.168.0.0/16 help description.  */
-  char * whitelist_file_arg;	/**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16.  */
-  char * whitelist_file_orig;	/**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 original value given at command line.  */
-  const char *whitelist_file_help; /**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 help description.  */
-  char * output_fields_arg;	/**< @brief Fields that should be output in result set.  */
-  char * output_fields_orig;	/**< @brief Fields that should be output in result set original value given at command line.  */
-  const char *output_fields_help; /**< @brief Fields that should be output in result set help description.  */
-  char * max_targets_arg;	/**< @brief Cap number of targets to probe (as a number or a percentage of the address space).  */
-  char * max_targets_orig;	/**< @brief Cap number of targets to probe (as a number or a percentage of the address space) original value given at command line.  */
-  const char *max_targets_help; /**< @brief Cap number of targets to probe (as a number or a percentage of the address space) help description.  */
-  int max_results_arg;	/**< @brief Cap number of results to return.  */
-  char * max_results_orig;	/**< @brief Cap number of results to return original value given at command line.  */
-  const char *max_results_help; /**< @brief Cap number of results to return help description.  */
-  int max_runtime_arg;	/**< @brief Cap length of time for sending packets.  */
-  char * max_runtime_orig;	/**< @brief Cap length of time for sending packets original value given at command line.  */
-  const char *max_runtime_help; /**< @brief Cap length of time for sending packets help description.  */
-  int rate_arg;	/**< @brief Set send rate in packets/sec.  */
-  char * rate_orig;	/**< @brief Set send rate in packets/sec original value given at command line.  */
-  const char *rate_help; /**< @brief Set send rate in packets/sec help description.  */
-  char * bandwidth_arg;	/**< @brief Set send rate in bits/second (supports suffixes G, M and K).  */
-  char * bandwidth_orig;	/**< @brief Set send rate in bits/second (supports suffixes G, M and K) original value given at command line.  */
-  const char *bandwidth_help; /**< @brief Set send rate in bits/second (supports suffixes G, M and K) help description.  */
-  int cooldown_time_arg;	/**< @brief How long to continue receiving after sending last probe (default='8').  */
-  char * cooldown_time_orig;	/**< @brief How long to continue receiving after sending last probe original value given at command line.  */
-  const char *cooldown_time_help; /**< @brief How long to continue receiving after sending last probe help description.  */
-  int seed_arg;	/**< @brief Seed used to select address permutation.  */
-  char * seed_orig;	/**< @brief Seed used to select address permutation original value given at command line.  */
-  const char *seed_help; /**< @brief Seed used to select address permutation help description.  */
-  int sender_threads_arg;	/**< @brief Threads used to send packets (default='1').  */
-  char * sender_threads_orig;	/**< @brief Threads used to send packets original value given at command line.  */
-  const char *sender_threads_help; /**< @brief Threads used to send packets help description.  */
-  int probes_arg;	/**< @brief Number of probes to send to each IP (default='1').  */
-  char * probes_orig;	/**< @brief Number of probes to send to each IP original value given at command line.  */
-  const char *probes_help; /**< @brief Number of probes to send to each IP help description.  */
-  const char *dryrun_help; /**< @brief Don't actually send packets help description.  */
-  char * source_port_arg;	/**< @brief Source port(s) for scan packets.  */
-  char * source_port_orig;	/**< @brief Source port(s) for scan packets original value given at command line.  */
-  const char *source_port_help; /**< @brief Source port(s) for scan packets help description.  */
-  char * source_ip_arg;	/**< @brief Source address(es) for scan packets.  */
-  char * source_ip_orig;	/**< @brief Source address(es) for scan packets original value given at command line.  */
-  const char *source_ip_help; /**< @brief Source address(es) for scan packets help description.  */
-  char * gateway_mac_arg;	/**< @brief Specify gateway MAC address.  */
-  char * gateway_mac_orig;	/**< @brief Specify gateway MAC address original value given at command line.  */
-  const char *gateway_mac_help; /**< @brief Specify gateway MAC address help description.  */
-  char * interface_arg;	/**< @brief Specify network interface to use.  */
-  char * interface_orig;	/**< @brief Specify network interface to use original value given at command line.  */
-  const char *interface_help; /**< @brief Specify network interface to use help description.  */
-  const char *vpn_help; /**< @brief Sends IP packets instead of Ethernet (for VPNs) help description.  */
-  char * probe_module_arg;	/**< @brief Select probe module (default='tcp_synscan').  */
-  char * probe_module_orig;	/**< @brief Select probe module original value given at command line.  */
-  const char *probe_module_help; /**< @brief Select probe module help description.  */
-  char * output_module_arg;	/**< @brief Select output module (default='simple_file').  */
-  char * output_module_orig;	/**< @brief Select output module original value given at command line.  */
-  const char *output_module_help; /**< @brief Select output module help description.  */
-  char * probe_args_arg;	/**< @brief Arguments to pass to probe module.  */
-  char * probe_args_orig;	/**< @brief Arguments to pass to probe module original value given at command line.  */
-  const char *probe_args_help; /**< @brief Arguments to pass to probe module help description.  */
-  char * output_args_arg;	/**< @brief Arguments to pass to output module.  */
-  char * output_args_orig;	/**< @brief Arguments to pass to output module original value given at command line.  */
-  const char *output_args_help; /**< @brief Arguments to pass to output module help description.  */
-  const char *list_output_modules_help; /**< @brief List available output modules help description.  */
-  const char *list_probe_modules_help; /**< @brief List available probe modules help description.  */
-  const char *list_output_fields_help; /**< @brief List all fields that can be output by selected probe module help description.  */
-  char * config_arg;	/**< @brief Read a configuration file, which can specify any of these options (default='/etc/zmap/zmap.conf').  */
-  char * config_orig;	/**< @brief Read a configuration file, which can specify any of these options original value given at command line.  */
-  const char *config_help; /**< @brief Read a configuration file, which can specify any of these options help description.  */
-  const char *quiet_help; /**< @brief Do not print status updates help description.  */
-  const char *summary_help; /**< @brief Print configuration and summary at end of scan help description.  */
-  int verbosity_arg;	/**< @brief Level of log detail (0-5) (default='3').  */
-  char * verbosity_orig;	/**< @brief Level of log detail (0-5) original value given at command line.  */
-  const char *verbosity_help; /**< @brief Level of log detail (0-5) help description.  */
-  const char *help_help; /**< @brief Print help and exit help description.  */
-  const char *version_help; /**< @brief Print version and exit help description.  */
-  
-  unsigned int target_port_given ;	/**< @brief Whether target-port was given.  */
-  unsigned int output_file_given ;	/**< @brief Whether output-file was given.  */
-  unsigned int blacklist_file_given ;	/**< @brief Whether blacklist-file was given.  */
-  unsigned int whitelist_file_given ;	/**< @brief Whether whitelist-file was given.  */
-  unsigned int output_fields_given ;	/**< @brief Whether output-fields was given.  */
-  unsigned int max_targets_given ;	/**< @brief Whether max-targets was given.  */
-  unsigned int max_results_given ;	/**< @brief Whether max-results was given.  */
-  unsigned int max_runtime_given ;	/**< @brief Whether max-runtime was given.  */
-  unsigned int rate_given ;	/**< @brief Whether rate was given.  */
-  unsigned int bandwidth_given ;	/**< @brief Whether bandwidth was given.  */
-  unsigned int cooldown_time_given ;	/**< @brief Whether cooldown-time was given.  */
-  unsigned int seed_given ;	/**< @brief Whether seed was given.  */
-  unsigned int sender_threads_given ;	/**< @brief Whether sender-threads was given.  */
-  unsigned int probes_given ;	/**< @brief Whether probes was given.  */
-  unsigned int dryrun_given ;	/**< @brief Whether dryrun was given.  */
-  unsigned int source_port_given ;	/**< @brief Whether source-port was given.  */
-  unsigned int source_ip_given ;	/**< @brief Whether source-ip was given.  */
-  unsigned int gateway_mac_given ;	/**< @brief Whether gateway-mac was given.  */
-  unsigned int interface_given ;	/**< @brief Whether interface was given.  */
-  unsigned int vpn_given ;	/**< @brief Whether vpn was given.  */
-  unsigned int probe_module_given ;	/**< @brief Whether probe-module was given.  */
-  unsigned int output_module_given ;	/**< @brief Whether output-module was given.  */
-  unsigned int probe_args_given ;	/**< @brief Whether probe-args was given.  */
-  unsigned int output_args_given ;	/**< @brief Whether output-args was given.  */
-  unsigned int list_output_modules_given ;	/**< @brief Whether list-output-modules was given.  */
-  unsigned int list_probe_modules_given ;	/**< @brief Whether list-probe-modules was given.  */
-  unsigned int list_output_fields_given ;	/**< @brief Whether list-output-fields was given.  */
-  unsigned int config_given ;	/**< @brief Whether config was given.  */
-  unsigned int quiet_given ;	/**< @brief Whether quiet was given.  */
-  unsigned int summary_given ;	/**< @brief Whether summary was given.  */
-  unsigned int verbosity_given ;	/**< @brief Whether verbosity was given.  */
-  unsigned int help_given ;	/**< @brief Whether help was given.  */
-  unsigned int version_given ;	/**< @brief Whether version was given.  */
-
-} ;
-
-/** @brief The additional parameters to pass to parser functions */
-struct cmdline_parser_params
-{
-  int override; /**< @brief whether to override possibly already present options (default 0) */
-  int initialize; /**< @brief whether to initialize the option structure gengetopt_args_info (default 1) */
-  int check_required; /**< @brief whether to check that all required options were provided (default 1) */
-  int check_ambiguity; /**< @brief whether to check for options already specified in the option structure gengetopt_args_info (default 0) */
-  int print_errors; /**< @brief whether getopt_long should print an error message for a bad option (default 1) */
-} ;
-
-/** @brief the purpose string of the program */
-extern const char *gengetopt_args_info_purpose;
-/** @brief the usage string of the program */
-extern const char *gengetopt_args_info_usage;
-/** @brief all the lines making the help output */
-extern const char *gengetopt_args_info_help[];
-
-/**
- * The command line parser
- * @param argc the number of command line options
- * @param argv the command line options
- * @param args_info the structure where option information will be stored
- * @return 0 if everything went fine, NON 0 if an error took place
- */
-int cmdline_parser (int argc, char **argv,
-  struct gengetopt_args_info *args_info);
-
-/**
- * The command line parser (version with additional parameters - deprecated)
- * @param argc the number of command line options
- * @param argv the command line options
- * @param args_info the structure where option information will be stored
- * @param override whether to override possibly already present options
- * @param initialize whether to initialize the option structure my_args_info
- * @param check_required whether to check that all required options were provided
- * @return 0 if everything went fine, NON 0 if an error took place
- * @deprecated use cmdline_parser_ext() instead
- */
-int cmdline_parser2 (int argc, char **argv,
-  struct gengetopt_args_info *args_info,
-  int override, int initialize, int check_required);
-
-/**
- * The command line parser (version with additional parameters)
- * @param argc the number of command line options
- * @param argv the command line options
- * @param args_info the structure where option information will be stored
- * @param params additional parameters for the parser
- * @return 0 if everything went fine, NON 0 if an error took place
- */
-int cmdline_parser_ext (int argc, char **argv,
-  struct gengetopt_args_info *args_info,
-  struct cmdline_parser_params *params);
-
-/**
- * Save the contents of the option struct into an already open FILE stream.
- * @param outfile the stream where to dump options
- * @param args_info the option struct to dump
- * @return 0 if everything went fine, NON 0 if an error took place
- */
-int cmdline_parser_dump(FILE *outfile,
-  struct gengetopt_args_info *args_info);
-
-/**
- * Save the contents of the option struct into a (text) file.
- * This file can be read by the config file parser (if generated by gengetopt)
- * @param filename the file where to save
- * @param args_info the option struct to save
- * @return 0 if everything went fine, NON 0 if an error took place
- */
-int cmdline_parser_file_save(const char *filename,
-  struct gengetopt_args_info *args_info);
-
-/**
- * Print the help
- */
-void cmdline_parser_print_help(void);
-/**
- * Print the version
- */
-void cmdline_parser_print_version(void);
-
-/**
- * Initializes all the fields a cmdline_parser_params structure 
- * to their default values
- * @param params the structure to initialize
- */
-void cmdline_parser_params_init(struct cmdline_parser_params *params);
-
-/**
- * Allocates dynamically a cmdline_parser_params structure and initializes
- * all its fields to their default values
- * @return the created and initialized cmdline_parser_params structure
- */
-struct cmdline_parser_params *cmdline_parser_params_create(void);
-
-/**
- * Initializes the passed gengetopt_args_info structure's fields
- * (also set default values for options that have a default)
- * @param args_info the structure to initialize
- */
-void cmdline_parser_init (struct gengetopt_args_info *args_info);
-/**
- * Deallocates the string fields of the gengetopt_args_info structure
- * (but does not deallocate the structure itself)
- * @param args_info the structure to deallocate
- */
-void cmdline_parser_free (struct gengetopt_args_info *args_info);
-
-/**
- * The config file parser (deprecated version)
- * @param filename the name of the config file
- * @param args_info the structure where option information will be stored
- * @param override whether to override possibly already present options
- * @param initialize whether to initialize the option structure my_args_info
- * @param check_required whether to check that all required options were provided
- * @return 0 if everything went fine, NON 0 if an error took place
- * @deprecated use cmdline_parser_config_file() instead
- */
-int cmdline_parser_configfile (const char *filename,
-  struct gengetopt_args_info *args_info,
-  int override, int initialize, int check_required);
-
-/**
- * The config file parser
- * @param filename the name of the config file
- * @param args_info the structure where option information will be stored
- * @param params additional parameters for the parser
- * @return 0 if everything went fine, NON 0 if an error took place
- */
-int cmdline_parser_config_file (const char *filename,
-  struct gengetopt_args_info *args_info,
-  struct cmdline_parser_params *params);
-
-/**
- * Checks that all the required options were specified
- * @param args_info the structure to check
- * @param prog_name the name of the program that will be used to print
- *   possible errors
- * @return
- */
-int cmdline_parser_required (struct gengetopt_args_info *args_info,
-  const char *prog_name);
-
-
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-#endif /* ZOPT_H */

zmap_conf_install.cmake.in

@@ -0,0 +1,6 @@
+foreach(conf_file ${CONF_FILES})
+	message(STATUS "${conf_file}")
+	if(NOT EXISTS "/etc/zmap/${conf_file}")
+		file(INSTALL "conf/${conf_file}" DESTINATION "/etc/zmap")
+	endif()
+endforeach()