Merge branch 'master' of github.com:zmap/zmap

INSTALL

@@ -29,3 +29,4 @@ followed by:
 Redis support is not enabled by default.  If you are want to use ZMap
 with Redis, you will first need to install Hiredis.  Then, rebuild
 ZMap with the command "make REDIS=true".
+

lib/blacklist.h

@@ -1,7 +1,7 @@
 #include <stdint.h>
 
-#ifndef _BLACKLIST_H
-#define _BLACKLIST_H
+#ifndef BLACKLIST_H
+#define BLACKLIST_H
 
 int blacklist_is_allowed(uint32_t s_addr);
 void blacklist_prefix(char *ip, int prefix_len);

lib/constraint.h

@@ -1,5 +1,5 @@
-#ifndef _CONSTRAINT_H
-#define _CONSTRAINT_H
+#ifndef CONSTRAINT_H
+#define CONSTRAINT_H
 
 typedef struct _constraint constraint_t;
 typedef int value_t;

lib/logger.c

@@ -53,6 +53,7 @@ static int LogLogVA(enum LogLevel level, const char *loggerName,
 		if (loggerName || logMessage) {
 			fputs("\n", log_output_stream);
 		}
+		fflush(log_output_stream);
 	}
 	return 0;
 }

lib/logger.h

@@ -1,8 +1,8 @@
 #include <stdio.h>
 #include <stdarg.h>
 
-#ifndef _LOGGER_H
-#define _LOGGER_H
+#ifndef LOGGER_H
+#define LOGGER_H
 
 enum LogLevel { LOG_FATAL, LOG_ERROR, LOG_WARN, LOG_INFO, LOG_DEBUG, LOG_TRACE,
 					NUM_LOGLEVELS };

lib/random.h

@@ -1,8 +1,8 @@
 #include <stdlib.h>
 #include <stdint.h>
 
-#ifndef _RANDOM_H
-#define _RANDOM_H
+#ifndef RANDOM_H
+#define RANDOM_H
 
 int random_bytes(void *dst, size_t n);
 

lib/redis.h

@@ -2,8 +2,8 @@
 #include <unistd.h>
 #include <hiredis/hiredis.h>
 
-#ifndef _REDIS_ZHELPERS_H
-#define _REDIS_ZHELPERS_H
+#ifndef REDIS_ZHELPERS_H
+#define REDIS_ZHELPERS_H
 
 int redis_init(void);
 

lib/rijndael-alg-fst.h

@@ -23,8 +23,8 @@
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#ifndef __RIJNDAEL_ALG_FST_H
-#define __RIJNDAEL_ALG_FST_H
+#ifndef RIJNDAEL_ALG_FST_H
+#define RIJNDAEL_ALG_FST_H
 
 #define MAXKC	(256/32)
 #define MAXKB	(256/8)
@@ -44,4 +44,4 @@ void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], in
 void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], int rounds);
 #endif /* INTERMEDIATE_VALUE_KAT */
 
-#endif /* __RIJNDAEL_ALG_FST_H */
+#endif /* RIJNDAEL_ALG_FST_H */

src/Makefile

@@ -7,7 +7,7 @@ PREFIX=/usr/local
 
 INSTALL=install
 INSTALLDATA=install -m 644
-mandir=/usr/share/man/man1/
+mandir=$(PREFIX)/man/man1/
 bindir=$(PREFIX)/sbin
 
 # Hardening and warnings for building with gcc
@@ -23,15 +23,16 @@ GCCWARNINGS = -Wall -Wformat=2 -Wno-format-nonliteral\
 GCCHARDENING=-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1
 LDHARDENING=-z relro -z now
 
-EXTRACFLAGS=-std=gnu99 -g -O2 $(EXTRA_CFLAGS) $(GCCHARDENING) $(GCCWARNINGS) -Werror
+EXTRACFLAGS=-std=gnu99 -g -O2 $(GCCHARDENING) $(GCCWARNINGS) $(EXTRA_CFLAGS) -Werror
 EXTRALDFLAGS= $(LDHARDENING)
 
 CFLAGS+=$(INCLUDE) $(EXTRACFLAGS)
 LDFLAGS+=$(EXTRALDFLAGS)
 
-modules=module_tcp_synscan.o module_icmp_echo.o module_udp.o #ADD YOUR MODULE HERE
+probemodules=module_tcp_synscan.o module_icmp_echo.o #module_udp.o #ADD YOUR PROBE MODULE HERE
+outputmodules= module_csv.o #ADD YOUR OUTPUT MODULE HERE
 
-objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt_compat.o zmap.o random.o output_modules.o module_simple_file.o module_extended_file.o packet.o probe_modules.o ${modules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o
+objects=constraint.o blacklist.o cyclic.o logger.o send.o recv.o state.o monitor.o zopt_compat.o zmap.o random.o output_modules.o packet.o probe_modules.o ${probemodules} ${outputmodules} validate.o rijndael-alg-fst.o get_gateway.o aesrand.o fieldset.o
 redis_objects=module_redis.o redis.o
 
 ifeq ($(REDIS), true)
@@ -55,6 +56,7 @@ zopt.c zopt.h: zopt.ggo
 install: zmap
 	$(INSTALL) zmap $(bindir)/zmap
 	test -d /etc/zmap || (mkdir /etc/zmap && $(INSTALLDATA) ../conf/* /etc/zmap/)
+	test -d $(mandir) || mkdir -p $(mandir)
 	$(INSTALLDATA) ./zmap.1 $(mandir)
 	@echo "\n**************\nSuccess! ZMap is installed. Try running (as root):\nzmap -p 80 -N 10 -B 1M -o -\n**************"
 

src/aesrand.h

@@ -8,8 +8,8 @@
 
 #include <stdint.h>
 
-#ifndef _AESRAND_H
-#define _AESRAND_H
+#ifndef AESRAND_H
+#define AESRAND_H
 
 void aesrand_init(uint32_t seed);
 

src/cyclic.h

@@ -8,8 +8,8 @@
 
 #include <stdint.h>
 
-#ifndef _CYCLIC_H
-#define _CYCLIC_H
+#ifndef CYCLIC_H
+#define CYCLIC_H
 
 int cyclic_init(uint32_t, uint32_t);
 

src/fieldset.c

@@ -0,0 +1,126 @@
+#include "fieldset.h"
+
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "../lib/logger.h"
+
+void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len)
+{
+	if (fds->len + len > MAX_FIELDS) {
+		log_fatal("fieldset", "out of room in field def set");
+	}
+	fielddef_t *open = &(fds->fielddefs[fds->len]);
+	memcpy(open, fs, len*sizeof(fielddef_t)); 
+	fds->len += len;
+}
+
+fieldset_t *fs_new_fieldset(void)
+{
+	fieldset_t *f = malloc(sizeof(fieldset_t));
+	if (!f) {
+		log_fatal("fieldset", "unable to allocate new fieldset");
+	}
+	memset(f, 0, sizeof(fieldset_t));
+	return f;	
+}
+
+static inline void fs_add_word(fieldset_t *fs, const char *name, int type,
+		int free_, size_t len, void *value)
+{
+	if (fs->len + 1 >= MAX_FIELDS) {
+		log_fatal("fieldset", "out of room in fieldset");
+	}
+	field_t *f = &(fs->fields[fs->len]);
+	fs->len++;
+	f->type = type;
+	f->name = name;
+	f->len = len;
+	f->value = value;
+	f->free_ = free_;
+}
+
+void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_)
+{
+	fs_add_word(fs, name, FS_STRING, free_, strlen(value), (void*) value);
+}
+
+void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value)
+{
+	fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), (void*) value);
+}
+
+void fs_add_binary(fieldset_t *fs, const char *name, size_t len,
+		void *value, int free_)
+{
+	fs_add_word(fs, name, FS_BINARY, free_, len, value);
+}
+
+uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index)
+{
+	return (uint64_t) fs->fields[index].value;
+}
+
+char* fs_get_string_by_index(fieldset_t *fs, int index)
+{
+	return (char*) fs->fields[index].value;
+}
+
+int fds_get_index_by_name(fielddefset_t *fds, char *name)
+{
+	for (int i=0; i < fds->len; i++) {
+		if (!strcmp(fds->fielddefs[i].name, name)) {
+			return i;
+		}
+	}
+	return -1;
+}
+
+void fs_free(fieldset_t *fs)
+{
+	if (!fs) {
+		return;
+	}
+	for (int i=0; i < fs->len; i++) {
+		field_t *f = &(fs->fields[i]);
+		if (f->free_) {
+			free(f->value);
+		}
+	}
+	free(fs);
+}
+
+void fs_generate_fieldset_translation(translation_t *t, 
+		fielddefset_t *avail, char** req, int reqlen)
+{
+	memset(t, 0, sizeof(translation_t));
+	if (!t) {
+		log_fatal("fieldset", "unable to allocate memory for translation");
+	}
+	for (int i=0; i < reqlen; i++) {
+		int l = fds_get_index_by_name(avail, req[i]);
+		if (l < 0) {
+			log_fatal("fieldset", "specified field (%s) not "
+					      "available in selected "
+					      "probe module.", req[i]);
+		} 
+		t->translation[t->len++] = l;
+	}
+}
+
+fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t)
+{
+	fieldset_t *retv = fs_new_fieldset();
+	if (!retv) {
+		log_fatal("fieldset", "unable to allocate space for translated field set");
+	}
+	for (int i=0; i < t->len; i++) {
+		int o = t->translation[i];
+		memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t));
+	}
+	retv->len = t->len;
+	return retv;
+}
+

src/fieldset.h

@@ -0,0 +1,93 @@
+/*
+ * ZMap Copyright 2013 Regents of the University of Michigan 
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include "types.h"
+
+#ifndef FIELDSET_H
+#define FIELDSET_H
+
+// maximum number of records that can be stored in a fieldset
+#define MAX_FIELDS 128
+
+// types of data that can be stored in a field
+#define FS_STRING 0
+#define FS_UINT64 1
+#define FS_BINARY 2
+
+// definition of a field that's provided by a probe module
+// these are used so that users can ask at the command-line
+// what fields are available for consumption
+typedef struct field_def {
+	const char *name;
+	const char *type;
+	const char *desc;
+} fielddef_t;
+
+typedef struct fielddef_set {
+	fielddef_t fielddefs[MAX_FIELDS];
+	int len;
+} fielddefset_t;
+
+// the internal field type used by fieldset
+typedef struct field {
+	const char *name;
+	int type;
+	int free_;
+	size_t len;
+	void *value;
+} field_t;
+
+// data structure that is populated by the probe module
+// and translated into the data structure that's passed
+// to the output module
+typedef struct fieldset {
+	int len;
+	field_t fields[MAX_FIELDS];
+} fieldset_t;
+
+// we pass a different fieldset to an output module than 
+// the probe module generates for us because a user may
+// only want certain fields and will expect them in a certain
+// order. We generate a translated fieldset that contains
+// only the fields we want to export to the output module.
+// a translation specifies how to efficiently convert the fs
+// povided by the probe module to the fs for the output module.
+typedef struct translation {
+	int len;
+	int translation[MAX_FIELDS];
+} translation_t;
+
+
+fieldset_t *fs_new_fieldset(void);
+
+char* fs_get_string_by_index(fieldset_t *fs, int index);
+
+int fds_get_index_by_name(fielddefset_t *fds, char *name);
+
+void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len);
+
+void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value);
+
+void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_);
+
+void fs_add_binary(fieldset_t *fs, const char *name, size_t len,
+		void *value, int free_);
+
+uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index);
+
+void fs_free(fieldset_t *fs);
+
+void fs_generate_fieldset_translation(translation_t *t, 
+		fielddefset_t *avail, char** req, int reqlen);
+
+fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t);
+
+#endif // FIELDSET_H
+

src/get_gateway.h

@@ -6,8 +6,8 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef _GET_GATEWAY_H
-#define _GET_GATEWAY_H
+#ifndef GET_GATEWAY_H
+#define GET_GATEWAY_H
 
 #include <netinet/in.h>
 

src/monitor.h

@@ -6,8 +6,8 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef _MONITOR_H
-#define _MONITOR_H
+#ifndef MONITOR_H
+#define MONITOR_H
 
 void monitor_run();
 

src/output_modules/module_csv.c

@@ -0,0 +1,103 @@
+/*
+ * ZMap Copyright 2013 Regents of the University of Michigan 
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <assert.h>
+
+#include "../../lib/logger.h"
+#include "../fieldset.h"
+
+#include "output_modules.h"
+
+static FILE *file = NULL;
+
+int csv_init(struct state_conf *conf, char **fields, int fieldlens)
+{
+	assert(conf);
+	if (conf->output_filename) {
+		if (!strcmp(conf->output_filename, "-")) {
+			file = stdout;
+		} else {
+			if (!(file = fopen(conf->output_filename, "w"))) {
+				log_fatal("csv", "could not open output file (%s)",
+					conf->output_filename);
+			}
+		}
+	} else {
+		log_warn("csv", "no output file selected. "
+				   "no results will be provided.");
+	}
+	if (fieldlens > 1 && file) {
+		for (int i=0; i < fieldlens; i++) {
+			if (i) {
+				fprintf(file, ", ");
+			}
+			fprintf(file, "%s", fields[i]);
+		}	
+	}
+	return EXIT_SUCCESS;
+}
+
+int csv_close(__attribute__((unused)) struct state_conf* c, 
+	__attribute__((unused)) struct state_send* s, 
+	__attribute__((unused)) struct state_recv* r)
+{
+	if (file) {
+		fflush(file);
+		fclose(file);
+	}
+	return EXIT_SUCCESS;
+}
+
+static void hex_encode(FILE *f, unsigned char* readbuf, size_t len) 
+{ 
+	for(size_t i=0; i < len; i++) { 
+		fprintf(f, "%02x", readbuf[i]); 
+	} 
+}
+
+int csv_process(fieldset_t *fs)
+{
+	if (!file) {
+		return EXIT_SUCCESS;
+	}
+	for (int i=0; i < fs->len; i++) {
+		field_t *f = &(fs->fields[i]);
+		if (i) {
+			fprintf(file, ", ");
+		}
+		if (f->type == FS_STRING) {
+			fprintf(file, "%s", (char*) f->value); 
+		} else if (f->type == FS_UINT64) {
+			fprintf(file, "%lu", (uint64_t) f->value); 
+		} else if (f->type == FS_BINARY) {
+			hex_encode(file, (unsigned char*) f->value, f->len);
+		} else {
+			log_fatal("csv", "received unknown output type");
+		}
+	}
+	fprintf(file, "\n");
+	return EXIT_SUCCESS;
+}
+
+output_module_t module_csv_file = {
+	.name = "csv",
+	.init = &csv_init,
+	.start = NULL,
+	.update = NULL,
+	.update_interval = 0,
+	.close = &csv_close,
+	.process_ip = &csv_process,
+};
+

src/output_modules/module_extended_file.c

@@ -1,109 +0,0 @@
-/*
- * ZMap Copyright 2013 Regents of the University of Michigan 
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <assert.h>
-
-#include <sys/time.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-
-#include "../../lib/logger.h"
-
-#include "output_modules.h"
-
-static FILE *file = NULL;
-#define UNUSED __attribute__((unused))
-
-int extendedfile_init(struct state_conf *conf)
-{
-	assert(conf);
-	if (conf->output_filename) {
-		if (!strcmp(conf->output_filename, "-")) {
-			file = stdout;
-		} else {
-			if (!(file = fopen(conf->output_filename, "w"))) {
-				perror("Couldn't open output file");
-				exit(EXIT_FAILURE);
-			}
-		}
-		fprintf(file, "response, saddr, daddr, sport, "
-						"dport, seq, ack, in_cooldown, is_repeat, timestamp\n");
-	}
-	return EXIT_SUCCESS;
-}
-
-static void fprint_tv(FILE *f, struct timeval *tv) 
-{
-	char time_string[40]; 
-	struct tm *ptm = localtime(&tv->tv_sec); 
-	strftime(time_string, sizeof (time_string),
-					"%Y-%m-%d %H:%M:%S", ptm);
-	long milliseconds = tv->tv_usec / 1000; 
-	fprintf(f, "%s.%03ld\n", time_string, milliseconds); 
-} 
-
-
-int extendedfile_ip(ipaddr_n_t saddr, ipaddr_n_t daddr,
-		const char *response_type, int is_repeat,
-		int in_cooldown, const u_char *packet, size_t buflen)
-{
-	struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)];
-	if (buflen < (sizeof(struct ethhdr) + ip_hdr->ihl*4 + sizeof(struct tcphdr)))
-		return EXIT_FAILURE;
-	struct tcphdr *tcp = (struct tcphdr *)((char *)ip_hdr + ip_hdr->ihl * 4);
-
-	if (file) {
-		struct in_addr addr;
-		addr.s_addr = saddr;
-		// inet_ntoa returns a <<const>> char * 
-		fprintf(file, "%s, %s, ", 
-				response_type, 
-				inet_ntoa(addr));
-		addr.s_addr = daddr;
-		fprintf(file, "%s, %u, %u, %u, %u, %i, %i,", 
-				inet_ntoa(addr),
-				ntohs(tcp->source),
-				ntohs(tcp->dest),
-				ntohl(tcp->seq),
-				ntohl(tcp->ack_seq),
-				in_cooldown,
-				is_repeat);
-		struct timeval t;
-		gettimeofday(&t, NULL);
-		fprint_tv(file, &t);
-		fflush(file);
-	}
-	return EXIT_SUCCESS;
-}
-
-int extendedfile_close(UNUSED struct state_conf* c, 
-		UNUSED struct state_send* s, UNUSED struct state_recv* r)
-{
-	if (file) {
-		fflush(file);
-		fclose(file);
-	}
-	return EXIT_SUCCESS;
-}
-
-output_module_t module_extended_file = {
-	.name = "extended_file",
-	.init = &extendedfile_init,
-	.start = NULL,
-	.update = NULL,
-	.update_interval = 0,
-	.close = &extendedfile_close,
-	.success_ip = &extendedfile_ip,
-	.other_ip = &extendedfile_ip
-};
-

src/output_modules/module_extended_file.h

@@ -1,20 +0,0 @@
-/*
- * ZMap Copyright 2013 Regents of the University of Michigan 
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- */
-
-#include <output_modules.h>
-
-int extendedfile_init(struct state_conf *conf);
-
-int extendedfile_ip(ipaddr_n_t saddr, ipaddr_n_t daddr,
-        port_n_t sport, port_n_t dport, struct timeval* t,
-        const char *response_type, int is_repeat,
-        int in_cooldown, const u_char *packet);
-
-int extendedfile_close(struct state_conf* c, struct state_send* s,
-		struct state_recv* r);
-

src/output_modules/module_simple_file.c

@@ -1,75 +0,0 @@
-/*
- * ZMap Copyright 2013 Regents of the University of Michigan 
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- */
-
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <assert.h>
-#include <logger.h>
-
-#include "output_modules.h"
-
-#define UNUSED __attribute__((unused))
-
-static FILE *file = NULL;
-
-int simplefile_init(struct state_conf *conf)
-{
-	assert(conf);
-	if (conf->output_filename) {
-		if (!strcmp(conf->output_filename, "-")) {
-			file = stdout;
-		} else {
-			if (!(file = fopen(conf->output_filename, "w"))) {
-				perror("Couldn't open output file");
-				exit(EXIT_FAILURE);
-			}
-		}
-	}
-	return EXIT_SUCCESS;
-}
-
-int simplefile_synack_newip(ipaddr_n_t saddr, UNUSED ipaddr_n_t daddr,
-		UNUSED const char *response_type,
-		int is_repeat, UNUSED int in_cooldown, UNUSED const u_char *packet,
-		UNUSED size_t buflen)
-{	
-	if (file && !is_repeat) {
-		struct in_addr addr;
-		addr.s_addr = saddr;
-		fprintf(file, "%s\n", inet_ntoa(addr));
-	}
-	fflush(file);
-	return EXIT_SUCCESS;
-}
-
-int simplefile_close(UNUSED struct state_conf* c, 
-	UNUSED struct state_send* s, 
-	UNUSED struct state_recv* r)
-{
-	if (file) {
-		fflush(file);
-		fclose(file);
-	}
-	return EXIT_SUCCESS;
-}
-
-output_module_t module_simple_file = {
-	.name = "simple_file",
-	.init = &simplefile_init,
-	.start = NULL,
-	.update = NULL,
-	.update_interval = 0,
-	.close = &simplefile_close,
-	.success_ip = &simplefile_synack_newip,
-	.other_ip = NULL,
-};

src/output_modules/output_modules.c

@@ -12,20 +12,17 @@
 
 #include "output_modules.h"
 
-extern output_module_t module_simple_file;
-extern output_module_t module_extended_file;
+extern output_module_t module_csv_file;
 // ADD YOUR MODULE HERE
 
 #ifdef REDIS
 extern output_module_t module_redis;
 #endif
 
-
 output_module_t* output_modules[] = {
-	&module_simple_file,
-	&module_extended_file,
+	&module_csv_file
 #ifdef REDIS
-	&module_redis,
+	//&module_redis,
 #endif
 	// ADD YOUR MODULE HERE
 };
@@ -50,3 +47,4 @@ void print_output_modules(void)
 		printf("%s\n", output_modules[i]->name);
 	}
 }
+

src/output_modules/output_modules.h

@@ -6,39 +6,35 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef HEADER_OUTPUT_MODULES_H
-#define HEADER_OUTPUT_MODULES_H
+#ifndef OUTPUT_MODULES_H
+#define OUTPUT_MODULES_H
 
 #include "../state.h"
+#include "../fieldset.h"
 
 // called at scanner initialization
-typedef int (*output_init_cb)(struct state_conf *);
+typedef int (*output_init_cb)(struct state_conf *, char **fields, int fieldslen);
 
 // called on packet receipt
-typedef int (*output_packet_cb)(ipaddr_n_t src_ip, ipaddr_n_t dst_ip,
-				const char* response_type,
-				int is_repeat, int in_cooldown,
-				const u_char* packetbuf, size_t buflen);
+typedef int (*output_packet_cb)(fieldset_t *fs);
 
 // called periodically during the scan
-typedef int (*output_update_cb)(struct state_conf*, struct state_send*, struct state_recv*);
-
+typedef int (*output_update_cb)(struct state_conf*, 
+		struct state_send*, struct state_recv*);
 
 typedef struct output_module {
 	const char *name;
 	unsigned update_interval;
-
 	output_init_cb init;
 	output_update_cb start;
 	output_update_cb update;
 	output_update_cb close;
-	output_packet_cb success_ip;
-	output_packet_cb other_ip;
-
+	output_packet_cb process_ip;
 } output_module_t;
 
 
 output_module_t* get_output_module_by_name(const char*);
+
 void print_output_modules(void);
 
 #endif // HEADER_OUTPUT_MODULES_H

src/probe_modules/module_icmp_echo.c

@@ -24,6 +24,7 @@
 #include <arpa/inet.h>
 
 #include "probe_modules.h"
+#include "../fieldset.h"
 #include "packet.h"
 #include "validate.h"
 
@@ -81,59 +82,15 @@ void icmp_echo_print_packet(FILE *fp, void* packet)
 			ntohs(icmp_header->icmp_cksum),
 			ntohs(icmp_header->icmp_id),
 			ntohs(icmp_header->icmp_seq));
-	struct in_addr *s = (struct in_addr *) &(iph->saddr);
-	struct in_addr *d = (struct in_addr *) &(iph->daddr);
-	char srcip[20];
-	char dstip[20];
-	// inet_ntoa is a const char * so we if just call it in
-	// fprintf, you'll get back wrong results since we're
-	// calling it twice.
-	strncpy(srcip, inet_ntoa(*s), 19);
-	strncpy(dstip, inet_ntoa(*d), 19);
-	fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n",
-			srcip,
-			dstip,
-			ntohl(iph->check));
-	fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | "
-			"dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n",
-			(int) ((unsigned char *) ethh->h_source)[0],
-			(int) ((unsigned char *) ethh->h_source)[1],
-			(int) ((unsigned char *) ethh->h_source)[2],
-			(int) ((unsigned char *) ethh->h_source)[3],
-			(int) ((unsigned char *) ethh->h_source)[4],
-			(int) ((unsigned char *) ethh->h_source)[5],
-			(int) ((unsigned char *) ethh->h_dest)[0],
-			(int) ((unsigned char *) ethh->h_dest)[1],
-			(int) ((unsigned char *) ethh->h_dest)[2],
-			(int) ((unsigned char *) ethh->h_dest)[3],
-			(int) ((unsigned char *) ethh->h_dest)[4],
-			(int) ((unsigned char *) ethh->h_dest)[5]);
+	fprintf_ip_header(fp, iph);
+	fprintf_eth_header(fp, ethh);
 	fprintf(fp, "------------------------------------------------------\n");
 }
 
-response_type_t* icmp_echo_classify_packet(const u_char *packet, uint32_t len)
-{
-	(void)len;
-	struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)];
-	struct icmp *icmp_hdr = (struct icmp*)((char *)ip_hdr
-					+ sizeof(struct iphdr)); 
-	switch (icmp_hdr->icmp_type) {
-		case ICMP_ECHOREPLY:
-			return &(module_icmp_echo.responses[0]);
-		case ICMP_UNREACH: 
-			return &(module_icmp_echo.responses[1]);
-		case ICMP_SOURCEQUENCH:
-			return &(module_icmp_echo.responses[2]);
-		case ICMP_REDIRECT:
-			return &(module_icmp_echo.responses[3]);
-		case ICMP_TIMXCEED:
-			return &(module_icmp_echo.responses[4]);
-		default:
-			return &(module_icmp_echo.responses[5]);
-	}
-}
 
-int icmp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src_ip, uint32_t *validation)
+
+int icmp_validate_packet(const struct iphdr *ip_hdr, 
+		uint32_t len, uint32_t *src_ip, uint32_t *validation)
 {
 	if (ip_hdr->protocol != IPPROTO_ICMP) {
 		return 0;
@@ -175,33 +132,47 @@ int icmp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src
 	return 1;
 }
 
-static response_type_t responses[] = {
-	{
-		.name = "echoreply",
-		.is_success = 1	
-	},
-	{
-		.name = "unreach",
-		.is_success = 0
-	},
-	{
-		.name = "sourcequench",
-		.is_success = 0
-	},
-	{
-		.name = "redirect",
-		.is_success = 0
-	},
-	{
-		.name = "timxceed",
-		.is_success = 0
-	},
-	{
-		.name = "other",
-		.is_success = 0
+void icmp_echo_process_packet(const u_char *packet,
+		__attribute__((unused)) uint32_t len, fieldset_t *fs)
+{
+	struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)];
+	struct icmphdr *icmp_hdr = (struct icmphdr*)((char *)ip_hdr + 4 *ip_hdr->ihl);
+	fs_add_uint64(fs, "type", ntohs(icmp_hdr->type));
+	fs_add_uint64(fs, "code", ntohs(icmp_hdr->code));
+	fs_add_uint64(fs, "icmp-id", ntohs(icmp_hdr->un.echo.id));
+	fs_add_uint64(fs, "seq", ntohs(icmp_hdr->un.echo.sequence));
+	switch (icmp_hdr->type) {
+		case ICMP_ECHOREPLY:
+			fs_add_string(fs, "classification", (char*) "echoreply", 0); 
+			fs_add_uint64(fs, "success", 1); 
+		case ICMP_UNREACH: 
+			fs_add_string(fs, "classification", (char*) "unreach", 0); 
+			fs_add_uint64(fs, "success", 0); 
+		case ICMP_SOURCEQUENCH:
+			fs_add_string(fs, "classification", (char*) "sourcequench", 0); 
+			fs_add_uint64(fs, "success", 0); 
+		case ICMP_REDIRECT:
+			fs_add_string(fs, "classification", (char*) "redirect", 0); 
+			fs_add_uint64(fs, "success", 0); 
+		case ICMP_TIMXCEED:
+			fs_add_string(fs, "classification", (char*) "timxceed", 0); 
+			fs_add_uint64(fs, "success", 0); 
+		default:
+			fs_add_string(fs, "classification", (char*) "other", 0); 
+			fs_add_uint64(fs, "success", 0); 
 	}
+}
+
+fielddef_t fields[] = {
+	{.name="type", .type="int", .desc="icmp message type"},
+	{.name="code", .type="int", .desc="icmp message sub type code"},
+	{.name="icmp-id", .type="int", .desc="icmp id number"},
+	{.name="seq", .type="int", .desc="icmp sequence number"},
+	{.name="classification", .type="string", .desc="probe module classification"},
+	{.name="success", .type="int", .desc="did probe module classify response as success"}
 };
 
+
 probe_module_t module_icmp_echo = {
 	.name = "icmp_echoscan",
 	.packet_length = 62,
@@ -211,9 +182,9 @@ probe_module_t module_icmp_echo = {
 	.thread_initialize = &icmp_echo_init_perthread,
 	.make_packet = &icmp_echo_make_packet,
 	.print_packet = &icmp_echo_print_packet,
-	.classify_packet = &icmp_echo_classify_packet,
+	.process_packet = &icmp_echo_process_packet,
 	.validate_packet = &icmp_validate_packet,
 	.close = NULL,
-	.responses = responses
-};
+	.fields = fields,
+	.numfields = 6};
 

src/probe_modules/module_tcp_synscan.c

@@ -13,6 +13,7 @@
 #include <stdint.h>
 #include <unistd.h>
 #include <string.h>
+#include <assert.h>
 
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
@@ -22,11 +23,18 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
+#include "../fieldset.h"
 #include "probe_modules.h"
 #include "packet.h"
 
 probe_module_t module_tcp_synscan;
-uint32_t num_ports = 1;
+static uint32_t num_ports;
+
+int synscan_global_initialize(struct state_conf *state)
+{
+	num_ports = state->source_port_last - state->source_port_first + 1;
+	return EXIT_SUCCESS;
+}
 
 int synscan_init_perthread(void* buf, macaddr_t *src,
 		macaddr_t *gw, port_h_t dst_port)
@@ -39,7 +47,6 @@ int synscan_init_perthread(void* buf, macaddr_t *src,
 	make_ip_header(ip_header, IPPROTO_TCP, len);
 	struct tcphdr *tcp_header = (struct tcphdr*)(&ip_header[1]);
 	make_tcp_header(tcp_header, dst_port);
-	num_ports = zconf.source_port_last - zconf.source_port_first + 1;
 	return EXIT_SUCCESS;
 }
 
@@ -49,15 +56,13 @@ int synscan_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip,
 	struct ethhdr *eth_header = (struct ethhdr *)buf;
 	struct iphdr *ip_header = (struct iphdr*)(&eth_header[1]);
 	struct tcphdr *tcp_header = (struct tcphdr*)(&ip_header[1]);
-	uint16_t src_port = zconf.source_port_first 
-					+ ((validation[1] + probe_num) % num_ports);
 	uint32_t tcp_seq = validation[0];
-	
 
 	ip_header->saddr = src_ip;
 	ip_header->daddr = dst_ip;
 
-	tcp_header->source = htons(src_port);
+	tcp_header->source = htons(get_src_port(num_ports,
+				probe_num, validation));
 	tcp_header->seq = tcp_seq;
 	tcp_header->check = 0;
 	tcp_header->check = tcp_checksum(sizeof(struct tcphdr),
@@ -79,69 +84,18 @@ void synscan_print_packet(FILE *fp, void* packet)
 			ntohs(tcph->dest),
 			ntohl(tcph->seq),
 			ntohl(tcph->check));
-	struct in_addr *s = (struct in_addr *) &(iph->saddr);
-	struct in_addr *d = (struct in_addr *) &(iph->daddr);
-	char srcip[20];
-	char dstip[20];
-	// inet_ntoa is a const char * so we if just call it in
-	// fprintf, you'll get back wrong results since we're
-	// calling it twice.
-	strncpy(srcip, inet_ntoa(*s), 19);
-	strncpy(dstip, inet_ntoa(*d), 19);
-	fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n",
-			srcip,
-			dstip,
-			ntohl(iph->check));
-	fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | "
-			"dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n",
-			(int) ((unsigned char *) ethh->h_source)[0],
-			(int) ((unsigned char *) ethh->h_source)[1],
-			(int) ((unsigned char *) ethh->h_source)[2],
-			(int) ((unsigned char *) ethh->h_source)[3],
-			(int) ((unsigned char *) ethh->h_source)[4],
-			(int) ((unsigned char *) ethh->h_source)[5],
-			(int) ((unsigned char *) ethh->h_dest)[0],
-			(int) ((unsigned char *) ethh->h_dest)[1],
-			(int) ((unsigned char *) ethh->h_dest)[2],
-			(int) ((unsigned char *) ethh->h_dest)[3],
-			(int) ((unsigned char *) ethh->h_dest)[4],
-			(int) ((unsigned char *) ethh->h_dest)[5]);
+	fprintf_ip_header(fp, iph);
+	fprintf_eth_header(fp, ethh);
 	fprintf(fp, "------------------------------------------------------\n");
 }
 
-response_type_t* synscan_classify_packet(const u_char *packet, uint32_t len)
-{
-	(void)len;
-	struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)];
-	struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr 
-					+ (sizeof(struct iphdr)));
-	if (tcp->rst) { // RST packet
-			return &(module_tcp_synscan.responses[1]);
-	} else { // SYNACK packet
-			return &(module_tcp_synscan.responses[0]);
-	}
-}
-// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise
-static inline int check_dst_port(uint16_t port, uint32_t *validation)
-{
-	if (port > zconf.source_port_last 
-					|| port < zconf.source_port_first) {
-		return EXIT_FAILURE;
-	}
-	int32_t to_validate = port - zconf.source_port_first;
-	int32_t min = validation[1] % num_ports;
-	int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports;
-
-	return (((max - min) % num_ports) >= ((to_validate - min) % num_ports));
-}
-
 int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len, 
-				__attribute__((unused))uint32_t *src_ip, uint32_t *validation)
+		__attribute__((unused))uint32_t *src_ip, 
+		uint32_t *validation)
 {
 	if (ip_hdr->protocol != IPPROTO_TCP) {
 		return 0;
 	}
-
 	if ((4*ip_hdr->ihl + sizeof(struct tcphdr)) > len) {
 		// buffer not large enough to contain expected tcp header 
 		return 0;
@@ -149,34 +103,51 @@ int synscan_validate_packet(const struct iphdr *ip_hdr, uint32_t len,
 	struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr + 4*ip_hdr->ihl);
 	uint16_t sport = tcp->source;
 	uint16_t dport = tcp->dest;
-
 	// validate source port
 	if (ntohs(sport) != zconf.target_port) {
 		return 0;
 	}
-
 	// validate destination port
-	if (!check_dst_port(ntohs(dport), validation)) {
+	if (!check_dst_port(ntohs(dport), num_ports, validation)) {
 		return 0;
 	}
-
 	// validate tcp acknowledgement number
 	if (htonl(tcp->ack_seq) != htonl(validation[0])+1) {
 		return 0;
 	}
-	
 	return 1;
 }
 
-static response_type_t responses[] = {
-	{
-		.is_success = 1,	
-		.name = "synack"
-	},
-	{
-		.is_success = 0,
-		.name = "rst"
+void synscan_process_packet(const u_char *packet,
+		__attribute__((unused)) uint32_t len, fieldset_t *fs)
+{
+	struct iphdr *ip_hdr = (struct iphdr *)&packet[sizeof(struct ethhdr)];
+	struct tcphdr *tcp = (struct tcphdr*)((char *)ip_hdr 
+					+ (sizeof(struct iphdr)));
+
+	fs_add_uint64(fs, "sport", (uint64_t) ntohs(tcp->source)); 
+	fs_add_uint64(fs, "dport", (uint64_t) ntohs(tcp->dest));
+	fs_add_uint64(fs, "seqnum", (uint64_t) ntohs(tcp->seq));
+	fs_add_uint64(fs, "acknum", (uint64_t) ntohl(tcp->ack_seq));
+	fs_add_uint64(fs, "window", (uint64_t) ntohs(tcp->window));
+
+	if (tcp->rst) { // RST packet
+		fs_add_string(fs, "classification", (char*) "rst", 0);
+		fs_add_uint64(fs, "success", 0);
+	} else { // SYNACK packet
+		fs_add_string(fs, "classification", (char*) "synack", 0);
+		fs_add_uint64(fs, "success", 1);
 	}
+}
+
+static fielddef_t fields[] = {
+	{.name = "sport",  .type = "int", .desc = "TCP source port"},
+	{.name = "dport",  .type = "int", .desc = "TCP destination port"},
+	{.name = "seqnum", .type = "int", .desc = "TCP sequence number"},
+	{.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"},
+	{.name = "window", .type = "int", .desc = "TCP window"},
+	{.name = "classification", .type="string", .desc = "packet classification"},
+	{.name = "success", .type="int", .desc = "is response considered success"}
 };
 
 probe_module_t module_tcp_synscan = {
@@ -185,12 +156,18 @@ probe_module_t module_tcp_synscan = {
 	.pcap_filter = "tcp && tcp[13] & 4 != 0 || tcp[13] == 18",
 	.pcap_snaplen = 96,
 	.port_args = 1,
+	.global_initialize = &synscan_global_initialize,
 	.thread_initialize = &synscan_init_perthread,
 	.make_packet = &synscan_make_packet,
 	.print_packet = &synscan_print_packet,
-	.classify_packet = &synscan_classify_packet,
+	.process_packet = &synscan_process_packet,
 	.validate_packet = &synscan_validate_packet,
 	.close = NULL,
-	.responses = responses,
-};
+	.helptext = "Probe module that sends a TCP SYN packet to a specific "
+		"port. Possible classifications are: synack and rst. A "
+		"SYN-ACK packet is considered a success and a reset packet "
+		"is considered a failed response.",
+
+	.fields = fields,
+	.numfields = 7};
 

src/probe_modules/module_udp.c

@@ -34,25 +34,26 @@ int udp_send_msg_len = 0;
 
 const char *udp_send_msg_default = "GET / HTTP/1.1\r\nHost: www\r\n\r\n"; 
 
-static int num_ports = 1;
+static int num_ports;
 
 probe_module_t module_udp;
 
-
-int udp_global_initialize(struct state_conf * zconf) {
+int udp_global_initialize(struct state_conf *conf) {
 	char *args, *c;
 	int i;
 	unsigned int n;
 
 	FILE *inp;
 
+	num_ports = conf->source_port_last - conf->source_port_first + 1;
+
 	udp_send_msg = strdup(udp_send_msg_default);
 	udp_send_msg_len = strlen(udp_send_msg);
 
-	if (! (zconf->probe_args && strlen(zconf->probe_args) > 0)) 
+	if (!(conf->probe_args && strlen(conf->probe_args) > 0)) 
 		return(0);
 	
-	args = strdup(zconf->probe_args);
+	args = strdup(conf->probe_args);
 	if (! args) exit(1);
 
 	c = strchr(args, ':');
@@ -108,14 +109,17 @@ int udp_global_initialize(struct state_conf * zconf) {
 			udp_send_msg[i] = (n & 0xff);
 		}
 	} else {
-		log_fatal("udp", "unknown UDP probe specification (expected file:/path, text:STRING, or hex:01020304)");
+		log_fatal("udp", "unknown UDP probe specification "
+				 "(expected file:/path, text:STRING, "
+				 "or hex:01020304)");
 		free(udp_send_msg);
 		free(args);
 		exit(1);
 	}
 
 	if (udp_send_msg_len > MAX_UDP_PAYLOAD_LEN) {
-		log_warn("udp", "warning: reducing UDP payload to %d bytes (from %d) to fit on the wire\n", 
+		log_warn("udp", "warning: reducing UDP payload to %d "
+			        "bytes (from %d) to fit on the wire\n", 
 				MAX_UDP_PAYLOAD_LEN, udp_send_msg_len);
 		udp_send_msg_len = MAX_UDP_PAYLOAD_LEN;
 	}
@@ -131,7 +135,6 @@ int udp_global_cleanup(__attribute__((unused)) struct state_conf *zconf,
 	return(0);
 }
 
-
 int udp_init_perthread(void* buf, macaddr_t *src,
 		macaddr_t *gw, __attribute__((unused)) port_h_t dst_port)
 {
@@ -154,8 +157,6 @@ int udp_init_perthread(void* buf, macaddr_t *src,
 
 	memcpy(payload, udp_send_msg, udp_send_msg_len);
 
-	num_ports = zconf.source_port_last - zconf.source_port_first + 1;
-
 	return EXIT_SUCCESS;
 }
 
@@ -165,12 +166,11 @@ int udp_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip,
 	struct ethhdr *eth_header = (struct ethhdr *)buf;
 	struct iphdr *ip_header = (struct iphdr*)(&eth_header[1]);
 	struct udphdr *udp_header = (struct udphdr*)(&ip_header[1]);
-	uint16_t src_port = zconf.source_port_first
-					+ ((validation[1] + probe_num) % num_ports);
 
 	ip_header->saddr = src_ip;
 	ip_header->daddr = dst_ip;
-	udp_header->source = src_port;
+	udp_header->source = get_src_port(num_ports, probe_num,
+					validation);
 
 	ip_header->check = 0;
 	ip_header->check = ip_checksum((unsigned short *) ip_header);
@@ -187,38 +187,11 @@ void udp_print_packet(FILE *fp, void* packet)
 			ntohs(udph->source),
 			ntohs(udph->dest),
 			ntohl(udph->check));
-	//ip_header = (struct iphdr*)(&eth_header[1])
-	struct in_addr *s = (struct in_addr *) &(iph->saddr);
-	struct in_addr *d = (struct in_addr *) &(iph->daddr);
-	char srcip[20];
-	char dstip[20];
-	// inet_ntoa is a const char * so we if just call it in
-	// fprintf, you'll get back wrong results since we're
-	// calling it twice.
-	strncpy(srcip, inet_ntoa(*s), 19);
-	strncpy(dstip, inet_ntoa(*d), 19);
-	fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n",
-			srcip,
-			dstip,
-			ntohl(iph->check));
-	fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | "
-			"dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n",
-			(int) ((unsigned char *) ethh->h_source)[0],
-			(int) ((unsigned char *) ethh->h_source)[1],
-			(int) ((unsigned char *) ethh->h_source)[2],
-			(int) ((unsigned char *) ethh->h_source)[3],
-			(int) ((unsigned char *) ethh->h_source)[4],
-			(int) ((unsigned char *) ethh->h_source)[5],
-			(int) ((unsigned char *) ethh->h_dest)[0],
-			(int) ((unsigned char *) ethh->h_dest)[1],
-			(int) ((unsigned char *) ethh->h_dest)[2],
-			(int) ((unsigned char *) ethh->h_dest)[3],
-			(int) ((unsigned char *) ethh->h_dest)[4],
-			(int) ((unsigned char *) ethh->h_dest)[5]);
+	fprintf_ip_header(fp, iph);
+	fprintf_eth_header(fp, ethh);
 	fprintf(fp, "------------------------------------------------------\n");
 }
 
-
 response_type_t* udp_classify_packet(const u_char *packet, uint32_t len)
 {
 	(void)len;
@@ -232,20 +205,6 @@ response_type_t* udp_classify_packet(const u_char *packet, uint32_t len)
 	}
 }
 
-// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise
-static inline int check_dst_port(uint16_t port, uint32_t *validation)
-{
-	if (port > zconf.source_port_last 
-					|| port < zconf.source_port_first) {
-		return EXIT_FAILURE;
-	}
-	int32_t to_validate = port - zconf.source_port_first;
-	int32_t min = validation[1] % num_ports;
-	int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports;
-
-	return (((max - min) % num_ports) >= ((to_validate - min) % num_ports));
-}
-
 int udp_validate_packet(const struct iphdr *ip_hdr, uint32_t len, 
 		__attribute__((unused))uint32_t *src_ip, uint32_t *validation)
 {
@@ -290,7 +249,7 @@ int udp_validate_packet(const struct iphdr *ip_hdr, uint32_t len,
 	if (dport != zconf.target_port) {
 		return 0;
 	}
-	if (!check_dst_port(sport, validation)) {
+	if (!check_dst_port(sport, num_ports, validation)) {
 		return 0;
 	}
 	return 1;

src/probe_modules/packet.c

@@ -6,7 +6,6 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-
 #include "packet.h"
 
 #include <stdlib.h>
@@ -36,6 +35,41 @@ void print_macaddr(struct ifreq* i)
 			(int) ((unsigned char *) &i->ifr_hwaddr.sa_data)[5]);
 }
 
+void fprintf_ip_header(FILE *fp, struct iphdr *iph)
+{
+	struct in_addr *s = (struct in_addr *) &(iph->saddr);
+	struct in_addr *d = (struct in_addr *) &(iph->daddr);
+	char srcip[20];
+	char dstip[20];
+	// inet_ntoa is a const char * so we if just call it in
+	// fprintf, you'll get back wrong results since we're
+	// calling it twice.
+	strncpy(srcip, inet_ntoa(*s), 19);
+	strncpy(dstip, inet_ntoa(*d), 19);
+	fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %u }\n",
+			srcip,
+			dstip,
+			ntohl(iph->check));
+}
+
+void fprintf_eth_header(FILE *fp, struct ethhdr *ethh)
+{
+	fprintf(fp, "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | "
+			"dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n",
+			(int) ((unsigned char *) ethh->h_source)[0],
+			(int) ((unsigned char *) ethh->h_source)[1],
+			(int) ((unsigned char *) ethh->h_source)[2],
+			(int) ((unsigned char *) ethh->h_source)[3],
+			(int) ((unsigned char *) ethh->h_source)[4],
+			(int) ((unsigned char *) ethh->h_source)[5],
+			(int) ((unsigned char *) ethh->h_dest)[0],
+			(int) ((unsigned char *) ethh->h_dest)[1],
+			(int) ((unsigned char *) ethh->h_dest)[2],
+			(int) ((unsigned char *) ethh->h_dest)[3],
+			(int) ((unsigned char *) ethh->h_dest)[4],
+			(int) ((unsigned char *) ethh->h_dest)[5]);
+}
+
 void make_eth_header(struct ethhdr *ethh, macaddr_t *src, macaddr_t *dst)
 {
 	memcpy(ethh->h_source, src, ETH_ALEN);

src/probe_modules/packet.h

@@ -6,8 +6,8 @@
 #include <netinet/ip_icmp.h>
 #include <netinet/udp.h>
 
-#ifndef HEADER_ZMAP_PACKET_H
-#define HEADER_ZMAP_PACKET_H
+#ifndef PACKET_H
+#define PACKET_H
 
 #define MAX_PACKET_SIZE 4096
 
@@ -20,6 +20,8 @@ void make_tcp_header(struct tcphdr*, port_h_t);
 void make_icmp_header(struct icmp *);
 void make_udp_header(struct udphdr *udp_header, port_h_t dest_port,
 				uint16_t len);
+void fprintf_ip_header(FILE *fp, struct iphdr *iph);
+void fprintf_eth_header(FILE *fp, struct ethhdr *ethh);
 
 static inline unsigned short in_checksum(unsigned short *ip_pkt, int len)
 {
@@ -79,4 +81,25 @@ static __attribute__((unused)) uint16_t tcp_checksum(unsigned short len_tcp,
 	return (unsigned short) (~sum);
 }
 
+// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise
+static __attribute__((unused)) inline int check_dst_port(uint16_t port,
+				int num_ports, uint32_t *validation)
+{
+	if (port > zconf.source_port_last 
+					|| port < zconf.source_port_first) {
+		return -1;
+	}
+	int32_t to_validate = port - zconf.source_port_first;
+	int32_t min = validation[1] % num_ports;
+	int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports;
+
+	return (((max - min) % num_ports) >= ((to_validate - min) % num_ports));
+}
+
+static __attribute__((unused)) inline uint16_t get_src_port(int num_ports, 
+				int probe_num, uint32_t *validation)
+{
+	return zconf.source_port_first + ((validation[1] + probe_num) % num_ports);
+}
+
 #endif

src/probe_modules/probe_modules.c

@@ -8,25 +8,35 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
+#include <time.h>
 
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <linux/if_packet.h>
+
+#include "../../lib/logger.h"
+#include "../fieldset.h"
 #include "probe_modules.h"
 
 extern probe_module_t module_tcp_synscan;
 extern probe_module_t module_icmp_echo;
-extern probe_module_t module_udp;
+//extern probe_module_t module_udp;
 // ADD YOUR MODULE HERE
 
 probe_module_t* probe_modules[] = {
 	&module_tcp_synscan,
 	&module_icmp_echo,
-	&module_udp
+//	&module_udp
 	// ADD YOUR MODULE HERE
 };
 
-
 probe_module_t* get_probe_module_by_name(const char* name)
 {
-	for (int i=0; i < (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); i++) {
+	int len = (int) (sizeof(probe_modules)/sizeof(probe_modules[0]));
+	for (int i=0; i < len; i++) {
 		if (!strcmp(probe_modules[i]->name, name)) {
 			return probe_modules[i];
 		}
@@ -36,7 +46,58 @@ probe_module_t* get_probe_module_by_name(const char* name)
 
 void print_probe_modules(void)
 {
-	for (int i=0; i < (int) (sizeof(probe_modules)/sizeof(probe_modules[0])); i++) {
+	int len = (int) (sizeof(probe_modules)/sizeof(probe_modules[0]));
+	for (int i=0; i < len; i++) {
 		printf("%s\n", probe_modules[i]->name);
 	}
 }
+
+char *make_ip_str(uint32_t ip)
+{
+	struct in_addr t;
+	t.s_addr = ip;
+	const char *temp = inet_ntoa(t);
+	char *retv = malloc(strlen(temp)+1);
+	assert (retv);
+	strcpy(retv, temp);
+	return retv;
+}
+
+void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip)
+{
+	fs_add_string(fs, "saddr", make_ip_str(ip->saddr), 1);
+	fs_add_string(fs, "daddr", make_ip_str(ip->daddr), 1);
+	fs_add_uint64(fs, "ipid", ntohl(ip->id));
+	fs_add_uint64(fs, "ttl", ntohl(ip->ttl));
+}
+
+#define TIMESTR_LEN 50
+
+void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown)
+{
+	fs_add_uint64(fs, "repeat", is_repeat);
+	fs_add_uint64(fs, "cooldown", in_cooldown);
+
+	char *timestr = malloc(TIMESTR_LEN+1);
+	if (!timestr) {
+		log_fatal("recv", "unable to allocate memory for "
+				  "timestamp string in fieldset.");
+	}
+	time_t now = time(0);
+	strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S%z",
+			localtime(&now));
+	fs_add_string(fs, "timestamp-str", timestr, 1);
+}
+
+fielddef_t ip_fields[] = {
+	{.name="saddr", .type="string", .desc="source IP address of response"},
+	{.name="daddr", .type="string", .desc="destination IP address of response"},
+	{.name="ipid", .type="int", .desc="IP identification number of response"},
+	{.name="ttl", .type="int", .desc="time-to-live of response packet"}
+};
+
+fielddef_t sys_fields[] = {
+	{.name="repeat", .type="int", .desc="Is response a repeat response from host"},
+	{.name="cooldown", .type="int", .desc="Was response received during the cooldown period"},
+	{.name="timestamp-str", .type="string", .desc="timestamp of when response arrived in ISO8601 format."}
+};

src/probe_modules/probe_modules.h

@@ -1,7 +1,8 @@
 #include "../state.h"
+#include "../fieldset.h"
 
-#ifndef HEADER_PROBE_MODULES_H
-#define HEADER_PROBE_MODULES_H
+#ifndef PROBE_MODULES_H
+#define PROBE_MODULES_H
 
 typedef struct probe_response_type {
 		const uint8_t is_success;
@@ -9,13 +10,21 @@ typedef struct probe_response_type {
 } response_type_t;
 
 typedef int (*probe_global_init_cb)(struct state_conf *);
-typedef int (*probe_thread_init_cb)(void* packetbuf, macaddr_t* src_mac, macaddr_t* gw_mac, port_n_t src_port);
-typedef int (*probe_make_packet_cb)(void* packetbuf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip,
-			uint32_t *validation, int probe_num);
+typedef int (*probe_thread_init_cb)(void* packetbuf, macaddr_t* src_mac,
+		macaddr_t* gw_mac, port_n_t src_port);
+
+typedef int (*probe_make_packet_cb)(void* packetbuf, ipaddr_n_t src_ip,
+		ipaddr_n_t dst_ip,
+		uint32_t *validation, int probe_num);
+
 typedef void (*probe_print_packet_cb)(FILE *, void* packetbuf);
-typedef int (*probe_close_cb)(struct state_conf*,  struct state_send*, struct state_recv*);
-typedef response_type_t* (*probe_classify_packet_cb)(const u_char* packetbuf, uint32_t len);
-typedef int (*probe_validate_packet_cb)(const struct iphdr *ip_hdr, uint32_t len, uint32_t *src_ip, uint32_t *validation);
+typedef int (*probe_close_cb)(struct state_conf*, 
+		struct state_send*, struct state_recv*);
+typedef int (*probe_validate_packet_cb)(const struct iphdr *ip_hdr,
+		uint32_t len, uint32_t *src_ip, uint32_t *validation);
+
+typedef void (*probe_classify_packet_cb)(const u_char* packetbuf,
+		uint32_t len, fieldset_t*);
 
 typedef struct probe_module {
 	const char *name;
@@ -27,19 +36,27 @@ typedef struct probe_module {
 	// source and target port numbers?
 	uint8_t port_args;
 
-	response_type_t *responses;
-
 	probe_global_init_cb global_initialize;
 	probe_thread_init_cb thread_initialize;
 	probe_make_packet_cb make_packet;
 	probe_print_packet_cb print_packet;
 	probe_validate_packet_cb validate_packet;
-	probe_classify_packet_cb classify_packet;
+	probe_classify_packet_cb process_packet;
 	probe_close_cb close;
+	fielddef_t *fields;
+	int numfields;
+	const char *helptext;
 
 } probe_module_t;
 
 probe_module_t* get_probe_module_by_name(const char*);
+
+void fs_add_ip_fields(fieldset_t *fs, struct iphdr *ip);
+void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown);
 void print_probe_modules(void);
 
+extern fielddef_t ip_fields[];
+extern fielddef_t sys_fields[];
+
 #endif // HEADER_PROBE_MODULES_H
+

src/recv.c

@@ -27,11 +27,13 @@
 #include <sys/socket.h>
 #include <sys/ioctl.h>
 #include <net/if.h>
+#include <assert.h>
 
 #include "../lib/logger.h"
 
 #include "state.h"
 #include "validate.h"
+#include "fieldset.h"
 #include "probe_modules/probe_modules.h"
 #include "output_modules/output_modules.h"
 
@@ -92,9 +94,16 @@ void packet_cb(u_char __attribute__((__unused__)) *user,
 	}
 
 	int is_repeat = check_ip(src_ip);
-	response_type_t *r = zconf.probe_module->classify_packet(bytes, buflen);
 
-	if (r->is_success) {
+	fieldset_t *fs = fs_new_fieldset();
+	fs_add_ip_fields(fs, ip_hdr);
+	zconf.probe_module->process_packet(bytes, buflen, fs);
+	fs_add_system_fields(fs, is_repeat, zsend.complete);
+	int success_index = zconf.fsconf.success_index;
+	assert(success_index < fs->len);
+	int is_success = fs_get_uint64_by_index(fs, success_index);
+	
+	if (is_success) {
 		zrecv.success_total++;
 		if (!is_repeat) {
 			zrecv.success_unique++;
@@ -106,20 +115,26 @@ void packet_cb(u_char __attribute__((__unused__)) *user,
 				zrecv.cooldown_unique++;
 			}
 		}
-		if (zconf.output_module && zconf.output_module->success_ip) {
-			zconf.output_module->success_ip(
-					ip_hdr->saddr, ip_hdr->daddr,
-					r->name, is_repeat, zsend.complete, bytes, buflen);
-		}
+		
 	} else {
 		zrecv.failure_total++;
-		if (zconf.output_module && zconf.output_module->other_ip) {
-			zconf.output_module->other_ip(
-					ip_hdr->saddr, ip_hdr->daddr,
-					r->name, is_repeat,	zsend.complete, bytes, buflen);
-		}
 	}
-
+	fieldset_t *o = NULL;
+	// we need to translate the data provided by the probe module
+	// into a fieldset that can be used by the output module
+	if (!is_success && zconf.filter_unsuccessful) {
+		goto cleanup;	
+	}
+	if (is_repeat && zconf.filter_duplicates) {
+		goto cleanup;
+	}
+	o = translate_fieldset(fs, &zconf.fsconf.translation); 
+	if (zconf.output_module && zconf.output_module->process_ip) {
+		zconf.output_module->process_ip(o);
+	}
+cleanup:
+	fs_free(fs);
+	free(o);	
 	if (zconf.output_module && zconf.output_module->update
 			&& !(zrecv.success_unique % zconf.output_module->update_interval)) {
 		zconf.output_module->update(&zconf, &zsend, &zrecv);
@@ -150,24 +165,37 @@ int recv_run(pthread_mutex_t *recv_ready_mutex)
 	num_src_ports = zconf.source_port_last - zconf.source_port_first + 1;
 	ip_seen = calloc(IP_SEEN_SIZE, sizeof(uint64_t));
 	if (!ip_seen) {
-		log_fatal("recv", "couldn't allocate address bitmap");
+		log_fatal("recv", "could not allocate address bitmap");
 	}
 	log_debug("recv", "using dev %s", zconf.iface);
-	char errbuf[PCAP_ERRBUF_SIZE];
-	pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen,
-					PCAP_PROMISC, PCAP_TIMEOUT, errbuf);
-	if (pc == NULL) {
-		log_fatal("recv", "couldn't open device %s: %s",
-						zconf.iface, errbuf);
-	}
-	struct bpf_program bpf;
-	if (pcap_compile(pc, &bpf, zconf.probe_module->pcap_filter, 1, 0) < 0) {
-		log_fatal("recv", "couldn't compile filter");
-	}
-	if (pcap_setfilter(pc, &bpf) < 0) {
-		log_fatal("recv", "couldn't install filter");
+	if (!zconf.dryrun) {
+		char errbuf[PCAP_ERRBUF_SIZE];
+		pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen,
+						PCAP_PROMISC, PCAP_TIMEOUT, errbuf);
+		if (pc == NULL) {
+			log_fatal("recv", "could not open device %s: %s",
+							zconf.iface, errbuf);
+		}
+		struct bpf_program bpf;
+		if (pcap_compile(pc, &bpf, zconf.probe_module->pcap_filter, 1, 0) < 0) {
+			log_fatal("recv", "couldn't compile filter");
+		}
+		if (pcap_setfilter(pc, &bpf) < 0) {
+			log_fatal("recv", "couldn't install filter");
+		}
 	}
 	log_debug("recv", "receiver ready");
+	if (zconf.filter_duplicates) {
+		log_debug("recv", "duplicate responses will be excluded from output"); 
+	} else {
+		log_debug("recv", "duplicate responses will be included in output"); 
+	}
+	if (zconf.filter_unsuccessful) {
+		log_debug("recv", "unsuccessful responses will be excluded from output"); 
+	} else {
+		log_debug("recv", "unsuccessful responses will be included in output"); 
+	}
+	
 	pthread_mutex_lock(recv_ready_mutex);
 	zconf.recv_ready = 1;
 	pthread_mutex_unlock(recv_ready_mutex);
@@ -176,12 +204,16 @@ int recv_run(pthread_mutex_t *recv_ready_mutex)
 		zconf.max_results = -1;
 	}
 	do {
-		if (pcap_dispatch(pc, 0, packet_cb, NULL) == -1) {
-			log_fatal("recv", "pcap_dispatch error");
-		}
-		if (zconf.max_results && zrecv.success_unique >= zconf.max_results) {
-			zsend.complete = 1;
-			break;
+		if (zconf.dryrun) {
+			sleep(1);
+		} else {
+			if (pcap_dispatch(pc, 0, packet_cb, NULL) == -1) {
+				log_fatal("recv", "pcap_dispatch error");
+			}
+			if (zconf.max_results && zrecv.success_unique >= zconf.max_results) {
+				zsend.complete = 1;
+				break;
+			}
 		}
 	} while (!(zsend.complete && (now()-zsend.finish > zconf.cooldown_secs)));
 	zrecv.finish = now();

src/recv.h

@@ -6,8 +6,8 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef _RECV_H
-#define _RECV_H
+#ifndef RECV_H
+#define RECV_H
 
 int recv_update_pcap_stats(void);
 int recv_run(pthread_mutex_t *recv_ready_mutex);

src/send.c

@@ -142,6 +142,21 @@ static int get_socket(void)
 	return sock;
 }
 
+static int get_dryrun_socket(void)
+{
+	// we need a socket in order to gather details about the system
+	// such as source MAC address and IP address. However, because
+	// we don't want to require root access in order to run dryrun,
+	// we just create a TCP socket.
+	int sock = socket(AF_INET, SOCK_STREAM, 0);
+	if (sock <= 0) {
+		log_fatal("send", "couldn't create socket. "
+			  "Error: %s\n", strerror(errno));
+	}
+	return sock;
+
+}
+
 static inline ipaddr_n_t get_src_ip(ipaddr_n_t dst, int local_offset)
 {
 	if (srcip_first == srcip_last) {
@@ -156,7 +171,12 @@ int send_run(void)
 {
 	log_debug("send", "thread started");
 	pthread_mutex_lock(&send_mutex);
-	int sock = get_socket();
+	int sock;
+	if (zconf.dryrun) {
+		sock = get_dryrun_socket();
+	} else {
+		sock = get_socket();
+	}
 	struct sockaddr_ll sockaddr;
 	// get source interface index
 	struct ifreq if_idx;

src/send.h

@@ -6,8 +6,8 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef _SEND_H
-#define _SEND_H
+#ifndef SEND_H
+#define SEND_H
 
 int send_init(void);
 int send_run(void);

src/state.c

@@ -38,9 +38,14 @@ struct state_conf zconf = {
 	.gw_mac_set = 0,
 	.source_ip_first = NULL,
 	.source_ip_last = NULL,
+	.raw_output_fields = NULL,
+	.output_fields = NULL,
+	.output_fields_len = 0,
 	.dryrun = 0,
 	.quiet = 0,
 	.summary = 0,
+	.filter_duplicates = 0,
+	.filter_unsuccessful = 0,
 	.recv_ready = 0,
 };
 

src/state.h

@@ -13,20 +13,25 @@
 #include <netinet/ether.h>
 #include <net/if.h>
 
-#ifndef _STATE_H
-#define _STATE_H
+#include "types.h"
+#include "fieldset.h"
+
+#ifndef STATE_H
+#define STATE_H
 
 #define MAX_PACKET_SIZE 4096
 
-typedef uint32_t ipaddr_n_t; // IPv4 address network order
-typedef uint32_t ipaddr_h_t; // IPv4 address host order
-typedef uint16_t port_n_t; // port network order
-typedef uint16_t port_h_t; // port host order
-typedef unsigned char macaddr_t;
-
 struct probe_module;
 struct output_module;
 
+struct fieldset_conf {
+	fielddefset_t defs;
+	fielddefset_t outdefs;
+	translation_t translation;
+	int success_index;
+	int classification_index;
+};
+
 // global configuration
 struct state_conf {
 	int log_level;
@@ -71,13 +76,20 @@ struct state_conf {
 	char *output_filename;
 	char *blacklist_filename;
 	char *whitelist_filename;
+	char *raw_output_fields;
+	char **output_fields;
+	struct fieldset_conf fsconf;
+	int output_fields_len;
 	int dryrun;
 	int summary;
 	int quiet;
+	int filter_duplicates;
+	int filter_unsuccessful;
 	int recv_ready;
 };
 extern struct state_conf zconf;
 
+
 // global sender stats
 struct state_send {
 	double start;

src/types.h

@@ -0,0 +1,11 @@
+#include <stdint.h>
+
+#ifndef TYPES_H_
+#define TYPES_H_
+typedef uint32_t ipaddr_n_t; // IPv4 address network order
+typedef uint32_t ipaddr_h_t; // IPv4 address host order
+typedef uint16_t port_n_t; // port network order
+typedef uint16_t port_h_t; // port host order
+typedef unsigned char macaddr_t;
+
+#endif

src/validate.h

@@ -6,8 +6,8 @@
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-#ifndef _VALIDATE_H
-#define _VALIDATE_H
+#ifndef VALIDATE_H
+#define VALIDATE_H
 
 #define VALIDATE_BYTES 16
 

src/zmap.c

@@ -10,18 +10,19 @@
 
 #include <stdlib.h>
 #include <stdio.h>
-
+#include <string.h>
+#include <unistd.h>
 #include <assert.h>
 #include <sched.h>
-#include <unistd.h>
 #include <errno.h>
-#include <string.h>
+
 #include <net/if.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
 #include <pcap/pcap.h>
+
 #include <pthread.h>
 
 #include "../lib/logger.h"
@@ -32,13 +33,45 @@
 #include "recv.h"
 #include "state.h"
 #include "monitor.h"
+#include "get_gateway.h"
+
 #include "output_modules/output_modules.h"
 #include "probe_modules/probe_modules.h"
-#include "get_gateway.h"
 
 pthread_mutex_t cpu_affinity_mutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+// splits comma delimited string into char*[]. Does not handle
+// escaping or complicated setups: designed to process a set
+// of fields that the user wants output 
+static void split_string(char* in, int *len, char***results)
+{
+        char** fields = calloc(MAX_FIELDS, sizeof(char*));
+        memset(fields, 0, sizeof(fields));
+        int retvlen = 0;
+        char *currloc = in; 
+        // parse csv into a set of strings
+        while (1) {
+                size_t len = strcspn(currloc, ", ");    
+                if (len == 0) {
+                        currloc++;
+                } else {
+                        char *new = malloc(len+1);
+			assert(new);
+                        strncpy(new, currloc, len);
+                        new[len] = '\0';             
+                        fields[retvlen++] = new;
+			assert(fields[retvlen-1]);
+                }   
+		if (len == strlen(currloc)) {
+			break;
+		}
+                currloc += len;
+        }
+        *results = fields;
+        *len = retvlen;
+}
+
 static void set_cpu(void)
 {
 	pthread_mutex_lock(&cpu_affinity_mutex);
@@ -81,7 +114,7 @@ static void *start_mon(__attribute__((unused)) void *arg)
 #define SI(w,x,y) printf("%s\t%s\t%i\n", w, x, y);
 #define SD(w,x,y) printf("%s\t%s\t%f\n", w, x, y);
 #define SU(w,x,y) printf("%s\t%s\t%u\n", w, x, y);
-#define SLU(w,x,y) printf("%s\t%s\t%lu\n", w, x, (long unsigned int)y);
+#define SLU(w,x,y) printf("%s\t%s\t%lu\n", w, x, (long unsigned int) y);
 #define SS(w,x,y) printf("%s\t%s\t%s\n", w, x, y);
 #define STRTIME_LEN 1024
 
@@ -131,7 +164,6 @@ static void summary(void)
 
 static void start_zmap(void)
 {
-	log_init(stderr, zconf.log_level);
 	log_info("zmap", "started");
 
 	// finish setting up configuration
@@ -179,7 +211,8 @@ static void start_zmap(void)
 
 	// initialization
 	if (zconf.output_module && zconf.output_module->init) {
-		zconf.output_module->init(&zconf);
+		zconf.output_module->init(&zconf, zconf.output_fields,
+				zconf.output_fields_len);
 	}
 	if (send_init()) {
 		exit(EXIT_FAILURE);
@@ -187,7 +220,6 @@ static void start_zmap(void)
 	if (zconf.output_module && zconf.output_module->start) {
 		zconf.output_module->start(&zconf, &zsend, &zrecv);
 	}
-
 	// start threads
 	pthread_t *tsend, trecv, tmon;
 	int r = pthread_create(&trecv, NULL, start_recv, NULL);
@@ -206,14 +238,14 @@ static void start_zmap(void)
 	assert(tsend);
 	log_debug("zmap", "using %d sender threads", zconf.senders);
 	for (int i=0; i < zconf.senders; i++) {
-		r = pthread_create(&tsend[i], NULL, start_send, NULL);
+		int r = pthread_create(&tsend[i], NULL, start_send, NULL);
 		if (r != 0) {
 			log_fatal("zmap", "unable to create send thread");
 			exit(EXIT_FAILURE);
 		}
 	}
 	if (!zconf.quiet) {
-		r = pthread_create(&tmon, NULL, start_mon, NULL);
+		int r = pthread_create(&tmon, NULL, start_mon, NULL);
 		if (r != 0) {
 			log_fatal("zmap", "unable to create monitor thread");
 			exit(EXIT_FAILURE);
@@ -222,14 +254,14 @@ static void start_zmap(void)
 
 	// wait for completion
 	for (int i=0; i < zconf.senders; i++) {
-		pthread_join(tsend[i], NULL);
+		int r = pthread_join(tsend[i], NULL);
 		if (r != 0) {
 			log_fatal("zmap", "unable to join send thread");
 			exit(EXIT_FAILURE);
 		}
 	}
 	log_debug("zmap", "senders finished");
-	pthread_join(trecv, NULL);
+	r = pthread_join(trecv, NULL);
 	if (r != 0) {
 		log_fatal("zmap", "unable to join recv thread");
 		exit(EXIT_FAILURE);
@@ -308,6 +340,11 @@ int main(int argc, char *argv[])
 	params->initialize = 1;
 	params->override = 0;
 	params->check_required = 0;
+
+	zconf.log_level = args.verbosity_arg;
+	log_init(stderr, zconf.log_level);
+	log_trace("zmap", "zmap main thread started");
+
 	if (cmdline_parser_ext(argc, argv, &args, params) != 0) {
 		exit(EXIT_SUCCESS);
 	}
@@ -338,7 +375,104 @@ int main(int argc, char *argv[])
 	if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) {
 		exit(EXIT_FAILURE);
 	}
+	// parse the provided probe and output module s.t. that we can support
+	// other command-line helpers (e.g. probe help)
+	if (!args.output_module_given) {
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "saddr";
+		zconf.filter_duplicates = 1;
+		zconf.filter_unsuccessful = 1;
+	} else if (!strcmp(args.output_module_arg, "simple_file")) {
+		log_warn("zmap", "the simple_file output interface has been deprecated and "
+                                 "will be removed in the future. Users should use the csv "
+				 "output module. Newer scan options such as output-fields "
+				 "are not supported with this output module."); 
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "saddr";
+		zconf.filter_duplicates = 1;
+		zconf.filter_unsuccessful = 1;
+	} else if (!strcmp(args.output_module_arg, "extended_file")) {
+		log_warn("zmap", "the extended_file output interface has been deprecated and "
+                                 "will be removed in the future. Users should use the csv "
+				 "output module. Newer scan options such as output-fields "
+				 "are not supported with this output module."); 
+		zconf.output_module = get_output_module_by_name("csv");
+		zconf.raw_output_fields = (char*) "classification, saddr, "
+						  "daddr, sport, dport, "
+						  "seqnum, acknum, cooldown, "
+						  "repeat, timestamp-str";
+		zconf.filter_duplicates = 0;
+	} else {
+		zconf.output_module = get_output_module_by_name(args.output_module_arg);
+		if (!zconf.output_module) {
+		  fprintf(stderr, "%s: specified output module (%s) does not exist\n",
+			  CMDLINE_PARSER_PACKAGE, args.output_module_arg);
+		  exit(EXIT_FAILURE);
+		}
+	}
+	zconf.probe_module = get_probe_module_by_name(args.probe_module_arg);
+	if (!zconf.probe_module) {
+		fprintf(stderr, "%s: specified probe module (%s) does not exist\n",
+				CMDLINE_PARSER_PACKAGE, args.probe_module_arg);
+	  exit(EXIT_FAILURE);
+	}
 
+	// now that we know the probe module, let's find what it supports
+	memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf));
+	// the set of fields made available to a user is constructed
+	// of IP header fields + probe module fields + system fields
+	fielddefset_t *fds = &(zconf.fsconf.defs);
+	gen_fielddef_set(fds, (fielddef_t*) &(ip_fields),
+		4);
+	gen_fielddef_set(fds, zconf.probe_module->fields,
+		zconf.probe_module->numfields);
+	gen_fielddef_set(fds, (fielddef_t*) &(sys_fields),
+		3);
+	if (args.list_output_fields_given) {
+		for (int i = 0; i < fds->len; i++) {
+			printf("%s (%s): %s\n", fds->fielddefs[i].name,
+				fds->fielddefs[i].type,
+				fds->fielddefs[i].desc);
+		}
+		exit(EXIT_SUCCESS);
+	}
+	// find the fields we need for the framework
+	zconf.fsconf.success_index =
+			fds_get_index_by_name(fds, (char*) "success");	
+	if (zconf.fsconf.success_index < 0) {
+		log_fatal("fieldset", "probe module does not supply "
+				      "required success field.");
+	}
+	zconf.fsconf.classification_index =
+			fds_get_index_by_name(fds, (char*) "classification");
+	if (zconf.fsconf.classification_index < 0) {
+		log_fatal("fieldset", "probe module does not supply "
+				      "required packet classification field.");
+	}
+	// process the list of requested output fields.
+	if (args.output_fields_given) {
+		zconf.raw_output_fields = args.output_fields_arg;
+	} else if (!zconf.raw_output_fields) {
+		zconf.raw_output_fields = (char*) "saddr";
+	}	
+	split_string(zconf.raw_output_fields, &(zconf.output_fields_len),
+			&(zconf.output_fields));
+	for (int i=0; i < zconf.output_fields_len; i++) {
+		log_debug("zmap", "requested output field (%i): %s",
+				i,
+				zconf.output_fields[i]);
+	}
+	// generate a translation that can be used to convert output
+	// from a probe module to the input for an output module
+	fs_generate_fieldset_translation(&zconf.fsconf.translation,
+			&zconf.fsconf.defs, zconf.output_fields,
+			zconf.output_fields_len);
+
+	SET_BOOL(zconf.dryrun, dryrun);
+	SET_BOOL(zconf.quiet, quiet);
+	SET_BOOL(zconf.summary, summary);
+	zconf.cooldown_secs = args.cooldown_time_arg;
+	zconf.senders = args.sender_threads_arg;
 	SET_IF_GIVEN(zconf.output_filename, output_file);
 	SET_IF_GIVEN(zconf.blacklist_filename, blacklist_file);
 	SET_IF_GIVEN(zconf.whitelist_filename, whitelist_file);
@@ -349,25 +483,8 @@ int main(int argc, char *argv[])
 	SET_IF_GIVEN(zconf.max_results, max_results);
 	SET_IF_GIVEN(zconf.rate, rate);
 	SET_IF_GIVEN(zconf.packet_streams, probes);
-	SET_BOOL(zconf.dryrun, dryrun);
-	SET_BOOL(zconf.quiet, quiet);
-	SET_BOOL(zconf.summary, summary);
-	zconf.cooldown_secs = args.cooldown_time_arg;
-	zconf.senders = args.sender_threads_arg;
-	zconf.log_level = args.verbosity_arg;
+	
 
-	zconf.output_module = get_output_module_by_name(args.output_module_arg);
-	if (!zconf.output_module) {
-	  fprintf(stderr, "%s: specified output module (%s) does not exist\n",
-		  CMDLINE_PARSER_PACKAGE, args.output_module_arg);
-	  exit(EXIT_FAILURE);
-	}
-	zconf.probe_module = get_probe_module_by_name(args.probe_module_arg);
-	if (!zconf.probe_module) {
-		fprintf(stderr, "%s: specified probe module (%s) does not exist\n",
-				CMDLINE_PARSER_PACKAGE, args.probe_module_arg);
-	  exit(EXIT_FAILURE);
-	}
 	if (zconf.probe_module->port_args) {
 		if (args.source_port_given) {
 			char *dash = strchr(args.source_port_arg, '-');

src/zopt.c

@@ -37,6 +37,7 @@ const char *gengetopt_args_info_help[] = {
   "  -o, --output-file=name        Output file",
   "  -b, --blacklist-file=path     File of subnets to exclude, in CIDR notation, \n                                  e.g. 192.168.0.0/16",
   "  -w, --whitelist-file=path     File of subnets to constrain scan to, in CIDR \n                                  notation, e.g. 192.168.0.0/16",
+  "  -f, --output-fields=fields    Fields that should be output in result set",
   "\nScan options:",
   "  -n, --max-targets=n           Cap number of targets to probe (as a number or \n                                  a percentage of the address space)",
   "  -N, --max-results=n           Cap number of results to return",
@@ -55,11 +56,12 @@ const char *gengetopt_args_info_help[] = {
   "  -i, --interface=name          Specify network interface to use",
   "\nAdvanced options:",
   "  -M, --probe-module=name       Select probe module  (default=`tcp_synscan')",
-  "  -O, --output-module=name      Select output module  (default=`simple_file')",
+  "  -O, --output-module=name      Select output module  (default=`csv')",
   "      --probe-args=args         Arguments to pass to probe module",
   "      --output-args=args        Arguments to pass to output module",
   "      --list-output-modules     List available output modules",
   "      --list-probe-modules      List available probe modules",
+  "      --list-output-fields      List all fields that can be output by selected \n                                  probe module",
   "\nAdditional options:",
   "  -C, --config=filename         Read a configuration file, which can specify \n                                  any of these options  \n                                  (default=`/etc/zmap/zmap.conf')",
   "  -q, --quiet                   Do not print status updates",
@@ -120,6 +122,7 @@ void clear_given (struct gengetopt_args_info *args_info)
   args_info->output_file_given = 0 ;
   args_info->blacklist_file_given = 0 ;
   args_info->whitelist_file_given = 0 ;
+  args_info->output_fields_given = 0 ;
   args_info->max_targets_given = 0 ;
   args_info->max_results_given = 0 ;
   args_info->max_runtime_given = 0 ;
@@ -140,6 +143,7 @@ void clear_given (struct gengetopt_args_info *args_info)
   args_info->output_args_given = 0 ;
   args_info->list_output_modules_given = 0 ;
   args_info->list_probe_modules_given = 0 ;
+  args_info->list_output_fields_given = 0 ;
   args_info->config_given = 0 ;
   args_info->quiet_given = 0 ;
   args_info->summary_given = 0 ;
@@ -159,6 +163,8 @@ void clear_args (struct gengetopt_args_info *args_info)
   args_info->blacklist_file_orig = NULL;
   args_info->whitelist_file_arg = NULL;
   args_info->whitelist_file_orig = NULL;
+  args_info->output_fields_arg = NULL;
+  args_info->output_fields_orig = NULL;
   args_info->max_targets_arg = NULL;
   args_info->max_targets_orig = NULL;
   args_info->max_results_orig = NULL;
@@ -183,7 +189,7 @@ void clear_args (struct gengetopt_args_info *args_info)
   args_info->interface_orig = NULL;
   args_info->probe_module_arg = gengetopt_strdup ("tcp_synscan");
   args_info->probe_module_orig = NULL;
-  args_info->output_module_arg = gengetopt_strdup ("simple_file");
+  args_info->output_module_arg = gengetopt_strdup ("csv");
   args_info->output_module_orig = NULL;
   args_info->probe_args_arg = NULL;
   args_info->probe_args_orig = NULL;
@@ -205,32 +211,34 @@ void init_args_info(struct gengetopt_args_info *args_info)
   args_info->output_file_help = gengetopt_args_info_help[2] ;
   args_info->blacklist_file_help = gengetopt_args_info_help[3] ;
   args_info->whitelist_file_help = gengetopt_args_info_help[4] ;
-  args_info->max_targets_help = gengetopt_args_info_help[6] ;
-  args_info->max_results_help = gengetopt_args_info_help[7] ;
-  args_info->max_runtime_help = gengetopt_args_info_help[8] ;
-  args_info->rate_help = gengetopt_args_info_help[9] ;
-  args_info->bandwidth_help = gengetopt_args_info_help[10] ;
-  args_info->cooldown_time_help = gengetopt_args_info_help[11] ;
-  args_info->seed_help = gengetopt_args_info_help[12] ;
-  args_info->sender_threads_help = gengetopt_args_info_help[13] ;
-  args_info->probes_help = gengetopt_args_info_help[14] ;
-  args_info->dryrun_help = gengetopt_args_info_help[15] ;
-  args_info->source_port_help = gengetopt_args_info_help[17] ;
-  args_info->source_ip_help = gengetopt_args_info_help[18] ;
-  args_info->gateway_mac_help = gengetopt_args_info_help[19] ;
-  args_info->interface_help = gengetopt_args_info_help[20] ;
-  args_info->probe_module_help = gengetopt_args_info_help[22] ;
-  args_info->output_module_help = gengetopt_args_info_help[23] ;
-  args_info->probe_args_help = gengetopt_args_info_help[24] ;
-  args_info->output_args_help = gengetopt_args_info_help[25] ;
-  args_info->list_output_modules_help = gengetopt_args_info_help[26] ;
-  args_info->list_probe_modules_help = gengetopt_args_info_help[27] ;
-  args_info->config_help = gengetopt_args_info_help[29] ;
-  args_info->quiet_help = gengetopt_args_info_help[30] ;
-  args_info->summary_help = gengetopt_args_info_help[31] ;
-  args_info->verbosity_help = gengetopt_args_info_help[32] ;
-  args_info->help_help = gengetopt_args_info_help[33] ;
-  args_info->version_help = gengetopt_args_info_help[34] ;
+  args_info->output_fields_help = gengetopt_args_info_help[5] ;
+  args_info->max_targets_help = gengetopt_args_info_help[7] ;
+  args_info->max_results_help = gengetopt_args_info_help[8] ;
+  args_info->max_runtime_help = gengetopt_args_info_help[9] ;
+  args_info->rate_help = gengetopt_args_info_help[10] ;
+  args_info->bandwidth_help = gengetopt_args_info_help[11] ;
+  args_info->cooldown_time_help = gengetopt_args_info_help[12] ;
+  args_info->seed_help = gengetopt_args_info_help[13] ;
+  args_info->sender_threads_help = gengetopt_args_info_help[14] ;
+  args_info->probes_help = gengetopt_args_info_help[15] ;
+  args_info->dryrun_help = gengetopt_args_info_help[16] ;
+  args_info->source_port_help = gengetopt_args_info_help[18] ;
+  args_info->source_ip_help = gengetopt_args_info_help[19] ;
+  args_info->gateway_mac_help = gengetopt_args_info_help[20] ;
+  args_info->interface_help = gengetopt_args_info_help[21] ;
+  args_info->probe_module_help = gengetopt_args_info_help[23] ;
+  args_info->output_module_help = gengetopt_args_info_help[24] ;
+  args_info->probe_args_help = gengetopt_args_info_help[25] ;
+  args_info->output_args_help = gengetopt_args_info_help[26] ;
+  args_info->list_output_modules_help = gengetopt_args_info_help[27] ;
+  args_info->list_probe_modules_help = gengetopt_args_info_help[28] ;
+  args_info->list_output_fields_help = gengetopt_args_info_help[29] ;
+  args_info->config_help = gengetopt_args_info_help[31] ;
+  args_info->quiet_help = gengetopt_args_info_help[32] ;
+  args_info->summary_help = gengetopt_args_info_help[33] ;
+  args_info->verbosity_help = gengetopt_args_info_help[34] ;
+  args_info->help_help = gengetopt_args_info_help[35] ;
+  args_info->version_help = gengetopt_args_info_help[36] ;
   
 }
 
@@ -318,6 +326,8 @@ cmdline_parser_release (struct gengetopt_args_info *args_info)
   free_string_field (&(args_info->blacklist_file_orig));
   free_string_field (&(args_info->whitelist_file_arg));
   free_string_field (&(args_info->whitelist_file_orig));
+  free_string_field (&(args_info->output_fields_arg));
+  free_string_field (&(args_info->output_fields_orig));
   free_string_field (&(args_info->max_targets_arg));
   free_string_field (&(args_info->max_targets_orig));
   free_string_field (&(args_info->max_results_orig));
@@ -386,6 +396,8 @@ cmdline_parser_dump(FILE *outfile, struct gengetopt_args_info *args_info)
     write_into_file(outfile, "blacklist-file", args_info->blacklist_file_orig, 0);
   if (args_info->whitelist_file_given)
     write_into_file(outfile, "whitelist-file", args_info->whitelist_file_orig, 0);
+  if (args_info->output_fields_given)
+    write_into_file(outfile, "output-fields", args_info->output_fields_orig, 0);
   if (args_info->max_targets_given)
     write_into_file(outfile, "max-targets", args_info->max_targets_orig, 0);
   if (args_info->max_results_given)
@@ -426,6 +438,8 @@ cmdline_parser_dump(FILE *outfile, struct gengetopt_args_info *args_info)
     write_into_file(outfile, "list-output-modules", 0, 0 );
   if (args_info->list_probe_modules_given)
     write_into_file(outfile, "list-probe-modules", 0, 0 );
+  if (args_info->list_output_fields_given)
+    write_into_file(outfile, "list-output-fields", 0, 0 );
   if (args_info->config_given)
     write_into_file(outfile, "config", args_info->config_orig, 0);
   if (args_info->quiet_given)
@@ -692,6 +706,7 @@ cmdline_parser_internal (
         { "output-file",	1, NULL, 'o' },
         { "blacklist-file",	1, NULL, 'b' },
         { "whitelist-file",	1, NULL, 'w' },
+        { "output-fields",	1, NULL, 'f' },
         { "max-targets",	1, NULL, 'n' },
         { "max-results",	1, NULL, 'N' },
         { "max-runtime",	1, NULL, 't' },
@@ -712,6 +727,7 @@ cmdline_parser_internal (
         { "output-args",	1, NULL, 0 },
         { "list-output-modules",	0, NULL, 0 },
         { "list-probe-modules",	0, NULL, 0 },
+        { "list-output-fields",	0, NULL, 0 },
         { "config",	1, NULL, 'C' },
         { "quiet",	0, NULL, 'q' },
         { "summary",	0, NULL, 'g' },
@@ -721,7 +737,7 @@ cmdline_parser_internal (
         { 0,  0, 0, 0 }
       };
 
-      c = getopt_long (argc, argv, "p:o:b:w:n:N:t:r:B:c:e:T:P:ds:S:G:i:M:O:C:qgv:hV", long_options, &option_index);
+      c = getopt_long (argc, argv, "p:o:b:w:f:n:N:t:r:B:c:e:T:P:ds:S:G:i:M:O:C:qgv:hV", long_options, &option_index);
 
       if (c == -1) break;	/* Exit from `while (1)' loop.  */
 
@@ -774,6 +790,18 @@ cmdline_parser_internal (
               additional_error))
             goto failure;
         
+          break;
+        case 'f':	/* Fields that should be output in result set.  */
+        
+        
+          if (update_arg( (void *)&(args_info->output_fields_arg), 
+               &(args_info->output_fields_orig), &(args_info->output_fields_given),
+              &(local_args_info.output_fields_given), optarg, 0, 0, ARG_STRING,
+              check_ambiguity, override, 0, 0,
+              "output-fields", 'f',
+              additional_error))
+            goto failure;
+        
           break;
         case 'n':	/* Cap number of targets to probe (as a number or a percentage of the address space).  */
         
@@ -1096,6 +1124,20 @@ cmdline_parser_internal (
                 additional_error))
               goto failure;
           
+          }
+          /* List all fields that can be output by selected probe module.  */
+          else if (strcmp (long_options[option_index].name, "list-output-fields") == 0)
+          {
+          
+          
+            if (update_arg( 0 , 
+                 0 , &(args_info->list_output_fields_given),
+                &(local_args_info.list_output_fields_given), optarg, 0, 0, ARG_NO,
+                check_ambiguity, override, 0, 0,
+                "list-output-fields", '-',
+                additional_error))
+              goto failure;
+          
           }
           
           break;

src/zopt.ggo

@@ -22,6 +22,9 @@ option "blacklist-file"		b "File of subnets to exclude, in CIDR notation, e.g. 1
 option "whitelist-file"		w "File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16"
 	   typestr="path"
 	   optional string
+option "output-fields"		f "Fields that should be output in result set"
+	   typestr="fields"
+	   optional string
 
 section "Scan options"
 
@@ -51,11 +54,11 @@ option "sender-threads"		T "Threads used to send packets"
 	   typestr="n"
 	   default="1"
 	   optional int
-option "probes"				P "Number of probes to send to each IP"
+option "probes"			P "Number of probes to send to each IP"
 	   typestr="n"
 	   default="1"
 	   optional int
-option "dryrun"				d "Don't actually send packets"
+option "dryrun"			d "Don't actually send packets"
 	   optional
 
 section "Network options" 
@@ -63,13 +66,13 @@ section "Network options"
 option "source-port"    	s "Source port(s) for scan packets"
 	   typestr="port|range"
 	   optional string
-option "source-ip"			S "Source address(es) for scan packets"
+option "source-ip"		S "Source address(es) for scan packets"
 	   typestr="ip|range"
 	   optional string
 option "gateway-mac"		G "Specify gateway MAC address"
 	   typestr="addr"
 	   optional string
-option "interface"			i "Specify network interface to use"
+option "interface"		i "Specify network interface to use"
 	   typestr="name"
 	   optional string
 
@@ -83,34 +86,36 @@ option "output-module"		O "Select output module"
 	   typestr="name"
 	   default="simple_file"
 	   optional string
-option "probe-args"			- "Arguments to pass to probe module"
+option "probe-args"		- "Arguments to pass to probe module"
 	   typestr="args"
 	   optional string
 option "output-args"		- "Arguments to pass to output module"
 	   typestr="args"
 	   optional string
-option "list-output-modules" - "List available output modules"
+option "list-output-modules"	- "List available output modules"
            optional
-option "list-probe-modules"	 - "List available probe modules"
+option "list-probe-modules"	- "List available probe modules"
            optional
+option "list-output-fields" 	- "List all fields that can be output by selected probe module"
+	   optional
 
 section "Additional options"
 
-option "config"				C "Read a configuration file, which can specify any of these options"
+option "config"			C "Read a configuration file, which can specify any of these options"
 	   typestr="filename"
 	   default="/etc/zmap/zmap.conf"
 	   optional string
-option "quiet"				q "Do not print status updates"
+option "quiet"			q "Do not print status updates"
        optional
-option "summary"			g "Print configuration and summary at end of scan"
+option "summary"		g "Print configuration and summary at end of scan"
        optional
-option "verbosity"			v "Level of log detail (0-5)"
+option "verbosity"		v "Level of log detail (0-5)"
 	   typestr="n"
 	   default="3"
 	   optional int
-option "help"				h "Print help and exit"
+option "help"	   		h "Print help and exit"
 	   optional
-option "version"			V "Print version and exit"
+option "version"   		V "Print version and exit"
 	   optional
 
 text "\nExamples:\n\

src/zopt.h

@@ -49,6 +49,9 @@ struct gengetopt_args_info
   char * whitelist_file_arg;	/**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16.  */
   char * whitelist_file_orig;	/**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 original value given at command line.  */
   const char *whitelist_file_help; /**< @brief File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16 help description.  */
+  char * output_fields_arg;	/**< @brief Fields that should be output in result set.  */
+  char * output_fields_orig;	/**< @brief Fields that should be output in result set original value given at command line.  */
+  const char *output_fields_help; /**< @brief Fields that should be output in result set help description.  */
   char * max_targets_arg;	/**< @brief Cap number of targets to probe (as a number or a percentage of the address space).  */
   char * max_targets_orig;	/**< @brief Cap number of targets to probe (as a number or a percentage of the address space) original value given at command line.  */
   const char *max_targets_help; /**< @brief Cap number of targets to probe (as a number or a percentage of the address space) help description.  */
@@ -103,6 +106,7 @@ struct gengetopt_args_info
   const char *output_args_help; /**< @brief Arguments to pass to output module help description.  */
   const char *list_output_modules_help; /**< @brief List available output modules help description.  */
   const char *list_probe_modules_help; /**< @brief List available probe modules help description.  */
+  const char *list_output_fields_help; /**< @brief List all fields that can be output by selected probe module help description.  */
   char * config_arg;	/**< @brief Read a configuration file, which can specify any of these options (default='/etc/zmap/zmap.conf').  */
   char * config_orig;	/**< @brief Read a configuration file, which can specify any of these options original value given at command line.  */
   const char *config_help; /**< @brief Read a configuration file, which can specify any of these options help description.  */
@@ -118,6 +122,7 @@ struct gengetopt_args_info
   unsigned int output_file_given ;	/**< @brief Whether output-file was given.  */
   unsigned int blacklist_file_given ;	/**< @brief Whether blacklist-file was given.  */
   unsigned int whitelist_file_given ;	/**< @brief Whether whitelist-file was given.  */
+  unsigned int output_fields_given ;	/**< @brief Whether output-fields was given.  */
   unsigned int max_targets_given ;	/**< @brief Whether max-targets was given.  */
   unsigned int max_results_given ;	/**< @brief Whether max-results was given.  */
   unsigned int max_runtime_given ;	/**< @brief Whether max-runtime was given.  */
@@ -138,6 +143,7 @@ struct gengetopt_args_info
   unsigned int output_args_given ;	/**< @brief Whether output-args was given.  */
   unsigned int list_output_modules_given ;	/**< @brief Whether list-output-modules was given.  */
   unsigned int list_probe_modules_given ;	/**< @brief Whether list-probe-modules was given.  */
+  unsigned int list_output_fields_given ;	/**< @brief Whether list-output-fields was given.  */
   unsigned int config_given ;	/**< @brief Whether config was given.  */
   unsigned int quiet_given ;	/**< @brief Whether quiet was given.  */
   unsigned int summary_given ;	/**< @brief Whether summary was given.  */