#!/usr/bin/gawk -f
#
function junkview_version() { return "junkview 2008-09-15a" }
#
# display frequency distribution for most recent firewall rejects in 
#  terms of port_number/protocol, also displays offenders' hits, IP 
#  addresses, country codes and matching network addresses
#
# Copyright (C) 2006 Grant Coady <grant@bugsplatter.id.au>
# See: <http://bugsplatter.id.au/junkview/> for related scripts and info
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License: http://www.gnu.org/licenses/licenses.html#GPL
#
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# usage
# ``````
# junkview [options] logfile [another_logfile] [...]
#
# zcat /path/to/logfile.n.gz | junkview [options]
#
# See also: junkshow bash script cron example
#
# logfile parser control
# ```````````````````````
# junkview filters input log data file records on two parameters: the 
#  optional timestamp filter and the required iptables --log-prefix word.
# 
# timestamp filter
# `````````````````
# not specifying a value sets it to zero, finish defaults to (now)
#
# start and finish are specified in epoch seconds, use the date 
#  function to perform the conversion from 'human readable' time 
#  into epoch seconds, eg. start=$(date -d "2006-03-06 12:00" +%s)
#
# options: 1) x hours before finish if start == 0, hours > 0
#	   2) x hours after start if start > 0, hours > 0
#	   3) start then finish if start > 0, start < finish
#	   4) disabled: all records if start == 0, hours == 0
#
# Note: hours, start and finish do not appear in /etc/junkview.conf since 
#  this defeats CLI override (as of 2006-04-03).
#
# iptables --log-prefix word
# ```````````````````````````
# You must specify the first word of iptables --log-prefix for records 
#  to be recognised for processing, and this prefix word must not clash 
#  with any other "kernel:" log entries.  You may specify the start of 
#  the prefix word, allowing for wildcard completion of first word.  
#
# You may also use a second --log-prefix word for packet classification, 
#  see 'classify' for an example.
#
# Note: normally read from /etc/junkview.conf, CLI parameter overrides the 
#  /etc/junkview.conf value.
#
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# option	description					default
# ------------	----------------------------------------------	--------
# -v hours=n	number of hours after start or before finish 
#		 to perform logfile parsing, see above		0
#
# -v start=n	start time in epoch seconds			0
#
# -v finish=n	finish time in epoch seconds			0
#
# -v year       log start year, see code comments below         ?
#
# -v prefix="s"	first word of firewall --log-prefix		n/a
#
# tail_log=n	non-zero to output records after processing	0
#
# verbose=n	output additional progress information,		0 
#		 1 = a little more info, 2 = data processing, 
#		 3 = report database file reads,
#		 4 = search values + lo, hi
#
# Note: See /etc/junkview.conf for the rest of configuration information
#
# credits:
#	Tim Bray: binary search algorithm
#	Mikhail Zotov: much testing, bug spotting and suggestions
#	comp.lang.awk, comp.unix.shell people --> dwarf stars ;)
#	ip2country data source: http://software77.net/
#
# reference:
#  gawk: http://www.gnu.org/software/gawk/manual/
#  binary search: http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary
#

#### setup processing parameters from /etc/junkview.conf and CLI
function check_cli_parms()
{
	# these database filepaths values from /etc/junkview.conf only
	datapath = junkconf["datapath"]
	ip2cfile = datapath "/" junkconf["ip2cfile"]
	namesfile = datapath "/" junkconf["namesfile"]
	if (junkconf["classify"])	# optional
		classify = datapath "/" junkconf["classify"]
	widthmax = junkconf["widthmax"]
	if (widthmax < 71 || widthmax > 131) widthmax = 79
	
	# general values, CLI overrides non-zero .conf value
	if (!client)		client		= junkconf["client"]
	if (!chartlen)		chartlen	= junkconf["chartlen"]
	if (!min_hits)		min_hits	= junkconf["min_hits"]
	if (!showaddr)		showaddr	= junkconf["showaddr"]
	if (!showhits)		showhits	= junkconf["showhits"]
	if (!maxports)		maxports	= junkconf["maxports"]
	if (!verbose)		verbose		= junkconf["verbose"]
	if (!nochart)		nochart		= junkconf["nochart"]
	if (!logchart)		logchart	= junkconf["logchart"]
	if (!nohostname)	nohostname	= junkconf["nohostname"]
	if (!dns_wait)		dns_wait	= junkconf["dns_wait"]
	if (!noshowport)	noshowport	= junkconf["noshowport"]
	if (!showdest)		showdest	= junkconf["showdest"]
	if (!hot_list)		hot_list	= junkconf["hot_list"]

	# allow CLI value of -1 to zero some parameters from .conf
	if (client < 0) client = 0
	if (dns_wait < 0) dns_wait = 0

	if (verbose > 2) {
		format = "  %-12s = %s\n"
		print "\nparameters:"
		printf format, "prefix",	prefix
		printf format, "datapath",	datapath
		printf format, "ip2cfile",	ip2cfile
		printf format, "namesfile",	namesfile
		printf format, "classify",	classify
		printf format, "client",	client
		printf format, "chartlen",	chartlen
		printf format, "hot_list",	hot_list
		printf format, "min_hits",	min_hits
		printf format, "showaddr",	showaddr
		printf format, "showdest",	showdest
		printf format, "showhits",	showhits
		printf format, "maxports",	maxports
		printf format, "nochart",	nochart
		printf format, "logchart",	logchart
		printf format, "nohostname",	nohostname
		printf format, "dns_wait",	dns_wait
		printf format, "noshowport",	noshowport
		printf format, "verbose",	verbose
		printf format, "widthmax",	widthmax
		print "done\n"
	}
}
#### setup logfile parser
function setup_timestamp_filter()
{
	if (hours < 0) hours = 0
	if (!finish) finish = systime()
	timestamp_filter = 0
	if (!start && hours) {
		start = finish - hours * 3600; ++timestamp_filter
	}
	else if (start && hours) {
		finish = start + hours * 3600; ++timestamp_filter
	}
	else if (start && start < finish) ++timestamp_filter

	if (year < 1970 || year > 2037) {
		year = strftime("%Y", systime()) # enable auto year adjust
		run_month = int(strftime("%m", systime()))
	}
	else {
		run_month = 99		# disable auto year adjust
	}
	if (timestamp_filter) {
		printf "Timestamp filter: \n  "
		if (hours)
			printf "%d hour%s from ", hours, (hours > 1 ? "s" : "")
		printf "%s to %s\n",
			strftime("%Y %b %d %H:%M:%S", start),
			strftime("%Y %b %d %H:%M:%S (%z)", finish)
	}
}
BEGIN {
	print "\n" junkview_version() " iptables log analysis\n"

	# make month name to number lookup
	n = split("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", k)
	for (i = 1; i <= n; i++) month[k[i]] = i

	# make flag name to first letter lookup
	n = split("SYN ACK FIN RST PSH CWR ECE", k)
	for (i = 1; i <= n; i++) xtcpflags[k[i]] = substr(k[i], 1, 1)

	read_database_file("/etc/junkview.conf", "configuration", "1.00", 1)

	#  CLI may override .conf value with -v prefix="string"
	if (!prefix) prefix = junkconf["prefix"]

	setup_timestamp_filter()

	if (!prefix) prefix = "JLE"	# Junkview Log Entry
	xprefix	= prefix "[^ ]*"	# prefix is "JLE* "
}
#### logfile parser -- data collection
verbose && FNR == 1 {
	print "Reading " FILENAME	# perhaps show filenames
}
!machine { machine = $4 }

timestamp_filter && !record_date_first {
	record_date_first = $1" "$2" "$3
}
timestamp_filter && !log_start_month {
	log_start_month = month[$1]
	# we looking at previous year log start?
	if (log_start_month > run_month)
		--year
}
timestamp_filter && log_start_month {
	# bump year if cross over new year
	if (month[$1] < last_rec_month)
		++year
	last_rec_month = month[$1]

	# timestamp filter
	split($3, k, ":")
	xdatestamp = mktime(year" "month[$1]" "$2" "k[1]" "k[2]" "k[3])
	if (xdatestamp < start || xdatestamp > finish)
		next
}
# check if last record repeated, may have multiple repeat records
/message repeated/ && xsrc_port_prot {
	if (tail_log) tail_recs[++tail_cnt] = $0
	xrepeat = $8			# last x times repeated value
	repeats += xrepeat		# report total repeats
	junk_class[xclass] += xrepeat
	junk_proto[xproto] += xrepeat
	junk_src_dst[xsrc_ip":"xdst_ip] += xrepeat
	raw_addr_port[xsrc_port_prot] += xrepeat
	next
}
!/message repeated/ && xsrc_port_prot { xsrc_port_prot = "" } # disable repeat

# looking for next record from iptables logger

$5 !~ /kernel:/ { next }		# skip non-kernel record
{ sub(/\[.*\]/, "") }			# eat optional 2.6 kernel timestamp
$6 !~ xprefix { next }			# skip if the not wanted --log-prefix

# found new iptables log record

tail_log { tail_recs[++tail_cnt] = $0 }

{
	xclass = $7			# prefix 2nd word is classify index

	sub(/kernel:.*SRC=/, "")	# $5 is now src_ip
	sub(/DST=/, "")			# $6 is now dst_ip

	split($5, k, ".")		# source IP stored as simple number
	xsrc_ip = sprintf("%010u", ((k[1]*256+k[2])*256+k[3])*256+k[4])

	split($6, k, ".")		# dest IP stored as simple number
	xdst_ip = sprintf("%010u", ((k[1]*256+k[2])*256+k[3])*256+k[4])

	sub(/LEN=.*PROTO=/, "")		# $7 is now protocol
	xproto = $7
	if (xproto ~ /(TCP|UDP)/) {
		sub(/SPT=.*DPT=/, "")	# $8 is now dest_port

		xdport = $8; xflags = ""
		if (xproto ~ /TCP/) {	# collect tcp flags
			sub(/WINDOW=.*RES=0x[0-9a-f][0-9a-f] /, "")
			xflags = "-"
			for (i = 9; i <= NF; i++) { # $9 is tcp-flags start
				if (xtcpflags[$i])
					xflags = xflags xtcpflags[$i]
				else	break
			}
		}
		xsrc_port_prot = xsrc_ip ":" xdport "/" xproto xflags
	}
	else if (xproto ~ /ICMP/) {
		sub(/TYPE=/, "")	# $8 is icmp type
		sub(/CODE=/, "")	# $9 is icmp code
		xsrc_port_prot = xsrc_ip ":" $8 "." $9 "/ICMP"
	}
	else {
		# protocol unknown, record source_ip and protocol
		xsrc_port_prot = xsrc_ip ":/" xproto
	}
	++junk_class[xclass]
	++junk_proto[xproto]
	++junk_src_dst[xsrc_ip":"xdst_ip]
	++raw_addr_port[xsrc_port_prot]
}
#### post processing
END {
	if (timestamp_filter) {
		record_date_last = $1" "$2" "$3
		printf "Log records found from %s to %s\n", \
		       record_date_first, record_date_last
	}
	if (dataload == 1) exit 1	# for data load timing
	records = NR
	check_cli_parms()		# setup post-process options
	convert_raw_list()
	if (verbose) {
		printf "Checked %d records plus %d repeats to find %d " \
			"from %s.\n", records, repeats, total, machine
		display_protocol_summary()
	}
	if (!nochart) chart_hits_per_port()
	display_classify_summary()
	if (!showhits) exit 0		# done if no hits summary requested
	if (client) {
		# 2008-08-13 convert to using ip2cn-server, check if server 
		#  is listening
		ip2cn_server = "/inet/tcp/0/localhost/4743"
		cmd = "netstat -l | grep 4743 | grep -c LISTEN"
		cmd | getline client; close(cmd)
	}
	if (!client) {
		# load database files when not in client mode or server not 
		#  listening
		read_database_files("Load database files")
	}
	if (verbose > 1) print "\nDiscover top offenders:"
	build_cons_hits_addr()
	build_ip2c_lookup()
	build_cons_hits_network()
	trim_network_addr_list()
	display_top_hits_by_host_dport()
	display_top_hits_by_net()
	if (tail_log) {			# dump log records after summary
		printf "\nRecords processed (%d):\n", tail_cnt
		x = 1; while (tail_recs[x]) print tail_recs[x++]
	}
}
#### hits per port chart
function autoscale(max, totl,   i, k, m, n, p) # set scale
{
	if (!logchart) {
		p = 100 * max / totl
		n = split("10 12 16 20 24 32 40 48 60 80 100", k)
		for (i = 1; i <= n; i++) if (k[i] >= p) break
		scale = k[i]
	} else {
		scale = 100
	}
}
function show_chart_bar(label, val,   pval, bar, barf, grid, width)
{
		barf = "|((((((((((((((((((((((((((((((((((((((((!" # bar
		bard = "|----------------------------------------!" # bar
	if (!logchart) {
		grid = "|         .         .         .         . " # grid
	} else {
		grid = "|     .      .        .        .        . " # grid
	}
	width = 40

	pval = 100 * val / total
	if (!logchart) {
		bar = int((pval * width + 0.5 * scale) / scale)
	} else {
		bar = int((pval ^ 0.5) * 4 + 0.5)
	}

	if (1) {
		printf "%11s %6d %s%s%5.1f\n", tolower(label), val, 
				substr(barf, 1, ++bar), 
				substr(grid, ++bar, width + 1), pval
	} else {
		printf "%11s %6d %s%s%5.1f\n", tolower(label), val,
				(bar <= width ? \
				substr(bard, 1, bar++) "+" : \
				substr(bard, 1, ++bar)),
				substr(grid, ++bar, width + 1), pval
	}
}
function show_chart_tick_scale(   tick)
{
	if (!logchart) {
		tick = "+ - - - - + - - - - + - - - - + - - - - + - - -"
		printf "%11s %6s %s\n", "total", total, tick
		printf "%20s %10.1f%% %8.1f%% %8.1f%% %8.1f%%\n", "0",
			scale * 0.25, scale * 0.5, scale * 0.75, scale
	} else {
		tick = "+ - - + - - -+- - - - + - - - -+- - - - + - - -"
		tnum =  "     2      10       30       60      100%"
		printf "%11s %6s %s\n", "total", total, tick
		printf "%20s%s\n", "0", tnum
	}
}
function chart_hits_per_port(   hits, i, j, k, kk, max, o, pf, xp)
{
	if (!total) return

	# find top n hits for barchart, change back to port order
	for (i = sort_hits_port_len; i > sort_hits_port_len - chartlen; i--) {
		if (!i) break
		split(sort_hits_port[i], k, ":")

		# promote ICMPs to list top by giving tcp/udp lead-zeroes
		if (k[2] !~ /ICMP/) {
			split(k[2], kk, "/")
			k[2] = sprintf("%07d/%s", kk[1], kk[2])
		}
		xp[i] = sprintf("%11s:%6d", k[2], k[1])
	}
	delete sort_hits_port		# chart_hits_per_port

	# display chart
	pf = asort(xp)
	print ""
	autoscale(max_hits, total)
	o = total
	for (i = 1; i <= pf; i++) {
		split(xp[i], k, ":")
		if (k[1] !~ /ICMP/) {	# remove lead-zero tcp/udp demotion
			split(k[1], kk, "/")
			k[1] = sprintf("%d/%s", kk[1], kk[2])
		}
		show_chart_bar(k[1], k[2])
		o -= k[2]
	}
	delete xp

	# display remaining hits
	if (o) show_chart_bar("others", o)
	show_chart_tick_scale()
}
#### read database files
function read_database_file(f, n, v, x)
{
	if (verbose > 1) printf "Open " f
	if (!(getline < f) > 0) {
		if (x) { print "\nfatal: " n " not recognised"; exit 3 }
		if (verbose > 1) print ", not recognised"; close(f); return

	}
	if ($1 != "junkview" || $2 != n || $3 != v) {
		if (x) { print "\nfatal: " n " not found"; exit 3 }
		if (verbose > 1) print ", not found"; close(f); return
	}
	if (verbose > 1) printf ", reading"
	while ((getline < f) > 0) {
		# fast path: largest data files first 
		if (n == "ip2c-data") {
			ip_data[++ip_size,1] = $1
			ip_data[ip_size,2] = $2
			ip_data[ip_size,3] = $3
			continue
		}
		else if (n == "cc2names") {
			split($0, k, ":"); cc_names[k[1]] = k[2]
			continue
		}
		# slow path, eat comments and blank lines
		sub(/#.*$/, ""); if (/^$/) continue
		else if (n == "configuration") {
			junkconf[$1] = $2
		}
		else if (n == "classify") {
			split($0, k, ":")
			split(k[1], kk) # remove lead blanks from key
			classtab[kk[1]] = k[2]
		}
	}
	close(f)
	if (verbose > 1) print ", done"
}
function read_database_files(s)
{
	if (verbose) print "\n" s
	read_database_file(ip2cfile,  "ip2c-data",  "1.00", 1)
	read_database_file(namesfile, "cc2names",   "1.00", 0)
}
#### classify summary
function display_classify_summary(   i, j, k, kk, tag)
{
	if (classify) {
		if (verbose > 1) print ""
		read_database_file(classify, "classify", "1.00", 0)
		print "\nClassify junk:"
		j = 0
		for (i in junk_class)
			kk[++j] = sprintf("%6d %s", junk_class[i], i)
		j = asort(kk)
		for (i = j; i > 0; i--) {
			split(kk[i], k)
			if (!(tag = classtab[k[2]])) tag = k[2]
			printf "%6d  %s\n", k[1], tag
		}
		delete classtab		# display_classify_summary
	}
	delete junk_class		# display_classify_summary
}
#### protocol summary
function display_protocol_summary(   i, j, k, kk)
{
	printf "Protocol summary: "
	j = 0
	for (i in junk_proto)
		kk[++j] = sprintf("%6d %s", junk_proto[i], i)
	j = asort(kk)
	for (i = j; i > 0; i--) {
		if (i < j) printf ", "
		split(kk[i], k)
		printf "%d %s", k[1], k[2]
	}
	print ""
	delete kk
	delete junk_proto
}			
#### hits summary
function convert_raw_list(   hp, i, j, k)
{
	if (verbose > 1) printf "Create sort_addr_port"
	# input: raw_addr_port[src_ip:dport/proto-flags] = hits)
	j = 0
	for (i in raw_addr_port) {
		split(i, k, ":")
		# output: addr, hits, port
		sort_addr_port[++j] = sprintf("%010d:%06d:%s", \
			      k[1], raw_addr_port[i], k[2])

		# output: hits / port (for chart)
		sub(/-.*$/, "", k[2])	# strip tcp-flags for chart
		hp[k[2]] += raw_addr_port[i]
	}
	if (verbose > 1) printf ", sort"
	sort_addr_port_len = asort(sort_addr_port)

	# discover total hits, max hits / port, build hits / port list
	if (verbose > 1) printf ", create sort_hits_port list"
	j = 0; total = 0
	for (i in hp) {
		hits = hp[i]
		total += hits
		if (hits > max_hits) { max_hits = hits }
		sort_hits_port[++j] = sprintf("%6d:%11s", hits, i)
	}
	delete hp
	if (verbose > 1) printf ", sort"
	sort_hits_port_len = asort(sort_hits_port)

	delete raw_addr_port		# convert_raw_list
	if (verbose > 1) print ", done."
}
#### ip2country search support
function search(a, k, s,   l, h, m)
{
	l = 1; h = s
	while (h - l > 1) {
		m = int((l + h) / 2); if (k[m,1] < a) l = m; else h = m
	}
	if (k[h,1] > a) --h; return h	# return match, or closest less than
}
function search_ip2country(a,   i, m, n, o, cc, min, max)
{
	# entry: dotquad IP, returns cidr/nn:cc or -:-- if not found
	if (no_ip2c) return "-:--"	# no database loaded
	if (client) {			# perhaps use optional server

		# junkview will exit if ip2cn-server disappears during run
		print a " +" |& ip2cn_server
		ip2cn_server |& getline o
		return o
	}
	# standalone operation
	if (!ip_size) {			# shutdown ip2c if no database loaded
		++no_ip2c; return "-:--" 
	}		
	i = search(a, ip_data, ip_size)	# lookup country code
	min = ip_data[i,1]; max = ip_data[i,2]; cc = ip_data[i,3]
	if (a < min || a > max) {
		return "-:--"
	}
	# find CIDR within allocation, as allocation may span several CIDRs
	m = 0x0fffffff; n = 4		# set start hostmask, netmask counter
	while (and(a, compl(m)) < min) { ++n; m = rshift(m, 1) }
	while (or(a, m) > max) { ++n; m = rshift(m, 1) }
	a = bin2dotquad(and(a, compl(m)), n)
	o = sprintf("%s:%s :%s:%010d:%010d", a, cc, cc_names[cc], min, max)
	return o
}
function bin2dotquad(addr, mask,   j, k)
{
	for (j = 4; j > 0; j--) { k[j] = and(addr, 255); addr /= 256 }
	if (!mask) return sprintf("%d.%d.%d.%d", k[1], k[2], k[3], k[4])
	else return sprintf("%d.%d.%d.%d/%d", k[1], k[2], k[3], k[4], mask)
}
function bin2dotfill(addr,   j, k)	# zero fill dotquad for debug
{
	for (j = 4; j > 0; j--) { k[j] = and(addr, 255); addr /= 256 }
	return sprintf("%3d.%3d.%3d.%3d", k[1], k[2], k[3], k[4])
}
function get_net_address(a, b,   m, n, x)
{
	# return cidr network address that includes a and b

	x = xor(a, b); m = 0xfffffffc; n = 30
	while (and(x, m) && n) { m = and(2 * m, 0xffffffff); --n }
	return bin2dotquad(and(a, m), n)
}
#### hits summary data processing
function build_cons_hits_addr(   a, h, i, j, k, o, p, run)
{
	if (verbose > 1) printf "Create addr_hits, hits_addr"
	# build tot_hits, addr, hits / port list (consolidate hits / addr)
	j = run = 0
	for (i = sort_addr_port_len; i >= 0; i--) { # go one past end array
		split(sort_addr_port[i], k, ":")
		o = sprintf("%d %s", k[2], k[3])
		if (!run) { a = k[1]; h = k[2]; p = o; ++run; continue }
		if (k[1] != a) {
			# new addr, store info..
			hits_addr_port[++j] = sprintf("%06d:%010d:%s",h,a,p)
			addr_hits_port[j]   = sprintf("%010d:%06d:%s",a,h,p)
			hits_addr_lookup[a] = sprintf("%06d:%s", h, p)

			# ..start next consolidation
			a = k[1]; h = k[2]; p = o
		}
		else {
			# code to go here to limit port detail list length
			h += k[2]
			p = p "!" o
		}
	}
	if (verbose > 1) printf ", sort addr_hits"
	addr_hits_port_len = asort(addr_hits_port)
	if (verbose > 1) printf ", sort hits_addr"
	hits_addr_port_len = asort(hits_addr_port)

	if (verbose > 1) print ", done."
}
function build_ip2c_lookup(   i, ip2c, k, kk, lo, hi, max, net, run)
{
	if (verbose > 1) printf "Create ip2c and netw lookup tables"
	# build lookup table: containing allocation, country code, name
	max = -1; run = 0
	for (i = 1; i <= sort_addr_port_len; i++) {
		split(sort_addr_port[i], k, ":") # k1 addr, k2 hits, k3 port
		if (verbose > 3) printf "\n  %s ", bin2dotfill(k[1])

		if (k[1] > max) {	# start new block
			if (run) {
				# NOTE: lo, hi become fields 6 and 7 !!
				netw_ip2c[kk[1]] =\
					 sprintf("%s:%d:%d", ip2c, lo, hi)
				if (verbose > 3)
					printf "` %08x-%08x, lo %08x, hi %08x, %s",\
					net, max, lo, hi, kk[1]
			}
			else {
				run++
			}
			ip2c = search_ip2country(k[1])
			split(ip2c, kk, ":")
			# kk1 net CIDR, kk2 cc, kk3 cname, kk4 min, kk5 max
			lo = k[1]; net = kk[4]; max = kk[5]
		}
		hi = k[1]; netw_lookup[hi] = kk[1]
	}
	# NOTE: lo, hi become fields 6 and 7 !!
	netw_ip2c[kk[1]] = sprintf("%s:%d:%d", ip2c, lo, hi) # store last block
	if (verbose > 3)
		printf "- %08x-%08x, lo %08x, hi %08x, %s", net, max, lo, hi, kk[1]
	delete sort_addr_port		# build_ip2c_lookup
	if (!client) {
		delete ip_data		# build_ip2c_lookup
	}

	if (verbose > 1) print ", done."
}
function build_cons_hits_network(   al, hits, i, j, k, net, nw, run)
{
	if (verbose > 1) printf "Consolidate hits per network"
	# build total_hits, net_addr, fake_net, addr_list
	j = run = 0
	for (i = 1; i <= addr_hits_port_len; i++) {
		split(addr_hits_port[i], k, ":")
		net = netw_lookup[k[1]]
		if (!run) {
			al = k[1]; hits = k[2]; nw = net; ++run; continue
		}
		if (net != nw) {
			if (hits >= min_hits) {
				hits_netw_addr[++j] = \
					sprintf("%06d:%s:%s", hits, nw, al)
				if (verbose > 3)
					printf "\n  %s ", hits_netw_addr[j]
			}
			al = k[1]; hits = k[2]; nw = net
		}
		else {
			al = al "!" k[1]; hits += k[2]
		}
	}
	# save last entry
	if (hits >= min_hits) {
		hits_netw_addr[++j] = sprintf("%06d:%s:%s", hits, nw, al)
		if (verbose > 3)
			printf "\n  %s ", hits_netw_addr[j]
	}
	delete addr_hits_port		# build_cons_hits_network
	if (verbose > 1) printf ", sort"
	hits_netw_addr_len = asort(hits_netw_addr)
	if (verbose > 1) print ", done."
}
function trim_network_addr_list(   al, ar, h, i, j, k, kk, k3, m, n, hl, nl)
{
	if (verbose > 1) printf "Trim hits per network for display"
	for (i = hits_netw_addr_len; i > hits_netw_addr_len - showhits; i--) {
		split(hits_netw_addr[i], kk, ":")
		n = split(kk[3], k, "!") # get address list
		if (n <= showaddr) {
			hits_netw_trim[i] = hits_netw_addr[i]; continue
		}
		delete hl		# build hits / addr list and sort
		for (j = 1; j <= n; j++) {
			split(hits_addr_lookup[k[j]], k3, ":")
			hl[j] = sprintf("%06d:%010d", k3[1], k[j])
		}
		m = asort(hl)
		h = kk[1]; n = 0; delete nl # now pick the top n hit addresses
		for (ar = m; ar > m - showaddr; ar--) {
			split(hl[ar], k3, ":")
			h -= k3[1]	# track remaining hits for this netw
			nl[++n] = sprintf("%010d", k3[2])
		}
		asort(nl)
		for (j = 1; j <= n; j++) { # rebuild addr_list in addr order
			if (j == 1) al = nl[j]
			else al = al "!" nl[j]
		}
		al = al ":" h ":" ar	# append remaining hits and addr count
		hits_netw_trim[i] = sprintf("%06d:%s:%s", kk[1], kk[2], al)
	}
	delete hits_netw_addr		# trim_network_addr_list
	if (verbose > 1) print ", done."
}
function lookup_hostname(a, w,   i, j, k, n, o)
{
	if (nohostname) return "-"
	split(a, k, "/")
	if (dns_wait)
		i = "host -W " dns_wait " " k[1]
	else
		i = "host " k[1]
	i | getline o; close(i)

	if (o ~ /has no PTR record/) {
		return "noname: no PTR record"
	}
	if (o ~ /connection timed out/) {
		return "noname: timed out"
	}
	i = split(o, k); o = k[i]	# get last field
	if (o ~ /NXDOMAIN/) return "noname: nxdomain"
	if (o ~ /SERVFAIL/) return "noname: servfail"
	sub(/\.$/, "", o)		# trim trail dot

	if (w) {			# trim name length to <w>
		n = length(o); if (n > w) o = "~" substr(o, n - w - 1, n)
	}
	return o
}
function trim_port_list(p, w,   i, j, k, m, o)
{
	split(p, k, "!")
	i = m = 1; o = k[i]; j = length(k[i])
	while (k[++i]) {
		if (j + length(k[i]) + 2 <= w) {
			o = o ", " k[i]; j += length(k[i]) + 2
		}
		else if (m < maxports) {
			o = o ":" k[i]; j = length(k[i]); ++m
		}
		else {
			o = o " ~"; break
		}
	}
	return tolower(o)
}
function hotlist_init(   i)
{
	if (!hot_list) return
	split(hot_list, hot_list_ports, ":")
	if (verbose > 1) {
		printf "Hot list ports: "
		for (i in hot_list_ports)
			printf "%d ", hot_list_ports[i]
		print ""
	}
}
function hotlist_check(p,   i, j, k3, k4, k5)
{
	if (!hot_list) return 0
	# this is where we need new hotlist display exception, port list is 
	#  'p' in the form: <count> <port>/<proto>[!...]
	split(p, k3, "!")
	for (i in k3) {
		split(k3[i], k4)
		split(k4[2], k5, "/")
		if (verbose > 2) print "port: " k5[1]
		for (j in hot_list_ports) {
			if (hot_list_ports[j] == k5[1])
				return 1
		}
	}
	return 0
}
function destination_init(   i, j, k, l)
{
	if (showdest) {
		if (verbose > 1) print "Build source, hits, dest lookup"
		j = 0
		for (i in junk_src_dst) {
			split(i, k, ":")
			sort_src_hit_dst[++j] = sprintf("%010d:%06d:%010d", \
					k[1], junk_src_dst[i], k[2])
		}
		sort_src_hit_dst_len = asort(sort_src_hit_dst)
	}
	delete junk_src_dst
	if (!showdest) return

	# add ranking (match number)
	l = 0
	for (i = sort_src_hit_dst_len; i > 0; i--) {
		split(sort_src_hit_dst[i], k, ":")
		if (k[1] != l ) {
			j = 0; l = k[1]
		}
		sort_src_hit_dst[i] = sprintf("%010d:%06d:%010d:%06d", \
			k[1], k[2], k[3], ++j)
	}
}
function destination_find(a, m,   j, k) # addr, match number
{
	for (j = sort_src_hit_dst_len; j > 0; j--) {
		split(sort_src_hit_dst[j], k, ":")
		if (a == k[1] && m == k[4])
			return sprintf("%d %s %s", \
				k[2], "->", bin2dotquad(k[3]))
	}
}
function max(a,b)
{
	if ( a > b) return a; else return b
}
function display_top_hits_by_host_dport(   a, h, i, j, k, kk, k3, n, y)
{
	if (noshowhost) return
	hotlist_init()
	destination_init()
	print "\nTop offenders by host, destination ports:"
	print \
"  hits code host                name/dest ports"
#1234567891123456789212345678931234567894123456789512345678961234567897123456789
#------  --  ---------------     -----------------------------------------------
	# display the top n items
	for (i = hits_addr_port_len; i > 0; i--) {

		split(hits_addr_port[i], kk, ":")
		if (hits_addr_port_len - i >= showhits || kk[1] < min_hits)
			if (!hotlist_check(kk[3])) continue

		split(netw_ip2c[netw_lookup[kk[2]]], k, ":")
		a = bin2dotquad(kk[2]); h = lookup_hostname(a, widthmax - 33)
		printf "%6d  %3s %-18s  %s\n", kk[1], k[2], a, h
		if (noshowport) continue

		x = trim_port_list(kk[3], widthmax - 33)
		y = max(split(x, k, ":"), showdest)
		for (x = 1; x <= y; x++) {

			if (x <= showdest) {
				# display most frequent destination ip as
				#  "nn  ->  nnn.nnn.nnn.nnn ..."
				split(destination_find(kk[2], x), k3)
				if (! k3[1]) continue
				printf "%6s  %-3s %-18s  %s\n",
					k3[1], k3[2], k3[3], k[x]
				continue
			}
			if (!k[x]) continue
			printf "%6s  %3s %-18s  %s\n", "", "", "", k[x]
		}
		print ""
	}
	delete netw_lookup
	delete hits_addr_port
	delete sort_src_hit_dst
}
function display_top_hits_by_net(   a, h, i, j, k, kk, k3, k4, m, n, x, y)
{
	if (noshow_net || no_ip2c || !hits_netw_addr_len) return
	print "Top offenders by network, host, destination ports:"
	if (noshowport)
		print \
"  hits code network addr        lookup addr         country"
	else
		print \
"  hits code network/host        lookup/dest ports   country"
#1234567891123456789212345678931234567894123456789512345678961234567897123456789
#------  --  ------------------  ------------------  ---------------------------
#               ---------------  -----------------------------------------------
	for (i = hits_netw_addr_len; i > hits_netw_addr_len - showhits; i--) {
		split(hits_netw_trim[i], kk, ":")
		if (!kk[1]) break
		split(netw_ip2c[kk[2]], k, ":")

		if (verbose > 3) {
			printf "netw_ip2c: lo %08x, hi %08x, %s\n",\
			       k[6], k[7], kk[2]
		}

		printf "%6d  %3s %-18s  %-18s  %s\n", kk[1], k[2], 
			get_net_address(k[6], k[7]), kk[2], k[3]

		# now display IPs per net
		n = split(kk[3], k3, "!")
		for (j = 1; j <= n; j++) {
			a = bin2dotquad(k3[j])
			if (noshowport) h = lookup_hostname(a, widthmax - 33)
			split(hits_addr_lookup[k3[j]], k4, ":")

			x = trim_port_list(k4[2], widthmax - 33)
			y = split(x, k, ":")
			for (x = 1; x <= y; x++) {
				if (!k[x]) break
				if (n > 1 && x == 1) {
					printf "%6d  %3s %-18s  %s\n",
						k4[1], "", a, k[x]
					continue
				}
				if (n == 1 && x == 1) {
					printf "%6s  %3s %-18s  %s\n",
						"", "", a, k[x]
					continue
				}
				if (!noshowport) {
					printf "%6s  %3s %-18s  %s\n",
						"", "", "", k[x]
					continue
				}
				if (noshowport && n > 1 && x == 1) {
					printf "%6d  %3s %-18s  %s\n",
						k4[1], "", a, h
					continue
				}
				if (noshowport && n == 1 && x == 1) {
					printf "%6s  %3s %-18s  %s\n",
						"", "", a, h
					continue
				}
			}
		}
		if (kk[4]) { # display remainder
			printf "%6d  %3s %-18s  %s\n", kk[4], "", 
				"from " kk[5] " more addr", ""
		}
		print ""
	}
	delete hits_netw_trim		# display_top_hits_by_net
	delete netw_ip2c		# display_top_hits_by_net
	delete hits_addr_lookup		# display_top_hits_by_net

	if (!noshowport)
	  print "key: tcp: a ack, c cwr, e ece, f fin, p psh, r rst, s syn"
}
# end