path: root/module/bsnmp-jails.c

/*
 * Copyright (c) 2008, Stefan Walter
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *     * Redistributions of source code must retain the above
 *       copyright notice, this list of conditions and the
 *       following disclaimer.
 *     * Redistributions in binary form must reproduce the
 *       above copyright notice, this list of conditions and
 *       the following disclaimer in the documentation and/or
 *       other materials provided with the distribution.
 *     * The names of contributors to this software may not be
 *       used to endorse or promote products derived from this
 *       software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 *
 * CONTRIBUTORS
 *  Stefan Walter <stef@memberwebs.com>
 */

#include "usuals.h"

#include <sys/param.h>
#include <sys/types.h>

#include <sys/jail.h>
#include <sys/limits.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/socket.h>
#include <sys/select.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/user.h>
#include <sys/wait.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <arpa/inet.h>

#include <ctype.h>
#include <fcntl.h>
#include <limits.h>
#include <kvm.h>
#include <paths.h>
#include <syslog.h>
#include <stdarg.h>
#include <unistd.h>

#include <bsnmp/snmpmod.h>
#include <pcap.h>

#include "common/hash.h"

#include "jails_tree.h"
#include "jails_oid.h"

/* -------------------------------------------------------------------
 * COMPAT
 */

struct xprison_v1 {
        int             pr_version;
        int             pr_id;
        char            pr_path[MAXPATHLEN];
        char            pr_host[MAXHOSTNAMELEN];
        u_int32_t       pr_ip;
};

#ifndef JAIL_MAX_IPS
#define JAIL_MAX_IPS    256
#endif

struct xprison_v2 {
        int             pr_version;
        int             pr_id;
        char            pr_path[MAXPATHLEN];
        char            pr_host[MAXHOSTNAMELEN];
        u_int32_t       pr_ips[JAIL_MAX_IPS];
        u_int           pr_nips;
};

/* -------------------------------------------------------------------
 * DECLARATIONS
 */

#define SNAP_LEN 64

/* our module handle */
static struct lmodule *module;

/* OIDs */
static const struct asn_oid oid_jails = OIDX_jails;

/* the Object Resource registration index */
static u_int reg_index = 0;

struct monitor {
	TAILQ_ENTRY(monitor)	link;

	int 			refs;

	char                    *device;
	pcap_t                  *handle;
	void                    *watch;
	struct bpf_program      filter;
	int                     filter_valid;
};

TAILQ_HEAD(monitor_list, monitor);

/* list of monitor structures */
static struct monitor_list monitors = TAILQ_HEAD_INITIALIZER (monitors);

/* Represents a 'top level' process */
struct ptop {
	pid_t 			pid;
	uint32_t		cpu_time;

	#define PROC_GONE    0
	#define PROC_EXISTS  1
	#define PROC_NEW     2
	int			status;
};

struct jaildat {
	uint32_t                index;
	TAILQ_ENTRY(jaildat)    link;
	int                     mark;

	/* Configuration */
	int                     jailid;
	char			*host;
	char                    *path;

	/* Monitor pointers for each address */
	char                    n_addrs;
	struct {
		struct sockaddr_in ip;
		struct monitor  *monitor;
	} addrs[JAIL_MAX_IPS];

	/* Stats gathered */
	uint64_t		in_octets;
	uint64_t		in_packets;
	uint64_t		out_octets;
	uint64_t 		out_packets;
	uint64_t		disk_space;
	uint64_t		disk_files;
	uint32_t		cpu_time_total;
	uint32_t		cpu_time_offset;
	uint32_t                n_processes;
	uint32_t		n_threads;

	/* Top process information */
	uint32_t		n_ptops;
	struct ptop		*ptops;
};

TAILQ_HEAD(jaildat_list, jaildat);

/* list of jail structures */
static struct jaildat_list jaildats = TAILQ_HEAD_INITIALIZER (jaildats);

/* number of if jail structures */
static u_int jaildat_count = 0;
static u_int jaildat_index = 0;

/* Hash of jail structures by id */
static hsh_t *jaildat_by_host = NULL;

/* Hash of jail structures by path */
static hsh_t *jaildat_by_path = NULL;

/* Hash of jail structures by address */
static hsh_t *jaildat_by_address = NULL;

/* Hash of jail structures by id */
static hsh_t *jaildat_by_id = NULL;

/* Timer for refreshing the jaildat info */
static void *timer_refresh = NULL;

/* Refresh interval in SNMP ticks */
static int refresh_interval = 3 * 100;

/* Timer for measuring the jails */
static void *timer_measure = NULL;

/* Measure interval in SNMP ticks */
static int measure_interval = 1800 * 100;

/* The monitor network filter */
static u_char *network_filter = NULL;

/* The KVM interface handle */
static kvm_t *kvm_handle = NULL;

/* -----------------------------------------------------------------------------
 * HELPERS
 */

static void
emsg(const char *format, ...)
{
	va_list va;
	va_start (va, format);
	vsyslog (LOG_ERR, format, va);
	vfprintf (stderr, format, va);
	fputc ('\n', stderr);
	va_end (va);
}

#if 0
static void
msg(const char *format, ...)
{
	va_list va;
	va_start (va, format);
	vfprintf (stderr, format, va);
	fputc ('\n', stderr);
	va_end (va);
}
#endif

typedef void* (*if_enumerator) (struct ifreq *ifr, void* data);

static void*
enumerate_ifs (if_enumerator func, void* data)
{
	int sockfd = -1;
	struct ifconf ifc;
	struct ifreq *ifr = NULL;
	unsigned char *ptr;
	unsigned char *buf = NULL;
	void *result = NULL;

	ASSERT (func);

	ifc.ifc_len = 32768;
	buf = malloc (ifc.ifc_len);
	ifc.ifc_req = (struct ifreq*)buf;
	if (!buf) {
		emsg ("couldn't allocate buffer to list interfaces: out of memory");
		goto cleanup;
	}

	sockfd = socket (AF_INET, SOCK_DGRAM, 0);
	if (sockfd < 0) {
		emsg ("couldn't create socket to list interfaces: %s", strerror (errno));
		goto cleanup;
	}

	if (ioctl (sockfd, SIOCGIFCONF, &ifc) < 0) {
		emsg ("couldn't list interfaces: %s", strerror (errno));
		goto cleanup;
	}

	#define IFR_SIZE(ifr)      \
		(max ((ifr)->ifr_addr.sa_len, sizeof((ifr)->ifr_addr)) + sizeof((ifr)->ifr_name))

	for (ptr = buf; ptr < buf + ifc.ifc_len; ) {
		ifr = (struct ifreq*)ptr;
		ptr += IFR_SIZE (ifr);

		result = (func) (ifr, data);
		if (result)
			break;
	}

cleanup:
	if (sockfd >= 0)
		close (sockfd);
	if (buf)
		free (buf);
	return result;
}

/* -----------------------------------------------------------------------------
 * MONITORING
 */

#pragma pack(1)

/* Ethernet header */
struct ethhdr {
	#define ETHER_ADDR_LEN	6
	u_char dhost[ETHER_ADDR_LEN]; /* Destination host address */
	u_char shost[ETHER_ADDR_LEN]; /* Source host address */
	u_short type; /* IP? ARP? RARP? etc */
};

/* IP4 header */
struct ip4hdr {
	uint8_t vhl;		/* version << 4 | header length >> 2 */
	uint8_t tos;		/* type of service */
	uint16_t len;		/* total length */
	uint16_t id;		/* identification */
	uint16_t off;		/* fragment offset field */
	#define IP_RF 0x8000		/* reserved fragment flag */
	#define IP_DF 0x4000		/* dont fragment flag */
	#define IP_MF 0x2000		/* more fragments flag */
	#define IP_OFFMASK 0x1fff	/* mask for fragmenting bits */
	uint8_t ttl;		/* time to live */
	uint8_t proto;		/* protocol */
	uint16_t sum;		/* checksum */
	struct in_addr src, dst; /* source and dest address */
};

/* IP6 header */
struct ip6hdr {
	int32_t flow;
	int16_t payload;
	int8_t next;
	int8_t hops;
	struct in6_addr src, dst;
};

#pragma pack()

static void
calculate_ip4 (const struct ip4hdr *hdr, uint32_t octets)
{
	struct sockaddr_in addr;
	struct jaildat *jail;

	ASSERT (hdr);

	/* Try incoming */
	memset (&addr, 0, sizeof (addr));
	addr.sin_family = AF_INET;
	addr.sin_len = sizeof (addr);
	addr.sin_port = 0;
	memcpy (&addr.sin_addr, &hdr->dst, sizeof (addr.sin_addr));

	jail = hsh_get (jaildat_by_address, &addr, addr.sin_len);
	if (jail) {
		jail->in_octets += octets;
		++jail->in_packets;
		return;
	}

	/* Try outgoing */
	memcpy (&addr.sin_addr, &hdr->src, sizeof (addr.sin_addr));

	jail = hsh_get (jaildat_by_address, &addr, addr.sin_len);
	if (jail) {
		jail->out_octets += octets;
		++jail->out_packets;
		return;
	}
}

static void
calculate_ip6 (const struct ip6hdr *hdr, uint32_t octets)
{
	struct sockaddr_in6 addr;
	struct jaildat *jail;

	ASSERT (hdr);

	/* Try incoming */
	memset (&addr, 0, sizeof (addr));
	addr.sin6_family = AF_INET6;
	addr.sin6_len = sizeof (addr);
	addr.sin6_port = 0;
	memcpy (&addr.sin6_addr, &hdr->dst, sizeof (addr.sin6_addr));

	jail = hsh_get (jaildat_by_address, &addr, addr.sin6_len);
	if (jail) {
		jail->in_octets += octets;
		++jail->in_packets;
		return;
	}

	/* Try outgoing */
	memcpy (&addr.sin6_addr, &hdr->src, sizeof (addr.sin6_addr));

	jail = hsh_get (jaildat_by_address, &addr, addr.sin6_len);
	if (jail) {
		jail->out_octets += octets;
		++jail->out_packets;
		return;
	}
}

static void
monitor_packet (u_char *data, const struct pcap_pkthdr *hdr, const u_char *bytes)
{
	struct ethhdr *eth;
	int minlen, octets, type;

	/* Short packet, don't care */
	if (hdr->len < sizeof (struct ethhdr))
		return;

	eth = (struct ethhdr*)bytes;
	bytes += sizeof (struct ethhdr);
	octets = hdr->len - sizeof (struct ethhdr);
	type = ntohs (eth->type);

	/* IPv4 packet? */
	if (type == 0x0800) {
		minlen = (sizeof (struct ethhdr) + sizeof (struct ip4hdr));
		if (hdr->len >= minlen && hdr->caplen >= minlen)
			calculate_ip4 ((const struct ip4hdr*)bytes, octets);

	/* IPv6 packet? */
	} else if (type == 0x86DD) {
		minlen = (sizeof (struct ethhdr) + sizeof (struct ip6hdr));
		if (hdr->len >= minlen && hdr->caplen >= minlen)
			calculate_ip6 ((const struct ip6hdr*)bytes, octets);
	}
}

static void
monitor_io (int fd, void *data)
{
	struct monitor* mon = (struct monitor*)data;
	int n_packets;

	n_packets = pcap_dispatch (mon->handle, -1, monitor_packet, (u_char*)mon);
	if (n_packets < 0)
		emsg ("couldn't capture packets in monitor: %s", pcap_geterr (mon->handle));
}

static void
monitor_free (struct monitor *mon)
{
	ASSERT (mon);

	if (mon->device)
		free (mon->device);
	if (mon->watch)
		fd_deselect (mon->watch);
	if (mon->handle)
		pcap_close (mon->handle);
	if (mon->filter_valid)
		pcap_freecode (&mon->filter);

	TAILQ_REMOVE (&monitors, mon, link);
	free (mon);
}

static struct monitor*
monitor_create (const char *device)
{
	char errbuf[PCAP_ERRBUF_SIZE];
	struct monitor* mon = NULL;
	int success = 0;
	int fd;

	ASSERT (device);
	ASSERT (network_filter);

	mon = calloc (1, sizeof (struct monitor));
	if (!mon) {
		emsg ("couldn't allocate monitor: out of memory");
		goto cleanup;
	}

	TAILQ_INSERT_TAIL(&monitors, mon, link);
	mon->device = strdup (device);

	ASSERT (SNAP_LEN >= sizeof (struct ethhdr) + sizeof (struct ip4hdr));
	ASSERT (SNAP_LEN >= sizeof (struct ethhdr) + sizeof (struct ip6hdr));

	mon->handle = pcap_open_live (mon->device, SNAP_LEN, 1, 100, errbuf);
	if (!mon->handle) {
		emsg ("couldn't open monitor on %s: %s", mon->device, errbuf);
		goto cleanup;
	}

	if (pcap_compile (mon->handle, &mon->filter, network_filter, 1, 0) < 0) {
		emsg ("couldn't compile monitor expression: %s", pcap_geterr (mon->handle));
		goto cleanup;
	}

	mon->filter_valid = 1;
	if (pcap_setfilter (mon->handle, &mon->filter) < 0) {
		emsg ("couldn't setup monitor expression: %s", pcap_geterr (mon->handle));
		goto cleanup;
	}

	if (pcap_setnonblock (mon->handle, 1, errbuf) < 0) {
		emsg ("couldn't set monitor in non-block mode: %s", errbuf);
		goto cleanup;
	}

	fd = pcap_get_selectable_fd (mon->handle);
	if (fd < 0) {
		emsg ("couldn't get selectable monitor: %s", pcap_geterr (mon->handle));
		goto cleanup;
	}

	mon->watch = fd_select (fd, monitor_io, mon, module);
	if (!mon->watch) {
		emsg ("couldn't listen to monitor: %s", strerror (errno));
		goto cleanup;
	}

	success = 1;

cleanup:
	if (!success && mon) {
		monitor_free (mon);
		mon = NULL;
	}

	return mon;
}

static void
monitor_ref (struct monitor *mon)
{
	ASSERT (mon);
	ASSERT (mon->refs >= 0);
	++mon->refs;
}

static void
monitor_unref (struct monitor *mon)
{
	ASSERT (mon);

	--mon->refs;
	ASSERT (mon->refs >= 0);

	if (mon->refs == 0)
		monitor_free (mon);
}

static struct monitor*
monitor_for_device (const char *device)
{
	struct monitor *mon;

	ASSERT (device);

	/* Mark and prepare for sweep below */
	TAILQ_FOREACH (mon, &monitors, link) {
		ASSERT (mon->device);
		if (strcmp (mon->device, device) == 0)
			return mon;
	}

	return monitor_create (device);
}

static void*
monitor_addr_enumerator (struct ifreq *ifr, void *data)
{
	struct sockaddr *addr = (struct sockaddr*)data;
	int match = 0;

	ASSERT (ifr);
	ASSERT (addr);

	switch (ifr->ifr_addr.sa_family) {
	case AF_INET:
		if (addr->sa_family == AF_INET) {
			struct sockaddr_in *sin = (struct sockaddr_in*)addr;
			if (memcmp (&(sin->sin_addr), &(((struct sockaddr_in*)&ifr->ifr_addr)->sin_addr),
			            sizeof (sin->sin_addr)) == 0)
				match = 1;
		}
		break;

	case AF_INET6:
		if (addr->sa_family == AF_INET6) {
			struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)addr;
			if (memcmp (&sin6->sin6_addr, &(((struct sockaddr_in6*)&ifr->ifr_addr)->sin6_addr),
			            sizeof (sin6->sin6_addr)) == 0)
				match = 1;
		}
		break;

	default:
		break;
	};

	if (!match)
		return NULL;

	return monitor_for_device (ifr->ifr_name);
}

static struct monitor*
monitor_for_address (struct sockaddr *addr)
{
	return (struct monitor*)enumerate_ifs (monitor_addr_enumerator, addr);
}

static int
monitor_test_filter (const char *filter)
{
	struct bpf_program bpf;
	int ret;

	ret = pcap_compile_nopcap (SNAP_LEN, DLT_EN10MB, &bpf, (char*)filter, 1, 0);
	if (ret >= 0)
		pcap_freecode (&bpf);

	return ret;
}

/* -----------------------------------------------------------------------------
 * MEASURE JAILS
 */

static void *measure_out_watch = NULL;
static int measure_out_fd = -1;

static void
measure_parse (char *line)
{
	unsigned long long value;
	struct jaildat *jail;
	char *p;

	/* Find the first colon */
	p = strchr (line, ':');
	if (!p) {
		emsg ("invalid output from measure process: %s", line);
		return;
	}

	/* Skip past the colon and spaces */
	*p = 0;
	++p;
	while (isspace (*p))
		++p;

	/* Parse the number */
	value = strtoull (p, &p, 10);

	/* Skip past any more spaces */
	while (isspace (*p))
		++p;

	/* Find a jail with this path */
	jail = hsh_get (jaildat_by_path, p, HSH_KEY_STRING);
	if (!jail)
		return;

	/* Update the various values */
	if (strcmp (line, "jail-space") == 0)
		jail->disk_space = value;
	else if (strcmp (line, "jail-files") == 0)
		jail->disk_files = value;
}

static void
measure_io (int fd, void *unused)
{
	char buffer[2048];
	char *line, *next;
	int r;

	ASSERT (measure_out_fd == fd);

	r = read (fd, buffer, sizeof (buffer) - 1);

	/* Error conditions */
	if (r < 0) {
		if (errno == EAGAIN || errno == EINTR)
			return;
		emsg ("couldn't read from jail measure: %s", strerror (errno));
	}

	/* Close and disconnect */
	if (r <= 0) {
		fd_deselect (measure_out_watch);
		measure_out_watch = NULL;
		close (measure_out_fd);
		measure_out_fd = -1;
		return;
	}

	/* Null terminate the line */
	ASSERT (r < sizeof (buffer));
	buffer[r] = 0;
	line = buffer;

	/* And parse each line received */
	while (line) {
		next = strchr (line, '\n');
		if (next)
			*(next++) = 0;
		while (isspace (*line))
			++line;
		if (*line)
			measure_parse (line);
		line = next;
	}
}

static char**
measure_get_command (void)
{
	struct jaildat *jail;
	char **args;
	int i;

	if (!jaildat_count)
		return NULL;

	args = calloc (jaildat_count + 4, sizeof (char*));
	if (!args) {
		emsg ("out of memory");
		return NULL;
	}

	args[0] = JAIL_MEASURE_PATH;
	args[1] = "-l";
	args[2] = "11"; /* ie: LOG_ERR | LOG_DAEMON */

	i = 3;
	TAILQ_FOREACH (jail, &jaildats, link)
		args[i++] = jail->path;

	return args;
}

static void
measure_all (void *unused)
{
	char **args;
	int pip[2];
	pid_t pid;
	int status;

	if (measure_out_watch) {
		ASSERT (measure_out_fd != -1);
		emsg ("jail measure already in progress, skipping");
		return;
	}

	args = measure_get_command ();
	if (!args)
		return;

	if (pipe (pip) < 0) {
		emsg ("couldn't create pipe to do jail measure: %s", strerror (errno));
		free (args);
		return;
	}

	pid = fork ();
	switch (pid) {
	case -1:
		emsg ("couldn't fork a measure process: %s", strerror (errno));
		free (args);
		close (pip[0]);
		close (pip[1]);
		return;

	case 0:
		/* Detach this new process */
		if (daemon (1, 1) < 0) {
			emsg ("couldn't daemonize the measure process: %s", strerror (errno));
			exit (1);
		}

		/* Dup pipe into stdout for process */
		close (pip[0]);
		dup2 (pip[1], 1);

		/* Run the measure process */
		execv (args[0], args);
		emsg ("couldn't execute the measure process: %s", strerror (errno));
		exit (1);
		break;

	default:
		break;
	};

	/* We no longer need this end */
	waitpid (pid, &status, 0);
	close (pip[1]);
	free (args);

	measure_out_watch = fd_select (pip[0], measure_io, NULL, module);
	if (!measure_out_watch) {
		emsg ("couldn't watch the measure process output: %s", strerror (errno));
		close (pip[0]);
	}

	measure_out_fd = pip[0];
}

static void
measure_start (void *unused)
{
	ASSERT (timer_measure);
	timer_measure = timer_start_repeat (measure_interval, measure_interval,
	                                    measure_all, NULL, module);
	measure_all (NULL);
}

/* -----------------------------------------------------------------------------
 * PROCESS LOOKUPS
 */

static int
process_compar_kp_pid (const void *a, const void *b)
{
	pid_t pida, pidb;

	pida = ((const struct kinfo_proc*)a)->ki_pid;
	pidb = ((const struct kinfo_proc*)b)->ki_pid;

	if (pida == pidb)
		return 0;

	/* Note we're sorting in reverse */
	return (pida < pidb) ? 1 : -1;
}

static int
process_compapre_kp_ppid_pid (const void *a, const void *b)
{
	pid_t pida, pidb;

	pida = ((const struct kinfo_proc*)a)->ki_ppid;
	pidb = ((const struct kinfo_proc*)b)->ki_pid;

	if (pida == pidb)
		return 0;

	/* Note we're sorting in reverse */
	return (pida < pidb) ? 1 : -1;
}

static int
process_compare_kp_ptop_pid (const void *a, const void *b)
{
	pid_t pida, pidb;

	pida = ((const struct kinfo_proc*)a)->ki_pid;
	pidb = ((const struct ptop*)b)->pid;

	if (pida == pidb)
		return 0;

	/* Note we're sorting in reverse */
	return (pida < pidb) ? 1 : -1;
}

static int
process_compare_ptop_pid (const void *a, const void *b)
{
	pid_t pida, pidb;

	pida = ((const struct ptop*)a)->pid;
	pidb = ((const struct ptop*)b)->pid;

	if (pida == pidb)
		return 0;

	/* Note we're sorting in reverse */
	return (pida < pidb) ? 1 : -1;
}

static void
process_refresh_all (void)
{
	struct kinfo_proc *kp, *tkp, *k;
	struct jaildat *jail;
	struct ptop *ptop;
	int nentries, i, jid;
	uint32_t cpu_time;
	void *alloc;
	int pagesize;

	/* Get a process listing */
	kp = kvm_getprocs (kvm_handle, KERN_PROC_PROC, 0, &nentries);
	if (kp == NULL) {
		emsg ("couldn't lookup process list: %s", kvm_geterr (kvm_handle));
		return;
	}

	/* Sort the input we get, in reverse */
	qsort (kp, nentries, sizeof (*kp), process_compar_kp_pid);
	pagesize = getpagesize ();

	/* Mark all processes in the jail for later sweep */
	TAILQ_FOREACH (jail, &jaildats, link) {
		for (i = 0; i < jail->n_ptops; ++i)
			jail->ptops[i].status = PROC_GONE;

		jail->n_processes = 0;
		jail->n_threads = 0;
		jail->cpu_time_total = jail->cpu_time_offset;
	}

	/* Okay now loop and add to each process's jail */
	for (i = 0; i < nentries; i++) {

		jid = kp[i].ki_jid;

		/* No jail? */
		if (jid == 0)
			continue;

		jail = hsh_get (jaildat_by_id, &jid, sizeof (jid));
		if (jail == NULL)
			continue;

		jail->n_processes += 1;
		jail->n_threads += kp[i].ki_numthreads;


		/* Find the top level process within jail to account to */
		tkp = &kp[i];
		for (;;) {
			if (tkp->ki_pid == tkp->ki_ppid)
				break;
			k = bsearch (tkp, kp, nentries, sizeof (*kp), process_compapre_kp_ppid_pid);
			if (k == NULL || k->ki_jid != jid)
				break;
			tkp = k;
		}

		/* Find top process for that pid */
		ptop = bsearch (tkp, jail->ptops, jail->n_ptops,
		                sizeof (struct ptop), process_compare_kp_ptop_pid);
		if (ptop == NULL) {
			alloc = realloc (jail->ptops, (jail->n_ptops + 1) * sizeof (struct ptop));
			if (alloc == NULL) {
				emsg ("out of memory");
				continue;
			}

			jail->ptops = alloc;
			ptop = jail->ptops + jail->n_ptops;
			jail->n_ptops += 1;

			ptop->pid = tkp->ki_pid;
			ptop->cpu_time = 0;
			ptop->status = PROC_NEW;
		}

		/* Account CPU time to this process */
		cpu_time = kp[i].ki_runtime + kp[i].ki_childtime.tv_usec;
		cpu_time /= 10000;
		cpu_time += kp[i].ki_childtime.tv_sec * 100;

		if (ptop->status == PROC_GONE) {
			ptop->cpu_time = 0;
			ptop->status = PROC_EXISTS;
		}

		ptop->cpu_time += cpu_time;
		jail->cpu_time_total += cpu_time;

		/* Sort the array if added */
		qsort (jail->ptops, jail->n_ptops, sizeof (struct ptop), process_compare_ptop_pid);
	}

	TAILQ_FOREACH (jail, &jaildats, link) {

		for (i = 0; i < jail->n_ptops; ++i) {
			if (jail->ptops[i].status != PROC_GONE)
				continue;

			/*
			 * Add time to the cpu_time_offset if it's a 'top level'
			 * process that's going away.
			 */

			jail->cpu_time_offset += jail->ptops[i].cpu_time;
			jail->cpu_time_total += jail->ptops[i].cpu_time;

			/* Remove nonexistant ptop */
			memmove (jail->ptops + i, jail->ptops + i + 1,
			         (jail->n_ptops - i) * sizeof (struct ptop));
			jail->n_ptops -= 1;
			i -= 1;
		}
	}
}

/* -----------------------------------------------------------------------------
 * JAIL LOOKUPS
 */

static void
jail_free (struct jaildat *jail)
{
	int i;

	ASSERT (jail);

	if (jail->jailid)
		hsh_rem (jaildat_by_id, &jail->jailid, sizeof (jail->jailid));

	if (jail->host) {
		hsh_rem (jaildat_by_host, jail->host, HSH_KEY_STRING);
		free (jail->host);
		jail->host = NULL;
	}

	/* Remove all addresses and release monitors */
	for (i = 0; i < jail->n_addrs; ++i) {
		hsh_rem (jaildat_by_address, &(jail->addrs[i].ip),
		         jail->addrs[i].ip.sin_len);
		if (jail->addrs[i].monitor)
			monitor_unref (jail->addrs[i].monitor);
		memset (&jail->addrs[i], 0, sizeof (jail->addrs[i]));
	}

	if (jail->path) {
		hsh_rem (jaildat_by_path, jail->path, HSH_KEY_STRING);
		free (jail->path);
	}
	jail->path = NULL;

	if (jail->index) {
		TAILQ_REMOVE (&jaildats, jail, link);
		jaildat_count--;
	}

	free (jail->ptops);
	jail->ptops = NULL;

	free (jail);
}

static int
jail_update (struct jaildat *jail, int jailid, const char *host,
             const char *path, u_int32_t *ips, u_int n_ips)
{
	struct monitor *mon = NULL;
	struct sockaddr_in addr;
	char *dup;
	int i, top;

	ASSERT (jail);

	/* Do the jail id */
	if (jail->jailid != jailid) {
		hsh_rem (jaildat_by_id, &jail->jailid, sizeof (jail->jailid));
		jail->jailid = jailid;
		hsh_set (jaildat_by_id, &jail->jailid, sizeof (jail->jailid), jail);
	}

	/* Do the host */
	if (!host)
		host = "";

	if (!jail->host || strcmp (jail->host, host) != 0) {
		dup = strdup (host);
		if (!dup)
			return -1;
		if (jail->host) {
			hsh_rem (jaildat_by_host, jail->host, HSH_KEY_STRING);
			free (jail->host);
		}
		jail->host = dup;
		hsh_set (jaildat_by_host, jail->host, HSH_KEY_STRING, jail);
	}

	/* Do the path */
	if (!path)
		path = "";

	if (!jail->path || strcmp (jail->path, path) != 0) {
		dup = strdup (path);
		if (!dup)
			return -1;
		if (jail->path) {
			hsh_rem (jaildat_by_path, jail->path, HSH_KEY_STRING);
			free (jail->path);
		}
		jail->path = dup;
		hsh_set (jaildat_by_path, jail->path, HSH_KEY_STRING, jail);
	}

	/* Do the addresses */
	top = (jail->n_addrs > n_ips ? jail->n_addrs : n_ips);
	for (i = 0; i < top; ++i) {

		/* Setup the address */
		if (i < n_ips) {
			memset (&addr, 0, sizeof (addr));
			addr.sin_family = AF_INET;
			addr.sin_len = sizeof (addr);
			addr.sin_port = 0;
			addr.sin_addr.s_addr = ntohl (ips[i]);

			/* Same? skip. */
			if (i < jail->n_addrs &&
			    memcmp (&jail->addrs[i].ip, &addr, sizeof (addr)) == 0)
				continue;
		}

		/* Remove old address */
		if (i < jail->n_addrs) {
			hsh_rem (jaildat_by_address, &(jail->addrs[i].ip),
			         jail->addrs[i].ip.sin_len);
			memset (&jail->addrs[i].ip, 0, sizeof (jail->addrs[i].ip));
		}

		/* Add new address */
		if (i < n_ips) {
			memcpy (&jail->addrs[i].ip, &addr, sizeof (addr));
			hsh_set (jaildat_by_address, &(jail->addrs[i].ip),
			         jail->addrs[i].ip.sin_len, jail);
		}

		/* Register new monitor */
		mon = NULL;
		if (i < n_ips) {
			mon = monitor_for_address ((struct sockaddr*)&addr);
			if (mon)
				monitor_ref (mon);
		}

		/* Free the old monitor */
		if (i < jail->n_addrs && jail->addrs[i].monitor)
			monitor_unref (jail->addrs[i].monitor);

		/* Assign the monitor */
		jail->addrs[i].monitor = mon;
	}

	jail->n_addrs = n_ips;

	return 0;
}

static struct jaildat*
jail_alloc (int jailid, const char *host, const char *path,
            u_int32_t *ips, u_int n_ips)
{
	struct jaildat *jail;

	jail = (struct jaildat*)calloc (1, sizeof (struct jaildat));
	if (!jail)
		return NULL;

	if (jail_update (jail, jailid, host, path, ips, n_ips) < 0) {
		jail_free (jail);
		return NULL;
	}

	jaildat_count++;
	jail->index = ++jaildat_index;
	INSERT_OBJECT_INT (jail, &jaildats);

	return jail;
}

static void
jail_refresh_v1 (struct xprison_v1 *xp, size_t len)
{
	struct jaildat *jail;
	int i;

	/* Make sure its kosher */
	if (len < sizeof (*xp) || len % sizeof (*xp)) {
		emsg ("not a valid v1 xprison. kernel and userland out of sync?");
		return;
	}

	/*
	 * Allocate new jails, and update old ones. We iterate backwards
	 * so that if there are multiple jails with the same host name
	 * (some possibly zombies), then we'll see the most recent one.
	 */
	for (i = (len / sizeof (*xp)) - 1; i >= 0; --i) {
		jail = hsh_get (jaildat_by_host, xp[i].pr_host, HSH_KEY_STRING);
		if (!jail)
			jail = jail_alloc (xp[i].pr_id, xp[i].pr_host, xp[i].pr_path,
			                   &xp[i].pr_ip, 1);
		else
			jail_update (jail, xp[i].pr_id, xp[i].pr_host, xp[i].pr_path,
			             &xp[i].pr_ip, 1);
		if (jail)
			jail->mark = 0;
	}
}

static void
jail_refresh_v2 (struct xprison_v2 *xp, size_t len)
{
	struct jaildat *jail;
	int i;

	/* Make sure its kosher */
	if (len < sizeof (*xp) || len % sizeof (*xp)) {
		emsg ("not a valid v2 xprison. kernel and userland out of sync?");
		return;
	}

	/*
	 * Allocate new jails, and update old ones. We iterate backwards
	 * so that if there are multiple jails with the same host name
	 * (some possibly zombies), then we'll see the most recent one.
	 */
	for (i = (len / sizeof (*xp)) - 1; i >= 0; --i) {
		jail = hsh_get (jaildat_by_host, xp[i].pr_host, HSH_KEY_STRING);
		if (!jail)
			jail = jail_alloc (xp[i].pr_id, xp[i].pr_host, xp[i].pr_path,
			                   xp[i].pr_ips, xp[i].pr_nips);
		else
			jail_update (jail, xp[i].pr_id, xp[i].pr_host, xp[i].pr_path,
			             xp[i].pr_ips, xp[i].pr_nips);
		if (jail)
			jail->mark = 0;
	}
}

static void
jail_refresh_all (void* unused)
{
        struct xprison *sxp = NULL;
	struct jaildat *jail, *tmp;
        size_t len;
	int i;

	/* Get the length of the list */
        if (sysctlbyname ("security.jail.list", NULL, &len, NULL, 0) == -1) {
		emsg ("couldn't lookup jail list: %s", strerror (errno));
		return;
	}

	/* Retrieve actual data */
	for (i = 0; i < 4; i++) {
		if (len <= 0)
			break;

		sxp = malloc (len);
		if (sxp == NULL) {
			emsg ("out of memory");
			return;
		}

                if (sysctlbyname ("security.jail.list", sxp, &len, NULL, 0) == -1) {
			free (sxp);
			sxp = NULL;
			if (errno == ENOMEM)
				continue;
                }

		if (sxp == NULL) {
			emsg ("couldn't retrieve jail list: %s", strerror (errno));
			return;
		}

                break;
        }

	/* Mark and prepare for sweep below */
	TAILQ_FOREACH (jail, &jaildats, link)
		jail->mark = 1;

	if (len > 0) {
		if (sxp->pr_version == 1)
			jail_refresh_v1 ((struct xprison_v1*)sxp, len);
		else if (sxp->pr_version == 2)
			jail_refresh_v2 ((struct xprison_v2*)sxp, len);
		else
			emsg ("unsupported kernel structure version: %d\n", sxp->pr_version);

		free (sxp);
	}

	/* Sweep any jails that are no longer */
	TAILQ_FOREACH_SAFE (jail, &jaildats, link, tmp) {
		if (jail->mark)
			jail_free (jail);
	}

	process_refresh_all ();

	/* Remove any jails that have no processes (kernel anomally) */
	TAILQ_FOREACH_SAFE (jail, &jaildats, link, tmp) {
		if (!jail->n_processes)
			jail_free (jail);
	}
}

/* -----------------------------------------------------------------------------
 * CALLBACKS/CONFIG
 */

int
op_jailconfig (struct snmp_context *ctx, struct snmp_value *value,
               u_int sub, u_int iidx, enum snmp_op op)
{
	asn_subid_t which = value->var.subs[sub - 1];
	int r = SNMP_ERR_NOERROR;

	if (op == SNMP_OP_GET) {
		switch (which) {
		case LEAF_jailNetworkFilter:
			return string_get (value, network_filter, -1);
		case LEAF_jailRefreshInterval:
			value->v.uint32 = refresh_interval;
			return SNMP_ERR_NOERROR;
		case LEAF_jailMeasureInterval:
			value->v.uint32 = measure_interval;
			return SNMP_ERR_NOERROR;
		default:
			ASSERT (0);
			return -1;
		};
	}

	/* Remainder only at initialization */
	if (community != COMM_INITIALIZE)
		return SNMP_ERR_NOT_WRITEABLE;

	switch (which) {
	case LEAF_jailNetworkFilter:
		switch (op) {
		case SNMP_OP_SET:
			if ((r = string_save (value, ctx, -1, &network_filter)) == SNMP_ERR_NOERROR) {
				if (monitor_test_filter (network_filter) < 0)
					r = SNMP_ERR_GENERR;
			}
			if (r != SNMP_ERR_NOERROR)
				string_rollback (ctx, &network_filter);
			break;
		case SNMP_OP_COMMIT:
			string_commit (ctx);
			break;
		case SNMP_OP_ROLLBACK:
			string_rollback (ctx, &network_filter);
			break;
		default:
			ASSERT (0);
			return SNMP_ERR_GENERR;
		};
		break;

	case LEAF_jailRefreshInterval:
		switch (op) {
		case SNMP_OP_SET:
			ctx->scratch->int1 = refresh_interval;
			refresh_interval = value->v.uint32;
			return SNMP_ERR_NOERROR;
		case SNMP_OP_ROLLBACK:
			refresh_interval = ctx->scratch->int1;
			return SNMP_ERR_NOERROR;
		case SNMP_OP_COMMIT:
			return SNMP_ERR_NOERROR;
		default:
			ASSERT (0);
			return SNMP_ERR_GENERR;
		};
		break;

	case LEAF_jailMeasureInterval:
		switch (op) {
		case SNMP_OP_SET:
			ctx->scratch->int1 = measure_interval;
			measure_interval = value->v.uint32;
			return SNMP_ERR_NOERROR;
		case SNMP_OP_ROLLBACK:
			measure_interval = ctx->scratch->int1;
			return SNMP_ERR_NOERROR;
		case SNMP_OP_COMMIT:
			return SNMP_ERR_NOERROR;
		default:
			ASSERT (0);
			return SNMP_ERR_GENERR;
		};
		break;
    	default:
		ASSERT (0);
		return SNMP_ERR_GENERR;
	};

	return r;
}

int
op_jailentry (struct snmp_context *ctx, struct snmp_value *value,
              u_int sub, u_int iidx, enum snmp_op op)
{
	asn_subid_t which = value->var.subs[sub - 1];
	struct jaildat *jail = NULL;

	switch (op) {
	case SNMP_OP_GETNEXT:
		jail = NEXT_OBJECT_INT (&jaildats, &value->var, sub);
		if (jail == NULL)
			return SNMP_ERR_NOSUCHNAME;
		value->var.len = sub + 1;
		value->var.subs[sub] = jail->index;
		break;

	case SNMP_OP_GET:
		jail = FIND_OBJECT_INT (&jaildats, &value->var, sub);
		if (jail == NULL)
			return SNMP_ERR_NOSUCHNAME;
		break;

	default:
		jail = FIND_OBJECT_INT (&jaildats, &value->var, sub);
		break;
	};

	if (op != SNMP_OP_GET && op != SNMP_OP_GETNEXT)
		return SNMP_ERR_NOT_WRITEABLE;

	switch (which) {
	case LEAF_jailIndex:
		value->v.integer = jail->index;
		return SNMP_ERR_NOERROR;
	case LEAF_jailHost:
		return string_get (value, jail->host, -1);
	case LEAF_jailInOctets:
		value->v.counter64 = jail->in_octets;
		return SNMP_ERR_NOERROR;
	case LEAF_jailInPackets:
		value->v.counter64 = jail->in_packets;
		return SNMP_ERR_NOERROR;
	case LEAF_jailOutOctets:
		value->v.counter64 = jail->out_octets;
		return SNMP_ERR_NOERROR;
	case LEAF_jailOutPackets:
		value->v.counter64 = jail->out_packets;
		return SNMP_ERR_NOERROR;
	case LEAF_jailProcesses:
		value->v.integer = jail->n_processes;
		return SNMP_ERR_NOERROR;
	case LEAF_jailThreads:
		value->v.integer = jail->n_threads;
		return SNMP_ERR_NOERROR;
	case LEAF_jailCpuTime:
		value->v.integer = jail->cpu_time_total;
		return SNMP_ERR_NOERROR;
	case LEAF_jailDiskSpace:
		value->v.counter64 = jail->disk_space;
		return SNMP_ERR_NOERROR;
	case LEAF_jailDiskFiles:
		value->v.counter64 = jail->disk_files;
		return SNMP_ERR_NOERROR;
	default:
		ASSERT (0);
		return SNMP_ERR_NOSUCHNAME;
	};
}

int
op_jail (struct snmp_context *ctx, struct snmp_value *value,
         u_int sub, u_int iidx, enum snmp_op op)
{
	asn_subid_t which = value->var.subs[sub - 1];

	switch (op) {
	case SNMP_OP_GET:
		break;

	case SNMP_OP_SET:
		return SNMP_ERR_NOT_WRITEABLE;

	case SNMP_OP_ROLLBACK:
	case SNMP_OP_COMMIT:
		return SNMP_ERR_NOERROR;

	default:
		ASSERT(0);
		break;
    	};

	switch (which) {
	case LEAF_jailCount:
		value->v.integer = jaildat_count;
		break;

	default:
		ASSERT(0);
		break;
	};

	return SNMP_ERR_NOERROR;
}


/* -----------------------------------------------------------------------------
 * MODULE
 */

/* Called, when the module is to be unloaded after it was successfully loaded */
static int
module_fini (void)
{
	struct jaildat *jail;
	struct monitor *mon;

	if (reg_index)
		or_unregister (reg_index);

	if (kvm_handle)
		kvm_close (kvm_handle);

	if (network_filter)
		free (network_filter);
	network_filter = NULL;

	if (jaildat_by_id)
		hsh_free (jaildat_by_id);
	jaildat_by_id = NULL;

	if (jaildat_by_address)
		hsh_free (jaildat_by_address);
	jaildat_by_address = NULL;

	if (jaildat_by_host)
		hsh_free (jaildat_by_host);
	jaildat_by_host = NULL;

	if (jaildat_by_path)
		hsh_free (jaildat_by_path);
	jaildat_by_path = NULL;

	while ((jail = TAILQ_FIRST(&jaildats)) != NULL)
		jail_free (jail);

	while ((mon = TAILQ_FIRST(&monitors)) != NULL)
		monitor_free (mon);

	if (timer_refresh)
		timer_stop (timer_refresh);
	timer_refresh = NULL;

	if (timer_measure)
		timer_stop (timer_measure);
	timer_measure = NULL;

	return 0;
}

/* the initialisation function */
static int
module_init (struct lmodule *mod, int argc, char *argv[])
{
	int success = 0;
	char errbuf[_POSIX2_LINE_MAX];

	module = mod;

	if (argc != 0) {
		syslog (LOG_ERR, "bad number of arguments for %s", __func__);
		return EINVAL;
	}

	/* Open the kernel interface */
	kvm_handle = kvm_openfiles (_PATH_DEVNULL, _PATH_DEVNULL, _PATH_DEVNULL,
	                            O_RDONLY, errbuf);
	if (kvm_handle == NULL) {
		syslog (LOG_ERR, "couldn't open kvm interface: %s", errbuf);
		return ENOENT;
	}

	/* Setup all our own structures */

	network_filter = strdup ("ip or ip6");
	if (!network_filter)
		goto cleanup;

	jaildat_by_id = hsh_create ();
	if (!jaildat_by_id)
		goto cleanup;

	jaildat_by_host = hsh_create ();
	if (!jaildat_by_host)
		goto cleanup;

	jaildat_by_path = hsh_create ();
	if (!jaildat_by_path)
		goto cleanup;

	jaildat_by_address = hsh_create ();
	if (!jaildat_by_address)
		goto cleanup;

	success = 1;

cleanup:
	if (!success) {
		emsg ("error initializing: out of memory");
		module_fini ();
		return ENOMEM;
	}

	return 0;
}

/* Module is started */
static void
module_start (void)
{
	reg_index = or_register (&oid_jails, "The MIB for jail interface data.", module);

	jail_refresh_all (NULL);

	timer_refresh = timer_start_repeat (refresh_interval, refresh_interval,
	                                    jail_refresh_all, NULL, module);
	if (!timer_refresh)
		emsg ("couldn't setup timer to refresh jails");

	/* Wait for some time before first measure */
	timer_measure = timer_start (measure_interval / 5, measure_start, NULL, module);
	if (!timer_measure)
		emsg ("couldn't setup timer to measure jails");
}

const struct snmp_module config = {
	.comment =      "This module implements SNMP monitoring of jails",
	.init =         module_init,
	.start =        module_start,
	.fini =         module_fini,
	.tree =         jails_ctree,
	.tree_size =    jails_CTREE_SIZE,
};