* Add counter for number of processes in jail

 * Add configurable refresh interval for jail data
 * Don't list jails without any processes.
 * Fix crasher when a jail quits.
 * Report jailCount correctly after jails quit.

6 files changed, 178 insertions, 31 deletions

diff --git a/ChangeLog b/ChangeLog
index b2a0957..e3e9b18 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,2 +1,7 @@
+0.2
+ - Add tracking of number of processes.
+ - Don't include jails that have no processes (FreeBSD has these
+   when a socket is hung in an open state).
+
 0.1
  - Initial Release
diff --git a/configure.in b/configure.in
index ba1602c..27b24a1 100644
--- a/configure.in
+++ b/configure.in
@@ -1,6 +1,6 @@
 # Process this file with autoconf to produce a configure script.
-AC_INIT(bsnmp-jails, 0.1, stef@memberwebs.com)
-AM_INIT_AUTOMAKE(bsnmp-jails, 0.1)
+AC_INIT(bsnmp-jails, 0.2, stef@memberwebs.com)
+AM_INIT_AUTOMAKE(bsnmp-jails, 0.2)
 
 AC_CONFIG_SRCDIR([module/bsnmp-jails.c])
 AM_CONFIG_HEADER([config.h])
@@ -55,6 +55,8 @@ AC_CHECK_HEADERS([pcap.h], ,
 # Check for libraries
 AC_SEARCH_LIBS(pcap_open_live, pcap, ,
 	[ echo "Must have a pcap library available"; exit 2 ] )
+AC_CHECK_LIB([kvm], [kvm_open], ,
+	[ echo "ERROR: Must have FreeBSD 5.1 or higher"; exit 1])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
diff --git a/doc/JAILS-MIB.txt b/doc/JAILS-MIB.txt
index 268f76b..f5fa26c 100644
--- a/doc/JAILS-MIB.txt
+++ b/doc/JAILS-MIB.txt
@@ -115,4 +115,11 @@ jailOutPackets OBJECT-TYPE
     DESCRIPTION "Number of packets sent by this jail."
     ::= { jailEntry 13 }
 
+jailProcesses OBJECT-TYPE
+    SYNTAX Integer32
+    MAX-ACCESS read-only
+    STATUS current
+    DESCRIPTION "Number of processes running in this jail."
+    ::= { jailEntry 20 }
+
 END
diff --git a/doc/bsnmp-jails.8 b/doc/bsnmp-jails.8
index d3a73a6..0cd5e07 100644
--- a/doc/bsnmp-jails.8
+++ b/doc/bsnmp-jails.8
@@ -74,9 +74,14 @@ The number of packets received by the jail over the network.
 The number of octets sent by the jail over the network.
 .It Ar jails.jailTable.jailEntry.jailOutPackets.X
 The number of packets sent by the jail over the network.
+.It Ar jails.jailTable.jailEntry.jailProcesses.X
+The number of processes running in the jail.
 .It Ar jails.jailNetworkFilter
 A tcpdump style filter for the network traffic. Only matched traffic
 is counted in the jail statistics.
+.It Ar jails.jailRefreshRate
+The interval (in SNMP time ticks) between refreshing the jail 
+process statistics and checking which jails are running.
 .El
 .Sh OPTIONS
 To activate the 
diff --git a/module/bsnmp-jails.c b/module/bsnmp-jails.c
index 8fbefb2..9e88d5f 100644
--- a/module/bsnmp-jails.c
+++ b/module/bsnmp-jails.c
@@ -42,22 +42,27 @@
 
 #include <sys/jail.h>
 #include <sys/limits.h>
+#include <sys/proc.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/select.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
+#include <sys/user.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
 #include <arpa/inet.h>
 
-#include <syslog.h>
-#include <unistd.h>
-#include <stdarg.h>
 #include <ctype.h>
 #include <fcntl.h>
+#include <limits.h>
+#include <kvm.h>
+#include <paths.h>
+#include <syslog.h>
+#include <stdarg.h>
+#include <unistd.h>
 
 #include <bsnmp/snmpmod.h>
 #include <pcap.h>
@@ -101,6 +106,7 @@ struct jaildat {
 	int                     mark;
 
 	/* Configuration */
+	int                     jailid;
 	char			*host;
 	char                    *path;
 	struct sockaddr_in	addr;
@@ -113,6 +119,7 @@ struct jaildat {
 	uint64_t		in_packets;
 	uint64_t		out_octets;
 	uint64_t 		out_packets;
+	uint32_t		n_processes;
 };
 
 TAILQ_HEAD(jaildat_list, jaildat);
@@ -129,12 +136,20 @@ static hsh_t *jaildat_by_host = NULL;
 /* Hash of jail structures by address */
 static hsh_t *jaildat_by_address = NULL;
 
+/* Hash of jail structures by id */
+static hsh_t *jaildat_by_id = NULL;
+
 /* Timer for refreshing the jaildat info */
 static void *timer_refresh = NULL;
 
 /* The monitor network filter */
 static u_char *network_filter = NULL;
 
+/* The KVM interface handle */
+static kvm_t *kvm_handle = NULL;
+
+/* Refresh interval in SNMP ticks */
+static int refresh_interval = 300;
 
 /* -----------------------------------------------------------------------------
  * HELPERS
@@ -245,7 +260,7 @@ struct ip6hdr {
 #pragma pack()
 
 static void
-process_ip4 (const struct ip4hdr *hdr, uint32_t octets)
+calculate_ip4 (const struct ip4hdr *hdr, uint32_t octets)
 {
 	struct sockaddr_in addr;
 	struct jaildat *jail;
@@ -278,7 +293,7 @@ process_ip4 (const struct ip4hdr *hdr, uint32_t octets)
 }
 
 static void
-process_ip6 (const struct ip6hdr *hdr, uint32_t octets)
+calculate_ip6 (const struct ip6hdr *hdr, uint32_t octets)
 {
 	struct sockaddr_in6 addr;
 	struct jaildat *jail;
@@ -330,7 +345,7 @@ monitor_packet (u_char *data, const struct pcap_pkthdr *hdr, const u_char *bytes
 		minlen = (sizeof (struct ethhdr) + sizeof (struct ip4hdr));
 		if (hdr->len >= minlen) {
 			ASSERT (hdr->caplen >= minlen);
-			process_ip4 ((const struct ip4hdr*)bytes, octets);
+			calculate_ip4 ((const struct ip4hdr*)bytes, octets);
 		}
 
 	/* IPv6 packet? */
@@ -338,7 +353,7 @@ monitor_packet (u_char *data, const struct pcap_pkthdr *hdr, const u_char *bytes
 		minlen = (sizeof (struct ethhdr) + sizeof (struct ip6hdr));
 		if (hdr->len >= minlen) {
 			ASSERT (hdr->caplen >= minlen);
-			process_ip6 ((const struct ip6hdr*)bytes, octets);
+			calculate_ip6 ((const struct ip6hdr*)bytes, octets);
 		}
 	}
 }
@@ -532,6 +547,41 @@ monitor_test_filter (const char *filter)
 }
 
 /* -----------------------------------------------------------------------------
+ * PROCESS LOOKUPS
+ */
+
+static void
+process_refresh_all (void)
+{
+	struct jaildat *jail;
+	struct kinfo_proc *kp;
+	int nentries, i, id;
+
+	/* Get a process listing */
+	kp = kvm_getprocs (kvm_handle, KERN_PROC_PROC, 0, &nentries);
+	if (kp == NULL) {
+		emsg ("couldn't lookup process list: %s", kvm_geterr (kvm_handle));
+		return;
+	}
+
+	/* Clear process counts for each jail */
+	TAILQ_FOREACH (jail, &jaildats, link)
+		jail->n_processes = 0;
+
+	/* Okay now loop and add to each process's jail */
+	for (i = 0; i < nentries; i++) {
+
+		id = kp[i].ki_jid;
+		if (id == 0)
+			continue;
+
+		jail = hsh_get (jaildat_by_id, &id, sizeof (id));
+		if (jail)
+			jail->n_processes++;
+	}
+}
+
+/* -----------------------------------------------------------------------------
  * JAIL LOOKUPS
  */
 
@@ -540,9 +590,16 @@ jail_free (struct jaildat *jail)
 {
 	ASSERT (jail);
 
-	if (jail->host)
+	if (jail->jailid)
+		hsh_rem (jaildat_by_id, &jail->jailid, sizeof (jail->jailid));
+
+	if (jail->host) {
+		hsh_rem (jaildat_by_host, jail->host, HSH_KEY_STRING);
 		free (jail->host);
-	jail->host = NULL;
+		jail->host = NULL;
+	}
+
+	hsh_rem (jaildat_by_address, &(jail->addr), jail->addr.sin_len);
 
 	if (jail->path)
 		free (jail->path);
@@ -552,14 +609,16 @@ jail_free (struct jaildat *jail)
 		monitor_unref (jail->monitor);
 	jail->monitor = NULL;
 
-	if (jail->index)
+	if (jail->index) {
 		TAILQ_REMOVE (&jaildats, jail, link);
+		jaildat_count--;
+	}
 
 	free (jail);
 }
 
 static int
-jail_update (struct jaildat *jail, const char *host,
+jail_update (struct jaildat *jail, int jailid, const char *host,
              const char *path, struct in_addr *addr)
 {
 	struct monitor *mon;
@@ -567,6 +626,12 @@ jail_update (struct jaildat *jail, const char *host,
 
 	ASSERT (jail);
 
+	if (jail->jailid != jailid) {
+		hsh_rem (jaildat_by_id, &jail->jailid, sizeof (jail->jailid));
+		jail->jailid = jailid;
+		hsh_set (jaildat_by_id, &jail->jailid, sizeof (jail->jailid), jail);
+	}
+
 	if (!host)
 		host = "";
 
@@ -615,7 +680,7 @@ jail_update (struct jaildat *jail, const char *host,
 }
 
 static struct jaildat*
-jail_alloc (const char *host, const char *path, struct in_addr *addr)
+jail_alloc (int jailid, const char *host, const char *path, struct in_addr *addr)
 {
 	struct jaildat *jail;
 
@@ -623,7 +688,7 @@ jail_alloc (const char *host, const char *path, struct in_addr *addr)
 	if (!jail)
 		return NULL;
 
-	if (jail_update (jail, host, path, addr) < 0) {
+	if (jail_update (jail, jailid, host, path, addr) < 0) {
 		jail_free (jail);
 		return NULL;
 	}
@@ -690,9 +755,9 @@ jail_refresh_all (void* unused)
 		in.s_addr = ntohl (xp->pr_ip);
 		jail = hsh_get (jaildat_by_host, xp->pr_host, HSH_KEY_STRING);
 		if (!jail)
-			jail = jail_alloc (xp->pr_host, xp->pr_path, &in);
+			jail = jail_alloc (xp->pr_id, xp->pr_host, xp->pr_path, &in);
 		else
-			jail_update (jail, xp->pr_host, xp->pr_path, &in);
+			jail_update (jail, xp->pr_id, xp->pr_host, xp->pr_path, &in);
 
 		jail->mark = 0;
 		xp++;
@@ -705,6 +770,14 @@ jail_refresh_all (void* unused)
 	}
 
 	free (sxp);
+
+	process_refresh_all ();
+
+	/* Remove any jails that have no processes (kernel anomally) */
+	TAILQ_FOREACH_SAFE (jail, &jaildats, link, tmp) {
+		if (!jail->n_processes)
+			jail_free (jail);
+	}
 }
 
 /* -----------------------------------------------------------------------------
@@ -718,16 +791,25 @@ op_jailconfig (struct snmp_context *ctx, struct snmp_value *value,
 	asn_subid_t which = value->var.subs[sub - 1];
 	int r = SNMP_ERR_NOERROR;
 
-	switch (which) {
-	case LEAF_jailNetworkFilter:
-
-		if (op == SNMP_OP_GET)
+	if (op == SNMP_OP_GET) {
+		switch (which) {
+		case LEAF_jailNetworkFilter:
 			return string_get (value, network_filter, -1);
+		case LEAF_jailRefreshInterval:
+			value->v.uint32 = refresh_interval;
+			return SNMP_ERR_NOERROR;
+		default:
+			ASSERT (0);
+			return -1;
+		};
+	}
 
-		/* Remainder only at initialization */
-		if (community != COMM_INITIALIZE)
-			return SNMP_ERR_NOT_WRITEABLE;
+	/* Remainder only at initialization */
+	if (community != COMM_INITIALIZE)
+		return SNMP_ERR_NOT_WRITEABLE;
 
+	switch (which) {
+	case LEAF_jailNetworkFilter:
 		switch (op) {
 		case SNMP_OP_SET:
 			if ((r = string_save (value, ctx, -1, &network_filter)) == SNMP_ERR_NOERROR) {
@@ -744,18 +826,34 @@ op_jailconfig (struct snmp_context *ctx, struct snmp_value *value,
 			string_rollback (ctx, &network_filter);
 			break;
 		default:
-			ASSERT(0);
-			break;
+			ASSERT (0);
+			return SNMP_ERR_GENERR;
 		};
+		break;
 
-		return r;
+	case LEAF_jailRefreshInterval:
+		switch (op) {
+		case SNMP_OP_SET:
+			ctx->scratch->int1 = refresh_interval;
+			refresh_interval = value->v.uint32;
+			return SNMP_ERR_NOERROR;
+		case SNMP_OP_ROLLBACK:
+			refresh_interval = ctx->scratch->int1;
+			return SNMP_ERR_NOERROR;
+		case SNMP_OP_COMMIT:
+			return SNMP_ERR_NOERROR;
+		default:
+			ASSERT (0);
+			return SNMP_ERR_GENERR;
+		};
+		break;
 
     	default:
-		break;
+		ASSERT (0);
+		return SNMP_ERR_GENERR;
 	};
 
-    ASSERT(0);
-    return -1;
+	return r;
 }
 
 int
@@ -806,6 +904,9 @@ op_jailentry (struct snmp_context *ctx, struct snmp_value *value,
 	case LEAF_jailOutPackets:
 		value->v.uint32 = jail->out_packets;
 		return SNMP_ERR_NOERROR;
+	case LEAF_jailProcesses:
+		value->v.integer = jail->n_processes;
+		return SNMP_ERR_NOERROR;
 	default:
 		ASSERT (0);
 		return SNMP_ERR_NOSUCHNAME;
@@ -862,10 +963,17 @@ module_fini (void)
 	if (reg_index)
 		or_unregister (reg_index);
 
+	if (kvm_handle)
+		kvm_close (kvm_handle);
+
 	if (network_filter)
 		free (network_filter);
 	network_filter = NULL;
 
+	if (jaildat_by_id)
+		hsh_free (jaildat_by_id);
+	jaildat_by_id = NULL;
+
 	if (jaildat_by_address)
 		hsh_free (jaildat_by_address);
 	jaildat_by_address = NULL;
@@ -892,6 +1000,7 @@ static int
 module_init (struct lmodule *mod, int argc, char *argv[])
 {
 	int success = 0;
+	char errbuf[_POSIX2_LINE_MAX];
 
 	module = mod;
 
@@ -900,10 +1009,24 @@ module_init (struct lmodule *mod, int argc, char *argv[])
 		return EINVAL;
 	}
 
+	/* Open the kernel interface */
+	kvm_handle = kvm_openfiles (_PATH_DEVNULL, _PATH_DEVNULL, _PATH_DEVNULL,
+	                            O_RDONLY, errbuf);
+	if (kvm_handle == NULL) {
+		syslog (LOG_ERR, "couldn't open kvm interface: %s", errbuf);
+		return ENOENT;
+	}
+
+	/* Setup all our own structures */
+
 	network_filter = strdup ("ip or ip6");
 	if (!network_filter)
 		goto cleanup;
 
+	jaildat_by_id = hsh_create ();
+	if (!jaildat_by_id)
+		goto cleanup;
+
 	jaildat_by_host = hsh_create ();
 	if (!jaildat_by_host)
 		goto cleanup;
@@ -932,7 +1055,8 @@ module_start (void)
 
 	jail_refresh_all (NULL);
 
-	timer_refresh = timer_start_repeat (500, 500, jail_refresh_all, NULL, module);
+	timer_refresh = timer_start_repeat (refresh_interval, refresh_interval,
+	                                    jail_refresh_all, NULL, module);
 	if (!timer_refresh)
 		emsg ("couldn't setup timer to refresh jails");
 }
diff --git a/module/jails-tree.def b/module/jails-tree.def
index fc70afd..6d5a79a 100644
--- a/module/jails-tree.def
+++ b/module/jails-tree.def
@@ -46,15 +46,19 @@
               (1 jailEntry : INTEGER op_jailentry
                 (0 jailIndex INTEGER GET)
                 (1 jailHost OCTETSTRING GET)
+
                 (10 jailInOctets COUNTER64 GET)
                 (11 jailInPackets COUNTER64 GET)
                 (12 jailOutOctets COUNTER64 GET)
                 (13 jailOutPackets COUNTER64 GET)
+
+                (20 jailProcesses INTEGER GET)
               )
             )
 
             # Valid only during configuration 
             (100 jailNetworkFilter OCTETSTRING op_jailconfig GET SET)
+            (101 jailRefreshInterval TIMETICKS op_jailconfig GET SET)
           )
         )
       )