Use SNMP GETNEXT requests for the table queries.

Make table queries more efficient by combining match/value in the same request whenever we've already gotten a match previously.

6 files changed, 407 insertions, 201 deletions

diff --git a/ChangeLog b/ChangeLog
index 1aff33a..0cc8f88 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,6 @@
 0.9 ?????
-    - Support failover between multiple agents
-    - Support table queries
+    - Support failover between multiple agents.
+    - Support table queries by looking up table indexes.
     - Major refactoring of internals. 
     
 0.8  [2007-07-16]
diff --git a/common/snmp-engine.c b/common/snmp-engine.c
index 35fae26..0bfb77d 100644
--- a/common/snmp-engine.c
+++ b/common/snmp-engine.c
@@ -305,10 +305,20 @@ host_cleanup (void)
  * ASYNC REQUEST PROCESSING
  */
 
+#define MAKE_REQUEST_ID(snmp, cb) \
+	((((snmp) & 0xFFFFFF) << 8) | (cb & 0xFF))
+#define REQUEST_ID_SNMP(id) \
+	((id) >> 8)
+#define REQUEST_ID_CB(id) \
+	((id) & 0xFF)
+
 struct request
 {
 	/* The SNMP request identifier */
-	uint id;
+	uint snmp_id;
+
+	/* References, useful since we have callbacks */
+	int refs;
 
 	mstime next_send;         /* Time of the next packet send */
 	mstime last_sent;         /* Time last sent */
@@ -332,7 +342,7 @@ struct request
 static int snmp_retries = 3;
 
 /* The last request id */
-static uint snmp_request_id = 100000;
+static uint snmp_request_id = 1;
 
 /* The SNMP socket we're communicating on */
 static int snmp_socket = -1;
@@ -353,8 +363,8 @@ static void
 request_release (struct request *req)
 {
 	/* It should no longer be referred to any of these places */
-	ASSERT (!hsh_get (snmp_preparing, &req->id, sizeof (req->id)));
-	ASSERT (!hsh_get (snmp_processing, &req->id, sizeof (req->id)));
+	ASSERT (!hsh_get (snmp_preparing, &req->snmp_id, sizeof (req->snmp_id)));
+	ASSERT (!hsh_get (snmp_processing, &req->snmp_id, sizeof (req->snmp_id)));
 
 	snmp_pdu_clear (&req->pdu);
 	free (req);
@@ -394,7 +404,7 @@ request_send (struct request* req, mstime when)
 		if (ret == -1)
 			log_error ("couldn't send snmp packet to: %s", req->host->hostname);
 		else
-			log_debug ("sent request #%d to: %s", req->id, req->host->hostname);
+			log_debug ("sent request #%d to: %s", req->snmp_id, req->host->hostname);
 	}
 }
 
@@ -406,18 +416,30 @@ request_failure (struct request *req, int code)
 
 	ASSERT (req);
 	ASSERT (code != 0);
+	ASSERT (hsh_get (snmp_processing, &req->snmp_id, sizeof (req->snmp_id)) == req);
 
-	log_debug ("failed request #%d to '%s' with code %d", req->id, req->host->hostname, code);
+	log_debug ("failed request #%d to '%s' with code %d", req->snmp_id, req->host->hostname, code);
 
 	/* For each request SNMP value... */
 	for (j = 0; j < req->pdu.nbindings; ++j) {
+
+		if (!req->callbacks[j].func)
+			continue;
+
 		/* ... let callback know */
-		if (req->callbacks[j].func)
-			(req->callbacks[j].func) (req->id, code, NULL, req->callbacks[j].arg);
+		(req->callbacks[j].func) (MAKE_REQUEST_ID (req->snmp_id, j),
+				          code, NULL, req->callbacks[j].arg);
+
+		/*
+		 * Request could have been freed by the callback, by calling the cancel
+		 * function, check and bail if so.
+		 */
+		if (hsh_get (snmp_processing, &req->snmp_id, sizeof (req->snmp_id)) != req)
+			return;
 	}
 
 	/* Remove from the processing list */
-	val = hsh_rem (snmp_processing, &req->id, sizeof (req->id));
+	val = hsh_rem (snmp_processing, &req->snmp_id, sizeof (req->snmp_id));
 	ASSERT (val == req);
 
 	/* And free the request */
@@ -429,24 +451,29 @@ request_get_dispatch (struct request* req, struct snmp_pdu* pdu)
 {
 	struct snmp_value* pvalue;
 	struct snmp_value* rvalue;
-	int i, j, last, processed;
+	int i, j, skipped, processed;
 	void *val;
 
 	ASSERT (req);
 	ASSERT (pdu);
-	ASSERT (req->id == pdu->request_id);
+	ASSERT (req->snmp_id == pdu->request_id);
 	ASSERT (pdu->error_status == SNMP_ERR_NOERROR);
 	ASSERT (req->pdu.type == SNMP_PDU_GET);
+	ASSERT (hsh_get (snmp_processing, &req->snmp_id, sizeof (req->snmp_id)) == req);
 
 	/*
 	 * For SNMP GET requests we check that the values that came back
 	 * were in fact for the same values we requested, and fix any
 	 * ordering issues etc.
 	 */
-	for (j = 0; j < req->pdu.nbindings; ++j) {
+	skipped = 0;
+	for (j = 0; j < SNMP_MAX_BINDINGS; ++j) {
+
+		if (!req->callbacks[j].func)
+			continue;
 
-		processed = 0;
 		rvalue = &(req->pdu.bindings[j]);
+		processed = 0;
 
 		/* ... dig out matching value from response */
 		for (i = 0; i < pdu->nbindings; ++i) {
@@ -455,37 +482,32 @@ request_get_dispatch (struct request* req, struct snmp_pdu* pdu)
 			if (asn_compare_oid (&(rvalue->var), &(pvalue->var)) != 0)
 				continue;
 
-			if (req->callbacks[j].func)
-				(req->callbacks[j].func) (req->id, SNMP_ERR_NOERROR,
-				                          pvalue, req->callbacks[j].arg);
+			(req->callbacks[j].func) (MAKE_REQUEST_ID (req->snmp_id, j),
+			                          SNMP_ERR_NOERROR, pvalue, req->callbacks[j].arg);
 
+			/*
+			 * Request could have been freed by the callback, by calling the cancel
+			 * function, check and bail if so.
+			 */
+			if (hsh_get (snmp_processing, &req->snmp_id, sizeof (req->snmp_id)) != req)
+				return;
+
+			req->callbacks[j].func = NULL;
 			processed = 1;
 			break;
 		}
 
-		/* If this one was processed, remove from request */
-		if (processed) {
-			last = --req->pdu.nbindings;
-
-			ASSERT (last >= 0);
-			if (last) {
-				memcpy (&req->callbacks[j], &req->callbacks[last], sizeof (req->callbacks[j]));
-				memcpy (&req->pdu.bindings[j], &req->pdu.bindings[last], sizeof (req->pdu.bindings[j]));
-			}
-			memset (&req->callbacks[last], 0, sizeof (req->callbacks[last]));
-			memset (&req->pdu.bindings[last], 0, sizeof (req->pdu.bindings[last]));
-
-			/* Process this index again, since we have a new request here */
-			--j;
-		}
+		/* Make note that we didn't find a match for at least one binding */
+		if (!processed && !skipped)
+			skipped = 1;
 	}
 
 	/* All done? then remove request */
-	if (req->pdu.nbindings == 0) {
+	if (!skipped) {
 
-		log_debug ("request #%d is complete", req->id);
+		log_debug ("request #%d is complete", req->snmp_id);
 
-		val = hsh_rem (snmp_processing, &req->id, sizeof (req->id));
+		val = hsh_rem (snmp_processing, &req->snmp_id, sizeof (req->snmp_id));
 		ASSERT (val == req);
 		request_release (req);
 	}
@@ -498,7 +520,7 @@ request_other_dispatch (struct request* req, struct snmp_pdu* pdu)
 
 	ASSERT (req);
 	ASSERT (pdu);
-	ASSERT (req->id == pdu->request_id);
+	ASSERT (req->snmp_id == pdu->request_id);
 	ASSERT (pdu->error_status == SNMP_ERR_NOERROR);
 	ASSERT (req->pdu.type != SNMP_PDU_GET);
 
@@ -520,12 +542,12 @@ request_other_dispatch (struct request* req, struct snmp_pdu* pdu)
 	ASSERT (req->pdu.nbindings == 1);
 
 	if (req->callbacks[0].func)
-		(req->callbacks[0].func) (req->id, SNMP_ERR_NOERROR,
+		(req->callbacks[0].func) (MAKE_REQUEST_ID (req->snmp_id, 0), SNMP_ERR_NOERROR,
 				          &(pdu->bindings[0]), req->callbacks[0].arg);
 
-	log_debug ("request #%d is complete", req->id);
+	log_debug ("request #%d is complete", req->snmp_id);
 
-	val = hsh_rem (snmp_processing, &req->id, sizeof (req->id));
+	val = hsh_rem (snmp_processing, &req->snmp_id, sizeof (req->snmp_id));
 	ASSERT (val == req);
 	request_release (req);
 }
@@ -573,7 +595,7 @@ request_response (int fd, int type, void* arg)
 	id = pdu.request_id;
 	req = hsh_get (snmp_processing, &id, sizeof (id));
 	if(!req) {
-		log_debug ("received extra or delayed packet from: %s", hostname);
+		log_debug ("received extra, cancelled or delayed packet from: %s", hostname);
 		return;
 	}
 
@@ -583,7 +605,7 @@ request_response (int fd, int type, void* arg)
 
 	/* Log any errors */
 	if(pdu.error_status == SNMP_ERR_NOERROR) {
-		log_debug ("response to request #%d from: %s", req->id, hostname);
+		log_debug ("response to request #%d from: %s", req->snmp_id, hostname);
 
 		if (req->pdu.type == SNMP_PDU_GET)
 			request_get_dispatch (req, &pdu);
@@ -593,9 +615,9 @@ request_response (int fd, int type, void* arg)
 	} else {
 		msg = snmp_get_errmsg (pdu.error_status);
 		if(msg)
-			log_debug ("failure for request #%d from: %s: %s", req->id, hostname, msg);
+			log_debug ("failure for request #%d from: %s: %s", req->snmp_id, hostname, msg);
 		else
-			log_debug ("failure for request #%d from: %s: %d", req->id, hostname,
+			log_debug ("failure for request #%d from: %s: %d", req->snmp_id, hostname,
 			           pdu.error_status);
 		request_failure (req, pdu.error_status);
 	}
@@ -641,7 +663,7 @@ request_flush (struct request *req, mstime when)
 
 	ASSERT (req->host->prepared == req);
 
-	val = hsh_rem (snmp_preparing, &req->id, sizeof (req->id));
+	val = hsh_rem (snmp_preparing, &req->snmp_id, sizeof (req->snmp_id));
 	ASSERT (val == req);
 
 	/* Don't let us add more onto this request via the host */
@@ -651,7 +673,7 @@ request_flush (struct request *req, mstime when)
 	/* Mark this packet to be sent now */
 	req->next_send = when;
 
-	if (!hsh_set (snmp_processing, &req->id, sizeof (req->id), req)) {
+	if (!hsh_set (snmp_processing, &req->snmp_id, sizeof (req->snmp_id), req)) {
 		log_errorx ("out of memory, discarding packets");
 		request_release (req);
 	}
@@ -698,7 +720,7 @@ request_prep_instance (struct host *host, mstime interval, mstime timeout, int r
 	/* See if we have one we can piggy back onto */
 	req = host->prepared;
 	if (req) {
-		ASSERT (hsh_get (snmp_preparing, &req->id, sizeof (req->id)));
+		ASSERT (hsh_get (snmp_preparing, &req->snmp_id, sizeof (req->snmp_id)));
 
 		/* We have one we can piggy back another request onto */
 		if (req->pdu.nbindings < SNMP_MAX_BINDINGS && req->pdu.type == reqtype)
@@ -719,10 +741,14 @@ request_prep_instance (struct host *host, mstime interval, mstime timeout, int r
 	}
 
 	/* Assign the unique id */
-	req->id = snmp_request_id++;
+	req->snmp_id = snmp_request_id++;
+
+	/* Roll around after a decent amount of ids */
+	if (snmp_request_id >= 0xFFFFFF)
+		snmp_request_id = 1;
 
 	/* Mark it down as something we want to prepare */
-	if (!hsh_set (snmp_preparing, &req->id, sizeof (req->id), req)) {
+	if (!hsh_set (snmp_preparing, &req->snmp_id, sizeof (req->snmp_id), req)) {
 		log_error ("out of memory");
 		free (req);
 		return NULL;
@@ -730,7 +756,7 @@ request_prep_instance (struct host *host, mstime interval, mstime timeout, int r
 
         /* Setup the packet */
         strlcpy (req->pdu.community, host->community, sizeof (req->pdu.community));
-        req->pdu.request_id = req->id;
+        req->pdu.request_id = req->snmp_id;
         req->pdu.version = host->version;
         req->pdu.type = reqtype;
         req->pdu.error_status = 0;
@@ -749,7 +775,7 @@ request_prep_instance (struct host *host, mstime interval, mstime timeout, int r
 	ASSERT (host->prepared == NULL);
         host->prepared = req;
 
-        log_debug ("preparing request #%d for: %s@%s", req->id,
+        log_debug ("preparing request #%d for: %s@%s", req->snmp_id,
                    req->host->community, req->host->hostname);
 
 	return req;
@@ -762,6 +788,9 @@ snmp_engine_request (const char *hostname, const char *community, int version,
 {
 	struct host *host;
 	struct request *req;
+	int callback_id;
+
+	ASSERT (func);
 
 	/* Lookup host for request */
 	host = host_instance (hostname, community, version, interval);
@@ -776,10 +805,11 @@ snmp_engine_request (const char *hostname, const char *community, int version,
 	ASSERT (req->pdu.nbindings < SNMP_MAX_BINDINGS);
 
 	/* Add the oid to that request */
-        req->pdu.bindings[req->pdu.nbindings].var = *oid;
-        req->pdu.bindings[req->pdu.nbindings].syntax = SNMP_SYNTAX_NULL;
-        req->callbacks[req->pdu.nbindings].func = func;
-        req->callbacks[req->pdu.nbindings].arg = arg;
+	callback_id = req->pdu.nbindings;
+        req->pdu.bindings[callback_id].var = *oid;
+        req->pdu.bindings[callback_id].syntax = SNMP_SYNTAX_NULL;
+        req->callbacks[callback_id].func = func;
+        req->callbacks[callback_id].arg = arg;
         req->pdu.nbindings++;
 
         /* All other than GET, only get one binding */
@@ -788,39 +818,62 @@ snmp_engine_request (const char *hostname, const char *community, int version,
         	request_flush (req, server_get_time ());
         }
 
-        if (!snmp_flush_pending) {
+        /* Otherwise flush on the idle callback */
+        else if (!snmp_flush_pending) {
         	server_oneshot (0, request_flush_cb, NULL);
         	snmp_flush_pending = 1;
         }
 
-        return req->id;
+        return MAKE_REQUEST_ID (req->snmp_id, callback_id);
 }
 
 void
-snmp_engine_cancel (int reqid)
+snmp_engine_cancel (int id)
 {
 	struct request *req;
+	int snmp_id, callback_id, i;
+	const char *during;
+
+	ASSERT (id);
+
+	snmp_id = REQUEST_ID_SNMP (id);
+	callback_id = REQUEST_ID_CB (id);
+
+	ASSERT (snmp_id > 0 && snmp_id < 0xFFFFFF);
+	ASSERT (callback_id >= 0 && callback_id < SNMP_MAX_BINDINGS);
 
 	/* Is it being processed? */
-	req = hsh_rem (snmp_processing, &reqid, sizeof (reqid));
+	req = hsh_rem (snmp_processing, &snmp_id, sizeof (snmp_id));
 	if (req) {
-		log_debug ("cancelling request #%d during processing", reqid);
-		request_release (req);
-		return;
-	}
+		during = "processing";
 
 	/* Is it being prepared? */
-	req = hsh_rem (snmp_preparing, &reqid, sizeof (reqid));
-	if (req) {
-
-		/* Remove it from the host in question */
-		ASSERT (req->host->prepared == req);
-		req->host->prepared = NULL;
+	} else {
+		req = hsh_rem (snmp_preparing, &snmp_id, sizeof (snmp_id));
+		if (req) {
+			during = "prep";
+			ASSERT (req->host->prepared == req);
+		}
+	}
 
-		log_debug ("cancelling request #%d during prep", reqid);
-		request_release (req);
+	if (!req)
 		return;
+
+	/* Remove this callback from the request */
+	req->callbacks[callback_id].func = NULL;
+	req->callbacks[callback_id].arg = NULL;
+
+	/* See if any callbacks exist in the request */
+	for (i = 0; i < SNMP_MAX_BINDINGS; ++i) {
+		if (req->callbacks[i].func)
+			return;
 	}
+
+	/* If not, free the request */
+	log_debug ("cancelling request #%d during %s", snmp_id, during);
+	if (req->host->prepared == req)
+		req->host->prepared = NULL;
+	request_release (req);
 }
 
 void
diff --git a/configure.in b/configure.in
index ea8d458..77b72c1 100644
--- a/configure.in
+++ b/configure.in
@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
-AC_INIT(rrdbot, 0.8, stef@memberwebs.com)
-AM_INIT_AUTOMAKE(rrdbot, 0.8)
+AC_INIT(rrdbot, 0.8.93, stef@memberwebs.com)
+AM_INIT_AUTOMAKE(rrdbot, 0.8.93)
 
 LDFLAGS="$LDFLAGS -L/usr/local/lib"
 CFLAGS="$CFLAGS -I/usr/local/include"
diff --git a/daemon/config.c b/daemon/config.c
index 70075ee..55ee597 100644
--- a/daemon/config.c
+++ b/daemon/config.c
@@ -227,8 +227,9 @@ parse_query (rb_item *item, char *query, config_ctx *ctx)
 	           asn_oid2str (&item->query_oid));
 
 	item->query_match = value;
-	item->query_last = 0;
-	item->query_value = 0;
+	memset (&item->query_last, 0, sizeof (item->query_last));
+	item->query_matched = 0;
+	item->query_searched = 0;
 }
 
 static rb_item*
diff --git a/daemon/poll-engine.c b/daemon/poll-engine.c
index db6074a..119c8b1 100644
--- a/daemon/poll-engine.c
+++ b/daemon/poll-engine.c
@@ -56,15 +56,18 @@
  */
 
 static void
-complete_request (rb_item *item, int code)
+complete_requests (rb_item *item, int code)
 {
 	int host;
 
 	ASSERT (item);
 
-	if (item->request)
-		snmp_engine_cancel (item->request);
-	item->request = 0;
+	if (item->field_request)
+		snmp_engine_cancel (item->field_request);
+	item->field_request = 0;
+	if (item->query_request)
+		snmp_engine_cancel (item->query_request);
+	item->query_request = 0;
 
 	/* If we have multiple host names then try the next host */
 	if (code != SNMP_ERR_NOERROR) {
@@ -77,16 +80,16 @@ complete_request (rb_item *item, int code)
 }
 
 static void
-cancel_request (rb_item *item, const char *reason)
+cancel_requests (rb_item *item, const char *reason)
 {
 	ASSERT (item);
 	ASSERT (reason);
-	ASSERT (item->request);
+	ASSERT (item->field_request || item->query_request);
 
         log_debug ("value for field '%s': %s", item->field, reason);
         item->vtype = VALUE_UNSET;
 
-        complete_request (item, -1);
+        complete_requests (item, -1);
 }
 
 static void
@@ -98,26 +101,36 @@ force_poll (rb_poller *poll, mstime when, const char *reason)
 	ASSERT (poll);
 	ASSERT (reason);
 
-	/* Now see if the entire request is done */
+	/* Now see if the all the requests are done */
 	for (item = poll->items; item; item = item->next) {
-		if (item->request) {
-			cancel_request (item, reason);
+		if (item->field_request || item->query_request) {
+			cancel_requests (item, reason);
 			forced = 1;
 		}
-		ASSERT (!item->request);
+		ASSERT (!item->field_request);
+		ASSERT (!item->query_request);
 	}
 
-	if (forced) {
-
-		/*
-		 * We note the failure has having taken place halfway between
-		 * the request and the current time.
-		 */
-		poll->last_polled = poll->last_request + ((when - poll->last_request) / 2);
+	if (!forced && !poll->polling)
+		return;
 
-		/* And send off our collection of values */
-		rb_rrd_update (poll);
+	/* Mark any non-matched queries as unset */
+	for (item = poll->items; item; item = item->next) {
+		if (item->has_query && !item->query_matched)
+			item->vtype = VALUE_UNSET;
 	}
+
+	/*
+	 * We note the failure has having taken place halfway between
+	 * the request and the current time.
+	 */
+	poll->last_polled = poll->last_request + ((when - poll->last_request) / 2);
+
+	/* And send off our collection of values */
+	rb_rrd_update (poll);
+
+	/* This polling cycle is no longer active */
+	poll->polling = 0;
 }
 
 static void
@@ -126,18 +139,28 @@ finish_poll (rb_poller *poll, mstime when)
 	rb_item *item;
 
 	ASSERT (poll);
+	ASSERT (poll->polling);
 
-	/* Now see if the entire request is done */
+	/* See if the all the requests are done */
 	for (item = poll->items; item; item = item->next) {
-		if (item->request)
+		if (item->field_request || item->query_request)
 			return;
 	}
 
+	/* Mark any non-matched queries as unset */
+	for (item = poll->items; item; item = item->next) {
+		if (item->has_query && !item->query_matched)
+			item->vtype = VALUE_UNSET;
+	}
+
 	/* Update the book-keeping */
 	poll->last_polled = when;
 
 	/* And send off our collection of values */
 	rb_rrd_update (poll);
+
+	/* This polling cycle is no longer active */
+	poll->polling = 0;
 }
 
 static void
@@ -146,10 +169,10 @@ field_response (int request, int code, struct snmp_value *value, void *arg)
 	rb_item *item = arg;
 	const char *msg = NULL;
 
-	ASSERT (item->request == request);
+	ASSERT (item->field_request == request);
 
 	/* Mark this item as done */
-	item->request = 0;
+	item->field_request = 0;
 
 	/* Errors result in us writing U */
 	if (code != SNMP_ERR_NOERROR) {
@@ -204,7 +227,7 @@ field_response (int request, int code, struct snmp_value *value, void *arg)
                 	           item->field);
 	}
 
-	complete_request (item, code);
+	complete_requests (item, code);
 
 	/* If the entire poll is done, then complete it */
 	finish_poll (item->poller, server_get_time ());
@@ -216,42 +239,182 @@ field_request (rb_item *item)
 	int req;
 
 	ASSERT (item);
-	ASSERT (!item->request);
+	ASSERT (!item->field_request);
 
         item->vtype = VALUE_UNSET;
 
 	req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
 	                           item->version, item->poller->interval, item->poller->timeout,
 	                           SNMP_PDU_GET, &item->field_oid, field_response, item);
-	item->request = req;
+	item->field_request = req;
 }
 
 /* Forward declaration */
-static void query_request (rb_item *item, int first);
+static void query_search_request (rb_item *item);
 
 static void
-query_response (int request, int code, struct snmp_value *value, void *arg)
+query_value_request (rb_item *item, asn_subid_t subid)
 {
-	rb_item *item = arg;
 	struct asn_oid oid;
-	int matched, req, found;
+	int req;
 
-	ASSERT (request == item->request);
+	ASSERT (item);
 	ASSERT (item->has_query);
+	ASSERT (!item->query_request);
+	ASSERT (!item->field_request);
+
+        item->vtype = VALUE_UNSET;
+
+	/* OID for the actual value */
+	oid = item->field_oid;
+	ASSERT (oid.len < ASN_MAXOIDLEN);
+	oid.subs[oid.len] = subid;
+	++oid.len;
+
+	log_debug ("query requesting value for table index: %u", subid);
+
+	req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
+	                           item->version, item->poller->interval, item->poller->timeout,
+	                           SNMP_PDU_GET, &oid, field_response, item);
+
+        /* Value retrieval is active */
+	item->field_request = req;
+}
+
+static void
+query_next_response (int request, int code, struct snmp_value *value, void *arg)
+{
+	rb_item *item = arg;
+	asn_subid_t subid;
+	int matched;
 
 	/*
-	 * This was the number we last appended.
+	 * Called when we get the next OID in a table
 	 */
-	ASSERT (item->query_value >= 0);
-	item->request = 0;
+
+	ASSERT (request == item->query_request);
+	ASSERT (!item->field_request);
+	item->query_request = 0;
+
+	if (code == SNMP_ERR_NOERROR) {
+		ASSERT (value);
+
+		/* Convert these result codes into 'not found' */
+		switch (value->syntax) {
+		case SNMP_SYNTAX_NOSUCHOBJECT:
+		case SNMP_SYNTAX_NOSUCHINSTANCE:
+		case SNMP_SYNTAX_ENDOFMIBVIEW:
+			code = SNMP_ERR_NOSUCHNAME;
+			break;
+
+		/*
+		 * Make sure that we haven't gone past the end. For it to
+		 * be in the table it must be exactly one longer (the table index)
+		 * and otherwise identical.
+		 */
+		default:
+			if (item->query_oid.len + 1 != value->var.len ||
+			    !asn_is_suboid (&item->query_oid, &value->var))
+				code = SNMP_ERR_NOSUCHNAME;
+			break;
+		};
+	}
+
+	if (code == SNMP_ERR_NOSUCHNAME)
+		log_debug ("query couldn't find table index that matches: %s",
+		           item->query_match ? item->query_match : "[null]");
+
+
+	/* Problems communicating with the server, or not found */
+	if (code != SNMP_ERR_NOERROR) {
+		memset (&item->query_last, 0, sizeof (item->query_last));
+		complete_requests (item, code);
+		return;
+	}
+
+	/* Save away the last OID we've seen */
+	item->query_last = value->var;
+	item->query_searched = 1;
+
+	ASSERT (value);
+
+	/* Match the query valu ereceived */
+	if (item->query_match)
+		matched = snmp_engine_match (value, item->query_match);
+
+	/* When query match is null, anything matches */
+	else
+		matched = 1;
+
+	item->query_matched = matched;
+	item->vtype = VALUE_UNSET;
+
+	if (matched) {
+		/* Do a query for the field value with this sub id */
+		subid = value->var.subs[value->var.len - 1];
+		query_value_request (item, subid);
+	} else {
+		/* Look for the next table index */
+		query_search_request (item);
+	}
+}
+
+static void
+query_search_request (rb_item *item)
+{
+	struct asn_oid *oid;
+	int req;
+
+	ASSERT (item);
+	ASSERT (item->has_query);
+	ASSERT (!item->query_request);
+	ASSERT (!item->field_request);
+
+	item->query_matched = 0;
+	item->vtype = VALUE_UNSET;
+
+	/* Start with the OID without any table index */
+	if (!item->query_searched) {
+		oid = &item->query_oid;
+		memset (&item->query_last, 0, sizeof (item->query_last));
+		log_debug ("query looking for first table index");
+
+	/* Go for the next one in the search */
+	} else {
+		ASSERT (item->query_last.len > 0);
+		oid = &item->query_last;
+		log_debug ("query looking for next table index");
+	}
+
+	req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
+	                           item->version, item->poller->interval, item->poller->timeout,
+	                           SNMP_PDU_GETNEXT, oid, query_next_response, item);
+
+	item->query_request = req;
+}
+
+static void
+query_match_response (int request, int code, struct snmp_value *value, void *arg)
+{
+	rb_item *item = arg;
+	int matched;
+
+	/*
+	 * Callback when SNMP request in query_request() completes.
+	 *
+	 * We receive a value back from the server when querying the table match OID,
+	 * whenever we queried it directly (without the search).
+	 */
+
+	ASSERT (request == item->query_request);
+	item->query_request = 0;
 
 	/* Problems communicating with the server? */
 	if (code != SNMP_ERR_NOERROR && code != SNMP_ERR_NOSUCHNAME) {
-		complete_request (item, code);
+		complete_requests (item, code);
 		return;
 	}
 
-	found = 0;
 	matched = 0;
 
 	if (code == SNMP_ERR_NOERROR) {
@@ -262,8 +425,6 @@ query_response (int request, int code, struct snmp_value *value, void *arg)
 		case SNMP_SYNTAX_NOSUCHOBJECT:
 		case SNMP_SYNTAX_NOSUCHINSTANCE:
 		case SNMP_SYNTAX_ENDOFMIBVIEW:
-			found = 0;
-			matched = 0;
 			break;
 
 		/* See if we have a match */
@@ -274,116 +435,102 @@ query_response (int request, int code, struct snmp_value *value, void *arg)
 			/* When query match is null, anything matches */
 			else
 				matched = 1;
-
-			found = 1;
 			break;
 		};
 	}
 
-	/*
-	 * When we had found this before, but then can no longer find it, we
-	 * start search again from the base.
-	 */
-	if (!matched && item->query_last != 0) {
-		log_debug ("last table index did not match, starting from zero");
-		item->query_last = 0;
-		query_request (item, 1);
-
-	/*
-	 * When we find no value at zero, then we skip ahead and see if
-	 * perhaps its a one based table
-	 */
-	} else if (!found && item->query_value == 0) {
-		log_debug ("no zero index in table, trying index one");
-		item->query_last = 0;
-		query_request (item, 0);
-
-	/*
-	 * Any other time we don't find a value, its game over for us,
-	 * we didn't find a match and are out of values.
-	 */
-	} else if (!found) {
-		item->query_last = 0;
-		log_warnx ("couldn't find match for query value: %s",
-		           item->query_match ? item->query_match : "");
-		complete_request (item, SNMP_ERR_NOSUCHNAME);
-
+	item->query_matched = matched;
+	if (matched)
+		return;
 
-	/*
-	 * Found a value but didn't match, so try next one.
-	 */
-	} else if (!matched) {
-		log_debug ("table index %d did not match, trying next", item->query_value);
-		item->query_last = 0;
-		query_request (item, 0);
+	log_debug ("query previous index did not match: %s",
+	           item->query_match ? item->query_match : "[null]");
 
 	/*
-	 * When we have a match send off a new request, built from the original
-	 * oid and the last numeric part of the query oid.
+	 * When it doesn't match cancel any pending value request, and
+	 * start a search for a match.
 	 */
-	} else {
-
-		log_debug ("table index %d matched query value: %s",
-		           item->query_value, item->query_match ? item->query_match : "");
-
-		/* Build up the OID */
-		oid = item->field_oid;
-		ASSERT (oid.len < ASN_MAXOIDLEN);
-		oid.subs[oid.len] = item->query_value;
-		++oid.len;
-
-		item->query_last = item->query_value;
-	        item->vtype = VALUE_UNSET;
-
-		req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
-		                           item->version, item->poller->interval, item->poller->timeout,
-		                           SNMP_PDU_GET, &oid, field_response, item);
-
-		item->request = req;
-	}
+	if (item->field_request)
+		snmp_engine_cancel (item->field_request);
+	item->field_request = 0;
+	query_search_request (item);
 }
 
 static void
-query_request (rb_item *item, int first)
+query_pair_request (rb_item *item, asn_subid_t subid)
 {
 	struct asn_oid oid;
 	int req;
 
 	ASSERT (item);
-	ASSERT (!item->request);
 	ASSERT (item->has_query);
+	ASSERT (!item->query_request);
+	ASSERT (!item->field_request);
+
+	log_debug ("query requesting match and value pair for index: %u", subid);
 
         item->vtype = VALUE_UNSET;
+        item->query_matched = 0;
 
-	/*
-	 * Build up an appropriate oid.
-	 *
-	 * We first try any oid that worked last time, and see if
-	 * it still has the same value, to avoid doing the brute
-	 * force search each time needlessly.
-	 */
+	/* OID for the value to match */
+	oid = item->query_oid;
+	ASSERT (oid.len < ASN_MAXOIDLEN);
+	oid.subs[oid.len] = subid;
+	++oid.len;
 
-	/* The first time the request has been called */
-	if (first)
-		item->query_value = item->query_last;
+	req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
+	                           item->version, item->poller->interval, item->poller->timeout,
+	                           SNMP_PDU_GET, &oid, query_match_response, item);
 
-	/* Try the next one in turn */
-	else
-		item->query_value = item->query_value + 1;
+	/* Query is active */
+	item->query_request = req;
 
-	/* Build up the OID */
-	oid = item->query_oid;
+	/* OID for the actual value */
+	oid = item->field_oid;
 	ASSERT (oid.len < ASN_MAXOIDLEN);
-	oid.subs[oid.len] = item->query_value;
+	oid.subs[oid.len] = subid;
 	++oid.len;
 
-	/* Make the request */
 	req = snmp_engine_request (item->hostnames[item->hostindex], item->community,
 	                           item->version, item->poller->interval, item->poller->timeout,
-	                           SNMP_PDU_GET, &oid, query_response, item);
+	                           SNMP_PDU_GET, &oid, field_response, item);
 
-        /* Mark item as active by this request */
-	item->request = req;
+        /* Value retrieval is active */
+	item->field_request = req;
+}
+
+static void
+query_request (rb_item *item)
+{
+	ASSERT (item);
+	ASSERT (!item->query_request);
+	ASSERT (!item->field_request);
+
+	item->query_searched = 0;
+	item->query_matched = 0;
+        item->vtype = VALUE_UNSET;
+
+	if (item->query_last.len) {
+
+	        /*
+	         * If we've done this query before, then we know the last matching table
+	         * index. We build a two part request that gets the match value for the
+	         * table index it was last seen on, and the actual value that we want.
+	         *
+	         * Doing this in one request is more efficient, then we check if the
+	         * match value matches the query in the response.
+	         */
+		query_pair_request (item, item->query_last.subs[item->query_last.len - 1]);
+
+	} else {
+
+		/*
+		 * We don't have a last matching table index, so start the search.
+		 * For indexes. We'll then query each of those indexes with the two
+		 * part request, as above.
+		 */
+		query_search_request (item);
+	}
 }
 
 static int
@@ -400,6 +547,8 @@ poller_timer (mstime when, void *arg)
 
 	/* Mark this poller as starting requests now */
 	poll->last_request = when;
+	ASSERT (!poll->polling);
+	poll->polling = 1;
 
 	/*
 	 * Send off the next query. This needs to be done after
@@ -407,7 +556,7 @@ poller_timer (mstime when, void *arg)
 	 */
 	for (item = poll->items; item; item = item->next) {
 		if (item->has_query)
-			query_request (item, 1);
+			query_request (item);
 		else
 			field_request (item);
 	}
diff --git a/daemon/rrdbotd.h b/daemon/rrdbotd.h
index f907de2..c370bb6 100644
--- a/daemon/rrdbotd.h
+++ b/daemon/rrdbotd.h
@@ -72,6 +72,7 @@ typedef struct _rb_item
 
     /* The oid that we are querying */
     struct asn_oid field_oid;
+    int field_request;
 
     /* Host names, with alternate hosts */
     #define MAX_HOSTNAMES 16
@@ -83,8 +84,10 @@ typedef struct _rb_item
     int has_query;
     struct asn_oid query_oid;
     const char* query_match;
-    asn_subid_t query_last;
-    int query_value;
+    int query_matched;
+    int query_searched;
+    struct asn_oid query_last;
+    int query_request;
 
     /* The last value / current request */
     union
@@ -98,9 +101,6 @@ typedef struct _rb_item
     #define VALUE_FLOAT 2
     int vtype;
 
-    /* A request in progress */
-    int request;
-
     /* Pointers to related */
     struct _rb_poller* poller;
 
@@ -123,6 +123,9 @@ typedef struct _rb_poller
     /* The things to poll. rb_poller owns this list */
     rb_item* items;
 
+    /* Polling is active */
+    int polling;
+
     /* Book keeping */
     mstime last_request;
     mstime last_polled;