Index: harvest/components/broker/standard/WWW/BrokerQuery.pl
diff -c harvest/components/broker/standard/WWW/BrokerQuery.pl:2.2 harvest/components/broker/standard/WWW/BrokerQuery.pl:2.3
*** harvest/components/broker/standard/WWW/BrokerQuery.pl:2.2	Sat Apr 19 15:23:39 1997
--- harvest/components/broker/standard/WWW/BrokerQuery.pl	Tue May 13 22:47:20 1997
***************
*** 8,14 ****
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'BrokerQuery.pl,v 2.2 1997/04/19 14:23:39 sxw Exp';
  #######################################################################
  #
  #  BrokerQuery.pl - Customizable WWW Interface to the Harvest Broker
--- 8,14 ----
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'BrokerQuery.pl,v 2.3 1997/05/13 21:47:20 sxw Exp';
  #######################################################################
  #
  #  BrokerQuery.pl - Customizable WWW Interface to the Harvest Broker
***************
*** 162,168 ****
  $maxlines	= $RQ{'maxlineflag'};
  $errors		= $RQ{'errorflag'};
  $broker 	= $RQ{'broker'} || $RQ{'host'};
! $verbflag	= $RQ{'verbflag'} eq "" ? 0 : 1;
  $version	= $RQ{'version'} eq "" ? 0 : 1;
  @atts		= split (/\s+/, $RQ{'attribute'});
  $sort		= $RQ{'sort'};
--- 162,168 ----
  $maxlines	= $RQ{'maxlineflag'};
  $errors		= $RQ{'errorflag'};
  $broker 	= $RQ{'broker'} || $RQ{'host'};
! $verbflag	= $RQ{'verbose'} eq "" ? 0 : 1;
  $version	= $RQ{'version'} eq "" ? 0 : 1;
  @atts		= split (/\s+/, $RQ{'attribute'});
  $sort		= $RQ{'sort'};
Index: harvest/components/broker/standard/WWW/nph-search
diff -c harvest/components/broker/standard/WWW/nph-search:2.3 harvest/components/broker/standard/WWW/nph-search:2.4
*** harvest/components/broker/standard/WWW/nph-search:2.3	Sun Apr 20 18:54:33 1997
--- harvest/components/broker/standard/WWW/nph-search	Tue May 13 22:47:20 1997
***************
*** 8,14 ****
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'nph-search,v 2.3 1997/04/20 17:54:33 sxw Exp';
  #######################################################################
  #
  #  nph-search - Customizable WWW Interface to the Harvest Broker
--- 8,14 ----
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'nph-search,v 2.4 1997/05/13 21:47:20 sxw Exp';
  #######################################################################
  #
  #  nph-search - Customizable WWW Interface to the Harvest Broker
***************
*** 194,200 ****
  $maxlines	= $RQ{'maxlineflag'};
  $errors		= $RQ{'errorflag'};
  $broker 	= $RQ{'broker'} || $RQ{'host'};
! $verbflag	= $RQ{'verbflag'} eq "" ? 0 : 1;
  $version	= $RQ{'version'} eq "" ? 0 : 1;
  @atts		= split (/\s+/, $RQ{'attribute'});
  $sort		= $RQ{'sort'};
--- 194,200 ----
  $maxlines	= $RQ{'maxlineflag'};
  $errors		= $RQ{'errorflag'};
  $broker 	= $RQ{'broker'} || $RQ{'host'};
! $verbflag	= $RQ{'verbose'} eq "" ? 0 : 1;
  $version	= $RQ{'version'} eq "" ? 0 : 1;
  @atts		= split (/\s+/, $RQ{'attribute'});
  $sort		= $RQ{'sort'};
Index: harvest/components/broker/standard/glimpse/Makefile.in
diff -c harvest/components/broker/standard/glimpse/Makefile.in:2.1 harvest/components/broker/standard/glimpse/Makefile.in:2.2
*** harvest/components/broker/standard/glimpse/Makefile.in:2.1	Wed Mar 26 00:30:16 1997
--- harvest/components/broker/standard/glimpse/Makefile.in	Thu May  8 17:03:18 1997
***************
*** 81,90 ****
  		$(INSTALL_PROGRAM) $(PROGAGREP) $(bindir)/$(binprefix)agrep
  		$(INSTALL_PROGRAM) $(PROGSERVER) $(bindir)/$(binprefix)glimpseserver
  		$(INSTALL_PROGRAM) $(PROG) $(bindir)/$(binprefix)glimpse
! 		# $(INSTALL_DATA) $(srcdir)/glimpse.1 $(mandir)/$(manprefix)glimpse.$(manext)
! 		# $(INSTALL_DATA) $(srcdir)/glimpseindex.1 $(mandir)/$(manprefix)glimpseindex.$(manext)
! 		# $(INSTALL_DATA) $(srcdir)/glimpseserver.1 $(mandir)/$(manprefix)glimpseserver.$(manext)
! 		# $(INSTALL_DATA) $(srcdir)/agrep/agrep.1 $(mandir)/$(manprefix)agrep.$(manext)
  
  installdirs:	mkinstalldirs
  		$(srcdir)/mkinstalldirs $(bindir) $(mandir)
--- 81,90 ----
  		$(INSTALL_PROGRAM) $(PROGAGREP) $(bindir)/$(binprefix)agrep
  		$(INSTALL_PROGRAM) $(PROGSERVER) $(bindir)/$(binprefix)glimpseserver
  		$(INSTALL_PROGRAM) $(PROG) $(bindir)/$(binprefix)glimpse
! # 		$(INSTALL_DATA) $(srcdir)/glimpse.1 $(mandir)/$(manprefix)glimpse.$(manext)
! # 		$(INSTALL_DATA) $(srcdir)/glimpseindex.1 $(mandir)/$(manprefix)glimpseindex.$(manext)
! # 		$(INSTALL_DATA) $(srcdir)/glimpseserver.1 $(mandir)/$(manprefix)glimpseserver.$(manext)
! # 		$(INSTALL_DATA) $(srcdir)/agrep/agrep.1 $(mandir)/$(manprefix)agrep.$(manext)
  
  installdirs:	mkinstalldirs
  		$(srcdir)/mkinstalldirs $(bindir) $(mandir)
Index: harvest/src/broker/Glimpse/index.c
diff -c harvest/src/broker/Glimpse/index.c:2.2 harvest/src/broker/Glimpse/index.c:2.4
*** harvest/src/broker/Glimpse/index.c:2.2	Tue Mar 25 18:38:47 1997
--- harvest/src/broker/Glimpse/index.c	Thu May 15 17:38:34 1997
***************
*** 1,4 ****
! static char rcsid[] = "index.c,v 2.2 1997/03/25 18:38:47 sxw Exp";
  /* 
   *  index.c -- Broker indexing/search support using Glimpse
   *
--- 1,4 ----
! static char rcsid[] = "index.c,v 2.4 1997/05/15 16:38:34 sxw Exp";
  /* 
   *  index.c -- Broker indexing/search support using Glimpse
   *
***************
*** 414,419 ****
--- 414,420 ----
  	char *opb = NULL;
  	char **opdata = NULL;
  	char *tmp = NULL;
+ 	char *tmp2 = NULL;
  	char *s = NULL;
  	int opsize = 0;
  	int obcnt = 0;
***************
*** 489,496 ****
  		/* 
  		 *  If there's a : in the output, then Glimpse is 
  		 *  displaying a matched line.
  		 */
! 		if ((tmp = strchr(inb, ':')) != NULL) {
  			if (s = strrchr(tmp, '\n'))
  				*s = '\0';
  			sprintf(opb, "Matched line: %s", ++tmp);
--- 490,504 ----
  		/* 
  		 *  If there's a : in the output, then Glimpse is 
  		 *  displaying a matched line.
+                  *
+ 		 *  URK - glimpse sometimes returns a corrupted matched line
+ 		 *  I don't know quite how we can get around this, but the
+ 		 *  following seems to work. It is, however, a kludgy hack
+ 		 *  which may break spectacularly if the storage manager is
+ 		 *  changed (sxw 14/05/97)
  		 */
! 		if ((tmp2 = strstr(inb, "/OBJ")) !=NULL) {
! 		  if ((tmp = strchr(tmp2, ':')) != NULL) {
  			if (s = strrchr(tmp, '\n'))
  				*s = '\0';
  			sprintf(opb, "Matched line: %s", ++tmp);
***************
*** 497,502 ****
--- 505,511 ----
  			Debug(102, 1, ("GL_user_query: got matched line %d\n", opsize));
  			if (opsize < MAX_OPDATA_SIZE)
  				opdata[opsize++] = xstrdup(opb);
+ 		  }
  		}
  		fd2 = fd1;
  	}
Index: harvest/src/broker/example/brokers/skeleton/admin/broker.conf.in
diff -c harvest/src/broker/example/brokers/skeleton/admin/broker.conf.in:2.0 harvest/src/broker/example/brokers/skeleton/admin/broker.conf.in:2.1
*** harvest/src/broker/example/brokers/skeleton/admin/broker.conf.in:2.0	Sun Mar 16 22:12:41 1997
--- harvest/src/broker/example/brokers/skeleton/admin/broker.conf.in	Sat May 10 00:28:52 1997
***************
*** 209,214 ****
--- 209,215 ----
  #  (i.e., ``-n'' to index numbers) use the 'GlimpseIndex-Flags' tag.
  #
  GlimpseIndex @THIS_BROKER_GLIMPSEINDEX@
+ GlimpseIndex-Flags -n
  
  #
  #  The GlimpseServer tag is the pathname to the glimpseserver program.
Index: harvest/src/gatherer/enumerate/httpenum-depth.c
diff -c harvest/src/gatherer/enumerate/httpenum-depth.c:2.5 harvest/src/gatherer/enumerate/httpenum-depth.c:2.6
*** harvest/src/gatherer/enumerate/httpenum-depth.c:2.5	Fri May  2 22:07:33 1997
--- harvest/src/gatherer/enumerate/httpenum-depth.c	Mon May  5 04:18:26 1997
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-depth.c,v 2.5 1997/05/02 21:07:33 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-depth.c,v 2.6 1997/05/05 03:18:26 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
***************
*** 208,213 ****
--- 208,226 ----
  
      Debug(42, 9, ("mark_retrieved: url='%s', md5='%s'\n", up->url, up->md5));
  
+     k.dptr = xstrdup(up->url);
+     k.dsize = strlen(k.dptr) + 1;
+     d.dptr = xstrdup(up->md5);
+     d.dsize = strlen(d.dptr) + 1;
+  
+     if (!gdbm_exists(urldbf, k) && gdbm_store(urldbf, k, d, GDBM_INSERT))
+         fatal("GDBM URLDB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+     if (!gdbm_exists(md5dbf, d) && gdbm_store(md5dbf, d, k, GDBM_INSERT))
+         fatal("GDBM MD5DB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+  
+     xfree(k.dptr);
+     xfree(d.dptr);
+  
      if (up->redir_from_url != (char *) NULL) {
  	Debug(42, 9, ("mark_retrieved: url='%s', md5='%s'\n",
  		up->redir_from_url, up->md5));
Index: harvest/src/gatherer/enumerate/robots-txt.c
diff -c harvest/src/gatherer/enumerate/robots-txt.c:2.1 harvest/src/gatherer/enumerate/robots-txt.c:2.2
*** harvest/src/gatherer/enumerate/robots-txt.c:2.1	Fri Mar 21 19:21:41 1997
--- harvest/src/gatherer/enumerate/robots-txt.c	Wed May 14 23:41:39 1997
***************
*** 1,4 ****
! static char rcsid[]="robots-txt.c,v 2.1 1997/03/21 19:21:41 sxw Exp";
  /*
   *  DEBUG: section  48, level 1, 5      Gatherer enumeration robots.txt stuff
   *
--- 1,4 ----
! static char rcsid[]="robots-txt.c,v 2.2 1997/05/14 22:41:39 sxw Exp";
  /*
   *  DEBUG: section  48, level 1, 5      Gatherer enumeration robots.txt stuff
   *
***************
*** 224,239 ****
  	if ((t = strtok(buf, ":\t ")) == NULL)
  	    continue;
  	if (!strcasecmp(t, "Disallow")) {
! 	    if (lastline != disallow) {
  	        *QT = Q;
  	        QT  = &Q->next;
  	    }
! 	    W = xmalloc(sizeof(word_list));
! 	    if ((t = strtok(NULL, " \t")))
! 		W->word = xstrdup(t);
! 	    *DAWLT = W;
! 	    DAWLT  = &W->next;
! 	    lastline = disallow;
  	} else if (!strcasecmp(t, "User-Agent")) {
  	    if (lastline != user_agent) {
  		Q = xmalloc(sizeof(record));
--- 224,244 ----
  	if ((t = strtok(buf, ":\t ")) == NULL)
  	    continue;
  	if (!strcasecmp(t, "Disallow")) {
! 	    if (lastline == user_agent) {
  	        *QT = Q;
  	        QT  = &Q->next;
+ 		lastline = disallow;
  	    }
! 	    else if (lastline == disallow) {
! 	        W = xmalloc(sizeof(word_list));
! 	        if ((t = strtok(NULL, " \t")))
! 		    W->word = xstrdup(t);
! 	        *DAWLT = W;
! 	        DAWLT  = &W->next;
! 	    }
! 	    else {
!       	        errorlog("Malformed robots.txt file from %s\n", server);
! 	    }
  	} else if (!strcasecmp(t, "User-Agent")) {
  	    if (lastline != user_agent) {
  		Q = xmalloc(sizeof(record));
Index: harvest/src/gatherer/essence/byname-posix.cf
diff -c harvest/src/gatherer/essence/byname-posix.cf:2.1 harvest/src/gatherer/essence/byname-posix.cf:2.2
*** harvest/src/gatherer/essence/byname-posix.cf:2.1	Fri Mar 21 19:21:45 1997
--- harvest/src/gatherer/essence/byname-posix.cf	Mon May 12 22:47:54 1997
***************
*** 7,13 ****
  #  a ^ first and a $ last.  The types are evaluated in the order specified.  
  #  All regular expressions are CASE SENSITIVE.
  #
! #  byname-posix.cf,v 2.1 1997/03/21 19:21:45 sxw Exp
  #
  
  #
--- 7,13 ----
  #  a ^ first and a $ last.  The types are evaluated in the order specified.  
  #  All regular expressions are CASE SENSITIVE.
  #
! #  byname-posix.cf,v 2.2 1997/05/12 21:47:54 sxw Exp
  #
  
  #
***************
*** 54,60 ****
  GNUInfo			^.*\.info$
  GhostscriptFont		^.*\.gsf$
  GifImage		^.*\.(gif|GIF)$
! HTML			^.*\.(html|htm|HTML|HTM)$
  IAFA			^.*\.iafa$
  Image			^.*\.(pbm|ras|ppm)$
  InputFile		^.*\.in$
--- 54,60 ----
  GNUInfo			^.*\.info$
  GhostscriptFont		^.*\.gsf$
  GifImage		^.*\.(gif|GIF)$
! HTML			^.*\.(html|htm|HTML|HTM|shtml)$
  IAFA			^.*\.iafa$
  Image			^.*\.(pbm|ras|ppm)$
  InputFile		^.*\.in$
