Index: harvest/ChangeLog
diff -c harvest/ChangeLog:2.43 harvest/ChangeLog:2.53
*** harvest/ChangeLog:2.43	Wed Feb  4 10:45:23 1998
--- harvest/ChangeLog	Tue Jun  2 18:23:49 1998
***************
*** 1,4 ****
--- 1,35 ----
  Changes in current version
+         - Change the content of the "description" attribute to be the first
+ 	  255 characters of the objects "body" or "partial-text" attribute
+ 	  (Peter Valkenburg)
+         - SGML summariser fixes -
+ 		* to allow mapping of Dublin Core directives to attributes
+ 		* to autmatically prefix meta generated elements with a 
+ 		  custom prefix
+ 		* to 'fix' meta generated attributes to only contain
+ 		  legal SOIF characters
+ 		(Hrvoje Stipetic)
+ 	- Added support for printing total number of items found when in
+ 	  perpage mode to nph-search (Hermann Straus)
+         - Altered broker behaviour so that objects with no TTL value are
+           never expired (Simon Wilkinson)
+         - Fixed bug in gather which would result in incorrect HELLO messages
+           being sent to servers (Dave Beckett)
+ 	- Fixed broker bug where duplicate copies of a URL could remain in
+       	  the broker in certain circumstances (Hrvoje Stipetic)
+ 	- Added reporting of number of objects added to broker output
+ 	  (Hrvoje Stipetic)
+ 	- Added broker version information to broker greeting message
+ 	  (Hrvoje Stipetic)
+ 	- Fixed bug where admin password could be written to broker log
+ 	  (Hrvoje Stipetic)
+ 	- Removed unnecessary print's in the nph-search script
+ 	- Fixed parsing of broker configuration files so that whitespace is
+ 	  correctly handled in arguments (Peter J. Scott & Hrvoje Stipetic)
+ 	- Added -V option to the broker to return the version number
+ 	  (Hrvoje Stipetic)
+ 	- Fixed definition of random() in stor_man.c for better
+ 	  portability (Otis Gospodnetic)
  	- Improved result ranking in nph-search for occurence of search
  	  string in URL and titles (Bruce R. Lewis)
  	- Fix to LocalMapping code to remove problem where FD 0 (stdin)
Index: harvest/components/broker/standard/WWW/BrokerAdmin.pl.in
diff -c harvest/components/broker/standard/WWW/BrokerAdmin.pl.in:2.2 harvest/components/broker/standard/WWW/BrokerAdmin.pl.in:2.3
*** harvest/components/broker/standard/WWW/BrokerAdmin.pl.in:2.2	Fri Sep 19 11:11:02 1997
--- harvest/components/broker/standard/WWW/BrokerAdmin.pl.in	Fri Feb 20 11:08:13 1998
***************
*** 1,6 ****
  #!@PERL@
  
! $rcsid = 'BrokerAdmin.pl.in,v 2.2 1997/09/19 15:11:02 sxw Exp';
  #############################################################################
  #
  #  BrokerAdmin.pl
--- 1,6 ----
  #!@PERL@
  
! $rcsid = 'BrokerAdmin.pl.in,v 2.3 1998/02/20 16:08:13 sxw Exp';
  #############################################################################
  #
  #  BrokerAdmin.pl
***************
*** 36,41 ****
--- 36,42 ----
  #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  # 
  
+ $prefix="@prefix@";
  $exec_prefix="@exec_prefix@";
  unshift(@INC, "@libdir@");
  require 'socket.ph';
Index: harvest/components/broker/standard/WWW/nph-search.in
diff -c harvest/components/broker/standard/WWW/nph-search.in:2.9 harvest/components/broker/standard/WWW/nph-search.in:2.11
*** harvest/components/broker/standard/WWW/nph-search.in:2.9	Wed Feb  4 10:45:24 1998
--- harvest/components/broker/standard/WWW/nph-search.in	Tue Jun  2 18:00:59 1998
***************
*** 8,14 ****
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'nph-search.in,v 2.9 1998/02/04 15:45:24 sxw Exp';
  #######################################################################
  #
  #  nph-search - Customizable WWW Interface to the Harvest Broker
--- 8,14 ----
  #  $BQLOG = "/usr/local/etc/httpd/logs/BrokerQuery.log";
  $BQLOG = "/dev/null";
  
! $rcsid = 'nph-search.in,v 2.11 1998/06/02 22:00:59 sxw Exp';
  #######################################################################
  #
  #  nph-search - Customizable WWW Interface to the Harvest Broker
***************
*** 508,517 ****
              }
              close ($S)  || &fatal ("socket: $!\n");
  
- 	    print "About to see whether to sort : '$sort'\n";
- 
              @OBJ = sort bynml @OBJ if ($sort eq 'by-NML');
! 	    print "Sorting ...\n" if ($sort eq 'by-NML');
  
  	    &rank_objects(@OBJ,@RANK) if ($sort eq 'by-rank');
  
--- 508,516 ----
              }
              close ($S)  || &fatal ("socket: $!\n");
  
              @OBJ = sort bynml @OBJ if ($sort eq 'by-NML');
! 
! 	    $totnumber= (scalar (@OBJ)) -2;
  
  	    &rank_objects(@OBJ,@RANK) if ($sort eq 'by-rank');
  
***************
*** 1007,1013 ****
      local ($objnum) = 0;
      local (%ratings);
      local (@object_index);
-     print "<pre>\n" if $verbflag;
      foreach $object (@objects) {
  	$rank = 1;
  	split(/\n/,$object);
--- 1006,1011 ----
***************
*** 1371,1376 ****
--- 1369,1375 ----
  		print TEMP &expand ($CFG{'ResultHeader'});
  		print TEMP @first_page;
          	print TEMP &expand ($CFG{'ResultSetEnd'});
+ 		print TEMP qq{<BR>Total: $totnumber} if ($more && defined $sort);
  		print TEMP "<p>\n<center>";
  		print TEMP qq{[<a href="/$tmp_www/$tmp_prefix-2.html">Next</a>]};
  	        print TEMP qq{[<a href="$hp_url">New Search</a>]} 
***************
*** 1383,1388 ****
--- 1382,1388 ----
  	# finish off current page
          print &expand ($CFG{'ResultSetEnd'}) if $more;
  	$previous_page = $current_page - 1; $next_page = $current_page + 1;
+ 	print qq{<BR>Total: $totnumber} if ($more && defined $sort);
  	print "<p>\n<center>";
  	print qq{[<a href="/$tmp_www/$tmp_prefix-$previous_page.html">Previous</a>]}
  		if $previous_page > 0;
Index: harvest/components/gatherer/standard/SGML/SGML.sum
diff -c harvest/components/gatherer/standard/SGML/SGML.sum:2.7 harvest/components/gatherer/standard/SGML/SGML.sum:2.8
*** harvest/components/gatherer/standard/SGML/SGML.sum:2.7	Thu Jan 29 12:29:13 1998
--- harvest/components/gatherer/standard/SGML/SGML.sum	Tue Jun  2 18:20:09 1998
***************
*** 6,12 ****
  #
  #  Usage: SGML.sum [-d file.decl] [-t file.tbl] DOCTYPE [file]
  #
! #  SGML.sum,v 2.7 1998/01/29 17:29:13 sxw Exp
  #
  #############################################################################
  #
--- 6,12 ----
  #
  #  Usage: SGML.sum [-d file.decl] [-t file.tbl] DOCTYPE [file]
  #
! #  SGML.sum,v 2.8 1998/06/02 22:20:09 sxw Exp
  #
  #############################################################################
  #
***************
*** 134,139 ****
--- 134,147 ----
  $decl		= "$sgmls_lib/$doctype/$doctype.decl";
  $table		= "$sgmls_lib/$doctype/$doctype.sum.tbl";	# default
  
+ # Set this to get prefixed META NAMEs (and HTTP-EQUIVs). (HS)
+ #$meta_prefix = 'meta-';
+ $meta_prefix = '';
+ 
+ # Set this to 1 to change all non-alphanumeric characters in SOIF attribute
+ # names to '_'.  The only exception is '-' that is not changed. (HS)
+ $fix_attrib_names = 0;
+ 
  # search for the .sum.tbl file; allows a per Gatherer customizations
  #
  $LIBPATH = $ENV{'SUMMARIZER_LIBPATH'};
***************
*** 385,405 ****
  	local ($tagvar, %VARS) = @_;
  	local (@atts) = ();
  	local ($att);
  
  	print STDERR "tagvar=$tagvar\n" if ($debug);
! 	if (defined ($SOIF{$tagvar})) {
! 		foreach $att (split (/,/, $SOIF{$tagvar})) {
  			if ($att =~ /\$/) {		# $NAME, $HTTP-EQUIV
  				$att =~ s/\$//;
  				if (defined $VARS{$att}) {
  					$att = $VARS{$att};
  					$att =~ tr/A-Z/a-z/;
  					push (@atts, $att);
  				}
  			} else {
  				push (@atts, $att);
  			}
  		}
  	}
  	@atts;
  }
--- 393,441 ----
  	local ($tagvar, %VARS) = @_;
  	local (@atts) = ();
  	local ($att);
+ 	local ($t);
+ 	local (@s);
+ 	local ($s);
+ 	local ($a);
+ 	local ($v);
+ 	local ($av);
  
  	print STDERR "tagvar=$tagvar\n" if ($debug);
! 
! 	# Support for HTML.sum table entry like this one:
! 	#  <TAG:ATTRX,ATTRY=VALUE>	soifattr1,soifattr2
! 	# Primary goal is to support Dublin Core. (HS)
! 	tagloop: foreach $t (@soifkeys) {
! 	    @s = split (',', $t);   # s[0] is tag:var, s[1..n] are att=val
! 	    $s = shift @s;
! 	    if ($s eq $tagvar) {
! 		foreach $av (@s) {
! 			($a,$v) = split (/=/, $av);
! 			print STDERR "A=$a\tV=$v\tVARS{$a}=$VARS{$a}\n"
! 				if ($debug);
! 			next tagloop unless ($VARS{$a} eq $v);
! 		}
! 		foreach $att (split (/,/, $SOIF{$t})) {
  			if ($att =~ /\$/) {		# $NAME, $HTTP-EQUIV
  				$att =~ s/\$//;
  				if (defined $VARS{$att}) {
  					$att = $VARS{$att};
  					$att =~ tr/A-Z/a-z/;
+ 					# Fix SOIF attribute names
+ 					$att =~ s/[^\w-]/_/g
+ 						if ($fix_attrib_names);
+ 					# Add custom prefix for META
+ 					# generated SOIF attributes
+ 					$att = $meta_prefix . $att
+ 						if ($tagvar =~ /^META/);
  					push (@atts, $att);
  				}
  			} else {
  				push (@atts, $att);
  			}
  		}
+ 		last;
+ 	    }
  	}
  	@atts;
  }
Index: harvest/src/broker/Makefile.in
diff -c harvest/src/broker/Makefile.in:2.6 harvest/src/broker/Makefile.in:2.8
*** harvest/src/broker/Makefile.in:2.6	Wed Nov 26 09:14:45 1997
--- harvest/src/broker/Makefile.in	Tue Jun  2 18:42:47 1998
***************
*** 1,7 ****
  #
  #  Makefile for the Harvest Broker
  #
! #  Makefile.in,v 2.6 1997/11/26 14:14:45 sxw Exp
  #
  
  srcdir = @srcdir@
--- 1,7 ----
  #
  #  Makefile for the Harvest Broker
  #
! #  Makefile.in,v 2.8 1998/06/02 22:42:47 sxw Exp
  #
  
  srcdir = @srcdir@
***************
*** 144,148 ****
  	cd $(srcdir)/example; find brokers -depth -print | cpio -pdvu $(prefix)/
  
  brkclient:	brkclient.o
! 	$(CC) -o $@ $@.o $(LDFLAGS) $(XTRA_LIBS)
  
--- 144,148 ----
  	cd $(srcdir)/example; find brokers -depth -print | cpio -pdvu $(prefix)/
  
  brkclient:	brkclient.o
! 	$(CC) -o $@ $@.o $(LDFLAGS) $(LIBS)
  
Index: harvest/src/broker/broker.h
diff -c harvest/src/broker/broker.h:2.3 harvest/src/broker/broker.h:2.4
*** harvest/src/broker/broker.h:2.3	Fri Nov 21 12:42:52 1997
--- harvest/src/broker/broker.h	Thu Mar  5 14:30:09 1998
***************
*** 4,10 ****
   *  DEBUG: none
   *  AUTHOR: Harvest derived
   *
!  *  broker.h,v 2.3 1997/11/21 17:42:52 sxw Exp
   *
   *  Harvest Indexer http://www.tardis.ed.ac.uk/harvest/
   *  ---------------------------------------------------
--- 4,10 ----
   *  DEBUG: none
   *  AUTHOR: Harvest derived
   *
!  *  broker.h,v 2.4 1998/03/05 19:30:09 sxw Exp
   *
   *  Harvest Indexer http://www.tardis.ed.ac.uk/harvest/
   *  ---------------------------------------------------
***************
*** 525,530 ****
--- 525,531 ----
  #define BULK_ERR  "400 - Bulk Object Transfer Error.\n"
  
  #define QMVERSION "200 - Harvest Broker Interface Version 0.3.\n"
+ #define BVERSION  "200 - Broker Version Harvest/"HARVEST_VERSION".\n"
  #define PIPECHK   "200 - Testing the socket to the client.\n"
  
  /* ============ INDEXER ============== */
Index: harvest/src/broker/collector.c
diff -c harvest/src/broker/collector.c:2.1 harvest/src/broker/collector.c:2.3
*** harvest/src/broker/collector.c:2.1	Fri Mar 21 12:19:53 1997
--- harvest/src/broker/collector.c	Tue Jun  2 10:11:49 1998
***************
*** 1,4 ****
! static char rcsid[] = "collector.c,v 2.1 1997/03/21 17:19:53 sxw Exp";
  /*
   *  collector.c -- Utility procs for add/delete/refresh objects in the Broker.
   *
--- 1,4 ----
! static char rcsid[] = "collector.c,v 2.3 1998/06/02 14:11:49 sxw Exp";
  /*
   *  collector.c -- Utility procs for add/delete/refresh objects in the Broker.
   *
***************
*** 118,124 ****
  extern char *obj_desc;
  extern int obj_desc_s;
  
! int up_nobjs, del_nobjs, ref_nobjs, recv_nobjs, ign_nobjs;
  time_t max_update_time = 0;
  
  GathererID *COL_gid = NULL;
--- 118,124 ----
  extern char *obj_desc;
  extern int obj_desc_s;
  
! int new_nobjs, up_nobjs, del_nobjs, ref_nobjs, recv_nobjs, ign_nobjs;
  time_t max_update_time = 0;
  
  GathererID *COL_gid = NULL;
***************
*** 191,199 ****
  	if (entry->lmt < 1)
  		entry->lmt = 0;
  	if (entry->refresh_rate < 1)
! 		entry->refresh_rate = (time_t) WEEK;	/* These also hard-   */
! 	if (entry->ttl < 1)	/* wired in gatherer/ */
! 		entry->ttl = (time_t) MONTH;	/* include/oid.h      */
  
  	return SUCCESS;
  }
--- 191,197 ----
  	if (entry->lmt < 1)
  		entry->lmt = 0;
  	if (entry->refresh_rate < 1)
! 		entry->refresh_rate = (time_t) WEEK;
  
  	return SUCCESS;
  }
***************
*** 205,232 ****
       reg_t *entry;
  {
  	reg_t *tmp;
  
  	/* 
! 	 *  We want to see if the new object already matches an object
  	 *  in the current Registry.  If it does, then if the new object's
  	 *  Update-Time is older than or the same as the Registry object's 
  	 *  Update-Time, then we ignore the new object.  Otherwise, we need 
! 	 *  to replace the Registry object with the new object.  We do this 
! 	 *  by deleting the Registry object, then adding the new object
  	 *  to the Registry.  The RG_Cleaner() will run periodically
  	 *  to compress the Registry.
  	 *
  	 *  If the new object is not in the Registry, then it's new
  	 *  so we add it to the Registry.
  	 */
! 	if ((tmp = RG_Object_Search_Entry(entry)) != NULL) {
  		if (tmp->update_time >= entry->update_time) {
  			(void) SM_Destroy_Obj(entry->FD);
  			RG_Free_Entry(entry);
  			ign_nobjs++;
  			return SUCCESS;
  		} else {
  			(void) RG_Clean_Entry(tmp);
  		}
  	}
  	if (RG_Register(entry) == ERROR) {
--- 203,234 ----
       reg_t *entry;
  {
  	reg_t *tmp;
+ 	int updating = 0;
  
  	/* 
! 	 *  We want to see if the new object already matches any objects
  	 *  in the current Registry.  If it does, then if the new object's
  	 *  Update-Time is older than or the same as the Registry object's 
  	 *  Update-Time, then we ignore the new object.  Otherwise, we need 
! 	 *  to replace the Registry objects with the new object.  We do this 
! 	 *  by deleting the Registry objects, then adding the new object
  	 *  to the Registry.  The RG_Cleaner() will run periodically
  	 *  to compress the Registry.
  	 *
  	 *  If the new object is not in the Registry, then it's new
  	 *  so we add it to the Registry.
  	 */
! 	while ((tmp = RG_Object_Search_Entry(entry)) != NULL) {
  		if (tmp->update_time >= entry->update_time) {
  			(void) SM_Destroy_Obj(entry->FD);
  			RG_Free_Entry(entry);
  			ign_nobjs++;
+ 			if (updating) del_nobjs++; /* There will be no update */
  			return SUCCESS;
  		} else {
  			(void) RG_Clean_Entry(tmp);
+ 			if (updating) del_nobjs++; /* One might be updated, */
+ 			updating = 1;		   /* others are deleted.   */
  		}
  	}
  	if (RG_Register(entry) == ERROR) {
***************
*** 235,241 ****
  	}
  	do_IND_New_Object(entry);
  	LOGUPDATE(entry);
! 	up_nobjs++;
  	return SUCCESS;
  }
  
--- 237,246 ----
  	}
  	do_IND_New_Object(entry);
  	LOGUPDATE(entry);
! 	if (updating) {
! 		up_nobjs++;
! 	} else
! 		new_nobjs++;
  	return SUCCESS;
  }
  
***************
*** 578,584 ****
  		if (sscanf(mspace, "%s %d %hd %[^\n]s\n",
  			host, &port, &type, queri) != 4)
  			continue;
! 		up_nobjs = del_nobjs = ref_nobjs = recv_nobjs = ign_nobjs = 0;
  		lupdate = COL_get_last_update(host, port);
  		max_update_time = lupdate;
  		ret = COL_Init_Connect(host, port, type, lupdate, queri);
--- 583,589 ----
  		if (sscanf(mspace, "%s %d %hd %[^\n]s\n",
  			host, &port, &type, queri) != 4)
  			continue;
! 		new_nobjs = up_nobjs = del_nobjs = ref_nobjs = recv_nobjs = ign_nobjs = 0;
  		lupdate = COL_get_last_update(host, port);
  		max_update_time = lupdate;
  		ret = COL_Init_Connect(host, port, type, lupdate, queri);
***************
*** 588,598 ****
  			(void) COL_reset_update(host, port, lupdate);
  		}
  		/* If we did something, then update the update time */
! 		if (up_nobjs > 0 || del_nobjs > 0 || ref_nobjs > 0) {
  			(void) COL_put_last_update(host, port, max_update_time);
  			do_indexing = 1;
  		}
! 		Log("Finished collection - received %d objects (updated %d, deleted %d, refreshed %d, ignored %d).\n", recv_nobjs, up_nobjs, del_nobjs, ref_nobjs, ign_nobjs);
  	}
  	(void) fclose(g_file);
  
--- 593,603 ----
  			(void) COL_reset_update(host, port, lupdate);
  		}
  		/* If we did something, then update the update time */
! 		if (new_nobjs > 0 || up_nobjs > 0 || del_nobjs > 0 || ref_nobjs > 0) {
  			(void) COL_put_last_update(host, port, max_update_time);
  			do_indexing = 1;
  		}
! 		Log("Finished collection - received %d objects (added %d, updated %d, deleted %d, refreshed %d, ignored %d).\n", recv_nobjs, new_nobjs, up_nobjs, del_nobjs, ref_nobjs, ign_nobjs);
  	}
  	(void) fclose(g_file);
  
Index: harvest/src/broker/main.c
diff -c harvest/src/broker/main.c:2.4 harvest/src/broker/main.c:2.8
*** harvest/src/broker/main.c:2.4	Mon Oct 27 17:47:07 1997
--- harvest/src/broker/main.c	Thu Mar  5 18:39:08 1998
***************
*** 1,4 ****
! static char rcsid[] = "main.c,v 2.4 1997/10/27 22:47:07 sxw Exp";
  /*
   *  main.c -- main module of the Harvest Broker.
   *
--- 1,4 ----
! static char rcsid[] = "main.c,v 2.8 1998/03/05 23:39:08 sxw Exp";
  /*
   *  main.c -- main module of the Harvest Broker.
   *
***************
*** 108,113 ****
--- 108,114 ----
  #include "broker.h"
  #include "log.h"
  #include <locale.h>
+ #include "../common/include/version.h"
  
  /* Global Variables - administrative */
  
***************
*** 321,326 ****
--- 322,332 ----
  {
  	int status;
  
+ 	if (argc == 2 && !strcmp(argv[1], "-V") ) {
+ 		printf("Harvest Broker version %s.\n", HARVEST_VERSION);
+ 		exit(0);
+ 	}
+ 
  	(void) setlocale (LC_ALL, "");
  
  	disconnect();		/* be a nice, robust daemon */
***************
*** 715,720 ****
--- 721,728 ----
  {
  	FILE *cfp;
  	char dspace[BUFSIZ], tag[BUFSIZ], value[BUFSIZ];
+ 	int llen, cc;
+ 	char *vptr, *eptr;
  
  	Debug(77,1,("Configuring Broker from the file: %s\n", cfile));
  
***************
*** 739,754 ****
  	}
  	while (fgets(dspace, BUFSIZ, cfp) != NULL) {
  		/* Parse config line */
! 		if (dspace[0] == '#') {
! 			continue;
! 		} else if (sscanf(dspace, "%s %s", tag, value) < 2) {
  			continue;
  		} else {
  			if (set_str(tag, value) == ERROR) {
! 				fclose(cfp);
! 				return ERROR;
! 			}
! 		}
  		memset(tag, 0, BUFSIZ);
  		memset(value, 0, BUFSIZ);
  		memset(dspace, 0, BUFSIZ);
--- 747,764 ----
  	}
  	while (fgets(dspace, BUFSIZ, cfp) != NULL) {
  		/* Parse config line */
! 		if (dspace[0] == '#') { 
! 		        continue;
! 		} else if (sscanf(dspace, "%s %[^\n]", tag, value) < 2) {
  			continue;
  		} else {
+ 			char *p = value+strlen(value)-1;
+ 			while (isspace(*p)) *p-- = '\0';
  			if (set_str(tag, value) == ERROR) {
!                         	fclose(cfp);
!                         	return ERROR;
!                         }
!                 }
  		memset(tag, 0, BUFSIZ);
  		memset(value, 0, BUFSIZ);
  		memset(dspace, 0, BUFSIZ);
Index: harvest/src/broker/query_man.c
diff -c harvest/src/broker/query_man.c:2.1 harvest/src/broker/query_man.c:2.3
*** harvest/src/broker/query_man.c:2.1	Fri Mar 21 12:20:09 1997
--- harvest/src/broker/query_man.c	Tue Jun  2 18:03:40 1998
***************
*** 1,4 ****
! static char rcsid[] = "query_man.c,v 2.1 1997/03/21 17:20:09 sxw Exp";
  /* 
   *  query_man.c -- Broker query manager interface
   *
--- 1,4 ----
! static char rcsid[] = "query_man.c,v 2.3 1998/06/02 22:03:40 sxw Exp";
  /* 
   *  query_man.c -- Broker query manager interface
   *
***************
*** 707,730 ****
  int n;
  char *s;
  {
! 	char *p, *q, *w, buf[BUFSIZ];
! 	int i, sz = strlen(s);
  
! 	/* try to remove password from broker.out */
! 	if (((p = (char *) strstr(s, "#password")) != NULL) ||
! 	    ((p = (char *) strstr(s, "#PASSWORD")) != NULL)) {
  		q = strchr(p+1, '#');
! 		if (q == NULL) 
! 			q = s + sz;
! 		else
! 			q--;
! 		for (i = 0, w = s; w < s + sz && *w; w++) {
! 			if (w < p || w > q)
! 				buf[i++] = *w;
! 			else
! 				buf[i++] = 'X';
! 		}
! 		buf[i] = '\0';
  		Log("Processing Admin Command %d: %s\n", n, buf);
  	} else {
  		Log("Processing Admin Command %d: %s\n", n, s);
--- 707,722 ----
  int n;
  char *s;
  {
! 	char *p, *q, *w, *t, buf[BUFSIZ];
  
! 	/* remove password from broker.out */
! 	for (t = buf, w = s; *w; ) *t++ = tolower(*w++);
! 	if ((p = (char *) strstr(buf, "#password")) != NULL) {
! 		p += strlen("#password");
  		q = strchr(p+1, '#');
! 		q = (q == NULL) ? t : q - 1;
! 		for (t = buf, w = s; *w; t++, w++) *t = (t < p || t > q) ? *w : 'X';
! 		*t = '\0';
  		Log("Processing Admin Command %d: %s\n", n, buf);
  	} else {
  		Log("Processing Admin Command %d: %s\n", n, s);
Index: harvest/src/broker/registry.c
diff -c harvest/src/broker/registry.c:2.2 harvest/src/broker/registry.c:2.3
*** harvest/src/broker/registry.c:2.2	Thu Jan 29 12:23:26 1998
--- harvest/src/broker/registry.c	Tue Jun  2 10:11:50 1998
***************
*** 1,4 ****
! static char rcsid[] = "registry.c,v 2.2 1998/01/29 17:23:26 sxw Exp";
  /* 
   *  registry.c -- Registry Module, Keeps track of all objects in the Broker.
   *
--- 1,4 ----
! static char rcsid[] = "registry.c,v 2.3 1998/06/02 14:11:50 sxw Exp";
  /* 
   *  registry.c -- Registry Module, Keeps track of all objects in the Broker.
   *
***************
*** 485,491 ****
  	tmp = Registry;
  	while (tmp) {
  		/* This object's TTL has expired? */
! 		if ((tmp->ttl + tmp->update_time) < now) {
  			Debug(70,5,("RG_Cleaner: Expiring: %d, %s\n", 
  				tmp->FD, tmp->url));
  			tmp2 = tmp;
--- 485,491 ----
  	tmp = Registry;
  	while (tmp) {
  		/* This object's TTL has expired? */
! 		if (((tmp->ttl + tmp->update_time) < now) && tmp->ttl>1) {
  			Debug(70,5,("RG_Cleaner: Expiring: %d, %s\n", 
  				tmp->FD, tmp->url));
  			tmp2 = tmp;
Index: harvest/src/broker/select_loop.c
diff -c harvest/src/broker/select_loop.c:2.1 harvest/src/broker/select_loop.c:2.2
*** harvest/src/broker/select_loop.c:2.1	Fri Mar 21 12:20:12 1997
--- harvest/src/broker/select_loop.c	Thu Mar  5 14:30:12 1998
***************
*** 1,4 ****
! static char rcsid[] = "select_loop.c,v 2.1 1997/03/21 17:20:12 sxw Exp";
  /*
   *  select_loop.c - Input socket processing for the Broker
   *
--- 1,4 ----
! static char rcsid[] = "select_loop.c,v 2.2 1998/03/05 19:30:12 sxw Exp";
  /*
   *  select_loop.c - Input socket processing for the Broker
   *
***************
*** 107,112 ****
--- 107,113 ----
   */
  #include "broker.h"
  #include "log.h"
+ #include "../common/include/version.h"
  #ifdef _HARVEST_AIX_
  #include <sys/select.h>
  #else
***************
*** 174,180 ****
  		setsocket_linger(msock, 30);	/* 30 sec linger time */
  
  		/* Immediately send the protocol version */
! 		if (write(msock, QMVERSION, strlen(QMVERSION)) < 0) {
  			log_errno("initial socket write");
  			(void)close(msock);
  			return err;
--- 175,182 ----
  		setsocket_linger(msock, 30);	/* 30 sec linger time */
  
  		/* Immediately send the protocol version */
! 		if (write(msock, QMVERSION, strlen(QMVERSION)) < 0 ||
! 		    write(msock, BVERSION, strlen(BVERSION)) < 0) {
  			log_errno("initial socket write");
  			(void)close(msock);
  			return err;
Index: harvest/src/broker/stor_man.c
diff -c harvest/src/broker/stor_man.c:2.2 harvest/src/broker/stor_man.c:2.3
*** harvest/src/broker/stor_man.c:2.2	Tue Jun 10 15:45:08 1997
--- harvest/src/broker/stor_man.c	Tue Mar  3 11:36:22 1998
***************
*** 1,4 ****
! static char rcsid[] = "stor_man.c,v 2.2 1997/06/10 19:45:08 sxw Exp";
  /*
   *   stor_man.c -- Stores and retreives Summary Objects in the Broker.
   *   This storage manager is based on the UNIX file system.
--- 1,4 ----
! static char rcsid[] = "stor_man.c,v 2.3 1998/03/03 16:36:22 sxw Exp";
  /*
   *   stor_man.c -- Stores and retreives Summary Objects in the Broker.
   *   This storage manager is based on the UNIX file system.
***************
*** 106,114 ****
--- 106,117 ----
   *    re-implementations of code complying to this set of standards.  
   *  
   */
+ #include <stdlib.h>
+ 
  #include "broker.h"
  #include "log.h"
  
+ 
  /* Global variables */
  extern char *DIRpath;
  extern char *brk_obj_url;
***************
*** 260,266 ****
  	return ((fd_t) lrand48());
  #elif defined(HAVE_RANDOM)
  #ifndef random
! 	extern int random();
  #endif
  	return ((fd_t) random());
  #else
--- 263,269 ----
  	return ((fd_t) lrand48());
  #elif defined(HAVE_RANDOM)
  #ifndef random
! 	extern long random();
  #endif
  	return ((fd_t) random());
  #else
Index: harvest/src/common/Makefile.in
diff -c harvest/src/common/Makefile.in:2.3 harvest/src/common/Makefile.in:2.4
*** harvest/src/common/Makefile.in:2.3	Tue Nov  4 13:15:25 1997
--- harvest/src/common/Makefile.in	Thu Mar  5 14:04:15 1998
***************
*** 1,9 ****
  #
  #  Makefile for the common source directory
  #
! #  Makefile.in,v 2.3 1997/11/04 18:15:25 sxw Exp
  #
! SUBDIRS		= md5 util url template regex
  
  prefix = @prefix@
  INSTALL_BINDIR	= $(prefix)/bin
--- 1,9 ----
  #
  #  Makefile for the common source directory
  #
! #  Makefile.in,v 2.4 1998/03/05 19:04:15 sxw Exp
  #
! SUBDIRS		= md5 util template url regex
  
  prefix = @prefix@
  INSTALL_BINDIR	= $(prefix)/bin
Index: harvest/src/common/include/template.h
diff -c harvest/src/common/include/template.h:2.0 harvest/src/common/include/template.h:2.1
*** harvest/src/common/include/template.h:2.0	Sun Mar 16 17:14:29 1997
--- harvest/src/common/include/template.h	Tue Jun  2 17:05:55 1998
***************
*** 136,141 ****
--- 136,142 ----
  #define T_TYPE		"Type"
  #define T_UPDATE	"Update-Time"
  #define T_URL		"URL"
+ #define T_UREFS		"URL-References"
  
  /* Backwards compatibility */
  #define T_FILETYPE	T_TYPE
Index: harvest/src/common/url/Makefile.in
diff -c harvest/src/common/url/Makefile.in:2.8 harvest/src/common/url/Makefile.in:2.10
*** harvest/src/common/url/Makefile.in:2.8	Tue Nov  4 13:15:32 1997
--- harvest/src/common/url/Makefile.in	Tue Jun  2 17:05:57 1998
***************
*** 1,7 ****
  #
  #  Makefile for simple liburl
  #
! #  Makefile.in,v 2.8 1997/11/04 18:15:32 sxw Exp
  #
  
  srcdir = @srcdir@
--- 1,7 ----
  #
  #  Makefile for simple liburl
  #
! #  Makefile.in,v 2.10 1998/06/02 21:05:57 sxw Exp
  #
  
  srcdir = @srcdir@
***************
*** 54,66 ****
  
  CFLAGS		= $(DEBUG) $(CACHE_FLAGS) $(INCLUDES)
  OBJS 		= rfc850.o rfc1738.o gopher.o ftp.o http.o url.o news.o \
! 		  url_relative.o $(CACHE_OBJS) $(CCACHE_OBJS) localmap.o
  SCRIPTS		= gopherget.pl newsget.pl test-socket-ph.pl
  PROGS		= ftpget
  UTILS		= urlget urlpurge
  LIBDIR		= ../lib
  LDFLAGS		= -L$(LIBDIR) $(CACHE_LDFLAGS) $(CCACHE_LDFLAGS)
! LIBS		= -lurl -lutil \
  		  $(MD5_LIB) $(CACHE_LIBS) $(CCACHE_LIBS) $(DEBUG_LIBS) \
  		  $(XTRA_LIBS)
  LIBFILE		= liburl.a
--- 54,66 ----
  
  CFLAGS		= $(DEBUG) $(CACHE_FLAGS) $(INCLUDES)
  OBJS 		= rfc850.o rfc1738.o gopher.o ftp.o http.o url.o news.o \
! 		  url_relative.o $(CACHE_OBJS) $(CCACHE_OBJS) localmap.o db.o
  SCRIPTS		= gopherget.pl newsget.pl test-socket-ph.pl
  PROGS		= ftpget
  UTILS		= urlget urlpurge
  LIBDIR		= ../lib
  LDFLAGS		= -L$(LIBDIR) $(CACHE_LDFLAGS) $(CCACHE_LDFLAGS)
! LIBS		= -lurl -lutil -ltemplate \
  		  $(MD5_LIB) $(CACHE_LIBS) $(CCACHE_LIBS) $(DEBUG_LIBS) \
  		  $(XTRA_LIBS)
  LIBFILE		= liburl.a
Index: harvest/src/common/url/http.c
diff -c harvest/src/common/url/http.c:2.6 harvest/src/common/url/http.c:2.8
*** harvest/src/common/url/http.c:2.6	Fri Nov 21 12:40:23 1997
--- harvest/src/common/url/http.c	Tue Jun  2 17:05:59 1998
***************
*** 1,4 ****
! static char rcsid[] = "http.c,v 2.6 1997/11/21 17:40:23 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
--- 1,4 ----
! static char rcsid[] = "http.c,v 2.8 1998/06/02 21:05:59 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
***************
*** 30,111 ****
   *  along with this program; if not, write to the Free Software
   *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
- /*
-  *  ----------------------------------------------------------------------
-  *  Copyright (c) 1994, 1995.  All rights reserved.
-  *  
-  *    The Harvest software was developed by the Internet Research Task
-  *    Force Research Group on Resource Discovery (IRTF-RD):
-  *  
-  *          Mic Bowman of Transarc Corporation.
-  *          Peter Danzig of the University of Southern California.
-  *          Darren R. Hardy of the University of Colorado at Boulder.
-  *          Udi Manber of the University of Arizona.
-  *          Michael F. Schwartz of the University of Colorado at Boulder.
-  *          Duane Wessels of the University of Colorado at Boulder.
-  *  
-  *    This copyright notice applies to software in the Harvest
-  *    ``src/'' directory only.  Users should consult the individual
-  *    copyright notices in the ``components/'' subdirectories for
-  *    copyright information about other software bundled with the
-  *    Harvest source code distribution.
-  *  
-  *  TERMS OF USE
-  *    
-  *    The Harvest software may be used and re-distributed without
-  *    charge, provided that the software origin and research team are
-  *    cited in any use of the system.  Most commonly this is
-  *    accomplished by including a link to the Harvest Home Page
-  *    (http://harvest.cs.colorado.edu/) from the query page of any
-  *    Broker you deploy, as well as in the query result pages.  These
-  *    links are generated automatically by the standard Broker
-  *    software distribution.
-  *    
-  *    The Harvest software is provided ``as is'', without express or
-  *    implied warranty, and with no support nor obligation to assist
-  *    in its use, correction, modification or enhancement.  We assume
-  *    no liability with respect to the infringement of copyrights,
-  *    trade secrets, or any patents, and are not responsible for
-  *    consequential damages.  Proper use of the Harvest software is
-  *    entirely the responsibility of the user.
-  *  
-  *  DERIVATIVE WORKS
-  *  
-  *    Users may make derivative works from the Harvest software, subject 
-  *    to the following constraints:
-  *  
-  *      - You must include the above copyright notice and these 
-  *        accompanying paragraphs in all forms of derivative works, 
-  *        and any documentation and other materials related to such 
-  *        distribution and use acknowledge that the software was 
-  *        developed at the above institutions.
-  *  
-  *      - You must notify IRTF-RD regarding your distribution of 
-  *        the derivative work.
-  *  
-  *      - You must clearly notify users that your are distributing 
-  *        a modified version and not the original Harvest software.
-  *  
-  *      - Any derivative product is also subject to these copyright 
-  *        and use restrictions.
-  *  
-  *    Note that the Harvest software is NOT in the public domain.  We
-  *    retain copyright, as specified above.
-  *  
-  *  HISTORY OF FREE SOFTWARE STATUS
-  *  
-  *    Originally we required sites to license the software in cases
-  *    where they were going to build commercial products/services
-  *    around Harvest.  In June 1995 we changed this policy.  We now
-  *    allow people to use the core Harvest software (the code found in
-  *    the Harvest ``src/'' directory) for free.  We made this change
-  *    in the interest of encouraging the widest possible deployment of
-  *    the technology.  The Harvest software is really a reference
-  *    implementation of a set of protocols and formats, some of which
-  *    we intend to standardize.  We encourage commercial
-  *    re-implementations of code complying to this set of standards.  
-  *  
-  */
  #include <stdio.h>
  #include <stdlib.h>
  #include <unistd.h>
--- 30,35 ----
***************
*** 265,270 ****
--- 189,200 ----
   * modification of User-Agent: and From: HTTP headers.  Uses
   * environment variables HARVEST_USER_AGENT and HARVEST_MAINTAINER_ADDRESS.
   *
+  * Patched by Judith Pluemer <judith@mathematik.uni-osnabrueck.de>
+  * to support incremental gathering on http-Servers. Additionally
+  * the Perl-Scripts uptime.pl and extract.pl are used.
+  *
+  * Patched by Simon Wilkinson <sxw@tardis.ed.ac.uk> to remove dependencies
+  * on perl scripts for incremental gatherering
   */
  int http_get(up)
       URL *up;
***************
*** 283,289 ****
--- 213,221 ----
      
  	Buffer *mimebuf = NULL;
  	Buffer *reqbuf = NULL;
+ 	Buffer *urlbuf = NULL;
  	URL *new_up = NULL;
+ 	FILE *dateizeiger;
  	char *bufp = NULL;
  	char *host = NULL;
  	char *http_proxy = getenv("http_proxy");
***************
*** 373,382 ****
--- 305,328 ----
  	  sprintf(buf, "Accept: %s\n",accept);
  	  add_buffer(reqbuf, buf, strlen(buf));
  	}
+ 
+         urlbuf = create_buffer(BUFSIZ);
+         sprintf(buf, "http://%s", up->host);
+ 	add_buffer(urlbuf, buf, strlen(buf));
+ 	if (up->port != 80) { 
+ 	  sprintf(buf, ":%d", up->port);
+           add_buffer(urlbuf, buf, strlen(buf)); 
+ 	}
+ 
+ 	/* FIXME: Is this right, or should we be using the escaped pathname? */
+ 	add_buffer(urlbuf,up->raw_pathname,strlen(up->raw_pathname));
+ 	up->lmt=urldb_getlmt(urlbuf);
+ 
  	if (up->lmt > 0) {
  		sprintf(buf, "If-Modified-Since: %s\r\n", mkrfc850(&(up->lmt)));
  		add_buffer(reqbuf, buf, strlen(buf));
  	}
+ 
  #ifdef HTTP_AUTHENTICATION
  	if (up->auth_realm) {
  		if ((tmp = http_make_auth(up->auth_realm))) {
***************
*** 440,445 ****
--- 386,392 ----
  				    up->host, up->port);
  			close(s);
  			fclose(up->fp);
+ 			free_buffer(urlbuf);
  			return 4;
  		}
  		if (in_http_code) {
***************
*** 451,457 ****
  			 * <sxw@tardis.ed.ac.uk> so Harvest will work with
  			 *  HTTP/1.1 compliant servers.
                           */
! 			if (memcmp(bufp, "HTTP/1", 6) != 0) {
  				tmp = strchr(bufp, '\n');
  				if (tmp != NULL)
  					*tmp = '\0';	/* get one line */
--- 398,405 ----
  			 * <sxw@tardis.ed.ac.uk> so Harvest will work with
  			 *  HTTP/1.1 compliant servers.
                           */
! 			if ((strlen(bufp) > 5) && 
! 			    (memcmp(bufp, "HTTP/1", 6) != 0)) {
  				tmp = strchr(bufp, '\n');
  				if (tmp != NULL)
  					*tmp = '\0';	/* get one line */
***************
*** 460,465 ****
--- 408,450 ----
  				in_http_header = 0;
  				in_http_data = 1;
  			} else {
+ 	                  if ((strchr(bufp,'3') != NULL) && 
+ 			      (memcmp(strchr(bufp,'3'), "304", 3) == 0)) {
+ 			    if (up->lmt > 0) {
+ 			      /* We've done a successful If-Modified-Since get
+ 				 so we now need to write the file out of the
+ 				 production db into where the data is expected
+ 			       */
+ 
+ #ifdef USE_MD5
+ 			      /* Pull the MD5 hash from the production db */
+ 			      up->md5 = urldb_getmd5(urlbuf);
+ #endif
+ 			      up->http_version = xstrdup("HTTP/1.0");
+ 			      up->http_reason_line=
+ 				xstrdup("HTTP/1.0 200 OK\n");
+ 
+ 			      /* Create a fake set of Mime headers for the
+ 				 item */
+ 			      mimebuf = create_buffer(BUFSIZ);
+ 			      sprintf(buf,"HTTP/1.1 200 OK\n");
+ 			      add_buffer(mimebuf, buf, strlen(buf));
+ 			      sprintf(buf,"Last-Modified: %s\n",
+ 				      mkrfc850(&(up->lmt)));
+ 			      add_buffer(mimebuf, buf, strlen(buf));
+ 			      sprintf(buf,"Content-Type: text/x-soif\n");
+ 			      add_buffer(mimebuf, buf, strlen(buf));
+ 			      
+ 			      /* Pull the SOIF out of the production database 
+ 				 and store it into the file */
+ 			      urldb_writesoif(urlbuf, up->fp);
+ 
+ 			      in_http_header = 0;
+ 			      in_http_data = 1;
+ 
+ 			      break; /* Don't parse any more of the stuff */
+ 			    }
+ 			  }
  				in_http_header = 1;
  				in_http_data = 0;
  
***************
*** 478,483 ****
--- 463,469 ----
  					errorlog("Failed HTTP/1.0 transfer for %s: %s (status code: %d)\n", up->url, bufp,up->http_status_code);
  					close(s);
  					fclose(up->fp);
+ 					free_buffer(urlbuf);
  					return 10;
  				}
  				up->http_reason_line = xstrdup(bufp);
***************
*** 489,494 ****
--- 475,481 ----
  					errorlog("HTTP/1.0 transfer error for %s: %s\n", up->url, up->http_reason_line);
  					close(s);
  					fclose(up->fp);
+ 					free_buffer(urlbuf);
  					return 10;
  				}
  				while (*bufp != '\n' && n > 0)
***************
*** 582,587 ****
--- 569,575 ----
  				fclose(up->fp);
  				up->fp=0;
  
+ 				free_buffer(urlbuf);
  			        return (-1); /* Return redirect info */
  			}
  	      redirect_fallthru:
***************
*** 604,609 ****
--- 592,598 ----
  				xfree(up->http_version);
  				up->http_version=NULL;
  				free_buffer(mimebuf);
+ 				free_buffer(urlbuf);
  				fclose(up->fp);
  				close(s);
  				return 10;	/* 'hard' error from httpd */
***************
*** 643,648 ****
--- 632,638 ----
  				xfree(up->http_version);
  				up->http_version=NULL;
  				free_buffer(mimebuf);
+ 				free_buffer(urlbuf);
  				close(s);
  				fclose(up->fp);
  				return http_get(up);
***************
*** 673,678 ****
--- 663,670 ----
  	/* Clean up */
  	fclose(up->fp);
  	close(s);
+ 
+ 	free_buffer(urlbuf);
  
  	return (in_http_data == 1 ? 0 : 10);
  }
Index: harvest/src/common/url/url.c
diff -c harvest/src/common/url/url.c:2.3 harvest/src/common/url/url.c:2.5
*** harvest/src/common/url/url.c:2.3	Wed Feb  4 09:39:06 1998
--- harvest/src/common/url/url.c	Tue Jun  2 17:06:00 1998
***************
*** 1,4 ****
! static char rcsid[] = "url.c,v 2.3 1998/02/04 14:39:06 sxw Exp";
  /*
   *  url.c - URL processing code 
   *
--- 1,4 ----
! static char rcsid[] = "url.c,v 2.5 1998/06/02 21:06:00 sxw Exp";
  /*
   *  url.c - URL processing code 
   *
***************
*** 243,248 ****
--- 243,250 ----
  	liburl_sleep_time = 1;	/* hard-coded default */
  	if ((s = getenv("HARVEST_URL_DELAY")) != NULL)
  		liburl_sleep_time = atoi(s);
+ 	if ((s = getenv("HARVEST_GATHERER_DBS")) != NULL)
+ 	        urldb_init(s);
  
  #ifdef USE_LOCAL_CACHE
  	if (use_local_cache)
Index: harvest/src/gatherer/enumerate/HTMLparse.c
diff -c harvest/src/gatherer/enumerate/HTMLparse.c:2.2 harvest/src/gatherer/enumerate/HTMLparse.c:2.3
*** harvest/src/gatherer/enumerate/HTMLparse.c:2.2	Fri Jun 13 15:20:50 1997
--- harvest/src/gatherer/enumerate/HTMLparse.c	Wed Feb 18 09:58:56 1998
***************
*** 1,4 ****
! static char rcsid[]="HTMLparse.c,v 2.2 1997/06/13 19:20:50 sxw Exp";
  /*  
   * HTMLparse.c taken from Mosaic 2.7b5 with some alterations for Harvest
   * 
--- 1,4 ----
! static char rcsid[]="HTMLparse.c,v 2.3 1998/02/18 14:58:56 sxw Exp";
  /*  
   * HTMLparse.c taken from Mosaic 2.7b5 with some alterations for Harvest
   * 
***************
*** 381,387 ****
  	while (*ptr != '\0')
  	  {
  	    if (*ptr == '\r')
! 	      *ptr = ' ';
  	    ptr++;
  	  }
  
--- 381,387 ----
  	while (*ptr != '\0')
  	  {
  	    if (*ptr == '\r')
! 	      *ptr = '\n';
  	    ptr++;
  	  }
  
Index: harvest/src/gatherer/enumerate/Makefile.in
diff -c harvest/src/gatherer/enumerate/Makefile.in:2.3 harvest/src/gatherer/enumerate/Makefile.in:2.4
*** harvest/src/gatherer/enumerate/Makefile.in:2.3	Tue Nov  4 13:15:38 1997
--- harvest/src/gatherer/enumerate/Makefile.in	Tue Jun  2 17:06:01 1998
***************
*** 1,7 ****
  #
  #  Makefile for the enumerators
  #
! #  Makefile.in,v 2.3 1997/11/04 18:15:38 sxw Exp
  #
  srcdir = @srcdir@
  VPATH = $(srcdir)
--- 1,7 ----
  #
  #  Makefile for the enumerators
  #
! #  Makefile.in,v 2.4 1998/06/02 21:06:01 sxw Exp
  #
  srcdir = @srcdir@
  VPATH = $(srcdir)
***************
*** 29,35 ****
  CFLAGS		= $(DEBUG) $(INCLUDE)
  LIBDIR		= ../../common/lib
  LDFLAGS		= -L$(LIBDIR)
! LIBS		= -lurl -lmd5 -lutil -lgdbm -lregex $(DEBUG_LIBS) $(XTRA_LIBS)
  FILTER_OBJS	= filter.o robots-txt.o
  HTML_OBJS	= HTMLparse.o
  PROGS		= enum fileenum ftpenum httpenum-breadth httpenum-depth gopherenum-breadth gopherenum-depth newsenum \
--- 29,35 ----
  CFLAGS		= $(DEBUG) $(INCLUDE)
  LIBDIR		= ../../common/lib
  LDFLAGS		= -L$(LIBDIR)
! LIBS		= -lurl -ltemplate -lmd5 -lutil -lgdbm -lregex $(DEBUG_LIBS) $(XTRA_LIBS)
  FILTER_OBJS	= filter.o robots-txt.o
  HTML_OBJS	= HTMLparse.o
  PROGS		= enum fileenum ftpenum httpenum-breadth httpenum-depth gopherenum-breadth gopherenum-depth newsenum \
Index: harvest/src/gatherer/enumerate/httpenum-depth.c
diff -c harvest/src/gatherer/enumerate/httpenum-depth.c:2.6 harvest/src/gatherer/enumerate/httpenum-depth.c:2.7
*** harvest/src/gatherer/enumerate/httpenum-depth.c:2.6	Sun May  4 23:18:26 1997
--- harvest/src/gatherer/enumerate/httpenum-depth.c	Tue Jun  2 17:06:02 1998
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-depth.c,v 2.6 1997/05/05 03:18:26 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-depth.c,v 2.7 1998/06/02 21:06:02 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
***************
*** 39,119 ****
   *  along with this program; if not, write to the Free Software
   *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
- /*  ----------------------------------------------------------------------
-  *  Copyright (c) 1994, 1995.  All rights reserved.
-  *  
-  *    The Harvest software was developed by the Internet Research Task
-  *    Force Research Group on Resource Discovery (IRTF-RD):
-  *  
-  *          Mic Bowman of Transarc Corporation.
-  *          Peter Danzig of the University of Southern California.
-  *          Darren R. Hardy of the University of Colorado at Boulder.
-  *          Udi Manber of the University of Arizona.
-  *          Michael F. Schwartz of the University of Colorado at Boulder.
-  *          Duane Wessels of the University of Colorado at Boulder.
-  *  
-  *    This copyright notice applies to software in the Harvest
-  *    ``src/'' directory only.  Users should consult the individual
-  *    copyright notices in the ``components/'' subdirectories for
-  *    copyright information about other software bundled with the
-  *    Harvest source code distribution.
-  *  
-  *  TERMS OF USE
-  *    
-  *    The Harvest software may be used and re-distributed without
-  *    charge, provided that the software origin and research team are
-  *    cited in any use of the system.  Most commonly this is
-  *    accomplished by including a link to the Harvest Home Page
-  *    (http://harvest.cs.colorado.edu/) from the query page of any
-  *    Broker you deploy, as well as in the query result pages.  These
-  *    links are generated automatically by the standard Broker
-  *    software distribution.
-  *    
-  *    The Harvest software is provided ``as is'', without express or
-  *    implied warranty, and with no support nor obligation to assist
-  *    in its use, correction, modification or enhancement.  We assume
-  *    no liability with respect to the infringement of copyrights,
-  *    trade secrets, or any patents, and are not responsible for
-  *    consequential damages.  Proper use of the Harvest software is
-  *    entirely the responsibility of the user.
-  *  
-  *  DERIVATIVE WORKS
-  *  
-  *    Users may make derivative works from the Harvest software, subject 
-  *    to the following constraints:
-  *  
-  *      - You must include the above copyright notice and these 
-  *        accompanying paragraphs in all forms of derivative works, 
-  *        and any documentation and other materials related to such 
-  *        distribution and use acknowledge that the software was 
-  *        developed at the above institutions.
-  *  
-  *      - You must notify IRTF-RD regarding your distribution of 
-  *        the derivative work.
-  *  
-  *      - You must clearly notify users that your are distributing 
-  *        a modified version and not the original Harvest software.
-  *  
-  *      - Any derivative product is also subject to these copyright 
-  *        and use restrictions.
-  *  
-  *    Note that the Harvest software is NOT in the public domain.  We
-  *    retain copyright, as specified above.
-  *  
-  *  HISTORY OF FREE SOFTWARE STATUS
-  *  
-  *    Originally we required sites to license the software in cases
-  *    where they were going to build commercial products/services
-  *    around Harvest.  In June 1995 we changed this policy.  We now
-  *    allow people to use the core Harvest software (the code found in
-  *    the Harvest ``src/'' directory) for free.  We made this change
-  *    in the interest of encouraging the widest possible deployment of
-  *    the technology.  The Harvest software is really a reference
-  *    implementation of a set of protocols and formats, some of which
-  *    we intend to standardize.  We encourage commercial
-  *    re-implementations of code complying to this set of standards.  
-  *  
-  */
  #include <stdio.h>
  #include <string.h>
  #include <signal.h>
--- 39,44 ----
***************
*** 446,531 ****
  	mark_retrieved(up);
  
      /* Are we dealing with an HTML file, if not we can't get href links */
!     if (up->http_mime_hdr != NULL &&
! 	strstr(up->http_mime_hdr, "text/html") == NULL) {
!           fprintf(stdout, "%s\t%s\n", up->url, up->md5);
! 	  fflush(stdout);
! 	  return (NULL);
      }
! 
!     /* Extract the HREF's */
!     if (pipe(pipefds) < 0) {
! 	log_errno("pipe");
  	return (NULL);
!     }
!     if ((pid = fork()) < 0) {
  	log_errno("fork");
  	return (NULL);
!     }
!     if (pid == 0) {		/* child: HTMLurls */
  	char *argv[64];
  	char *enum_url = NULL;
! 
  	enum_url = (char *) xmalloc(strlen(up->url) + 20);
  	sprintf(enum_url, "ENUMERATOR_URL=%s", up->url);
  	putenv(enum_url);
! 
  	close(pipefds[0]);	/* child wont read from pipe */
  	dup2(pipefds[1], 1);	/* stdout -> write:pipe */
  	close(pipefds[1]);	/* close pipe, its now stdout */
! 
  	argv[0]="HTMLurls";
  	argv[1]="--base-url";
  	argv[2]=up->url;
  	argv[3]=up->filename;
  	argv[4]=NULL;
! 
  	execvp(argv[0], argv);
  	sprintf(buf, "execvp: %s", argv[0]);
  	log_errno(buf);
  	_exit(1);
!     }
!     close(pipefds[1]);		/* parent wont write */
!     if ((fp = fdopen(pipefds[0], "r")) == NULL) {
  	log_errno("fdopen");
  	return (NULL);
!     }
!     /* 
!      *  For each HREF pointer, convert it to a URL, and add it to
!      *  the list of URLs to return.
!      */
!     Tail = &head;
!     while (fgets(buf, BUFSIZ, fp) != NULL) {
  	if ((s = strrchr(buf, '\n')) != NULL)
! 	    *s = '\0';		/* strip newline */
  	Debug(42, 1, ("httpenum-depth: Input: %s\n", buf));
! 
  	/* Make a link in the list for this URL */
  	l = (list_t *) xmalloc(sizeof(list_t));
  	l->ptr = (void *) xstrdup(buf);
  	l->next = (list_t *) NULL;
  	*Tail = l;
  	Tail = &(l->next);
! 
  	nurls++;
!     }
!     fclose(fp);
!     close(pipefds[0]);
! 
!     /* Wait for HTMLurls to finish and decide on whether to index the page
!      * according to its exit code 
!      */
!     if ((err = waitpid(pid, &status, 0)) != pid) {
  	Debug(42, 1, ("WARNING: waiting for child %d got %d...\n",
! 		pid, err));
      }
  
!     /* If we had an exit code of 0 then index it, otherwise don't */
!     if (WEXITSTATUS(status) == 0) {
!       fprintf(stdout, "%s\t%s\n", up->url, up->md5);
!       fflush(stdout);
      }
!     
      Debug(42, 1, ("Adding %d URLs from %s to workload\n", nurls, up->url));
      return (head);
  }
--- 371,484 ----
  	mark_retrieved(up);
  
      /* Are we dealing with an HTML file, if not we can't get href links */
!     if (up->http_mime_hdr == NULL || 
! 	(strstr(up->http_mime_hdr, "text/html") == NULL &&
! 	 strstr(up->http_mime_hdr, "text/x-soif") == NULL)) {
!       fprintf(stdout, "%s\t%s\n", up->url, up->md5);
!       fflush(stdout);
!       return (NULL);
      }
!       
!     /* If we were given an HTML document, then use HTMLurls to summarise it */
!     if (strstr(up->http_mime_hdr, "text/html") !=NULL ) {
!       /* Extract the HREF's */
!       if (pipe(pipefds) < 0) {
!       	log_errno("pipe");
  	return (NULL);
!       }
!       if ((pid = fork()) < 0) {
  	log_errno("fork");
  	return (NULL);
!       }
!       if (pid == 0) {		/* child: HTMLurls */
  	char *argv[64];
  	char *enum_url = NULL;
! 	
  	enum_url = (char *) xmalloc(strlen(up->url) + 20);
  	sprintf(enum_url, "ENUMERATOR_URL=%s", up->url);
  	putenv(enum_url);
! 	
  	close(pipefds[0]);	/* child wont read from pipe */
  	dup2(pipefds[1], 1);	/* stdout -> write:pipe */
  	close(pipefds[1]);	/* close pipe, its now stdout */
! 	
  	argv[0]="HTMLurls";
  	argv[1]="--base-url";
  	argv[2]=up->url;
  	argv[3]=up->filename;
  	argv[4]=NULL;
! 	
  	execvp(argv[0], argv);
  	sprintf(buf, "execvp: %s", argv[0]);
  	log_errno(buf);
  	_exit(1);
!       }
!       close(pipefds[1]);		/* parent wont write */
!       if ((fp = fdopen(pipefds[0], "r")) == NULL) {
  	log_errno("fdopen");
  	return (NULL);
!       }
!       /* 
!        *  For each HREF pointer, convert it to a URL, and add it to
!        *  the list of URLs to return.
!        */
!       Tail = &head;
!       while (fgets(buf, BUFSIZ, fp) != NULL) {
  	if ((s = strrchr(buf, '\n')) != NULL)
! 	  *s = '\0';		/* strip newline */
  	Debug(42, 1, ("httpenum-depth: Input: %s\n", buf));
! 	
  	/* Make a link in the list for this URL */
  	l = (list_t *) xmalloc(sizeof(list_t));
  	l->ptr = (void *) xstrdup(buf);
  	l->next = (list_t *) NULL;
  	*Tail = l;
  	Tail = &(l->next);
! 	
  	nurls++;
!       }
!       fclose(fp);
!       close(pipefds[0]);
!       
!       /* Wait for HTMLurls to finish and decide on whether to index the page
!        * according to its exit code 
!        */
!       if ((err = waitpid(pid, &status, 0)) != pid) {
  	Debug(42, 1, ("WARNING: waiting for child %d got %d...\n",
! 		      pid, err));
!       }
!     
!       /* If we had an exit code of 0 then index it, otherwise don't */
!       if (WEXITSTATUS(status) == 0) {
! 	fprintf(stdout, "%s\t%s\n", up->url, up->md5);
! 	fflush(stdout);
!       }
      }
+     
+     /* If its a SOIF document then use the SOIF summariser to do it */
+     if (strstr(up->http_mime_hdr, "text/x-soif") !=NULL ) {
+       char *refs=urldb_getrefs(up->url);
+       char *ptr;
  
!       /* Flush the URL, but as a SOIF one */
!       fprintf(stdout, "SOIF:%s\t%s\n", up->url, up->md5);
! 
!       ptr=strtok(refs, '\n');
!       while(ptr!=NULL) {
! 	/* Make a link in the list for this URL */
! 	l = (list_t *) xmalloc(sizeof(list_t));
! 	l->ptr = (void *) xstrdup(ptr);
! 	l->next = (list_t *) NULL;
! 	*Tail = l;
! 	Tail = &(l->next);
! 	
! 	nurls++;
! 
! 	ptr=strtok(NULL, '/n');
!       }
!       xfree(refs);
      }
! 
      Debug(42, 1, ("Adding %d URLs from %s to workload\n", nurls, up->url));
      return (head);
  }
Index: harvest/src/gatherer/essence/summarize.c
diff -c harvest/src/gatherer/essence/summarize.c:2.0 harvest/src/gatherer/essence/summarize.c:2.1
*** harvest/src/gatherer/essence/summarize.c:2.0	Sun Mar 16 17:18:17 1997
--- harvest/src/gatherer/essence/summarize.c	Tue Jun  2 18:23:51 1998
***************
*** 834,874 ****
  		return;
  
  	/* Try to build a Description attribute based on other fields */
! 	avp = extract_AVPair(t->list, T_TITLE);
! 	if (avp == NULL)
! 		avp = extract_AVPair(t->list, T_ABSTRACT);
  	if (avp == NULL)
  		avp = extract_AVPair(t->list, T_PARTTEXT);
  
  	/* Cannot find any data to use, or not enough */
  	if (avp == NULL || avp->vsize < 2)
  		return;
  
! 	/* Locate the first line of the data */
! 	for (i = 0, gotdata = 0; i < avp->vsize; i++) {
! 		if (avp->value[i] == '\n') {
! 			if (gotdata == 1)
! 				break;
! 		} else if (gotdata == 0 && isalnum(avp->value[i])) {
  			gotdata = 1;
  		}
  	}
! 	n = i;
! 
! 	/* If we got a reasonable chunk then use i bytes of it */
! 	if (gotdata && n > 0) {
! 		/* Copy the data and strip the newlines */
! 		s = xmalloc(n + 1);
! 		for (i = 0, j = 0; j < n; j++) {
! 			if (avp->value[j] != '\n')
! 				s[i++] = avp->value[j];
! 		}
! 		s[i] = '\0';
! 
  		/* We know for sure that T_DESCRIPTION isn't in t->list */
! 		FAST_add_AVList(t->list, T_DESCRIPTION, s, i);
! 		xfree(s);
  	}
  }
  
  /*
--- 834,874 ----
  		return;
  
  	/* Try to build a Description attribute based on other fields */
! 	/* These heuristics only apply if the Summarizer failed to make one */
! 	avp = extract_AVPair(t->list, T_ABSTRACT);
  	if (avp == NULL)
  		avp = extract_AVPair(t->list, T_PARTTEXT);
+ 	if (avp == NULL)
+ 		avp = extract_AVPair(t->list, "body");  /* By SGML summarizer */
+ 	if (avp == NULL)
+ 		avp = extract_AVPair(t->list, T_TITLE); /* Last resort */
  
  	/* Cannot find any data to use, or not enough */
  	if (avp == NULL || avp->vsize < 2)
  		return;
  
! 	/* Extract a reasonable chunk of the data */
! 	if ( (s = xmalloc(BUFSIZ / 4 + 1)) == NULL)
! 		return;
! 	for (i = 0, j = 0, gotdata = 0; i < avp->vsize; i++) {
! 		if (j >= BUFSIZ / 4)
! 			break;
! 		if (isspace(avp->value[i])) {
! 			/* Squash white space into a single blank / newline */
! 			if (gotdata)
! 				s[j++] = (avp->value[i] == '\n' ? '\n' : ' ');
! 			gotdata = 0;
! 		} else {
! 			s[j++] = avp->value[i];
  			gotdata = 1;
  		}
  	}
! 	s[j] = '\0';
! 	if (j > 0) {
  		/* We know for sure that T_DESCRIPTION isn't in t->list */
! 		FAST_add_AVList(t->list, T_DESCRIPTION, s, j);
  	}
+ 	xfree(s);
  }
  
  /*
Index: harvest/src/gatherer/gatherd/gather.c
diff -c harvest/src/gatherer/gatherd/gather.c:2.3 harvest/src/gatherer/gatherd/gather.c:2.4
*** harvest/src/gatherer/gatherd/gather.c:2.3	Fri Nov 21 12:41:49 1997
--- harvest/src/gatherer/gatherd/gather.c	Tue Mar 17 19:02:02 1998
***************
*** 1,4 ****
! static char rcsid[] = "gather.c,v 2.3 1997/11/21 17:41:49 sxw Exp";
  /*
   *  gather.c - Simple, portable client to retrieve data from a Gatherer,
   *  and print it to stdout.  Uses the GNU zip compression to transmit
--- 1,4 ----
! static char rcsid[] = "gather.c,v 2.4 1998/03/18 00:02:02 sxw Exp";
  /*
   *  gather.c - Simple, portable client to retrieve data from a Gatherer,
   *  and print it to stdout.  Uses the GNU zip compression to transmit
***************
*** 162,168 ****
  	FILE *ofp = NULL;
  	char *p = NULL;
  	char *server_host = NULL;
! 	char *this_host = getfullhostname();
  	int s;
  	int server_port;
  	int timestamp = 0;
--- 162,168 ----
  	FILE *ofp = NULL;
  	char *p = NULL;
  	char *server_host = NULL;
! 	char *this_host = xstrdup(getfullhostname());
  	int s;
  	int server_port;
  	int timestamp = 0;
Index: harvest/src/gatherer/mgmt/Gatherer.in
diff -c harvest/src/gatherer/mgmt/Gatherer.in:2.5 harvest/src/gatherer/mgmt/Gatherer.in:2.6
*** harvest/src/gatherer/mgmt/Gatherer.in:2.5	Mon Sep 22 12:05:13 1997
--- harvest/src/gatherer/mgmt/Gatherer.in	Tue Jun  2 17:06:03 1998
***************
*** 7,13 ****
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.5 1997/09/22 16:05:13 sxw Exp
  #
  #############################################################################
  #
--- 7,13 ----
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.6 1998/06/02 21:06:03 sxw Exp
  #
  #############################################################################
  #
***************
*** 260,265 ****
--- 260,266 ----
  &run_system("/bin/csh -f $gcmd 2>> $vals{'Errorlog-File'}");	# actually run the Gatherer
  unlink($gcmd) if ($debug == 0);
  unlink($gcmdinput) if ($debug == 0);
+ unlink("$ENV{'TMPDIR'}/All-Files") if ( -e "$ENV{'TMPDIR'}/All-Files");
  &install_gatherer() if ($automatic == 1);
  exit(0);		# END OF PROGRAM
  
***************
*** 427,432 ****
--- 428,436 ----
  		chmod(0644, $f);
  	}
  	$ENV{'TMPDIR'} = $vals{"Working-Directory"};
+ 	$templatename = $vals{"Data-Directory"} . "/All-Templates.gz";
+ 	if (-e $templatename) {
+ 		system("$ENV{HARVEST_HOME}/bin/gzip -dc $templatename > $ENV{'TMPDIR'}/All-Files");}	
  
  	if (defined(%mapping)) {
  		$tfile = "$vals{'Working-Directory'}/localmap.cf";
