Index: harvest/CONTRIBUTORS
diff -c harvest/CONTRIBUTORS:1.1 harvest/CONTRIBUTORS:1.2
*** harvest/CONTRIBUTORS:1.1	Tue Jun  2 18:34:00 1998
--- harvest/CONTRIBUTORS	Tue Jun  2 19:58:24 1998
***************
*** 21,26 ****
--- 21,27 ----
  Bruce R. Lewis
  Dean Marino
  David A. Nowitz
+ Judith Pluemer
  Andy Powell
  Tim Riker
  Dan Riley
Index: harvest/ChangeLog
diff -c harvest/ChangeLog:2.53 harvest/ChangeLog:2.55
*** harvest/ChangeLog:2.53	Tue Jun  2 18:23:49 1998
--- harvest/ChangeLog	Wed Jun 10 10:47:59 1998
***************
*** 1,4 ****
--- 1,10 ----
  Changes in current version
+ 	- Added support for setting transfer timeout times through
+ 	  envirnoment variables, and config file settings (Peter Scott)
+         - Fixed problem with broker registry not correctly handling deletion
+ 	  of duplicate objects (Hrvoje Stipetic)
+ 	- Added support for relative URLs in client pull inforamtion, and
+ 	  for the HTML refresh directive. (Hrvoje Stipetic)
          - Change the content of the "description" attribute to be the first
  	  255 characters of the objects "body" or "partial-text" attribute
  	  (Peter Valkenburg)
Index: harvest/src/broker/Makefile.in
diff -c harvest/src/broker/Makefile.in:2.8 harvest/src/broker/Makefile.in:2.9
*** harvest/src/broker/Makefile.in:2.8	Tue Jun  2 18:42:47 1998
--- harvest/src/broker/Makefile.in	Wed Jun 10 10:32:04 1998
***************
*** 1,7 ****
  #
  #  Makefile for the Harvest Broker
  #
! #  Makefile.in,v 2.8 1998/06/02 22:42:47 sxw Exp
  #
  
  srcdir = @srcdir@
--- 1,7 ----
  #
  #  Makefile for the Harvest Broker
  #
! #  Makefile.in,v 2.9 1998/06/10 14:32:04 sxw Exp
  #
  
  srcdir = @srcdir@
***************
*** 48,54 ****
  		  -I$(srcdir)
  CFLAGS		= $(DEBUG) $(INCLUDE)
  LDFLAGS		= -L../common/lib
! LIBS		= -ltemplate -lutil $(DEBUG_LIBS) $(LEXLIB) $(XTRA_LIBS)
  
  PROGS		= broker 
  UTILS		= dumpregistry brkclient 
--- 48,54 ----
  		  -I$(srcdir)
  CFLAGS		= $(DEBUG) $(INCLUDE)
  LDFLAGS		= -L../common/lib
! LIBS		= -ltemplate $(DEBUG_LIBS) $(LEXLIB) $(XTRA_LIBS) -lutil
  
  PROGS		= broker 
  UTILS		= dumpregistry brkclient 
Index: harvest/src/broker/registry.c
diff -c harvest/src/broker/registry.c:2.3 harvest/src/broker/registry.c:2.4
*** harvest/src/broker/registry.c:2.3	Tue Jun  2 10:11:50 1998
--- harvest/src/broker/registry.c	Wed Jun 10 10:33:54 1998
***************
*** 1,4 ****
! static char rcsid[] = "registry.c,v 2.3 1998/06/02 14:11:50 sxw Exp";
  /* 
   *  registry.c -- Registry Module, Keeps track of all objects in the Broker.
   *
--- 1,4 ----
! static char rcsid[] = "registry.c,v 2.4 1998/06/10 14:33:54 sxw Exp";
  /* 
   *  registry.c -- Registry Module, Keeps track of all objects in the Broker.
   *
***************
*** 547,553 ****
  	back_fn = UTIL_make_admin_filename("Registry.old");
  	old_fn = UTIL_make_admin_filename("Registry");
  	new_fn = UTIL_make_admin_filename("Registry.compressing");
! 	finish_registry_file();
  	if (set_registry_file(new_fn) != SUCCESS) {
  		Log("An error occured while trying to open Registry.compressing .\n");
  		goto reinstate_reg;
--- 547,553 ----
  	back_fn = UTIL_make_admin_filename("Registry.old");
  	old_fn = UTIL_make_admin_filename("Registry");
  	new_fn = UTIL_make_admin_filename("Registry.compressing");
!       RG_Registry_Shutdown();
  	if (set_registry_file(new_fn) != SUCCESS) {
  		Log("An error occured while trying to open Registry.compressing .\n");
  		goto reinstate_reg;
***************
*** 810,816 ****
  			if (!RG_Object_URL_Match(entry,tmp->item)) continue;
  			if (tmp->item->update_time <= entry->update_time) {
  				Debug (70,1,("RG_Uniqify: Removing duplicate: %s\n", tmp->item->url));
! 				RG_Unregister (tmp->item);
  				n++;
  			}
  		}
--- 810,816 ----
  			if (!RG_Object_URL_Match(entry,tmp->item)) continue;
  			if (tmp->item->update_time <= entry->update_time) {
  				Debug (70,1,("RG_Uniqify: Removing duplicate: %s\n", tmp->item->url));
! 				RG_Clean_Entry(tmp->item);
  				n++;
  			}
  		}
Index: harvest/src/common/include/version.h
diff -c harvest/src/common/include/version.h:2.5 harvest/src/common/include/version.h:2.6
*** harvest/src/common/include/version.h:2.5	Wed Feb  4 12:29:24 1998
--- harvest/src/common/include/version.h	Wed Jun 10 12:29:48 1998
***************
*** 3,7 ****
   */
  #ifndef HARVEST_VERSION
  /* UPDATE VERSION HERE */
! #define HARVEST_VERSION	"1.5.17"
  #endif
--- 3,7 ----
   */
  #ifndef HARVEST_VERSION
  /* UPDATE VERSION HERE */
! #define HARVEST_VERSION	"1.5.19"
  #endif
Index: harvest/src/common/url/ftpget.c
diff -c harvest/src/common/url/ftpget.c:2.1 harvest/src/common/url/ftpget.c:2.2
*** harvest/src/common/url/ftpget.c:2.1	Thu Jun 12 21:07:54 1997
--- harvest/src/common/url/ftpget.c	Wed Jun 10 10:48:01 1998
***************
*** 1,4 ****
! static char rcsid[] = "ftpget.c,v 1.5 1996/01/05 20:28:25 duane Exp";
  /* 
   *  ftpget.c - An FTP client used by liburl and cached.
   *
--- 1,4 ----
! static char rcsid[] = "ftpget.c,v 2.2 1998/06/10 14:48:01 sxw Exp";
  /* 
   *  ftpget.c - An FTP client used by liburl and cached.
   *
***************
*** 1621,1626 ****
--- 1621,1628 ----
  	int httpify_flag = 0;
  	int rc;
  
+ 	if ((t = getenv ("HARVEST_XFER_TIMEOUT")) !=NULL)
+ 		timeout = atol (t);
  	if ((t = strrchr(argv[0], '/'))) {
  		progname = xstrdup(t + 1);
  	} else
Index: harvest/src/common/url/http.c
diff -c harvest/src/common/url/http.c:2.8 harvest/src/common/url/http.c:2.9
*** harvest/src/common/url/http.c:2.8	Tue Jun  2 17:05:59 1998
--- harvest/src/common/url/http.c	Wed Jun 10 10:48:02 1998
***************
*** 1,4 ****
! static char rcsid[] = "http.c,v 2.8 1998/06/02 21:05:59 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
--- 1,4 ----
! static char rcsid[] = "http.c,v 2.9 1998/06/10 14:48:02 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
***************
*** 378,386 ****
  		if (n == 0)
  			break;	/* nothing left to do */
  		if (n < 0) {
! 			if (read_timeout == 1)
  				errorlog("HTTP timeout: %s:%d (%d seconds).\n",
! 				    up->host, up->port, XFER_TIMEOUT);
  			else
  				errorlog("HTTP read: %s:%d failed.\n",
  				    up->host, up->port);
--- 378,390 ----
  		if (n == 0)
  			break;	/* nothing left to do */
  		if (n < 0) {
! 			if (read_timeout == 1) {
! 				char *t =NULL;
! 				int to = XFER_TIMEOUT;
!                                 if ((t = getenv("HARVEST_XFER_TIMEOUT")) != NULL)
!                                 	to = atol (t);
  				errorlog("HTTP timeout: %s:%d (%d seconds).\n",
! 				        up->host, up->port, to);
  			else
  				errorlog("HTTP read: %s:%d failed.\n",
  				    up->host, up->port);
***************
*** 685,693 ****
  
  	/* read until timeout or amount of requested bytes read */
  	while (1) {
  		FD_ZERO(&readDetect);
  		FD_SET(s, &readDetect);
! 		timeout.tv_sec = XFER_TIMEOUT;
  		timeout.tv_usec = 0;
  
  		/* wait for data for seconds */
--- 689,701 ----
  
  	/* read until timeout or amount of requested bytes read */
  	while (1) {
+ 		char *t = NULL;
+ 		int to = XFER_TIMEOUT;
+ 		if ((t = getenv("HARVEST_XFER_TIMEOUT")) != NULL)
+ 			to = atol (t);
  		FD_ZERO(&readDetect);
  		FD_SET(s, &readDetect);
! 		timeout.tv_sec = to;
  		timeout.tv_usec = 0;
  
  		/* wait for data for seconds */
Index: harvest/src/common/url/newsget.pl
diff -c harvest/src/common/url/newsget.pl:2.0 harvest/src/common/url/newsget.pl:2.1
*** harvest/src/common/url/newsget.pl:2.0	Sun Mar 16 17:16:01 1997
--- harvest/src/common/url/newsget.pl	Wed Jun 10 10:48:03 1998
***************
*** 87,93 ****
  
  $debug	= 0;
  
! $XFER_TIMEOUT = 120;	# timeout of 2 minutes between reads
  alarm (300);		# limit process to 5 minutes until start reading
  
  $nntp_sock = -1;
--- 87,93 ----
  
  $debug	= 0;
  
! $XFER_TIMEOUT = $ENV{'HARVEST_XFER_TIMEOUT'} || 120;	# timeout of 2 minutes between reads
  alarm (300);		# limit process to 5 minutes until start reading
  
  $nntp_sock = -1;
Index: harvest/src/common/url/url.c
diff -c harvest/src/common/url/url.c:2.5 harvest/src/common/url/url.c:2.6
*** harvest/src/common/url/url.c:2.5	Tue Jun  2 17:06:00 1998
--- harvest/src/common/url/url.c	Wed Jun 10 10:33:55 1998
***************
*** 1,4 ****
! static char rcsid[] = "url.c,v 2.5 1998/06/02 21:06:00 sxw Exp";
  /*
   *  url.c - URL processing code 
   *
--- 1,4 ----
! static char rcsid[] = "url.c,v 2.6 1998/06/10 14:33:55 sxw Exp";
  /*
   *  url.c - URL processing code 
   *
***************
*** 131,136 ****
--- 131,138 ----
  int liburl_sleep_time = 1;
  #ifdef USE_LOCAL_CACHE
  int use_local_cache = 1;
+ #else
+ int use_local_cache = 0;
  #endif
  
  /* Local Functions */
Index: harvest/src/gatherer/enumerate/HTMLurls.c
diff -c harvest/src/gatherer/enumerate/HTMLurls.c:2.3 harvest/src/gatherer/enumerate/HTMLurls.c:2.4
*** harvest/src/gatherer/enumerate/HTMLurls.c:2.3	Mon Oct 20 12:43:46 1997
--- harvest/src/gatherer/enumerate/HTMLurls.c	Wed Jun 10 10:33:57 1998
***************
*** 1,4 ****
! static char rcsid[] = "HTMLurls.c,v 2.3 1997/10/20 16:43:46 sxw Exp";
  /* 
   *  HTMLurls - Prints all of the URLs from an HTML file.
   *
--- 1,4 ----
! static char rcsid[] = "HTMLurls.c,v 2.4 1998/06/10 14:33:57 sxw Exp";
  /* 
   *  HTMLurls - Prints all of the URLs from an HTML file.
   *
***************
*** 237,249 ****
  
  /*
   * process_metapull - extract the URL from the META tag for client pull
!  * <META HTTP-EQUIV="REFRESH" CONTENT="<secs>; URL=http://www.some.dom/">
!  * HS, 20-Oct-1997
   */
  void process_metapull(s)
       char *s;
  {
!     char *p, *q, *tmps;
  
      /* Find the HTTP-EQUIV in the META */
      if ((tmps = strstr_icase(s, "http-equiv")) == NULL)
--- 237,250 ----
  
  /*
   * process_metapull - extract the URL from the META tag for client pull
!  * <META HTTP-EQUIV="REFRESH" CONTENT="<secs>; URL=URL">	Netscape 1.1
!  * <META HTTP-EQUIV="REFRESH" CONTENT="<secs>,URI">		W3C, HTML 4.0 (REC-html40-19980424)
!  * HS, 20-Oct-1997, 03-May-1998
   */
  void process_metapull(s)
       char *s;
  {
!     char *p, *q, *tmps, *v;
  
      /* Find the HTTP-EQUIV in the META */
      if ((tmps = strstr_icase(s, "http-equiv")) == NULL)
***************
*** 275,287 ****
      if ((p = strchr(tmps, '=')) == NULL)    /* Find equals */
        return;
  
!     if ((tmps = strstr_icase(p, "url")) == NULL) /* Find 'URL' */
!       return;
!     if ((p = strchr(tmps, '=')) == NULL)    /* Find equals */
        return;
!     p++;                                    /* skip '=' */
!     while (isspace(*p))
!       p++;                                  /* skip spaces */
  
      q = xstrdup(p);                         /* copy name */
      if ((p = strchr(q, '\"')) != NULL) {    /* terminate string on '"' */
--- 276,300 ----
      if ((p = strchr(tmps, '=')) == NULL)    /* Find equals */
        return;
  
!     if ((p = strpbrk(p, ",;")) == NULL)     /* There is no URL in this refresh, not interesting */
        return;
! 
!     p++;				    /* Skip delimiter, ';' or ',' */
!     while (isspace(*p)) p++;                /* skip spaces */
! 
!     /* p is now first char of something, it may be 'URL=' or the URL itself */
!     q = p;	/* save p for later */
! 
!     /* If there is URL=, skip it */
!     if (strncasecmp(p, "url", 3) == 0) { /* there is URL */
!       q += 3;
!       while (isspace(*q)) q++;          /* skip spaces */
!       if (*q == '=') { 			/* if there is URL = , skip it */
!         q++;                            /* skip '=' */
!         while (isspace(*q)) q++;          /* skip spaces */
! 	p = q;
!       } /* else - this is HTML 4.0 conformant (?) refresh with (relative) URL starting with letters 'url' */
!     }
  
      q = xstrdup(p);                         /* copy name */
      if ((p = strchr(q, '\"')) != NULL) {    /* terminate string on '"' */
***************
*** 292,297 ****
--- 305,317 ----
      }
  
      /* q should now contain the URL */
+ 
+     /* Relative URLs are processed by various browsers, so enable them */
+     if (base != (char *) NULL) {
+       v = q;
+       q = url_parse_relative(v, base);
+       xfree(v);
+     }
  
      if (q != (char *) NULL) {             /* Just in case... */
        add_buffer(urls, q, strlen(q));     /* Add URL to urls */
Index: harvest/src/gatherer/enumerate/httpenum-breadth.c
diff -c harvest/src/gatherer/enumerate/httpenum-breadth.c:2.7 harvest/src/gatherer/enumerate/httpenum-breadth.c:2.8
*** harvest/src/gatherer/enumerate/httpenum-breadth.c:2.7	Tue Jul  1 15:29:58 1997
--- harvest/src/gatherer/enumerate/httpenum-breadth.c	Wed Jun 10 10:33:58 1998
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-breadth.c,v 2.7 1997/07/01 19:29:58 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-breadth.c,v 2.8 1998/06/10 14:33:58 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
***************
*** 699,704 ****
--- 699,705 ----
      xfree(md5db_filename);
  
      Log("Statistics for URL '%s':\n", tree_root);
+     while (depth_hist[max_depth] == 0) max_depth--;
      for (i = 0, sum = 0; i < 100; i++) {
  	if (i > max_depth && depth_hist[i] == 0)
  	    break;
***************
*** 756,761 ****
--- 757,764 ----
      /* Mark the RootNode */
      tree_root = xstrdup(up->url);
      Tail = &head;
+ 
+     Log("Processing URL '%s':\n", tree_root);
  
      printf("%s\n", up->url);	/* Print tree root */
      /*
Index: harvest/src/gatherer/enumerate/httpenum-depth.c
diff -c harvest/src/gatherer/enumerate/httpenum-depth.c:2.7 harvest/src/gatherer/enumerate/httpenum-depth.c:2.8
*** harvest/src/gatherer/enumerate/httpenum-depth.c:2.7	Tue Jun  2 17:06:02 1998
--- harvest/src/gatherer/enumerate/httpenum-depth.c	Mon Jun  8 14:08:28 1998
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-depth.c,v 2.7 1998/06/02 21:06:02 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-depth.c,v 2.8 1998/06/08 18:08:28 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
***************
*** 463,469 ****
        /* Flush the URL, but as a SOIF one */
        fprintf(stdout, "SOIF:%s\t%s\n", up->url, up->md5);
  
!       ptr=strtok(refs, '\n');
        while(ptr!=NULL) {
  	/* Make a link in the list for this URL */
  	l = (list_t *) xmalloc(sizeof(list_t));
--- 463,469 ----
        /* Flush the URL, but as a SOIF one */
        fprintf(stdout, "SOIF:%s\t%s\n", up->url, up->md5);
  
!       ptr=strtok(refs, "\n");
        while(ptr!=NULL) {
  	/* Make a link in the list for this URL */
  	l = (list_t *) xmalloc(sizeof(list_t));
***************
*** 474,480 ****
  	
  	nurls++;
  
! 	ptr=strtok(NULL, '/n');
        }
        xfree(refs);
      }
--- 474,480 ----
  	
  	nurls++;
  
! 	ptr=strtok(NULL, "\n");
        }
        xfree(refs);
      }
Index: harvest/src/gatherer/mgmt/Gatherer.in
diff -c harvest/src/gatherer/mgmt/Gatherer.in:2.6 harvest/src/gatherer/mgmt/Gatherer.in:2.7
*** harvest/src/gatherer/mgmt/Gatherer.in:2.6	Tue Jun  2 17:06:03 1998
--- harvest/src/gatherer/mgmt/Gatherer.in	Wed Jun 10 10:48:05 1998
***************
*** 7,13 ****
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.6 1998/06/02 21:06:03 sxw Exp
  #
  #############################################################################
  #
--- 7,13 ----
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.7 1998/06/10 14:48:05 sxw Exp
  #
  #############################################################################
  #
***************
*** 196,201 ****
--- 196,202 ----
  		"Refresh-Rate",
  		"Time-To-Live",
  	        "Top-Directory",
+ 		"Transfer-Timeout",
  	        "User-Agent",
  		"Working-Directory",
  	);
***************
*** 470,475 ****
--- 471,480 ----
          if (defined($vals{"User-Agent"})) {
                  $ENV{'HARVEST_USER_AGENT'} = $vals{"User-Agent"};
          }
+ 
+ 	if (defined($vals{"Transfer-Timeout"})) {
+ 		$ENV{'HARVEST_XFER_TIMEOUT'} = $vals{'Transfer-Timeout"};
+ 	}
  
          if (defined($vals{"Maintainer"})) {
                  $ENV{'HARVEST_MAINTAINER_ADDRESS'} = $vals{"Maintainer"};
