Index: harvest/ChangeLog
diff -c harvest/ChangeLog:2.6 harvest/ChangeLog:2.8
*** harvest/ChangeLog:2.6	Tue Apr 29 16:13:45 1997
--- harvest/ChangeLog	Fri May  2 22:15:38 1997
***************
*** 1,4 ****
--- 1,11 ----
  Changes in current version
+ 	- Fix for Year 2000 logging and file naming problem (logs
+ 	  should now be produced using 4 digits for the year) (Simon Wilkinson,
+ 	  suggested by Craig Counterman)
+ 	- Fixed gatherers so they don't revisit pages that they've previously
+ 	  found to be broken (Simon Wilkinson, suggested by David A. Nowitz)
+ 	- Fixed breadth first httpenum so it correctly handles META robots
+ 	  control tags (Simon Wilkinson)
  	- Rewrote Troff.sum in perl (David A. Nowitz)
  	- Added DOCTYPES and tidied up HTML generally (Craig Counterman)
  	- Fixed &times bug in &timestamp call (Craig Counterman)
Index: harvest/components/broker/standard/glimpse/agrep/agrep.c
diff -c harvest/components/broker/standard/glimpse/agrep/agrep.c:2.0 harvest/components/broker/standard/glimpse/agrep/agrep.c:2.1
*** harvest/components/broker/standard/glimpse/agrep/agrep.c:2.0	Sun Mar 16 21:57:58 1997
--- harvest/components/broker/standard/glimpse/agrep/agrep.c	Fri May  2 21:03:12 1997
***************
*** 4,9 ****
--- 4,10 ----
   * selectively modified many routines to work with our text-compression algo.
   */
  #include <sys/stat.h>
+ #include <sys/types.h>
  #include "agrep.h"
  #include "checkfile.h"
  #include <errno.h>
Index: harvest/src/broker/log.c
diff -c harvest/src/broker/log.c:2.1 harvest/src/broker/log.c:2.2
*** harvest/src/broker/log.c:2.1	Fri Mar 21 17:20:00 1997
--- harvest/src/broker/log.c	Fri May  2 21:34:44 1997
***************
*** 1,4 ****
! static char rcsid[] = "log.c,v 2.1 1997/03/21 17:20:00 sxw Exp";
  /*
   *  log.c -- Collects statistics information for the broker and logs it.
   *
--- 1,4 ----
! static char rcsid[] = "log.c,v 2.2 1997/05/02 20:34:44 sxw Exp";
  /*
   *  log.c -- Collects statistics information for the broker and logs it.
   *
***************
*** 326,332 ****
--- 326,336 ----
  
  #ifdef HAVE_STRFTIME
  	timebuf = xmalloc(BUFSIZ);
+ #ifdef Y2K_TIME
+ 	(void)strftime(timebuf, BUFSIZ, "LOG.%Y%m%d", localtime(&Cur_Time));
+ #else
  	(void)strftime(timebuf, BUFSIZ, "LOG.%y%m%d", localtime(&Cur_Time));
+ #endif	
  	f2 = UTIL_make_admin_filename(timebuf);
  #else
  	f2 = UTIL_make_admin_filename("LOG.old");
Index: harvest/src/common/include/config.h
diff -c harvest/src/common/include/config.h:2.1 harvest/src/common/include/config.h:2.2
*** harvest/src/common/include/config.h:2.1	Fri Mar 21 19:21:00 1997
--- harvest/src/common/include/config.h	Fri May  2 21:35:49 1997
***************
*** 1,7 ****
  /*
   *  config.h - Master configuration file for the Harvest system.
   * 
!  *  config.h,v 2.1 1997/03/21 19:21:00 sxw Exp
   *
   *  AUTHOR: Harvest derived
   *
--- 1,7 ----
  /*
   *  config.h - Master configuration file for the Harvest system.
   * 
!  *  config.h,v 2.2 1997/05/02 20:35:49 sxw Exp
   *
   *  AUTHOR: Harvest derived
   *
***************
*** 121,126 ****
--- 121,133 ----
  #endif
  
  /*
+  *  Y2K_TIME - define to log time in form YYYYMMDD rather than YYYMMDD
+  */
+ #ifndef Y2K_TIME
+ #define Y2K_TIME
+ #endif
+ 
+ /*
   *  USE_LOCAL_CACHE - define to use the Gatherer's local disk cache 
   */
  #ifndef USE_LOCAL_CACHE
***************
*** 250,255 ****
--- 257,269 ----
  #define HTTP_MAX_TRANSFER        (10<<20)
  #endif
  
+ /*
+  *  DONT_RETRY_FAILS - define to mark failed URLs as visited, and so
+  *  stop repeated attempts to fetch them
+  */
+ #ifndef DONT_RETRY_FAILS
+ #define DONT_RETRY_FAILS
+ #endif
  /* 
   *  NO_STRDUP - define if standard C library doesn't have strdup(3).
   */
Index: harvest/src/common/util/log.c
diff -c harvest/src/common/util/log.c:2.0 harvest/src/common/util/log.c:2.1
*** harvest/src/common/util/log.c:2.0	Sun Mar 16 22:16:29 1997
--- harvest/src/common/util/log.c	Fri May  2 21:36:35 1997
***************
*** 362,368 ****
--- 362,372 ----
  	t = time(NULL);
  	lt = localtime(&t);
  	if (lt != NULL) {
+ 	  #ifdef Y2K_TIME
+ 	    (void) strftime(buf + strlen(buf), 63, ": %Y%m%d %H:%M:%S", lt);
+           #else
  	    (void) strftime(buf + strlen(buf), 63, ": %y%m%d %H:%M:%S", lt);
+ 	  #endif
  	}
      }
  #endif
Index: harvest/src/gatherer/enumerate/gopherenum-breadth.c
diff -c harvest/src/gatherer/enumerate/gopherenum-breadth.c:2.1 harvest/src/gatherer/enumerate/gopherenum-breadth.c:2.3
*** harvest/src/gatherer/enumerate/gopherenum-breadth.c:2.1	Fri Mar 21 19:21:33 1997
--- harvest/src/gatherer/enumerate/gopherenum-breadth.c	Fri May  2 22:07:26 1997
***************
*** 1,4 ****
! static char rcsid[] = "gopherenum-breadth.c,v 2.1 1997/03/21 19:21:33 sxw Exp";
  /*
   *  gopherenum-breadth.c - RootNode URL enumerator for Gopher URLs
   *
--- 1,4 ----
! static char rcsid[] = "gopherenum-breadth.c,v 2.3 1997/05/02 21:07:26 sxw Exp";
  /*
   *  gopherenum-breadth.c - RootNode URL enumerator for Gopher URLs
   *
***************
*** 164,169 ****
--- 164,170 ----
  
  /* Local functions */
  static void usage();
+ static void mark_failed();
  static void mark_retrieved();
  static void sigdie();
  static int url_in_db();
***************
*** 200,205 ****
--- 201,228 ----
  
  /* ---------------------------------------------------------------------- */
  
+ /* 
+  *  mark_failed() - Mark that a URL failed to be retrieved, so that the
+  *  enumerator doesn't try it again. This option may not be wanted by
+  *  some users and so should be configurable.
+  */
+ static void mark_failed(URL *up) {
+     datum k,d;
+  
+     Debug(42, 9, ("mark_failed: url='%s'",up->url));
+  
+     k.dptr = xstrdup(up->url);
+     k.dsize = strlen(k.dptr) + 1;
+     d.dptr = xstrdup("FailedAccess");
+     d.dsize = strlen(d.dptr) + 1;
+ 
+     if (!gdbm_exists(urldbf, k) && gdbm_store(urldbf, k, d, GDBM_INSERT))
+         fatal("GDBM URLDB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+  
+     xfree(k.dptr);
+     xfree(d.dptr);
+ }
+ 
  /*
   *  mark_retrieved() - Mark that the given URL was successfully retrieved,
   *  so that the URL is not retrieved again.  This prevents cycles in the
***************
*** 394,399 ****
--- 417,427 ----
      }
      if (url_retrieve(up)) {	/* Grab the URL; success? */
  	Debug(43, 1, ("Cannot Retrieve URL: %s\n", up->url));
+ 
+ #ifdef DONT_RETRY_FAILS
+         mark_failed(up);
+ #endif
+ 
  	return 0;
      }
      if (up->md5 && md5_in_db(up->md5)) {	/* Have we been here? */
Index: harvest/src/gatherer/enumerate/gopherenum-depth.c
diff -c harvest/src/gatherer/enumerate/gopherenum-depth.c:2.1 harvest/src/gatherer/enumerate/gopherenum-depth.c:2.3
*** harvest/src/gatherer/enumerate/gopherenum-depth.c:2.1	Fri Mar 21 19:21:35 1997
--- harvest/src/gatherer/enumerate/gopherenum-depth.c	Fri May  2 22:07:29 1997
***************
*** 1,4 ****
! static char rcsid[] = "gopherenum-depth.c,v 2.1 1997/03/21 19:21:35 sxw Exp";
  /*
   *  gopherenum-depth.c - Depth First RootNode URL enumerator for Gopher URLs
   *
--- 1,4 ----
! static char rcsid[] = "gopherenum-depth.c,v 2.3 1997/05/02 21:07:29 sxw Exp";
  /*
   *  gopherenum-depth.c - Depth First RootNode URL enumerator for Gopher URLs
   *
***************
*** 155,160 ****
--- 155,161 ----
  /* Local functions */
  static void process_url();
  static void usage();
+ static void mark_failed();
  static void mark_retrieved();
  static void sigdie();
  static int url_in_db();
***************
*** 162,167 ****
--- 163,190 ----
  
  /* ---------------------------------------------------------------------- */
  
+ /* 
+  *  mark_failed() - Mark that a URL failed to be retrieved, so that the
+  *  enumerator doesn't try it again. This option may not be wanted by
+  *  some users and so should be configurable.
+  */
+ static void mark_failed(URL *up) {
+     datum k,d;
+  
+     Debug(42, 9, ("mark_failed: url='%s'",up->url));
+  
+     k.dptr = xstrdup(up->url);
+     k.dsize = strlen(k.dptr) + 1;
+     d.dptr = xstrdup("FailedAccess");
+     d.dsize = strlen(d.dptr) + 1;
+  
+     if (!gdbm_exists(urldbf, k) && gdbm_store(urldbf, k, d, GDBM_INSERT))
+         fatal("GDBM URLDB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+  
+     xfree(k.dptr);
+     xfree(d.dptr);
+ }
+ 
  /*
   *  mark_retrieved() - Mark that the given URL was successfully retrieved,
   *  so that the URL is not retrieved again.  This prevents cycles in the
***************
*** 321,326 ****
--- 344,354 ----
  
      if (url_retrieve(up)) {	/* Grab the URL; success? */
  	Debug(43, 1, ("Cannot Retrieve URL: %s\n", up->url));
+ 
+ #ifdef DONT_RETRY_FAILS
+ 	mark_failed(up);
+ #endif
+ 
  	return (NULL);
      }
      if (up->md5 && md5_in_db(up->md5)) {	/* Have we been here? */
Index: harvest/src/gatherer/enumerate/httpenum-breadth.c
diff -c harvest/src/gatherer/enumerate/httpenum-breadth.c:2.2 harvest/src/gatherer/enumerate/httpenum-breadth.c:2.5
*** harvest/src/gatherer/enumerate/httpenum-breadth.c:2.2	Tue Mar 25 00:21:18 1997
--- harvest/src/gatherer/enumerate/httpenum-breadth.c	Fri May  2 22:07:31 1997
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-breadth.c,v 2.2 1997/03/25 00:21:18 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-breadth.c,v 2.5 1997/05/02 21:07:31 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
***************
*** 174,179 ****
--- 174,180 ----
  
  /* Local functions */
  static void usage();
+ static void mark_failed();
  static void mark_retrieved();
  static void sigdie();
  static int url_in_db();
***************
*** 242,247 ****
--- 243,270 ----
  
  /* ---------------------------------------------------------------------- */
  
+ /* 
+  *  mark_failed() - Mark that a URL failed to be retrieved, so that the
+  *  enumerator doesn't try it again. This option may not be wanted by
+  *  some users and so should be configurable.
+  */
+ static void mark_failed(URL *up) {
+     datum k,d;
+  
+     Debug(42, 9, ("mark_failed: url='%s'",up->url));
+  
+     k.dptr = xstrdup(up->url);
+     k.dsize = strlen(k.dptr) + 1;
+     d.dptr = xstrdup("FailedAccess");
+     d.dsize = strlen(d.dptr) + 1;
+  
+     if (!gdbm_exists(urldbf, k) && gdbm_store(urldbf, k, d, GDBM_INSERT))
+         fatal("GDBM URLDB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+  
+     xfree(k.dptr);
+     xfree(d.dptr);
+ }
+ 
  /*
   *  mark_retrieved() - Mark that the given URL was successfully retrieved,
   *  so that the URL is not retrieved again.  This prevents cycles in the
***************
*** 282,290 ****
  	xfree(k.dptr);
  	xfree(d.dptr);
      }
-     /* Print URL to stdout to enumerate; flush to keep pipe moving */
-     fprintf(stdout, "%s\t%s\n", up->url, up->md5);	/* URL <tab> MD5 */
-     fflush(stdout);
  
      if (++nurls >= url_max) {
  	Log("Truncating RootNode %s at %d LeafNode URLs\n",
--- 305,310 ----
***************
*** 485,490 ****
--- 505,515 ----
  
      if (status) {	/* Grab the URL; success? */
  	Debug(42, 1, ("Cannot Retrieve URL: %s\n", up->url));
+ 
+ #ifdef DONT_RETRY_FAILS
+ 	mark_failed(up);
+ #endif
+ 
  	return 0;
      }
      if (up->md5 && md5_in_db(up->md5)) {	/* Have we been here? */
***************
*** 497,504 ****
  
      /* Are we dealing with an HTML file, if not we can't get href links */
      if (up->http_mime_hdr != NULL &&
! 	strstr(up->http_mime_hdr, "text/html") == NULL)
  	return 0;
  
      /* Extract the HREF's */
  
--- 522,532 ----
  
      /* Are we dealing with an HTML file, if not we can't get href links */
      if (up->http_mime_hdr != NULL &&
! 	strstr(up->http_mime_hdr, "text/html") == NULL) {
!         fprintf(stdout, "%s\t%s\n", up->url, up->md5);	/* URL <tab> MD5 */
!         fflush(stdout);
  	return 0;
+     }
  
      /* Extract the HREF's */
  
***************
*** 552,561 ****
      }
      fclose(fp);
      close(pipefds[0]);
!     if ((err = waitpid(pid, NULL, NULL)) != pid) {
  	Debug(42, 1, ("WARNING: waiting for child %d got %d...\n",
  		pid, err));
      }
      Debug(42, 1, ("Adding %d URLs from %s to workload\n", nurls, up->url));
      return 1;
  }
--- 580,600 ----
      }
      fclose(fp);
      close(pipefds[0]);
! 
!     /* Wait for HTMLurls to finish and decide on whether to index the page
!      * according to its exit code 
!      */
!     if ((err = waitpid(pid, &status, 0)) != pid) {
  	Debug(42, 1, ("WARNING: waiting for child %d got %d...\n",
  		pid, err));
      }
+ 
+     /* If we had an exit code of 0 then index it, otherwise don't */
+     if (WEXITSTATUS(status) == 0) {
+       fprintf(stdout, "%s\t%s\n", up->url, up->md5);
+       fflush(stdout);
+     }
+ 
      Debug(42, 1, ("Adding %d URLs from %s to workload\n", nurls, up->url));
      return 1;
  }
Index: harvest/src/gatherer/enumerate/httpenum-depth.c
diff -c harvest/src/gatherer/enumerate/httpenum-depth.c:2.3 harvest/src/gatherer/enumerate/httpenum-depth.c:2.5
*** harvest/src/gatherer/enumerate/httpenum-depth.c:2.3	Tue Mar 25 00:21:22 1997
--- harvest/src/gatherer/enumerate/httpenum-depth.c	Fri May  2 22:07:33 1997
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-depth.c,v 2.3 1997/03/25 00:21:22 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-depth.c,v 2.6 1997/05/05 03:18:26 sxw Exp";
  /*
   *  httpenum-depth.c - Depth First RootNode URL enumerator for HTTP URLs
   *
***************
*** 163,168 ****
--- 163,169 ----
  static void process_url();
  static void usage();
  static void mark_retrieved();
+ static void mark_failed();
  static void sigdie();
  static int url_in_db();
  static int md5_in_db();
***************
*** 173,178 ****
--- 174,201 ----
  
  /* ---------------------------------------------------------------------- */
  
+ /* 
+  *  mark_failed() - Mark that a URL failed to be retrieved, so that the
+  *  enumerator doesn't try it again. This option may not be wanted by
+  *  some users and so should be configurable.
+  */
+ static void mark_failed(URL *up) {
+     datum k,d;
+ 
+     Debug(42, 9, ("mark_failed: url='%s'",up->url));
+ 
+     k.dptr = xstrdup(up->url);
+     k.dsize = strlen(k.dptr) + 1;
+     d.dptr = xstrdup("FailedAccess");
+     d.dsize = strlen(d.dptr) + 1;
+ 
+     if (!gdbm_exists(urldbf, k) && gdbm_store(urldbf, k, d, GDBM_INSERT))
+       fatal("GDBM URLDB: %s: %s", k.dptr, gdbm_strerror(gdbm_errno));
+ 
+     xfree(k.dptr);
+     xfree(d.dptr);
+ }
+ 
  /*
   *  mark_retrieved() - Mark that the given URL was successfully retrieved,
   *  so that the URL is not retrieved again.  This prevents cycles in the
***************
*** 407,412 ****
--- 417,427 ----
  
      if (status) {	/* Grab the URL; success? */
  	Debug(42, 1, ("Cannot Retrieve URL: %s\n", up->url));
+ 
+ #ifdef DONT_RETRY_FAILS
+ 	mark_failed(up);
+ #endif
+ 
  	return (NULL);
      }
      if (up->md5 && md5_in_db(up->md5)) {	/* Have we been here? */
