Index: harvest/ChangeLog
diff -c harvest/ChangeLog:2.18 harvest/ChangeLog:2.21
*** harvest/ChangeLog:2.18	Mon Jun 16 21:51:56 1997
--- harvest/ChangeLog	Thu Jul 10 19:49:27 1997
***************
*** 1,4 ****
--- 1,20 ----
  Changes in current version
+         - Replaced all of the HSR tools with new, fixed versions supplied by
+ 	  Mic Bowman
+ 	- Added changes to SGML.sum to make it compatible with nsgmls, part
+ 	  of SP. Error reports now on by default (Craig Counterman)
+ 	- Altered broker/Glimpse/index.c so that case and word matching are
+ 	  disabled when an errorflag is provided (Craig Counterman)
+ 	- Fixed broker/Glimpse/index.c so it compiles on an SGI (Simon Woods)
+ 	- Altered http code so it now sends a user controllable User-Agent
+ 	  and Maintainer address which can be set in the config file
+ 	  (Hrvoje Stipetic)
+ 	- Altered httpenum-breadth so that the count of objects retrieved
+ 	  is accurate (Hrvoje Stipetic)
+ 	- Altered summarisers Makefile so that Pdf.sum is now installed
+ 	  by default (Simon Wilkinson)
+ 	- Added -DAGREP_POINTER=1 to glimpse Makefiles - fixes segmentation
+ 	  faults with picky mallocs (Dan Riley)
  	- Replaced HTMLparse with a newer version from the Mosaic 2.7b5
  	  which handles comments correctly (Simon Wilkinson)
          - Changed Postscript.sum to a sh script, so stderr doesn't corrupt the
Index: harvest/components/broker/standard/glimpse/Makefile.in
diff -c harvest/components/broker/standard/glimpse/Makefile.in:2.2 harvest/components/broker/standard/glimpse/Makefile.in:2.3
*** harvest/components/broker/standard/glimpse/Makefile.in:2.2	Thu May  8 17:03:18 1997
--- harvest/components/broker/standard/glimpse/Makefile.in	Mon Jun 30 20:09:39 1997
***************
*** 32,38 ****
  COMPRESSDIR	= compress
  COMPRESSSRC	= $(srcdir)/$(COMPRESSDIR)
  
! CFLAGS		= @CFLAGS@ -DSTRUCTURED_QUERIES=1 -DISO_CHAR_SET $(DEBUG_TOP)
  ALL_CFLAGS      = $(CFLAGS) $(DEFS) -I$(INDEXSRC) -I$(AGREPSRC) -I$(TEMPLATESRC)/include
  LINKFLAGS	= @LDFLAGS@ -I$(INDEXSRC) -I$(AGREPSRC) -I$(TEMPLATESRC)/include
  
--- 32,38 ----
  COMPRESSDIR	= compress
  COMPRESSSRC	= $(srcdir)/$(COMPRESSDIR)
  
! CFLAGS		= @CFLAGS@ -DSTRUCTURED_QUERIES=1 -DISO_CHAR_SET=1 -DAGREP_POINTER=1 $(DEBUG_TOP)
  ALL_CFLAGS      = $(CFLAGS) $(DEFS) -I$(INDEXSRC) -I$(AGREPSRC) -I$(TEMPLATESRC)/include
  LINKFLAGS	= @LDFLAGS@ -I$(INDEXSRC) -I$(AGREPSRC) -I$(TEMPLATESRC)/include
  
Index: harvest/components/gatherer/standard/SGML/SGML.sum
diff -c harvest/components/gatherer/standard/SGML/SGML.sum:2.1 harvest/components/gatherer/standard/SGML/SGML.sum:2.3
*** harvest/components/gatherer/standard/SGML/SGML.sum:2.1	Mon Jun  9 18:51:05 1997
--- harvest/components/gatherer/standard/SGML/SGML.sum	Thu Jul 10 18:48:26 1997
***************
*** 88,94 ****
  
  $start_t	= time;
  
! $syntax_check	= 0;	# set to 1 if you want sgmls to print error messages
  $debug		= 0;
  $print_times    = 0;	# set to 1 if you want SGML.sum to print time info
  $print_times    = 1 if ($debug);
--- 88,94 ----
  
  $start_t	= time;
  
! $syntax_check	= 1;	# set to 1 if you want sgmls to print error messages
  $debug		= 0;
  $print_times    = 0;	# set to 1 if you want SGML.sum to print time info
  $print_times    = 1 if ($debug);
***************
*** 167,173 ****
  #
  #    <!DOCTYPE HTML SYSTEM>
  #
! if ($sgml =~ /^\s*<!DOCTYPE\s+(\S+)\s+(\S+)\s+("[^"]+")\s+"([^"]+)"\s*>/i) {
          $xdoctype = $1;
          $dtdfile = $4;
          $sgml =~ s/^\s*<!DOCTYPE[^>]+>/<!DOCTYPE $xdoctype SYSTEM>/i
--- 167,173 ----
  #
  #    <!DOCTYPE HTML SYSTEM>
  #
! if ($sgml =~ /^\s*<!DOCTYPE\s+(\S+)\s+(\S+)\s+(\"[^\"]+\")\s+\"([^\"]+)\"\s*>/i) {
          $xdoctype = $1;
          $dtdfile = $4;
          $sgml =~ s/^\s*<!DOCTYPE[^>]+>/<!DOCTYPE $xdoctype SYSTEM>/i
***************
*** 182,194 ****
  #  but NOT:
  #  <!DOCTYPE HTML PUBLIC "-//Sun Micorsystems Corp.//DTD HotJava HTML//EN">
  #
! if ($sgml =~ /^\s*<!DOCTYPE\s+(\S+)\s+(\S+)\s+"([^-][^"]+)"\s*>/i) {
!         $xdoctype = $1;
!         $dtdfile = $3;
!         $sgml =~ s/^\s*<!DOCTYPE[^>]+>/<!DOCTYPE $xdoctype SYSTEM>/i
!                 unless ( -f $dtdfile );
! 	undef $xdoctype;
! 	undef $dtdfile;
  }
  
  print T "<!DOCTYPE $doctype SYSTEM>\n"
--- 182,202 ----
  #  but NOT:
  #  <!DOCTYPE HTML PUBLIC "-//Sun Micorsystems Corp.//DTD HotJava HTML//EN">
  #
! # or  <!DOCTYPE HTML SYSTEM "http:...">
! if ($sgml =~ /^\s*<!DOCTYPE\s+(\S+)\s+(\S+)\s+\"(http:\/\/[^\"]+)\"\s*>/i) {
!     $xdoctype = $1;
!     $dtdfile = $3;
!     $sgml =~ s/^\s*<!DOCTYPE[^>]+>/<!DOCTYPE $xdoctype SYSTEM "$dtdfile">/i
! 	unless ( -f $dtdfile );
!     undef $xdoctype;
!     undef $dtdfile;
! } elsif ($sgml =~ /^\s*<!DOCTYPE\s+(\S+)\s+(\S+)\s+\"([^-][^\"]+)\"\s*>/i) {
!     $xdoctype = $1;
!     $dtdfile = $3;
!     $sgml =~ s/^\s*<!DOCTYPE[^>]+>/<!DOCTYPE $xdoctype SYSTEM>/i
! 	unless ( -f $dtdfile );
!     undef $xdoctype;
!     undef $dtdfile;
  }
  
  print T "<!DOCTYPE $doctype SYSTEM>\n"
***************
*** 200,212 ****
  # Open the pipe from 'sgmls' for reading...
  #
  if ($syntax_check) {
! 	$cmd = "$sgmls_cmd -m $catalog $decl $T";
  } else {
! 	$cmd = "$sgmls_cmd -f /dev/null -m $catalog $decl $T";
  }
! open (SGMLS, "$cmd|")   || (unlink ($T), die "$cmd: $!\n");
  $foo = &parse_sgml ('NIL', ());
  close SGMLS;
  unlink ($T) unless ($debug);
  
  
--- 208,233 ----
  # Open the pipe from 'sgmls' for reading...
  #
  if ($syntax_check) {
!     $Terr = &temperrnam;
!     $cmd = "$sgmls_cmd -f $Terr -m $catalog $decl $T";
  } else {
!     $cmd = "$sgmls_cmd -f /dev/null -m $catalog $decl $T";
  }
! open (SGMLS, "$cmd|")   || (unlink ($T), unlink ($Terr), die "$cmd: $!\n");
  $foo = &parse_sgml ('NIL', ());
  close SGMLS;
+ if (-s $Terr) {
+     if (open(Terr,$Terr)) {
+ 	print STDERR "\n";
+ 	foreach (<Terr>) {
+ 	    chop;
+ 	    s/^[^:]*:[^:]*://;
+ 	    ($line_no, $eno, $e, $error_message) = split(':');
+ 	    print STDERR "SGML error in $ENV{SUMMARIZER_URL} at line $line_no: $error_message\n";
+ 	}
+     }
+ }
+ unlink ($Terr) unless ($debug);
  unlink ($T) unless ($debug);
  
  
***************
*** 368,373 ****
--- 389,404 ----
          local ($f) = sprintf ("sgml%d%06x", $$, rand (0xFFFFFF));
          return "$ENV{'TMPDIR'}/$f"
                  if (defined $ENV{'TMPDIR'} && -d $ENV{'TMPDIR'});
+         return "/usr/tmp/$f"
+                 if (-d "/usr/tmp");
+         return "/tmp/$f";
+ }
+ 
+ sub temperrnam {
+         srand time;
+         local ($f) = sprintf ("err%d%06x", $$, rand (0xFFFFFF));
+         return "$ENV{'TMPDIR'}/$f"
+                 if (defined $ENV{'TMPDIR'} && -d $ENV{'TMPDIR'});
          return "/usr/tmp/$f"
                  if (-d "/usr/tmp");
          return "/tmp/$f";
Index: harvest/components/gatherer/standard/summarizers/Makefile.in
diff -c harvest/components/gatherer/standard/summarizers/Makefile.in:2.0 harvest/components/gatherer/standard/summarizers/Makefile.in:2.1
*** harvest/components/gatherer/standard/summarizers/Makefile.in:2.0	Sun Mar 16 22:08:21 1997
--- harvest/components/gatherer/standard/summarizers/Makefile.in	Tue Jul  1 20:20:38 1997
***************
*** 116,121 ****
--- 116,122 ----
  		  NewsArticle.sum \
  		  NewsGroup.sum \
  		  Object.sum \
+ 		  Pdf.sum \
  		  PostScript.sum \
  		  RCS.sum \
  		  RFC.sum \
Index: harvest/src/broker/Glimpse/index.c
diff -c harvest/src/broker/Glimpse/index.c:2.4 harvest/src/broker/Glimpse/index.c:2.6
*** harvest/src/broker/Glimpse/index.c:2.4	Thu May 15 17:38:34 1997
--- harvest/src/broker/Glimpse/index.c	Thu Jul 10 17:45:14 1997
***************
*** 1,4 ****
! static char rcsid[] = "index.c,v 2.4 1997/05/15 16:38:34 sxw Exp";
  /* 
   *  index.c -- Broker indexing/search support using Glimpse
   *
--- 1,4 ----
! static char rcsid[] = "index.c,v 2.6 1997/07/10 16:45:14 sxw Exp";
  /* 
   *  index.c -- Broker indexing/search support using Glimpse
   *
***************
*** 129,134 ****
--- 129,141 ----
  #define MAX_OPDATA_SIZE 1024
  #endif
  
+ /* some machines do not have this defined in netinet/in.h 
+ /* eg. SGI */
+ 
+ #ifndef IPPORT_USERRESERVED
+ #define IPPORT_USERRESERVED 5000
+ #endif
+ 
  /* Global variables */
  extern char *DIRpath;
  extern char *brk_obj_url;
***************
*** 740,745 ****
--- 747,755 ----
  	if (GL_errflag > 0) {
  		sprintf(tbuf, "-%d", GL_errflag);
  		argv[argc++] = xstrdup(tbuf);
+ 		/* We can't do an error match with -i or -w so disable them */
+ 		GL_caseflag=0;
+ 		GL_wordflag=0;
  	}
  	argv[argc++] = xstrdup("-L");
  	sprintf(tbuf, "%d", GL_maxresults < 1 ? 1000 : GL_maxresults);
Index: harvest/src/broker/example/brokers/AdminTools/README
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/README:1.1
*** /dev/null	Thu Jul 10 19:50:52 1997
--- harvest/src/broker/example/brokers/AdminTools/README	Tue Jul  1 20:41:51 1997
***************
*** 0 ****
--- 1,11 ----
+ cgi-bin		bdaddopt.cgi, bdapply.cgi, bdhome.cgi, 
+ 		bdtool.cgi, bdupdate.cgi
+ 
+ lib		hsrlibrary.pl, cgi.pl, configure.pl, bdlibrary.pl
+ 		bdhome.html.in, bdmanual.html.in, bdtool.html.in
+ 		broker.tags, gatherer.tags
+ 
+ tools 		bddisplay, bdlogs, bdmanual, bdstop
+ 
+ hsr.conf	make sure all the .Home points to the correct directory on your
+ 		machine
Index: harvest/src/broker/example/brokers/AdminTools/hsr.conf
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/hsr.conf:1.1
*** /dev/null	Thu Jul 10 19:50:52 1997
--- harvest/src/broker/example/brokers/AdminTools/hsr.conf	Thu Jul 10 19:46:53 1997
***************
*** 0 ****
--- 1,27 ----
+ ! -----------------------------------------------------------------
+ ! Standard configuration file for admin tools
+ ! -----------------------------------------------------------------
+ CGI.Home http://gondor.transarc.com/cgi-bin/test/hsr/cgi-bin
+ CGI.BDAddOpt   bdaddopt.cgi
+ CGI.BDApply    bdapply.cgi
+ CGI.BDHome     bdhome.cgi
+ CGI.BDTool     bdtool.cgi
+ CGI.BDUpdate   bdupdate.cgi
+ 
+ Library.Home /local/httpd/cgi-bin/test/hsr/lib
+ Library.BDHome.Skel  bdhome.html.in
+ Library.BDNew.Skel   bdnew.html.in
+ Library.BDTool.Skel  bdtool.html.in
+ Library.BDManual.Skel   bdmanual.html.in
+ 
+ Tags.Home      /local/httpd/cgi-bin/test/hsr/lib
+ Tags.BROKER    broker.tags
+ Tags.GATHERER  gatherer.tags
+ 
+ Descriptions.Home /local/httpd/cgi-bin/test/hsr/lib
+ Descriptions.List descriptions
+ 
+ Help.Home   /local/httpd/cgi-bin/test/hsr/help
+ 
+ Tools.Home  /local/httpd/cgi-bin/test/hsr/tools
+ Tools.List  tools
Index: harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdaddopt.cgi
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdaddopt.cgi:1.1
*** /dev/null	Thu Jul 10 19:50:52 1997
--- harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdaddopt.cgi	Thu Jul 10 19:47:00 1997
***************
*** 0 ****
--- 1,89 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdaddopt.cgi
+ # 
+ # Author: Mic Bowman and Satya Venneti
+ #
+ # Adds optional fields to the description file
+ # -----------------------------------------------------------------
+ $ENV{"HARVEST_HOME"} = "/usr/local/harvest" 
+ 	if (!defined($ENV{"HARVEST_HOME"}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ require 'bdlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_AddOpt
+ # -----------------------------------------------------------------
+ sub BD_AddOpt {
+     local(*obj,*tags) = @_;
+     local($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is);
+ 
+     # Update the object
+     print STDERR "Tag: ", $in{"TAG"}, "\n";
+     print STDERR "Value: ", $in{"VALUE"}, "\n";
+ 
+     $obj{$in{"TAG"}} = $in{"VALUE"};
+ 
+     # Add update attributes
+     ($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is) = localtime(time);
+     $min = "0$min" if ($min < 10);
+     $hour = "0$hour" if ($hour < 10);
+     $obj{"Last-Update-Time"} = "$hour:$min " . @HSR_Month[$mo] . " $md, 19$yr";
+     $obj{"Last-Update"} = "Add Optional Attribute";
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$shome,$sfile,$sop);
+     local(%obj,%tags);
+ 
+     # Initialize
+     &HSR_Initialize;
+     &CGIHeader;
+     undef %obj;
+     undef %tags;
+ 
+     # Process the arguments
+     &CGIParse;
+ 
+     &CGIError("Server type unspecified") 
+ 	if ((! defined $in{"TYPE"}) || ($in{"TYPE"} !~ /\S/));
+     &CGIError("Server home unspecified") 
+ 	if ((! defined $in{"HOME"}) || ($in{"HOME"} !~ /\S/));
+     &CGIError("Server description unspecified") 
+ 	if ((! defined $in{"FILE"}) || ($in{"FILE"} !~ /\S/));
+     &CGIError("Missing parameter: TAG") 
+ 	if ((! defined $in{"TAG"})  || ($in{"TAG"} !~ /\S/));
+     &CGIError("Missing parameter: VALUE") 
+ 	if ((! defined $in{"VALUE"})  || ($in{"VALUE"} !~ /\S/));
+ 
+     $stype = $in{"TYPE"}; delete $in{"TYPE"};
+     $shome = $in{"HOME"}; delete $in{"HOME"};
+     $sfile = $in{"FILE"}; delete $in{"FILE"};
+ 
+     # Make sure the file is in the descriptions home directory
+     &CGIError("Invalid description file: $sfile")
+ 	if (index($sfile,$Config{"Descriptions.Home"}) != 0);
+ 
+     # Update the description file
+     %tags = &BD_ReadTags("BROKER");
+ 
+     &CGIError("Unable to update non-existant description file.")
+ 	if (! -r $sfile);
+ 
+     ($type,$url,%obj) = &HSR_ParseDescription($sfile);
+ 
+     &BD_AddOpt(*obj,*tags);
+     &HSR_WriteDescription($sfile,$stype,$obj{"Description-File"},%obj);
+     &HSR_DisplayDescription($sfile);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdapply.cgi
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdapply.cgi:1.1
*** /dev/null	Thu Jul 10 19:50:52 1997
--- harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdapply.cgi	Thu Jul 10 19:47:08 1997
***************
*** 0 ****
--- 1,83 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdtool.pl
+ # 
+ # Author: Mic Bowman and Satya Venneti
+ #
+ # Create the initial page for the validation database maintenance
+ # scripts.
+ # -----------------------------------------------------------------
+ $ENV{"HARVEST_HOME"} = "/usr/local/harvest" 
+ 	if (!defined($ENV{"HARVEST_HOME"}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ require 'bdlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_Apply
+ # -----------------------------------------------------------------
+ sub BD_Apply {
+     local($stype,$shome,$sfile,$sop) = @_;
+     local($type,$desc);
+     local($thome,%tlist,$tname);
+ 
+     $thome = $Config{"Tools.Home"};
+     %tlist = &BD_ReadTools;
+     $tname = join('&',$sop,$stype);
+ 
+     &CGIError("This tool is not defined: $sop")
+ 	if ((! defined $tlist{$tname}) || ($tlist{$tname} !~ /\S/));
+     &CGIError("Unable to locate tool: $sop")
+ 	if (! -r "$thome/$sop");
+ 
+     # Be very careful about what we pass as parameters to the system call
+     $desc = $tlist{$tname};
+     &CGIError("Invalid server path: $shome")
+ 	if ($shome !~ m@^[-a-zA-Z/.]+$@); # No special characters
+     &CGIError("Invalid description file: $sfile")
+ 	if ($sfile !~ m@^[-a-zA-Z/.]+$@); # No special characters
+     
+     # Create the pipe and read the output
+     open(SPIPE,"$thome/$sop $stype $sfile $shome|") ||
+ 	&CGIError("Unable to apply tool $sop");
+ 
+     while (<SPIPE>) {
+ 	print $_;
+     }
+ 
+     close(SPIPE);
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$shome,$sfile,$sop);
+ 
+     &HSR_Initialize;
+ 
+     &CGIHeader;
+     &CGIParse;
+     &CGIError("Server type unspecified") 
+ 	if ((! defined $in{"TYPE"}) || ($in{"TYPE"} !~ /\S/));
+     &CGIError("Server home unspecified") 
+ 	if ((! defined $in{"HOME"}) || ($in{"HOME"} !~ /\S/));
+     &CGIError("Server description unspecified") 
+ 	if ((! defined $in{"FILE"}) || ($in{"FILE"} !~ /\S/));
+     &CGIError("No operation specified")
+ 	if ((! defined $in{"OP"}) || ($in{"OP"} !~ /\S/));
+ 
+     $stype = $in{"TYPE"};
+     $shome = $in{"HOME"};
+     $sfile = $in{"FILE"};
+     $sop = $in{"OP"};
+ 
+     &BD_Apply($stype,$shome,$sfile,$sop);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdhome.cgi
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdhome.cgi:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdhome.cgi	Thu Jul 10 19:47:10 1997
***************
*** 0 ****
--- 1,70 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdhome.pl
+ # 
+ # Author: Mic Bowman and Satya Venneti
+ #
+ # Create the initial page for the description file maintenance
+ # scripts.
+ # -----------------------------------------------------------------
+ $ENV{"HARVEST_HOME"} = "/usr/local/harvest" 
+ 	if (!defined($ENV{"HARVEST_HOME"}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ require 'bdlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_BuildList
+ # -----------------------------------------------------------------
+ sub BD_BuildList {
+     local($toolurl);
+     local(%dlist);
+     local($key,$type,$home);
+     local($url,$output);
+ 
+     $toolurl = &HSR_FileName("CGI","BDTool");
+     %dlist = &BD_ReadDescriptions();
+ 
+     $output = "";
+     foreach $key (sort keys %dlist) {
+ 	($type, $home) = split('&',$dlist{$key},2);
+ 	$url = "<a href=\"$toolurl?TYPE=$type&HOME=$home&FILE=$key\">$key</A>";
+ 	$output = $output . "<LI>" . $url . "\n";
+     }
+ 
+     return($output);
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($filename,$applyurl);
+ 
+     &HSR_Initialize;
+ 
+     &CGIHeader;
+ 
+     $descriptions = &BD_BuildList();
+     $applyurl = &HSR_FileName("CGI","BDApply");
+     $filename = &HSR_FileName("Library","BDHome.Skel");
+ 
+     open(SKEL,"<$filename") ||
+ 	&CGIError("Unable to generate update form, check configuration.");
+ 
+     while (<SKEL>) {
+ 	$_ =~ s/\@APPLYURL\@/$applyurl/;
+ 	$_ =~ s/\@DESCRIPTIONS\@/$descriptions/;
+ 	print $_;
+     }
+ 
+     close(SKEL);
+ 
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdtool.cgi
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdtool.cgi:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdtool.cgi	Thu Jul 10 19:47:11 1997
***************
*** 0 ****
--- 1,86 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdtool.pl
+ # 
+ # Author: Mic Bowman and Satya Venneti
+ #
+ # Create the initial page for the validation database maintenance
+ # scripts.
+ # -----------------------------------------------------------------
+ $ENV{"HARVEST_HOME"} = "/usr/local/harvest" 
+ 	if (!defined($ENV{"HARVEST_HOME"}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ require 'bdlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_BuildList
+ # -----------------------------------------------------------------
+ sub BD_BuildList {
+     local($stype) = @_;
+     local(%tlist);
+     local($key,$desc,$file,$type);
+     local($line,$output);
+ 
+     %tlist = &BD_ReadTools;
+ 
+     foreach $key (sort keys %tlist) {
+ 	$desc = $tlist{$key};
+ 	($file,$type) = split('&',$key,2);
+ 	next if ($type ne $stype);
+ 
+ 	$line = "<INPUT type=\"RADIO\" name=\"OP\" value=\"$file\"> $desc";
+ 	$output = $output . $line . "<BR>\n";
+     }
+ 
+     return($output);
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$shome,$sfile);
+     local($filename,$applyurl);
+ 
+     &HSR_Initialize;
+ 
+     &CGIHeader;
+     &CGIParse;
+     &CGIError("Server type unspecified") 
+ 	if ((! defined $in{"TYPE"}) || ($in{"TYPE"} !~ /\S/));
+     &CGIError("Server home unspecified") 
+ 	if ((! defined $in{"HOME"}) || ($in{"HOME"} !~ /\S/));
+     &CGIError("Server description unspecified") 
+ 	if ((! defined $in{"FILE"}) || ($in{"FILE"} !~ /\S/));
+ 
+     $stype = $in{"TYPE"};
+     $shome = $in{"HOME"};
+     $sfile = $in{"FILE"};
+     $tools = &BD_BuildList($stype);
+ 
+     $applyurl = &HSR_FileName("CGI","BDApply");
+     $filename = &HSR_FileName("Library","BDTool.Skel");
+ 
+     open(SKEL,"<$filename") ||
+ 	&CGIError("Unable to generate update form, check configuration.");
+ 
+     while (<SKEL>) {
+ 	$_ =~ s/\@FILE\@/$sfile/g;
+ 	$_ =~ s/\@TYPE\@/$stype/g;
+ 	$_ =~ s/\@HOME\@/$shome/g;
+ 	$_ =~ s/\@TOOLS\@/$tools/g;
+ 	$_ =~ s/\@APPLYURL\@/$applyurl/g;
+ 	print $_;
+     }
+ 
+     close(SKEL);
+ 
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdupdate.cgi
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdupdate.cgi:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/cgi-bin/bdupdate.cgi	Thu Jul 10 19:47:12 1997
***************
*** 0 ****
--- 1,107 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdtool.pl
+ # 
+ # Author: Mic Bowman and Satya Venneti
+ #
+ # Create the initial page for the validation database maintenance
+ # scripts.
+ # -----------------------------------------------------------------
+ $ENV{"HARVEST_HOME"} = "/usr/local/harvest" 
+ 	if (!defined($ENV{"HARVEST_HOME"}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ require 'bdlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_Update
+ # -----------------------------------------------------------------
+ sub BD_Update {
+     local(*obj,*tags) = @_;
+     local($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is);
+ 
+     # Update the object
+     foreach $key (keys %in) {
+ 	if ($in{$key} =~ /^\s*$/) {
+ 	    delete $obj{$key};
+ 	} else {
+ 	    $obj{$key} = $in{$key};
+ 	}
+     }
+ 
+     # Ensure that the mandatory attributes are defined
+     foreach $key (keys %tags) {
+ 	($mandatory,$manual,$auto,$desc) = split('&',$tags{$key});
+ 	next if ($manual ne "Y"); # Skip automatic tags
+ 	if ($mandatory eq "Y") {
+ 	    if ((! defined $obj{$key}) || ($obj{$key} =~ /^\s*$/)) {
+ 		&CGIError("Mandatory attributes is not defined: $key");
+ 	    }
+ 	}
+     }
+ 
+     # Add update attributes
+     ($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is) = localtime(time);
+     $min = "0$min" if ($min < 10);
+     $hour = "0$hour" if ($hour < 10);
+     $obj{"Last-Update-Time"} = "$hour:$min " . @HSR_Month[$mo] . " $md, 19$yr";
+     $obj{"Last-Update"} = "Manual Update";
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$shome,$sfile,$sop);
+     local(%obj,%tags,%dlist);
+ 
+     # Initialize
+     &HSR_Initialize;
+     &CGIHeader;
+     undef %obj;
+     undef %tags;
+ 
+     # Process the arguments
+     &CGIParse;
+ 
+     &CGIError("Server type unspecified") 
+ 	if ((! defined $in{"TYPE"}) || ($in{"TYPE"} !~ /\S/));
+     &CGIError("Server home unspecified") 
+ 	if ((! defined $in{"HOME"}) || ($in{"HOME"} !~ /\S/));
+     &CGIError("Server description unspecified") 
+ 	if ((! defined $in{"FILE"}) || ($in{"FILE"} !~ /\S/));
+ 
+     $stype = $in{"TYPE"}; delete $in{"TYPE"};
+     $shome = $in{"HOME"}; delete $in{"HOME"};
+     $sfile = $in{"FILE"}; delete $in{"FILE"};
+ 
+     # Make sure the file is in the descriptions home directory
+     &CGIError("Invalid description file: $sfile")
+ 	if (index($sfile,$Config{"Descriptions.Home"}) != 0);
+ 
+     # Update the description file
+     %tags = &BD_ReadTags("BROKER");
+ 
+     if (-r $sfile) {
+ 	($type,$url,%obj) = &HSR_ParseDescription($sfile);
+ 	&CGIError("Requested type is different than stored type: $type")
+ 	    if ($type ne $stype);
+     }
+ 
+     &BD_Update(*obj,*tags);
+     &HSR_WriteDescription($sfile,$stype,$obj{"Description-File"},%obj);
+ 
+     %dlist = &BD_ReadDescriptions;
+     if ((! defined $dlist{$sfile}) || ($dlist{$sfile} =~ /^\s*$/)) {
+ 	&BD_AddDescription($stype,$sfile,$shome);
+     }
+ 
+     &HSR_DisplayDescription($sfile);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/descriptions/description
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/descriptions/description:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/descriptions/description	Tue Jul  1 20:42:08 1997
***************
*** 0 ****
--- 1,19 ----
+ ! -----------------------------------------------------------------
+ ! Server Description file
+ !
+ ! This file contains information about the location and type of
+ ! various server description files.  The format of the file is as 
+ ! follows:
+ !
+ !       Type File Home
+ !
+ ! Type -- the type of server described, usually BROKER or GATHERER
+ ! File -- the name of the file where the description is kept
+ ! Home -- the root of the server directory, automatic tools use this
+ !         to find server specific files.
+ !
+ ! Specifically the fields are defined by the following regular expression
+ !       ^\s*(\S+)\s+(\S+)\s+(\S.*)$
+ !
+ ! Note that this file should be updated automatically.
+ ! -----------------------------------------------------------------
Index: harvest/src/broker/example/brokers/AdminTools/lib/bdhome.html.in
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/bdhome.html.in:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/bdhome.html.in	Thu Jul 10 19:47:18 1997
***************
*** 0 ****
--- 1,62 ----
+ <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+ <HTML>
+ <HEAD>
+ <TITLE>Harvest Description Administration</TITLE>
+ <LINK rev=made href="mailto:harvest+feedback@transarc.com">
+ </HEAD>
+ 
+ <BODY>
+ <H1>Harvest Description Administration</H1>
+ 
+ <P> Use this form to maintain and update the description file for your
+ Harvest broker or gatherer.  If you make changes to the location of your
+ description file, be sure to register the changes with the
+ <A HREF="http://harvest.transarc.com/Harvest/brokers/HSR/hsr-registration.html">
+ Harvest Server Registry</A>.  </P>
+ 
+ <P> The local Harvest registry contains the following description files.
+ Select one to display the maintenance page. </P>
+ 
+ <DIR>
+   @DESCRIPTIONS@
+ </DIR>
+ 
+ <HR>
+ 
+ <H2> New Description </H2>
+ 
+ <P> If the description file does not exist in the previous list, use
+ this form to add it to the local registry. </P>
+ 
+ <FORM METHOD="POST" ACTION="@APPLYURL@">
+ <INPUT type="HIDDEN" name="OP" value="bdmanual">
+ <H4>Specify the name of the description file (file system path)</H4>
+ <INPUT type="TEXT" size="70" name="FILE">
+ <BR>
+ 
+ <H4>Specify the directory in which the server resides (file system path)</H4>
+ <INPUT type="TEXT" size="70" name="HOME">
+ <BR>
+ 
+ <H4>Specify the type of the server:</H4>
+ <INPUT type="RADIO" name="TYPE" value="BROKER" CHECKED> Broker
+ <BR>
+ <INPUT type="RADIO" name="TYPE" value="GATHERER"> Gatherer
+ <P>
+ <INPUT TYPE="submit" VALUE="Submit">
+ <INPUT TYPE="reset" VALUE="Clear">
+ 
+ <BR>
+ <BR>
+ </FORM>
+ 
+ <HR>
+ <ADDRESS>
+ <A HREF="http://harvest.cs.colorado.edu">Harvest Information System</A>.
+ <!-- hhmts start -->
+ Last modified: Fri Jul  7 08:32:03 EDT 1995
+ <!-- hhmts end -->
+ </ADDRESS>
+ 
+ </BODY>
+ </HTML>
Index: harvest/src/broker/example/brokers/AdminTools/lib/bdlibrary.pl
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/bdlibrary.pl:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/bdlibrary.pl	Thu Jul 10 19:47:19 1997
***************
*** 0 ****
--- 1,125 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdlibrary.pl
+ # 
+ # Author: Mic Bowman 
+ #
+ # A library of routines for description maintenance for the Harvest
+ # HSR.
+ # -----------------------------------------------------------------
+ 
+ # -----------------------------------------------------------------
+ # BD_ReadDescriptions -- read a list of descriptions
+ # -----------------------------------------------------------------
+ sub BD_ReadDescriptions {
+     local(%list);
+     local($filename);
+ 
+     $filename = &HSR_FileName("Descriptions","List");
+     
+     open(DFILE,"<$filename") ||
+ 	&CGIError("Unable to locate description list");
+ 
+     while (<DFILE>) {
+ 	chop;
+ 	next if (/^\s*!/);	# Skip comments
+ 	next if (/^\s*$/);	# Skip blanks
+ 	($type, $file, $home) = /^\s*(\S+)\s+(\S+)\s+(\S+)\s*$/;
+ 	$list{$file} = join('&',$type,$home);
+     }
+ 
+     return (%list);
+ }
+ 
+ # -----------------------------------------------------------------
+ # BD_AddDescription
+ # -----------------------------------------------------------------
+ sub BD_AddDescription {
+     local($type,$file,$home) = @_;
+     local($filename);
+ 
+     $filename = &HSR_FileName("Descriptions","List");
+     
+     open(DFILE,">>$filename") ||
+ 	&CGIError("Unable to locate description list");
+ 
+     print DFILE "$type $file $home\n";
+ 
+     close(DFILE);
+ }
+ 
+ # -----------------------------------------------------------------
+ # BD_ReadTags -- read the list of tags
+ # -----------------------------------------------------------------
+ sub BD_ReadTags {
+     local($stype) = @_;
+     local(%list);
+     local($filename);
+     local($name,$mandatory,$manual,$auto,$desc);
+ 
+     $filename = &HSR_FileName("Tags",$stype);
+ 
+     open(TFILE,"<$filename") ||
+ 	&CGIError("Unable to open tags file $filename");
+ 
+     while (<TFILE>) {
+ 	chop;
+ 	next if (/^\s*!/);	# Skip comments
+ 	next if (/^\s*$/);	# Skip blanks
+ 	
+ 	($name,$btype,$mandatory,$manual,$auto,$desc) =
+ 	    /^\s*(\S+)\s+([SPN])\s+([YN])\s+([YN])\s+([YN])\s+(\S.*)$/;
+ 
+ 	$list{$name} = join('&',$btype,$mandatory,$manual,$auto,$desc);
+     }
+ 
+     return (%list);
+ }
+ 
+ # -----------------------------------------------------------------
+ # BD_ReadTools -- read a list of tools
+ # -----------------------------------------------------------------
+ sub BD_ReadTools {
+     local(%list);
+     local($filename);
+ 
+     $toolhome = $Config{"Tools.Home"};
+     $filename = &HSR_FileName("Tools","List");
+ 
+     open(TFILE,"<$filename") ||
+ 	&CGIError("Unable to locate tools list");
+ 
+     while (<TFILE>) {
+ 	chop;
+ 	next if (/^\s*!/);	# Skip comments
+ 	next if (/^\s*$/);	# Skip blanks
+ 	($type, $file, $desc) = /^\s*(\S+)\s+(\S+)\s+(\S.*)$/;
+ 	$list{join('&',$file,$type)} = $desc;
+     }
+ 
+     return (%list);
+ }
+ 
+ # -----------------------------------------------------------------
+ # BD_InputBox -- Create a string with a box
+ # -----------------------------------------------------------------
+ sub BD_InputBox {
+     local($btype,$tag,$val) = @_;
+     local($value);
+     
+     $value = "";
+     if ($btype eq "S") {
+ 	$value = "value=\"$val\"" if ($val !~ /^\s*$/);
+ 	$box = "<input type=TEXT name=\"$tag\" size=70 $value>";
+     } elsif ($btype eq "P") {
+ 	$box = "<textarea name=\"$tag\" rows=\"5\" cols=\"70\">";
+ 	$box = $box . "$val</textarea>";
+     } elsif ($btype eq "N") {
+ 	$value = "value=\"$val\"" if ($val !~ /^\s*$/);
+ 	$box = "<input type=TEXT name=\"$tag\" size=20 $value>";
+     }
+     return($box);
+ }
+ 
+ # -----------------------------------------------------------------
+ 1
Index: harvest/src/broker/example/brokers/AdminTools/lib/bdmanual.html.in
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/bdmanual.html.in:1.1
*** /dev/null	Thu Jul 10 19:50:53 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/bdmanual.html.in	Thu Jul 10 19:47:19 1997
***************
*** 0 ****
--- 1,88 ----
+ <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+ <HTML>
+ <HEAD>
+ <TITLE>Manual Update of Server Description</TITLE>
+ <LINK rev=made href="mailto:harvest+feedback@transarc.com">
+ </HEAD>
+ 
+ <BODY>
+ <H1>Manual Update of Server Description</H1>
+ 
+ <P> Use this form to perform manual updates of the description file
+ <PRE>
+ 	@FILE@
+ </PRE>
+ <HR>
+ 
+ <form action="@UPDATEURL@" method="POST">
+ <input type="HIDDEN" name="TYPE" value="@TYPE@">
+ <input type="HIDDEN" name="FILE" value="@FILE@">
+ <input type="HIDDEN" name="HOME" value="@HOME@">
+ 
+ <H2>Mandatory Attributes</H2>
+ 
+ <P> Mandatory attributes are required for the description file.  The
+ update will fail if you do not provide values for the mandatory fields.
+ </P>
+ 
+ @MANDATORY@
+ 
+ <H2>Recommended Attributes</H2>
+ 
+ <P> Recommended attributes have a predefined meaning that is common to
+ all Harvest servers.  For example, the tag <EM>Organization</EM> is
+ always the name of the organization that maintains the server.  However,
+ recommended attributes need not be provided for all servers. </P>
+ 
+ @RECOMMENDED@
+ 
+ <H2>Optional Attributes </H2>
+ 
+ <P> Optional attributes have no predefined meaning.  Two different
+ servers may have optional attributes with the same tag, but different
+ meanings.  Use the form below to add new optional attributes.</P>
+ 
+ @OPTIONAL@
+ 
+ <P>
+ <INPUT TYPE="submit" VALUE="Update">
+ <INPUT TYPE="reset" VALUE="Clear">
+ <P>
+ 
+ </form>
+ 
+ <HR>
+ <H2>Add New Optional Attributes</H2>
+ 
+ <form action="@ADDOPTURL@" method="POST">
+ <input type="HIDDEN" name="TYPE" value="@TYPE@">
+ <input type="HIDDEN" name="FILE" value="@FILE@">
+ <input type="HIDDEN" name="HOME" value="@HOME@">
+ <H4>Tag</H4>
+ 
+ <P> Specify the field name for the new attribute:<BR>
+ <INPUT NAME="TAG" TYPE=TEXT SIZE=40>
+ </P>
+ 
+ <H4>Value</H4>
+ <P> Specify the value for the new attribute:<BR>
+ <INPUT NAME="VALUE" TYPE=TEXT SIZE=70>
+ </P>
+ 
+ <P>
+ <INPUT TYPE="submit" VALUE="Add Attribute">
+ <INPUT TYPE="reset" VALUE="Clear">
+ <P>
+ 
+ </form>
+ 
+ <HR>
+ <ADDRESS>
+ <A HREF="http://harvest.cs.colorado.edu">Harvest Information System</A>.
+ <!-- hhmts start -->
+ Last modified: Thu Jul  6 13:06:16 EDT 1995
+ <!-- hhmts end -->
+ </ADDRESS>
+ 
+ </BODY>
+ </HTML>
Index: harvest/src/broker/example/brokers/AdminTools/lib/bdtool.html.in
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/bdtool.html.in:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/bdtool.html.in	Thu Jul 10 19:47:20 1997
***************
*** 0 ****
--- 1,46 ----
+ <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+ <HTML>
+ <HEAD>
+ <TITLE>Description File Tools</TITLE>
+ <LINK rev=made href="mailto:harvest+feedback@transarc.com">
+ </HEAD>
+ 
+ <BODY>
+ <H1>Description File Tools</H1>
+ 
+ <P> The following tools can be used to create or maintain the
+ description file for your Harvest server. This page applies
+ the tools to the server description contained in the file:</P>
+ <PRE>
+ 	@FILE@
+ </PRE>
+ 
+ <HR>
+ 
+ <form action="@APPLYURL@" method="POST">
+ 
+ <input type="HIDDEN" name="TYPE" value="@TYPE@">
+ <input type="HIDDEN" name="FILE" value="@FILE@">
+ <input type="HIDDEN" name="HOME" value="@HOME@">
+ 
+ <H4>Select a tool to apply:</H4>
+ 
+ @TOOLS@
+ 
+ <P>
+ <INPUT TYPE="submit" VALUE="Submit">
+ <INPUT TYPE="reset" VALUE="Clear">
+ <P>
+ 
+ </form>
+ 
+ <HR>
+ <ADDRESS>
+ <A HREF="http://harvest.cs.colorado.edu">Harvest Information System</A>.
+ <!-- hhmts start -->
+ Last modified: Wed Jul  5 14:47:32 EDT 1995
+ <!-- hhmts end -->
+ </ADDRESS>
+ 
+ </BODY>
+ </HTML>
Index: harvest/src/broker/example/brokers/AdminTools/lib/broker.tags
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/broker.tags:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/broker.tags	Thu Jul 10 19:47:21 1997
***************
*** 0 ****
--- 1,40 ----
+ ! -----------------------------------------------------------------
+ ! Broker Tags File
+ !
+ ! This file contains descriptions of the mandatory and recommended
+ ! tags for broker summaries.  The manual update tools use this
+ ! file to verify that a given object contains the correct fields.
+ ! The format of the file is as follows:
+ !
+ ! 	Name BoxType Mandatory Manual Automatic Description
+ !
+ ! Name -- the tag name
+ ! BoxType -- the type of box for displaying the value
+ !	S -- String (TEXT 70)
+ !	N -- Number (TEXT 20)
+ !	P -- Paragraph (TEXTAREA)
+ ! Mandatory -- is this a mandatory attribute?
+ ! Manual -- can this attribute be updated manually?
+ ! Automatic -- can this attribute be updated automatically?
+ ! Description -- short description of the tag
+ !
+ ! Specifically the fields are defined by the following regular expression
+ ! 	^\s*(\S+)\s+([SPN])\s+([YN])\s+([YN])\s+([YN])\s+(\S.*)$
+ ! -----------------------------------------------------------------
+ 
+ ! --------------- Mandatory ---------------
+ Description		S Y Y N	A one-line description to be displayed when this object is returned as part of a query.
+ Description-File	S Y N N	The public URL of the file that contains the broker description.
+ Host 			S Y Y Y	The name of the host on which the broker resides.
+ Last-Update		S Y N Y	The name of the program that last updated the broker description.
+ Last-Update-Time 	S Y N Y	The time of the last update.
+ Maintainer		S Y Y N	The email address of the broker administrator.
+ Port 			S Y Y Y	The port number on which the broker listens.
+ Version 		N Y Y Y	The version number of the Harvest software (e.g. 1.2).
+ 
+ ! --------------- Recommended ---------------
+ Extended-Description	P N Y N A paragraph that describes the contents of the broker.
+ Gatherer 		P N Y Y A list of host:port pairs that identify gatherers from which this broker collects objects.
+ Keywords 		S N Y Y A list of words that characterize the contents of the broker.
+ Organization 		S N Y N The name of the organization that maintains the broker.
+ 
Index: harvest/src/broker/example/brokers/AdminTools/lib/cgi.pl
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/cgi.pl:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/cgi.pl	Thu Jul 10 19:47:22 1997
***************
*** 0 ****
--- 1,93 ----
+ : # *-*-perl-*-*
+     eval 'exec perl -S $0 "$@"'
+     if $running_under_some_shell;  
+ 
+ # -----------------------------------------------------------------
+ # cgi.pl
+ # 
+ # Author: Mic Bowman
+ #
+ # CGI library routines.
+ # -----------------------------------------------------------------
+ 
+ # -----------------------------------------------------------------
+ # Globals
+ # -----------------------------------------------------------------
+ 
+ # -----------------------------------------------------------------
+ # CGIParse -- from cgi-lib, parses the POST and GET arguments
+ # -----------------------------------------------------------------
+ sub CGIParse {
+     if (@_) {
+ 	local (*in) = @_;
+     }
+ 
+     local ($i, $loc, $key, $val);
+ 
+     # Read in text
+     if ($ENV{'REQUEST_METHOD'} eq "POST") {
+ 	for ($i = 0; $i < $ENV{'CONTENT_LENGTH'}; $i++) {
+ 	    $in .= getc;
+ 	}
+     } elsif ($ENV{'REQUEST_METHOD'} eq "GET") {
+ 	$in = $ENV{'QUERY_STRING'};
+     } else {
+ 	&CGIError("This program processes POST and GET messages only.");
+     }
+ 
+     @in = split(/&/,$in);
+ 
+     foreach $i (0 .. $#in) {
+ 	# Convert plus's to spaces
+ 	$in[$i] =~ s/\+/ /g;
+ 
+ 	# Convert %XX from hex numbers to alphanumeric
+ 	$in[$i] =~ s/%(..)/pack("c",hex($1))/ge;
+ 
+ 	# Split into key and value.
+ 	$loc = index($in[$i],"=");
+ 	$key = substr($in[$i],0,$loc);
+ 	$val = substr($in[$i],$loc+1);
+ 	$in{$key} .= '&' if (defined($in{$key})); # \0 is the separator
+ 	$in{$key} .= $val;
+     }
+ 
+     return 1; # just for fun
+ }
+ 
+ # -----------------------------------------------------------------
+ # CGIError -- prints a nicely formated error message
+ # -----------------------------------------------------------------
+ sub CGIError {
+     local($msg) = @_;
+ 
+     print STDOUT <<EOM;
+ <HTML>
+ <HEAD>
+ <TITLE>Script Processing Error</TITLE>
+ </HEAD>
+ <BODY>
+ <H1>Script Processing Error</H1>
+ An error occured while processing the request:
+ <PRE>
+     <B>$msg</B>
+ </PRE>
+ </BODY>
+ </HTML>
+ EOM
+     exit(1);
+ }
+ 
+ # -----------------------------------------------------------------
+ # CGIHeader -- Mime content header
+ # -----------------------------------------------------------------
+ sub CGIHeader {
+     print "HTTP/1.0 200 OK\n";
+     print "MIME-Version: 1.0\n";
+     print "Content-Type: text/html\n";
+     print "Content-Transfer-Encoding: binary\n";
+     print "\n";
+ }
+ 
+ # -----------------------------------------------------------------
+ 1
Index: harvest/src/broker/example/brokers/AdminTools/lib/configure.pl
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/configure.pl:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/configure.pl	Thu Jul 10 19:47:23 1997
***************
*** 0 ****
--- 1,64 ----
+ : # *-*-perl-*-*
+     eval 'exec perl -S $0 "$@"'
+     if $running_under_some_shell;  
+ 
+ # -----------------------------------------------------------------
+ # Config.pl
+ # 
+ # Author: Mic Bowman
+ #
+ # A perl interface to the config files used by the Transarc common
+ # library configuration modules.  Each file contains a series of
+ # lines that contain a variable and the value to be assigned to that
+ # variable.  Whitespace separates the value from the variable. The
+ # environment variable CONFIG_PRECONF contains the name of a program
+ # to apply to the configuration file before it is read.
+ # -----------------------------------------------------------------
+ 
+ # -----------------------------------------------------------------
+ # Globals
+ # -----------------------------------------------------------------
+ $config_preconf = $ENV{"CONFIGURE_PREPROCESS"};
+ 
+ # -----------------------------------------------------------------
+ # ReadConfiguration(file,initial)
+ #
+ # Read a file of configuration records; add them to the initial
+ # configuration array.  If necessary, run the preprocessor on the
+ # configuration file. A typical call looks like this:
+ #	&ReadConfiguration("test.conf",*TEST);
+ # -----------------------------------------------------------------
+ sub ReadConfiguration {
+     local($config_file, *CONFIG) = @_;
+     
+     # Open the configuration file
+     if ((defined $config_preconf) && (-x $config_preconf)) {
+ 	open(CFILE,"$config_preconf $config_file|") || 
+ 	    die "Cannot create pipe to process configuration file.\n";
+     } else {
+ 	open(CFILE,"<$config_file") ||
+ 	    die "Cannot open configuration file: $config_file.\n";
+     }
+ 
+     # Read the configuration records
+     while (<CFILE>) {
+ 	next if (/^\s*!/);	# skip comments
+ 	next if (/^\s*$/);	# skip blank lines
+ 	($var,$val) = /^\s*([a-zA-Z]\S*)\s+(\S.*)\s*$/;
+ 	if ($val =~ /^\"(.*)\"[^\"]*$/) { # "
+ 	    ($val)  = ($val =~ /^\"(.*)\"[^\"]*$/);
+ 	}
+ 	$val =~ s/\\\\/\\--/g;
+ 	$val =~ s/\\e/\033/g;
+ 	$val =~ s/\\n/\n/g;
+ 	$val =~ s/\\r/\r/g;
+ 	$val =~ s/\\t/\t/g;
+ 	$val =~ s/\\--/\\/g;
+ 	    
+ 	$CONFIG{$var} = $val;
+     }
+ 
+     close(CFILE);
+ }
+ 
+ 1
Index: harvest/src/broker/example/brokers/AdminTools/lib/gatherer.tags
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/gatherer.tags:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/gatherer.tags	Thu Jul 10 19:47:23 1997
***************
*** 0 ****
--- 1,42 ----
+ ! -----------------------------------------------------------------
+ ! Gatherer Tags File
+ !
+ ! This file contains descriptions of the mandatory and recommended
+ ! tags for gatherer summaries.  The manual update tools use this
+ ! file to verify that a given object contains the correct fields.
+ ! The format of the file is as follows:
+ !
+ ! 	Name BoxType Mandatory Manual Automatic Description
+ !
+ ! Name -- the tag name
+ ! BoxType -- the type of box for displaying the value
+ !	S -- String (TEXT 70)
+ !	N -- Number (TEXT 20)
+ !	P -- Paragraph (TEXTAREA)
+ ! Mandatory -- is this a mandatory attribute?
+ ! Manual -- can this attribute be updated manually?
+ ! Automatic -- can this attribute be updated automatically?
+ ! Description -- short description of the tag
+ !
+ ! Specifically the fields are defined by the following regular expression
+ ! 	^\s*(\S+)\s+([SPN])\s+([YN])\s+([YN])\s+([YN])\s+(\S.*)$
+ ! -----------------------------------------------------------------
+ 
+ ! --------------- Mandatory ---------------
+ Description		S Y Y N	A one-line description to be displayed when this object is returned as part of a query.
+ Description-File	S Y N N	The public URL of the file that contains the gatherer description.
+ Host 			S Y Y Y	The name of the host on which the gatherer resides.
+ Last-Update		S Y N Y	The name of the program that last updated the gatherer description.
+ Last-Update-Time 	S Y N Y	The time of the last update.
+ Maintainer		S Y Y N	The email address of the gatherer administrator.
+ Port 			S Y Y Y	The port number on which the gatherer listens.
+ Version 		N Y Y Y	The version number of the Harvest software (e.g. 1.2).
+ 
+ ! --------------- Recommended ---------------
+ Extended-Description	P N Y N A paragraph that describes the contents of the gatherer.
+ Broker	 		P N Y Y A list of host:port pairs that identify brokers that collect from this gatherer.
+ Keywords 		S N Y Y A list of words that characterize the contents of the gatherer.
+ Organization 		S N Y N The name of the organization that maintains the gatherer.
+ Leaf-Nodes		P N Y Y A list of leaf nodes for this gatherer.
+ Root-Nodes		P N Y Y A list of root nodes for this gatherer.
+ 
Index: harvest/src/broker/example/brokers/AdminTools/lib/hsrlibrary.pl
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/lib/hsrlibrary.pl:1.1
*** /dev/null	Thu Jul 10 19:50:54 1997
--- harvest/src/broker/example/brokers/AdminTools/lib/hsrlibrary.pl	Thu Jul 10 19:47:24 1997
***************
*** 0 ****
--- 1,288 ----
+ : # *-*-perl-*-*
+     eval 'exec perl -S $0 "$@"'
+     if $running_under_some_shell;  
+ 
+ # -----------------------------------------------------------------
+ # hsrlibrary.pl
+ # 
+ # Author: Mic Bowman
+ #
+ # Collection of functions used by most HSR programs.
+ # 
+ # -----------------------------------------------------------------
+ require 'configure.pl';
+ 
+ # -----------------------------------------------------------------
+ # Globals
+ # -----------------------------------------------------------------
+ $HSR_Configuration = "hsr.conf";
+ @HSR_Month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", 
+ 	      "Jul", "Aug", "Sep", "Oct", "Nov", "Dec");
+ 
+ # -----------------------------------------------------------------
+ # HSR_Initialize
+ #
+ # The hsr configuration file is found by searching the following
+ # locations:
+ #	- The first argument of the command line
+ #       - The HSR_CONFIGURATION variable
+ #       - The HSR_HOME environment variable (plus hsr.conf)
+ #       - The current directory (plus hsr.conf)
+ # -----------------------------------------------------------------
+ sub HSR_Initialize {
+     local($tag);
+ 
+     undef %Config;
+ 
+     # Process the configuration file
+     if (defined $ENV{"HSR_CONFIGURATION"}) {
+ 	&ReadConfiguration($ENV{"HSR_CONFIGURATION"},*Config);
+     } elsif (defined $ENV{"HSR_HOME"}) {
+ 	&ReadConfiguration($ENV{"HSR_HOME"} . "/$HSR_Configuration",*Config);
+     } else {
+ 	&ReadConfiguration($HSR_Configuration,*Config);
+     }
+ 
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_FileName -- create a file name from the configuration variables
+ # -----------------------------------------------------------------
+ sub HSR_FileName
+ {
+     local($cffamily,$cfclass) = @_;
+     local($dir,$file);
+ 
+     if (defined $Config{"$cffamily.$cfclass"}) {
+ 	$file = $Config{"$cffamily.$cfclass"};
+ 	return $file if ($file =~ m@^/.+$@);
+     }
+ 
+     if (defined $Config{"$cffamily.Home"}) {
+ 	$dir = $Config{"$cffamily.Home"};
+ 	$file = "$dir/$file";
+     }
+ 
+     return $file;
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_ParseSOIF() -- parse a SOIF object from IFILE.
+ # -----------------------------------------------------------------
+ sub HSR_ParseSOIF {
+     local($ifile);
+     $ifile = @_[0];
+ 
+     return () if (eof($ifile));
+ 
+     local($template_type) = "UNKNOWN";
+     local($url) = "UNKNOWN";
+     local(%SOIF);
+     undef %SOIF;
+     local ($attr, $vsize, $value, $end_value);
+ 
+     while (<$ifile>) {
+ 	last if (/^\@\S+\s*{\s*\S+\s*$/o);
+     }
+     if (/^\@(\S+)\s*{\s*(\S+)\s*$/o) {
+ 	$template_type = $1, $url = $2;
+ 	$SOIF{"Type"} = $template_type;
+ 	$SOIF{"Description"} = $url;
+     } else {
+ 	return ($template_type, $url, %SOIF);	# done
+     }
+ 
+     while (<$ifile>) {
+ 	if (/^\s*([^{]+){(\d+)}:\t(.*\n)/o) {
+ 	    $attr = $1;
+ 	    $vsize = $2;
+ 	    $value = $3;
+ 	    if (length($value) < $vsize) {
+ 		$nleft = $vsize - length($value);
+ 		$end_value = "";
+ 		$x = read($ifile, $end_value, $nleft);
+ 		die "Cannot read $nleft bytes: $!" 
+ 		    if ($x != $nleft);
+ 		$value .= $end_value;
+ 		undef $end_value;
+ 	    }
+ 	    chop ($value) if ($value =~ /\n$/);
+ 	    $SOIF{$attr} = $value;
+ 	    undef $value;
+ 	    undef $end_value;
+ 	    next;
+ 	} 
+ 	last if (/^}/o);
+     }
+ 
+     return ($template_type, $url, %SOIF);
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_ParseDescription
+ # -----------------------------------------------------------------
+ sub HSR_ParseDescription {
+     local($file) = @_;
+     local($url,$type);
+     local(%obj);
+ 
+     $url = "";
+     $type = "UNKNOWN";
+ 
+     open(DFILE,"<$file") ||
+ 	die "Unable to open description file ($file): $!\n";
+ 
+     while (<DFILE>) {
+         last if (m@</HEAD>@);
+ 
+ 	if (($key, $val) = m@<META NAME="([^"]+)" +CONTENT="(.*)">@) {
+ 	    $obj{$key} = $val;
+ 	}
+     }
+ 
+     $key = "";
+     $val = "";
+     while (<DFILE>) {
+ 	if (m@<DT>(\S*)</DT><DD>(\S*)</DD>@) {
+ 	    $key = $1; $val = $2;
+ 	    $obj{$key} = $val if ($key !~ /^\s*$/);
+ 	} 
+ 	last if (m@</BODY>@);
+     }
+ 
+     if (defined $obj{"HarvestServerType"}) {
+ 	$type = $obj{"HarvestServerType"};
+     } elsif (defined $obj{"Type"}) {
+ 	$type = $obj{"Type"};
+     }
+ 
+     if (defined $obj{"Description-File"}) {
+ 	$url = $obj{"Description-File"};
+     }
+     close(DFILE);
+ 
+     return ($type,$url,%obj);
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_WriteDescription
+ # -----------------------------------------------------------------
+ sub HSR_WriteDescription {
+     local($output);
+     local($file,$type,$url,%obj) = @_;
+     local($key);
+ 
+     system("cp $file $file.old");
+ 
+     # Create the description file
+     $output = "";
+     foreach $key (sort keys %obj) {
+ 	next if (!defined $obj{$key});
+ 	$val = $obj{$key};
+ 	$output .= "<DT>$key</DT><DD>$val</DD>\n";
+     }
+ 	
+     open(DFILE,">$file") ||
+ 	die "Unable to open description file $file: $!\n";	
+ 
+     print DFILE "<HTML>\n<HEAD>\n";
+ 
+     foreach $key (keys %obj) {
+ 	next if (!defined $obj{$key});
+ 	$val = $obj{$key};
+ 	print DFILE "<META NAME=\"$key\" CONTENT=\"$val\">\n";
+     }
+ 
+     print DFILE "<TITLE>Description for ", $obj{"Description"}, " Harvest Broker</TITLE>\n";
+     print DFILE "</HEAD>\n";
+     print DFILE "<BODY>\n";
+     print DFILE "<H4>Description for ", $obj{"Description"}, " Harvest Broker</H4>\n";
+     print DFILE "<HR>\n";
+ 
+     print DFILE "$output";
+     print DFILE "</BODY>\n</HTML>\n";
+ 
+     close(DFILE);
+     chmod(0666, "$file");
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_DisplayDescription
+ # -----------------------------------------------------------------
+ sub HSR_DisplayDescription {
+     local($file) = @_;
+ 
+     open(DFILE,"<$file") ||
+ 	&CGIError("Unable to open description file $sfile");
+ 
+     while (<DFILE>) {
+ 	print $_;
+     }
+ 
+     close(DFILE);
+ }
+ 
+ # -----------------------------------------------------------------
+ # MakeAttribute() -- make a single attribute
+ # -----------------------------------------------------------------
+ sub HSR_MakeAttribute {
+     local($k, $v, $a) = @_;
+     return "" if (($v =~ /^\s*$/) || ($k =~ /^\s*$/));
+ 
+     $k = substr($k,9) if ($k =~ /^Gatherer-/) ;
+     $k = substr($k,7) if ($k =~ /^Broker-/) ;
+ 
+     $a = $k . "{" . length($v) . "}:\t" . $v . "\n";
+     return $a;
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_DumpSOIFObjects() -- Dump the contents of the information array
+ # to the output file.
+ # -----------------------------------------------------------------
+ sub HSR_DumpSOIFObjects {
+     local($ofile,*info) = @_;
+     local($k,$t,$u);
+ 
+     foreach $k (sort keys %info) {
+ 	($t, $u) = split('&',$k,2);
+ 	next if (! defined $info{$k});
+ 	print $ofile "\@$t { $u\n";
+ 	print $ofile $info{$k};
+ 	print $ofile "}\n";
+     }
+ }
+     
+ # -----------------------------------------------------------------
+ # HSR_AddSOIFObject() -- Add an object to the information array.
+ # -----------------------------------------------------------------
+ sub HSR_AddSOIFObject {
+     local($type,$url,%obj,*info) = @_;
+     local($output);
+ 
+     $output = "";
+     foreach $k (sort keys %obj) {
+ 	$output .= &HSR_MakeAttribute($k,$obj{$k});
+     }
+     $info{join('&',$type,$url)} = $output;
+ }
+ 
+ # -----------------------------------------------------------------
+ # HSR_ReadSOIFObjects() -- Read SOIF objects from a file and add them to
+ # the information structure.
+ # -----------------------------------------------------------------
+ sub HSR_ReadSOIFObjects {
+     local($ifile,*info) = @_;
+     local($output);
+     local($stype, $surl, %sobj);
+ 
+     ($stype, $surl, %sobj) = &HSR_ParseSOIF("$ifile");
+     while (defined $stype) {
+ 	&HSR_AddSOIFObject($stype,$surl,%sobj,*info);
+     } continue {
+ 	($stype, $surl, %sobj) = &HSR_ParseSOIF("$ifile");
+     }    
+ }
+ 
+ # -----------------------------------------------------------------
+ 1
Index: harvest/src/broker/example/brokers/AdminTools/tools/README
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/tools/README:1.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/AdminTools/tools/README	Tue Jul  1 20:42:19 1997
***************
*** 0 ****
--- 1,24 ----
+ All the tools can be run interactively via Netscape by opening the 
+ AdminTools/cgi-bin/bdhome.cgi file.
+ 
+ To run the tools(except bdmanual, since it requires the fields to be updated 
+ from the Netscape forms interface) from the command line, the arguments to be 
+ supplied are : 
+ TYPE  description_file  server_home_dir
+ 
+ Eg: BROKER /local/harvest/brokers/Test/decsription.html /local/harvest/brokers/Test
+ 
+ bddisplay	display the contents of the description file
+ 
+ bdlogs		adds to the description file,
+ 		*  Commonly Asked Queries
+ 		*  Commonly returned objects 
+ 
+ bdmanual	Perform a manual update of the server description file.
+ 
+ bdstop		parses the .glimpse_index file in the broker directory 
+ 		and extracts keywords from it.  It adds the unique keywords
+ 		present in the broker and their frequency of occurance.  
+ 		This is obtained by taking the keywords in the broker minus
+ 		the words in the stop list.  The stop list contains words
+ 		that are common among all the brokers in the HSR.
Index: harvest/src/broker/example/brokers/AdminTools/tools/bddisplay
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/tools/bddisplay:1.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/AdminTools/tools/bddisplay	Thu Jul 10 19:47:29 1997
***************
*** 0 ****
--- 1,41 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bddisplay
+ # 
+ # Author: Mic Bowman
+ #
+ # display the contents of the description file
+ # -----------------------------------------------------------------
+ $ENV{'HARVEST_HOME'} = "/usr/local/harvest"
+ 	if (!defined($ENV{'HARVEST_HOME'}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'bdlibrary.pl';
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$sfile,$shome) = @ARGV;
+     local(%obj,%tags,%dlist);
+     local($filename);
+ 
+     # Initialize
+     &HSR_Initialize;
+ 
+     # Make sure the file is in the descriptions home directory
+     &CGIError("Invalid description file: $sfile")
+ 	if (index($sfile,$Config{"Descriptions.Home"}) != 0);
+ 
+     &CGIError("Unable to read description file $sfile")
+ 	if (! -r $sfile);
+ 
+     &HSR_DisplayDescription($sfile);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/tools/bdlogs
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/tools/bdlogs:1.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/AdminTools/tools/bdlogs	Thu Jul 10 19:47:30 1997
***************
*** 0 ****
--- 1,231 ----
+ #!/usr/local/bin/perl
+ 
+ # dont use eval for cgi scripts
+ 
+ #-----------------------------------------------------------------------------
+ # bdlogs  - Creates a better decription file by adding to it:
+ #		  a.) Commonly Asked Queries
+ #		  b.) Embedded Objects - most commonly returned
+ #
+ # Author : Raji Gopalakrishnan
+ # Version: 2.0
+ #-----------------------------------------------------------------------------
+ $ENV{'HARVEST_HOME'} = "/usr/local/harvest"
+     unless (defined($ENV{'HARVEST_HOME'}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'bdlibrary.pl';
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ 
+ #-----------------------------------------------------------------------------
+ # Globals
+ #-----------------------------------------------------------------------------
+ undef @SortedQ;
+ undef @SortedO;
+ undef $brokerdir;
+ 
+ #-----------------------------------------------------------------------------
+ #   Subroutines
+ #-----------------------------------------------------------------------------
+ 
+ #-----------------------------------------------------------------------------
+ # SortArr - sort associative array based on values
+ #-----------------------------------------------------------------------------
+ 
+ sub SortArr{
+     local(%arr, $k, @vals);
+     %arr = @_;
+     foreach $k (keys %arr)
+     {
+ 	$arr{$k} .= "|" . $k;
+     }
+     @vals = sort values %arr;
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # ProcessLog - processes log file
+ #-----------------------------------------------------------------------------
+ 
+ sub ProcessLog{
+     local (%Queries, %Objects,$object,$query,$totalqs);
+     #%Queries keeps track of how many times a query is asked
+     #%Objects keeps track of how many times an object is accessed
+ 
+     undef %Queries;
+     undef %Objects;
+     $totalqs = 0;
+ 
+     while (<IFILE>)
+     {
+ 
+     if(/^QUERY\s.*/)
+     {
+ 	$totalqs++;
+ 	$_ = substr($_, rindex($_ , "#END")); 
+ 	chop;
+ 	s/#END\s//;
+ 	$query = $_;
+         if (! defined $Queries{$query} )
+ 	{
+ 	    $Queries{$query}=1;
+ 	}
+ 	else
+ 	{
+ 	    $Queries{$query}++;
+ 	}
+     }
+     if(/^QUERY\-RETURN.*/)
+     {
+ 	s/(\w+\-\w+\s+\d+::\s+)(\d+)(.*)/$2/;
+ 	chop;
+         $object = $_;
+ 	if (! defined $Objects{$object} )
+ 	{
+ 	    $Objects{$object}=1;
+ 	}
+ 	else
+ 	{
+ 	    $Objects{$object}++;
+ 	}
+     }
+ 
+     }    
+ 
+ 
+ #@SortedQ contains queries sorted in ascending order of their frequency of 
+ #occurance
+ #@SortedO contains objects sorted in ascending order of their frequency of
+ #occurance
+ #an entry in the above arrays would look like: frequency|query/object
+ 
+     @SortedQ = &SortArr (%Queries);
+     @SortedO = &SortArr (%Objects);
+     return ($totalqs);
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # GetData() - get data from logfile
+ #-----------------------------------------------------------------------------
+ sub GetData {
+     local ($filename) = @_;
+     open(IFILE, $filename) || &CGIError("Unable to open $filename ");
+     local ($totalqs) = &ProcessLog;
+     close (IFILE);
+     return ($totalqs);
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # EmbedObject - reads object directories and embeds the objects into desc file
+ #  Repeat for the last how many ever entries in the array @SortedO
+ #  Here, we do it only for the last entry
+ #-----------------------------------------------------------------------------
+ sub EmbedObject {
+     local(*obj) = @_;
+     local($cnt,$max,$object,$dir,$filename,$k);
+     local($type,$url,%SOIF);
+ 
+     $max = 3;
+     $max = $Config{"Objects"} if (defined $Config{"Objects"});
+ 
+     for ($cnt = 1; $cnt < $max; $cnt++) {
+ 	$object = $SortedO[$#SortedO - $cnt + 1];
+ 	last if (! defined $object);
+ 
+ 	$object =~ s/(^\d+\|)(.*)/$2/;
+ 	$dir = substr($object, -2, 2);
+ 	$dir =~ s/^0//;
+ 	$filename = "$brokerdir/objects/$dir/OBJ$object";
+ 
+ 	#aug 4th - if you cant access this object, it has been deleted - 
+ 	#get the next one!!
+ 	unless (open(IFILE, "<$filename")){next};
+ 	($type, $url , %SOIF) = &HSR_ParseSOIF('IFILE');
+ 	close(IFILE);
+ 
+ 	foreach $k (keys %SOIF)	{
+ 	    $obj{"Embed<$cnt>-$k"} = $SOIF{$k};
+ 	}
+     }
+ 
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # EmbedQuery - embeds queries into description file
+ #-----------------------------------------------------------------------------
+ sub EmbedQuery {
+     local(*obj) = @_;
+     local($qstring,$query,$cnt,$max);
+ 
+     $max = 2;
+     $max = $Config{"Queries"} if (defined $Config{"Queries"});
+ 
+     $qstring = "";
+ 
+     # do for how many ever popular queries you want to put in.
+     for ($cnt = 0; $cnt < $max;$cnt++) {
+ 	$query =  $SortedQ[$#SortedQ-$cnt];
+ 	$query =~ s/(^\d+\|)(.*)/$2/;
+ 	$qstring .= $query . "\n";
+     }
+ 
+     $obj{"Common-Queries"}  = $qstring;
+ }
+ 
+ #-----------------------------------------------------------------------------
+ #   Append - append stuff to appropriate description file
+ #-----------------------------------------------------------------------------
+ sub AppendFields {
+ 
+     local($totalqs,*obj) = @_;
+     local($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is);
+ 
+     #Add Total Number of Queries field!
+     $obj{"Total-Number-of-Queries"} = $totalqs;
+ 
+     #Add Number of Unique Queries filed!
+     $obj{"Number-of-Unique-Queries"} = $#SortedQ+1;
+ 
+     #Add Total number of Objects field!
+     $obj{"Number-of-Objects"} = $#SortedO+1;
+ 
+     ($sec,$min,$hour,$md,$mo,$yr,$wd,$yday,$is) = localtime(time);
+     $min = "0$min" if ($min < 10);
+     $hour = "0$hour" if ($hour < 10);
+     $obj{"Last-Update-Time"} = "$hour:$min " . @HSR_Month[$mo] . " $md, 19$yr";
+     $obj{"Last-Update"} = "Update Logs";
+ }
+ 
+ #-----------------------------------------------------------------------------
+ #   Main
+ #-----------------------------------------------------------------------------
+ sub Main {
+     local($type,$descfile,$bdir) = @ARGV;
+     local($ntype,$url,%obj,$totalqs);
+ 
+     &HSR_Initialize;
+     if (! defined $type || ! defined $descfile || ! defined $bdir) {
+ 	&CGIError("Usage : bdlogs <type> <descfile> <brokerdirectory>");
+     }
+ 
+     # Parse description file 
+     local($ntype,$url,%obj) = &HSR_ParseDescription($descfile);
+     &CGIError("Type mismatch in description file $ntype") 
+ 	if ($ntype ne $type);
+     
+     #Global variable $brokerdir
+     $brokerdir = $bdir;
+ 
+     # Process the log files and update the object
+     $totalqs = &GetData("$brokerdir/admin/LOG");
+     &EmbedObject(*obj);
+     &EmbedQuery(*obj);
+     &AppendFields($totalqs,*obj);
+     &HSR_WriteDescription ($descfile,$type,$url,%obj);
+     &HSR_DisplayDescription($descfile);
+     exit(0);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ 
Index: harvest/src/broker/example/brokers/AdminTools/tools/bdmanual
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/tools/bdmanual:1.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/AdminTools/tools/bdmanual	Thu Jul 10 19:47:31 1997
***************
*** 0 ****
--- 1,128 ----
+ #!/usr/local/bin/perl
+ # -----------------------------------------------------------------
+ # bdmanual
+ # 
+ # Author: Mic Bowman
+ #
+ # Perform a manual update of the server description file.
+ # -----------------------------------------------------------------
+ $ENV{'HARVEST_HOME'} = "/usr/local/harvest"
+ 	if (!defined($ENV{'HARVEST_HOME'}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ 
+ require 'bdlibrary.pl';
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ 
+ # -----------------------------------------------------------------
+ # BD_Update
+ # -----------------------------------------------------------------
+ sub BD_Update {
+     local(*obj,*tags) = @_;
+     local($mandatory,$manual,$auto,$desc);
+     local($man,$rec,$opt);
+     local($val,$box,$line);
+ 
+     $man = $rec = $opt = "";
+ 
+     # Create mandatory and recommended fields
+     foreach $key (sort keys %tags) {
+ 	($btype,$mandatory,$manual,$auto,$desc) = split('&',$tags{$key});
+ 	next if ($manual ne "Y"); # Skip automatic tags
+ 	$box = &BD_InputBox($btype,$key,$obj{$key});
+ 	$line = "<H4>$key</H4>\n<P>$desc<BR>$box</P>\n";
+ 	
+ 	$man = $man . $line if ($mandatory eq "Y");
+ 	$rec = $rec . $line if ($mandatory eq "N");
+     }
+ 
+     foreach $key (sort keys %obj) {
+ 	next if (defined $tags{$key});
+ 	$box = &BD_InputBox("S",$key,$obj{$key});
+ 	$line = "<H4>$key</H4>\n$box\n<P>\n";
+ 	$opt = $opt . $line;
+     }
+ 
+     return($man,$rec,$opt);
+ }
+ 
+ # -----------------------------------------------------------------
+ # -----------------------------------------------------------------
+ sub BD_Create {
+     local(*tags) = @_;
+     local($mandatory,$manual,$auto,$desc);
+     local($man,$rec,$opt);
+     local($val,$box,$line);
+     
+     $man = $rec = $opt = "";
+ 
+     foreach $key (sort keys %tags) {
+ 	($btype,$mandatory,$manual,$auto,$desc) = split('&',$tags{$key});
+ 	next if ($manual ne "Y"); # Skip automatic tags
+ 	$box = &BD_InputBox($btype,$key,"");
+ 	$line = "<H4>$key</H4>\n<P>$desc<BR>$box</P>\n";
+ 	
+ 	$man = $man . $line if ($mandatory eq "Y");
+ 	$rec = $rec . $line if ($mandatory eq "N");
+     }
+ 
+     # Add the description file
+     ($btype,$mandatory,$manual,$auto,$desc) = 
+ 	split('&',$tags{"Description-File"});
+     $box = &BD_InputBox($btype,"Description-File","");
+     $line = "<H4>Description-File</H4>\n<P>$desc<BR>$box</P>\n";
+     $man = $line . $man;
+ 
+     return ($man,$rec,$opt);
+ }
+ 
+ # -----------------------------------------------------------------
+ # Main
+ # -----------------------------------------------------------------
+ sub Main
+ {
+     local($stype,$sfile,$shome) = @ARGV;
+     local($type,$url,%obj,%tags);
+     local($updateurl, $addopturl);
+     local($filename);
+ 
+     &HSR_Initialize;
+     %tags = &BD_ReadTags($stype);
+ 
+     # Create the form entries
+     if (-r $sfile) {
+ 	($type,$url,%obj) = &HSR_ParseDescription($sfile);
+ 	&CGIError("Requested type is different than stored type: $type")
+ 	    if ($type ne $stype);
+ 	($man,$rec,$opt) = &BD_Update(*obj,*tags);
+     } else {
+ 	($man,$rec,$opt) = &BD_Create(*tags);
+     }
+ 
+     # Process the skeleton
+     $updateurl = &HSR_FileName("CGI","BDUpdate");
+     $addopturl = &HSR_FileName("CGI","BDAddOpt");
+     $filename = &HSR_FileName("Library","BDManual.Skel");
+ 
+     open(SFILE,"<$filename") ||
+ 	&CGIError("Unable to find skeleton file, check configuration");
+ 
+     while (<SFILE>) {
+ 	$_ =~ s/@FILE@/$sfile/g;
+ 	$_ =~ s/@TYPE@/$stype/g;
+ 	$_ =~ s/@HOME@/$shome/g;
+ 	$_ =~ s/@UPDATEURL@/$updateurl/g;
+ 	$_ =~ s/@ADDOPTURL@/$addopturl/g;
+ 	$_ =~ s/@MANDATORY@/$man/g;
+ 	$_ =~ s/@RECOMMENDED@/$rec/g;
+ 	$_ =~ s/@OPTIONAL@/$opt/g;
+ 	print $_;
+     }
+ 
+     close(SFILE);
+ }
+ 
+ # -----------------------------------------------------------------
+ &Main;
+ exit(0);
Index: harvest/src/broker/example/brokers/AdminTools/tools/bdstop
diff -c /dev/null harvest/src/broker/example/brokers/AdminTools/tools/bdstop:1.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/AdminTools/tools/bdstop	Thu Jul 10 19:47:32 1997
***************
*** 0 ****
--- 1,278 ----
+ #!/afs/tr/project/tools/bin/perl5
+ 
+ #-----------------------------------------------------------------------------
+ # bdstop
+ #
+ # Author     : Raji Gopalakrishnan
+ # Date       : Jun 20, 1996
+ # Version    : 2.0
+ # Description: This program parses the .glimpse_index file in the broker 
+ #	       directory and extracts keywords from it.
+ #-----------------------------------------------------------------------------
+ 
+ $ENV{'HARVEST_HOME'} = "/local/harvest"
+ 	if (!defined($ENV{'HARVEST_HOME'}));
+ $ENV{"HSR_HOME"} = "$ENV{'HARVEST_HOME'}/brokers/AdminTools";
+ push(@INC,$ENV{"HSR_HOME"} . "/lib");
+ require 'bdlibrary.pl';
+ require 'cgi.pl';
+ require 'hsrlibrary.pl';
+ 
+ #-----------------------------------------------------------------------------
+ # subroutines
+ #-----------------------------------------------------------------------------
+ 
+ 
+ #-----------------------------------------------------------------------------
+ # Process - process the array of words to remove suffixes & prefixes wherever
+ # possible.
+ #-----------------------------------------------------------------------------
+ sub Process {
+     local(@temp) = @_;
+     local($prev, $cand, @list);
+     @list = sort @temp;
+ 
+     $prev = $list[0];
+     for($i=0; $i < $#list; $i++) {
+ 	$word = $list[$i];
+ 	$cand = $word;
+ 
+ 	if ($cand =~ /^.*e[ds]$/ && $prev =~ /^.*e$/) {
+ 	    substr($cand, -1, 1) = "";
+ 	} elsif ($cand =~ /^.*e[ds]$/) {
+ 	    substr($cand, -2, 2) = "";
+ 	} 
+ 
+ 	$list[$i] = $cand if ($cand eq $prev);
+ 	if ($cand =~ /^.*s$/) {
+ 	    substr($cand, -1, 1) = "";
+ 	    $list[$i] = $cand if ($cand eq $prev);
+         }
+ 
+ 	$cand =~ s/ly// if ($cand =~ /^.*ly$/);
+ 	$list[$i] = $cand if ($cand eq $prev);
+ 	substr($cand, -2, 2) = "" if ($cand =~ /^.*al$/);
+ 	$list[$i] = $cand if ($cand eq $prev);
+ 
+ 	substr($cand, -2, 2) = "" if ($cand =~ /^.*ic$/);
+ 	$cand =~ s/ing// if ($cand =~ /^.*ing$/);
+ 	$cand =~ s/able// if ($cand =~ /^.*able$/);
+ 	$cand =~ s/ible// if ($cand =~ /^.*ible$/);
+ 
+ 	substr($cand, -3, 3) = "" if ($cand =~ /^.*ant$/);
+ 	substr($cand, -3, 3) = "" if ($cand =~ /^.*ive$/);
+ 	substr($cand, -4, 4) = "" if ($cand =~ /^.*ance$/);
+ 	substr($cand, -4, 4) = "" if ($cand =~ /^.*ment$/);
+ 	substr($cand, -3, 3) = "" if ($cand =~ /^.*ion$/);
+ 
+ 	if ($cand eq $prev) {
+ 	    $list[$i] = $cand;
+ 	} else {
+ 	    $cand = $cand . "e";
+ 	    $list[$i] = $cand if ($cand eq $prev);
+ 	}
+ 	$prev = $list[$i];
+ 
+ 	if ($list[$i] eq $cand) {
+ 	    $count{$list[$i]} += $count{$word};
+ 	    $numdbs{$list[$i]} += $numdbs{$word};
+ 	}
+     }
+     return (@list);
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # ParseIndex - parses the index file
+ #-----------------------------------------------------------------------------
+ 
+ sub ParseIndex {
+ 
+     local($brokerdir) = @_;
+     local(%seen);
+ 
+     local($filename) = $brokerdir . "/.glimpse_index";
+     open (IFILE, $filename) || &CGIError("Unable to open $filename");
+     while (<IFILE>)
+     {
+ 	($word,$junk) = split (/\002/,$_);
+ 	
+ 	# do some preprocessing of the words.
+ 	
+ 	substr($word, -3, 3) = "y" if ($word =~ /^.*ies$/);
+ 	substr($word, -5, 5) = "le" if ($word =~ /^.*ility$/);
+ 
+ 	# remove words that don't contain atleast one vowel (filtering out junk)
+ 	push(@words,$word) if ($word =~ /^[a-z]{3,25}/ && $word =~ /^.*[aeiou].*$/);
+     } 
+     close IFILE;
+ 
+     foreach (@words) {
+ 	unless ($seen{$_}++) {
+ 	    &GetStats($brokerdir, $_);
+ 	}
+     }
+ 
+     @words = &Process(@words);
+     return (@words);
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # ModifyStopList - takes an array of words as an argument and modifies 
+ #                  existing stoplist to intersection of stoplist and the array
+ #                  or, if stoplist does not exist , creates a stoplist = array
+ #                  returns an array of words = original array - new stoplist
+ #-----------------------------------------------------------------------------
+ 
+ sub ModifyStopList {
+    
+     local(@newarray) = @_;
+     local(@stoplist);
+     local(@InStopNotinNew);
+ 
+     unless (open (SLIST, $stoplistname)) {
+ 	&WriteStop(@newarray); 
+         return (@newarray);
+     }
+ 
+     while (<SLIST>) {
+ 	chop;
+ 	push(@stoplist, $_);
+     }
+     close(SLIST);
+ 
+     @InStopNotinNew = &diff(*newarray, *stoplist);
+     #modify stoplist to intersection of stoplist and newarray
+     @stoplist = &diff( *InStopNotinNew, *stoplist);
+     &WriteStop(@stoplist);
+     #modify new array to difference of newarray and modified stoplist
+     @newarray = &diff( *stoplist, *newarray);
+     return (@newarray);
+ }
+ 
+ #-----------------------------------------------------------------------------
+ # Diff - computes the difference between arrays
+ #-----------------------------------------------------------------------------
+ 	    
+ sub diff {
+     local(%MARK, @result);
+     local (*array1, *array2) = @_;
+     grep($MARK{$_}++,@array1);
+     @result = grep(!$MARK{$_},@array2);
+     return @result;
+ }
+      
+ 
+ #-----------------------------------------------------------------------------
+ # WriteArray - writes an array onto STDOUT - in html format
+ #-----------------------------------------------------------------------------
+ 
+ sub WriteArray {
+ 
+     local(@array) = @_;
+     local(%seen);
+ 
+     ($type, $url, %obj) = &HSR_ParseDescription($descfile);
+     undef $obj{"Keywords"};
+     undef $obj{"keywords"};
+     
+     print STDOUT <<EOM;
+ <HTML>
+ <HEAD>
+ <TITLE>List of Unique Keywords in this broker</TITLE>
+ </HEAD>
+ <BODY>
+ <H1>List of Unique Keywords in this broker</H1>
+ <P> This is a list of keywords unique to this broker with their corresponding
+ frequencies and the number of databases they have occured in.  It is computed by 
+ taking the difference of the set of all keywords of this broker and the 
+ Stoplist.  The StopList consists of words common to all brokers in the HSR.</P>
+ EOM
+ 
+     for $key (sort @array) {
+ 	if ($key =~ /^[a-z]+$/) {
+ 	    unless ($seen{$key}++) {
+ 		#  include only words that occur in the broker
+ 		if ($count{$key} > 0) {
+ 	            print STDOUT $key, "\t$count{$key}\t$numdbs{$key}<BR>\n"; 
+ 	            $obj{"Keywords"} = $obj{"Keywords"} . $key . "\t" . $count{$key} . "\t" . $numdbs{$key} . "\t";
+ 		}
+ 	    }
+ 	}
+     }
+     print STDOUT "</BODY>\n</HTML>\n";
+     &HSR_WriteDescription($descfile, $type, $url, %obj);
+ }
+ 
+ 
+ #-----------------------------------------------------------------------------
+ # GetStats - for a given word, calculates 
+ # 	     * the number of objects that contain the word, and 
+ #	     * the sum of the number of docs that contain the word.
+ #
+ # this information is used by the HSR to find suitable brokers to send queries to.
+ #-----------------------------------------------------------------------------
+ sub GetStats {
+     local ($brokerdir, $word) = @_;
+     local ($objdir) = $brokerdir . "/objects/*/*";
+     local ($resfile) = $ENV{"HSR_HOME"} . "/tmp/result";
+ 
+     $numdbs{$word} = 0;
+     $count{$word} = 0;
+ 
+     $gcmd = "/local/harvest/lib/broker/glimpse -H \'" . $brokerdir . "\' -c -w -y " . $word;
+     $gcmd .= " > " . $resfile;
+ 
+     system ("$gcmd");
+     open (RFILE, "$resfile") || die "Cannot open file $resfile : $!\n" if -e $resfile;
+ 
+     while (<RFILE>) {
+ 	if (/^(.*):\s*(\d+)/) {
+ 	    $numdbs{$word}++;
+ 	    $count{$word} += $2;
+ 	}
+         last if (eof(RFILE));
+     }
+     close (RFILE);
+     system ("rm $resfile") ;
+ }
+ 
+ 
+ #-----------------------------------------------------------------------------
+ # WriteStop - writes an array into the stoplist file
+ #-----------------------------------------------------------------------------
+ 
+ sub WriteStop {
+ 
+     local(@array) = @_;
+     open (OFILE, ">$stoplistname") ||&CGIError("Unable to open $stoplistname");
+     for (sort @array) {
+ 	if ($_ =~ /^[a-z]+/) {
+             print OFILE $_, "\n" unless $seen{$_}++;
+ 	}
+     }
+     close OFILE;
+ }
+ 
+ 
+ 
+ #-----------------------------------------------------------------------------
+ # Main
+ #-----------------------------------------------------------------------------
+ sub Main{
+     local($type, $descfile, $bdir) = @ARGV;
+     local (@uniquearray,@words);
+     &HSR_Initialize;
+ 
+     $stoplistname = $Config{"Library.Home"} . "/stop.list";
+ 
+     if (! defined $type || ! defined $descfile || ! defined $bdir) 
+     { &CGIError("Usage: bdstop <type> <description file> <broker directory>");}
+     @words = &ParseIndex($bdir);
+     @uniquearray= &ModifyStopList(@words);
+     &WriteArray(@uniquearray);
+     exit(0); 
+ 
+ }
+ 
+ &Main;
+ 
Index: harvest/src/broker/example/brokers/skeleton/RunUpdate.in
diff -c harvest/src/broker/example/brokers/skeleton/RunUpdate.in:2.2 harvest/src/broker/example/brokers/skeleton/RunUpdate.in:2.3
*** harvest/src/broker/example/brokers/skeleton/RunUpdate.in:2.2	Tue Apr 22 19:38:32 1997
--- harvest/src/broker/example/brokers/skeleton/RunUpdate.in	Tue Jul  1 20:42:27 1997
***************
*** 1,70 ****
! #!/bin/sh
! #
! #  Updates the description for the @THIS_BROKER_NAME@ broker.
  #  The Broker's configuration file is available as $1
  #
  #  Created by @CONTACT_EMAIL@
  #
! HARVEST_HOME=@THIS_HARVEST_HOME@; export HARVEST_HOME
! PATH=${HARVEST_HOME}/bin:${HARVEST_HOME}/lib/broker:$PATH:/bin; export PATH
! d=@THIS_BROKER_DIR@
  
! Version="1.3"
  
! # set some variables for the HTML file...
  
! IndexType=`egrep -i -e '^Indexer-Type' $1 | awk '{print $2}'`
! WebServer=`egrep -i -e '^Web-Server' $1 | awk '{print $2}'`
! BrokerHPP=`egrep -i -e '^Broker-Home-Page' $1 | awk '{print $2}'`
! Port=`egrep -i -e '^Port' $1 | awk '{print $2}'`
! BrokerHP="http://${WebServer}${BrokerHPP}"
! ctime=`date`
  
! # change into the broker directory
! cd $d
  
! cpoints=`egrep -v -e '^#' admin/Collection.conf`
  
! # commands to update description file go here...
! desc="${d}/description.html"
! mv $desc $desc.old
! cat <<EOM >$desc
! <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
! <HTML>
! <HEAD>
! <META NAME="HarvestServerType" CONTENT="BROKER">
! <META NAME="HarvestServerHost" CONTENT="@THIS_BROKER_HOST@">
! <META NAME="HarvestServerPort" CONTENT="$Port">
! <TITLE>Description for @THIS_BROKER_NAME@ Harvest Broker</TITLE>
! </HEAD>
! <BODY>
! <H2>Description for @THIS_BROKER_NAME@ Harvest Broker</H2>
! <HR>
! <H4>Mandatory information</H4>
! <DT>Description</DT><DD>UNKNOWN</DD>
! <DT>Description-File</DT><DD>@THIS_BROKER_DESC_URL@</DD>
! <DT>Host</DT><DD>@THIS_BROKER_HOST@</DD>
! <DT>Last-Update</DT><DD>@THIS_BROKER_ADMIN_PROCESS@</DD>
! <DT>Last-Update-Time</DT><DD>$ctime</DD>
! <DT>Maintainer</DT><DD>@CONTACT_EMAIL@</DD>
! <DT>Name</DT><DD>@THIS_BROKER_NAME@</DD>
! <DT>Port</DT><DD>$Port</DD>
! <DT>Version</DT><DD>$Version</DD>
! <HR>
! <H4>Recommended information</H4>
! <DT>Extended-Description</DT><DD>UNKNOWN</DD>
! <DT>Gatherer</DT><DD>$cpoints</DD>
! <DT>Keywords</DT><DD>UNKNOWN</DD>
! <DT>Organization</DT><DD>UNKNOWN</DD>
! <HR>
! <H4>Optional information</H4>
! <DT>Broker-Home-Page</DT><DD>$BrokerHP</DD>
! <DT>Index-Type</DT><DD>$IndexType</DD>
! <DT>Replicas</DT><DD>@REPLICAS@</DD>
! </BODY>
! </HTML>
! EOM
  
! # make sure the HTTP server can serve the description file
! chmod o+r $desc
! exit 0
--- 1,48 ----
! : # *-*-perl-*-*
!     eval 'exec perl -S $0 "$@"'
!     if $running_under_some_shell;
! 
! #  Updates the description if any information has been 
! #  changed in the config file for the @THIS_BROKER_NAME@ broker.
  #  The Broker's configuration file is available as $1
  #
  #  Created by @CONTACT_EMAIL@
  #
! $ENV{'HARVEST_HOME'} = "@THIS_HARVEST_HOME@"
!     unless (defined($ENV{'HARVEST_HOME'}));
! $ENV{'BROKER_DIR'} = "@THIS_BROKER_DIR@";
  
! push(@INC,$ENV{'HARVEST_HOME'} . "/brokers/AdminTools/lib");
! require 'hsrlibrary.pl';
  
! $filename = "$ENV{'BROKER_DIR'}/admin/broker.conf";
! $dfile = "$ENV{'BROKER_DIR'}/description.html";
  
! open (CFILE, "<$filename") ||
!     die "Unable to open configuration file $filename\n";
  
! # read the description file and get the information.
! local($type,$url,%obj) = &HSR_ParseDescription($dfile);
  
! # reads values from the config file
! while (<CFILE>) {
!     next if /#.*/;
!     last if (eof(CFILE));
  
!     # if any of the values in the config file have been changed,
!     # update the description file to reflect these changes.
!     if (/^(\S*)\s+(\S*)$/) {
! 	$key = $1; $val = $2;
! 	if (defined ($obj{"$key"}) && $val ne $obj{"$key"}) {
! 	    $obj{"$key"} = $val;
! 	    $update = "TRUE";
! 	}
!     }
! }
  
! # update the description file only if necessary.
! if ($update eq "TRUE") {
!     $obj{'Last-Update-Time'} = `date`;
!     system("cp $dfile $dfile.old");
!     &HSR_WriteDescription ($dfile,$type,$url,%obj);
! }
Index: harvest/src/broker/example/brokers/skeleton/description.html.in
diff -c /dev/null harvest/src/broker/example/brokers/skeleton/description.html.in:2.1
*** /dev/null	Thu Jul 10 19:50:55 1997
--- harvest/src/broker/example/brokers/skeleton/description.html.in	Tue Jul  1 20:42:27 1997
***************
*** 0 ****
--- 1,40 ----
+ <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+ <HTML>
+ <HEAD>
+ <META NAME="Indexer-Type" CONTENT="@THIS_BROKER_INDEXER_TYPE@">
+ <META NAME="Type" CONTENT="BROKER">
+ <META NAME="Description-File" CONTENT="@THIS_BROKER_DESC_URL@">
+ <META NAME="Description" CONTENT="@THIS_BROKER_DESCRIPTION@">
+ <META NAME="Port" CONTENT="@THIS_BROKER_PORT@">
+ <META NAME="Version" CONTENT="2.0">
+ <META NAME="Host" CONTENT="@THIS_HTTP_HOST@">
+ <META NAME="Broker-Home-Page" CONTENT="http://@THIS_HTTP_HOST@/Harvest/brokers/@THIS_BROKER_SHORT_NAME@/query.html">
+ <META NAME="Maintainer" CONTENT="@CONTACT_EMAIL@">
+ 
+ <TITLE>Description for @THIS_BROKER_NAME@ Harvest Broker</TITLE>
+ </HEAD>
+ <BODY>
+ <H4>Description for @THIS_BROKER_NAME@ Harvest Broker</H4>
+ <HR>
+ <H4>Mandatory information</H4>
+ <DT>Description</DT><DD>@THIS_BROKER_DESCRIPTION@</DD>
+ <DT>Description-File</DT><DD>@THIS_BROKER_DESC_URL@</DD>
+ <DT>Host</DT><DD>@THIS_BROKER_HOST@</DD>
+ <DT>Last-Update</DT><DD>@THIS_BROKER_ADMIN_PROCESS@</DD>
+ <DT>Last-Update-Time</DT><DD>@UPDATE_TIME@</DD>
+ <DT>Maintainer</DT><DD>@CONTACT_EMAIL@</DD>
+ <DT>Name</DT><DD>@THIS_BROKER_NAME@</DD>
+ <DT>Port</DT><DD>@THIS_BROKER_PORT@</DD>
+ <DT>Version</DT><DD>2.0</DD>
+ <HR>
+ <H4>Recommended information</H4>
+ <DT>Extended-Description</DT><DD>UNKNOWN</DD>
+ <DT>Keywords</DT><DD>UNKNOWN</DD>
+ <DT>Organization</DT><DD>UNKNOWN</DD>
+ <HR>
+ <H4>Optional information</H4>
+ <DT>Broker-Home-Page</DT><DD>http://@THIS_HTTP_HOST@/Harvest/brokers/@THIS_BROKER_SHORT_NAME@/query.html</DD>
+ <DT>Indexer-Type</DT><DD>@THIS_BROKER_INDEXER_TYPE@</DD>
+ <DT>Replicas</DT><DD>@REPLICAS@</DD>
+ </BODY>
+ </HTML>
Index: harvest/src/common/url/http.c
diff -c harvest/src/common/url/http.c:2.2 harvest/src/common/url/http.c:2.3
*** harvest/src/common/url/http.c:2.2	Mon Mar 24 23:56:13 1997
--- harvest/src/common/url/http.c	Tue Jul  1 20:33:33 1997
***************
*** 1,4 ****
! static char rcsid[] = "http.c,v 2.2 1997/03/24 23:56:13 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
--- 1,4 ----
! static char rcsid[] = "http.c,v 2.3 1997/07/01 19:33:33 sxw Exp";
  /*
   *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
   *
***************
*** 241,246 ****
--- 241,251 ----
   *
   *  Uses the HTTP/1.0 protocol as described in:
   *      http://info.cern.ch/hypertext/WWW/Protocols/HTTP/HTTP2.html
+  *
+  * Patched by Hrvoje Stipetic <hrvoje.stipetic@hck.hr> to support
+  * modification of User-Agent: and From: HTTP headers.  Uses
+  * environment variables HARVEST_USER_AGENT and HARVEST_MAINTAINER_ADDRESS.
+  *
   */
  int http_get(up)
       URL *up;
***************
*** 323,331 ****
  	reqbuf = create_buffer(BUFSIZ);
  	sprintf(buf, "GET %s HTTP/1.0\r\n", request);
  	add_buffer(reqbuf, buf, strlen(buf));
! 	sprintf(buf, "User-Agent: Harvest/%s\r\n", HARVEST_VERSION);
  	add_buffer(reqbuf, buf, strlen(buf));
! 	sprintf(buf, "From: %s@%s\r\n", getmylogin(), getfullhostname());
  	add_buffer(reqbuf, buf, strlen(buf));
  /* A minor HTTP/1.1 addition - make Harvest send the Host: header --JLa */
          sprintf(buf, "Host: %s\r\n",up->host);
--- 328,345 ----
  	reqbuf = create_buffer(BUFSIZ);
  	sprintf(buf, "GET %s HTTP/1.0\r\n", request);
  	add_buffer(reqbuf, buf, strlen(buf));
! 	if (getenv("HARVEST_USER_AGENT") != NULL)
! 		sprintf(buf, "User-Agent: %s (Harvest/%s)\r\n",
! 			getenv("HARVEST_USER_AGENT"), HARVEST_VERSION);
! 	else
! 		sprintf(buf, "User-Agent: Harvest/%s\r\n", HARVEST_VERSION);
  	add_buffer(reqbuf, buf, strlen(buf));
! 	if (getenv("HARVEST_MAINTAINER_ADDRESS") != NULL)
! 		sprintf(buf, "From: %s\r\n",
! 			getenv("HARVEST_MAINTAINER_ADDRESS"));
! 	else 
! 		sprintf(buf, "From: %s@%s\r\n",
! 			getmylogin(), getfullhostname());
  	add_buffer(reqbuf, buf, strlen(buf));
  /* A minor HTTP/1.1 addition - make Harvest send the Host: header --JLa */
          sprintf(buf, "Host: %s\r\n",up->host);
Index: harvest/src/gatherer/enumerate/httpenum-breadth.c
diff -c harvest/src/gatherer/enumerate/httpenum-breadth.c:2.6 harvest/src/gatherer/enumerate/httpenum-breadth.c:2.7
*** harvest/src/gatherer/enumerate/httpenum-breadth.c:2.6	Mon May 26 17:50:26 1997
--- harvest/src/gatherer/enumerate/httpenum-breadth.c	Tue Jul  1 20:29:58 1997
***************
*** 1,4 ****
! static char rcsid[] = "httpenum-breadth.c,v 2.6 1997/05/26 16:50:26 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
--- 1,4 ----
! static char rcsid[] = "httpenum-breadth.c,v 2.7 1997/07/01 19:29:58 sxw Exp";
  /*
   *  httpenum-breadth.c - Breadth first RootNode URL enumerator for HTTP URLs
   *
***************
*** 526,532 ****
  	strstr(up->http_mime_hdr, "text/html") == NULL) {
          fprintf(stdout, "%s\t%s\n", up->url, up->md5);	/* URL <tab> MD5 */
          fflush(stdout);
! 	return 0;
      }
  
      /* Extract the HREF's */
--- 526,532 ----
  	strstr(up->http_mime_hdr, "text/html") == NULL) {
          fprintf(stdout, "%s\t%s\n", up->url, up->md5);	/* URL <tab> MD5 */
          fflush(stdout);
! 	return -1;
      }
  
      /* Extract the HREF's */
***************
*** 533,545 ****
  
      if (pipe(pipefds) < 0) {
  	log_errno("pipe");
! 	return 0;
      }
      if ((pid = fork()) < 0) {
  	log_errno("fork");
  	close(pipefds[0]);
  	close(pipefds[1]);
! 	return 0;
      }
      if (pid == 0) {		/* child: HTMLurls */
  	enum_url = (char *) xmalloc(strlen(up->url) + 20);
--- 533,545 ----
  
      if (pipe(pipefds) < 0) {
  	log_errno("pipe");
! 	return -1;
      }
      if ((pid = fork()) < 0) {
  	log_errno("fork");
  	close(pipefds[0]);
  	close(pipefds[1]);
! 	return -1;
      }
      if (pid == 0) {		/* child: HTMLurls */
  	enum_url = (char *) xmalloc(strlen(up->url) + 20);
***************
*** 564,570 ****
      close(pipefds[1]);		/* parent wont write */
      if ((fp = fdopen(pipefds[0], "r")) == NULL) {
  	log_errno("fdopen");
! 	return 0;
      }
      /* 
       *  For each HREF pointer, convert it to a URL, and add it to
--- 564,570 ----
      close(pipefds[1]);		/* parent wont write */
      if ((fp = fdopen(pipefds[0], "r")) == NULL) {
  	log_errno("fdopen");
! 	return -1;
      }
      /* 
       *  For each HREF pointer, convert it to a URL, and add it to
***************
*** 678,684 ****
  static void sigdie(x)
       int x;
  {
!     int i;
  #ifdef USE_HOST_CACHE
      dump_host_cache(42, 9);
  #endif
--- 678,684 ----
  static void sigdie(x)
       int x;
  {
!     int i, sum;
  #ifdef USE_HOST_CACHE
      dump_host_cache(42, 9);
  #endif
***************
*** 698,708 ****
      (void) unlink(md5db_filename);
      xfree(md5db_filename);
  
!     for (i = 0; i < 100; i++) {
  	if (i > max_depth && depth_hist[i] == 0)
  	    break;
! 	Log("Found %8d objects at depth %d\n", depth_hist[i], i);
      }
  
      Debug(42, 1, ("httpenum-breadth: exiting (signal %d)\n", x));
      exit(0);
--- 698,711 ----
      (void) unlink(md5db_filename);
      xfree(md5db_filename);
  
!     Log("Statistics for URL '%s':\n", tree_root);
!     for (i = 0, sum = 0; i < 100; i++) {
  	if (i > max_depth && depth_hist[i] == 0)
  	    break;
! 	Log("    Found %8d objects at depth %d\n", depth_hist[i], i);
! 	sum += depth_hist[i];
      }
+     Log("Total for URL '%s': %8d objects\n", tree_root, sum);
  
      Debug(42, 1, ("httpenum-breadth: exiting (signal %d)\n", x));
      exit(0);
***************
*** 769,777 ****
  	url = (char *) l->ptr;
  	depth = l->depth;
  
- 	if (depth < 100)
- 	    depth_hist[depth]++;
- 
  	if (max_depth > 0 && depth > max_depth) {
  	    if (not_visited)
  		fprintf(not_visited, "[DEPTH] %s\n", url);
--- 772,777 ----
***************
*** 801,807 ****
  	    continue;
  	}
  	/* search for more links from this one */
! 	http_enum(up, depth + 1);
  	url_close(up);
      }
  
--- 801,809 ----
  	    continue;
  	}
  	/* search for more links from this one */
! 	if (http_enum(up, depth + 1) != 0 && depth < 100)
! 	    /* Increase retrieved count only for new URLs - HrS */
! 	    depth_hist[depth]++;
  	url_close(up);
      }
  
Index: harvest/src/gatherer/mgmt/Gatherer.in
diff -c harvest/src/gatherer/mgmt/Gatherer.in:2.1 harvest/src/gatherer/mgmt/Gatherer.in:2.2
*** harvest/src/gatherer/mgmt/Gatherer.in:2.1	Fri Mar 21 19:15:43 1997
--- harvest/src/gatherer/mgmt/Gatherer.in	Tue Jul  1 20:42:39 1997
***************
*** 7,13 ****
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.1 1997/03/21 19:15:43 sxw Exp
  #
  #############################################################################
  #
--- 7,13 ----
  #
  #  Usage: Gatherer [-manual | -export | -debug | -background] file.cf
  #
! #  Gatherer.in,v 2.2 1997/07/01 19:42:39 sxw Exp
  #
  #############################################################################
  #
***************
*** 188,193 ****
--- 188,194 ----
  		"Local-Mapping",
  	        "Locale",
  		"Log-File",
+ 	        "Maintainer",
  		"Errorlog-File",
  		"Post-Summarizing",
  		"Refresh-Rate",
***************
*** 454,459 ****
--- 455,464 ----
                  $ENV{'HARVEST_USER_AGENT'} = $vals{"User-Agent"};
          }
  
+         if (defined($vals{"Maintainer"})) {
+                 $ENV{'HARVEST_MAINTAINER_ADDRESS'} = $vals{"Maintainer"};
+         }
+ 
          if (defined($vals{"Locale"})) {
                  $ENV{'LC_CTYPE'} = $vals{"Locale"};
          }
