/* gs-tree.c
 *
 * collect information for the dir tree reports
 *  part of the getstats program
 */


#include <stdio.h>
#include <string.h>
#include <sys/types.h>			/* for mkdir, stat */
#include <sys/stat.h>			/* for mkdir, stat */
#include <errno.h>			/* for errno */
#include "compat.h"			/* for bzero */
#include "gs-tree.h"
#include "gs-domains.h"
#include "gs-extract.h"

extern vhost *thisvhost;
extern int debug, filesopen, errno;
extern int countbytes;
extern long glob_first_date, glob_last_date;
extern sortType sortkey;
extern char wdate[];			/* date is part of extract file name */
extern char reqmask[];
extern int printhtml, counthosts, ignoreimages, ignorepublic, dtrlevels;

xtnode	xtskip = { NULL };

#define REQTRUNCATE	42
#define GRANULARITY	 6

/* max granularity in the directory tree report is specified by
 * GRANULARITY.  if a request has more slashes they will not be
 * broken out further.
 *   /aaaa/
 *       bbbb/
 *           cccc/
 *              dddd/eeee/ffff/gggg
 */



/* for one request add a series of path names to the request list
 *  for that treebranch.  also adds to the domain tree for this
 *  directory, if necessary
 *
 * the request /!/~abra/ca/dabra adds (or increments) these nodes:
 *   fullreq		request
 *
 *   /			/		<- total for this vhost
 *   /!/		!/		<- total for personal dirs
 *   /!/~abra/		~abra/
 *   /!/~abra/ca/	ca/
 *   /!/~abra/ca/dabra	dabra
 *
 * the first few nodes might have domain trees in them
 * don't break it down further than dtrlevels
 */
void AddTreeEntry(r, vh)
logr *r;
vhost *vh;
{
  treebranch *br;
  char sofar[REQUESTLEN];		/* pieces of path parsed so far */
  char *pname;				/* one piece of the path */
  char *p1, *p2;
  int level, traslash, new_node;

  if (*r->request != '/')
    return;		/* all access requests should start with a slash */
  br = vh->tree_root;
  if (strchr(r->request, '^')) {
    fprintf(stderr, "AddTreeEntry filename contains CTRL char %s\n", r->request);
    return;
  }
  if (debug)
    printf("ATE: r->req %s, vhost %s\n", r->request, vh->hostname);
  pname = sofar;
  p1 = sofar;
  p2 = r->request;
  *p1++ = *p2++;			/* copy the slash */
  *p1 = '\0';
  traslash = TRUE;
  for (level = 0;
       (level < GRANULARITY) &&
	 (level <= dtrlevels) &&
	 traslash;			/* not leaf node */
       level++) {		/* loop through the directory nodes */
    
			/* add to the dir tree data structure for this dir */
    br = AddTreeReq(br, pname, sofar, r, &new_node, TRUE);
    if (!br) {
      fprintf(stderr, "AddTreeEntry null filename in %s\n", r->request);
      return;
    }
    if (new_node) {
      if (pname[strlen(pname)-1]=='/' &&
	  pname[1]!=PCHAR) {			/* official directory */
	br->dom = NewDomainInfo(sofar, r, br);  /* gets domain report */
	br->xt  = NewExtractInfo(sofar, r, br); /*  and extract file */
      } else {					/* plain file or tilde dir */
	br->wantdomain = FALSE;
	br->wantextract = FALSE;
      }
    }
			/* add to the domain data structure for this dir */
    if (br->wantdomain && br->dom)
      AddDomEntry(r, br->dom->dom_root, br->dom->numeric_host);

    /* append this record to the appropriate extract file
     * each record can be appended to multiple extract files.
     */
    if (br->wantextract && br->xt)	/* treebranch links to xtinfo */
      WriteExtract(r, br->xt);		/* append to extract file */

				/* add the dir tree info for each level */
    pname = p1;			/* copy up to the next slash */
    while (*p2 && (*p2 != '/'))
      *p1++ = *p2++;
    traslash = (*p2 == '/');
    if (traslash)			/* trailing slash found? */
      *p1++ = *p2++;			/*  yes, copy it */
    *p1 = '\0';
  }
					/* leaf node */
  if (level > dtrlevels) return;
  if (!AddTreeReq(br, pname, sofar, r, &new_node, TRUE))
    fprintf(stderr, "AddTreeEntry no filename in %s, ref: %s\n",
				r->request, r->referer);
  return;
}  /* AddTreeEntry */



/* AddRedirEntry
 * this function is called for all requests that had a redirect status:
 *  301 or 302.
 */
void AddRedirEntry(r, vh)
logr *r;
vhost *vh;
{
    treebranch *br = vh->redir_root;
    char sofar[REQUESTLEN];		/* pieces of path parsed so far */
    char *pname;			/* one piece of the path */
    char *p1, *p2;
    int j, level, traslash, new_node;

    if (*r->request != '/')
	return;		/* all access requests should start with a slash */
    if (r->request[1]==PCHAR)		/* ignore redirects from ~user */
	return;
    pname = sofar;
    p1 = sofar;
    p2 = r->request;
    *p1++ = *p2++;			/* copy leading slash */
    *p1 = '\0';
    traslash = TRUE;
    for (level = 0;
	 (level < GRANULARITY) &&
	 (level <= dtrlevels) &&
	 traslash;			/* not leaf node */
	 level++) {		/* loop through the directory nodes */

		/* add to the dir tree data structure for this dir */
	    br = AddTreeReq(br, pname, sofar, r, &new_node, TRUE);
	    if (!br) {
		fprintf(stderr, "AddRedirEntry null filename in %s\n",
						r->request);
		return;
	    }

		/* add the dir tree info for each leaf */
	    pname = p1;			/* copy up to the next slash */
	    while (*p2 && (*p2 != '/'))
		*p1++ = *p2++;
	    traslash = (*p2 == '/');
	    if (traslash)		/* trailing slash found? */
		*p1++ = *p2++;		/*  yes, copy it */
	    *p1 = '\0';
    }
					/* leaf node */
    if (level > dtrlevels) return;
    if (!AddTreeReq(br, pname, sofar, r, &new_node, TRUE))
	fprintf(stderr, "AddRedirEntry no filename in %s\n", r->request);
    return;
}  /* AddRedirEntry */



/* AddErrorEntry
 * this function is called for all requests that failed with an
 * error status: 400 to 599.
 *
 * find the referer in the dir tree (return if none)
 * ignore CGI errors, ignore 403 errors from off-site users
 * skip errors from personal web pages
 * if the referer is one of our web servers, add an error record
 */
void AddErrorEntry(r, vh)
logr *r;
vhost *vh;
{
    treebranch *br;
    char proto[STFLEN+1];
    char refhost[ADDRLEN+1];
    char refpath[REQUESTLEN+1+strlen(DEFAULTFILE)];
    char sofar[REQUESTLEN];		/* pieces of path parsed so far */
    char *pname;			/* one piece of the path */
    char *p1, *p2;
    int j, errstatus, level, traslash, new_node;
    ernode *e;

    if ((*r->referer == '\0') ||	/* no referer, fail return */
	(*r->referer == '-'))
	return;
    errstatus = atoi(r->status);

    if (!strncmp(r->request, "/cgi-bin/", 9))
	return;				/* ignore CGI errors */
    if (!strncmp(r->request, "/sec-cgi-bin/", 13))
	return;
    if (!r->localreq && errstatus==403)
	return;			/* ignore 403 errors from off-site users */

    ParseReferer(r->referer, proto, refhost, refpath);

    /* we're only interested in finding bad links on the current vhost,
     * where the referer host matches the virtual host
     */
    if (strcmp(refhost, vh->hostname))
	return;
			/* all access requests should start with a slash */
    if (*refpath != '/') {
	fprintf(stderr,
		"AddErrorEntry invalid ref %s\n", refpath);
	fprintf(stderr, "%s %s %s [%s] \"%s %s %s\" %s %s \"ref %s\"\n\n\n",
		r->address, r->ident, r->user, r->date,
		r->rprefix, r->request, r->rsuffix,
		r->status, r->nbytstr, r->referer);
	return;
    }

    if (refpath[1]=='~')		/* referer starts with "/~" */
	return;

    br = vh->erro_root;
    pname = sofar;
    p1 = sofar;
    p2 = refpath;
    *p1++ = *p2++;			/* copy leading slash */
    while (*p2 && (*p2 != '/'))		/*  and up to the next slash */
	*p1++ = *p2++;
    traslash = (*p2 == '/');
    if (traslash)			/* trailing slash found? */
	*p1++ = *p2++;			/*  yes, copy it */
    *p1 = '\0';
    for (level = 1;
	 (level < GRANULARITY) &&
	 (level <= dtrlevels) &&
	 traslash;			/* not leaf node */
	 level++) {		/* loop through the directory nodes */

		/* add to the dir tree data structure for this dir */
	    br = AddTreeReq(br, pname, sofar, r, &new_node, FALSE);
	    if (!br) {
		fprintf(stderr, "AddErrorEntry null filename in %s\n",
			refpath);
		return;
	    }

		/* add the dir tree info for each leaf */
	    pname = p1;			/* copy up to the next slash */
	    while (*p2 && (*p2 != '/'))
		*p1++ = *p2++;
	    traslash = (*p2 == '/');
	    if (traslash)		/* trailing slash found? */
		*p1++ = *p2++;		/*  yes, copy it */
	    *p1 = '\0';
    }
					/* leaf node */
    if (level > dtrlevels) return;
    br = AddTreeReq(br, pname, sofar, r, &new_node, FALSE);
    if (!br) {
	fprintf(stderr, "AddErrorEntry no filename in %s\n", refpath);
	return;
    }
		/* any previous errors for this URL and error code? */
    for (e = br->er; e; e = e->next)
	if (!strcmp(e->url, r->request) && (e->errcode==errstatus))
	    break;

    if (e) {			/* update existing record */
	e->errcount++;
	if (r->datesecs < e->date1secs) { /* first and last date */
	    e->date1secs = r->datesecs;
	    sprintf(e->date1, "%.17s", r->date);
	} else if (r->datesecs > e->date2secs) {
	    e->date2secs = r->datesecs;
	    sprintf(e->date2, "%.17s", r->date);
	}
    } else {			/* create an error record */
	e = NewError(r->request);
	e->next = br->er;
	br->er = e;
	e->errcode = errstatus;
	e->errcount = 1;
	e->date1secs = r->datesecs;
	e->date2secs = r->datesecs;
	sprintf(e->date1, "%.17s", r->date);
	sprintf(e->date2, "%.17s", r->date);
	return;
    }
    return;
}  /* AddErrorEntry */



/* parse the referer, which takes this form:
 *   http://www.columbia.edu/cu/libraries/
 */
void ParseReferer(ref, pro, hos, path)
char *ref;
char *pro;
char *hos;
char *path;
{
    char *cp, *cr;

    if (cp = strchr(ref, ':')) {	/* look for protocol */
	*cp = '\0';
	sprintf(pro, "%.*s", STFLEN, ref);
	*cp++ = ':';
    } else {
	cp = ref;
	*pro = '\0';
    }
    if (*cp == '/') {
	cp++;
	if (*cp == '/') cp++;
	if (cr = strchr(cp, '/')) {
	    char *q;
	    *cr = '\0';			/* remove the slash */
	    sprintf(hos, "%.*s", ADDRLEN, cp); /* get host name */
	    *cr = '/';			/* put back the slash */
	    if (q = strchr(cr, '#'))	/* chop off relative portion */
		*q = '\0';
	    if (q = strchr(cr, '"'))	/* chop off quoted string */
		*q = '\0';
	    if (q = strchr(cr, '?'))	/* chop off CGI arguments */
		*q = '\0';
	    sprintf(path, "%.*s", REQUESTLEN, cr);
	} else {			/* no slash */
	    sprintf(hos, "%.*s", ADDRLEN, cp);
	    strcpy(path, "/");		/* use empty refpath */
	}
    } else {
	*hos = '\0';			/* no host name */
	strcpy(path, "/");		/* use empty refpath */
    }
    if (path[strlen(path)-1] == '/')	/* if ref ends in a slash */
	strcat(path, DEFAULTFILE);	/* append index.html */

					/* canonicalize referer hostname */
    if (!strcmp(hos, "www") ||
	!strcmp(hos, "www.cc.columbia.edu") ||
	!strncmp(hos, "jonapot", 7) ||
	!strncmp(hos, "kwaziwai", 8))
		strcpy(hos, "www.columbia.edu");
    return;
} /* ParseReferer */



#define LEFTBOXWIDTH		42
#define RIGHTBOXWIDTH		100-LEFTBOXWIDTH

void PrintVHostSummary(rpt, thisvhost, maxlevel)
FILE *rpt;
vhost *thisvhost;
int maxlevel;
{
  float bytes, hours;
  int col, requests, hoursi;
  domainbranch *br;
  char *spc;
  if (*reqmask)
    fprintf(rpt, "Reporting on requests that begin with \"%s\"\n\n",
	    reqmask);
  fprintf(rpt, "<!-- begin summary -->\n");
  fprintf(rpt, "<table border=1 cellpadding=4 width=100%% bgcolor=\"%s\">\n", C_NORM);
  fprintf(rpt, "<tr valign=middle><td width=%d%% align=center>", LEFTBOXWIDTH);
  fprintf(rpt, "<table border=0 cellspacing=0 cellpadding=7");
  fprintf(rpt," bgcolor=\"%s\">\n", C_NORM);

					/* top left box */
  fprintf(rpt, "<tr><th colspan=2>Totals for this<br>Reporting Period</th>");
  fprintf(rpt, "<th>Avg.<br>per Hour</th>");
  fprintf(rpt, "<th>Avg.<br>per Min.</th></tr>\n");
  requests = thisvhost->totaldocnum;
  if (!requests) return;
  bytes = thisvhost->bytecount;
  hours = (float) (glob_last_date - glob_first_date) / SECSPERHOUR;
  hoursi = (int) (hours + 0.5);
  isequence(rpt, "<b>Total requests</b>", requests, hoursi);
  fsequence(rpt, "Total bytes", bytes, hours, hoursi);
  PrintCuTotals(rpt, hours, hoursi);
  if (counthosts) {
    donode *dome=thisvhost->tree_root->sub->dom;
    if (!dome->unique_hosts)
      CountUniqueHosts(dome);		    /* count unique hosts */
    isequence(rpt, "Unique hosts", dome->unique_hosts, hoursi);
  }
  fprintf(rpt, "</table></td><td width=%d%% align=center>\n", RIGHTBOXWIDTH);
  fprintf(rpt, "<table border=0 bgcolor=\"%s\">\n", C_NORM);

					/* top right box */
  fprintf(rpt, "<tr><th colspan=3>Breakdown by Document Type</th></tr>\n");
  icount2(rpt, requests, "GIF",		thisvhost->gifdocnum);
  icount2(rpt, requests, "HTML",	thisvhost->htmldocnum);
  icount2(rpt, requests, "JPEG",	thisvhost->jpgdocnum);
  icount2(rpt, requests, "PDF",		thisvhost->pdfdocnum);
  icount2(rpt, requests, "CGI",		thisvhost->cginum);
  icount2(rpt, requests, "SEC CGI",	thisvhost->seccginum);
  icount2(rpt, requests, "other",	thisvhost->otherdocnum);

  fprintf(rpt, "</table></td></tr>\n");

  fprintf(rpt, "<tr valign=middle><td width=%d%% align=center>", LEFTBOXWIDTH);
  fprintf(rpt, "<table border=0 cellspacing=0 cellpadding=4");
  fprintf(rpt, " bgcolor=\"%s\">\n", C_NORM);

					/* bottom left box */
  fprintf(rpt, "<tr><th colspan=3>Redirected Requests</th></tr>\n");
  icount3(rpt, C_NORM, "301", "Permanent Redirect",	thisvhost->red301);
  icount3(rpt, C_NORM, "302", "Temporary Redirect",	thisvhost->red302);

  fprintf(rpt, "<tr><th colspan=3 bgcolor=\"%s\">Failed Requests</th></tr>\n",
	  C_RED);
  icount3(rpt, C_RED,  "400", "Bad syntax",		thisvhost->fail400);
  icount3(rpt, C_RED,  "401", "Access denied",		thisvhost->fail401);
  icount3(rpt, C_RED,  "402", "Payment required",	thisvhost->fail402);
  icount3(rpt, C_RED,  "403", "Protected",		thisvhost->fail403);
  icount3(rpt, C_RED,  "404", "Not found",		thisvhost->fail404);
  icount3(rpt, C_RED,  "405", "Method not allowed",	thisvhost->fail405);
  icount3(rpt, C_RED,  "406", "Not acceptable",		thisvhost->fail406);
  icount3(rpt, C_RED,  "407", "Proxy auth required",	thisvhost->fail407);
  icount3(rpt, C_RED,  "408", "Timeout",		thisvhost->fail408);
  icount3(rpt, C_RED,  "409", "Conflict",		thisvhost->fail409);
  icount3(rpt, C_RED,  "410", "Gone",			thisvhost->fail410);
  icount3(rpt, C_RED,  "411", "Length required",	thisvhost->fail411);
  icount3(rpt, C_RED,  "412", "Precondition failed",  	thisvhost->fail412);
  icount3(rpt, C_RED,  "413", "Too large",		thisvhost->fail413);
  icount3(rpt, C_RED,  "500", "Server error",		thisvhost->fail500);
  icount3(rpt, C_RED,  "501", "Not implemented",	thisvhost->fail501);

  fprintf(rpt, "</table></td><td width=%d%% align=center>\n", RIGHTBOXWIDTH);
					/* bottom right box */
  PrintTopTen(rpt, thisvhost);
  fprintf(rpt, "</td></tr></table>\n");
  fprintf(rpt, "<!-- end summary -->\n");
  return;
} /* PrintVHostSummary */



void PrintCuTotals(rpt, hours, hoursi)
FILE *rpt;
float hours;
int hoursi;
{
  treebranch *treebr=thisvhost->tree_root->sub;
  domainbranch *br;
  if (!treebr) return;
  br=treebr->dom->dom_root;
  if (!br) return;
  for (br=br->sub;
       br && strcmp(br->fulldom, ".edu");
       br = br->next) ;
  if (!br) return;
  for (br=br->sub;
       br && strcmp(br->fulldom, ".columbia.edu");
       br = br->next) ;
  if (!br) return;
  isequence(rpt, "CU requests", br->nreq, hoursi);
  fsequence(rpt, "CU bytes", br->nbytes, hours, hoursi);
  return;
} /* PrintCuTotals */



void tablerow2(fil, col1, col2)
FILE *fil;
char *col1, *col2;
{
  fprintf(fil, "<tr><td align=right>");
  fprintf(fil, "<font face=\"courier\">%s</font></td>\n", col1);
  fprintf(fil, "  <td><font face=\"courier\">%s</font></td></font></tr>\n",
	  col2);
  return;
} /* tablerow2 */

void tablerow2b(fil, col1, col2, perc)
FILE *fil;
char *col1, *col2, *perc;
{
  fprintf(fil, " <tr><td align=right><font color=\"%s\">%s</font></td>\n",
	  D_GREEN, col1);
  fprintf(fil, "  <td align=center>%s</td>\n", col2);
  fprintf(fil, "  <td>%s</td></tr>\n", perc);
  return;
} /* tablerow2b */

void tablerow3(fil, color, col1, col2, col3)
FILE *fil;
char *color, *col1, *col2, *col3;
{
  fprintf(fil, "<tr bgcolor=\"%s\"><td><font size=\"-1\">%s</font></td>\n",
	  color, col1);
  fprintf(fil, "  <td>%s</td><td align=right>%s</td></tr>\n", col2, col3);
  return;
} /* tablerow3 */

void tablerow4(fil, col1, col2, col3, col4)
FILE *fil;
char *col1, *col2, *col3, *col4;
{
  fprintf(fil, "<tr><td bgcolor=\"%s\">%s:</td>\n", C_GREEN, col1);
  fprintf(fil, "  <td align=right bgcolor=\"%s\"><font face=\"courier\">%s</font></td>\n",
	  C_GREEN, col2);
  fprintf(fil, "  <td align=right><font face=\"courier\">%s</font></td>\n", col3);
  fprintf(fil, "  <td align=right><font face=\"courier\">%s</font></td></tr>\n", col4);
  return;
} /* tablerow4 */

void icount2(fil, tot, type, val1)	/* int, title, bar graph */
FILE *fil;
int tot;
char *type;
int val1;				/* integer with commas */
{
  char	hits[ADDRLEN],
	perc[REQUESTLEN];
  int percent;
  if (!val1) return;
  comm(hits, val1, 1);
  percent = 100*val1/tot;
  sprintf(perc, "<table width=100%%>\n  <tr><td width=%d%% bgcolor=\"%s\">&nbsp;</td>\n    <td width=%d%%>%d%%</td></tr></table>\n",
	  percent, D_GREEN, 100-percent, percent);
  tablerow2b(fil, hits, type, perc);
  return;
} /* icount2 */

void icount3(fil, color, cod, nam, val1) /* code, title, int */
FILE *fil;
char *color, *cod, *nam;
int val1;				/* integer with commas */
{
  char buf1[ADDRLEN];
  if (!val1) return;
  comm(buf1, val1, 1);
  tablerow3(fil, color, cod, nam, buf1);
  return;
} /* icount3 */

void isequence(fil, nam, val1, hoursi)	/* title, int, int, int */
FILE *fil;
char *nam;
int val1, hoursi;			/* integers with commas */
{
  int val2=0, val3=0;
  char	buf1[ADDRLEN],
	buf2[ADDRLEN],
	buf3[ADDRLEN];
  if (!val1) return;
  comm(buf1, val1, 1);
  if (hoursi) {
    val2 = val1 / hoursi;		/* avg requests per hour */
    val3 = val2 / 60;			/* avg requests per minute */
    comm(buf2, val2, 1);
    comm(buf3, val3, 1);
  } else {
    strcpy(buf2, "&nbsp;");
    strcpy(buf3, "&nbsp;");
  }
  tablerow4(fil, nam, buf1, buf2, buf3);
  return;
} /* isequence */

void fsequence(fil, nam, val1, hours, hoursi)
FILE *fil;
char *nam;
float val1, hours;			/* floats with K, M, G */
int hoursi;
{
  float val2=0.0, val3=0.0;
  char	buf1[ADDRLEN],
	buf2[ADDRLEN],
	buf3[ADDRLEN];
  if (val1==0.0) return;
  gigs(buf1, val1, 0);
  if (hoursi) {
    val2 = val1 / hours;		/* avg bytes per hour */
    val3 = val2 / 60.0;			/* avg bytes per minute */
    gigs(buf2, val2, 0);
    gigs(buf3, val3, 0);
  } else {
    strcpy(buf2, "&nbsp;");
    strcpy(buf3, "&nbsp;");
  }
  tablerow4(fil, nam, buf1, buf2, buf3);
  return;
} /* fsequence */



void PrintTopTen(rpt, thisvhost)
FILE *rpt;
vhost *thisvhost;
{
  int i;
  if (!thisvhost || !ignorepublic) {	/* personal dirs have no top 10 */
    fprintf(rpt, "&nbsp;");
    return;
  }
  if (!thisvhost->tree_topreq[0])	/* create top 10 list */
    TopTenDTR(thisvhost, thisvhost->tree_root->sub);
  if (!thisvhost->bottom_req) {
    fprintf(rpt, "&nbsp;");
    return;
  }
  fprintf(rpt, "<table width=100%% border=0 cellspacing=0 cellpadding=4");
  fprintf(rpt, " bgcolor=\"%s\">\n", C_NORM);

  fprintf(rpt, "<tr><th colspan=2>Top %d Documents</th></tr>\n", TOPNUM);

  for (i = 0; i < TOPNUM; i++) {
    char req_comm[DATELEN];
    treebranch *tree_ptr = thisvhost->tree_topreq[i];
    if (tree_ptr && !tree_ptr->fullreq) {
      printf("null fullreq found in PrintTopTen\n");
      exit(3);
    }
    if (tree_ptr && *tree_ptr->fullreq) {
      char unam[REQUESTLEN], fnam[REQUESTLEN];
      char *s, *t;
      comm(req_comm, tree_ptr->nreq,  10);

      for (s = tree_ptr->fullreq, t = unam;
	   *s && (t-unam < REQTRUNCATE);
	   s++)
	if (*s == '<') {
	  *t++='&';
	  *t++='l';
	  *t++='t';
	  *t++=';';
	} else if (*s == '>') {
	  *t++='&';
	  *t++='g';
	  *t++='t';
	  *t++=';';
	} else if (*s == '&') {
	  *t++='&';
	  *t++='a';
	  *t++='m';
	  *t++='p';
	  *t++=';';
	} else
	  *t++=*s;

      *t='\0';
      if (strchr(unam, '&'))
	strcpy(fnam, unam);
      else
	sprintf(fnam, "<a href=\"%s//%s%s\">%s</a>",
		thisvhost->proto, thisvhost->hostname, unam, unam);
      tablerow2(rpt, req_comm, fnam);
    }
  }
  fprintf(rpt, "</table>\n");
  return;
} /* PrintTopTen */



/* PrintTreeHeader
 *
 * prints the header for the Dir Tree Report
 */
void PrintTreeHeader(rpt, maxlevel)
FILE *rpt;
int maxlevel;
{
  int i;
  if (!thisvhost) return;
  fprintf(rpt, "<center><h2>Directory Tree Report - sorted ");
  if (sortkey==N_ALPHA)
    fprintf(rpt, "alphabetically");
  else if (sortkey==N_BYTES)
    fprintf(rpt, "by the number of bytes transferred");
  else
    fprintf(rpt, "by the number of requests");
  fprintf(rpt, "</h2></center>\n");

  if (ignoreimages) fprintf(rpt,
"<p>Showing <b>HTML documents</b> requested from this server,\n");
  else fprintf(rpt,
"<p>Showing <b>all document types</b> requested from this server (HTML, GIF, JPG, etc.),\n");

  if (maxlevel==1)
    fprintf(rpt, "including top level files and directories only.\n");
  else
    fprintf(rpt, "including the top %d directory levels.\n", maxlevel);

  if (!ignorepublic)
    fprintf(rpt, "  Including personal home pages only.\n");
  fprintf(rpt, "</p><pre>\n");
  fprintf(rpt, "   Total       CU      Offsite     Total\n");
  fprintf(rpt, "   Bytes      Hits       Hits       Hits     Dir/file Requested\n",
TOPNUM);
  fprintf(rpt, " --------  ---------  ---------  ----------  ---------------------\n");

  return;
} /* PrintTreeHeader */



/* go through the list of nodes at the current level, printing
 * them in order by sortkey (then alphabetical)
 * after printing each directory, call self recursively on
 * that branch
 */
void PrintTreeLevel(rpt, br, level, maxlevel)
FILE *rpt;
treebranch *br;
int level;
int maxlevel;
{
  treebranch *this, *best;
  if (!br) return;
  do {		/* sorted by number of requests, then alphabetically */
    for (this = br, best = NULL;
	 this;
	 this = this->next) {	/* find the best candidate and print it */

      if (this->printed) continue;
      if (*this->request==PCHAR) {		/* "!/" */
	best = this;				/* personal web dir summary */
	break;					/*  at the top of the list */
      }
      if (level==1 && !ignorepublic && *this->request!=PCHAR)
	this->printed = TRUE;			/* show only personal dirs */

      else if (!this->sub &&			/* leaf node */
	  ignoreimages && !this->html)
	this->printed = TRUE;	/* only show html in this report */
      
      else if (this->sub &&			/* directory node */
	       ignoreimages && this->hreq==0)
	this->printed = TRUE;	/* skip dir with no HTML requests */

      else if (this->sub &&			/* directory node */
	       !ignoreimages && this->nreq==0)
	this->printed = TRUE;	/* skip dir with no requests */
      
      else if (sortkey==N_ALPHA) {
	best = this;		/* already sorted alphabetically */
	break;
      } else if (best == NULL)
	best = this;		/* first node is the best so far */
      else if (CompareTreeNodes(this, best, sortkey))
	best = this;		/* put this before best */

    } /* for */
      /* inner loop is finished, print the best not-yet-printed node */
    if (best) {
      PrintTreeRec(rpt, best, level);	/* print this node */
      best->printed = TRUE;

      if (level==1 && ignorepublic && *best->request==PCHAR)
	continue;

      if (best->sub && (level < maxlevel))	/* recurse */
	PrintTreeLevel(rpt, best->sub, level+1, maxlevel);
    }
  } while (best);
  return;
} /* PrintTreeLevel */



/* PrintRedirHeader
 *
 * prints the header for the Redirect Report
 */
void PrintRedirHeader(rpt, maxlevel)
FILE *rpt;
int maxlevel;
{
    if (!thisvhost) return;
    fprintf(rpt, "<center><h2>Redirect Report</h2></center>\n");
    fprintf(rpt, "<p>Showing requests that were redirected.\n");
    if (maxlevel==1)
	fprintf(rpt, "  Including top level files and directories only.</p><pre>\n");
    else
	fprintf(rpt, "  Including the top %d directory levels.</p><pre>\n", maxlevel);

    fprintf(rpt, "     CU      Offsite     Total\n");
    fprintf(rpt, "    Hits       Hits       Hits     Dir/file Requested\n");
    fprintf(rpt, " ---------  ---------  ----------  ---------------------\n");

    return;
} /* PrintRedirHeader */



/* go through the list of nodes at the current level
 */
void PrintRedirLevel(rpt, br, level, maxlevel)
FILE *rpt;
treebranch *br;
int level;
int maxlevel;
{
    treebranch *this, *best;
    if (!br) return;
    /* print one level, alphabetically */

    for (this = br; this; this = this->next) {
	/* if we are looking at the top level of the tree,
	 * check to see if this is a ~user or official directory
	 * as desired for the report
	 * if ((level == 1) && (ignorepublic == (this->request[1] == '~')))
	 *   continue;
	 */
	PrintRedirRec(rpt, this, level);	/* print this node */
	if (this->sub && (level < maxlevel))	/* recurse */
	    PrintRedirLevel(rpt, this->sub, level+1, maxlevel);
    }
    return;
} /* PrintRedirLevel */



/* PrintErrorLevel
 * go through the list of nodes at the current level, printing
 * those with broken links (output not sorted)
 * after printing each directory, call self recursively on that branch
 *
 * returns the number of records written
 */
int PrintErrorLevel(rpt, br, thisdir, level, maxlevel)
FILE *rpt;
treebranch *br;
char *thisdir;
int level;
int maxlevel;
{
    treebranch *this, *best;
    int nrecs = 0;
    if (br == NULL) return nrecs;
    /* print the official directories or the public_html directories,
     * depending on br
     * print one level, not sorted
     */

    for (this = br; this; this = this->next) {
	if (this->er)				/* this file has bad links */
	    nrecs += PrintErrorRec(rpt, thisdir, this, level); /* print them */
	else if (this->sub && (level < maxlevel)) { /* this is a directory */
	    char sofar[REQUESTLEN];		/* recurse */
	    sprintf(sofar, "%s%s", thisdir, this->request);
	    nrecs += PrintErrorLevel(rpt, this->sub, sofar, level+1, maxlevel);
	}
    } /* for */
    return nrecs;
} /* PrintErrorLevel */



/* create the top ten list for Dir Tree Report
 */
void TopTenDTR(thisvhost, br)
vhost *thisvhost;
treebranch *br;
{
  treebranch *this;
  if (!thisvhost) return;
  for (this = br; this; this = this->next) {
    int i, j;
    if ((this->request[0]==PCHAR) &&	/* ignore "!/" subtree */
	(this->request[1]=='/'))	/*  for personal web pages */
      continue;
    if (this->sub) {			/* non-empty directory, recurse */
      TopTenDTR(thisvhost, this->sub);
      continue;
    }
    if (strchr(this->request, '/'))	/* empty directory, ignore */
      continue;
    if (!this->html)			/* keep only html files */
      continue;
    i = thisvhost->bottom_req;
    while (i > 0)
      if (thisvhost->tree_topreq[i-1] &&
	  thisvhost->tree_topreq[i-1]->nreq >= this->nreq)
	break;
      else
	i--;

    if (thisvhost->bottom_req < TOPNUM) /* list is now longer */
      thisvhost->bottom_req++;
    if (i < thisvhost->bottom_req) {	/* new item for list */
      for (j = TOPNUM-1;		/* move rest of list down */
	   j > i;
	   j--)
	thisvhost->tree_topreq[j]=thisvhost->tree_topreq[j-1];

      thisvhost->tree_topreq[i] = this; /* insert new list item */
    }
  }
  return;
} /* TopTenDTR */



/* turn off all the "printed" flags at the current level
 *  and all levels below
 */
void ResetTreeLevel(br)
treebranch *br;
{
  treebranch *this;
  for (this = br; this; this = this->next) {
    if (this->printed) {
      this->printed = FALSE;		/* not printed */
      if (this->sub) ResetTreeLevel(this->sub);
    }
  }
} /* ResetTreeLevel */



/* compare two nodes according to sortkey
 *
 * returns TRUE if node1 comes before node2
 *         FALSE otherwise
 */
int CompareTreeNodes(node1, node2, skey)
treebranch *node1;
treebranch *node2;
sortType skey;
{
    if (skey==N_ALPHA) ;
    else if (ignoreimages && skey==N_BYTES)	/* sort by number of bytes */
	if      (node1->hbytes < node2->hbytes)	return FALSE;
	else if (node1->hbytes==node2->hbytes)  ;
	else					 return TRUE;
    else if (!ignoreimages && skey==N_BYTES)	/* sort by number of bytes */
	if      (node1->nbytes < node2->nbytes) return FALSE;
	else if (node1->nbytes==node2->nbytes)  ;
	else					return TRUE;
    else if (ignoreimages)		/* sort by number of html requests */
	if (node1->hreq < node2->hreq)		return FALSE;
	else if (node1->hreq == node2->hreq)	;
	else					return TRUE;
    else				/* sort by total hits */
	if (node1->nreq < node2->nreq)		return FALSE;
	else if (node1->nreq == node2->nreq)	;
	else					return TRUE;
    /* if the primary sort key is the same, sort alphabetically */
    return strcmp(node1->request, node2->request) < 0;
} /* CompareTreeNodes */



/* print the number of bytes, the number of requests, and the
 * file or directory of this node.  indent the file name according
 * to the tree level (three spaces per level).
 *
 * examine the global variable ignoreimages
 */
void PrintTreeRec(rpt, br, level)
FILE *rpt;
treebranch *br;
int level;
{
    char byt_megs[DATELEN],
	 req_cCU[DATELEN], req_cOS[DATELEN], req_comm[DATELEN];
    int i;
    if (ignoreimages) {			/* count HTML files only */
	gigs(byt_megs, br->hbytes,  9);
	comm(req_cCU,  br->hreqCU,  9);
	comm(req_cOS,  br->hreq - br->hreqCU, 9);
	comm(req_comm, br->hreq,   10);
    } else {
	gigs(byt_megs, br->nbytes,  9);
	comm(req_cCU,  br->nreqCU,  9);
	comm(req_cOS,  br->nreq - br->nreqCU, 9);
	comm(req_comm, br->nreq,   10);
    }
    fprintf(rpt, "%s  %s  %s  %s  ", byt_megs,
			req_cCU, req_cOS, req_comm);
    for (i = level; i>0; i--)		/* print (level-2)*3+1 spaces */
	if (i==1) fprintf(rpt, " ");
	else	  fprintf(rpt, "   ");	/* indent 3 spaces for each slash */

			/* print the last segment (after rightmost slash) */
    PrintFilnam(rpt, br->request);
    return;
} /* PrintTreeRec */



/* print the number of requests and the file or directory of this node.
 * indent the file name according to the tree level (three spaces per level).
 */
void PrintRedirRec(rpt, br, level)
FILE *rpt;
treebranch *br;
int level;
{
    char req_cCU[DATELEN], req_cOS[DATELEN], req_comm[DATELEN];
    int i;
    comm(req_cCU,  br->nreqCU,  9);
    comm(req_cOS,  br->nreq - br->nreqCU, 9);
    comm(req_comm, br->nreq,   10);
    fprintf(rpt, " %s  %s  %s  ", req_cCU, req_cOS, req_comm);
    for (i = level; i>0; i--)		/* print (level-2)*3+1 spaces */
	if (i==1) fprintf(rpt, " ");
	else	  fprintf(rpt, "   ");	/* indent 3 spaces for each slash */

			/* print the last segment (after rightmost slash) */
    PrintFilnam(rpt, br->request);
    return;
} /* PrintRedirRec */



/* print all the errors for this web page
 * returns the number of records written
 */
int PrintErrorRec(rpt, thisdir, br, level)
FILE *rpt;
char *thisdir;
treebranch *br;
int level;
{
  int nrecs=0;
  ernode *e;
  for (e = br->er; e; e = e->next) {	/* every bad link on this page */
    char *eurl=e->url;
    /* cvt personal web requests from "/!/~user/file" to "/~user/file" */
    if ((eurl[0]=='/') && (eurl[1]==PCHAR) && (eurl[2]=='/'))
      eurl+=2;				/* skip leading "/!" */
    nrecs++;
    fprintf(rpt, "%s%s\t%s\t%d\t%d\t%s\t%s\n",
	    thisdir, br->request, eurl, e->errcode, e->errcount,
	    e->date1, e->date2);
  }
  return nrecs;
} /* PrintErrorRec */



/* print the file or directory of this node
 */
void PrintFilnam(rpt, request)
FILE *rpt;
char *request;
{
  char *s;
  int j;
  if (!thisvhost) return;
  if ((request[0]==PCHAR) &&
      (request[1]=='/') &&
      (request[2]=='\0'))
    request = "Personal web pages";
  for (s = request, j = 0;
       *s && j < REQTRUNCATE;
       s++, j++)
    if      (*s == '<') fprintf(rpt,"&lt;");
    else if (*s == '>') fprintf(rpt,"&gt;");
    else if (*s == '&') fprintf(rpt,"&amp;");
    else		putc(*s, rpt);
  putc('\n', rpt);
  return;
} /* PrintFilnam */



/* NewBranch
 *
 * creates a new node of the tree
 * find the file ending (html, gif, jpg, etc.)
 * returns a pointer to the new node
 */
treebranch *NewBranch(request, fullreq)
char *request;
char *fullreq;
{
  int per, plim;
  treebranch *rt = (struct treebranch *) emalloc(sizeof(treebranch));
  char *req;
  int offset=strlen(fullreq)-strlen(request);
  if (offset < 0) {
    fprintf(stderr, "NBr: request %s is longer than %s\n", request, fullreq);
    exit(9);
  }
  rt->fullreq = e_strdup(fullreq ? fullreq : "");
  req=rt->fullreq+offset;
  if (strcmp(req, request)) {
    rt->request = e_strdup(request ? request : "");
    fprintf(stderr, "NBr: request %s not in fullreq %s\n", request, fullreq);
  } else
    rt->request = req;

  rt->nreqCU  = 0;
  rt->nreq    = 0;
  rt->nbytes  = 0.0;
  rt->hreqCU  = 0;
  rt->hreq    = 0;
  rt->hbytes  = 0.0;
  rt->date    = 0;
  rt->html        = FALSE;
  rt->printed     = FALSE;
  rt->wantdomain  = TRUE;
  rt->wantextract = TRUE;
  rt->dom     = NULL;			/* domain tree */
  rt->xt      = NULL;			/* extract files */
  rt->er      = NULL;			/* errors */
  rt->next    = NULL;
  rt->sub     = NULL;
  return rt;
} /* NewBranch */



/* NewError
 *
 * creates a new error node
 * returns a pointer to the new node
 */
ernode *NewError(request)
char *request;
{
    ernode *er = (struct ernode *) emalloc(sizeof(ernode));
    if (request)
	er->url = e_strdup(request);
    else
	er->url = e_strdup("");
    er->errcode  =0;
    er->errcount =0;
    er->date1secs=BIGDATE;
    er->date2secs=BIGDATE;
    *er->date1   ='\0';
    *er->date2   ='\0';
    er->next = NULL;
    return er;
} /* NewError */



/* AddTreeReq
 *
 * if this node exists, and count_it is TRUE (request was satisfied)
 * then update the counters
 * create a new node if necessary
 *
 * returns a pointer to the new or updated node on success
 *         NULL on failure
 *         new_node is TRUE if a new node was created, otherwise FALSE
 */
treebranch *AddTreeReq(br, request, fullreq, r, new_node, count_it)
treebranch *br;
char *request;
char *fullreq;
logr *r;			/* for datesecs, nbytes, html, localreq */
int *new_node;
int count_it;
{
  treebranch *this, *prev;
  if (debug)
    printf("ATR: br->req %s, req %s, fullreq %s\n",
	   br->request, request, fullreq);
  if (!request || *request=='\0' || !fullreq || *fullreq=='\0')
    return '\0';
  /* keep the lists sorted alphabetically so that searching is faster */
  for (prev = NULL, this = br->sub, *new_node = TRUE;
       this;
       prev = this, this = this->next) {
    int comp = strcmp(request, this->request);
    if (comp < 0)
      break;				/* not found */
    else if (comp==0) {			/* update existing entry */
      if (r->datesecs > this->date)	/* this rec is later */
	this->date = r->datesecs;	/* most recent access date */
      *new_node = FALSE;
      if (count_it) {			/* not an error, so count it */
	if (r->localreq) this->nreqCU++; /* CU hit */
	this->nreq++;			/* total hits */
	this->nbytes += r->nbytes;	/* bytes delivered */

	if (r->html) {			/* file type is {htm|html|shtml} */
	  if (r->localreq) this->hreqCU++; /* CU hit */
	  this->hreq++;			/* total hits */
	  this->hbytes += r->nbytes;	/* bytes delivered */
	}
      }
      break;
    }
  }

  if (*new_node) {	/* this request not found in the list, so add it */
    /* don't use r->request because this could be an error record
     * which uses referer instead
     */
    treebranch *rt = NewBranch(request, fullreq);
    if (prev)				/* first on the list? */
      prev->next = rt;			/*  no */
    else				/*  yes */
      br->sub = rt;
    rt->next = this;
    rt->date = r->datesecs;
    rt->html = r->html;
    this = rt;
    if (count_it) {
      if (r->localreq) this->nreqCU = 1;
      this->nreq = 1;
      this->nbytes = r->nbytes;
      if (r->html) {			/* file type is {htm|html|shtml} */
	if (r->localreq) this->hreqCU = 1; /* CU hit */
	this->hreq = 1;			/* total hits */
	this->hbytes = r->nbytes;	/* bytes delivered */
      }
    }
  }
  return this;
} /* AddTreeReq */
