/*
** getstats
**
** Version 1 written Apr 94 by Kevin Hughes <kevinh@eit.com>
** Version 2 written Jan 95 by Ben Beecher <beecher@columbia.edu>
** Version 3 written Dec 97 by Ben Beecher <beecher@columbia.edu>
**
*/

#include <stdio.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

#include "gs-struct.h"
#include "getstats.h"
#include "gs-parse.h"
#include "gs-domains.h"
#include "gs-tree.h"
#include "gs-extract.h"


#define PROGNAME  "getstats"
#define VERSION   "3.0"

#define LOCALDOM  ".columbia.edu"
#define DOCURL    "http://www.columbia.edu/httpd/getstats/"
#define ACISURL   "http://www.columbia.edu/acis/mailalias.html"

#define EXITCODE  0
#define MILESTONE 200000

vhost *vhostlist=NULL, *thisvhost=NULL;
    /* every web server (vhost) we find in the access log gets a
     * separate data structure, and gets a complete set of reports
     * the main host is www.columbia.edu
     */

vhost *NewVHost(char *host);

float	onekil = 1024.0,
	onemeg = 1024.0 * 1024.0,
	onegig = 1024.0 * 1024.0 * 1024.0;

int failshort, faildate, failmin, failhost, failprefix, failreq;
int truncline, truncreq, truncref;

sortType sortkey = N_BYTES;	/* default is # of bytes */

int	debug = FALSE;
int	filesopen = 0;
char	logcopy[MAXLINE+1];	/* current input line, for error messages */
int	grecs = 0;		/* input record count, for error messages */

int	milestone = MILESTONE;
int	wantextract = FALSE;	/* TRUE if any vhost wants extracts */
char	*cmdfilnam = "-";	/* default is STDIN */

char	defaulthost[ADDRLEN]=""; /* log records without vhost use this */
				 /* set this with SET DEFAULTHOST command */
char	wdate[DATELEN]="";	/* date is part of extract and domain rpt */
				/* set this with SET DATE command */
int	printhtml=TRUE,		/* the only supported output format is HTML */
	ignoreimages=FALSE,	/* keep everything by default */
				/* change this with SET IGNOREIMAGES command */
	ignorepublic=TRUE;	/* skip /~user/ requests by default */
				/* change this with SET IGNOREPUBLIC command */
int	countbytes=TRUE,	/* counting bytes? */
	counthosts=FALSE;	/* counting unique hosts? */

long	glob_first_date = BIGDATE,
	glob_last_date = 0;
char	firstdate[SHORTDATELEN] = "01/01/1970",
	lastdate[SHORTDATELEN]  = "01/01/1970",
	msgfile[ADDRLEN] = "",	/* parse errors go here */
	reqmask[MASKLEN] = "";
int	dtrlevels = DTRLEVELS,
	domlevels = DOMLEVELS;
time_t	starttime, stoptime;



int main(argc, argv)
int argc;
char **argv;
{
    do_args(argc, argv);
    time(&starttime);
    readcmdfile();
    exit(EXITCODE);
} /* main */



#define NCMDFIELDS	4

/* parse and execute each line of the command file
 *
 */
void readcmdfile(void)
{
    vhost *v;
    char cmdline[CMDLEN];
    char *cmd_result = '\0';
    FILE *cmd;

    if (strcmp(cmdfilnam, "-") == 0)
	cmd = stdin;
    else if ((cmd = fopen(cmdfilnam, "r")) == NULL) {
	char errmsg[200];
	sprintf(errmsg, "Can't open command file \"%s\" because", cmdfilnam);
	perror(errmsg);
	return;
    }
    if (debug)
	printf("rcf: %d files, %s is open\n", ++filesopen, cmdfilnam);
    while (fgets(cmdline, CMDLEN, cmd)) { /* read a line */
	int nwords;
	char *word[NCMDFIELDS+1],
	     *c, *command;

	if ((*cmdline=='#') || (*cmdline=='\n'))
	    continue;			/* ignore blank lines and comments */
	c = strchr(cmdline, '\n');	/* erase the trailing newline */
	if (c) *c = '\0';
	getwords(cmdline, word, &nwords);
	command = word[0];
	if (!strcmp(command, "read"))
	    cmd_result = DoReadCommand(word, nwords);
	else if (!strcmp(command, "set"))
	    cmd_result = DoSetCommand(word, nwords);
	else if (!strcmp(command, "write"))
	    cmd_result = DoWriteCommand(word, nwords);
	else
	    cmd_result = "valid commands are READ, SET, and WRITE";
	if (cmd_result) {		/* syntax error */
	    printf("invalid command: %s\n%s\n", cmdline, cmd_result);
	    fclose(cmd);
	    return;
	}
    } /* while read a line */
    fclose(cmd);
    if (debug) printf("rcf: %d files, %s is closed\n", --filesopen, cmdfilnam);

    for (v=vhostlist; v; v=v->next) {
	printf("\nFinishing host %s\n", v->hostname);
	WriteDomainReports(v);
	if (!v->printed)
	    printf("no reports written for this host!\n");
	CompressExtractFiles(v);
    }
    return;
} /* readcmdfile */



/* getwords
 *
 * create an array of pointers to individual command line fields
 * if any field begins with a doublequote, strip the quotes and
 * allow it to contain spaces.  any fields not surrounded by quotes
 * are converted to lower case.
 */
void getwords(buffer, word, nwords)
char *buffer;
char *word[];
int *nwords;
{
    static char bcopy[MAXLINE+1];
    char *c = bcopy;
    int i;
   
    strcpy(bcopy, buffer);
    for (i = 0; i <= NCMDFIELDS && *c; i++) {
	int done;
	char stopchar = ' ';
	if (*c == '"') {
	    stopchar = '"';
	    c++;
	}
	word[i] = c;			/* begin */
	done = FALSE;
	while (!done) {			/* parse one field */
	    char *d;
	    if (*c == '\0')
		  done = TRUE;
	    else if (*c == stopchar) {
		*c++ = '\0';		/* end */
		done = TRUE;
		while (*c == ' ')
		      *c++ = '\0';
	    } else
		  c++;
	} /* while not done */
	if (stopchar != '"')		/* if not a quoted field */
	    makelower(word[i]);		/* cvt to lower case */
    } /* for */
    *nwords = i;
} /* getwords */



/* parse and execute the READ command
 *
 * returns '\0' on success, or
 *         pointer to message string on error
 */
char *DoReadCommand(word, nwords)
char *word[];
int nwords;
{
    char *format = word[1],
	 *accfilnam = word[2];

    if (nwords != 3)
	return "wrong number of arguments";
    if (!strcmp(format, "hash")) {
	printf("READ HASH command is no longer necessary\n");
	return '\0';			/* not an error */
    }
    if (strcmp(format, "1.0") && strcmp(format, "std"))
	return "valid input formats are 1.0 and STD";

    if (*defaulthost=='\0')
	return "you must SET DEFAULTHOST before you read log files";
    if (wantextract && *wdate=='\0')
	return "you must SET DATE before you create extract files";
    ShowIgnoredExtract(vhostlist, dtrlevels);
    if (readlogfile(accfilnam, msgfile, format))
	exit(1);			/* can't read input file */
    time(&stoptime);			/* what time did we finish? */
    return '\0';
} /* DoReadCommand */



/* parse and execute the SET command
 *
 * returns '\0' on success, or
 *         pointer to message string on error
 */
char *DoSetCommand(word, nwords)
char *word[];
int nwords;
{
    int temp;
    char *varname = word[1],
	 *value   = word[2],
	 *value2  = word[3];

    if (!strcmp(varname, "domroot") || !strcmp(varname, "xtroot")) {
	if (nwords != 4) return "SET DOMROOT and SET XTROOT take 4 arguments";
    } else if (nwords != 3) return "SET command takes 3 arguments";

    if (!strcmp(varname, "countbytes"))		/* COUNTBYTES */
	if (!strcmp(value, "no"))
	    countbytes = FALSE;
	else if (!strcmp(value, "yes"))
	    countbytes = TRUE;
	else
	    return "new value must be YES or NO";
    else if (!strcmp(varname, "counthosts"))	/* COUNTHOSTS */
	if (!strcmp(value, "no"))
	    counthosts = FALSE;
	else if (!strcmp(value, "yes"))
	    counthosts = TRUE;
	else
	    return "new value must be YES or NO";
    else if (!strcmp(varname, "date"))		/* DATE */
	strcpy(wdate, value);
    else if (!strcmp(varname, "defaulthost"))	/* DEFAULTHOST */
	strcpy(defaulthost, value);
    else if (!strcmp(varname, "domlevels")) {	/* DOMLEVELS */
	temp = atoi(value);
	if ((temp < 0) || (12 < temp))
	    return "domlevels must be in the range 0 to 12";
	domlevels = temp;
    } else if (!strcmp(varname, "domroot")) {	/* DOMROOT */
	set_doroot(value, value2);
    } else if (!strcmp(varname, "dtrlevels")) {	/* DTRLEVELS */
	temp = atoi(value);
	if ((temp < 0) || (12 < temp))
	    return "dtrlevels must be in the range 0 to 12";
	dtrlevels = temp;
    } else if (!strcmp(varname, "htmloutput"))	/* HTMLOUTPUT */
	if (!strcmp(value, "no"))
	    return "only html format is supported now";
	else if (!strcmp(value, "yes"))
	    ;
	else
	    return "new value must be YES or NO";
    else if (!strcmp(varname, "icon"))		/* ICON */
	if (!thisvhost)
	  printf("you must SET SERVERNAME before you can SET ICON\n");
	else strcpy(thisvhost->iconurl, value);
    else if (!strcmp(varname, "ignoreimages"))	/* IGNOREIMAGES */
	if (!strcmp(value, "no"))
	    ignoreimages = FALSE;
	else if (!strcmp(value, "yes"))
	    ignoreimages = TRUE;
	else
	    return "new value must be YES or NO";
    else if (!strcmp(varname, "ignorepublic")) /* IGNOREPUBLIC */
	if (!strcmp(value, "no"))
	    ignorepublic = FALSE;
	else if (!strcmp(value, "yes"))
	    ignorepublic = TRUE;
	else
	    return "new value must be YES or NO";
    else if (!strcmp(varname, "milestone")) {	/* MILESTONE */
	temp = atoi(value);
	if (temp <= 0)
	    return "milestone must be positive";
	milestone = temp;
    } else if (!strcmp(varname, "msgfile"))	/* MSGFILE */
	strcpy(msgfile, value);
    else if (!strcmp(varname, "reqmask"))	/* REQMASK */
	strcpy(reqmask, value);
    else if (!strcmp(varname, "servername")) {	/* SERVERNAME */
	set_vhost(value);
    } else if (!strcmp(varname, "sortkey")) {	/* SORTKEY */
	if      (!strcmp(value, "alpha"))
	    sortkey = N_ALPHA;
	else if (!strcmp(value, "nbytes"))
	    sortkey = N_BYTES;
	else if (!strcmp(value, "nreq"))
	    sortkey = N_REQ;
	else
	    return "valid sort keys are ALPHA, NBYTES, and NREQ";
    } else if (!strcmp(varname, "xtroot")) {	/* XTROOT */
	set_xtroot(value, value2);
    } else
return "valid SET commands are COUNTBYTES, COUNTHOSTS, DATE, DEFAULTHOST, DOMLEVELS, DOMROOT, DTRLEVELS, HTMLOUTPUT, ICON, IGNOREIMAGES, IGNOREPUBLIC, MILESTONE, MSGFILE, REQMASK, SERVERNAME, SORTKEY, and XTROOT";
    return '\0';
} /* DoSetCommand */



/* parse and execute the WRITE command
 *
 * returns '\0' on success, or
 *         pointer to message string on error
 */
char *DoWriteCommand(word, nwords)
char *word[];
int nwords;
{
    int lev;
    char *rptype = word[1],
	 *rptfilnam;

    if (nwords < 2) return "wrong number of arguments";
    if (!strcmp(rptype, "hash")) {	/* dump the hash table */
	printf("WRITE HASH command is no longer necessary\n");
	return '\0';
    }
    if (!thisvhost) {
	printf("you must SET SERVERNAME before writing a report\n");
	return '\0';
    }
    if (!thisvhost->totaldocnum &&	/* no valid access records read */
	   strcmp(rptype, "error")) {	/* not doing the error report */
	if (*reqmask) printf("reqmask is %s, ", reqmask);
	printf("no logs for %s\n", thisvhost->hostname);
	return '\0';
    }
    lev = atoi(word[2]);
    rptfilnam = word[3];
    if (nwords != 4) return "wrong number of arguments";
    if ((lev < 1) || (12 < lev))
	return "nlevels must be in the range 1 to 12";

    if ((thisvhost->bytecount==0.0) &&
	(sortkey==N_BYTES))		/* have no size information */
	return "try sorting by NREQ because log contains no size information";

    if (!strcmp(rptype, "dirtree")) {	/* dir tree report */
	if (lev > dtrlevels) {		/* asked for too many levels */
	    printf("can't produce %d level DTR report because DTRLEVELS=%d\n",
		lev, dtrlevels);
	    return '\0';
	}
	thisvhost->printed = TRUE;

	return WriteReportDTR(rptfilnam, lev);

    } else if (!strcmp(rptype, "domain")) { /* domain report */
	if (lev > domlevels) {
	    printf("can't produce %d level DOM report because DOMLEVELS=%d\n",
		lev, domlevels);
	    return '\0';
	}
	if (!thisvhost->tree_root)
	    return "DWC: can't create DOM, tree root is NULL";
	if (!thisvhost->tree_root->sub)
	    return '\0';
	if (!WriteReportDOM(rptfilnam, lev,
			thisvhost, thisvhost->tree_root->sub->dom))
	    thisvhost->printed = TRUE;
	return '\0';
    } else if (!strcmp(rptype, "error")) { /* error report shows bad links */
	if (lev > dtrlevels) {
	    printf("can't produce %d level ERR report because DTRLEVELS=%d\n",
		lev, dtrlevels);
	    return '\0';
	}
	thisvhost->printed = TRUE;

	return WriteReportERR(rptfilnam, lev);

    } else if (!strcmp(rptype, "redir")) { /* redirects */
	if (lev > dtrlevels) {
	    printf("can't produce %d level RED report because DTRLEVELS=%d\n",
		lev, dtrlevels);
	    return '\0';
	}
	thisvhost->printed = TRUE;

	return WriteReportRED(rptfilnam, lev);

    } else
	return "valid report types are DIRTREE, DOMAIN, ERROR, and REDIR";
} /* DoWriteCommand */



void WriteReportTop(rpt, thisvhost)
FILE *rpt;
vhost *thisvhost;
{
    char title[ADDRLEN];

    if (!thisvhost) return;
    if (strcmp(firstdate, lastdate))
	sprintf(title, "Web Server Statistics: %s to %s", firstdate, lastdate);
    else sprintf(title, "Web Server Statistics: %s", firstdate);

    fprintf(rpt, "<html><head>\n<title>%s</title>\n</head>\n", title);
    fprintf(rpt,
    "<body bgcolor=white text=black link=blue vlink=purple alink=red>\n");
    fprintf(rpt, "<table border=0 width=100%%>");
    fprintf(rpt, "<tr><td width=1%%><img align=right src=\"%s\" alt=\"\">\n",
					thisvhost->iconurl);
    fprintf(rpt, "</td><td width=99%% valign=bottom><p><font size=+3>\n");
    fprintf(rpt, " %s<br><br>\n", title);
    fprintf(rpt, " <code>%s</code></font></p>\n</td></tr></table>\n",
					thisvhost->sername);
    return;
} /* WriteReportTop */



#define NRCOLUMNS 3
/* prints the error text and the error count
 */
void PrintEx(rpt, errtext, errcount, suffix, col)
FILE *rpt;
char *errtext;
int  errcount;
char *suffix;
int  *col;
{
    char desc[ADDRLEN+9];
    int in, out;
    if (!errcount) return;
    for (in=0, out=0; errtext[in] && out<ADDRLEN;)
	if (errtext[in]==' ') {		/* cvt space to &nbsp; */
	    desc[out++]='&';
	    desc[out++]='n';
	    desc[out++]='b';
	    desc[out++]='s';
	    desc[out++]='p';
	    desc[out++]=';';
	    in++;
	} else
	    desc[out++]=errtext[in++];

    desc[out]='\0';
    *col = ((*col)+1) % NRCOLUMNS;
    fprintf(rpt, "%s:&nbsp;%d%s", desc, errcount,
		*col ? suffix : "<br>\n");
    return;
}


void WriteReportBottom(rpt)		/* common footer for all reports */
FILE *rpt;
{
    fprintf(rpt, "</pre><p><hr size=3><p>Created by");
    fprintf(rpt, " <a href=\"%s\">%s %s</a> on %s, ",
			DOCURL, PROGNAME, VERSION, getlocaltime());
    printrunning(rpt, starttime, stoptime);

    fprintf(rpt, "</p><address><dl><dt>Academic Information Systems\n");
    fprintf(rpt, "<dd>Help Line: <code>212.854.1919</code><br>\n");
    fprintf(rpt, "Email: <code><a href=%s>", ACISURL);
    fprintf(rpt, "consultant@columbia.edu</a></code>\n");
    fprintf(rpt, "</dl></address></body>\n</html>\n");
    return;
} /* WriteReportBottom */



/* write the directory tree report
 *
 * examine global variables ignoreimages, ignorepublic
 */
char *WriteReportDTR(rptfilnam, lev)
char *rptfilnam;
int lev;
{
  FILE *rpt;
  if (!thisvhost) return '\0';
  if (!(thisvhost->tree_root))
    return "tree root is NULL";
  if ((rpt = fopen(rptfilnam, "w")) == '\0')
    return "unable to create report file";
    
  WriteReportTop(rpt, thisvhost);
  PrintVHostSummary(rpt, thisvhost, lev);
  PrintTreeHeader(rpt, lev);
  PrintTreeLevel(rpt, thisvhost->tree_root->sub, 0, lev);
  WriteReportBottom(rpt);
  fclose(rpt);
  ResetTreeLevel(thisvhost->tree_root->sub);	/* reset the printed flag */
  return '\0';
} /* WriteReportDTR */



/* write the redirect report
 *
 * examine global variables ignoreimages, ignorepublic
 */
char *WriteReportRED(rptfilnam, lev)
char *rptfilnam;
int lev;
{
    FILE *rpt;
    if (!thisvhost) return '\0';
    if (!(thisvhost->redir_root))
	return "redir root is NULL";
    if ((rpt = fopen(rptfilnam, "w")) == '\0')
	return "unable to create report file";
    
    WriteReportTop(rpt, thisvhost);
    PrintRedirHeader(rpt, lev);
    PrintRedirLevel(rpt, thisvhost->redir_root->sub, 0, lev);
    WriteReportBottom(rpt);
    fclose(rpt);
    return '\0';
} /* WriteReportRED */



/* write the error report
 */
char *WriteReportERR(rptfilnam, lev)
char *rptfilnam;
int lev;
{
    char temp[DATELEN];
    FILE *rpt;
    int nrecs;
    if (!thisvhost) return '\0';
    if (!thisvhost->erro_root)
	return "error root is NULL";
    if ((rpt = fopen(rptfilnam, "w")) == '\0')
	return "unable to create report file";
    nrecs = PrintErrorLevel(rpt, thisvhost->erro_root->sub, "", 1, lev);
    comm(temp, nrecs, 10);
    printf("%s errors written to %s\n", temp, rptfilnam);
    fclose(rpt);
    return '\0';
} /* WriteReportERR */



/* read the access log
 *
 * the "std" format is used by httpd 1.3 and later versions
 * each record contains a status code and the number of bytes transferred.
 * if the file name ends in .gz, use gunzip to uncompress the log file
 *
 * returns TRUE if unable to open the log file
 *         FALSE otherwise
 */
int readlogfile(accfilnam, msgfilnam, format)
char *accfilnam;
char *msgfilnam;
char *format;
{
    int		acc_eof, reqmask_len = strlen(reqmask);
    FILE	*accp=0, *msg=0;
    char	accline[MAXLINE+1];
    char	temp[DATELEN];
    int		(*ParseAccRec) ();

    grecs=0; truncline=0; truncreq=0; truncref=0;
    failshort=0; faildate=0; failmin=0; failhost=0; failprefix=0;
    failreq=0;
    if (!strcmp(format, "1.0"))			/* "1.0" */
	ParseAccRec = ParseAccNCSA;
    else if (!strcmp(format, "std"))		/* "std" */
	ParseAccRec = ParseAccSTD;
    else {
	printf("readlogfile: undefined input format: %s\n", format);
	exit(3);
    }

    if (strstr(accfilnam, ".gz")) {	/* pipe through gunzip */
	if (read_uncompressed(&accp, accfilnam) < 0)
	    return TRUE;		/* error */
    } else if ((accp = fopen(accfilnam, "r")) == 0) {
	fprintf(stderr, "Can't open log file %s", accfilnam);
	perror(" because");
	return TRUE;
    }
    if (debug) printf("rlf: %d files, %s is open\n", ++filesopen, accfilnam);

    if (*msgfilnam == '\0')		/* parse errors go here */
	msg = stderr;
    else if ((msg = fopen(msgfilnam, "a")) == 0) {
	fprintf(stderr, "Can't open msg file %s", msgfilnam);
	perror(" because");
	msg = stderr;
    } else if (debug)
	printf("rlf: %d files, %s is open\n", ++filesopen, msgfilnam);
    do {
	logr r;
	char *tt;
	if (acc_eof = readaline(accline, MAXLINE, accp))
	    continue;
	if (++grecs % milestone == 0) {
	    comm(temp, grecs, 10);
	    printf("%s records read\n", temp);
	}
	if (!ParseAccRec(accline, &r, msg))
	    continue;

	/* convert multiple slashes to single slash, and convert %7E to ~ */
	if (tt=strstr(r.referer, "://"))
	    unescape_url(tt+3);

	if (tt=strchr(r.request, ' '))	/* truncate request on space */
	    *tt = '\0';
	if (tt=strchr(r.request, '?'))	/*  or question mark */
	    *tt = '\0';
	if (tt=strchr(r.request, '#'))	/*  or pound sign */
	    *tt = '\0';
	if (tt=strchr(r.request, '"'))	/* replace quoted string */
	    strcpy(tt, "<quoted-string>");

	if (tt=strchr(r.referer, ' '))	/* truncate referer on space */
	    *tt = '\0';
	if (tt=strchr(r.referer, '?'))	/*  or question mark */
	    *tt = '\0';
	if (tt=strchr(r.referer, '#'))	/*  or pound sign */
	    *tt = '\0';
	if (tt=strchr(r.referer, '"'))	/* replace quoted string */
	    strcpy(tt, "<quoted-string>");

	/* skip requests that don't begin with the request mask string */
	if (reqmask_len && strncmp(reqmask, r.request, reqmask_len))
	    continue;
	AnalyzeInput(&r);
	CountThisRecord(&r, msg);
    } while (!acc_eof);
    fclose(accp);
    if (debug)
	printf("rlf  %d files, %s is closed\n", --filesopen, accfilnam);
    if (*msgfilnam) {
      fclose(msg);
      if (debug)
	printf("rlf  %d files, %s is closed\n", --filesopen, msgfilnam);
    }
    comm(temp, grecs, 10);
    printf("%s records read from %s\n", temp, accfilnam);
    if (truncline) {
	comm(temp, truncline, 10);
	printf("%s records truncated, longer than %d bytes (MAXLINE)\n",
		temp, MAXLINE);
    }
    if (truncreq) {
	comm(temp, truncreq, 10);
	printf("%s req fields truncated, longer than %d bytes (REQUESTLEN)\n",
		temp, REQUESTLEN);
    }
    if (truncref) {
	comm(temp, truncref, 10);
	printf("%s ref fields truncated, longer than %d bytes (REQUESTLEN)\n",
		temp, REQUESTLEN);
    }
    if (failshort)  printf("too short: %d   ",   failshort);
    if (faildate)   printf("bad date: %d   ",    faildate);
    if (failmin)    printf("fewer than 7 fields: %d   ", failmin);
    if (failprefix) printf("invalid prefix: %d   ", failprefix);
    if (failhost)   printf("hostname too long: %d   ", failhost);
    if (failreq)    printf("null request: %d",   failreq);
    putc('\n', stdout);
    return FALSE;
} /* readlogfile */



void AnalyzeInput(r)
logr *r;
{
    int TrailingSlash = FALSE;
    int reqlen, per, plim;
    if (countbytes)			/* cvt byte count to float */
	r->nbytes = atol(r->nbytstr);
					/* get datesecs and shortdate */
    convtoyearsecs(r, "%d/%b/%Y:%T");	/* "31/Jan/1995:01:02:03" */

    reqlen = strlen(r->request);
    if (r->request[reqlen-1] == '/') {		/* req ends in a slash */
	r->request[reqlen-1] = '\0';		/* remove the slash */
	reqlen--;
	TrailingSlash = TRUE;
    }
    for (per=reqlen, plim=FTYPLEN;
		per>0, plim>0;
		per--, plim--)
	if (r->request[per]=='.')
	    break;
    if (r->request[per]=='.')	/* file type is html, gif, jpg, etc. */
	strcpy(r->filtyp, r->request+per+1);
    else
	*r->filtyp = '\0';
    makelower(r->filtyp);
    if ((strcmp(r->filtyp, "html") ==0) ||
	(strcmp(r->filtyp, "htm")  ==0) ||
	(strcmp(r->filtyp, "shtml")==0))
	r->html = TRUE;
    else
	r->html = FALSE;

	/* trailing slash implies the default filename, "index.html"
	 * unless it comes after a dot, e.g. "/campus.gif/"
	 */
    if (TrailingSlash && (*r->filtyp=='\0')) {
	r->request[reqlen++] = '/';		/* put back the slash */
	strcpy(r->request+reqlen, DEFAULTFILE);	/* append "index.html" */
	reqlen += strlen(DEFAULTFILE);
	strcpy(r->filtyp, "html");
	r->html = TRUE;
    }
    makelower(r->address);
    r->localreq = strstr(r->address, LOCALDOM) != '\0';
    return;
} /* AnalyzeInput */



/** count this record **
 *
 * maintain a list of vhost records, each with a separate data structure
 *  for that virtual host.
 * the treebranch data structure will be used for the dir tree reports,
 *  the home page totals, and the error reports.
 * the domainbranch data structures are used for the domain reports.
 */
void CountThisRecord(r, err)
logr *r;
FILE *err;
{
    vhost *v;
    char *webserver;
    char *p1, *p2;
    int reqlen;

    if (strstr(r->shortdate, "00/")) {		/* can't have 00 month or day */
	fprintf(err, "CountThisRecord invalid date %s datesecs %d short %s\n",
		r->date, r->datesecs, r->shortdate);
	fprintf(err, "%s %s %s [%s] \"%s %s %s\" %s %s\n\n\n",
		r->address, r->ident, r->user, r->date,
		r->rprefix, r->request, r->rsuffix,
		r->status, r->nbytstr);
	faildate++;
	return;
    }
    if (*r->vhost)
	webserver = r->vhost;
    else					/* record contains no vhost */
	webserver = defaulthost;		/*  so use the default */
    for (v=vhostlist; v; v=v->next)		/* find the vhost record */
	if (!strcmp(webserver, v->hostname))	/*  got it */
	    break;

    if (!v)					/* create new vhost record */
	v=NewVHost(webserver);

    r->virthost = v;				/* save ptr to vhost */


    /* prepend "/!" to /~user/file.html, so it becomes
     * /!/~user/file.html
     * that way the personal home pages can be counted separately
     */
    if (r->request[0] == '/' &&
	r->request[1] == '~') {
      char stretch[REQUESTLEN+1+15];
      sprintf(stretch, "/%c/~%s", PCHAR, r->request+2);
      strcpy(r->request, stretch);
    }


    /* since we are reading separate log files from each host
     * we can not assume the first record contains the earliest date and
     * the last record contains the latest date.  so we examine the dates.
     */
    if (r->datesecs < glob_first_date) {
	glob_first_date = r->datesecs;
	strcpy(firstdate, r->shortdate);
    }
    if (r->datesecs > glob_last_date) {
	glob_last_date = r->datesecs;
	strcpy(lastdate, r->shortdate);
    }

    if (!strcmp(r->status, "301") ||		/* redirect */
	!strcmp(r->status, "302")) {
      if (!strcmp(r->status, "301")) v->red301++;
      else			     v->red302++;
				/* see if we need to append "index.html" */
      if ((reqlen = strlen(r->request)) &&	/* request is not NULL, and */
	  (r->request[reqlen-1] == '/')) {	/*  request ends in slash */
	strcpy(r->request+reqlen, DEFAULTFILE);	/* append "index.html" */
      }
      AddRedirEntry(r, v);
      return;
    }

    if ((*r->status=='4') ||
	(*r->status=='5')) {			/* error 4xx or 5xx */
      int errstat = atoi(r->status);
      if      (errstat == 400) v->fail400++;
      else if (errstat == 401) v->fail401++;
      else if (errstat == 402) v->fail402++;
      else if (errstat == 403) v->fail403++;
      else if (errstat == 404) v->fail404++;
      else if (errstat == 405) v->fail405++;
      else if (errstat == 406) v->fail406++;
      else if (errstat == 407) v->fail407++;
      else if (errstat == 408) v->fail408++;
      else if (errstat == 409) v->fail409++;
      else if (errstat == 410) v->fail410++;
      else if (errstat == 411) v->fail411++;
      else if (errstat == 412) v->fail412++;
      else if (errstat == 413) v->fail413++;
      else if (errstat == 500) v->fail500++;
      else if (errstat == 501) v->fail501++;
      else fprintf(stderr, "unexpected error status: %d\n", errstat);
      AddErrorEntry(r, v);
      return;
    }
    if (!strncmp(r->request, "/cgi-bin", 8))	v->cginum++;
    else if (!strncmp(r->request, "/sec-cgi-bin", 12))	v->seccginum++;
    else if (r->html)					v->htmldocnum++;
    else if (!strcmp(r->filtyp, "gif"))			v->gifdocnum++;
    else if (!strcmp(r->filtyp, "jpeg"))		v->jpgdocnum++;
    else if (!strcmp(r->filtyp, "jpg"))			v->jpgdocnum++;
    else if (!strcmp(r->filtyp, "pdf"))			v->pdfdocnum++;
    else						v->otherdocnum++;
    v->totaldocnum++;
    v->bytecount += r->nbytes;

    /* remove ^I garbage since the request was successful (dumb browser) */
    for (p1=r->request; *p1;)			/* remove ^I garbage */
	if (*p1!='^') p1++;
	else if (*(p1+1)=='\0') *p1='\0';
	else for (p2=p1; *p2=*(p2+2); p2++);	/* shift left 2 bytes */

    /* check again to see if we need to append "index.html" */
    if ((reqlen = strlen(r->request)) &&	/* request is not NULL, and */
	(r->request[reqlen-1] == '/')) {	/*  request ends in slash */
	strcpy(r->request+reqlen, DEFAULTFILE);	/* append "index.html" */
    }

    AddTreeEntry(r, v);				/* add to directory tree */
    return;
} /* CountThisRecord */



void printbreak(rpt)
FILE *rpt;
{
    int i = BREAKLEN;

    if (countbytes)
	i += 13;
    if (!printhtml) {
	while (i--)
	    putc('-', rpt);
	putc('\n', rpt);
    }
}



void printrunning(rpt, starttime, stoptime)
FILE *rpt;
time_t starttime;
time_t stoptime;
{
    int minutes;
    int seconds = (long) stoptime - (long) starttime;
    int elapsed = seconds;

    int hours = seconds / SECSPERHOUR;
    seconds -= hours * SECSPERHOUR;

    minutes = seconds / SECSPERMIN;
    seconds -= minutes * SECSPERMIN;

    fprintf(rpt, "running time was");
    if (elapsed==0) {
	fprintf(rpt, " less than a second.\n");
	return;
    }
    if (hours)
	fprintf(rpt, " %d hour%s", hours, (hours == 1) ? "" : "s");
    if (minutes)
	fprintf(rpt, " %d minute%s", minutes, (minutes == 1) ? "" : "s");
    if (seconds)
	fprintf(rpt, " %d second%s", seconds, (seconds == 1) ? "" : "s");
    fprintf(rpt, "\n");
    return;
}



int numlen(num)
long num;
{
    int i = 0;
    while (num /= 10)
	i++;
    return i;
}



char *e_strdup(string)
char *string;
{
    char *p = (char *) emalloc(strlen(string) + 1);
    strcpy(p, string);
    return p;
}



void *emalloc(i)
int i;
{
    void *p;
    static char errmsg[100]; 
    if (p = (void *) malloc(i))
	return p;
    sprintf(errmsg, "Can't allocate %d bytes, insufficient memory", i);
    progerr(errmsg);
}



void progerr(errorstr)
char *errorstr;
{
  printf("%s: %s\n", PROGNAME, errorstr);
  printf("most recent input line is %d:\n%s\n", grecs, logcopy);
  exit(3);
}



/* gigs
 * display a floating point number using Kilo, Mega, or Giga units
 */
void gigs(dest, value, width)
char *dest;
float value;
int width;
{
  if (width) {
    if      (value >= onegig) sprintf(dest, "%*.2f G", width-2, value/onegig);
    else if (value >= onemeg) sprintf(dest, "%*.2f M", width-2, value/onemeg);
    else if (value >= onekil) sprintf(dest, "%*.2f K", width-2, value/onekil);
    else		      sprintf(dest, "%*.0f  ", width-2, value);
  } else {
    if      (value >= onegig) sprintf(dest, "%.2fG", value/onegig);
    else if (value >= onemeg) sprintf(dest, "%.2fM", value/onemeg);
    else if (value >= onekil) sprintf(dest, "%.2fK", value/onekil);
    else		      sprintf(dest, "%.0f ", value);
  }
  return;
} /* gigs */



/* comm
 * add commas to the number of requests
 * if width is too narrow, more space will be used
 */
void comm(dest, value, width)
char *dest;
int value;
int width;
{
    char temp[DATELEN];
    char *t;
    int tlen, i;

    sprintf(temp, "%d", value);
    tlen = strlen(temp);
    if (tlen < 4)			/* 0 to 999 */
	sprintf(dest, "%*s", width, temp);
    else if (tlen < 7) {		/* 1,000 to 999,999 */
	if (width < tlen+1)
	    width = tlen+1;
	for (i = width-tlen-1; i>0; i--)
	    *dest++ = ' ';
	t = temp;
	for (i = tlen-3; i>0; i--)
	    *dest++ = *t++;
	*dest++ = ',';			/* add the comma */
	while (*t)
	    *dest++ = *t++;
	*dest = '\0';

    } else {				/* 1,000,000 and up */
	if (width < tlen+2)		/* make room for 2 commas */
	    width = tlen+2;
	for (i = width-tlen-2; i>0; i--)
	    *dest++ = ' ';
	t = temp;
	for (i = tlen-6; i>0; i--)
	    *dest++ = *t++;
	*dest++ = ',';			/* add the comma */
	for (i = 3; i>0; i--)
	    *dest++ = *t++;
	*dest++ = ',';			/* add the comma */
	while (*t)
	    *dest++ = *t++;
	*dest = '\0';
    }
    return;
} /* comm */



/* NewVHost
 *
 * creates a new vhost node
 * returns a pointer to the new node
 */
vhost *NewVHost(host)
char *host;
{
    int i;
    vhost *v, *w;
    if (!host || !*host) return NULL;
    v = (struct vhost *) emalloc(sizeof(vhost));
    strcpy(v->hostname, host);
    *v->proto   = '\0';
    *v->sername = '\0';
    *v->iconurl = '\0';
					/* initialize data structures */
    v->tree_root  = NewBranch("DTR", "DTR"); /* hits */
    v->erro_root  = NewBranch("ERR", "ERR"); /* errors */
    v->redir_root = NewBranch("RED", "RED"); /* redirects */
    v->dominfo  = NULL;
    *v->doroot  = '\0';
    v->xtinfo   = NULL;
    *v->xtroot  = '\0';
    v->xtfiles     = 0;			/* potential files */
    v->xtcreated   = 0;			/* actual files */
    v->xtoverwrite = 0;			/* files overwritten */
    v->xtrecords   = 0;
    v->xtwrites    = 0;
    v->xtfilesopen = 0;

    v->bytecount    = 0.0;
    v->printed      = FALSE;
    v->htmldocnum   = 0;
    v->gifdocnum    = 0;
    v->jpgdocnum    = 0;
    v->pdfdocnum    = 0;
    v->cginum       = 0;
    v->seccginum    = 0;
    v->otherdocnum  = 0;
    v->totaldocnum  = 0;
    v->red301       = 0;
    v->red302       = 0;
    v->fail400      = 0;
    v->fail401      = 0;
    v->fail402      = 0;
    v->fail403      = 0;
    v->fail404      = 0;
    v->fail405      = 0;
    v->fail406      = 0;
    v->fail407      = 0;
    v->fail408      = 0;
    v->fail409      = 0;
    v->fail410      = 0;
    v->fail411      = 0;
    v->fail412      = 0;
    v->fail413      = 0;
    v->fail500      = 0;
    v->fail501      = 0;

    for (i = 0; i < TOPNUM; i++)	/* initialize top ten list */
	v->tree_topreq[i] = NULL;
    v->bottom_req   = 0;
    v->next         = NULL;
    if (!vhostlist) {			/* empty list */
	vhostlist = v;			/* is no longer empty */
	return v;
    }
    for (w=vhostlist; w; w=w->next)	/* add this vhost record */
	if (!w->next) {			/*  to the end of the list */
	    w->next = v;
	    return v;
	}
    printf("impossible error\n");	/* can't get here */
    return v;
} /* NewVHost */



/* parse the server name to get the host name, then set global thisvhost */
void set_vhost(sername)
char *sername;
{
    char proto[ADDRLEN];		/* http: */
    char hostn[ADDRLEN];		/* www.columbia.edu */
    char *a;
    strcpy(proto, sername);
    if (a=strstr(proto, "://")) {
	*(a+1) = '\0';
	strcpy(hostn, a+3);
    } else {
	*proto = '\0';
	strcpy(hostn, sername);
    }
    a = strchr(hostn, '/');
    if (a) *a='\0';
    a = strchr(hostn, ' ');
    if (a) *a='\0';

				/* find the matching vhost record */
    for (thisvhost=vhostlist; thisvhost; thisvhost=thisvhost->next)
	if (!strcmp(hostn, thisvhost->hostname))
	    break;

    if (!thisvhost) {
	printf("no log records for this host: %s\n", hostn);
	return;
    }
    strcpy(thisvhost->proto, proto);
    strcpy(thisvhost->sername, sername);
    return;
} /* set_vhost */



void set_doroot(serhost, root)
char *serhost;
char *root;
{
    vhost *v;
    for (v=vhostlist; v; v=v->next)		/* find the vhost record */
	if (!strcmp(serhost, v->hostname))	/*  got it */
	    break;

    if (!v)					/* not found */
	v=NewVHost(serhost);			/* create new vhost record */

    strcpy(v->doroot, root);
    get_do_list(v, "");				/* create list of subdirs */
    return;
} /* set_doroot */



void set_xtroot(serhost, root)
char *serhost;
char *root;
{
    vhost *v;
    wantextract = TRUE;			/* at least one vhost wants extracts */
    for (v=vhostlist; v; v=v->next)		/* find the vhost record */
	if (!strcmp(serhost, v->hostname))	/*  got it */
	    break;

    if (!v)					/* not found */
	v=NewVHost(serhost);			/* create new vhost record */

    strcpy(v->xtroot, root);
    get_xt_list(v, "", 0);			/* create list of subdirs */
    return;
} /* set_xtroot */



char *getlocaltime(void)		/* in the form January 1 1998 14:45 */
{
	static char s[DATELEN];
	time_t tp;
 
	time(&tp);
	strftime(s, DATELEN, "%B %d %Y %R", localtime(&tp));
	return s;
}



/*****************************************/

int do_args(argc, argv)
int argc;
char **argv;
{
    extern int optind;
    extern char *optarg;
    int c;

    while ((c = getopt(argc, argv, "c:")) != EOF)
      switch(c) {
	case 'c':
	  cmdfilnam = optarg;		/* get input file name */
	  break;
	case '?':
	  usage();
	  break;
	}
    if (optind < argc)			/* command options not valid */
      usage();

    return;
}


int usage(void)
{
    printf("Usage: %s [-c commandfile]\n", PROGNAME);
    exit(3);
}
