/* gs-domains.c
 *
 * collect domain info and create domain reports
 */


#include <stdio.h>
#include <string.h>
#include <ctype.h>			/* for isdigit */
#include <dirent.h>			/* for opendir, readdir */
#include <sys/types.h>			/* for lookuphost, mkdir, stat */
#include <sys/stat.h>			/* for mkdir, stat */
#include <sys/socket.h>			/* for lookuphost */
#include <netdb.h>			/* for lookuphost */
#include <netinet/in.h>			/* for lookuphost */
#include <arpa/inet.h>			/* for gethostbyaddr */
#include "compat.h"			/* for bzero */
#include "gs-topdomains.h"		/* declare doma */
#include "gs-domains.h"

extern char wdate[];			/* date is part of output file name */
extern vhost *thisvhost;
extern sortType sortkey;
extern int debug, domlevels;
extern char domain_report[];
extern int counthosts;			/* count every host name and IP address */
					/* for "unique visits" counter */

donode *NewDomainInfo(sofar, r, br)
char *sofar;
logr *r;
treebranch *br;
{
    donode *dolink;
    dolink = wantDomain(r->virthost, sofar); /* search linked list */
    if (dolink)				/* found it */
	br->dom = dolink;		/* link to doinfo */
    else
	br->wantdomain = FALSE;
    return dolink;
} /* NewDomainInfo */



/* NewDomain
 *
 * add a new directory that will receive domain reports
 * creates a new donode which holds two domain trees for the current directory
 * this node is an extension of the treebranch record
 * returns a pointer to the new node
 */
donode *NewDomain(vh, dir)
vhost *vh;
char *dir;
{
    int i;
    donode *domm;
    if (!vh) return NULL;
    domm = (struct donode *) emalloc(sizeof(donode));
    domm->dom_root     = NewDomBranch("Host names", "Host names");
    domm->numeric_host = NewDomBranch("Numeric host addresses",
				      "Numeric host addresses");
    for (i = 0; i < TOPNUM; i++)	/* initialize top list */
	domm->dom_top[i] = NULL;
    domm->bottom_dom = 0;
    domm->unique_hosts = 0;
    domm->dirnam = e_strdup(dir);
    domm->next = vh->dominfo;		/* join the list */
    vh->dominfo = domm;			/* with new node at the head */
    return domm;
} /* NewDomain */



#define NODOMAINFILE	".nodomain"

/* make a list of subdirectories under doroot
 *
 * include the top directory (doroot)
 * omit any directories that contain a .nodomain file
 * returns 0 on success
 */
int get_do_list(vh, dir)
vhost *vh;
char *dir;
{
    DIR *mydir;
    struct dirent *myfile;
    char dirname[REQUESTLEN];
    int wantdo = 1;

    if (!vh) {
	printf("get_do_list: virtual host is NULL\n");
	return 1;
    }
    if (!*vh->doroot) {
	printf("doroot is undefined on host %s\n", vh->hostname);
	return 1;
    }
    sprintf(dirname, "%s/%s", vh->doroot, dir);
    mydir = opendir(dirname);
    if (!mydir) {
	fprintf(stderr, "can't open %s", dirname);
	perror(" because");
	exit(3);
    }
    while (myfile = readdir(mydir)) {
	struct stat stbuf;
	char filnam[REQUESTLEN], path[REQUESTLEN];
	if (!strcmp(myfile->d_name, NODOMAINFILE)) {
	    wantdo = 0;
	    continue;
	}
	if (*(myfile->d_name) == '.')		/* skip dot files */
	    continue;
	sprintf(filnam, "%s/%s", dir, myfile->d_name);
	sprintf(path, "%s/%s", vh->doroot, filnam);
	if (stat(path, &stbuf)) {
	    fprintf(stderr, "can't stat %s", path);
	    perror(" because");
	    continue;
	}
	if (S_ISDIR(stbuf.st_mode))		/* found a subdirectory */
	    if (get_do_list(vh, filnam))	/* call myself recursively */
		return 1;			/*  failure propagates back */
    }
    closedir(mydir);

    if (wantdo) {				/* include the top dir "" */
	char dirnam[REQUESTLEN];
	sprintf(dirnam, "%s/", dir);
	NewDomain(vh, dirnam);
    }
    return 0;
} /* get_do_list */



/* if this is one of the selected directories for domain reports
 * returns pointer to donode,
 * otherwise NULL
 */
donode *wantDomain(vh, dir)
vhost *vh;
char *dir;
{
    char woodir[REQUESTLEN];
    int dlen;
    donode *k;

    if (!vh) return NULL;
    if (!*vh->doroot) return NULL;	/* doing domains for this vhost? */
    strcpy(woodir, dir);
    dlen = strlen(woodir);
    for (k = vh->dominfo; k; k = k->next)
	if (!strcmp(woodir, k->dirnam))
	    return k;

    return NULL;
} /* wantDomain */



/* AddDomEntry
 *
 * for one request increment a series of nodes
 *
 * for example, a request from host neb.cc.columbia.edu
 * adds (or increments) these nodes in the request list:
 *   0) Host names
 *   1) .edu
 *   2) .columbia.edu
 *   3) .cc.columbia.edu
 * don't break it down further than domlevels levels
 *
 * a request from a numeric IP address adds (or increments)
 * these nodes:
 *   0) Numeric host names		<-- always
 *   1) nn.nn.nn.nn			<-- only if counthosts
 *
 */
void AddDomEntry(r, dns_hosts, num_hosts)
logr *r;
domainbranch *dns_hosts;
domainbranch *num_hosts;
{
  char pname[DOMAINLEN]="";
  char *dom = r->address;
  int level, dot;
  domainbranch *br;		/* points to dns_hosts or num_hosts */

  if (*r->request != '/')
    return;		/* access requests normally start with a slash */

		/* r->address contains the IP address or the host name */
  if (isip(r->address)) {		/* numeric host name */
    br = num_hosts;			/* count these separately */
    br->nreq++;				/* bump the root node */
    br->nbytes += r->nbytes;
    if (!counthosts) return;
    strcpy(pname, dom);			/* one entry for each IP address */
    br = AddDomReq(br, pname, pname, r->nbytes);
    return;
  }

  br = dns_hosts;
  br->nreq++;				/* bump the root node */
  br->nbytes += r->nbytes;
  for (level=1, dot=strlen(dom);
       ((level <= domlevels) || counthosts) && dot;
       level++) {			/*  moving right to left */
    char *in, *out;
					/* count the rightmost piece */
    do dot--; while (dot && dom[dot]!='.'); /* find previous dot */

    in=dom+dot;
    out=pname;				/* pname gets a piece */
    *out++ = *in++;			/* copy the leading dot */
    while (*in && (*in!='.'))		/* copy the rest of this piece */
      *out++ = *in++;
    *out++ = '\0';
    br = AddDomReq(br, pname, dom+dot, r->nbytes);
  }
  return;
} /* AddDomEntry */



/* PrintDomainHeader
 *
 * prints the header for the domain report, including top ten domains
 */
void PrintDomainHeader(rpt, dome, maxlevel)
FILE *rpt;
donode *dome;
int maxlevel;
{
  domainbranch *br;
  int i, top_domains = 0;

  /* these are things we only want to do once: */
  if (!dome->bottom_dom)		/* first report for this vhost:dir */
    TopTenDOM(dome, dome->dom_root->sub, "", 1);	/* create top 10 list */
  if (!dome->unique_hosts)
    CountUniqueHosts(dome);				/* count unique hosts */

  fprintf(rpt, "<center><h2>Top %d Domains", TOPNUM);
  if (dome->dirnam && strcmp(dome->dirnam, "/"))
    fprintf(rpt, " for %s", dome->dirnam);
  fprintf(rpt, "</h2></center><pre>\n");

  fprintf(rpt, "  Bytes   :     Req's  : Top %d Domains\n", TOPNUM);
  fprintf(rpt, " -------- : ---------- : --------------\n");

  for (i = 0; i < TOPNUM; i++) {
    char byt_megs[DATELEN],
	 req_comm[DATELEN];
    if (dome->dom_top[i]) {
      gigs(byt_megs, dome->dom_top[i]->nbytes, 9);
      comm(req_comm, dome->dom_top[i]->nreq, 10);
      fprintf(rpt, "%s : %s : %s\n",
	      byt_megs, req_comm, dome->dom_top[i]->fulldom);
    }
  }
  fprintf(rpt, "</pre><p><hr><p>\n");

  /* the top ten list is done, show the domain report */

  for (br = dome->dom_root->sub; br; br = br->next)
    top_domains++;		/* count the top level domains */

  fprintf(rpt, "<center><h2>Domain Report");
  if (dome->dirnam && strcmp(dome->dirnam, "/"))
    fprintf(rpt, " for %s", dome->dirnam); /* show dir if not top-level */
  fprintf(rpt, "</h2></center>\n");

  if (maxlevel==1)
    fprintf(rpt, "<p>This report indicates from which top-level domains the web site received requests.</p>\n<p>");
  else
    fprintf(rpt, "<p>This report indicates from which domains the web site received requests, including %d domain levels.</p>\n<p>", maxlevel);

  if (counthosts)
    fprintf(rpt, "%d unique hosts sent requests to this site.\n",
	    dome->unique_hosts);
  fprintf(rpt, "%d top-level domain%s sent requests to this site.</p><pre>\n",
	    top_domains, top_domains==1 ? "" : "s");

    fprintf(rpt, "   Bytes  :  Requests  : Domain\n");
    fprintf(rpt, " -------- : ---------- : ------\n");
    return;
} /* PrintDomainHeader */



/* PrintDomainLevel
 * go through the list of nodes at the current level, printing
 * them in order by sortkey (secondary key is alphabetical)
 * after printing each node, call PrintDomainLevel recursively on
 * that branch
 */
void PrintDomainLevel(rpt, br, level, maxlevel)
FILE *rpt;
domainbranch *br;
int level;
int maxlevel;
{
  domainbranch *this, *best;
  do {
    for (this = br, best = NULL;
	 this;
	 this = this->next)
      if (this->printed)
	;				/* printed already */
      else if (best == NULL)
	best = this;			/* first node is the best so far */
      else if (CompareDomainNodes(this, best, sortkey))
	best = this;			/* put this before best */

    if (best) {
      PrintDomainName(rpt, best, level);
      best->printed = TRUE;
      if (level < maxlevel)		/* recurse */
	PrintDomainLevel(rpt, best->sub, level+1, maxlevel);
    }
  } while (best);
  return;
} /* PrintDomainLevel */



/* create the top ten lists for this domain report
 *
 * each directory gets a set of domain reports
 *  and each set of domain reports gets a top ten list
 */
void TopTenDOM(dome, br, suffix, level)
donode *dome;
domainbranch *br;
char *suffix;
int level;
{
    domainbranch *this;
    if (!thisvhost) return;
    for (this = br; this; this = this->next) {
	int i, j, add;
	if (*this->domain == '\0')	/* ignore null domain name */
	    continue;
	if (level < domlevels) {
	  if (this->sub)
	    TopTenDOM(dome, this->sub, this->fulldom, level+1); /* recurse */
	  continue;
	}
	/* only get here if level=domlevels */
	/* see if it should be added to the top ten list */
	i = dome->bottom_dom;
	while (i > 0)
	    if (dome->dom_top[i-1] &&
		dome->dom_top[i-1]->nreq >= this->nreq)
		break;
	    else
		i--;

	if (dome->bottom_dom < TOPNUM) /* list is now longer */
	    dome->bottom_dom++;
	if (i < dome->bottom_dom) {	/* new item for list */
	    for (j = dome->bottom_dom-1; /* move rest of list down */
		 j > i;
		 j--)
		dome->dom_top[j]=dome->dom_top[j-1];

	    dome->dom_top[i] = this;	/* insert the new list item */
	}
    }
} /* TopTenDOM */



/* turn off all the "printed" flags at the current level
 * call self recursively on the next level
 */
void ResetDomainLevel(br)
domainbranch *br;
{
    domainbranch *this;

    for (this = br; this; this = this->next) {
	this->printed = FALSE;				/* not printed */
	ResetDomainLevel(this->sub);
    }
} /* ResetDomainLevel */



/* compare two nodes according to sortkey
 *
 * returns TRUE if node1 comes before node2
 *         FALSE otherwise
 */
int CompareDomainNodes(node1, node2, skey)
domainbranch *node1;
domainbranch *node2;
sortType skey;
{
    if (skey == N_BYTES)		/* sort by number of bytes */
	if (node1->nbytes < node2->nbytes)
	    return FALSE;
	else if (node1->nbytes == node2->nbytes)
	    ;
	else
	    return TRUE;
    else				/* sort by number of requests */
	if (node1->nreq < node2->nreq)
	    return FALSE;
	else if (node1->nreq == node2->nreq)
	    ;
	else
	    return TRUE;
    /* if the primary sort key is the same, sort alphabetically */
    return strcmp(node1->domain, node2->domain) < 0;
} /* CompareDomainNodes */



/* PrintDomainName
 *
 */
void PrintDomainName(rpt, br, level)
FILE *rpt;
domainbranch *br;
int level;
{
    char byt_megs[DATELEN],
	 req_comm[DATELEN];
    gigs(byt_megs, br->nbytes, 9);
    comm(req_comm, br->nreq, 10);
    fprintf(rpt, "%s : %s : ", byt_megs, req_comm);

    if (level == 0)
	fprintf(rpt, "%s\n", br->fulldom);
    else if (br->fulldom[strlen(br->fulldom)-1]=='.')	/* numeric */
	fprintf(rpt, "%*s%s\n", level, " ", br->fulldom);
    else if (level == 1)		/* top level, get description */
	fprintf(rpt, " %s%s\n", br->domain, FindDomainName(br->domain));
    else				/* indent 1 space for each dot */
	fprintf(rpt, "%*s%s\n", level, " ", br->fulldom);
    return;
} /* PrintDomainName */



/* FindDomainName
 *
 * look up the domain code in the list
 */
char *FindDomainName(dom)
char *dom;
{
    static char desc[DOMAINLEN];
    int i;
    for (i = 0; i < MAX_DOMAIN; i++)
	if (!strcmp(dom, doma[i][1])) {
	    sprintf(desc, " (%s)", doma[i][0]);
	    return desc;
	}
    return "";
} /* FindDomainName */



/* NewDomBranch
 *
 * returns a pointer to the new node
 */
domainbranch *NewDomBranch(domain, fulldom)
char *domain;
char *fulldom;
{
  domainbranch *rt = (struct domainbranch *) emalloc(sizeof(domainbranch));
  char *pnam;
  int offset=strlen(fulldom)-strlen(domain);
  if (offset < 0) {
    fprintf(stderr, "NDB: domain %s is longer than %s\n", domain, fulldom);
    exit(9);
  }
  rt->fulldom = e_strdup(fulldom ? fulldom : "");
  pnam=rt->fulldom+offset;
  if (strcmp(pnam, domain))
    rt->domain = e_strdup(domain ? domain : "");
  else
    rt->domain = pnam;
  rt->nreq    = 0;
  rt->nbytes  = 0.0;
  rt->printed = FALSE;
  rt->next    = NULL;
  rt->sub     = NULL;
  return rt;
} /* NewDomBranch */



/* AddDomReq
 *
 * if this node exists, then update the counters
 * otherwise create a new node
 * returns a pointer to the new or updated node
 */
domainbranch *AddDomReq(br, domain, fulldom, nbytes)
domainbranch *br;
char *domain;
char *fulldom;
float nbytes;
{
    domainbranch *this, *prev = '\0';
    int new_node = TRUE;

    if (!domain)
	progerr("AddDomReq gets null domain");
    /* keep the lists sorted alphabetically so that searching is faster */
    for (this = br->sub; this; ) {
	int comp = strcmp(domain, this->domain);
	if (comp < 0)
	    break;			/* not found */
	else if (comp==0) {		/* update existing entry */
	    this->nreq++;		/* number of accesses */
	    this->nbytes += nbytes;	/* number of bytes */
	    new_node = FALSE;
	    break;
	} else {			/* keep looking */
	    prev = this;
	    this = this->next;
	}
    }

    if (new_node) {	/* this domain not found in the list, so add it */
	domainbranch *rt = NewDomBranch(domain, fulldom);
	if (prev)			/* first on the list? */
	    prev->next = rt;		/*  no */
	else				/*  yes */
	    br->sub = rt;
	rt->next = this;
	rt->nreq = 1;
	rt->nbytes = nbytes;
	this = rt;
    }
    return this;
} /* AddDomReq */



int isip(address)
char *address;
{
  return isdigit(address[strlen(address)-1]);	/* last character a digit? */
}



void makelower(string)
char *string;
{
  while (*string)
    *string++ = tolower(*string);
}



/* create a domain report for each directory on this vhost's dominfo list
 * the reports are created in the domtree hierarchy
 */
void WriteDomainReports(vh)
vhost *vh;
{
    char rptfilnam[REQUESTLEN];
    char dobuf    [REQUESTLEN];
    donode *domm;
    int rcount=0;
    if (!vh) return;
    for (domm = vh->dominfo; domm; domm = domm->next) {
	sprintf(rptfilnam, "%s%s%s.domreq%d.html",
				vh->doroot, domm->dirnam, wdate, domlevels);
	if (!WriteReportDOM(rptfilnam, domlevels, vh, domm)) {
	    vh->printed = TRUE;		/* at least one report printed */
	    rcount++;
	}
    }
    if (rcount) {
	comm(dobuf, rcount, 10);
	printf("%s domain reports written\n", dobuf);
    }
    return;
} /* WriteDomainReports */



void CountUniqueHosts(dome)
donode *dome;
{
  char legend[REQUESTLEN];
  domainbranch *nub;
  int numcount = 0;

  int CountChildren(parent)
  domainbranch *parent;
  {
    domainbranch *nub;
    int numcount = 0;
    for (nub = parent->sub; nub; nub = nub->next)
      if (nub->sub) numcount += CountChildren(nub);
      else	  numcount++;
    return numcount;
  } /* CountChildren */

  if (!counthosts) return;

  /* use recursive function to get the number of unique hosts */
  numcount = CountChildren(dome->dom_root);
  sprintf(legend, "%d unique host names", numcount);
  free(dome->dom_root->fulldom);
  dome->dom_root->fulldom = e_strdup(legend);
  dome->unique_hosts = numcount;

  /* get the number of unique IP addresses */
  for (numcount=0, nub = dome->numeric_host->sub; nub; nub = nub->next)
    numcount++;
  sprintf(legend, "%d unique IP addresses", numcount);
  free(dome->numeric_host->fulldom);
  dome->numeric_host->fulldom = e_strdup(legend);
  dome->unique_hosts += numcount;
} /* CountUniqueHosts */



/* write one domain report
 * returns FALSE on success
 */
int WriteReportDOM(rptfilnam, lev, thisvhost, dome)
char *rptfilnam;
int lev;
vhost *thisvhost;
donode *dome;
{
  FILE *rpt;
  if (!dome) {
    printf("%s\nno domain report written because db is empty\n",
	   rptfilnam);
    return TRUE;
  }
  if (!dome->dom_root) {
    printf("WRD: can't create DOM, domain root is NULL\n");
    return TRUE;
  }
  if ((rpt = fopen(rptfilnam, "w")) == '\0') {
    printf("WRD: unable to create %s\n", rptfilnam);
    return TRUE;
  }
  WriteReportTop(rpt, thisvhost);

  PrintDomainHeader(rpt, dome, lev);
  PrintDomainLevel(rpt, dome->dom_root, 0, lev);
  ResetDomainLevel(dome->dom_root);	/* reset the printed flag */
  fprintf(rpt, "\n");
  PrintDomainLevel(rpt, dome->numeric_host, 0, 0); /* print one line */
  dome->numeric_host->printed = FALSE;	/* reset the printed flag */

  WriteReportBottom(rpt);
  fclose(rpt);
  return FALSE;
} /* WriteReportDOM */
