/* gs-extract.c
 *
 * writes the extract files
 *  part of the getstats program
 */


#include <stdio.h>
#include <string.h>
#include <dirent.h>			/* for opendir, readdir */
#include <sys/types.h>			/* for mkdir, stat */
#include <sys/stat.h>			/* for mkdir, stat */
#include "gs-struct.h"
#include "gs-extract.h"
#include "getstats.h"			/* for GZIPBIN */

extern char wdate[];			/* date is part of extract file name */
extern int debug, filesopen;
static char *gzipbin = GZIPBIN;

/* max number of extract files we can write */
#define XTMAXFIL	10000

/* number of outgoing extract records to hold in the buffer */
#define XTHOLD		    1

/* max number of extract files we can open simultaneously */
#define XTMAXOPEN	   50



xtnode *NewExtractInfo(sofar, r, br)
char *sofar;
logr *r;
treebranch *br;
{
  xtnode *xtlink;
  xtlink = wantExtract(r->virthost, sofar); /* search linked list */
  if (xtlink) {				/* found it */
    br->xt = xtlink;			/* link to xtinfo */
    mkExtractFile(r->virthost,		/* create empty file */
		  *sofar ? sofar : "/",
		  br->xt);
  } else
    br->wantextract = FALSE;
  return xtlink;
} /* NewExtractInfo */



/* NewExtract
 *
 * creates a new xtnode with information about the
 *  extract file for this directory
 *  this node is an extension of the treebranch record
 * returns a pointer to the new node
 */
xtnode *NewExtract(vh, dir, lev)
vhost *vh;
char *dir;
int lev;
{
  xtnode *no;
  if (!vh) return NULL;
  no = (struct xtnode *) emalloc(sizeof(xtnode));

  no->mex = NULL;			/* no output file yet */
  no->dirnam = e_strdup(dir);
  no->level = lev;
  no->nreq = 0;				/* lines written to disk */
  no->nbuff = 0;			/* lines in the buffer */
  no->outbuf = e_strdup("");		/* the buffer */
  no->next = vh->xtinfo;		/* join the list */
  vh->xtinfo = no;			/* with new node at the head */
  vh->xtfiles++;
  return no;
} /* NewExtract */



/* make a list of subdirectories under xtroot
 *
 * include the top directory (xtroot) by default,
 * but omit any directories that contain a ".noextract" file
 *
 * returns 0 on success
 */
int get_xt_list(vh, dir, lev)
vhost *vh;
char *dir;
int lev;
{
    DIR *mydir;
    struct dirent *myfile;
    char dirname[REQUESTLEN];
    int wantex = 1;

    if (!vh) {
	fprintf(stderr, "get_xt_list: virtual host is NULL\n");
	return 1;
    }
    if (!*vh->xtroot) {
	fprintf(stderr, "xtroot is undefined on host %s\n", vh->hostname);
	return 1;
    }
    sprintf(dirname, "%s%s", vh->xtroot, dir);
    mydir = opendir(dirname);
    if (!mydir) {
	fprintf(stderr, "unable to open %s ", dirname);
	perror("because");
	exit(3);
    }
    while (myfile = readdir(mydir)) {
	struct stat stbuf;
	char filnam[REQUESTLEN], path[REQUESTLEN];
	if (!strcmp(myfile->d_name, ".noextract")) {
	    wantex = 0;
	    continue;
	}
	if (*(myfile->d_name) == '.')		/* skip dot files */
	    continue;
	sprintf(filnam, "%s/%s", dir, myfile->d_name);
	sprintf(path, "%s%s", vh->xtroot, filnam);
	if (stat(path, &stbuf)) {
	    fprintf(stderr, "unable to stat %s ", path);
	    perror("because");
	    continue;
	}
	if (S_ISDIR(stbuf.st_mode))		/* found a subdirectory */
	    if (get_xt_list(vh, filnam, lev+1))	/* call myself recursively */
		return 1;			/*  failure propagates back */

    }
    closedir(mydir);

    if (wantex) {				/* include the top dir "" */
	if (vh->xtfiles >= XTMAXFIL) {
	    printf("%s wants more than %d extract files, increase XTMAXFIL\n",
		vh->hostname, XTMAXFIL);
	    return 1;				/* fail return */
	}
	NewExtract(vh, dir, lev);
    }
    return 0;
} /* get_xt_list */



/* go through the list of extract files for this vhost
 * if dir is one of the selected directories for extract files
 * returns pointer to xtnode, otherwise NULL
 */
xtnode *wantExtract(vh, dir)
vhost *vh;
char *dir;
{
  char wexdir[REQUESTLEN];
  int dlen;
  xtnode *k;
  if (!vh) return NULL;
  strcpy(wexdir, dir);
  dlen = strlen(wexdir);
  if (wexdir[dlen-1] == '/')		/* remove trailing slash */
    wexdir[dlen-1] = '\0';
  for (k = vh->xtinfo; k; k = k->next)
    if (!strcmp(wexdir, k->dirnam))
      return k;
  return NULL;
} /* wantExtract */



/* go through the list of extract files for each vhost, printing the
 * directories that will be ignored because they exceed dtrlevels
 */
void ShowIgnoredExtract(vhostlist, dtrlevels)
vhost *vhostlist;
int dtrlevels;
{
    char wexdir[REQUESTLEN];
    int dlen, header;

    vhost *v;
    xtnode *k;
    for (v=vhostlist; v; v=v->next) {
	header=0;
	for (k = v->xtinfo; k; k = k->next) {
	    if (dtrlevels < k->level) {
		if (!header) {
printf("these directories on host %s will not get extract files because DTRLEVELS=%d\n",
					v->hostname, dtrlevels);
		    header=1;
		}
		printf("      %s\n", k->dirnam);
	    }
	}
    }
} /* ShowIgnoredExtract */



/* first time we see this directory during this run
 * 1) remove the old compressed file, if any
 * 2) overwrite existing uncompressed file, if any
 * 3) create a new, empty file, and bump the counter
 *
 * returns 0 on success, 1 otherwise
 */
int mkExtractFile(vh, dir, xtlik)
vhost *vh;
char *dir;
xtnode *xtlik;
{
  struct stat s;
  char thisfilnam[REQUESTLEN];
  char thisfilgz [REQUESTLEN];

  if (!vh) return 1;
  sprintf(thisfilnam, "%s%shttpd.access.%s", vh->xtroot, dir, wdate);
  sprintf(thisfilgz,  "%s.gz", thisfilnam);
  if (stat(thisfilgz, &s) == 0) {	/* .gz file exists */
    if (unlink(thisfilgz)) {		/* remove the .gz file */
      fprintf(stderr, "unable to remove old extract %s ", thisfilgz);
      perror("because");
      return 1;
    }
    vh->xtoverwrite++;			/* overwrite the .gz file */
  }
					/* output file name */
  if (stat(thisfilnam, &s) == 0) {	/* uncompressed file exists */
    vh->xtoverwrite++;			/* overwrite uncompressed file */
    fprintf(stderr, "overwriting %s\n",  thisfilnam);
  }
  if (!(xtlik->mex = fopen(thisfilnam, "w"))) { /* create empty file */
    fprintf(stderr, "unable to create %s ", thisfilnam);
    perror("because");
    return 1;
  }
  if (debug) printf("MEF: %d files, %d xt files\n     %s is open\n",
		    ++filesopen, vh->xtfilesopen+1, thisfilnam);
  vh->xtcreated++;
  if (vh->xtfilesopen >= XTMAXOPEN) {
    fclose(xtlik->mex);
    xtlik->mex = NULL;
  } else
    vh->xtfilesopen++;
  return 0;
} /* mkExtractFile */



#define MAX_ADDR_PIECES 12
/* remove first part of host name (replace with "x"),
 * or remove the last part of IP address (replace with "0"),
 * for privacy
 */
void censor(newaddress, address)
char *newaddress;
char *address;
{
    char piece[MAX_ADDR_PIECES][ADDRLEN];
    int i,p=0;
    char *a=address, *b=piece[p];
    while (*a)				/* separate the pieces */
	if (*a=='.') {			/* end of this piece */
	    *b='\0';			/* start a new piece */
	    p++;
	    a++;
	    if (p<MAX_ADDR_PIECES)
		b=piece[p];
	    else {
		fprintf(stderr, "host name %s has more than %d pieces\n",
				address, MAX_ADDR_PIECES);
		break;
	    }
	} else
	    *b++ = *a++;
    *b='\0';
    p++;

    /* address is numeric IP address if the first char of last piece is numeric */
    if (isdigit(piece[p-1][0]))
	strcpy(piece[p-1], "0");	/* replace last piece with "0" */
    else
	strcpy(piece[0], "x");		/* replace first piece with "x" */
    for (i=0, b=newaddress; i<p; i++) {	/* put the pieces back together */
	if (i) *b++ = '.';
	a=piece[i];
	while (*a) *b++ = *a++;
    }
} /* censor */



/* if the file is currently open, write to it
 * otherwise, append one record to the xtnode buffer
 *   when the buffer fills, flush to disk
 */
void WriteExtract(r, xtlik)
logr *r;
xtnode *xtlik;
{
  char	tmpadd[ADDRLEN]="",
	tmpreq[REQUESTLEN]="",
	tmpref[REQUESTLEN]="",
	tmpcli[REQUESTLEN]="";
  char wexbuf[MAXLINE*XTHOLD];

  censor(tmpadd, r->address);	/* change first part of host name to "x" */
  if (*r->rprefix || *r->request || *r->rsuffix)
    sprintf(tmpreq, "\"%s %s %s\"", r->rprefix, r->request, r->rsuffix);
  else
    strcpy(tmpreq, "\"-\"");

  if (*r->referer)
    sprintf(tmpref, " \"ref %s\"", r->referer);
  if (*r->client)
    sprintf(tmpcli, " \"client %s\"", r->client);

  sprintf(wexbuf, "%s%s %s %s [%s] %s %s %s%s%s\n",
	  xtlik->outbuf ? xtlik->outbuf : "",
	  tmpadd, r->ident, r->user, r->date, tmpreq,
	  r->status, r->nbytstr, tmpref, tmpcli);

  xtlik->nbuff++;			/* records in the buffer */
  if (xtlik->outbuf)			/* free the old buffer, if any */
    free(xtlik->outbuf);
  xtlik->outbuf = e_strdup(wexbuf);	/* malloc a new buffer */

  if (xtlik->mex ||			/* file is currently open, or */
      (xtlik->nbuff >= XTHOLD)) /* buffer is full */
    FlushExtractBuffer(r->virthost, xtlik); /* write it to disk */
  return;
} /* WriteExtract */



/* flush the buffer for this extract file, and create an empty buffer */
void FlushExtractBuffer(vh, xtlik)
vhost *vh;
xtnode *xtlik;
{
  char thisfilnam[REQUESTLEN]="";
  sprintf(thisfilnam, "%s%s/httpd.access.%s", vh->xtroot,
			xtlik->dirnam, wdate);
					/* append to extract file */
  if (xtlik->mex) {			/* file is currently open */
    fputs(xtlik->outbuf, xtlik->mex);	/* write to disk */
  } else if ((xtlik->mex = fopen(thisfilnam, "a")) == NULL) {
    fprintf(stderr, "unable to append to %s ", thisfilnam);
    perror("because");
    exit(9);
  } else {
    fputs(xtlik->outbuf, xtlik->mex);	/* write to disk */
    fclose(xtlik->mex);
    xtlik->mex = NULL;
  }
  free(xtlik->outbuf);			/* free the old buffer */
  xtlik->outbuf = NULL;			/* buffer is now empty */

  vh->xtwrites++;			/* number of disk writes */
  vh->xtrecords += xtlik->nbuff;	/* extract records written */
  xtlik->nreq += xtlik->nbuff;		/* records in this extract file */
  xtlik->nbuff = 0;			/* buffer is empty */
  return;
} /* FlushExtractBuffer */



void CompressExtractFiles(vh)
vhost *vh;
{
    struct stat s;
    char xtbuf     [REQUESTLEN],
	 xtfilename[REQUESTLEN];
    int xtcomp=0;
    xtnode *mo;

    if (!vh) return;
    if (vh->xtfiles==0) return;
					/* compress new extract files */
    for (mo = vh->xtinfo; mo; mo = mo->next) {
	if (mo->nbuff)			/* buffer not empty */
	    FlushExtractBuffer(vh, mo);	/* write it to disk */
	if (mo->mex) {			/* file is still open */
	    fclose(mo->mex);		/*  so close it */
	    mo->mex = NULL;
	    vh->xtfilesopen--;
	    if (debug) printf(
		"CEF: %d files, %d xtfiles\n     xt %s:%s is closed\n",
		--filesopen, vh->xtfilesopen, vh->hostname, mo->dirnam);
	}
	if (mo->nreq == 0)		/* no records in this file */
	    continue;			/*  so don't compress it */

	sprintf(xtfilename, "%s%s/httpd.access.%s",
					vh->xtroot, mo->dirnam, wdate);
	sprintf(xtbuf, "%s %s", gzipbin, xtfilename);
	if (system(xtbuf)==0)		/* compress the file */
	    xtcomp++;
    }
    comm(xtbuf, vh->xtrecords, 10);
    printf("%s extract records written to %d out of %d files\n",
			xtbuf, vh->xtcreated, vh->xtfiles);
    if (vh->xtrecords != vh->xtwrites) {
	comm(xtbuf, vh->xtwrites, 1);
	printf("using %s disk writes\n", xtbuf);
    }
    if (vh->xtoverwrite) {
	comm(xtbuf, vh->xtoverwrite, 10);
	printf("%s extract files overwritten\n", xtbuf);
    }

    comm(xtbuf, xtcomp, 10);
    printf("%s extract files compressed\n", xtbuf);
    return;
} /* CompressExtractFiles */
