/*
 * DEXPIRE.C	- diablo expire.
 *
 *	remove directories in time order until sufficient space is 
 *	available.
 *
 *	When sufficient space is available, scan remaining files and
 *	then scan history and set the expired flags as appropriate.
 *
 * (c)Copyright 1997, Matthew Dillon, All Rights Reserved.  Refer to
 *    the COPYRIGHT file in the base directory of this distribution 
 *    for specific rights granted.
 *
 * Modification by Nickolai Zeldovich to store msgid hashes when
 * expiring articles to allow for better overview expiration.
 */

#include "defs.h"
#include <sys/param.h>
#ifndef _AIX
#include <sys/mount.h>
#endif
#ifdef _AIX
#include <sys/statfs.h>
#endif

#if USE_SYSV_STATFS
#include <sys/statfs.h>
#define f_bavail f_bfree
#endif

/*
 * The solaris statvfs is rather limited, but we suffer with reduced
 * capability (and hence take a possible performance hit).
 */
#if USE_SUN_STATVFS
#include <sys/statvfs.h>	/* god knows if this hack will work */
#define f_bsize	f_frsize	/* god knows if this hack will work */
#define fsid_t u_long
#define statfs statvfs
#endif

#if USE_SYS_VFS			/* this is mainly for linux	*/
#include <sys/vfs.h>
#endif

/*
 * For each directory (D.*) that contains files (B.*), we point to the
 * Partiton that contains this directory (P.* or N.*)
 *
 * The old spool layout will have Partiton = /news/spool/news
 *
 * The spooldir format will have Partiton = /news/spool/news/N.nn
 *
 * The new format will have Partition = /news/spool/news/P.nn
 * or Partition = /news/spool/news/P.nn/N.nn
 *
 */
typedef struct FileSystem {
    dev_t		dev;		/* The unique filesystem ID */
    double		fsfree;		/* Space free on filesystem  (MB) */
    long		fsfreefiles;	/* Inodes free on filesystem */
    struct FileSystem	*next;
} FileSystem;

/*
 * Linked list of all P.* and N.* directories with a pointer to the
 * underlying filesystem.
 */

typedef struct Partition {
    char		*partname;	/* The full directory path */
    double		minfree; 	/* Free Space target  (MB) */
    long		minfreefiles; 	/* Free Inodes target */
    int			spaceok;	/* Is space ok */
    double		partsize;	/* Space used in partition  (MB) */
    time_t		age;		/* Age of oldest dir in part (secs) */
    double		maxsize; 	/* Max size allowed */
    long		keeptime; 	/* How long to keep articles */
    int			expmethod; 	/* Sync or dirsize expire method */
    struct FileSystem	*filesys;	/* Which filesys does this lie on */
    struct Partition	*next;
} Partition;

/*
 * Entry for D.* directories with a pointer to the containing partition.
 * The partition size of updated when spool entires are removed and its
 * spaceok flag is set when further spool entries within the partition do
 * not need to be cleaned up.
 */

typedef struct SpoolEnt {
    char		*dirname;	/* The relative path to directory */
    double		dirsize;	/* Space used in directory */
    struct Partition	*partn;		/* Which spool partition */
    uint8		removed;	/* Has it been removed */
} SpoolEnt;

struct FileSystem *FileSystems = NULL;
struct Partition *Partitions = NULL;
struct SpoolEnt **SpoolEntries;
int		entryIdx = 0;
int		entryMax = 64;
double		FreeSpaceTargetList[100];
uint16		*SinglePart = NULL;
time_t		TimeNow;

int VerboseOpt = 1;
int NotForReal = 1;
int SoftUpdates = -1;
int UseDirSize = 0;
int MaxPass = 0;
int UnexpireOpt = 0;
int DoCleanup = 1;
int TestOpt = 0;
int UpdateHistory = 0;		/* Do we run the history update */
int HistoryUpdateOpt = -1;	/* Do we actually update the history entry 
				 *	0 = No
				 *	1 = Force
				 *	2 = Not entries
				 *	-1 = if changes
				 */
int WriteHashesToFileOpt = 0;
const char *HistoryFile = NULL;
off_t HistoryEnd;
int HistoryFd = -1;

int CleanAllSpools(void);
int spoolEntSort(const void *s1, const void *s2);
void addFreeSpaceTarget(char *which);
int cleanupDirectories(void);
long removeDirectory(char *partname, char *dirname, int *count, int *ccount, double *size);
void scanDirectory(Partition *scanpart);
double findSize(char *pathname, char *dirname);
void printPartition(Partition *p);
int updateHistory(void);
int findNode(const char *path, int createMe);
void freeNodes(void);
void dumpNodeTable(void);
time_t getAge(char *dirname);
double freeSpaceOn(char *path, int logit, long *freefiles);
FileSystem *findFileSys(char *path);
void DumpSpoolEntries(void);

void
Usage(void)
{
	printf("This program performs an expire run on a diablo spool\n");
	printf("dexpire -a|n [-c0] [-f historyfile] [-h0|1] [-k] [-O nn] [-o] [-q] [-R s:n] [-S nn] [-s n] [-u] [-v] [-z] [-C diablo.config] [-d[n]] [-V]\n");
	printf("\t-a\tactually make changes (nothing is done without this option)\n");
	printf("\t-c0\tdon't perform file removal pass\n");
	printf("\t-f\tspecify the history file to use\n");
	printf("\t-h0\tdon't scan history file at all\n");
	printf("\t-h1\tforce scan of history file\n");
	printf("\t-k\tdon't update history entries\n");
	printf("\t-n\trun through process, but don't make any changes\n");
	printf("\t-O\tset number of dexpire iterations before exit\n");
	printf("\t-o\twrite ID hashes to a file\n");
	printf("\t-q\tbe relatively quiet\n");
	printf("\t-R\tset a freespacetarget for a spool (spoolnum:target)\n");
	printf("\t-S\tonly expire the specified spool number\n");
	printf("\t-s\tsync before disk free space checks\n");
	printf("\t-t\ttest expired status to spool status only\n");
	printf("\t-u\t'unexpire' all articles marked as expired in dhistory\n");
	printf("\t-v\tbe more verbose\n");
	printf("\t-z\texpire based on directory size, rather than disk space\n");
	printf("\t-C file\tspecify diablo.config to use\n");
	printf("\t-d[n]\tset debug [with optional level]\n");
	printf("\t-V\tprint version and exit\n");
	exit(0);
}


int
main(int ac, char **av)
{
    int n;
    int allOk = 0;
    int count;

    LoadDiabloConfig(ac, av);

    for (n = 1; n < ac; ++n) {
	char *ptr = av[n];

	if (*ptr == '-') {
	    ptr += 2;
	    switch(ptr[-1]) {
	    case 'a':
		NotForReal = 0;
		break;
	    case 'c':
		DoCleanup = strtol(ptr, NULL, 0);;
		break;
	    case 'f':
		HistoryFile = *ptr ? ptr : av[++n];
		break;
	    case 'h':
		if (*ptr == '0')
		    HistoryUpdateOpt = 0;
		else
		    HistoryUpdateOpt = 1;
		break;
	    case 'k':
		HistoryUpdateOpt = 2;
		break;
	    case 'n':
		NotForReal = 1;
		break;
	    case 'O':
		if (*ptr)
		    MaxPass = strtol(ptr, NULL, 0);
		else
		    MaxPass = 1;
		break;
	    case 'o':
		WriteHashesToFileOpt = 1;
		break;
	    case 'q':
		VerboseOpt = 0;
		break;
	    case 'R':
		addFreeSpaceTarget(*ptr ? ptr : av[++n]);
		break;
	    case 'S':
		SinglePart = (uint16 *)malloc(sizeof(uint16));
		*SinglePart = (uint16)strtol((*ptr ? ptr : av[++n]), NULL, 0);
		break;
	    case 's':
		if (*ptr)
		    SoftUpdates = strtol(ptr, NULL, 0);
		else
		    SoftUpdates = 1;
		break;
	    case 't':
		TestOpt = 1;
		VerboseOpt = 1;
		break;
	    case 'u':
		UnexpireOpt = 1;
		break;
	    case 'v':
		VerboseOpt = (*ptr) ? strtol(ptr, NULL, 0) : 1;
		break;
	    case 'z':
		if (*ptr)
		    UseDirSize = strtol(ptr, NULL, 0);
		else
		    UseDirSize = 1;
		break;
	    /* Common options */
	    case 'C':		/* parsed by LoadDiabloConfig */
		if (*ptr == 0)
		    ++n;
		break;
	    case 'd':
		VerboseOpt++;
		if (isdigit((int)(unsigned char)*ptr)) {
		    DebugOpt = strtol(ptr, NULL, 0);
		} else {
		    --ptr;
		    while (*ptr == 'd') {
			++DebugOpt;
			++ptr;
		    }
		}
		break;
	    case 'V':
		PrintVersion();
		break;
	    default:
		fprintf(stderr, "Illegal option: %s\n", ptr - 2);
		exit(1);
	    }
	} else {
	    fprintf(stderr, "Illegal argument: %s\n", ptr);
	    exit(1);
	}
    }

    /*
     * this isn't an error, but a request to list 
     * valid arguments, then exit.
     */

    if (ac == 1) {
	Usage();
    }

    if (NotForReal)
	MaxPass = 1;
    if (NotForReal && HistoryUpdateOpt == 1)
	HistoryUpdateOpt = 2;
    count = 1;
    if (HistoryFile == NULL)
	HistoryFile = strdup(PatDbExpand(DHistoryPat));
    TimeNow = time(NULL);
    while (!allOk) {
	if (VerboseOpt)
	    printf("Spool scan pass %d\n", count);
	allOk = CleanAllSpools();
	if (allOk || (MaxPass > 0 && --MaxPass == 0))
	    break;
	count++;
	sleep(10);
    }
    return(0);
}

int
CleanAllSpools(void)
{
    int n;
    int allOk = 1;

    UpdateHistory = (HistoryUpdateOpt > 0);
    FileSystems = NULL;
    Partitions = NULL;

    /*
     * Removal Scan
     */

    SpoolEntries = malloc(entryMax * sizeof(SpoolEnt *));
    entryIdx = 0;
     
    LoadSpoolCtl(0, 1);

    /*
     * Open and get a copy of the history file size before we scan the spools
     * because we shouldn't check history entries added beyond this point
     */
    if (HistoryUpdateOpt != 0 || UnexpireOpt) {
	struct stat sb;

	HistoryFd = open(HistoryFile, O_RDWR, 0644);
	if (HistoryFd == -1) {
	    fprintf(stderr, "Unable to open history(%s): %s\n",
						HistoryFile, strerror(errno));
	    exit(1);
	}
	if (fstat(HistoryFd, &sb) != 0) {
	    fprintf(stderr, "Unable to stat history(%s): %s\n",
						HistoryFile, strerror(errno));
	    exit(1);
	}
	HistoryEnd = sb.st_size;
    }

    /*
     * Get into spool directory
     */
    if (chdir(PatExpand(SpoolHomePat)) != 0) {
	fprintf(stderr, "Unable to chdir(%s)\n", PatExpand(SpoolHomePat));
	exit(1);
    }

    /*
     * Scan through each of the spool objects
     */
    if (!UnexpireOpt) {
	int i;
	uint16 spoolnum;
	char *path;
	double maxsize = 0.0;
	double minfree = 0.0;
	long minfreefiles = 0;
	long keeptime = 0;
	int expmethod = EXM_SYNC;
	
	for (i = GetFirstSpool(&spoolnum, &path, &maxsize, &minfree, &minfreefiles, &keeptime, &expmethod); i;
		i = GetNextSpool(&spoolnum, &path, &maxsize, &minfree, &minfreefiles, &keeptime, &expmethod))  {
	    if (SinglePart && *SinglePart != spoolnum)
		continue;
	    {
		Partition *p = (Partition *)malloc(sizeof(Partition));;
	
		if (VerboseOpt)
		    printf("Spool Object: %02d\n", spoolnum);
		p->partname = (char *)malloc(strlen(path) + 2);
		strcpy(p->partname, path);
		if (path[strlen(path) - 1] != '/')
		    strcat(p->partname, "/");
		p->partsize = 0.0;
		p->age = 0;
		p->minfree = minfree;
		p->minfreefiles = minfreefiles;
		p->maxsize = maxsize;
		p->keeptime = keeptime;
		if (UseDirSize)
		    p->expmethod = EXM_DIRSIZE;
		else
		    p->expmethod = expmethod;
		p->spaceok = 0;
		p->filesys = findFileSys(p->partname);
		p->next = Partitions;
		Partitions = p;
		if (FreeSpaceTargetList[spoolnum] > 0.0)
		    p->minfree = FreeSpaceTargetList[spoolnum];
		scanDirectory(p);
	    }
	}
    }

    if (chdir(PatExpand(SpoolHomePat)) != 0) {
	fprintf(stderr, "Unable to chdir(%s)\n", PatExpand(SpoolHomePat));
	exit(1);
    }

    /*
     * Sort directory	'0' <= 'a' so we can safely sort numerically
     *			with strcmp.
     */


    if (entryIdx) {
	qsort(SpoolEntries, entryIdx, sizeof(SpoolEnt *), spoolEntSort);
	if (DoCleanup)
	    allOk = cleanupDirectories();
    }

    /*
     * History file update scan
     */

    if (chdir(PatExpand(SpoolHomePat)) != 0) {
	fprintf(stderr, "Unable to chdir(%s)\n", PatExpand(SpoolHomePat));
	exit(1);
    }

    if (UpdateHistory || UnexpireOpt) {
	int n;
	printf("DExpire updating history file\n");
	n = updateHistory();
	printf("DExpire history file update complete, %d articles %smarked %sexpired\n",
					n,
					NotForReal ? "would be " : "",
					UnexpireOpt ? "un" : "");
    } else {
	printf("DExpire history file will not be updated\n");
    }

    if (HistoryUpdateOpt != 0 || UnexpireOpt)
	close(HistoryFd);

    /*
     * Cleanup all the allocated space
     */
    for (n = 0; n < entryIdx; ++n) {
	free(SpoolEntries[n]->dirname);
	free(SpoolEntries[n]);
    }
    free(SpoolEntries);

    while (Partitions) {
	Partition *next;

	next=Partitions->next;
	free(Partitions);
	Partitions=next;        
    }

    while (FileSystems) {
	FileSystem *next;

	next=FileSystems->next;
	free(FileSystems);
	FileSystems=next;        
    }
    return(allOk);
}

int
spoolEntSort(const void *s1, const void *s2)
{
    struct SpoolEnt *ent1 = *(struct SpoolEnt **)s1;
    struct SpoolEnt *ent2 = *(struct SpoolEnt **)s2;

    return(strcmp(ent1->dirname, ent2->dirname));
}

void
addFreeSpaceTarget(char *which)
{
    char *p;
    int part;

    p = strchr(which, ':');
    if (p == NULL) {
	fprintf(stderr, "-R parameter needs 'n=n'\n");
	exit(1);
    }
    *p = '\0';
    part = atoi(which);
    if (part < 0 || part >= 100) {
	fprintf(stderr, "Invalid spool number for -R (%s)\n", which);
	exit(1);
    }
    FreeSpaceTargetList[part] = strtod(++p, NULL);
    if (FreeSpaceTargetList[part] < 0.0) {
	fprintf(stderr, "Invalid free space target in -R (%d)\n", part);
	exit(1);
    }
}

long
removeDirectory(char *partname, char *dirname, int *count, int *ccount, double *size)
{
    DIR *dir;
    char pwd[PATH_MAX];
    char tdir[PATH_MAX];
    char *ptr;
    int c = 0;

    if (getcwd(pwd, sizeof(pwd)) == NULL) {
	fprintf(stderr, "Unable to determine current directory\n");
	return(c);
    }

    sprintf(tdir,"%s/%s", partname, dirname);
    if (chdir(tdir) != 0) {
	fprintf(stderr, "Unable to chdir(%s)\n", tdir);
	return(c);
    }

    if (VerboseOpt) {
	printf("    -%s%s ... ", partname, dirname);
	fflush(stdout);
    }

    if ((dir = opendir(".")) != NULL) {
	den_t *den;
	struct stat sb;

	while ((den = readdir(dir)) != NULL) {
	    if ((den->d_name[0] == 'B' && den->d_name[1]=='.') ||
		    (strlen(den->d_name) > 8 && den->d_name[8] =='.')
	    ) {
		if (size != NULL && stat(den->d_name, &sb) == 0)
		    *size += sb.st_size;
		if (!NotForReal)
		    remove(den->d_name);
		if (count)
		    ++*count;
		c++;
	    } else if (WildCmp("*.core", den->d_name) == 0 ||
				WildCmp("core.*", den->d_name) == 0 ||
				strcmp(den->d_name, "core") == 0
	    ) {
		if (size != NULL && stat(den->d_name, &sb) == 0)
		    *size += sb.st_size;
		if (!NotForReal)
		    remove(den->d_name);
		if (ccount)
		    ++*ccount;
		c++;
	    }
	}
	closedir(dir);
    }
    if (chdir("..") != 0) {
	fprintf(stderr, "Unable to chdir(..)\n");
	return(c);
    }
    strcpy(tdir, dirname);
    if ((ptr = strchr(tdir, 'D')))
	*ptr = 'A';
    if (!NotForReal && rename(dirname, tdir) < 0) {
	printf("unable to rename directory: %s\n", dirname);
    } else {
	if (VerboseOpt)
	    printf("done.\n");
	errno = 0;
	if (!NotForReal && rmdir(tdir) < 0)
	    fprintf(stderr, "Unable to rmdir(\"%s\"): %s\n",
					tdir, strerror(errno));
	else
	    c++;
    }
    chdir(pwd);
    return(c);
}

/*
 * cleanupDirectories() - work through each partition and cleanup
 * space on it until enough space is free
 */
int
cleanupDirectories(void)
{
    int i;
    int dircount = 0;
    int count = 0;
    int ccount = 0;
    double tremsize = 0.0;
    int allclean = 1;

    if (entryIdx)
	printf("%d directories (%s - %s)\n", entryIdx, SpoolEntries[0]->dirname,
				SpoolEntries[entryIdx-1]->dirname);

    if (VerboseOpt)
	printf("Cleaning up directories\n");
    /*
     * Remove files a directory at a time.  The spool directories are
     * named A.* or D.*.  We remove a directory by renaming it from D.
     * to A., removing the files, then removing the directory.  The
     * rename is required to prevent Diablo from recreating files in
     * the directory (and thus potentially corrupting articles by 
     * reusing history keys).
     */

    for (i = 0; i < entryIdx; ++i) {
	int spaceok = 1;
	int filesok = 1;
	int sizeok = 1;
	int ageok = 1;
	Partition *part;
	double spacefree;
	long freefiles;
	double remsize = 0.0;
	long c;

	part = SpoolEntries[i]->partn;
	if (part->spaceok)
	    continue;

	spacefree = part->filesys->fsfree;
	freefiles = part->filesys->fsfreefiles;
#if 0
	/*
	 * XXX We don't need to do this because it has already been done
	 * as part of the directory scan and we do it after removing
	 * each directory
	 */
	if (part->expmethod == EXM_SYNC && part->minfree) {
	    spacefree = freeSpaceOn(part->partname, 1, &freefiles);
	}
#endif
	if (VerboseOpt) {
	    printf("Checking %s on %s\n", SpoolEntries[i]->dirname,
							part->partname);
	    if (part->maxsize)
		printf("\tsize=%s/%s\n", ftos(part->partsize),
						ftos(part->maxsize));
	    if (part->keeptime)
		printf("\tkeeptime=%u/%lu secs\n", part->age, part->keeptime);
	    if (part->minfree)
		printf("\tfreespace=%s/%s\n", ftos(spacefree),
						ftos(part->minfree));
	    if (part->minfreefiles)
		printf("\tfreefiles=%ld/%ld\n", freefiles, part->minfreefiles);
	    printf("\texpire method %s",
			(part->expmethod == EXM_SYNC) ? "sync" : "dirsize");

	}
	if (part->minfree && spacefree < part->minfree)
	    spaceok = 0;
	if (part->minfreefiles && freefiles < part->minfreefiles)
	    filesok = 0;
	if (part->maxsize && part->partsize > part->maxsize)
	    sizeok = 0;
	if (part->keeptime && part->age > part->keeptime)
	    ageok = 0;
	if (spaceok && sizeok && ageok && filesok) {
	    part->spaceok = 1;
	    if (VerboseOpt)
		printf("\t\t: ok\n");
	    continue;
	}
	if (VerboseOpt) {
	    printf("\t\t: need attention ");
	    if (!spaceok)
		printf("(freespace < minfree) ");
	    if (!filesok)
		printf("(files < minfiles) ");
	    if (!sizeok)
		printf("(size > maxsize) ");
	    if (!ageok)
		printf("(age > keeptime) ");
	    printf("\n");
	}
	if (DebugOpt > 1)
	    printPartition(part);

	/*
	 * XXX We should probably include the size of the dir in remsize
	 */
	c = removeDirectory(part->partname, SpoolEntries[i]->dirname,
						&count, &ccount, &remsize);
	tremsize += remsize;
	++dircount;
	allclean = 0;
	SpoolEntries[i]->removed = 1;
	if (HistoryUpdateOpt != 0)
	    UpdateHistory = 1;
	part->filesys->fsfree += remsize;
	part->filesys->fsfreefiles += c;
	if (part->expmethod == EXM_DIRSIZE || part->maxsize) {
	    SpoolEntries[i]->dirsize = remsize;
	    part->partsize -= remsize;
	} else if (!NotForReal && (part->minfree || part->minfreefiles))
	    part->filesys->fsfree = freeSpaceOn(part->partname, 1,
					&part->filesys->fsfreefiles);
	if (i + 1 < entryIdx)
	    part->age = TimeNow - getAge(SpoolEntries[i + 1]->dirname);
	if (VerboseOpt) {
	    if (part->expmethod == EXM_DIRSIZE || part->maxsize)
		printf("     Removed %s (used: %s free: %s age: %s)\n",
					ftos(remsize),
					ftos(part->partsize),
					ftos(part->filesys->fsfree),
					dtlenstr(part->age));
	    else if (part->minfree)
		printf("     Removed %s (free: %s age: %s)\n",
					ftos(remsize),
					ftos(part->filesys->fsfree),
					dtlenstr(part->age));
	    else if (part->minfreefiles)
		printf("     Removed %ld (freefiles: %ld age: %s)\n",
					c,
					part->filesys->fsfreefiles,
					dtlenstr(part->age));
	    else 
		printf("     Removed %s (age: %s)\n",
					ftos(remsize),
					dtlenstr(part->age));
	}
    }
    printf("%d files %s (%d directories) removed", count,
						ftos(tremsize), dircount);
    if (ccount)
	printf(", and %d core files removed!", ccount);
    printf("\n");
    fflush(stdout);
    return(allclean);
}

void
printPartition(Partition *p)
{
    printf("partname: %s\n", p->partname);
    printf("size: %s\n", ftos(p->partsize));
    printf("minfree: %s\n", ftos(p->minfree));
    printf("spaceok: %d\n", p->spaceok);
    printf("maxsize: %s\n", ftos(p->maxsize));
    printf("keeptime: %s\n", dtlenstr(p->keeptime));
    printf("age: %s\n", dtlenstr(p->age));
    printf("free: %s\n", ftos(p->filesys->fsfree));
    printf("freefiles: %ld\n", p->filesys->fsfreefiles);
}

/*
 * scanDirectory(): Scan a Diablo spool directory and act according to
 *    the type of partition/directory/file found:
 *
 *	P.01 		= A spool partition (dexpire.ctl Sn option)
 *	N.00 		= A spool partition (DiabloSpoolDirs)
 *	D.00f49e76	= A spool directory (gets renamed to A.*)
 *	A.00f49e76	= A spool directory that can be removed
 *	B.051b		= A spool file
 *
 * All spool directories on partitions that need clearing are stored
 * in the array "SpoolEntries[]"
 */
void
scanDirectory(Partition *scanpart)
{
    DIR *dir;
    den_t *den;
 
    /*
     * Check to see if this directory/partition actually needs
     * to be cleaned up. If we aren't updating history, we don't
     * bother checking the partition.
     *
     */

    if (DebugOpt > 1)
	printf("Scanning directory: %s\n", scanpart->partname);

    if ((dir = opendir(scanpart->partname)) == NULL) {
	fprintf(stderr, "Unable to scan directory: %s (%s)\n",
				scanpart->partname, strerror(errno));
	return;
    }
    {
	while ((den = readdir(dir)) != NULL) {
	    if (den->d_name[1] != '.' || (strcmp(den->d_name, "..") == 0))
		continue;
	    switch (den->d_name[0]) {
	    case 'N':
		{
		Partition *p = malloc(sizeof(Partition));
		
		    p->partname = (char *)malloc(strlen(scanpart->partname) +
						strlen(den->d_name) + 2);
		    sprintf(p->partname, "%s%s/", scanpart->partname,
						den->d_name);
		    p->partsize = 0;
		    p->age = 0;
		    p->spaceok = 0;
		    p->keeptime = scanpart->keeptime;
		    p->maxsize = scanpart->maxsize;
		    p->minfree = scanpart->minfree;
		    p->minfreefiles = scanpart->minfreefiles;
		    p->filesys = findFileSys(p->partname);
		    p->expmethod = scanpart->expmethod;
		    p->next = scanpart->next;
		    scanpart->next = p;
		    scanDirectory(p);
		    scanpart->partsize += p->partsize;
		    if (p->age > scanpart->age)
			scanpart->age = p->age;
		    break;
		}
	    case 'B':
		fprintf(stderr, "Skipping unexpected spool file %s%s\n",
					scanpart->partname, den->d_name);
		break;
	    case 'A':
		removeDirectory(scanpart->partname, den->d_name, NULL, NULL, NULL);
		break;
	    case 'D':
		if (entryIdx == entryMax) {
		    entryMax = entryMax * 2;
		    SpoolEntries = realloc(SpoolEntries, entryMax * sizeof(SpoolEnt *));
		    if (!SpoolEntries) {
			fprintf(stderr, "unable to realloc in scan for queue dirs\n");
			exit(1);
		    }
		}
		SpoolEntries[entryIdx] = malloc(sizeof(SpoolEnt));
		if (!SpoolEntries[entryIdx]) {
		    fprintf(stderr, "unable to malloc in scan for queue dirs\n");
		    exit(1);
		}

		SpoolEntries[entryIdx]->partn = scanpart;
		SpoolEntries[entryIdx]->removed = 0;
		SpoolEntries[entryIdx]->dirsize = 0.0;
		SpoolEntries[entryIdx]->dirname = strdup(den->d_name);

		if (scanpart->expmethod == EXM_DIRSIZE || scanpart->maxsize) {
		    SpoolEntries[entryIdx]->dirsize =
				findSize(scanpart->partname, den->d_name);
		    scanpart->partsize += SpoolEntries[entryIdx]->dirsize;
		}
		{
		    time_t t;
		    t = getAge(den->d_name);
		    if ((TimeNow - t) > scanpart->age)
			scanpart->age = TimeNow - t;
		}
		if (DebugOpt > 2)
		    printf("ADD DIR %s%s dirsize=%s age=%u partsize=%s\n",
				scanpart->partname,
				den->d_name,
				ftos(SpoolEntries[entryIdx]->dirsize),
				scanpart->age,
				ftos(scanpart->partsize));
		++entryIdx;
		break;
	    case 'P':
		/* Skip spool object names */
		break;
	    default:
		printf("Skipping unknown file/dir: %s%s\n",
					scanpart->partname, den->d_name);
	    }
	}

    }
    closedir(dir);
}

/*
 * Find the size of a spool directory (in MB)
 */
double
findSize(char *pathname, char *dirname)
{
    char p[PATH_MAX];
    char f[PATH_MAX];
    DIR *dir;
    den_t *den;
    double size = 0;
    struct stat sb;

    strcpy(p, pathname);
    strcat(p, dirname);
    if ((dir = opendir(p)) == NULL) {
	fprintf(stderr, "Unable to find size of directory: %s (%s)\n",
					p, strerror(errno));
	return(0);
    }
    strcat(p, "/");
    while ((den = readdir(dir)) != NULL) {
	if (strcmp(den->d_name, "..") == 0)
	    continue;
	strcpy(f, p);
	strcat(f, den->d_name);
	if (stat(f, &sb) == 0)
	    size += sb.st_size;
    }
    closedir(dir);
    return(size);
}

int
updateHistory(void)
{
    uint32 countExp = 0;
    uint32 numEnt = 0;
    off_t countEnt = 0;
    off_t countTestSpool = 0;
    off_t countTestHistory = 0;
    off_t countTestExpired = 0;
    off_t countTestValid = 0;
    int lastPerc = 0;
    FILE *DExpOverList = NULL;
    char path[PATH_MAX];
    int i;
    char spoolHome[512];
    int spoolHomeLen;
    off_t npos = -1;
    off_t bpos = -1;
    History *h;

    /*
     * Write expired article msgid hashes to a file if requested.
     */

    if (WriteHashesToFileOpt == 1) {
	const char *filename = PatDbExpand(DExpireOverListPat);

	DExpOverList = fopen(filename, "a");
	if (DExpOverList == NULL)
	    fprintf(stderr, "Error opening %s: %s\n", filename,
						      strerror(errno));
    }

    /*
     * scan all directories in the spool.   Expire history records by
     * directory.  We can't expire history records by file anymore 
     * because 'reader mode' expire may create new files with 'old' gmt
     * times.
     */

    if (VerboseOpt) {
	printf("Scanning history with %d directory entries ....\n", entryIdx);
	fflush(stdout);
    }

    snprintf(spoolHome, sizeof(spoolHome), "%s", PatExpand(SpoolHomePat));
    spoolHomeLen = strlen(spoolHome);

    for (i = 0; i < entryIdx; ++i) {
	int partnameLen;

	if (SpoolEntries[i]->removed)
	    continue;

	partnameLen = strlen(SpoolEntries[i]->partn->partname);

	/* If partname has a "/news/spool/news/" prefix, strip it off */
	if (partnameLen > spoolHomeLen &&
	    strncmp(SpoolEntries[i]->partn->partname, spoolHome,
		    spoolHomeLen) == 0 &&
	    SpoolEntries[i]->partn->partname[spoolHomeLen] == '/'
	    ) {
	    snprintf(path, sizeof(path), "%s%s",
		     SpoolEntries[i]->partn->partname + spoolHomeLen + 1,
		     SpoolEntries[i]->dirname);
	} else {
	    /* No "/news/spool/news/" prefix; go for absolute path */
	    snprintf(path, sizeof(path), "%s%s",
					SpoolEntries[i]->partn->partname,
					SpoolEntries[i]->dirname);
	}

	if (DebugOpt > 1)
	    printf("NODEPATH: %s\n", path);
	(void)findNode(path, 1);
    }

    /*
     * Scan history file and update the expiration
     *
     * The history file was opened before the spool scan to make sure
     * we get the right history file at this point and that we know
     * where the end of the file is before the spool entry hash is built
     */

    {
	int n;
	HistHead hh;
	History hist[65536];

	if ((n = read(HistoryFd, &hh, sizeof(hh))) != sizeof(hh)) {
	    if (n == -1)
		fprintf(stderr, "Unable to read history header from %s (%s)\n",
						HistoryFile, strerror(errno));
	    else 
		fprintf(stderr, "Read %d bytes from history %s, expected %d\n",
						n, HistoryFile, sizeof(hh));

	    exit(1);
	}
	if (hh.hmagic != HMAGIC) {
	    fprintf(stderr, "corrupted history file or old version of history file: %x : %x\n", hh.hmagic, HMAGIC);
	    exit(1);
	}
	if (hh.version > HVERSION) {
	    fprintf(stderr, "wrong dhistory file version (%d), expected %d\n",
		hh.version,
		HVERSION
	    );
	    exit(1);
	}

	lseek(HistoryFd, hh.headSize + sizeof(HistIndex) * hh.hashSize, 0);

	numEnt = (HistoryEnd -
			(hh.headSize + sizeof(HistIndex) * hh.hashSize)) /
					sizeof(History);
	if (VerboseOpt) {
	    printf("History contains %d entries ....\n", numEnt);
	    fflush(stdout);
	}

	npos = lseek(HistoryFd, 0L, 1);
	while ((n = read(HistoryFd, hist, sizeof(hist))) > 0) {
	    int i;
	    int changed = 0;

	    bpos = npos;
	    npos += n;
	    n /= sizeof(History);

	    for (i = 0; i < n; ++i) {
		/*
		 * Don't scan beyond the stored history position
		 */
		if (bpos + i * sizeof(History) >= HistoryEnd)
		    break;

		countEnt++;
		if (VerboseOpt && numEnt > 0 &&
				countEnt * 100 / numEnt >= lastPerc + 10) {
		    lastPerc = countEnt * 100 / numEnt;
		    printf("\t%-10ld of %-10d (%d%%) complete  %llu  %u   \n",
					(long)countEnt, numEnt, lastPerc,
					bpos, countExp);
		    fflush(stdout);
		}

		h = &hist[i];
		path[0] = 0;

		if (TestOpt) {
		    int res;

		    ArticleFileName(path, sizeof(path), h, ARTFILE_DIR_REL);
		    res = findNode(path, 0);

		    if (res == 0) {
			if (H_EXPIRED(h->exp)) {
			    countTestSpool++;
			    printf("%08x.%08x expired @%llu (index=%llu) but on spool (%s)\n",
					h->hv.h1, h->hv.h2,
					npos + i * sizeof(History),
					countEnt, path);
			    findNode(path, 0);
			} else {
			    countTestValid++;
			}
		    } else {
			if (H_EXPIRED(h->exp)) {
			    countTestExpired++;
			} else {
			    countTestHistory++;
			    printf("%08x.%08x not expired @%llu (index=%llu) and not on spool (%s)\n",
					h->hv.h1, h->hv.h2,
					npos + i * sizeof(History),
					countEnt, path);
			}
		    }
		    continue;
		}

		/*
		 * skip if the article has already expired or if it
		 * is a new article that we may not have scanned, or
		 * if it is an expansion slot.
		 */

		if (SinglePart != NULL && *SinglePart != H_SPOOL(h->exp))
		    continue;
		if (!UnexpireOpt && H_EXPIRED(h->exp))
		    continue;
		if (UnexpireOpt && !H_EXPIRED(h->exp))
		    continue;
		if (h->hv.h1 == 0 && h->hv.h2 == 0)
		    continue;

		if (!UnexpireOpt)
		    ArticleFileName(path, sizeof(path), h, ARTFILE_DIR_REL);

		if (UnexpireOpt || findNode(path, 0) < 0) {
		    if (!UnexpireOpt && VerboseOpt > 1) {
			printf("Unable to find path %s (%08x.%08x), %s history record\n",
			    path,
			    h->hv.h1, h->hv.h2,
			    ((HistoryUpdateOpt != 2) ? "expiring" : "would expire")
			);
		    }
		    if (UnexpireOpt || HistoryUpdateOpt != 2) {
			changed = 1;
			if (UnexpireOpt)
			    h->exp &= ~EXPF_EXPIRED;
			else
			    h->exp |= EXPF_EXPIRED;
			lseek(
			    HistoryFd,
			    bpos + sizeof(History) * i + offsetof(History, exp),
			    0
			);
			if (!NotForReal)
			    write(HistoryFd, &h->exp, sizeof(h->exp));

			if (WriteHashesToFileOpt == 1)
			    fwrite(&h->hv, sizeof(hash_t), 1, DExpOverList);
		    }
		    ++countExp;
		}
	    }
	    if (changed)
		lseek(HistoryFd, npos, 0);
	}
    }

    if (WriteHashesToFileOpt == 1)
	fclose(DExpOverList);

    if (TestOpt) {
	printf("Total entries scanned      : %12llu\n", countEnt);
	printf("Total !expired + on spool  : %12llu\n", countTestValid);
	printf("Total expired + ! on spool : %12llu\n", countTestExpired);
	printf("Total expired + on spool   : %12llu\n", countTestSpool);
	printf("Total !expired + ! on spool: %12llu\n", countTestHistory);
    }
    return(countExp);
}

typedef struct ENode {
    struct ENode *no_Next;
    char	*no_Path;
} ENode;

#define EHSIZE	16384
#define EHMASK	(EHSIZE - 1)

ENode	*NodeAry[EHSIZE];

/*
 * Find a path in the node array
 * returns:	0 for found
 *		1 for created
 *		-1 for not found
 */
int
findNode(const char *path, int createMe)
{
    unsigned int hv = 0xA4FC3244;
    int i;
    ENode **pnode;
    ENode *node;
    unsigned int index;

    for (i = 0; path[i]; ++i)
	hv = (hv << 5) ^ path[i] ^ (hv >> 23);

    index = (hv ^ (hv >> 16)) & EHMASK;
    for (pnode = &NodeAry[index]; 
	(node = *pnode) != NULL; 
	pnode = &node->no_Next
    ) {
	if (strcmp(path, node->no_Path) == 0)
	    return(0);
    }
    if (createMe) {
	node = malloc(sizeof(ENode) + strlen(path) + 1);
	if (!node) {
	  fprintf(stderr, "unable to malloc in findNode\n");
	  exit(1);
	}
	node->no_Next = NULL;
	node->no_Path = (char *)(node + 1);
	*pnode = node;
	strcpy(node->no_Path, path);
	return(1);
    }
    return(-1);
}

void
freeNodes(void)
{
    int i;

    for (i = 0; i < EHSIZE; ++i) {
	ENode *node;

	while ((node = NodeAry[i]) != NULL) {
	    NodeAry[i] = node->no_Next;
	    free(node);
	}
    }
}

void
dumpNodeTable(void)
{
    int i;

    for (i = 0; i < EHSIZE; ++i) {
	ENode *node;

	node = NodeAry[i];
	while (node != NULL) {
	    printf("NODEent: %d : %s\n", i, node->no_Path);
	    node = node->no_Next;
	}
    }
}

FileSystem *
findFileSys(char *path)
{
    FileSystem *f;
    FileSystem *pf;
    struct stat sb;

    if (stat(path, &sb) == -1) {
	fprintf(stderr, "Unable to stat %s\n", path);
	exit(1);
    }

    for (f = FileSystems, pf = NULL; f != NULL; pf = f, f = f->next)
	if (sb.st_dev == f->dev)
	    return(f);

    f = (FileSystem *)malloc(sizeof(FileSystem));
    if (pf == NULL)
	FileSystems = f;
    else
	pf->next = f;

    f->dev = sb.st_dev;
    f->fsfree = freeSpaceOn(path, 0, &f->fsfreefiles);
    f->next = 0;
    if (DebugOpt > 1)
	printf("ADD FS: %s\n", path);
    return(f);
}

/*
 * Return a time value (in secs) of a directory name in hex
 * specified as D.xxxxxxxxx, where 'xxxxxxxxx' is (time / 60) in hex
 */
time_t
getAge(char *dirname)
{
    time_t t = 0;

    sscanf(dirname, "D.%x", &t);
    if (t)
	t = t * 60;
    return(t);
}

/*
 * freeSpaceOn() - return the space free (MB) in a directory/partition
 *
 */

double
freeSpaceOn(char *path, int syncit, long *freefiles)
{
    struct statfs stmp;
    double avail;

    if (DebugOpt)
	printf("Check space on %s\n", path);

    /*
     * This code does not significantly slow dexpire down, but it does give
     * the system sync a chance to update the bitmaps so statfs returns a
     * more accurate value.  Certain filesystems such as FreeBSD and BSDI
     * w/ softupdates are so decoupled and so fast that dexpire might remove 
     * 80% of the spool before statfs() realizes that sufficient free space 
     * remains.
     */
    if (SoftUpdates > 0 && syncit) {
	sync();
	sync();
	sleep(SoftUpdates);
	sync();
    }

#if USE_SYSV_STATFS
    if (statfs(path, &stmp, sizeof(stmp), 0) != 0) {
#else
    if (statfs(path, &stmp) != 0) {
#endif
	fprintf(stderr, "dexpire: unable to statfs %s (%s)\n", path,
						strerror(errno));
	return(1);
    }
    avail = stmp.f_bavail * 1.0 * stmp.f_bsize;
    if (freefiles != NULL)
	*freefiles = stmp.f_ffree;
    if (DebugOpt)
	printf("Tested fs avail: %s (%ld inodes)\n", ftos(avail), (long)stmp.f_ffree);
    return(avail);
}

void
DumpSpoolEntries(void)
{
    char path[PATH_MAX];
    int i;
    char spoolHome[PATH_MAX];
    int spoolHomeLen;

    snprintf(spoolHome, sizeof(spoolHome), "%s", PatExpand(SpoolHomePat));
    spoolHomeLen = strlen(spoolHome);
    printf("List of directories on spool:\n");
    for (i = 0; i < entryIdx; ++i) {
	int partnameLen;

	if (SpoolEntries[i]->removed)
	    continue;

	partnameLen = strlen(SpoolEntries[i]->partn->partname);

	/* If partname has a "/news/spool/news/" prefix, strip it off */
	if (partnameLen > spoolHomeLen &&
	    strncmp(SpoolEntries[i]->partn->partname, spoolHome,
		    spoolHomeLen) == 0 &&
	    SpoolEntries[i]->partn->partname[spoolHomeLen] == '/'
	    ) {
	    snprintf(path, sizeof(path), "%s%s",
		     SpoolEntries[i]->partn->partname + spoolHomeLen + 1,
		     SpoolEntries[i]->dirname);
	} else {
	    /* No "/news/spool/news/" prefix; go for absolute path */
	    snprintf(path, sizeof(path), "%s%s",
					SpoolEntries[i]->partn->partname,
					SpoolEntries[i]->dirname);
	}

	printf("path= %s  dirname=%s  partition=%s  removed=%d  found=%d\n",
		path,
                SpoolEntries[i]->dirname,
                SpoolEntries[i]->partn->partname,
                SpoolEntries[i]->removed,
		findNode(path, 0));
    }
}