/*
 * UTIL/DEXPIREOVER.C
 *
 * (c)Copyright 1998, Matthew Dillon, All Rights Reserved.  Refer to
 *    the COPYRIGHT file in the base directory of this distribution
 *    for specific rights granted.
 *
 * 	In this incarnation, dexpireover cleans up overview information as
 *	specified in the 'x' fields in dexpire.ctl (see the sample dexpire.ctl)
 *
 *	This is still very rough.  I still need to add in a free-space-target
 *	option and have dexpireover adjust the expiration dynamically based on
 *	free space.  IT DOESN'T DO THIS YET!! You have to make sure your
 *	expiration ('x' option) in dexpire.ctl is not set too high.
 *
 *	I also need to have a remote-server scanning option to allow 
 *	dexpireover to adjust expirations based on remote server retentions.
 *	It does not do this yet either.
 *
 *	Modifications by Nickolai Zeldovich to allow spool-based expiration
 *	(ExpireBySpool and ExpireFromFile)
 *
 *	Specifying the -pN option will fork N dexpireover processes and
 *	perform the expiration process in parallel. This is useful to speed
 *	up ExpireBySpool where dhistory lookups take a long time.
 *
 *	LOCKING INFO
 *
 *	The data file will have an advisory lock at offset 4 from dreaderd
 *	when it has the file open (can't rewrite data)
 *	The info file will have an advisory lock at offset 4 from dreaderd
 *	when it has the file open (can't resize)
 *
 *	od_HFd = data. file
 *	ov_OFd = over. file
 *
 */

#include <dreaderd/defs.h>

typedef struct Group {
    struct Group *gr_Next;
    int		gr_State;
    int		gr_StartNo;
    int		gr_EndNo;
    int		gr_CTS;
    int		gr_LMTS;
    int		gr_Iter;
    char	*gr_GroupName;
    char	*gr_Flags;
    char	*gr_Hash;
    int		gr_UpdateFlag;
} Group;

#define GRF_DESCRIPTION 0x00000001
#define GRF_STARTNO     0x00000002
#define GRF_ENDNO       0x00000004
#define GRF_FLAGS       0x00000008
#define GRF_FROMLOCAL   0x00000800
#define GRF_NEW         0x00001000
#define GRF_FROMREMOTE  0x00002000
#define GRF_MODIFIED    0x00008000
#define GRF_EDITEDBEG   0x00010000

#define GHSIZE		1024
#define GHMASK		(GHSIZE-1)

typedef struct ReplaceFile {
    struct ReplaceFile	*rf_Next;
    int			rf_Fd;
    int			rf_ArtBase;
} ReplaceFile;

#define DEXPOVER_READ_BUFFER_SIZE	4096
#define DEXPOVER_HASH_SIZE		32768

/* Since we only look at the first char of the first-level directory,
 * we do not support more than 16 forks.
 */
#define	MAX_PAR_COUNT			16

/*
 * These aren't really buckets, they're parts of a bucket
 */

typedef struct bucket_t {
    struct bucket_t *next;
    hash_t hash_item;
    short valid;
} bucket_t;

KPDB  *KDBActive;
Group *GHash[GHSIZE];

void ScanDirectories(void);
void scanDirectory(const char *dirpath, char *dirname, int *level);
void DeleteJunkFile(const char *dirPath, const char *name);
void ProcessOverviewFile(const char *dirPath, const char *name, int type);
int getFirstArtAge(Group *group, int fd, OverHead *oh);
char *allocTmpCopy(const char *buf, int bufLen);
Group *EnterGroup(const char *groupName, int begNo, int endNo, int lmts, int cts, int iter, const char *flags);
Group *FindGroupByHash(char *Hash, int iter);
int SetField(char **pptr, const char *str);
void ExpireByDays(Group *group, int fd, OverHead *oh, int expireSecs);
void ExpireBySpool(Group *group, int fd, OverHead *oh);
void ExpireFromFile(Group *group, int fd, OverHead *oh, int expireSecs);
void RewriteData(Group *group, int fd, OverHead *oh, const char *dirPath);
void rewriteDataFile(Group *group, ReplaceFile **prf, const char *cacheBase, int cacheSize, const OverArt *oa, OverArt *ob, const char *dirPath);
void ResizeGroup(Group *group, int fd, OverHead *oh, int maxArts);
int nearestPower(int n);
void ReadDExpOverList(void);
int expOverListCheckExpired(hash_t *hv);
int hexCharToInt(char c);

int UpdateBegArtNoOpt = 0;
int UpdateCTSOpt = 0;
int RewriteDataOpt = 0;
int BadGroups = 0;
int ResizedGroups = 0;
int NoResizedGroups = 0;
int GoodGroups = 0;
int ActiveUpdated = 0;
int VerboseOpt = -1;
int ResizeOpt = -1;
int ForReal = 1;
int OldGroups = 0;
int UseExpireByDays = 0;
int UseExpireBySpool = 0;
int UseExpireFromFile = 0;
int ParallelCount = 0;
int FactorExpire = 0;
int LockWaitTime= 1;
int MustExit = 0;
char *Wild;
bucket_t *dexpover_msgid_hash;
int ParallelIdx = 0;
int ParallelPid[MAX_PAR_COUNT];

void
sigInt(int sigNo)
{
    printf("Exit signal caught - exiting\n");
    ++MustExit;
    if (MustExit > 3)
	exit(1);
}

/*
 * We use alarms to cancel a lock wait, so just ignore the alarm
 */
void
sigAlarm(int sigNo)
{
 ;;
}

void Usage(char *progname)
{
    fprintf(stderr, "Expire the reader header database\n");
    fprintf(stderr, "dexpireover [-a] [-e] [-f active] [-l#] [-NB] [-n] [-O#] [-o] [-p#] [-R] [-s] [-U] [-v#] [-w wildmat] [-x] [-y] [-C diablo.config] [-d[n]] [-V]\n");
    fprintf(stderr, "\t-a\t\tDo a standard header expire run (-NB -U -s -y)\n");
    fprintf(stderr, "\t-e\t\tExpire by checking a local spool\n");
    fprintf(stderr, "\t-f file\t\tSpecify the name of the active file\n");
    fprintf(stderr, "\t-l#\t\tWait N seconds for lock files\n");
    fprintf(stderr, "\t-NB\t\tUpdate the begining article number (NB) in active\n");
    fprintf(stderr, "\t-n\t\tDon't actually make any changes (dry run)\n");
    fprintf(stderr, "\t-O#\t\tRemove groups not used within # days\n");
    fprintf(stderr, "\t-o\t\tExpire from file of msgid hashes\n");
    fprintf(stderr, "\t-p#\t\tRun # parallel expire runs (speed up)\n");
    fprintf(stderr, "\t-R\t\tRewrite header data files\n");
    fprintf(stderr, "\t-s\t\tResize group indexes\n");
    fprintf(stderr, "\t-U\t\tAdd CTS (group create time) in active if not present\n");
    fprintf(stderr, "\t-v[#]\t\tVerbose mode\n");
    fprintf(stderr, "\t-w wildmat\tSpecify a wildmat of groups to expire\n");
    fprintf(stderr, "\t-x\t\tUse factoring for expire (see man page)\n");
    fprintf(stderr, "\t-y\t\tActually expire headers based on dexpire.ctl\n");
    fprintf(stderr, "\t-C file\tspecify diablo.config to use\n");
    fprintf(stderr, "\t-d[n]\tset debug [with optional level]\n");
    fprintf(stderr, "\t-V\tprint version and exit\n");
    exit(1);
}

int
main(int ac, char **av)
{
    int i;
    char *dbfile = NULL;

    LoadDiabloConfig(ac, av);

    for (i = 1; i < ac; ++i) {
	char *ptr = av[i];
	if (*ptr != '-') {
	    fprintf(stderr, "Unexpected argument: %s\n", ptr);
	    Usage(av[0]);
	}
	ptr += 2;
	switch(ptr[-1]) {
	case 'a':
	    UseExpireByDays = 1;
	    UpdateBegArtNoOpt = 1;
	    UpdateCTSOpt = 1;
	    ResizeOpt = 1;
	    break;
	case 'e':
	    UseExpireBySpool = 1;
	    break;
	case 'f':
	    dbfile = (*ptr) ? ptr : av[++i];
	    break;
	case 'l':
	    LockWaitTime = strtol((*ptr) ? ptr : av[++i], NULL, 0);
	    break;
	case 'N':
	    while (*ptr) {
		switch(*ptr) {
		case 'B':
		    UpdateBegArtNoOpt = 1;
		    break;
		default:
		    break;
		}
		++ptr;
	    }
	    break;
	case 'n':
	    ForReal = 0;
	    if (VerboseOpt < 0)
		VerboseOpt = 1;
	    break;
	case 'O':
	    if (*ptr)
		OldGroups = strtol(ptr, NULL, 0);
	    else
		OldGroups = 30 * 3;	/* 3 months by default */
	    break;
	case 'o':
	    UseExpireFromFile = 1;
	    break;
	case 'p':
	    if (*ptr)
		ParallelCount = strtol(ptr, NULL, 0);
	    else
		ParallelCount = 1;
	    if (ParallelCount > MAX_PAR_COUNT)
		ParallelCount = MAX_PAR_COUNT;
	    /* Note that a parcount of 1 doesn't do anything useful. */
	    break;
	case 'R':
	    RewriteDataOpt = 1;
	    break;
	case 's':
	    ResizeOpt = 1;
	    break;
	case 'U':
	    UpdateCTSOpt = 1;
	    break;
	case 'v':
	    VerboseOpt = (*ptr) ? strtol(ptr, NULL, 0) : 1;
	    break;
	case 'w':
	    Wild = (*ptr) ? ptr : av[++i];
	    break;
	case 'x':
	    FactorExpire = 1;
	    break;
	case 'y':
	    UseExpireByDays = (*ptr) ? strtol(ptr, NULL, 0) : 1;
	    break;
	/* Common options */
	case 'C':           /* parsed by LoadDiabloConfig */
	    if (*ptr == 0)
		++i;
	    break;
	case 'd':
	    DebugOpt = (*ptr) ? strtol(ptr, NULL, 0) : 1;
	    break;
	case 'V':
	    PrintVersion();
	    break;
	default:
	    fprintf(stderr, "Unknown option: %s\n", ptr - 2);
	    Usage(av[0]);
	}
    }

    if (!UseExpireByDays && ResizeOpt <= 0 && !RewriteDataOpt &&
			!UseExpireFromFile && !UseExpireBySpool && !OldGroups)
	Usage(av[0]);
    /*
     * Read in the list of expired msgid hashes, if we are using it
     */

    if (UseExpireFromFile)
	ReadDExpOverList();

    /*
     * fork off parallel copies of dexpireover at this point
     */

    if (ParallelCount) {
	char *stdout_buffer;

	for (ParallelIdx=0; ParallelIdx < ParallelCount; ParallelIdx++) {
	    int pid;

	    pid = fork();
	    if(pid == 0)
		break;
	    ParallelPid[ParallelIdx] = pid;
	}

	stdout_buffer = (char *)malloc(BUFSIZ);
	setvbuf(stdout, stdout_buffer, _IOLBF, BUFSIZ);

	if (ParallelIdx == ParallelCount) {
	    pid_t pid;
	    int remaining = ParallelCount;

	    while (remaining) {
		while (remaining && ((pid = wait3(NULL, 0, NULL)) > 0)) {
		    for (i=0; i<ParallelCount; i++)
			if(ParallelPid[i] == pid) {
			    ParallelPid[i] = 0;
			    --remaining;
			}
		}
	    }

	    printf("Parallelizing dexpireover (%d forks) finished.\n",
	       ParallelCount);
	    exit(0);
	}
    }

    /*
     * Open active file database
     */

    if (VerboseOpt)
	printf("Loading active file\n");
    if (dbfile) {
	KDBActive = KPDBOpen(dbfile, O_RDWR);
    } else {
	KDBActive = KPDBOpen(PatDbExpand(ReaderDActivePat), O_RDWR);
    }
    if (KDBActive == NULL) {
	fprintf(stderr, "Unable to open dactive.kp\n");
	exit(1);
    }
    if (OldGroups && Wild == NULL) {
	fprintf(stderr, "group wildcard must be specified if -O option used\n");
	Usage(av[0]);
    }

    LoadExpireCtl(1);

    /*
     * Open the history file if we are going to expire based on local spool
     */

    if (UseExpireBySpool)
	HistoryOpen(NULL, 0);

    /*
     * scan dactive.kp
     */

    if (VerboseOpt)
	printf("Hashing newsgroups\n");
    {
	int recLen;
	int recOff;
	int cts0 = (int)time(NULL);

	for (recOff = KPDBScanFirst(KDBActive, 0, &recLen);
	     recOff;
	     recOff = KPDBScanNext(KDBActive, recOff, 0, &recLen)
	) {
	    int groupLen;
	    int flagsLen;
	    const char *rec = KPDBReadRecordAt(KDBActive, recOff, 0, NULL);
	    const char *group = KPDBGetField(rec, recLen, NULL, &groupLen, NULL);
	    const char *flags = KPDBGetField(rec, recLen, "S", &flagsLen, "y");
	    int begNo = strtol(KPDBGetField(rec, recLen, "NB", NULL, "-1"), NULL, 10);
	    int endNo = strtol(KPDBGetField(rec, recLen, "NE", NULL, "-1"), NULL, 10);
	    int lmts = (int)strtoul(KPDBGetField(rec, recLen, "LMTS", NULL, "0"), NULL, 16);
	    int cts = (int)strtoul(KPDBGetField(rec, recLen, "CTS", NULL, "0"), NULL, 16);
	    int iter = (int)strtoul(KPDBGetField(rec, recLen, "ITER", NULL, "0"), NULL, 16);
	    Group *grp;

	    if (cts == 0)	/* enter non-zero cts only if group has no CTS field */
		cts = cts0;
	    else
		cts = 0;

	    if (group)
		group = allocTmpCopy(group, groupLen);
	    if (flags)
		flags = allocTmpCopy(flags, flagsLen);

	    /*
	     * ignore bad group or group that does not match the wildcard
	     */

	    if (group == NULL)
		continue;
	    if (Wild && WildCmp(Wild, group) != 0)
		continue;

	    grp = EnterGroup(
		group,
		begNo,
		endNo,
		lmts,
		cts,
		iter,
		flags
	    );
	    grp->gr_State &= ~(GRF_NEW|GRF_MODIFIED);
	}
    }

    rsignal(SIGINT, sigInt);
    rsignal(SIGHUP, sigInt);
    rsignal(SIGTERM, sigInt);
    rsignal(SIGALRM, SIG_IGN);

    /*
     * Scan /news/spool/group/ and do the actual expire, over resize
     * and data rewrite
     *
     */

    ScanDirectories();

    /*
     * Writeback active file
     */

    if (UpdateBegArtNoOpt || OldGroups || UpdateCTSOpt) {
	int t0 = (int)time(NULL);	/* int-sized for LMTS compare */
	int t;
	int i;
	int count = 0;

	t = t0 - OldGroups * (60 * 60 * 24);

	for (i = 0; i < GHSIZE; ++i) {
	    Group *group;

	    for (group = GHash[i]; group; group = group->gr_Next) {
		/*
		 * If we have a new group not previously in the database,
		 * we only add it if SyncGroupsOpt is set.
		 */
		int add = 0;

		if (OldGroups) {
		    if (group->gr_LMTS) {
			/*
			 * Existing LMTS
			 */
			if ((int)(t - group->gr_LMTS) > 0) {
			    if (ForReal)
				KPDBDelete(KDBActive, group->gr_GroupName);
			    if (VerboseOpt)
				printf("%s: stale group deleted\n", group->gr_GroupName);
			    add = 1;
			    group->gr_State &= ~GRF_MODIFIED;	/* prevent NB update */
			} 
		    } else {
			/*
			 * no LMTS in record, add one
			 */
			if (ForReal) {
			    char tsBuf[16];
			    sprintf(tsBuf, "%08x", (int)t0);
			    KPDBWriteEncode(KDBActive, group->gr_GroupName, "LMTS", tsBuf, 0);
			    add = 1;
			}
			if (VerboseOpt)
			    printf("%s: added missing LMTS\n", group->gr_GroupName);
		    }
		}

		if (UpdateCTSOpt) {
		    if (group->gr_CTS) {
			if (ForReal) {
			    char tsBuf[16];
			    sprintf(tsBuf, "%08x", group->gr_CTS);
			    KPDBWriteEncode(KDBActive, group->gr_GroupName, "CTS", tsBuf, 0);
			}
			if (VerboseOpt)
			    printf("%s: added missing CTS\n", group->gr_GroupName);
			add = 1;
		    }
		}

		if (ForReal && (group->gr_State & GRF_MODIFIED)) {
		    if (group->gr_State & GRF_EDITEDBEG) {
			char startBuf[16];
			sprintf(startBuf, "%010d", group->gr_StartNo);
			KPDBWriteEncode(KDBActive, group->gr_GroupName, "NB", startBuf, 0);
			add = 1;
		    }
		}
		count += add;
	    }
	}
	printf("Updated article range in %d groups\n", count);
    }
    if (KDBActive)
	KPDBClose(KDBActive);

    /*
     * Close history if we had it open
     */
    if (UseExpireBySpool)
	HistoryClose();

    return(0);
}

/*
 * Scan /news/spool/group/ and do the actual expire, over resize
 * and data rewrite
 *
 * This works by scanning all files in the directories, mapping the
 * filename (hash) to a newsgroup name and performing the following,
 * depending on file type:
 *
 * over.* :
 * *.o.* :
 *		1) resize over (if requested)
 *		2) expire articles (by days, spool or file)
 *		3) rewrite data (if requested)
 *
 * data.*:
 * *.d.*:
 *		1) if the range of articles for the file falls below NB,
 *		   then delete the file
 *
 * If a newsgroup for a file cannot be found, the file is removed
 */

void
ScanDirectories(void)
{
    DIR *dir;
    int level = 0;

    if (VerboseOpt)
	printf("Scanning group directories\n");

    chdir(PatExpand(GroupHomePat));
    if ((dir = opendir(".")) != NULL) {
	den_t *den;
	struct stat st;

	while ((den = readdir(dir)) != NULL) {
	    if (isalnum((int)den->d_name[0]) &&
	    	stat(den->d_name, &st) == 0 && S_ISDIR(st.st_mode) &&
		/*
		 * We explicitly use the first char, because overview
		 * sizes appear to be not evenly distributed wrt second
		 * char.
		 */
		(ParallelCount ? ((GFIndex(den->d_name[0]) % ParallelCount) ==
				      ParallelIdx) : 1)
	    )
		scanDirectory(PatExpand(GroupHomePat), den->d_name, &level);
	}
	closedir(dir);
    }

    printf("Scanned %d files, %d were bad, %d/%d indexes resized successfully\n",
	GoodGroups + BadGroups, 
	BadGroups, 
	ResizedGroups, 
	NoResizedGroups + ResizedGroups
    );

}

void
scanDirectory(const char *dirpath, char *dirname, int *level)
{
    DIR *dir2;
    char path[PATH_MAX];
    char origpath[PATH_MAX];

    sprintf(path, "%s/%s", dirpath, dirname);

    if (getcwd(origpath, sizeof(origpath)) == NULL) {
	printf("Unable to getcwd(%s): %s\n", dirname, strerror(errno));
	return;
    }
    if (chdir(dirname) != 0) {
	printf("Unable to chdir(%s)\n", dirname);
	return;
    }

    if ((dir2 = opendir(".")) != NULL) {
	den_t *den2;
	char tbuf[24];
	int tint;
	struct stat st;

	while ((den2 = readdir(dir2)) != NULL) {
	    if (strcmp(den2->d_name, ".") == 0 || strcmp(den2->d_name, "..") == 0)
		continue;
	    if (stat(den2->d_name, &st) == 0 && S_ISDIR(st.st_mode)) {
		if (*level <= 2)
		    scanDirectory(path, den2->d_name, level);
	    } else if (strncmp(den2->d_name, "over.", 5) == 0 ||
		strncmp(den2->d_name, "o.", 2) == 0 ||
		strstr(den2->d_name, ".o.") != NULL)
		ProcessOverviewFile(path, den2->d_name, 1);
	    if (MustExit)
		exit(1);
	}
	rewinddir(dir2);
	while ((den2 = readdir(dir2)) != NULL) {
	    /*
	     * delete junk files from previously interrupted
	     * dexpireover -R
	     */
	    if (strncmp(den2->d_name, ".data.", 6) == 0 ||
		strncmp(den2->d_name, ".d.", 3) == 0 ||
		sscanf(den2->d_name, ".%[^.]s.%d.d.", tbuf, &tint) == 2) {
		DeleteJunkFile(path, den2->d_name);
		continue;
	    }

	    /*
	     * process data. files
	     */
	    if (strncmp(den2->d_name, "data.", 5) == 0 ||
		strstr(den2->d_name, ".d.") != NULL)
		ProcessOverviewFile(path, den2->d_name, 2);
	    if (MustExit)
		exit(1);
	}
	closedir(dir2);
    }
    if (chdir(origpath) != 0) {
	printf("FATAL: Unable to return with chdir(%s)\n", dirname);
	exit(1);
    }
}

void
DeleteJunkFile(const char *dirPath, const char *name)
{
    char path[PATH_MAX];

    snprintf(path, sizeof(path), "%s/%s", dirPath, name);
    if (VerboseOpt)
	printf("Removing old temp file: %s\n", path);
    if (ForReal) {
	remove(path);
    }
}

/*
 * ProcessOverviewFile() - process over. and data. files.  All over. files
 *			   are processed first. 
 *
 *	When processing over. files, we may resize the index array (-s)
 *	and/or cleanup the file (-R).
 *
 *	When processing data. files, we typically remove whole files. 
 *	If the -R option was used, however, we rewrite the files.  We can
 *	safely copy/rename-over data. files as long as we are able to
 *	lock the associated over. file.
 */

void
ProcessOverviewFile(const char *dirPath, const char *name, int type)
{
    long artBase = -1;
    Group *group;
    char path[PATH_MAX];
    char Hash[PATH_MAX];
    int iter = 0;

    snprintf(path, sizeof(path), "%s/%s", dirPath, name);

    if (DebugOpt > 0)
	printf("ProcessOverviewFile(%s)\n", path);

    bzero(Hash, sizeof(Hash));

    if (ExtractGroupHashInfo(name, Hash, &artBase, &iter) == HASHGRP_NONE)
	return;

    if (DebugOpt > 2)
	printf("File: %s  type=%d  artBase=%ld  iter=%d  Hash=%s\n",
						name, type, artBase, iter, Hash);

    if ((group = FindGroupByHash(Hash, iter)) == NULL) {
	/*
	 * If we gave a wildcard, we can't remove stale groups because
	 * we do not have a full group list.
	 */

	if (Wild == NULL) {
	    ++BadGroups;
	    printf("Group for over file not found, removing %s\n", path);
	    if (ForReal)
		remove(path);
	}
	return;
    }
    ++GoodGroups;

    if (DebugOpt > 2)
	printf("Maps to group: %s\n", group->gr_GroupName);

    if (type == 1) {
	/*
	 * over. file	(fixed length file)
	 */
	int fd;
	OverHead oh;

	if ((fd = open(path, O_RDWR)) >= 0) {
	    int r;
	    r = (read(fd, &oh, sizeof(oh)) == sizeof(oh));
	    if (oh.oh_Version > OH_VERSION ||
		oh.oh_ByteOrder != OH_BYTEORDER)
		r = 0;
	    if (r && oh.oh_Version > 1 &&
				strcmp(oh.oh_Gname, group->gr_GroupName) != 0)
		r = 0;
	    if (r) {
		int expireSecs;
		OverExpire save;
		int recLen;
		const char *rec = KPDBReadRecord(KDBActive, group->gr_GroupName,
                                                        KP_LOCK, &recLen);

		/*
		 * Refresh the EndNo
		 */
		group->gr_EndNo = strtol(KPDBGetField(rec, recLen, "NE",
							NULL, "-1"), NULL, 10);
		KPDBUnlock(KDBActive, rec);

		if (group->gr_EndNo - group->gr_StartNo >= oh.oh_MaxArts) {
		    if (VerboseOpt)
			printf("adjust %s startno %d to %d (endno=%d  maxarts=%d)\n",
					group->gr_GroupName,
					group->gr_StartNo,
					group->gr_EndNo - oh.oh_MaxArts + 1,
					group->gr_EndNo,
					oh.oh_MaxArts);
		    group->gr_StartNo = group->gr_EndNo - oh.oh_MaxArts + 1;
		    group->gr_State |= GRF_EDITEDBEG | GRF_MODIFIED;
		}

		/*
		 * Figure out expire values
		 */
		expireSecs = GetOverExpire(group->gr_GroupName, &save);

		/*
		 * Force regeneration of over. file if RewriteDataOpt, else
		 * only regenerate if ResizeOpt and a size differential.  Valid
		 * overview index sizes run in steps if the nearest higher
		 * power of 2 divided by 3.
		 *
		 */
		if (ResizeOpt > 0 || RewriteDataOpt > 0) {
		    int numArts = group->gr_EndNo - group->gr_StartNo + 1;
		    int maxArts = oh.oh_MaxArts;
		    int artAge;

		    /*
		     * Find out the age of the first article in the
		     * group and calculate an age factor based on this
		     * and the expire value in dexpire.ctl.
		     *
		     * We don't let the age factor get too large
		     * if the age of the first articles is less than an
		     * hour because this can lead to some bloated files.
		     * We also adjust the age factor if there are a
		     * large number of articles in the group and the
		     * age factor is low as we don't need a huge
		     * number of spare slots.
		     */
		    artAge = getFirstArtAge(group, fd, &oh);
		    if (FactorExpire && artAge > 0 && expireSecs > 0) {
			float ageFactor;

			if (artAge <= 0)
			    artAge = 1;
			ageFactor = (expireSecs * 1.0) / (artAge * 1.0);
			if (ageFactor < 1.5)
			    ageFactor = 1.5;
			if (numArts > 20000 && ageFactor == 1.5)
			    ageFactor = 1.3;
			if (artAge < 3600 && ageFactor > 50.0)
			    ageFactor = 50.0;
			if (numArts > 500 && ageFactor > 20.0)
			    ageFactor = 20.0;
			if (numArts > 5000 && ageFactor > 5.0)
			    ageFactor = 5.0;
			if (ageFactor > 400.0)
			    ageFactor = 400.0;
			numArts = ageFactor * numArts;
		    } else if (numArts < maxArts / 2)
			numArts = maxArts - nearestPower(maxArts) / 3;
		    else if (numArts > maxArts * 2 / 3)
			numArts = maxArts + nearestPower(maxArts) / 3;
		    else
			numArts = maxArts;

		    /*
		     * The minimum is somewhat contrived, but if we can
		     * fit the index file into a fragment for unused groups
		     * we save a considerable amount of space.
		     */

		    if (save.oe_MaxArts > 0 && numArts > save.oe_MaxArts)
			numArts = save.oe_MaxArts;
		    if (save.oe_MinArts >= 0 && numArts < save.oe_MinArts)
			numArts = save.oe_MinArts;

		    {
			int needResize = RewriteDataOpt;
			int d = abs(maxArts - numArts);
			/*
			 * We only rewrite if there has been a radical
			 * change in the number of articles needed.
			 */
			if (numArts < 100 && d > 10)
				needResize = 1;
			if (numArts >= 100 && numArts < 1000 && d > 20)
				needResize = 1;
			if (numArts >= 1000 && d > 200)
				needResize = 1;
			if (needResize)
			    ResizeGroup(group, fd, &oh, numArts);
		    }
		}

		if (UseExpireBySpool)
		    ExpireBySpool(group, fd, &oh);
		else if (UseExpireFromFile)
		    ExpireFromFile(group, fd, &oh, expireSecs);
		else if (UseExpireByDays)
		    ExpireByDays(group, fd, &oh, expireSecs);

		/*
		 * Rewrite data files associated with over. file if -R.
		 */
		if (RewriteDataOpt > 0)
		    RewriteData(group, fd, &oh, dirPath);

	    } else {
		printf("group %s, file \"%s\" bad file header\n",
		    group->gr_GroupName,
		    path
		);
		if (oh.oh_Version > OH_VERSION)
		    printf("   expected version %d, got version %d\n",
						OH_VERSION, oh.oh_Version);
	    }
	    close(fd);
	}
    } else {
	/*
	 * data. file, modulo OD_HARTS.  OD_HARTS constant in second
	 * part of conditional is a fudge to make 100% sure we do not
	 * delete a brand new data file.
	 */
	int recLen;
	const char *rec = KPDBReadRecord(KDBActive, group->gr_GroupName,
                                                        KP_LOCK, &recLen);
	/*
	 * Refresh the EndNo
	 */
	group->gr_EndNo = strtol(KPDBGetField(rec, recLen, "NE",
							NULL, "-1"), NULL, 10);
	KPDBUnlock(KDBActive, rec);

	if (artBase + OD_HARTS <= group->gr_StartNo ||
	    artBase >= group->gr_EndNo + OD_HARTS
	) {
	    if (VerboseOpt)
		printf("Deleting stale overview data for %s: %s (artBase=%ld  StartNo=%d  EndNo=%d\n",
				group->gr_GroupName, path,
				artBase, group->gr_StartNo, group->gr_EndNo
		);
	    if (ForReal)
		remove(path);
	} 
    }
}

int
getFirstArtAge(Group *group, int fd, OverHead *oh)
{
    const OverArt *oaBase;
    struct stat st;
    int n;
    time_t t = time(NULL);
    int dt = 0;

    if (fstat(fd, &st) != 0)
	return(-1);

    /*
     * Calculate number of overview records
     */

    n = (st.st_size - oh->oh_HeadSize) / sizeof(OverArt);

    oaBase = xmap(NULL, n * sizeof(OverArt), PROT_READ, MAP_SHARED, fd,
			oh->oh_HeadSize);
    if (oaBase == NULL) {
	fprintf(stderr, "Unable to xmap over.* file for group %s\n",
			group->gr_GroupName);
	return(-1);
    }

    {
        int i = (group->gr_StartNo & 0x7FFFFFFF) % oh->oh_MaxArts;
	const OverArt *oa = &oaBase[i];

	/*
	 * Get the age of the first valid article
	 */
	if (oa && i <= n)
	    dt = (int)(t - oa->oa_TimeRcvd);
    }
    xunmap((void *)oaBase, n * sizeof(OverArt));
    return(dt);
}

char *
allocTmpCopy(const char *buf, int bufLen)
{
    static char *SaveAry[8];
    static int SaveCnt;
    char **pptr;

    SaveCnt = (SaveCnt + 1) % arysize(SaveAry);
    pptr = &SaveAry[SaveCnt];
    if (*pptr)
	free(*pptr);
    *pptr = malloc(bufLen + 1);
    memcpy(*pptr, buf, bufLen);
    (*pptr)[bufLen] = 0;
    return(*pptr);
}

Group *
EnterGroup(const char *groupName, int begNo, int endNo, int lmts, int cts, int iter, const char *flags)
{
    hash_t hv = hhash(GFHash(groupName, &DOpts.ReaderGroupHashMethod));
    Group **pgroup = &GHash[hv.h1 & GHMASK];
    Group *group;

    while ((group = *pgroup) != NULL) {
	if (strcmp(groupName, group->gr_GroupName) == 0)
	    break;
	pgroup = &group->gr_Next;
    }
    if (group == NULL) {
	*pgroup = group = calloc(sizeof(Group) + strlen(groupName) + 1, 1);
	group->gr_State = GRF_NEW;
	group->gr_GroupName = (char *)(group + 1);
	group->gr_Hash = strdup(GFHash(groupName, &DOpts.ReaderGroupHashMethod));
	group->gr_Iter = iter;
	strcpy(group->gr_GroupName, groupName);
    }

    /*
     * update fields
     */
    if (begNo >= 0) {
	group->gr_State |= GRF_STARTNO;
	if (group->gr_StartNo != begNo) {
	    group->gr_State |= GRF_MODIFIED;
	    group->gr_StartNo = begNo;
	}
    }
    if (endNo >= 0) {
	group->gr_State |= GRF_ENDNO;
	if (endNo < group->gr_EndNo) {
	    printf("*** Would be adjusting NE down - not doing it ***\n");
	    printf("adjust %s NE from %d to %d\n", group->gr_GroupName,
				endNo, group->gr_EndNo);
	} else if (group->gr_EndNo != endNo) {
	    group->gr_EndNo = endNo;
	    group->gr_State |= GRF_MODIFIED;
	}
    }
    group->gr_LMTS = lmts;

    if (cts) {
	group->gr_CTS = cts;
    }

    if (flags) {
	group->gr_State |= GRF_FLAGS;
	if (SetField(&group->gr_Flags, flags))
	    group->gr_State |= GRF_MODIFIED;
    }
    return(group);
}

Group *
FindGroupByHash(char *Hash, int iter)
{
    Group *group;
    hash_t hv = hhash(Hash);

    for (group = GHash[hv.h1 & GHMASK]; group; group = group->gr_Next) {
	if (strcmp(group->gr_Hash, Hash) == 0 && group->gr_Iter == iter)
	    break;
    }
    return(group);
}

int
SetField(char **pptr, const char *str)
{
    if (*pptr && strcmp(*pptr, str) == 0)
	return(0);
    if (*pptr)
	free(*pptr);
    *pptr = strcpy(malloc(strlen(str) + 1), str);
    return(1);
}

/*
 * Scan overview records from beginning article to ending article
 */

void
ExpireByDays(Group *group, int fd, OverHead *oh, int expireSecs)
{
    const OverArt *oaBase;
    struct stat st;
    int count = 0;
    int jumped = 0;
    int expired = 0;
    int canceled = 0;
    int stale = 0;
    int n;
    time_t t = time(NULL);

    if (fstat(fd, &st) != 0)
	return;

    /*
     * Calculate number of overview records
     */

    n = (st.st_size - oh->oh_HeadSize) / sizeof(OverArt);

    oaBase = xmap(NULL, n * sizeof(OverArt), PROT_READ, MAP_SHARED, fd, oh->oh_HeadSize);
    if (oaBase == NULL) {
	fprintf(stderr, "Unable to xmap over.* file for group %s\n", group->gr_GroupName);
	return;
    }

    /*
     * Delete expired overview
     */

    {
	int i;

	for (i = 0; i < n; ++i) {
	    const OverArt *oa = &oaBase[i];

	    if (oa && oa->oa_ArtNo > 0) {
		int dt = (int)(t - oa->oa_TimeRcvd);

		if (VerboseOpt > 2)
		    printf("DT %d/%d %08lx %08lx\n", dt, expireSecs, (long)t, (long)oa->oa_TimeRcvd);

		if (expireSecs > 0 && 
		    (dt > expireSecs || dt < -(60 * 60 * 24))
		) {
		    OverArt copy = *oa;

		    copy.oa_ArtNo = -2;		/* EXPIRED */
		    if (ForReal) {
			lseek(fd, oh->oh_HeadSize + i * sizeof(OverArt), 0);
			write(fd, &copy, sizeof(OverArt));
		    }
		    ++count;
		}
	    }
	}
    }

    {
	/*
	 * Update active file begin sequence number
	 */
	while (group->gr_StartNo <= group->gr_EndNo) {
	    int i = (group->gr_StartNo & 0x7FFFFFFF) % n;
	    const OverArt *oa = &oaBase[i];

	    if (VerboseOpt > 2)
		printf("test %d vs %d (i = %d)\n", oa->oa_ArtNo, group->gr_StartNo, i);
	    if (oa->oa_ArtNo == group->gr_StartNo)
		break;
	    ++group->gr_StartNo;
	    switch(oa->oa_ArtNo) {
	    case -2:
		++expired;
		break;
	    case -1:
		++canceled;
		break;
	    default:
		++stale;
		break;
	    }
	    ++jumped;
	}
	if (jumped)
	    group->gr_State |= GRF_EDITEDBEG | GRF_MODIFIED;
    }
    if (VerboseOpt && (jumped || count)) {
	printf("expired %-4d NB += %-4d (%3d can, %3d stale, %3d exprd) left %-4d expires in %6.2f days, grp=%s\n", 
	    count,
	    jumped,
	    canceled, stale, expired,
	    group->gr_EndNo - group->gr_StartNo + 1,
	    ((expireSecs>0) ? (double)expireSecs / (60.0 * 60.0 * 24.0) :-1.0),
	    group->gr_GroupName
	);
    }
    xunmap((void *)oaBase, n * sizeof(OverArt));
}

/*
 * Scan overview records from beginning article to ending article
 *
 * Expire by checking the history file for the expired bit
 */

void
ExpireBySpool(Group *group, int fd, OverHead *oh)
{
    const OverArt *oaBase;
    struct stat st;
    int count = 0;
    int jumped = 0;
    int expired = 0;
    int canceled = 0;
    int stale = 0;
    int n;

    if (fstat(fd, &st) != 0)
	return;

    /*
     * Calculate number of overview records
     */

    n = (st.st_size - oh->oh_HeadSize) / sizeof(OverArt);

    oaBase = xmap(NULL, n * sizeof(OverArt), PROT_READ, MAP_SHARED, fd, oh->oh_HeadSize);
    if (oaBase == NULL) {
	fprintf(stderr, "Unable to xmap over.* file for group %s\n", group->gr_GroupName);
	return;
    }

    /*
     * Delete expired overview
     */

    {
	int i;

	for (i = 0; i < n; ++i) {
	    const OverArt *oa = &oaBase[i];

	    if (oa && oa->oa_ArtNo > 0) {
		hash_t dh = oa->oa_MsgHash;
		History dh_lookup;

		/*
		 * Make sure that the history entry exists. It's possible
		 * that dexpire already removed the article, and dhistory
		 * was cleaned, so the msgID doesn't exist.
		 *
		 * If the article does not exist or is expired, then expire
		 * its overview entry as well.
		 */
		if ((HistoryLookupByHash(dh, &dh_lookup) == -1) ||
						H_EXPIRED(dh_lookup.exp)) {
		    OverArt copy = *oa;

		    copy.oa_ArtNo = -2;		/* EXPIRED */
		    if (ForReal) {
			lseek(fd, oh->oh_HeadSize + i * sizeof(OverArt), 0);
			write(fd, &copy, sizeof(OverArt));
		    }
		    ++count;
		}
	    }
	}
    }

    {
	/*
	 * Update history file begin sequence number
	 */
	while (group->gr_StartNo <= group->gr_EndNo) {
	    int i = (group->gr_StartNo & 0x7FFFFFFF) % n;
	    const OverArt *oa = &oaBase[i];

	    if (VerboseOpt > 2)
		printf("test %d vs %d (i = %d)\n", oa->oa_ArtNo, group->gr_StartNo, i);
	    if (oa->oa_ArtNo == group->gr_StartNo)
		break;
	    ++group->gr_StartNo;
	    switch(oa->oa_ArtNo) {
	    case -2:
		++expired;
		break;
	    case -1:
		++canceled;
		break;
	    default:
		++stale;
		break;
	    }
	    ++jumped;
	}
	if (jumped)
	    group->gr_State |= GRF_EDITEDBEG | GRF_MODIFIED;
    }
    if (VerboseOpt && (jumped || count)) {
	printf("expired %-4d NB += %-4d (%3d can, %3d stale, %3d exprd) left %-4d expires by spool, grp=%s\n", 
	    count,
	    jumped,
	    canceled, stale, expired,
	    group->gr_EndNo - group->gr_StartNo + 1,
	    group->gr_GroupName
	);
    }
    xunmap((void *)oaBase, n * sizeof(OverArt));
}

/*
 * Similar to ExpireBySpool but uses a file generated by dexpire as a list
 * of msgid hashes of messages which are expired.
 * 
 * The 'x' parameter in dexpire.ctl is also checked, to punt articles which
 * have been laying in overview for a long time and somehow escaped being
 * written to the dexpover.dat file.
 */

void
ExpireFromFile(Group *group, int fd, OverHead *oh, int expireSecs)
{
    const OverArt *oaBase;
    struct stat st;
    int count = 0;
    int jumped = 0;
    int expired = 0;
    int canceled = 0;
    int stale = 0;
    int n;
    time_t t = time(NULL);

    if (fstat(fd, &st) != 0)
	return;

    /*
     * Calculate number of overview records
     */

    n = (st.st_size - oh->oh_HeadSize) / sizeof(OverArt);

    oaBase = xmap(NULL, n * sizeof(OverArt), PROT_READ, MAP_SHARED, fd, oh->oh_HeadSize);
    if (oaBase == NULL) {
	fprintf(stderr, "Unable to xmap over.* file for group %s\n", group->gr_GroupName);
	return;
    }

    /*
     * Delete expired overview
     */

    {
	int i;

	for (i = 0; i < n; ++i) {
	    const OverArt *oa = &oaBase[i];

	    if (oa && oa->oa_ArtNo > 0) {
		int dt = (int)(t - oa->oa_TimeRcvd);
		hash_t *hv = (hash_t *)(&(oa->oa_MsgHash));

		if (VerboseOpt > 2)
		    printf("DT %d/%d %08lx %08lx\n", dt, expireSecs, (long)t, (long)oa->oa_TimeRcvd);

		if ((expOverListCheckExpired(hv) == 0) ||
		    ((expireSecs > 0) && 
		     (dt > expireSecs || dt < -(60 * 60 * 24)))
		) {
		    OverArt copy = *oa;

		    copy.oa_ArtNo = -2;		/* EXPIRED */
		    if (ForReal) {
			lseek(fd, oh->oh_HeadSize + i * sizeof(OverArt), 0);
			write(fd, &copy, sizeof(OverArt));
		    }
		    ++count;
		}
	    }
	}
    }

    {
	/*
	 * Update history file begin sequence number
	 */
	while (group->gr_StartNo <= group->gr_EndNo) {
	    int i = (group->gr_StartNo & 0x7FFFFFFF) % n;
	    const OverArt *oa = &oaBase[i];

	    if (VerboseOpt > 2)
		printf("test %d vs %d (i = %d)\n", oa->oa_ArtNo, group->gr_StartNo, i);
	    if (oa->oa_ArtNo == group->gr_StartNo)
		break;
	    ++group->gr_StartNo;
	    switch(oa->oa_ArtNo) {
	    case -2:
		++expired;
		break;
	    case -1:
		++canceled;
		break;
	    default:
		++stale;
		break;
	    }
	    ++jumped;
	}
	if (jumped)
	    group->gr_State |= GRF_EDITEDBEG | GRF_MODIFIED;
    }
    if (VerboseOpt && (jumped || count)) {
	printf("expired %-4d NB += %-4d (%3d can, %3d stale, %3d exprd) left %-4d expires in %6.2f days, grp=%s\n", 
	    count,
	    jumped,
	    canceled, stale, expired,
	    group->gr_EndNo - group->gr_StartNo + 1,
	    ((expireSecs>0) ? (double)expireSecs / (60.0 * 60.0 * 24.0) :-1.0),
	    group->gr_GroupName
	);
    }
    xunmap((void *)oaBase, n * sizeof(OverArt));
}

/*
 * Rewrite the data.* files associated with an overview file
 */


void
RewriteData(Group *group, int fd, OverHead *oh, const char *dirPath)
{
    const OverArt *oaBase;
    OverArt *obBase;
    ReplaceFile   *rfBase = NULL;
    struct stat st;
    int i;
    int n;
    int ok = 1;

    /*
     * If not for real, do not do anything
     */

    if (ForReal == 0)
	return;

    /*
     * We need an exclusive lock for this to work, and if the
     * number of links has fallen to zero we are in a race with
     * another expireover.
     */

    nrsignal(SIGALRM, sigAlarm);
    alarm(LockWaitTime);
    if (hflock(fd, 4, XLOCK_EX) < 0) {
	nrsignal(SIGALRM, SIG_IGN);
	if (VerboseOpt > 2)
	    printf("unable to rewrite data for grp=%s - file in use\n",
							group->gr_GroupName);
	return;
    }
    nrsignal(SIGALRM, SIG_IGN);


    if (fstat(fd, &st) != 0 || st.st_nlink == 0) {
	hflock(fd, 4, XLOCK_UN);
	return;
    }

    /*
     * Calculate number of overview records
     */

    n = (st.st_size - oh->oh_HeadSize) / sizeof(OverArt);

    oaBase = xmap(NULL, n * sizeof(OverArt), PROT_READ, MAP_SHARED, fd, oh->oh_HeadSize);
    if (oaBase == NULL) {
	fprintf(stderr, "Unable to xmap over.* file for group %s\n", group->gr_GroupName);
	hflock(fd, 4, XLOCK_UN);
	return;
    }

    obBase = calloc(n, sizeof(OverArt));

    /*
     * Scan entire overview index, building replacement data.* files on the
     * fly.  We have already cleaned it up, so there should not be any
     * garbarge in the index.
     */

    {
	const char *cacheBase = NULL;
	int cacheSize = 0;
	int cacheArtBase = -1;

	for (i = 0; i < n; ++i) {
	    const OverArt *oa = &oaBase[i];

	    if (oa->oa_ArtNo <= 0)
		continue;

	    if (
		cacheArtBase == -1 ||
		oa->oa_ArtNo < cacheArtBase ||
		oa->oa_ArtNo >= cacheArtBase + OD_HARTS
	    ) {
		char path[PATH_MAX];
		struct stat st;
		int cfd;

		if (cacheBase != NULL) {
		    xunmap((void *)cacheBase, cacheSize);
		    cacheSize = 0;
		    cacheBase = NULL;
		}
		cacheArtBase = oa->oa_ArtNo & ~OD_HMASK;
		snprintf(path, sizeof(path), "%s/%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_DATA, cacheArtBase, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
		if (DebugOpt > 1)
		    printf("Opening: %s\n", path);
		if ((cfd = open(path, O_RDONLY)) >= 0 &&
		    fstat(cfd, &st) == 0
		) {
		    cacheBase = xmap(NULL, st.st_size, PROT_READ, MAP_SHARED, cfd, 0);
		    cacheSize = st.st_size;
		}
		if (cfd >= 0)
		    close(cfd);
	    }
	    rewriteDataFile(group, &rfBase, cacheBase, cacheSize, oa,
							&obBase[i], dirPath);
	}
	if (cacheBase != NULL) {
	    xunmap((void *)cacheBase, cacheSize);
	    cacheSize = 0;
	    cacheBase = NULL;
	}
    }

    /*
     * Remove original data.* files so we can replace the over. file
     * safely.  If we are broken after this point, portions or all of the
     * overview information relating to this group will be lost.
     */
    {
	ReplaceFile *rf = rfBase;

	while (rf) {
	    char path1[PATH_MAX];
	    snprintf(path1, sizeof(path1), "%s/%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_DATA, rf->rf_ArtBase, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
	    if (DebugOpt > 1)
		printf("Remove: %s\n", path1);
	    if (ForReal)
		remove(path1);
	    rf = rf->rf_Next;
	}
    }

    /*
     * Replace the over. file.  If an error occurs we pretty much have to
     * blow the file away because we already deleted the data. files.
     */

    {
	char path1[PATH_MAX];
	char path2[PATH_MAX];
	int ovFd;

	snprintf(path1, sizeof(path1), "%s/.%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_OVER, 0, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
	snprintf(path2, sizeof(path2), "%s/%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_OVER, 0, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));

	/* Make sure we are using the latest versions */
	if (oh->oh_Version != OH_VERSION) {
	    printf("Upgrading overinfo header version %d->%d in %s\n",
						oh->oh_Version, OH_VERSION,
						group->gr_GroupName);
	    oh->oh_Version = OH_VERSION;
	    strncpy(oh->oh_Gname, group->gr_GroupName,
						sizeof(oh->oh_Gname) - 1);
	    oh->oh_Gname[sizeof(oh->oh_Gname) - 1] = '\0';
	    oh->oh_HeadSize = sizeof(OverHead);
	}
	ovFd = open(path1, O_RDWR|O_CREAT|O_TRUNC, 0644);
	if (ovFd >= 0) {
	    if (write(ovFd, oh, oh->oh_HeadSize) != oh->oh_HeadSize)
		ok = 0;
	    if (write(ovFd, obBase, n*sizeof(OverArt)) != n*sizeof(OverArt))
		ok = 0;
	    close(ovFd);
	    if (ok) {
		if (DebugOpt > 1)
		    printf("Renaming over: %s -> %s\n", path1, path2);
		if (ForReal && rename(path1, path2) < 0) {
		    printf("Error renaming %s -> %s (%s)\n", path1, path2,
							strerror(errno));
		    ok = 0;
		}
	    }
	} else {
	    ok = 0;
	}
	if (ok == 0) {
	    printf("Rewrite of %s over. file failed, removing\n",
							group->gr_GroupName);
	    if (ForReal)
		remove(path1);
	}
    }

    /*
     * Rename the temporary .data files to the real ones.
     */
    i = 0;
    while (rfBase) {
	char path1[PATH_MAX];
	char path2[PATH_MAX];
	ReplaceFile *rf = rfBase;

	snprintf(path1, sizeof(path1), "%s/.%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_DATA, rf->rf_ArtBase, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
	snprintf(path2, sizeof(path2), "%s/%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_DATA, rf->rf_ArtBase, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
	if (DebugOpt > 1)
	    printf("Renaming data: %s -> %s\n", path1, path2);
	if (ForReal && rename(path1, path2) < 0) {
	    printf("Error renaming %s -> %s (%s)\n", path1, path2,
							strerror(errno));
	    remove(path1);
	}

	if (rf->rf_Fd >= 0)
	    close(rf->rf_Fd);

	rfBase = rf->rf_Next;
	free(rf);
	i++;
    }
    if (VerboseOpt)
	printf("Rebuilt %d data file(s) for %s\n", i, group->gr_GroupName);

    /*
     * cleanup
     */
    hflock(fd, 4, XLOCK_UN);
    xunmap((void *)oaBase, n * sizeof(OverArt));
    free(obBase);
    return;
}

/*
 * rewriteDataFile() - rewrite the file from cacheFd/oa-params to
 *		       ob, maintaining the ReplaceFile list.
 */

void
rewriteDataFile(Group *group, ReplaceFile **prf, const char *cacheBase, int cacheSize, const OverArt *oa, OverArt *ob, const char *dirPath)
{
    int artBase = oa->oa_ArtNo & ~OD_HMASK;
    ReplaceFile *rf;

    if (VerboseOpt > 1)
	printf("Rewriting: %s:%d\n", group->gr_GroupName, artBase);

    /*
     * Locate the artBase file in our rewrite 'cache'.  Note: it's possible to
     * negatively cache an ArtBase, where rf_Fd will be < 0.
     */

    while ((rf = *prf) != NULL) {
	if (rf->rf_ArtBase == artBase)
	    break;
	prf = &rf->rf_Next;
    }
    if (rf == NULL) {
	char path[PATH_MAX];
	rf = calloc(1, sizeof(ReplaceFile));

	snprintf(path, sizeof(path), "%s/.%s", dirPath,
					GFName(group->gr_GroupName,
					GRPFTYPE_DATA, artBase, 0,
					group->gr_Iter,
					&DOpts.ReaderGroupHashMethod));
	if (DebugOpt > 1)
            printf("Creating: %s\n", path);
	rf->rf_Fd = open(path, O_RDWR|O_CREAT|O_TRUNC, 0644);
	if (rf->rf_Fd == -1)
	    printf("Error opening %s: %s\n", path, strerror(errno));
	rf->rf_ArtBase = artBase;
	*prf = rf;
    }

    /*
     * copy data from cacheFd/oa-params to rf->rf_Fd and fill in ob.  It is
     * possible for cacheBase to be NULL if there is no valid data source
     * for the file.
     */
    if (
	cacheBase != NULL && 
	rf->rf_Fd >= 0 && 
	oa->oa_SeekPos >= 0 &&
	oa->oa_Bytes > 0 &&
	oa->oa_SeekPos + oa->oa_Bytes < cacheSize &&
	cacheBase[oa->oa_SeekPos + oa->oa_Bytes] == 0 && 	  /* guard */
	(oa->oa_SeekPos == 0 || cacheBase[oa->oa_SeekPos-1] == 0) /* guard */

    ) {
	*ob = *oa;
	ob->oa_SeekPos = lseek(rf->rf_Fd, 0L, 1);
	if (write(rf->rf_Fd, cacheBase + oa->oa_SeekPos, oa->oa_Bytes + 1) != oa->oa_Bytes + 1) {
	    lseek(rf->rf_Fd, ob->oa_SeekPos, 0);
	    ftruncate(rf->rf_Fd, ob->oa_SeekPos);
	    ob->oa_ArtNo = -2;
	    ob->oa_SeekPos = 0;
	    ob->oa_Bytes = 0;
	    printf("copy failed %s:%d, write error\n", group->gr_GroupName, oa->oa_ArtNo);
	}
    } else if (oa->oa_SeekPos == -1) {
	; /* do nothing */
    } else {
	printf("copy failed %s:%d, %s\n",
	    group->gr_GroupName,
	    oa->oa_ArtNo,
	    ((cacheBase == NULL) ? "source-missing" :
	    (rf->rf_Fd < 0) ? "dest-failure" :
	    (oa->oa_Bytes <= 0) ? "source-bounds1" :
	    (oa->oa_SeekPos + oa->oa_Bytes >= cacheSize) ? "source-bounds2" :
	    "source-corrupt")
	);
    }
}

/*
 * Resize a newsgroup's over.* index file, if possible.  If called via the -s
 * option, only groups that need resizing are rebuilt.  If called via the -R
 * option, the group is always rebuild AND the associated data files are 
 * rebuilt.
 */

void
ResizeGroup(Group *group, int fd, OverHead *oh, int newSize)
{
    int oldSize = oh->oh_MaxArts;
    struct stat st;

    nrsignal(SIGALRM, sigAlarm);
    alarm(LockWaitTime);
    if (hflock(fd, 4, XLOCK_EX) < 0) {
	nrsignal(SIGALRM, SIG_IGN);
	++NoResizedGroups;
	if (VerboseOpt)
	    printf("resize maxArts from %d to %d failed, file in use grp=%s\n",
					oldSize, newSize, group->gr_GroupName);
	return;
    }
    nrsignal(SIGALRM, SIG_IGN);

    if (fstat(fd, &st) < 0 || st.st_nlink == 0) {
	hflock(fd, 4, XLOCK_UN);
	++NoResizedGroups;
	if (VerboseOpt) {
	    printf("resize maxArts from %d to %d failed, file in use grp=%s\n",
					oldSize, newSize, group->gr_GroupName);
	}
	return;
    }

    /*
     * Resize a group.  We 'own' the overview index file (because other 
     * processes must get a shared lock on offset 4 and we got the exclusive
     * lock).  We can do anything we want with it, but we must rewrite the
     * file in-place to maintain lock consistancy.
     *
     * Resize the group by copying the existing data into an array, validating
     * it based on known information, then putting it back.
     */

    {
	OverArt *oa = calloc(oldSize, sizeof(OverArt));
	OverArt *ob = calloc(newSize, sizeof(OverArt));
	int n;
	int i;
	int recLen;
	const char *rec = KPDBReadRecord(KDBActive, group->gr_GroupName,
							KP_LOCK, &recLen);

	lseek(fd, oh->oh_HeadSize, 0);
	n = read(fd, oa, oldSize * sizeof(OverArt)) / sizeof(OverArt);

	/*
	 * We need to get the NE field again now that we have locked the
	 * over.* file because it could have been adjusted by dreaderd
	 * since we originally got it.
	 */
	group->gr_EndNo = strtol(KPDBGetField(rec, recLen, "NE", NULL, "-1"),
								NULL, 10);
	KPDBUnlock(KDBActive, rec);

	if (group->gr_EndNo - group->gr_StartNo >= newSize) {
	    if (VerboseOpt)
		printf("adjust %s startno %d to %d (endno=%d  newsize=%d)\n",
				group->gr_GroupName,
				group->gr_StartNo,
				group->gr_EndNo - newSize + 1,
				group->gr_EndNo,
				newSize);
	    group->gr_StartNo = group->gr_EndNo - newSize + 1;
	    group->gr_State |= GRF_EDITEDBEG | GRF_MODIFIED;
	}

	/*
	 * Run through and copy validated entries
	 */

	for (i = group->gr_StartNo; i <= group->gr_EndNo; ++i) {
	    OverArt *op = &oa[(i & 0x7FFFFFFF) % oh->oh_MaxArts];

	    if (op->oa_ArtNo == i)
		ob[(i & 0x7FFFFFFF) % newSize] = *op;
	}

	/*
	 * Rewrite.  Only if ForReal.  Note that oh_MaxArts isn't updated 
	 * (for use in other parts of dexpireover) if not for real.
	 */

	if (ForReal) {
	    /* Take this opportunity to set the correct version */
	    if (oh->oh_Version != OH_VERSION) {
		printf("Upgrading overinfo header version %d->%d\n",
						oh->oh_Version, OH_VERSION);
		oh->oh_Version = OH_VERSION;
		strncpy(oh->oh_Gname, group->gr_GroupName,
						sizeof(oh->oh_Gname) - 1);
		oh->oh_Gname[sizeof(oh->oh_Gname) - 1] = '\0';
		oh->oh_HeadSize = sizeof(OverHead);
	    }
	    lseek(fd, 0L, 0);
	    ftruncate(fd, oh->oh_HeadSize + newSize * sizeof(OverArt));
	    oh->oh_MaxArts = newSize;
	    write(fd, oh, sizeof(OverHead));
	    write(fd, ob, newSize * sizeof(OverArt));
	}

	free(ob);
	free(oa);
    }
    hflock(fd, 4, XLOCK_UN);
    ++ResizedGroups;

    if (VerboseOpt && oldSize != newSize) {
	printf("resized maxArts from %d to %d grp=%s\n", oldSize, newSize, group->gr_GroupName);
    }
    return;
}

int
nearestPower(int n)
{
    int i;

    for (i = 1; i < n; i <<= 1)
	;
    return(i);
}

void
ReadDExpOverList()
{
    FILE *DExpOverList;
    hash_t read_buffer[DEXPOVER_READ_BUFFER_SIZE];
    int i, n;
    char path[128];

    dexpover_msgid_hash =
	(struct bucket_t *)malloc(DEXPOVER_HASH_SIZE * sizeof(struct bucket_t));
    for(i=0; i<DEXPOVER_HASH_SIZE; i++) {
	dexpover_msgid_hash[i].valid=0;
	dexpover_msgid_hash[i].next=NULL;
    }

    snprintf(path, 128, "%s.bak", PatDbExpand(DExpireOverListPat));
    rename(PatDbExpand(DExpireOverListPat), path);
    DExpOverList = fopen(path, "r");

    if(DExpOverList == NULL) return;

    while((n = fread(read_buffer, sizeof(hash_t),
		     DEXPOVER_READ_BUFFER_SIZE, DExpOverList))) {
	for(i=0; i<n; i++) {
	    int hashval;
	    struct bucket_t *chain;

	    hashval = (read_buffer[i].h1)&(DEXPOVER_HASH_SIZE-1);
	    chain = &dexpover_msgid_hash[hashval];

	    while((chain->valid == 1) && (chain->next != NULL))
		chain = chain->next;

	    if(chain->valid == 1) {
		chain->next = (struct bucket_t *)malloc(sizeof(struct bucket_t));
		chain = chain->next;
	    }

	    chain->valid = 1;
	    chain->hash_item = read_buffer[i];
	    chain->next = NULL;
	}
    }

    fclose(DExpOverList);
}

int
hexCharToInt(char c)
{
    return
	(c == '0') ? 0 :
	(c == '1') ? 1 :
	(c == '2') ? 2 :
	(c == '3') ? 3 :
	(c == '4') ? 4 :
	(c == '5') ? 5 :
	(c == '6') ? 6 :
	(c == '7') ? 7 :
	(c == '8') ? 8 :
	(c == '9') ? 9 :

	(c == 'a') ? 10 :
	(c == 'b') ? 11 :
	(c == 'c') ? 12 :
	(c == 'd') ? 13 :
	(c == 'e') ? 14 :
	(c == 'f') ? 15 :

	(c == 'A') ? 10 :
	(c == 'B') ? 11 :
	(c == 'C') ? 12 :
	(c == 'D') ? 13 :
	(c == 'E') ? 14 :
	(c == 'F') ? 15 :

	-1;
}

int
expOverListCheckExpired(hash_t *hv)
{
    int hashval;
    bucket_t *chain;

    hashval = (hv->h1)&(DEXPOVER_HASH_SIZE-1);
    chain = &dexpover_msgid_hash[hashval];

    while(chain && chain->valid) {
	if((chain->hash_item.h1 == hv->h1) &&
	   (chain->hash_item.h2 == hv->h2)) {
	    return 0;
	}
	chain = chain->next;
    }

    return -1;
}