/*
* DREADERD/GROUP.C
*
* Group/Overview routines. Group control and overview information is
* arranged in a two-level directory scheme by hash code under
* /news/spool/group. The first level consists of 256 subdirectories.
* The group is stored as a KPDB database file (dactive.kp)
*
* The header information is stored in 2 files:
* over.0.HASH Overview index
* data.nnnn.HASH Overview data
*
* The overview data file is a list of NUL terminated article
* headers in no particular order.
*
* The overview index file is binary file containing a header
* (struct OverInfo) and fixed length records (struct OverArt)
* with each article number placed in a slot based on the
* maximum number of possible articles in the group (ov_MaxArts).
*
* Each header is temporarily stored in a memory reference list
* (struct OverData) linked from OverInfo->ov_HData.
*
* (c)Copyright 1998, Matthew Dillon, All Rights Reserved. Refer to
* the COPYRIGHT file in the base directory of this distribution
* for specific rights granted.
*/
#include "defs.h"
#define OVHSIZE 64
#define OVHMASK (OVHSIZE-1)
#define HMAPALIGN (1024 * 64)
#define HMAPSIZE (1024 * 1024)
Prototype void NNFeedOverview(Connection *conn);
Prototype void FlushOverCache(void);
Prototype OverInfo *GetOverInfo(const char *group);
Prototype void PutOverInfo(OverInfo *ov);
Prototype const char *GetOverRecord(OverInfo *ov, int artno, int *plen, int *aleno, TimeRestrict *tr, int *TimeRcvd);
Prototype OverInfo *FindCanceledMsg(const char *group, const char *msgid, int *partNo, int *pvalidGroups);
Prototype int CancelOverArt(OverInfo *ov, int artNo);
Prototype int NNTestOverview(Connection *conn);
Prototype const char *NNRetrieveHead(Connection *conn, int *povlen, const char **pmsgid, int *TimeRcvd, int *grpIter, int *endNo);
const OverArt *GetOverArt(OverInfo *ov, int artNo, off_t *ppos);
void AssignArticleNo(Connection *conn, ArtNumAss **pan, const char *group, const char *xref, int approved, const char *art, int artLen, const char *msgid);
int WriteOverview(Connection *conn, ArtNumAss *an, const char *group, const char *xref, const char *art, int artLen, const char *msgid);
OverData *MakeOverHFile(OverInfo *ov, int artNo, int create);
void FreeOverInfo(OverInfo *ov);
void FreeOverData(OverData *od);
OverInfo *OvHash[OVHSIZE];
int NumOverInfo;
void
NNFeedOverview(Connection *conn)
{
int artLen;
int l = 0;
char *art = MBNormalize(&conn->co_ArtBuf, &artLen);
char *line;
int appr = 0;
char ch;
int err = 0;
char *newsgroups = NULL;
char *msgid = NULL;
char *subject = NULL;
char *date = NULL;
char *xref = NULL;
char *control = NULL;
char *supers = NULL; /* Supersedes */
/*
* Scan headers, look for Newsgroups: line, Subject:, Date:, From:, and
* Message-Id:. If any are missing, the article is bad. If there is an
* Xref: line, save that too and use it to calculate line numbers if
* Xref operation is enabled.
*
* We allow an LF-only line to terminate the headers as well as CR+LF,
* because some news systems are totally broken.
*/
for (line = art; line < art + artLen; line += l + 1) {
for (l = line - art; l < artLen; ++l) {
if (art[l] == '\n') {
if (l + 1 >= artLen || /* past end of article */
l == line - art || /* blank line */
(art[l+1] != ' ' && art[l+1] != '\t') /* !header ext */
) {
break;
}
}
}
l -= line - art;
ch = tolower(*line);
if (l == 0 || (l == 1 && line[0] == '\r')) {
/* out of headers */
break;
} else if (ch == 'n' && strncasecmp(line, "Newsgroups:", 11) == 0) {
newsgroups = zallocStrTrim(&conn->co_MemPool, line + 11, l - 11);
} else if (ch == 'm' && strncasecmp(line, "Message-ID:", 11) == 0) {
msgid = zallocStrTrim(&conn->co_MemPool, line + 11, l - 11);
} else if (ch == 's' && strncasecmp(line, "Subject:", 8) == 0) {
subject = zallocStrTrim(&conn->co_MemPool, line + 8, l - 8);
} else if (ch == 'd' && strncasecmp(line, "Date:", 5) == 0) {
date = zallocStrTrim(&conn->co_MemPool, line + 5, l - 5);
} else if (ch == 'x' && strncasecmp(line, "Xref:", 5) == 0) {
xref = zallocStrTrim(&conn->co_MemPool, line + 5, l - 5);
} else if (ch == 'c' && strncasecmp(line, "Control:", 8) == 0) {
control = zallocStrTrim(&conn->co_MemPool, line + 8, l - 8);
} else if (ch == 's' && strncasecmp(line, "Supersedes:", 11) == 0) {
supers = zallocStrTrim(&conn->co_MemPool, line + 11, l - 11);
} else if (ch == 'a' && strncasecmp(line, "Approved:", 9) == 0) {
appr = 1;
}
}
if (conn->co_Flags & COF_POSTTOOBIG) {
conn->co_Auth.dr_PostFailCount++;
if (conn->co_Flags & COF_IHAVE) {
MBLogPrintf(conn, &conn->co_TMBuf, "437 Rejected, too big\r\n");
} else {
MBLogPrintf(conn, &conn->co_TMBuf, "439 %s too big\r\n", conn->co_IHaveMsgId);
}
conn->co_Flags &= ~COF_POSTTOOBIG;
} else if (newsgroups == NULL || msgid == NULL || subject == NULL ||
date == NULL || strcmp(msgid, "<>") == 0
) {
/*
* failure
*/
conn->co_Auth.dr_PostFailCount++;
if (conn->co_Flags & COF_IHAVE) {
MBLogPrintf(conn, &conn->co_TMBuf, "437 Rejected, headers missing\r\n");
} else {
MBLogPrintf(conn, &conn->co_TMBuf, "439 %s\r\n", conn->co_IHaveMsgId);
}
} else if (conn->co_ByteCounter == 0.0 && conn->co_BytesHeader == 0) {
conn->co_Auth.dr_PostFailCount++;
if (conn->co_Flags & COF_IHAVE) {
MBLogPrintf(conn, &conn->co_TMBuf, "437 Rejected, Bytes header missing for header-only feed\r\n");
} else {
MBLogPrintf(conn, &conn->co_TMBuf, "439 %s headerOnlyFeed requires Bytes header\r\n", conn->co_IHaveMsgId);
}
} else if (FindCancelCache(msgid) == 0) {
char logbuf[1024];
conn->co_Auth.dr_PostFailCount++;
snprintf(logbuf, sizeof(logbuf), "%s cancel cache", msgid);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
if (conn->co_Flags & COF_IHAVE) {
MBLogPrintf(conn, &conn->co_TMBuf, "437 Article Already Cancelled\r\n");
} else {
MBLogPrintf(conn, &conn->co_TMBuf, "439 %s Article Already Cancelled\r\n", conn->co_IHaveMsgId);
}
} else {
/*
* write out overview information
*/
char *group;
char *ngroup = NULL;
ArtNumAss *ANABase = NULL;
/*
* if it is a control message, we don't really care what the newsgroups
* line says. instead, we cobble up "control.<type>" or just "control"
*/
if (conn->co_Flags & COF_WASCONTROL) {
char cmsgtype[64];
char *cptr;
cptr = control;
while (*cptr == ' ' || *cptr == '\t') {
cptr++;
}
snprintf(cmsgtype, sizeof(cmsgtype), "control%s%s", *cptr ? "." : "", cptr);
if (((cptr = strchr(cmsgtype, ' '))) || ((cptr = strchr(cmsgtype, '\t')))) {
*cptr = '\0';
}
zfreeStr(&conn->co_MemPool, &newsgroups);
newsgroups = zallocStr(&conn->co_MemPool, cmsgtype);
}
if (DebugOpt)
printf("Feed overview %s %s\n", msgid, newsgroups);
/*
* pass 1 - assign article numbers
*/
for (group = newsgroups; *group; group = ngroup) {
char c;
char whspc = 0;
char *whspptr;
/* Strip leading spaces */
while (*group == ' ' || *group == '\t') {
group++;
}
if (! *group) {
break;
}
for (ngroup = group; *ngroup && *ngroup != ','; ++ngroup)
;
c = *ngroup;
*ngroup = 0;
/*
* Skip groups with names that are too long
*/
if (ngroup - group > MAXGNAME)
continue;
/* Strip trailing space or tab from group name */
whspptr = strpbrk(group, " \t");
if (whspptr) {
whspc = *whspptr;
*whspptr = 0;
}
AssignArticleNo(conn, &ANABase, group, xref, appr, art, artLen, msgid);
/* Repair string back to its former state */
if (whspptr) {
*whspptr = whspc;
}
*ngroup = c;
if (*ngroup == ',')
++ngroup;
}
/*
* Supersedes is allowed on non-control messages. We execute the
* cancel AND post the article. Note: we do not allow supersedes
* on Control: messages. (XXX is this still true with the new logic? JG)
*/
if (supers) {
if (DebugOpt)
printf("has Supersedes: %s %s\n", msgid, newsgroups);
ExecuteSupersedes(conn, supers, art, artLen);
}
err = 0;
for (group = newsgroups; *group; group = ngroup) {
char c;
char whspc = 0;
char *whspptr;
for (ngroup = group; *ngroup && *ngroup != ','; ++ngroup)
;
c = *ngroup;
*ngroup = 0;
/*
* Skip groups with names that are too long
*/
if (ngroup - group > MAXGNAME)
continue;
/* Strip trailing space or tab from group name */
whspptr = strpbrk(group, " \t");
if (whspptr) {
whspc = *whspptr;
*whspptr = 0;
}
err += WriteOverview(conn, ANABase, group, xref, art, artLen, msgid);
/* Repair string back to its former state */
if (whspptr) {
*whspptr = whspc;
}
*ngroup = c;
if (*ngroup == ',')
++ngroup;
}
while (ANABase) {
ArtNumAss *an = ANABase;
ANABase = an->an_Next;
zfree(&conn->co_MemPool, an, sizeof(ArtNumAss));
}
if (conn->co_Flags & COF_WASCONTROL) {
if (DebugOpt)
printf("Control message: %s %s\n", msgid, newsgroups);
LogCmd(conn, 'c', control);
if (DRIncomingLogPat != NULL)
LogIncoming("%s c %s %s", conn->co_Auth.dr_Host,
msgid, control);
ExecuteControl(conn, control, art, artLen);
}
if (!err) {
conn->co_Auth.dr_PostCount++;
if (conn->co_Flags & COF_IHAVE) {
MBLogPrintf(conn, &conn->co_TMBuf, "235\r\n");
} else {
MBLogPrintf(conn, &conn->co_TMBuf, "239 %s\r\n", conn->co_IHaveMsgId);
}
}
}
zfreeStr(&conn->co_MemPool, &newsgroups);
zfreeStr(&conn->co_MemPool, &msgid);
zfreeStr(&conn->co_MemPool, &subject);
zfreeStr(&conn->co_MemPool, &date);
zfreeStr(&conn->co_MemPool, &xref);
zfreeStr(&conn->co_MemPool, &control);
zfreeStr(&conn->co_MemPool, &conn->co_IHaveMsgId);
MBFree(&conn->co_ArtBuf);
NNCommand(conn);
}
void
AssignArticleNo(Connection *conn, ArtNumAss **pan, const char *group, const char *xref, int approved, const char *art, int artLen, const char *msgid)
{
const char *rec;
int recLen;
int groupLen = strlen(group);
int activeArtBeg;
int activeArtEnd;
int aabegchanged = 0;
int foundXRef = 0;
int ts;
char aabegbuf[16];
char aaendbuf[16];
int artNo;
char logbuf[1024];
if (DOpts.ReaderAutoAddToActive) {
/*
* locate group in active file and lock, create if it does not exist
* You will have to manually add a GD, M, and fix S as appropriate
* through some external process, if you use this.
*/
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) == NULL) {
if (ValidGroupName(group) < 0) {
/* logit(LOG_ERR, "group %s illegal", group); */
} else {
char tsBuf[64];
KPDBWrite(KDBActive, group, "NB", "0000000001", KP_LOCK);
KPDBWrite(KDBActive, group, "NE", "0000000000", KP_LOCK_CONTINUE);
sprintf(tsBuf, "%08x", (int)time(NULL));
KPDBWrite(KDBActive, group, "CTS", tsBuf, KP_LOCK_CONTINUE);
KPDBWrite(KDBActive, group, "LMTS", tsBuf, KP_LOCK_CONTINUE);
KPDBWrite(KDBActive, group, "S", "y", KP_UNLOCK);
}
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) == NULL) {
snprintf(logbuf, sizeof(logbuf), "%s %s not in dactive", msgid, group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host,
"", logbuf);
return;
}
}
} else {
/*
* locate group in active file and lock
*/
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) == NULL) {
snprintf(logbuf, sizeof(logbuf), "%s %s not in dactive", msgid, group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
return;
}
}
/*
* silently drop postings to moderated groups that do not have an
* approved header.
*/
if (approved == 0) {
int flagsLen;
const char *flags = KPDBGetField(rec, recLen, "S", &flagsLen, "y");
while (flagsLen > 0) {
if (*flags == 'm') {
KPDBUnlock(KDBActive, rec);
snprintf(logbuf, sizeof(logbuf), "%s %s unapproved", msgid, group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host,
"", logbuf);
return;
}
--flagsLen;
++flags;
}
}
/*
* assign article number. Locate Xref: line if Xref's are enabled
*/
activeArtEnd = strtol(KPDBGetField(rec, recLen, "NE", NULL, "-1"), NULL,10);
activeArtBeg = strtol(KPDBGetField(rec, recLen, "NB", NULL, "-1"), NULL,10);
ts = (int)strtoul(KPDBGetField(rec, recLen, "LMTS", NULL, "0"), NULL, 16);
artNo = activeArtEnd + 1;
if (xref) {
const char *test;
for (test = strchr(xref, ' '); test; test = strchr(test, ' ')) {
++test;
if (strncmp(test, group, groupLen) == 0 && test[groupLen] == ':') {
artNo = strtol(test + groupLen + 1, NULL, 10);
foundXRef = 1;
break;
}
}
}
/*
* If we did not find an XRef entry and we are in xref-slave mode,
* drop the newsgroup on the floor.
*/
if (foundXRef == 0 && DOpts.ReaderXRefSlaveHost != NULL) {
KPDBUnlock(KDBActive, rec);
snprintf(logbuf, sizeof(logbuf), "%s %s no valid xref present in slave mode", msgid, group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
return;
}
if (artNo < 1)
artNo = 1;
if (activeArtEnd < artNo) {
activeArtEnd = artNo;
if (activeArtBeg > activeArtEnd) {
activeArtBeg = activeArtEnd;
aabegchanged = 1;
}
} else if (activeArtBeg > artNo) {
activeArtBeg = artNo;
aabegchanged = 1;
if (activeArtEnd < activeArtBeg)
activeArtEnd = activeArtBeg;
} else if (activeArtBeg > activeArtEnd) {
activeArtBeg = activeArtEnd = artNo;
aabegchanged = 1;
}
{
int nts = (int)time(NULL);
if (nts != ts) {
char tsBuf[64];
sprintf(tsBuf, "%08x", nts);
KPDBWrite(KDBActive, group, "LMTS", tsBuf, KP_LOCK_CONTINUE);
}
}
sprintf(aabegbuf, "%010d", activeArtBeg);
sprintf(aaendbuf, "%010d", activeArtEnd);
if (aabegchanged)
KPDBWrite(KDBActive, group, "NB", aabegbuf, KP_LOCK_CONTINUE); /* continuing lock */
KPDBWrite(KDBActive, group, "NE", aaendbuf, KP_UNLOCK); /* and unlock */
{
ArtNumAss *an = zalloc(&conn->co_MemPool, sizeof(ArtNumAss));
an->an_Next = *pan;
*pan = an;
an->an_GroupName = group;
an->an_GroupLen = strlen(group);
an->an_ArtNo = artNo;
}
}
int
WriteOverview(Connection *conn, ArtNumAss *an, const char *group, const char *xref, const char *art, int artLen, const char *msgid)
{
int artNo = 0;
const char *body;
char *xtmp = NULL;
int xtmpLen = 16 + strlen(conn->co_Auth.dr_VServerDef->vs_ClusterName);
char logbuf[1024];
int err = 0;
/*
* Locate article number assignment
*/
{
ArtNumAss *scan;
for (scan = an; scan; scan = scan->an_Next) {
if (scan->an_GroupName == group) {
artNo = scan->an_ArtNo;
}
xtmpLen += scan->an_GroupLen + 15;
}
}
if (artNo == 0)
return(0);
/*
* XXX We should find some way to aggregate these into one log entry.
*/
snprintf(logbuf, sizeof(logbuf), "%s %s:%d", msgid, group, artNo);
/*
* Locate start of body (we may have to append our own Xref: line)
*/
{
int l;
int lnl = 1;
for (l = 0; l < artLen; ++l) {
/*
* blank line terminates headers
*/
if (art[l] == '\r' && (l + 1 >= artLen || art[l+1] == '\n')) {
if (lnl)
break;
}
lnl = 0;
if (art[l] == '\n')
lnl = 1;
}
body = art + l;
}
/*
* Write overview record.
*/
{
off_t pos;
OverInfo *ov;
const OverArt *oa;
hash_t hv = hhash(msgid);
int actLen = 0;
int iovLen = 0;
struct iovec iov[3];
if ((ov = GetOverInfo(group)) == NULL) {
logit(LOG_ERR, "Error in GetOverInfo(%s) msgid=%s", group, msgid);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
return(1);
}
if (MakeOverHFile(ov, artNo, 1) == NULL) {
logit(LOG_ERR, "Error in MakeOverHFile() msgid=%s", msgid);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
return(1);
}
hflock(ov->ov_HCache->od_HFd, 0, XLOCK_EX);
pos = lseek(ov->ov_HCache->od_HFd, 0L, 2);
errno = 0;
if (xref) {
iov[0].iov_base = (void *)art;
iov[0].iov_len = artLen + 1;
iovLen = 1;
} else {
ArtNumAss *scan;
int soff;
xtmp = zalloc(&conn->co_MemPool, xtmpLen);
sprintf(xtmp, "Xref: %s", DOpts.ReaderXRefHost);
soff = strlen(xtmp);
for (scan = an; scan; scan = scan->an_Next) {
xtmp[soff++] = ' ';
memcpy(xtmp + soff, scan->an_GroupName, scan->an_GroupLen);
soff += scan->an_GroupLen;
sprintf(xtmp + soff, ":%d", scan->an_ArtNo);
soff += strlen(xtmp + soff);
}
sprintf(xtmp + soff, "\r\n");
soff += 2;
iov[0].iov_base = (void *)art;
iov[0].iov_len = body - art;
iov[1].iov_base = xtmp;
iov[1].iov_len = soff;
iov[2].iov_base = (void *)body;
iov[2].iov_len = (art + artLen + 1) - body;
iovLen = 3;
}
if (art[0] == 0)
logit(LOG_ERR, "Warning: art[0] is NIL! %s", xtmp);
{
int i;
for (i = 0; i < iovLen; ++i)
actLen += iov[i].iov_len;
}
if (DOpts.ReaderXRefSlaveHost &&
(oa = GetOverArt(ov, artNo, NULL)) != NULL &&
oa->oa_MsgHash.h1 == hv.h1 &&
oa->oa_MsgHash.h2 == hv.h2 &&
oa->oa_ArtNo == artNo
) {
/*
* We can detect duplicate articles in XRef slave mode. If
* we see one, do not do anything.
*/
LogCmd(conn, 'd', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s d %s%s", conn->co_Auth.dr_Host, "", logbuf);
; /* EMPTY */
} else if (DOpts.ReaderXOverMode == 0) {
/*
* Do not write xover info at all. This mode is not really
* supported by the reader but may eventually be supported
* in 100% nntp-cache mode if/when we develop it.
*/
logit(LOG_INFO, "ReaderXOverMode0 (%d:%s)", artNo, msgid);
LogCmd(conn, 'm', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s m %s%s", conn->co_Auth.dr_Host, "", logbuf);
; /* EMPTY */
} else if (
DOpts.ReaderXOverMode == 2 ||
writev(ov->ov_HCache->od_HFd, iov, iovLen) == actLen
) {
/*
* Our write of the overview data succeeded or we were asked not
* to write out the overview data. Write out the overview
* article record.
*/
OverArt ovart = { 0 };
off_t ovpos = 0;
hflock(ov->ov_OFd, 0, XLOCK_EX);
(void)GetOverArt(ov, artNo, &ovpos);
ovart.oa_ArtNo = artNo;
if (DOpts.ReaderXOverMode == 2)
ovart.oa_SeekPos = -1;
else
ovart.oa_SeekPos = pos;
ovart.oa_Bytes = actLen - 1; /* do not include \0 */
ovart.oa_MsgHash = hv;
ovart.oa_TimeRcvd = (int)time(NULL);
ovart.oa_ArtSize = (conn->co_ByteCounter > 0.0 ? conn->co_ByteCounter : conn->co_BytesHeader);
lseek(ov->ov_OFd, ovpos, 0);
write(ov->ov_OFd, &ovart, sizeof(ovart));
hflock(ov->ov_OFd, 0, XLOCK_UN);
LogCmd(conn, '+', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s + %s%s", conn->co_Auth.dr_Host, "", logbuf);
} else {
ftruncate(ov->ov_HCache->od_HFd, pos);
logit(LOG_ERR, "error writing overview data file for %s", group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
err = 1;
}
hflock(ov->ov_HCache->od_HFd, 0, XLOCK_UN);
PutOverInfo(ov);
}
if (xtmp)
zfree(&conn->co_MemPool, xtmp, xtmpLen);
return(err);
}
void FlushOverCache(void)
{
OverInfo **pov;
OverInfo *ov;
int i;
static int OI = 0;
for (i = 0; i < OVHSIZE; ++i) {
int ai = OI;
OI = (ai + 1) & OVHMASK;
pov = &OvHash[ai];
while ((ov = *pov) != NULL) {
if (ov->ov_Refs == 0) {
*pov = ov->ov_Next;
FreeOverInfo(ov);
--NumOverInfo;
} else {
pov = &ov->ov_Next;
}
}
}
}
OverInfo *
GetOverInfo(const char *group)
{
OverInfo **pov = &OvHash[shash(group) & OVHMASK];
OverInfo *ov;
while ((ov = *pov) != NULL) {
if (strcmp(group, ov->ov_Group) == 0)
break;
pov = &ov->ov_Next;
}
if (ov == NULL) {
struct stat st;
char *path;
int iter = 0;
int endNo = 0;
bzero(&st, sizeof(st));
/*
* If our cache has grown too large, attempt to free up
* a bunch of overview structures. Depending on the load,
* we may not be able to.
*/
if (NumOverInfo >= OV_CACHE_MAX) {
int i;
int freeup = OV_CACHE_MAX / 2;
static int OI = 0;
for (i = 0; i < OVHSIZE && freeup; ++i) {
int ai = OI;
OI = (ai + 1) & OVHMASK;
pov = &OvHash[ai];
while ((ov = *pov) != NULL) {
if (ov->ov_Refs == 0) {
*pov = ov->ov_Next;
FreeOverInfo(ov);
--NumOverInfo;
--freeup;
} else {
pov = &ov->ov_Next;
}
}
}
}
ov = zalloc(&SysMemPool, sizeof(OverInfo));
ov->ov_Group = zallocStr(&SysMemPool, group);
{
const char *rec;
int recLen;
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) == NULL) {
return(NULL);
}
iter = strtol(KPDBGetField(rec, recLen, "ITER", NULL, "0"), NULL, 10);
endNo = strtol(KPDBGetField(rec, recLen, "NE", NULL, "-1"), NULL, 10);
KPDBUnlock(KDBActive, rec);
}
path = zalloc(&SysMemPool, strlen(MyGroupHome) + 48);
again:
{
const char *gfname = GFName(group, GRPFTYPE_OVER, 0, 1, iter,
&DOpts.ReaderGroupHashMethod);
sprintf(path, "%s/%s", MyGroupHome, gfname);
ov->ov_OFd = -1;
if (MakeGroupDirectory(path) == -1)
logit(LOG_ERR, "Error on overview dir open/create: %s (%s)",
path, strerror(errno));
else
ov->ov_OFd = xopen(O_RDWR|O_CREAT, 0644, "%s", path);
}
if (ov->ov_OFd < 0) {
logit(LOG_ERR, "Error on overview open/create for group %s: %s (%s)",
group, path, strerror(errno));
FreeOverInfo(ov);
ov = NULL;
} else {
OverHead oh;
int r;
/*
* Leave a shared lock on the over.* file so expireover knows when
* it is OK to resize the file. If the file was renamed-over,
* we have to re-open it.
*/
hflock(ov->ov_OFd, 4, XLOCK_SH);
if (fstat(ov->ov_OFd, &st) < 0 || st.st_nlink == 0) {
hflock(ov->ov_OFd, 4, XLOCK_UN);
close(ov->ov_OFd);
ov->ov_OFd = -1;
goto again;
}
/*
* check if new overview file or illegal overview file and size
* accordingly
*/
r = (read(ov->ov_OFd, &oh, sizeof(oh)) != sizeof(oh));
if (r == 0 && oh.oh_ByteOrder != OH_BYTEORDER) {
logit(LOG_CRIT, "Incorrect overview byte order for %s (%s)",
group, path);
r = -1;
}
if (r == 0 && oh.oh_Version > OH_VERSION) {
logit(LOG_CRIT, "Incorrect overview version for %s (%s)",
group, path);
r = -1;
}
if (r != 0) {
hflock(ov->ov_OFd, 0, XLOCK_EX);
/*
* we have to test again after we got the lock in case
* another process had a lock and adjusted the file.
*/
lseek(ov->ov_OFd, 0L, 0);
if (read(ov->ov_OFd, &oh, sizeof(oh)) != sizeof(oh) ||
oh.oh_ByteOrder != OH_BYTEORDER
) {
OverExpire save;
GetOverExpire(group, &save);
/*
* If 'aInitArts' option not given in expireCtl
*/
if (save.oe_InitArts == 0)
save.oe_InitArts = DEFARTSINGROUP;
if (save.oe_InitArts > 65536)
save.oe_InitArts = 65536;
ftruncate(ov->ov_OFd, 0);
st.st_size = sizeof(oh) + sizeof(OverArt) * save.oe_InitArts;
ftruncate(ov->ov_OFd, st.st_size);
fsync(ov->ov_OFd);
lseek(ov->ov_OFd, 0L, 0);
bzero(&oh, sizeof(oh));
oh.oh_Version = OH_VERSION;
oh.oh_ByteOrder = OH_BYTEORDER;
oh.oh_HeadSize = sizeof(oh);
oh.oh_MaxArts = save.oe_InitArts;
strncpy(oh.oh_Gname, group, sizeof(oh.oh_Gname) - 1);
write(ov->ov_OFd, &oh, sizeof(oh));
fsync(ov->ov_OFd);
}
hflock(ov->ov_OFd, 0, XLOCK_UN);
}
if (oh.oh_Version > 1 && strcmp(oh.oh_Gname, group) != 0) {
hflock(ov->ov_OFd, 4, XLOCK_UN);
close(ov->ov_OFd);
ov->ov_OFd = -1;
iter++;
goto again;
}
if (iter > 0) {
char tsBuf[64];
sprintf(tsBuf,"%06d", iter);
KPDBWrite(KDBActive, group, "ITER", tsBuf, 0);
}
ov->ov_Iter = iter;
ov->ov_endNo = endNo;
ov->ov_Size = st.st_size;
ov->ov_MaxArts = (st.st_size - oh.oh_HeadSize) / sizeof(OverArt);
ov->ov_Head = xmap(NULL, ov->ov_Size, PROT_READ, MAP_SHARED, ov->ov_OFd, 0);
if (ov->ov_Head == NULL) {
logit(LOG_ERR, "Error on overview mmap for group %s (%s)",
group, strerror(errno));
FreeOverInfo(ov);
ov = NULL;
} else {
xadvise(ov->ov_Head, ov->ov_Size, XADV_SEQUENTIAL);
++NumOverInfo;
pov = &OvHash[shash(group) & OVHMASK];
ov->ov_Next = *pov;
*pov = ov;
}
}
zfree(&SysMemPool, path, strlen(MyGroupHome) + 48);
}
if (ov)
++ov->ov_Refs;
return(ov);
}
const OverArt *
GetOverArt(OverInfo *ov, int artno, off_t *ppos)
{
const OverArt *oa;
int ovpos = ov->ov_Head->oh_HeadSize +
((artno & 0x7FFFFFFF) % ov->ov_MaxArts) * sizeof(OverArt);
/*
* memory map the overview data. Check overview record to
* see if we actually have the requested information.
*/
oa = (const OverArt *)((const char *)ov->ov_Head + ovpos);
if (ppos)
*ppos = ovpos;
if (DebugOpt > 2)
printf("OA %08lx %d,%d\n", (long)oa, oa->oa_ArtNo, artno);
return(oa);
}
const char *
GetOverRecord(OverInfo *ov, int artno, int *plen, int *alen, TimeRestrict *tr, int *TimeRcvd)
{
int hvpos;
int xpos;
int xsize;
const OverArt *oa;
OverData *od;
oa = GetOverArt(ov, artno, NULL);
if (oa == NULL || oa->oa_ArtNo != artno) {
if (plen)
*plen = 0;
return(NULL);
}
if (tr && tr->tr_Time > oa->oa_TimeRcvd)
return(NULL);
if (TimeRcvd)
*TimeRcvd = oa->oa_TimeRcvd;
if (alen)
*alen = oa->oa_ArtSize;
if (plen == NULL)
return((const char *)1);
if (oa->oa_SeekPos == -1)
return(NULL);
if ((od = MakeOverHFile(ov, artno, 0)) == NULL)
return(NULL);
/*
* hvpos / oa->oa_Bytes. Include the guard character(s) in our
* calculations.
*/
hvpos = oa->oa_SeekPos;
xsize = oa->oa_Bytes + 1;
if ((xpos = hvpos) != 0) {
--xpos;
++xsize;
}
if (
od->od_HMapBase == NULL ||
xpos < od->od_HMapPos ||
xpos + xsize > od->od_HMapPos + od->od_HMapBytes
) {
struct stat st;
if (od->od_HMapBase) {
xunmap((void *)od->od_HMapBase, od->od_HMapBytes);
od->od_HMapBase = NULL;
od->od_HMapBytes = 0;
od->od_HMapPos = 0;
}
st.st_size = 0;
fstat(od->od_HFd, &st);
/*
* Make sure the file is big enough to map requested header. It
* is possible for it to not be.
*/
if (xpos + xsize > st.st_size)
return(NULL);
if (xpos > HMAPSIZE/2)
od->od_HMapPos = (xpos - HMAPSIZE/2) & ~(HMAPALIGN-1);
else
od->od_HMapPos = xpos & ~(HMAPALIGN-1);
od->od_HMapBytes = HMAPSIZE;
if (od->od_HMapBytes + od->od_HMapPos > st.st_size)
od->od_HMapBytes = st.st_size - od->od_HMapPos;
od->od_HMapBase = xmap(NULL, od->od_HMapBytes, PROT_READ, MAP_SHARED, od->od_HFd, od->od_HMapPos);
if (od->od_HMapBase == NULL) {
logit(LOG_CRIT, "mmap() failed B %s", strerror(errno));
exit(1);
}
}
/*
* Return base of record, length in *plen. But check for corruption...
* if the overview starts with a nul we have a problem.
*/
*plen = oa->oa_Bytes;
{
const char *r = od->od_HMapBase + hvpos - od->od_HMapPos;
if (*r == 0)
return(NULL);
if (xpos < hvpos && r[-1] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%d", ov->ov_Group, artno);
return(NULL);
}
if (r[oa->oa_Bytes] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%d", ov->ov_Group, artno);
return(NULL);
}
return(r);
}
}
/*
* FindCancelMsgId() - Locate cancel by message-id in group, return OverInfo
* and article number if found.
*
* Increment *pvalidGroups if the group is valid, leave
* alone otherwise. Whether or not the message-id could
* be located.
*/
OverInfo *
FindCanceledMsg(const char *group, const char *msgid, int *partNo, int *pvalidGroups)
{
const char *rec;
int recLen;
OverInfo *ov = NULL;
/*
* Make sure group is valid before calling GetOverInfo() or we will
* create random over. files for illegal groups. Don't lock the record,
* meaning that we have to re-read it after calling GetOverInfo (otherwise
* someone else can update the numeric fields in the record while we are
* trying to process them).
*/
*partNo = -1;
if ((rec = KPDBReadRecord(KDBActive, group, 0, &recLen)) != NULL) {
++*pvalidGroups;
if ((ov = GetOverInfo(group)) != NULL) {
hash_t hv = hhash(msgid);
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) != NULL) {
int artBeg;
int artEnd;
artBeg = strtol(KPDBGetField(rec, recLen, "NB", NULL, "-1"), NULL, 10);
artEnd = strtol(KPDBGetField(rec, recLen, "NE", NULL, "-1"), NULL, 10);
if (artEnd - artBeg > ov->ov_MaxArts)
artBeg = artEnd - ov->ov_MaxArts;
while (artEnd >= artBeg) {
off_t ovpos = 0;
const OverArt *oa = GetOverArt(ov, artEnd, &ovpos);
if (oa->oa_ArtNo == artEnd &&
bcmp(&oa->oa_MsgHash, &hv, sizeof(hv)) == 0
) {
*partNo = oa->oa_ArtNo;
break;
}
--artEnd;
} /* while */
KPDBUnlock(KDBActive, rec);
}
if (*partNo == -1) {
PutOverInfo(ov);
ov = NULL;
}
} /* if */
}
return(ov);
}
/*
* CancelOverMsgId() - cancel overview by message-id given article number.
*/
int
CancelOverArt(OverInfo *ov, int artNo)
{
int r = 0;
if (ov != NULL) {
off_t ovpos = 0;
const OverArt *oa;
hflock(ov->ov_OFd, 0, XLOCK_EX);
oa = GetOverArt(ov, artNo, &ovpos);
if (oa->oa_ArtNo == artNo) {
OverArt copy = *oa;
copy.oa_ArtNo = -1; /* CANCELED! */
lseek(ov->ov_OFd, ovpos, 0);
write(ov->ov_OFd, ©, sizeof(copy));
r = 1;
}
hflock(ov->ov_OFd, 0, XLOCK_UN);
}
return(r);
}
OverData *
MakeOverHFile(OverInfo *ov, int artNo, int create)
{
int artBase = artNo & ~OD_HMASK;
OverData **pod;
OverData *od;
int count = 0;
if (create)
create = O_CREAT;
if (ov->ov_HCache && artBase == ov->ov_HCache->od_ArtBase)
return(ov->ov_HCache);
for (pod = &ov->ov_HData; (od = *pod) != NULL; pod = &od->od_Next) {
if (artBase == od->od_ArtBase)
break;
++count;
}
if (od == NULL) {
const char *gfname = GFName(ov->ov_Group, GRPFTYPE_DATA, artBase, 1,
ov->ov_Iter, &DOpts.ReaderGroupHashMethod);
*pod = od = zalloc(&SysMemPool, sizeof(OverData));
errno = 0;
od->od_HFd = xopen(O_RDWR|create, 0644, "%s/%s", MyGroupHome, gfname);
if (od->od_HFd < 0) {
if (create) {
logit(LOG_ERR, "Unable to open/create %s/%s: %s",
MyGroupHome,
gfname,
strerror(errno)
);
}
FreeOverData(od);
*pod = od = NULL;
} else {
od->od_ArtBase = artBase;
if (count > DOpts.ReaderThreads) {
OverData *t = ov->ov_HData;
ov->ov_HData = t->od_Next;
FreeOverData(t);
}
}
}
ov->ov_HCache = od;
return(od);
}
/*
* PutOverInfo() - release the ref count, but do not immediately unmap or
* free the data (other routines depend on this).
*/
void
PutOverInfo(OverInfo *ov)
{
if (ov != NULL)
--ov->ov_Refs;
}
void
FreeOverInfo(OverInfo *ov)
{
OverData *od;
while ((od = ov->ov_HData) != NULL) {
ov->ov_HData = od->od_Next;
FreeOverData(od);
}
if (ov->ov_Head)
xunmap((void *)ov->ov_Head, ov->ov_Size);
if (ov->ov_OFd >= 0) {
/*
* remove shared lock and close
*/
hflock(ov->ov_OFd, 4, XLOCK_UN);
close(ov->ov_OFd);
}
zfreeStr(&SysMemPool, &ov->ov_Group);
bzero(ov, sizeof(OverInfo));
zfree(&SysMemPool, ov, sizeof(OverInfo));
}
void
FreeOverData(OverData *od)
{
if (od->od_HMapBase) {
xunmap((void *)od->od_HMapBase, od->od_HMapBytes);
od->od_HMapBase = NULL;
}
if (od->od_HFd >= 0) {
close(od->od_HFd);
od->od_HFd = -1;
}
zfree(&SysMemPool, od, sizeof(OverData));
}
int
NNTestOverview(Connection *conn)
{
OverInfo *ov;
int r = -1;
if ((ov = GetOverInfo(conn->co_GroupName)) != NULL) {
if (GetOverRecord(ov, conn->co_ArtNo, NULL, NULL, NULL, NULL) != NULL)
r = 0;
PutOverInfo(ov);
}
return(r);
}
const char *
NNRetrieveHead(Connection *conn, int *povlen, const char **pmsgid, int *TimeRcvd, int *grpIter, int *endNo)
{
OverInfo *ov;
const char *res = NULL;
*povlen = 0;
if (pmsgid != NULL)
*pmsgid = "<>";
if (conn->co_GroupName == NULL) {
return(NULL);
}
if ((ov = GetOverInfo(conn->co_GroupName)) != NULL) {
if (grpIter != NULL)
*grpIter = ov->ov_Iter;
if (endNo != NULL)
*endNo = ov->ov_endNo;
if ((res = GetOverRecord(ov, conn->co_ArtNo, povlen, NULL, NULL, TimeRcvd)) != NULL) {
const char *scan = res;
int scanLen = *povlen;
/*
* Locate and extract the Message-ID
*/
while (scanLen > 0) {
int i;
char ch = tolower(scan[0]);
for (i = 0; i < scanLen && scan[i] != '\n'; ++i)
;
if (pmsgid != NULL && ch == 'm' &&
strncasecmp(scan, "Message-ID:", 11) == 0) {
int b = 11;
int e;
char buf[MAXMSGIDLEN];
while (b < scanLen && (scan[b] == ' ' || scan[b] == '\t'))
++b;
e = b;
while (e < scanLen && (scan[e] != '>'))
++e;
if (e < scanLen)
++e;
if (e - b < MAXMSGIDLEN) {
bcopy(scan + b, buf, e - b);
buf[e-b] = 0;
*pmsgid = MsgId(buf, NULL);
}
/*
* Stop scanning through headers
*/
break;
}
if (i == 1 && scan[0] == '\r') {
*povlen -= scanLen;
break;
}
if (i < scanLen)
++i;
scanLen -= i;
scan += i;
}
}
PutOverInfo(ov);
}
if (pmsgid != NULL && strcmp(*pmsgid, "<>") == 0)
res = NULL;
return(res);
}
syntax highlighted by Code2HTML, v. 0.9.1