/*
fetchnews -- post articles to and get news from upstream server(s)
Written by Arnt Gulbrandsen <agulbra@troll.no> and copyright 1995
Troll Tech AS, Postboks 6133 Etterstad, 0602 Oslo, Norway, fax +47
22646949.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>
and Randolf Skerka <Randolf.Skerka@gmx.de>.
Copyright of the modifications 1997.
Modified by Kent Robotti <robotti@erols.com>. Copyright of the
modifications 1998.
Modified by Markus Enzenberger <enz@cip.physik.uni-muenchen.de>.
Copyright of the modifications 1998.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>.
Copyright of the modifications 1998, 1999.
Modified by Ralf Wildenhues <ralf.wildenhues@gmx.de>.
Copyright of the modifications 2002.
Modified by Jonathan Larmour <jifl@jifvik.org>.
Copyright of the modifications 2002.
Modified by Richard van der Hoff <richard@rvanderhoff.org.uk>
Copyright of the modifications 2002.
Enhanced and modified by Matthias Andree <matthias.andree@gmx.de>.
Copyright of the modifications 2000 - 2006.
See file COPYING for restrictions on the use of this software.
*/
#include "leafnode.h"
#include "fetchnews.h"
#include "mastring.h"
#include "ln_log.h"
#include "mysigact.h"
#include <sys/types.h>
#include <ctype.h>
#include "system.h"
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <setjmp.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <syslog.h>
#include <sys/resource.h>
#include <unistd.h>
#include <utime.h>
#include <sys/wait.h>
#include "groupselect.h"
int verbose = 0;
int debug = 0;
static time_t now;
/* Variables set by command-line options which are specific for fetch */
static unsigned long extraarticles = 0;
static int usesupplement = 1;
static int postonly = 0; /* if 1, don't read files from upstream */
static int noexpire = 0; /* if 1, don't automatically unsubscribe */
static int forceactive = 0; /* if 1, reread complete active file */
static sigjmp_buf jmpbuffer;
static volatile sig_atomic_t canjump;
static int age( /*@null@*/ const char *date);
static int postarticles(const struct server *current_server);
static void
ignore_answer(FILE * f)
{
char *l;
while (((l = mgetaline(f)) != NULL) && strcmp(l, "."));
}
static RETSIGTYPE
sig_int(int signo)
{
if (canjump == 0)
return; /* ignore unexpected signals */
if (signo == SIGINT || signo == SIGTERM) {
canjump = 0;
alarm(0);
siglongjmp(jmpbuffer, signo);
}
}
static void
usage(void)
{
fprintf(stderr, "Usage: fetchnews [-q] [-v] [-x #] [-l] [-n] [-f] [-P] [-w]\n"
" -q: quiet, suppress some warnings, cancels -v\n"
" -v: more verbose (may be repeated), cancels -q\n"
" -x: check for # extra articles in each group\n"
" -l: do not use supplementary servers\n"
" -n: do not automatically expire unread groups\n"
" -f: force reload of groupinfo file\n"
" -P: only post outgoing articles, don't fetch any\n"
" -w: wait, run XOVER updater in foreground\n");
}
/**
* check whether any of the newsgroups is on server
* return TRUE if yes, FALSE otherwise
*/
static int
isgrouponserver(const struct server *current_server,
char *newsgroups /** string will be destroyed! */)
{
char *p, *q;
int retval;
if (!newsgroups)
return FALSE;
retval = FALSE;
p = newsgroups;
do {
q = strchr(p, ',');
if (q)
*q++ = '\0';
switch (gs_match(current_server->group_pcre, p)) {
case 1:
xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", p);
putaline();
if (nntpreply(current_server) == 211) {
if (debug > 1)
syslog(LOG_DEBUG, "%s is matched by only_groups_pcre and is on server", p);
retval = TRUE;
}
break;
case 0:
if (debug > 1)
syslog(LOG_DEBUG, "%s not matched by only_group_pcre", p);
if (current_server->only_groups_match_all) {
if (debug > 1)
syslog(LOG_DEBUG, "not posting article to this server, "
"only_groups_match_all is set");
return FALSE;
}
break;
default:
break;
}
p = q;
if (p)
while (*p && isspace((unsigned char)*p))
p++;
} while (q);
return retval;
}
/*
* check whether message-id is on server - msgid is expected
* to contain the angle brackets, i. e. <id@example.org>
* return 1 if yes, 0 if no, -1 for error
*/
static int
ismsgidonserver(const struct server *current_server, char *msgid)
{
int r;
if (!msgid)
return 0;
xsnprintf(lineout, SIZE_lineout, "%s %s\r\n",
stat_is_evil ? "HEAD" : "STAT", msgid);
putaline();
r = nntpreply(current_server);
if (r == 498 && lastreply() == NULL)
return -1;
if (r >= 220 && r <= 223) {
if (stat_is_evil)
ignore_answer(nntpin);
return 1;
} else
return 0;
}
int
age( /*@null@*/ const char *date)
{
char monthname[4]; /* RATS: ignore */
int month;
int year;
int day;
const char *d;
time_t tmp;
struct tm time_struct;
if (!date)
return 1000; /* large number: OLD */
d = date;
if (!(strncasecmp(d, "date:", 5)))
d += 5;
while (isspace((unsigned char)*d))
d++;
if (isalpha((unsigned char)*d)) {
while (*d && !isspace((unsigned char)*d)) /* skip "Mon" or "Tuesday," */
d++;
}
/* RFC 822 says we have 1*LWSP-char between tokens */
while (isspace((unsigned char)*d))
d++;
/* parsing with sscanf leads to crashes */
day = atoi(d);
while (isdigit((unsigned char)*d) || isspace((unsigned char)*d))
d++;
if (!isalpha((unsigned char)*d)) {
syslog(LOG_INFO, "Unable to parse %s", date);
return 1003;
}
monthname[0] = *d++;
monthname[1] = *d++;
monthname[2] = *d++;
monthname[3] = '\0';
if (strlen(monthname) != 3) {
syslog(LOG_INFO, "Unable to parse month in %s", date);
return 1004;
}
while (isalpha((unsigned char)*d))
d++;
while (isspace((unsigned char)*d))
d++;
year = atoi(d);
if ((year < 1970) && (year > 99)) {
syslog(LOG_INFO, "Unable to parse year in %s", date);
return 1005;
} else if (!(day > 0 && day < 32)) {
syslog(LOG_INFO, "Unable to parse day in %s", date);
return 1006;
} else {
if (!strcasecmp(monthname, "jan"))
month = 0;
else if (!strcasecmp(monthname, "feb"))
month = 1;
else if (!strcasecmp(monthname, "mar"))
month = 2;
else if (!strcasecmp(monthname, "apr"))
month = 3;
else if (!strcasecmp(monthname, "may"))
month = 4;
else if (!strcasecmp(monthname, "jun"))
month = 5;
else if (!strcasecmp(monthname, "jul"))
month = 6;
else if (!strcasecmp(monthname, "aug"))
month = 7;
else if (!strcasecmp(monthname, "sep"))
month = 8;
else if (!strcasecmp(monthname, "oct"))
month = 9;
else if (!strcasecmp(monthname, "nov"))
month = 10;
else if (!strcasecmp(monthname, "dec"))
month = 11;
else {
syslog(LOG_INFO, "Unable to parse %s", date);
return 1001;
}
if (year < 70) /* years 2000-2069 in two-digit form */
year += 100;
else if (year > 1970) /* years > 1970 in four-digit form */
year -= 1900;
memset(&time_struct, 0, sizeof(time_struct));
time_struct.tm_sec = 0;
time_struct.tm_min = 0;
time_struct.tm_hour = 0;
time_struct.tm_mday = day;
time_struct.tm_mon = month;
time_struct.tm_year = year;
time_struct.tm_isdst = 0;
tmp = mktime(&time_struct);
if (tmp == -1)
return 1002;
return ((now - tmp) / SECONDS_PER_DAY);
}
}
/*
* Get body of a single message of which the header has already been
* downloaded and append it to the file with the header.
* Returns 0 if file could not be retrieved, 1 otherwise.
*/
static int
getbody_insitu(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
const char *c;
int rc = 0;
char *l;
char *messageid;
FILE *f;
char s[SIZE_s+1];
off_t pos;
if (!chdirgroup(group->name, FALSE))
return 0;
/* extract message-id: header */
xsnprintf(s, SIZE_s, "%lu", id);
messageid = getheader(s, "Message-ID:");
if (!messageid)
return 0;
/* check whether we can retrieve the body */
if (verbose > 2)
printf("%s: BODY %s\n", group->name, messageid);
xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
putaline();
if (nntpreply(current_server) != 222) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: Retrieving body %s failed. No response",
group->name, messageid);
rc = 0;
goto getbody_bail;
}
xsnprintf(s, SIZE_s, "%lu", id);
c = lookup(messageid);
if (!(f = fopen(c, "a"))) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: cannot open %s for appending", group->name, c);
rc = 0;
goto getbody_bail;
}
pos = ftell(f);
fputc('\n', f); /* blank line -- separate header and body */
debug--;
while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
if (*l == '.')
++l;
fputs(l, f);
fputc('\n', f);
}
debug = debugmode;
if (!l) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
"Transmission interrupted.", group->name, messageid);
ftruncate(fileno(f), pos);
fclose(f);
rc = 0;
goto getbody_bail;
}
/* abort when disk is full */
if (fclose(f) && errno == ENOSPC) {
truncate(s, pos);
raise(SIGINT);
return 0;
}
rc = 1;
getbody_bail:
if (messageid)
free(messageid);
return rc;
}
static int
getbody_newno(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
const char *c;
int rc = 0;
char *l;
char *p, *q;
char *messageid;
char *newsgroups; /* I hope this is enough */
char *xref;
FILE *f, *g;
char s[SIZE_s+1];
if (!chdirgroup(group->name, FALSE))
return 0;
/* extract message-id: and xref: headers */
xsnprintf(s, SIZE_s, "%lu", id);
if (!(f = fopen(s, "r"))) {
syslog(LOG_INFO, "%s: cannot open %s for reading -- possibly expired",
group->name, s);
return 1; /* pretend to have read file successfully so that
it is purged from the list */
}
messageid = NULL;
newsgroups = NULL;
xref = NULL;
debug--;
while ((l = getaline(f)) != NULL) {
if (!newsgroups && !strncmp(l, "Newsgroups:", 11)) {
p = l+11;
SKIPLWS(p);
newsgroups = critstrdup(p, "getbody");
}
if (!messageid && !strncmp(l, "Message-ID:", 11)) {
p = l+11;
SKIPLWS(p);
messageid = critstrdup(p, "getbody");
}
if (!xref && !strncmp(l, "Xref:", 5)) {
p = l + 5;
SKIPLWS(p);
xref = critstrdup(p, "getbody");
}
}
debug = debugmode;
fclose(f);
/* check whether we can retrieve the body */
if (verbose > 2)
printf("%s: BODY %s\n", group->name, messageid);
xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
putaline();
if (nntpreply(current_server) != 222) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. No response",
group->name, messageid);
rc = 0;
goto getbody_bail;
}
xsnprintf(s, SIZE_s, "%lu", id);
c = lookup(messageid);
log_unlink(c, 0); /* make space for new file */
if (!(f = fopen(c, "w"))) {
ln_log(LNLOG_SERR, LNLOG_CGROUP, "%s: cannot open %s for writing", group->name, c);
link(s, c); /* if we can't open new file restore old one */
rc = 0;
goto getbody_bail;
}
/* copy all headers except Xref: into new file */
g = fopen(s, "r");
if (!g) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: open %s failed", group->name, s);
rc = 0;
goto getbody_bail;
}
debug--;
while ((l = getaline(g)) != NULL) {
/* skip xref: headers */
if (strncasecmp(l, "Xref:", 5) != 0)
fprintf(f, "%s\n", l);
}
debug = debugmode;
fclose(g);
/* create a whole bunch of new hardlinks */
store(c, f, newsgroups, messageid);
/* retrieve body */
fprintf(f, "\n"); /* Empty line between header and body. */
debug--;
while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
if (*l == '.')
++l;
fputs(l, f);
fputc('\n', f);
}
debug = debugmode;
if (!l) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
"Transmission interrupted.", group->name, messageid);
fprintf(f, "\n\t[ Leafnode: ]\n"
"\t[ An error occured while " "retrieving this message. ]\n");
fclose(f);
rc = 0;
goto getbody_bail;
}
/* abort when disk is full */
if (fclose(f) && errno == ENOSPC)
raise(SIGINT);
/* Remove old article files since we don't need them anymore.
This is done by evaluating the old Xref: header.
*/
if (!xref) {
/* no Xref: header --> make a fake one */
xref = critmalloc(50 + strlen(fqdn) + strlen(group->name), "getbody");
sprintf(xref, "%s %s:%lu", fqdn, group->name, id); /* RATS: ignore */
}
if (debugmode)
syslog(LOG_DEBUG, "xref: %s", xref);
c = strchr(xref, ' ');
#ifdef __LCLINT__
assert(c != NULL); /* we know c != NULL */
#endif /* __LCLINT__ */
while ((c++) && (*c) && (q = strchr(c, ':')) != NULL) {
*q++ = '\0';
if ((p = strchr(q, ' ')) != NULL)
*p = '\0';
/* c points to the newsgroup, q to the article number */
if (!chdirgroup(c, FALSE)) {
return 0;
}
if (unlink(q))
syslog(LOG_NOTICE,
"retrieved body, but unlinking headers-only file %s/%s failed",
c, q);
else if (debugmode)
syslog(LOG_DEBUG,
"retrieved body, now unlinking headers-only file %s/%s", c,
q);
c = strchr(q, ' ');
}
rc = 1;
getbody_bail:
if (xref)
free(xref);
if (messageid)
free(messageid);
if (newsgroups)
free(newsgroups);
return rc;
}
static int
getbody(const struct server *cs, struct newsgroup *group, unsigned long id) {
static int (*func)(const struct server *, struct newsgroup *, unsigned long);
if (!func) {
func = db_situ ? getbody_insitu : getbody_newno;
}
return func(cs, group, id);
}
/*
* Get bodies of messages that have marked for download.
* The group must already be selected at the remote server and
* the current directory must be the one of the group.
*/
static void
getmarked(const struct server *current_server, struct newsgroup *group)
{
int n, i;
int had_bodies = 0;
FILE *f;
mastr *filename = mastr_new(PATH_MAX);
unsigned long id[BODY_DOWNLOAD_LIMIT]; /* RATS: ignore */
char *t;
/* #1 read interesting.groups file */
n = 0;
mastr_vcat(filename, spooldir, "/interesting.groups/", group->name, NULL);
if (!(f = fopen(mastr_str(filename), "r")))
ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for reading", mastr_str(filename));
else {
struct stat st;
if (fstat(fileno(f), &st) == 0 && st.st_size > 0) {
had_bodies = 1;
if (verbose)
printf("%s: getting bodies of marked messages...\n",
group->name);
while ((t = getaline(f)) && n < BODY_DOWNLOAD_LIMIT) {
if (sscanf(t, "%lu", &id[n]) == 1)
++n;
}
}
fclose(f);
}
/* #2 get bodies */
if (delaybody || had_bodies) {
syslog(LOG_INFO, "%s: marked bodies %d", group->name, n);
if (verbose > 1)
printf("%s: marked bodies %d\n", group->name, n);
}
for (i = 0; i < n; ++i)
if (getbody(current_server, group, id[i]))
id[i] = 0;
/* #3 write back ids of all articles which could not be retrieved */
if (had_bodies) {
if (!(f = fopen(mastr_str(filename), "w")))
ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for writing", mastr_str(filename));
else {
for (i = 0; i < n; ++i)
if (id[i] != 0)
fprintf(f, "%lu\n", id[i]);
fclose(f);
}
}
if (delaybody || had_bodies) {
if (verbose)
printf("%s: Done getting article bodies.\n", group->name);
}
mastr_delete(filename);
}
/** count number of colons in the string s_in. */
static int count_colons(const char *s_in) {
int ngs;
const char *t;
ngs = 0;
t = s_in;
for(;;) {
t += strcspn(t, ":");
if (!*t) break;
t++;
ngs++;
if (ngs < 0) {
/* overflow */
return INT_MAX;
}
}
return ngs;
}
/*
* get newsgroup from a server. "server" is the last article that
* was previously read from this group on that server
*/
static unsigned long
getgroup(const struct server *current_server,
/*@null@*/ struct newsgroup *g,
unsigned long server)
{
#define HD_MAX 10
static char *hd[HD_MAX];
const char *hnames[HD_MAX] = { "Path: ", "Message-ID: ", "From: ",
"Newsgroups: ", "Subject: ", "Date: ",
"References: ", "Lines: ", "Xref: ", ""
};
/* order of headers in XOVER */
enum enames { /* 0: art. no. */ h_sub = 1, h_fro, h_dat, h_mid,
h_ref, h_byt, h_lin, h_xref };
/* XOVER fields: 0 Subject, 1 From, 2 Date, 3 Message-ID, 4
* References, 5 Bytes, 6 Lines, 7 Xref:full (optional) */
unsigned long fetched, killed;
unsigned long h;
long n;
unsigned long last;
unsigned long window; /* last ARTICLE n command sent */
char *l;
FILE *f;
const char *c;
struct stat st;
long outstanding = 0, j;
unsigned long i;
unsigned long *stufftoget;
int localmaxage = maxage;
int maxagelimit = -1;
const char *limitfrom = "";
int expdays;
if (!g)
abort();
if (g->first > g->last && g->first - g->last > 1)
g->last = g->first - 1;
if ((expdays = lookup_expiredays(g->name)) > 0) {
if (localmaxage > expdays) {
maxagelimit = expdays;
limitfrom = "groupexpire";
}
} else {
if (localmaxage > expiredays) {
maxagelimit = expiredays;
limitfrom = "global expire";
}
}
if (*limitfrom && localmaxage > maxagelimit) {
if (clamp_maxage) {
syslog(LOG_NOTICE, "clamping maxage for %s to %s %d",
g->name, limitfrom, maxagelimit);
localmaxage = maxagelimit;
} else {
fprintf(stderr,
"warning: group %s: maxage of %d is inappropriate for your "
"applicable %s of %d. This can cause excessive downloads of "
"articles that were previously downloaded and expired. "
"Fix your configuration.\n", g->name, localmaxage, limitfrom,
maxagelimit);
syslog(LOG_WARNING,
"warning: group %s: maxage of %d is inappropriate for your "
"applicable %s of %d. This can cause excessive downloads of "
"articles that were previously downloaded and expired. "
"Fix your configuration.", g->name, localmaxage, limitfrom,
maxagelimit);
}
}
if (server == 0ul)
server = 1ul;
/* skip */
if (gs_match(current_server->group_pcre, g->name) != 1) {
if (verbose > 1) {
printf("%s: skipped %s, not in only_groups_pcre\n",
current_server->name, g->name);
}
return server;
}
/* skip */
if ((l = getenv("LN_SKIP_GROUPS"))) {
char *x, *y = critstrdup(l, "getgroup");
for (x = strtok(y, ","); x; x = strtok(NULL, ",")) {
if (wildmat(g->name, x)) {
if (verbose) {
printf("%s: skipped %s, LN_SKIP_GROUPS=%s\n",
current_server->name, g->name, l);
syslog(LOG_INFO, "%s: skipped %s, LN_SKIP_GROUPS=%s",
current_server->name, g->name, l);
}
free(y);
return server;
}
}
free(y);
}
xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", g->name);
putaline();
n = nntpreply(current_server);
if (n == 498) {
return 0;
}
l = lastreply();
if (n == 411) { /* group not available on server */
if (verbose > 1)
printf("%s: no such group\n", g->name);
return server;
}
if (sscanf(l, "%3ld %lu %lu %lu ", &n, &h, &window, &last) < 4 || n != 211)
{
fprintf(stderr, "Warning: %s: cannot parse server reply \"%s\"\n", g->name, l);
syslog(LOG_WARNING, "Warning: %s: cannot parse server reply \"%s\"", g->name, l);
return 0;
}
if (h == 0) {
if (verbose > 1)
printf("%s: upstream group is empty\n", g->name);
return server;
}
if (extraarticles) {
if (server > extraarticles)
i = server - extraarticles;
else
i = 1;
if (i < window)
i = window;
if (verbose > 1 && server > 1)
printf("%s: backing up from %lu to %lu\n", g->name, server, i);
server = i;
}
if (server > last + 1) {
syslog(LOG_INFO,
"%s: last seen article was %lu, server now has %lu-%lu",
g->name, server, window, last);
if (server > last + 5) {
if (verbose)
printf("%s: switched upstream servers? %lu > %lu\n",
g->name, server - 1, last);
server = window; /* insane - recover thoroughly */
} else {
if (verbose)
printf("%s: rampant spam cancel? %lu > %lu\n", g->name, server - 1, last);
server = last - 5; /* a little bit too much */
}
}
if (initiallimit && server == 1 && last > server
&& last - server > initiallimit) {
if (verbose > 1)
printf("%s: skipping articles %lu-%lu inclusive (initial limit)\n",
g->name, server, last - initiallimit);
syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (initial limit)",
g->name, server, last - initiallimit);
server = last - initiallimit + 1;
}
if (artlimit && last > server && last - server > artlimit) {
if (verbose > 1)
printf("%s: skipping articles %lu-%lu inclusive (article limit)\n",
g->name, server, last - artlimit - 1);
syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (article limit)",
g->name, server, last - artlimit - 1);
server = last - artlimit;
}
getmarked(current_server, g);
if (window < server)
window = server;
if (window < 1)
window = 1;
server = window;
if (server > last) {
if (verbose > 1)
printf("%s: no new articles\n", g->name);
syslog(LOG_INFO, "%s: no new articles\n", g->name);
return server;
}
if (verbose > 1)
printf("%s: considering articles %lu - %lu\n", g->name, server, last);
syslog(LOG_INFO, "%s: considering articles %lu - %lu\n", g->name, server,
last);
fetched = 0;
killed = 0;
stufftoget =
(unsigned long *)malloc(sizeof(stufftoget[0]) * (last + 1 - server));
if (!stufftoget) {
ln_log(LNLOG_SERR, LNLOG_CGROUP, "not enough memory for XHDRs");
return server;
}
memset(stufftoget, 0, sizeof(stufftoget[0]) * (last + 1 - server));
if (!current_server->noxover) {
int tmp;
/* Try XOVER */
xsnprintf(lineout, SIZE_lineout, "XOVER %lu-%lu\r\n", server, last);
putaline();
if (nntpreply(current_server) == 224) {
mastr *ol = mastr_new(1024);
debug--;
while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
/*@dependent@*/ char *fields[HD_MAX];
unsigned long art;
const char *t;
char *q;
mastr_cpy(ol, l);
/* split xover */
/*@+loopexec@*/
for (i = 0; l && l[0] && i < HD_MAX; i++) {
char *y;
fields[i] = l;
if ((y = strchr(l, '\t'))) {
y[0] = '\0';
l = y + 1;
} else {
l = NULL;
};
};
/*@=loopexec@*/
/* short line -- log and skip */
if (i < 7) {
ln_log(LNLOG_SWARNING, LNLOG_CTOP,
"%s: %s: Warning: got unparsable XOVER line from server, "
"too few fields (%lu): \"%s\"",
current_server->name, g->name, i, mastr_str(ol));
continue;
}
for (; i < HD_MAX; i++)
fields[i] = NULL;
art = strtoul(fields[0], &q, 10);
if (q && art >= server && art <= last) {
long artlines; /* invalid: -1 */
unsigned long artbytes; /* invalid: 0 */
if (fields[h_lin]) artlines = strtol(fields[h_lin], NULL, 10);
else artlines = -1;
if (fields[h_byt]) artbytes = strtoul(fields[h_byt], NULL, 10);
else artbytes = 0;
if (maxbytes && fields[h_byt]
&& (strtoul(fields[h_byt], NULL, 10) > maxbytes)) {
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too many bytes (%lu > %lu)",
g->name, art, fields[h_mid],
strtoul(fields[h_byt], NULL, 10), maxbytes);
} else if (maxbytes && artbytes != 0
&& artbytes > maxbytes) {
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too large (%lu > %lu)",
g->name, art, fields[h_mid],
artbytes, maxbytes);
} else if (maxlines && artlines != -1
&& artlines > maxlines) {
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too many lines (%ld > %ld)",
g->name, art, fields[h_mid],
artlines, maxlines);
} else if (minlines && artlines != -1
&& artlines < minlines) {
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too few lines (%ld < %ld)",
g->name, art, fields[h_mid],
artlines, minlines);
} else if (localmaxage && fields[h_dat]
&& (age(fields[h_dat]) > localmaxage)) {
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too old (%d > %d) days",
g->name, art, fields[h_mid],
age(fields[h_dat]), localmaxage);
} else if (crosspostlimit && fields[h_xref] && (tmp = count_colons(fields[h_xref]) - 1) > crosspostlimit) {
/* -1 to skip over the header's name, "Xref:" */
killed++;
ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
"too many groups in Xref: header (%d > %ld)",
g->name, art, fields[h_mid],
tmp, crosspostlimit);
} else if (lstat(t = lookup(fields[h_mid]), &st) == 0) {
killed++;
ln_log(LNLOG_SDEBUG, 4, "%s: killed %lu (%s), already fetched before",
g->name, art, fields[h_mid]);
} else {
stufftoget[outstanding] = art;
outstanding++;
if (verbose > 2)
printf("%s: will fetch %lu (%s)\n", g->name, art, t);
}
}
}
mastr_delete(ol);
debug = debugmode;
goto have_outstanding;
}
ln_log(LNLOG_SINFO, 2, "XOVER failed, trying XHDR");
}
xsnprintf(lineout, SIZE_lineout, "XHDR Message-ID %lu-%lu\r\n", server,
last);
putaline();
if (nntpreply(current_server) == 221) {
debug--;
while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
unsigned long art;
char *t;
art = strtoul(l, &t, 10);
if (t && isspace((unsigned char)*t)) {
while (isspace((unsigned char)*t))
t++;
if (art >= server && art <= last && stat(lookup(t), &st) != 0) {
stufftoget[outstanding] = art;
outstanding++;
if (verbose > 2)
printf("%s: will fetch %lu (%s)\n", g->name, art, t);
}
}
}
debug = debugmode;
if (!l) {
free(stufftoget);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout after XHDR",
current_server->name, g->name);
return 0;
}
} else {
free(stufftoget);
return server;
}
if (outstanding == 0) {
free(stufftoget);
return last + 1;
}
syslog(LOG_INFO, "%s: will fetch %ld article%s", g->name, outstanding, PLURAL(outstanding));
if (verbose > 1)
printf("%s: will fetch %ld article%s\n", g->name, outstanding, PLURAL(outstanding));
if (minlines || maxlines) {
xsnprintf(lineout, SIZE_lineout, "XHDR Lines %lu-%lu\r\n", server,
last);
putaline();
if (nntpreply(current_server) == 221) {
while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
unsigned long art;
long lines = 0;
char *t;
art = strtoul(l, &t, 10);
if (t)
lines = strtol(t, NULL, 10);
for (j = 0; j < outstanding; j++) {
if (art == stufftoget[j])
break;
}
if (j < outstanding)
if ((minlines && lines < minlines)
|| (maxlines && lines > maxlines)) {
stufftoget[j] = 0;
syslog(LOG_INFO, "%s: Killed article %lu: %ld line%s",
g->name, art, lines, PLURAL(lines));
if (verbose > 2)
printf("%s: Killed article %lu: %ld line%s.\n",
g->name, art, lines, PLURAL(lines));
killed++;
}
}
if (!l) { /* timeout */
free(stufftoget);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout after XHDR Lines",
current_server->name, g->name);
return 0;
}
}
}
if (maxbytes) {
xsnprintf(lineout, SIZE_lineout, "XHDR Bytes %lu-%lu\r\n", server,
last);
putaline();
if (nntpreply(current_server) == 221) {
while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
unsigned long art, bytes = 0;
char *t;
art = strtoul(l, &t, 10);
if (t)
bytes = strtoul(t, NULL, 10);
for (j = 0; j < outstanding; j++) {
if (art == stufftoget[j])
break;
}
if (j < outstanding && (bytes > maxbytes)) {
stufftoget[j] = 0;
syslog(LOG_INFO, "%s: Killed article %lu (%lu > %lu bytes)",
g->name, art, bytes, maxbytes);
if (verbose > 2)
printf("%s: Killed article %lu (%lu > %lu bytes).\n",
g->name, art, bytes, maxbytes);
killed++;
}
}
if (!l) { /* timeout */
free(stufftoget);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout after XHDR Bytes",
current_server->name, g->name);
return 0;
}
}
}
if (localmaxage) {
xsnprintf(lineout, SIZE_lineout, "XHDR Date %lu-%lu\r\n", server, last);
putaline();
if (nntpreply(current_server) == 221) {
debug--;
while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
unsigned long art;
long aage = 0;
char *t;
art = strtoul(l, &t, 10);
if (t)
aage = age(t);
for (j = 0; j < outstanding; j++) {
if (art == stufftoget[j])
break;
}
if (j < outstanding && (aage > localmaxage)) {
stufftoget[j] = 0;
syslog(LOG_INFO,
"%s: Killed article %lu (%ld > %d = localmaxage)", g->name,
art, aage, localmaxage);
if (verbose > 2)
printf("%s: Killed article %lu (%ld > %d = localmaxage).\n",
g->name, art, aage, localmaxage);
killed++;
}
}
if (!l) { /* timeout */
free(stufftoget);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout after XHDR Date",
current_server->name, g->name);
return 0;
}
debug = debugmode;
}
}
/* now we have a list of articles in stufftoget[] */
/* let's get the header and possibly bodies of these */
have_outstanding:
for (i = 0; outstanding > 0; i++) {
int takethis = 1;
int requested_body;
const char *cmd;
outstanding--;
if (!stufftoget[i])
continue;
if (stufftoget[i] < server) {
if (verbose > 2)
printf("%s: skipping %lu - not available or too old\n",
g->name, stufftoget[i]);
syslog(LOG_INFO, "%s: skipping %lu - not available or too old",
g->name, stufftoget[i]);
continue;
}
debug = debugmode;
requested_body = ((!filterfile || article_despite_filter) && !delaybody);
cmd = requested_body ? "ARTICLE" : "HEAD";
xsnprintf(lineout, SIZE_lineout, "%s %lu\r\n", cmd, stufftoget[i]);
putaline();
l = mgetaline(nntpin);
/* timeout */
if (!l)
{
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"Server disconnection or timeout before article "
"could be retrieved #1");
free(stufftoget);
return 0;
}
/* check proper reply code */
if (sscanf(l, "%3ld %lu", &n, &h) < 2 || ((n / 10) != 22)) {
if (verbose > 2)
printf("%s %s %lu: reply %s (%ld more up in the air)\n",
g->name, cmd, stufftoget[i], l, outstanding);
syslog(LOG_INFO, "%s %s %lu: reply %s (%ld more up in the air)",
g->name, cmd, stufftoget[i], l, outstanding);
continue;
}
/* anything below this line will have to make sure that data is
* drained properly in case */
debug--;
if (verbose > 2)
printf("%s: receiving article %lu (%ld more up in the air)\n",
g->name, stufftoget[i], outstanding);
for (h = 0; h < 10; h++) {
if (hd[h])
free(hd[h]);
hd[h] = critstrdup("", "getgroup");
}
c = NULL;
n = 9; /* "other" header */
while ((l = getfoldedline(nntpin, mgetaline)) && *l && strcmp(l, ".")) {
/* regexp pattern matching */
if (filterfile && dofilter(l)) {
killed++;
if (verbose > 2)
printf(".filtered article %lu: match on \"%s\"\n",
stufftoget[i], l);
syslog(LOG_INFO, "filtered article %lu: match on \"%s\"",
stufftoget[i], l);
takethis = 0;
free(l);
l = NULL;
continue;
}
n = 0;
while (strncasecmp(l, hnames[n], strlen(hnames[n])))
n++;
if (n < 9 && hd[n] && *(hd[n]))
/* second occurance of the same recognized header
* is treated as if it was not listed
* in hnames (as "other header") */
n = 9;
hd[n] = critrealloc(hd[n], strlen(hd[n]) + strlen(l) + 2,
"Fetching article header");
if (strlen(hd[n]))
strcat(hd[n], "\n"); /* RATS: ignore */
strcat(hd[n], l); /* RATS: ignore */
if (debugmode > 1 && verbose > 3 && hnames[n] && *hnames[n])
printf("...saw header %s\n", hnames[n]);
free(l);
l = NULL;
} /* end while */
/* if server ended the fetch prematurely, assume we didn't
* request a body to ignore - ignoring would cause timeout
* waiting for data that is never sent.
*
* SourceForge bug 873149, reported 2004-01-08 by Toni Viemerö,
* sourceforge user "skithund" */
if (l == NULL) {
/* timeout - don't flush body */
requested_body = FALSE;
} else if (strcmp(l, ".") == 0 && requested_body) {
ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "%s: %s:%lu: article without blank line after header, format violation",
current_server->name, g->name, stufftoget[i]);
requested_body = FALSE;
}
if (l)
free(l);
if (!takethis) {
if (requested_body) ignore_answer(nntpin);
continue; /* filtered article */
}
debug = debugmode;
if (!l) { /* timeout */
free(stufftoget);
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"Server disconnection or timeout before article "
"could be retrieved #2");
return 0;
}
/* check headers */
for (h = 0; h < 6; h++) {
if (!hd[h] || !*(hd[h])) {
if (verbose)
printf("Discarding article %lu - no %s found\n",
stufftoget[i], hnames[h]);
syslog(LOG_NOTICE,
"Discarding article %lu - no %s found",
stufftoget[i], hnames[h]);
killed++;
if (requested_body) ignore_answer(nntpin);
takethis = 0;
break;
}
}
/* mandatory header missing */
if (!takethis)
continue;
if (localmaxage && age(hd[5]) > localmaxage) {
if (verbose > 2)
printf("Discarding article %lu - older than %d day%s\n",
stufftoget[i], localmaxage, PLURAL(localmaxage));
syslog(LOG_INFO, "Discarding article %lu %s - older than %d day%s",
stufftoget[i], hd[4], localmaxage, PLURAL(localmaxage));
killed++;
if (requested_body) ignore_answer(nntpin);
continue;
}
if (minlines || maxlines) {
char *t;
t = strchr(hd[7], ' ');
if (t) {
n = strtol(t, NULL, 10);
if (minlines && n < minlines) {
if (verbose > 2)
printf("Discarding article %lu - %ld < minlines\n",
stufftoget[i], n);
syslog(LOG_INFO,
"Discarding article %lu %s -- %ld < minlines",
stufftoget[i], hd[4], n);
killed++;
if (requested_body) ignore_answer(nntpin);
continue;
}
if (maxlines && n > maxlines) {
if (verbose > 2)
printf("Discarding article %lu - %ld > maxlines\n",
stufftoget[i], n);
syslog(LOG_INFO,
"Discarding article %lu %s -- %ld > maxlines",
stufftoget[i], hd[4], n);
killed++;
if (requested_body) ignore_answer(nntpin);
continue;
}
}
}
if (crosspostlimit) {
char *t;
t = hd[3];
n = 1; /* number of groups the article is posted to */
while ((t = strchr(t, ',')) != NULL) {
t++;
n++;
}
if (crosspostlimit < n) {
if (verbose > 2)
printf("Discarding article %lu - posted to %ld groups "
"(max. %ld)\n", stufftoget[i], n, crosspostlimit);
syslog(LOG_INFO,
"Discarding article %lu %s - posted to %ld groups "
"(max. %ld)", stufftoget[i], hd[4], n, crosspostlimit);
killed++;
if (requested_body) ignore_answer(nntpin);
continue;
}
}
/* store articles */
f = NULL;
c = lookup(strchr(hd[1], '<')); /* lookup also replaces '/' with '@' */
if (!c) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "lookup of %s failed", hd[1]);
if (requested_body) ignore_answer(nntpin);
continue;
}
if (!stat(c, &st)) {
syslog(LOG_INFO, "article %s already stored", c);
if (requested_body) ignore_answer(nntpin);
continue; /* for some reasons, article is already there */
} else if (errno == ENOENT) {
f = fopen(c, "w");
if (!f) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to create article %s: %m", c);
if (requested_body) ignore_answer(nntpin);
continue;
}
} else {
ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to store article %s: %m", c);
if (requested_body) ignore_answer(nntpin);
continue;
}
for (h = 0; h < 10; h++)
if (h != 8 && hd[h] && *(hd[h]))
fprintf(f, "%s\n", hd[h]);
h = 0;
/* replace tabs and other odd signs with spaces */
while (h < 8) {
char *p1;
char *p2;
p1 = p2 = hd[h];
while (p1 && *p1) {
if (isspace((unsigned char)*p1)) {
*p2 = ' ';
do {
p1++;
} while (isspace((unsigned char)*p1));
} else {
*p2 = *p1++;
}
p2++;
}
*p2 = '\0';
h++;
}
if (fflush(f)) {
(void)fclose(f);
(void)unlink(c);
if (requested_body) ignore_answer(nntpin);
continue;
}
/* generate hardlinks; this procedure also increments g->last */
store(c, f, *hd[3] ? hd[3] + strlen(hnames[3]) : "",
*hd[1] ? hd[1] + strlen(hnames[1]) : "");
if (delaybody) {
if (fclose(f)) {
int e = errno;
(void)truncate(c, 0);
(void)unlink(c);
if (e == ENOSPC)
raise(SIGINT);
} else {
fetched++;
}
continue;
}
if (!requested_body) {
xsnprintf(lineout, SIZE_lineout, "BODY %lu\r\n", stufftoget[i]);
putaline();
l = mgetaline(nntpin);
if (!l) { /* timeout */
(void)fflush(f);
(void)ftruncate(fileno(f), 0);
(void)fclose(f);
unlink(c);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout after BODY %lu",
current_server->name, g->name, stufftoget[i]);
free(stufftoget);
return 0;
}
if (sscanf(l, "%3ld", &n) != 1 || (n / 10 != 22)) {
if (verbose > 2)
printf("BODY %lu: reply %s\n", stufftoget[i], l);
syslog(LOG_NOTICE, "BODY %lu: reply %s", stufftoget[i], l);
(void)fflush(f);
(void)ftruncate(fileno(f), 0);
(void)fclose(f);
unlink(c);
continue;
}
}
debug--;
fputs("\n", f); /* empty line between header and body */
while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".")) {
if (*l == '.')
l++;
clearerr(f);
fputs(l, f);
fputc('\n', f);
if (feof(f)) {
l = NULL;
break;
}
}
debug = debugmode;
fetched++;
if (fflush(f)) {
l = NULL;
}
if (fclose(f)) {
l = NULL;
}
if (l == NULL) { /* article didn't terminate with a .: error */
(void)truncate(c, 0);
(void)unlink(c);
ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
"warning: %s: %s: server disconnect or timeout retrieving article %lu",
current_server->name, g->name, stufftoget[i]);
free(stufftoget);
return 0;
}
}
syslog(LOG_INFO, "%s: %lu article%s fetched (to %lu), %lu killed",
g->name, fetched, PLURAL(fetched), g->last, killed);
if (verbose > 1)
printf("%s: %lu article%s fetched, %lu killed\n",
g->name, fetched, PLURAL(fetched), killed);
free(stufftoget);
return last + 1;
}
/* return 1 == success, 0 == failure */
static int expire_interesting(void) {
DIR *d;
struct dirent *de;
char s[SIZE_s+1];
xsnprintf(s, SIZE_s, "%s/interesting.groups/", spooldir);
d = opendir(s);
if (d == NULL) {
ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot open %s for reading: %m", s);
return 0;
}
while ((de = readdir(d))) {
struct stat st;
xsnprintf(s, SIZE_s, "%s/interesting.groups/%s", spooldir, de->d_name);
if (stat(s, &st) < 0)
continue;
/* reading a newsgroup changes the ctime; if the newsgroup is
newly created, the mtime is changed as well */
if (((st.st_mtime == st.st_ctime) &&
(now - st.st_ctime > (timeout_short * SECONDS_PER_DAY))) ||
(now - st.st_ctime > (timeout_long * SECONDS_PER_DAY))) {
if (verbose > 1)
printf("unsubscribing from %s\n", de->d_name);
syslog(LOG_INFO, "unsubscribing from %s (current time: %ld): "
"ctime age %ld, mtime age %ld", de->d_name, (long)now,
(long)now - st.st_ctime, (long)now - st.st_mtime);
unlink(s);
}
}
(void)closedir(d);
return 1;
}
static void formatserver(char *d, size_t len, const struct server *serv, const char *suffix)
{
if (serv->port == 0 || serv->port == 119)
xsnprintf(d, len, "%s/leaf.node/%s%s", spooldir, serv->name, suffix);
else
xsnprintf(d, len, "%s/leaf.node/%s:%u%s", spooldir, serv->name,
serv->port, suffix);
}
/** get active file from current_server.
* \returns 0 for success, non-zero for error.
*/
static int
nntpactive(struct server *current_server, time_t *stamp)
{
struct stat st;
char *l, *p;
struct stringlist *groups = NULL;
struct stringlist *helpptr = NULL;
char timestr[64]; /* RATS: ignore */
long reply = 0l;
int error, merge = 0;
char s[SIZE_s+1];
int try_xgtitle = 1;
static time_t cur_date;
static int cur_date_init = 0;
time_t t;
if (!cur_date_init) {
cur_date = time(NULL);
cur_date_init = 1;
}
formatserver(s, SIZE_s, current_server, "");
t = time(NULL);
if (active && !forceactive && (stat(s, &st) == 0)) {
if (verbose)
printf("%s: getting new newsgroups\n", current_server->name);
/* to avoid a compiler warning we print out a four-digit year;
* but since we need only the last two digits, we skip them
* in the next line
*/
*stamp = st.st_mtime;
(void)strftime(timestr, sizeof(timestr),
"%Y%m%d %H%M%S", gmtime(&st.st_mtime));
xsnprintf(lineout, SIZE_lineout, "NEWGROUPS %s GMT\r\n", timestr + 2);
putaline();
/* we used to expect 231 here, but some broken servers (MC-link
* Custom News-server V1.06) return 215 instead.
* Just accept any 2XX code as success.
*/
if ((reply = nntpreply(current_server)) < 200 || reply >= 300) {
char *e = lastreply();
if (!e) e = "server disconnect or timeout";
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading new newsgroups failed, reason \"%s\"",
current_server->name, e);
return -1;
}
while ((l = mgetaline(nntpin)) && (strcmp (l, "."))) {
p = l;
while (*p && !isspace((unsigned char)*p))
p++;
if (*p)
*p = '\0';
if (gs_match(current_server->group_pcre, l)) {
merge++;
insertgroup(l, 1, 0, cur_date);
prependtolist(&groups, l);
}
}
if (!l) { /* timeout */
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading new newsgroups failed, server disconnect or timeout.",
current_server->name);
return -1;
}
ln_log(LNLOG_SINFO, LNLOG_CSERVER,
"%s: got %d new newsgroups.",
current_server->name, merge);
if (merge) {
mergegroups(); /* merge groups into active */
merge = 0;
}
helpptr = groups;
if (verbose && helpptr && current_server->descriptions)
printf("%s: getting newsgroup descriptions\n",
current_server->name);
while (helpptr != NULL) {
if (current_server->descriptions) {
error = 0;
if (try_xgtitle) {
xsnprintf(lineout, SIZE_lineout, "XGTITLE %s\r\n",
helpptr->string);
putaline();
reply = nntpreply(current_server);
}
if (!try_xgtitle || reply != 282) {
try_xgtitle = 0;
xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS %s\r\n",
helpptr->string);
putaline();
reply = nntpreply(current_server);
if (reply && (reply != 215))
error = 1;
}
if (!error) {
l = mgetaline(nntpin);
if (l && *l && strcmp(l, ".")) {
p = l;
while (*p && !isspace((unsigned char)*p))
p++;
while (isspace((unsigned char)*p)) {
*p = '\0';
p++;
}
if (reply == 215 || reply == 282)
changegroupdesc(l, *p ? p : NULL);
do {
l = mgetaline(nntpin);
error++;
} while (l && *l && strcmp(l, "."));
if (error > 1) {
current_server->descriptions = 0;
syslog(LOG_WARNING, "warning: %s does not process "
"LIST NEWSGROUPS %s correctly: use nodesc\n",
current_server->name, helpptr->string);
fprintf(stderr, "warning: %s does not process LIST "
"NEWSGROUPS %s correctly: use nodesc\n",
current_server->name, helpptr->string);
}
}
}
} /* if ( current_server->descriptions ) */
helpptr = helpptr->next;
}
freelist(groups);
} else {
ln_log(LNLOG_SINFO, LNLOG_CSERVER,
"%s: getting all newsgroups (debug: active: %s, forceactive: %s)",
current_server->name,
active ? "set" : "nil", forceactive ? "true" : "false");
xsnprintf(lineout, SIZE_lineout, "LIST\r\n");
putaline();
if (nntpreply(current_server) != 215) {
char *e = lastreply();
if (!e) e = "server disconnect or timeout";
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading all newsgroups failed, reason \"%s\".",
current_server->name, e);
return -2;
}
debug--;
while ((l = mgetaline(nntpin)) && (strcmp(l, "."))) {
p = l;
while (*p && !isspace((unsigned char)*p))
p++;
while (isspace((unsigned char)*p)) {
*p = '\0';
p++;
}
if (gs_match(current_server->group_pcre, l)) {
insertgroup(l, 1, 0, cur_date);
}
}
mergegroups();
if (!l) { /* timeout */
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading all newsgroups failed, server disconnect or timeout.",
current_server->name);
return -2;
}
if (current_server->descriptions) {
if (verbose)
printf("%s: getting newsgroup descriptions\n",
current_server->name);
xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS\r\n");
putaline();
l = mgetaline(nntpin);
/* correct reply starts with "215". However, INN 1.5.1 is broken
and immediately returns the list of groups */
if (l) {
if (debug)
syslog(LOG_DEBUG, "<%s", l);
reply = strtol(l, &p, 10);
if ((reply == 215) && (*p == ' ' || *p == '\0')) {
l = mgetaline(nntpin); /* get first description */
} else if (*p != ' ' && *p != '\0') {
int dummy = 0;
/* INN 1.5.1: line already contains description */
(void)dummy;
} else {
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading newsgroups descriptions failed: %s",
current_server->name, l);
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"Workaround: Add \"nodesc = 1\" (without quotes) below the server = %s line.", current_server->name);
return -2;
}
} else {
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading newsgroups descriptions failed: server disconnect or timeout.",
current_server->name);
return -2;
}
while (l && (strcmp(l, "."))) {
p = l;
while (*p && !isspace((unsigned char)*p))
p++;
while (isspace((unsigned char)*p)) {
*p = '\0';
p++;
}
changegroupdesc(l, *p ? p : NULL);
l = mgetaline(nntpin);
}
if (!l) { /* timeout */
ln_log(LNLOG_SERR, LNLOG_CSERVER,
"%s: reading newsgroup descriptions failed, server disconnect or timeout.",
current_server->name);
return -2;
}
}
debug = debugmode;
/* mark active for /this/ server fetched */
{
FILE *f = fopen(s, "a");
if (!f) {
ln_log(LNLOG_SERR, LNLOG_CGROUP,
"cannot open \"%s\": %m", s);
} else {
if (fclose(f))
ln_log(LNLOG_SERR, LNLOG_CGROUP,
"cannot close \"%s\": %m", s);
}
}
}
*stamp = t;
return 0;
}
static int
movetofailed(const char *name) {
char s[SIZE_s + 1];
xsnprintf(s, SIZE_s, "%s/failed.postings/%s", spooldir, name);
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"moving file %s to failed.postings", name);
if (rename(name, s)) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"unable to move failed posting to %s: %m", s);
return -1;
} else {
return 0;
}
}
/*
* post all spooled articles
*
* if all postings succeed, returns 1
* if there are no postings to post, returns 1
* if a posting is strange for some reason, returns 0
* returns -1 if server should be skipped
*/
static int
postarticles(const struct server *current_server)
{
struct stat st;
char *line;
DIR *d;
struct dirent *de;
FILE *f;
int r, haveid, n;
char *p, *q;
int savedir;
n = 0;
savedir = open(".", O_RDONLY);
if (savedir < 0) {
ln_log(LNLOG_SERR, LNLOG_CTOP,
"postarticles: Unable to save current working directory: %m");
return 0;
}
if (chdir(spooldir) || chdir("out.going")) {
ln_log(LNLOG_SERR, LNLOG_CTOP,
"postarticles: Unable to cd to %s/out.going: %m",
spooldir);
fchdir(savedir);
close(savedir);
return 0;
}
d = opendir(".");
if (!d) {
ln_log(LNLOG_SERR, LNLOG_CTOP,
"postarticles: Unable to opendir %s/out.going: %m", spooldir);
fchdir(savedir);
close(savedir);
return 0;
}
while ((de = readdir(d)) != NULL) {
haveid = 0;
f = NULL;
if (!strcmp(".", de->d_name) || !strcmp("..", de->d_name)) {
continue;
}
p = q = NULL;
if ((lstat(de->d_name, &st) != 0)) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"postarticles: cannot stat %s: %m",
de->d_name);
continue;
}
if (!S_ISREG(st.st_mode)) {
ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
"postarticles: %s is not a regular file",
de->d_name);
movetofailed(de->d_name);
continue;
}
if (!(st.st_mode & S_IRUSR)) {
ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
"postarticles: skipping %s, not complete",
de->d_name);
continue;
}
f = fopen(de->d_name, "r");
if (f == NULL) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"postarticles: cannot open file %s for reading: %m",
de->d_name);
movetofailed(de->d_name);
continue;
}
p = fgetheader(f, "Newsgroups:");
q = fgetheader(f, "Message-ID:");
if (p == NULL || q == NULL) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"postarticles: file %s lacks Newsgroups "
"and/or Message-ID header (this cannot happen)",
de->d_name);
movetofailed(de->d_name);
goto free_cont;
}
if (!current_server->post_anygroup) {
char *pp = critstrdup(p, "postarticles");
if (!isgrouponserver(current_server, pp)) {
ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
"%s: postarticles: file %s: only_groups_pcre excluded "
"or server does not carry newsgroups %s",
current_server->name, de->d_name, p);
free(pp);
goto free_cont;
}
free(pp);
}
haveid = ismsgidonserver(current_server, q);
switch(haveid) {
case 0:
ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
"%s: postarticles: trying to post file %s Message-ID %s",
current_server->name, de->d_name, q);
xsnprintf(lineout, SIZE_lineout, "POST\r\n");
putaline();
r = nntpreply(current_server);
if (r != 340) {
char *e = lastreply();
if (e == NULL) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: postarticles: server disconnect or timeout"
" while trying to post file %s)",
current_server->name, de->d_name);
goto free_ret;
}
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: postarticles: server replied \"%s\" to our POST command, skipping server",
current_server->name, e);
goto free_ret;
} else {
/* server replied 340, it is willing to accept our article */
int postok = 0;
debug--;
while ((line = getaline(f)) != NULL) {
/* can't use putaline() here because
line length of lineout is restricted */
if (line[0] == '.')
fputc('.', nntpout);
fputs(line, nntpout);
fputs("\r\n", nntpout);
};
fflush(nntpout);
debug = debugmode;
xsnprintf(lineout, SIZE_lineout, ".\r\n");
putaline();
line = mgetaline(nntpin);
if (!line) { /* timeout: posting failed */
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: postarticles: server disconnect or timeout"
" after sending article %s)",
current_server->name, de->d_name);
goto free_ret;
}
line = critstrdup(line, "postarticles");
if (strncmp(line, "240", 3) == 0) {
postok = 1;
} else if (ismsgidonserver(current_server, q) == 1) {
syslog(LOG_NOTICE,
"%s: postarticles: posting resulted in \"%s\", "
"but article is available upstream, assuming OK.",
current_server->name, line);
printf("%s: postarticles: posting resulted in \"%s\",\n"
"but article is available upstream, assuming OK.",
current_server->name, line);
postok = 1;
}
if (postok) {
if (verbose > 2)
printf("%s: postarticles: POST of article %s OK\n",
current_server->name, de->d_name);
n++;
if (unlink(de->d_name)) {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"postarticles: unable to unlink posted article %s: %m",
de->d_name);
}
} else {
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: postarticles: Article file %s Message-ID %s"
" was rejected: \"%s\"",
current_server->name, de->d_name, q, line);
movetofailed(de->d_name);
}
free(line);
}
haveid = 0;
break;
case 1:
syslog(LOG_INFO, "%s: postarticles: Message-ID of %s already in use"
" upstream -- article discarded\n",
current_server->name, de->d_name);
if (verbose > 2)
printf("%s: postarticles: %s already available upstream\n",
current_server->name, de->d_name);
unlink(de->d_name);
break;
case -1:
ln_log(LNLOG_SERR, LNLOG_CARTICLE,
"%s: postarticles: skipping server",
current_server->name);
goto free_ret;
}
free_cont:
fclose(f);
if (p) free(p);
if (q) free(q);
continue;
free_ret:
if (p) free(p);
if (q) free(q);
fchdir(savedir);
close(savedir);
closedir(d);
return -1;
} /* while de = readdir */
closedir(d);
if (verbose)
printf("%s: %d article%s posted.\n", current_server->name, n, PLURAL(n));
syslog(LOG_INFO, "%s: %d article%s posted.", current_server->name, n, PLURAL(n));
fchdir(savedir);
close(savedir);
return 1;
}
static int
processupstream(const struct server *serv, time_t stamp)
{
FILE *f;
DIR *d;
struct dirent *de;
struct newsgroup *g;
int havefile;
unsigned long newserver = 0;
char *l;
char *oldfile;
struct stat st1, st2;
int have_st1 = 0;
char s[SIZE_s+1];
int aborting = 0;
struct stringlist *ngs = NULL, *a, *b = NULL;
/* read server info */
formatserver(s, SIZE_s, serv, "");
oldfile = critstrdup(s, "processupstream");
havefile = 0;
if ((f = fopen(s, "r")) != NULL) {
if (fstat(fileno(f), &st1)) {
int e = errno;
ln_log(LNLOG_SERR, LNLOG_CSERVER, "Cannot stat %s: %s", s, strerror(e));
} else {
have_st1 = 1;
}
/* a sorted array or a tree would be better than a list */
ngs = NULL;
debug--;
if (verbose > 1)
printf("%s: reading server info from %s\n", serv->name, s);
syslog(LOG_INFO, "%s: reading server info from %s", serv->name, s);
while (((l = getaline(f)) != NULL) && (strlen(l))) {
a = (struct stringlist *)critmalloc(sizeof(struct stringlist)
+ strlen(l),
"Reading server info");
strcpy(a->string, l); /* RATS: ignore */
a->next = NULL;
if (ngs == NULL)
ngs = a;
else
b->next = a;
b = a;
}
havefile = 1;
debug = debugmode;
fclose(f);
}
xsnprintf(s, SIZE_s, "%s/interesting.groups", spooldir);
d = opendir(s);
if (!d) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "opendir %s: %m", s);
free(oldfile);
return -1;
}
formatserver(s, SIZE_s, serv, "~");
if (stat(s, &st2) == 0 && (!have_st1 || st2.st_mtime >
st1.st_mtime) && (f = fopen(s, "r"))) {
int e = 0;
/* roll in changes of a previous crash */
syslog(LOG_INFO, "Merging in %s from previous run", s);
if (verbose > 1)
printf("Merging in %s from previous run\n", s);
while (((l = getaline(f)) != NULL)) {
char *p, *t = strchr(l, ' ');
if (!t || !*++t)
continue;
(void)strtoul(t, &p, 10);
if (p && !*p)
replaceinlist(&ngs, l, (size_t)(t-l));
}
(void)fclose(f); /* read-only file, assume no error */
xsnprintf(s, SIZE_s, "%s/leaf.node/%s.new", spooldir, serv->name);
f = fopen(s, "w");
if (!f) {
e = 1;
} else {
a = ngs;
while (a && a->string && !ferror(f)) {
(void)fputs(a->string, f);
(void)fputc('\n', f);
a = a->next;
}
if (fclose(f))
e = 1;
}
if (e) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "open %s: %m", s);
(void)unlink(s);
return -1;
}
if (rename(s, oldfile)) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "rename %s -> %s: %m", s, oldfile);
(void)unlink(s);
return -1;
}
}
formatserver(s, SIZE_s, serv, "~");
(void)unlink(s);
f = fopen(s, "w");
if (f == NULL) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "Could not open %s for writing: %s",
s, strerror(errno));
} else {
/* make sure that at least SERVERINFO~ is complete */
if (mysetvbuf(f, NULL, _IOLBF, 4096)) {
/* try to at least use unbuffered then */
mysetvbuf(f, NULL, _IONBF, 0);
}
}
while ((de = readdir(d))) {
if (isalnum((unsigned char)*(de->d_name))) {
g = findgroup(de->d_name);
if (g != NULL) {
unsigned long newhigh;
xsnprintf(s, SIZE_s, "%s ", g->name);
newhigh = 1ul;
l = havefile ? findinlist(ngs, s) : NULL;
if (l && *l) {
char *t;
l = strchr(l, ' ');
if (l) {
newhigh = strtoul(l, &t, 10);
if (t == l || *t)
newhigh = 1ul;
}
}
newserver = getgroup(serv, g, newhigh);
/* run this independent of delaybody mode, because
* the admin may have switched delaybody off recently,
* and we still want users to be able to retrieve
* articles. */
if (newserver) newhigh = newserver;
if (f != NULL && newhigh > 0) {
fprintf(f, "%s %lu\n", g->name, newhigh);
}
if (!newserver) {
fprintf(stderr, "Warning: aborting fetch from %s due to previous condition.\n", serv->name);
syslog(LOG_WARNING, "Warning: aborting fetch from %s due to previous condition.", serv->name);
aborting = 1;
break;
}
} else {
if (verbose > 1)
printf("%s not found in groupinfo file\n", de->d_name);
syslog(LOG_NOTICE, "%s not found in groupinfo file", de->d_name);
}
} /* if isalnum */
} /* while readdir */
closedir(d);
if (f != NULL) {
int ren = 1;
formatserver(s, SIZE_s, serv, "~");
if (ferror(f))
ren = 0;
if (fflush(f))
ren = 0;
if (fclose(f))
ren = 0;
if (!aborting) {
if (ren) {
struct utimbuf ut;
if (rename(s, oldfile)) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot rename %s to %s: %m",
s, oldfile);
}
ut.modtime = ut.actime = stamp;
if (utime(oldfile, &ut)) {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot set proper time for %s to %lu: %m",
s, (unsigned long)stamp);
}
} else {
ln_log(LNLOG_SERR, LNLOG_CSERVER, "write error on %s, old version of %s kept",
s, oldfile);
}
}
}
free(oldfile);
freelist(ngs);
return 0;
}
/*
* checks whether all newsgroups have to be retrieved anew
* returns 0 if yes, time of last update if not
* mtime is the time when active was fetched fully
* atime is the time when active was last updated
*/
static time_t
checkactive(void)
{
struct stat st;
char *s = activeread();
if (stat(s, &st)) {
free(s);
return 0;
}
if ((now - st.st_mtime) < (timeout_active * SECONDS_PER_DAY)) {
if (debugmode)
syslog(LOG_DEBUG,
"Last LIST done %d seconds ago: NEWGROUPS\n",
(int)(now - st.st_mtime));
free(s);
return st.st_atime;
} else {
if (debugmode)
syslog(LOG_DEBUG, "Last LIST done %d seconds ago: LIST\n",
(int)(now - st.st_mtime));
free(s);
return 0;
}
}
static int
updateactive(void)
{
struct stat st;
struct utimbuf buf;
FILE *f;
char *s = activeread();
int rc = 0;
if (stat(s, &st)) {
/* active.read probably doesn't exist */
(void)unlink(s); /* delete it in case it's junk */
if ((f = fopen(s, "w")) != NULL) {
if (fsync(fileno(f))) rc = -1;
if (fclose(f)) rc = -1;
} else {
/* f == NULL, open error */
rc = -1;
}
} else {
buf.actime = (now < st.st_atime) ? st.st_atime : now;
/* now < update may happen through HW failures */
buf.modtime = st.st_mtime;
if (utime(s, &buf)) {
rc = -1;
}
}
if (rc)
ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot update or create %s: %m", s);
free(s);
return rc;
}
static void
error_refetch(const char *e) {
ln_log(LNLOG_SERR, LNLOG_CTOP,
"ERROR: FETCHNEWS MUST REFETCH THE WHOLE ACTIVE FILE NEXT RUN.");
ln_log(LNLOG_SERR, LNLOG_CTOP, "REASON: %s", e);
}
/** re-add non-expiring groups to active */
static void
addnonexpiring(void)
{
struct stringlist *t, *l = get_grouplist();
int expdays;
for (t=l; t != NULL; t = t->next) {
char *x = t->string;
if ((expdays = lookup_expiredays(x)) >= 0) {
if (expdays == 0 || !(expdays = lookup_expire(x)))
expdays = expire;
} else {
expdays = -1;
}
if (expdays == -1) {
insertgroup(x, 0, 0, 0);
}
}
freelist(l);
}
int
main(int argc, char **argv)
{
/* the volatile keyboard avoids clobbering by siglongjmp */
volatile time_t lastrun;
volatile int rc = 0, skip_servers = 0;
volatile int anypost = 0, waitchild = 0, quiet;
struct server *current_server;
volatile int need_refetch = 0;
int option, reply;
pid_t pid;
verbose = quiet = 0;
postonly = waitchild = 0;
myopenlog("fetchnews");
if (!initvars(argv[0]))
exit(1);
while ((option = getopt(argc, argv, "Pfhlnvx:qw")) != -1) {
if (option == 'v') {
verbose++;
quiet = 0;
} else if (option == 'h') {
usage();
exit(EXIT_SUCCESS);
} else if (option == 'x') {
char *nptr, *endptr;
nptr = optarg;
endptr = NULL;
extraarticles = strtoul(nptr, &endptr, 0);
if (!nptr || !*nptr || !endptr || *endptr || !extraarticles) {
usage();
exit(1);
}
syslog(LOG_NOTICE, "fetchnews: run with option -x %lu", extraarticles);
} else if (option == 'l') {
usesupplement = 0; /* don't use supplementary servers */
} else if (option == 'n') {
noexpire = 1;
} else if (option == 'f') {
forceactive = 1;
} else if (option == 'P') {
postonly = 1;
} else if (option == 'q') {
verbose = 0;
quiet = 1;
} else if (option == 'w') {
waitchild = 1;
} else {
usage();
exit(1);
}
}
/* Set line buffering to ensure that logging gets displayed promptly. */
if (mysetvbuf(stdout, NULL, _IOLBF, 4096)) {
/* Try to at least use unbuffered then */
mysetvbuf(stdout, NULL, _IONBF, 0);
}
now = time(NULL);
umask(2);
if (!readconfig(0)) {
fprintf(stderr, "Reading configuration failed, exiting "
"(see syslog for more information).\n");
freeconfig();
exit(1);
}
if (debugmode)
syslog(LOG_DEBUG, "leafnode %s: verbosity level is %d, debugmode is %d",
version, verbose, debugmode);
if (verbose || debugmode) {
printf("leafnode %s: verbosity level is %d, debugmode is %d\n",
version, verbose, debugmode);
if (verbose > 1 && noexpire) {
printf("Don't automatically unsubscribe unread newsgroups.\n");
}
}
if (forceactive) {
ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
"Forced active fetch requested from command-line (option -f).");
}
if (try_lock(timeout_lock)) {
ln_log(LNLOG_SERR, LNLOG_CTOP,
"Cannot obtain lock file, aborting.");
freeconfig();
exit(1);
}
if (!postonly) {
readactive();
if (!active) {
addnonexpiring();
fakeactive(); /* we need proper lowwater/highwater marks
for the groups that exist */
ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
"Forced active fetch after trouble reading active file.");
forceactive = 1;
}
readfilter(filterfile);
}
lastrun = 0;
if (!postonly && !forceactive) {
lastrun = checkactive();
if (!lastrun) {
ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Active has not been fetched completely in previous run");
ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "or has never been fetched, forcing active fetch.");
forceactive = 1;
}
}
if (!postonly && !forceactive) {
int staterror;
struct stat st;
char s[SIZE_s+1];
xsnprintf(s, SIZE_s, "%s/leaf.node/groupinfo", spooldir);
if ((staterror = stat(s, &st)) && errno != ENOENT) {
/* this should happen only when the problem occurs after
* readactive() above, but needs to be checked nonetheless */
ln_log(LNLOG_SERR, LNLOG_CTOP, "Cannot open %s: %m", s);
unlink(lockfile);
freeconfig();
exit(EXIT_FAILURE);
}
if (staterror || st.st_size < 7) {
ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
"Groupinfo file %s not present or too small, "
"forcing active fetch.", s);
forceactive = 1;
}
}
if (forceactive) {
oldactive = active;
oldactivesize = activesize;
active = NULL;
activesize = 0;
if (killactiveread()) exit(1);
addnonexpiring();
}
if (mysigact(SIGINT, 0, sig_int, SIGALRM) != 0)
fprintf(stderr, "Can't catch SIGINT.\n");
if (mysigact(SIGTERM, 0, sig_int, SIGALRM) != 0)
fprintf(stderr, "Can't catch SIGTERM.\n");
if (mysigact(SIGPIPE, 0, sig_int, SIGALRM) != 0)
fprintf(stderr, "Can't catch SIGPIPE.\n");
{
int sig = sigsetjmp(jmpbuffer, 1);
if (sig != 0) {
ln_log(LNLOG_SWARNING, LNLOG_CTOP,
"fetchnews: caught signal %d, shutting down.", sig);
nntpquit();
if (!rc)
rc = 2;
if (forceactive) {
error_refetch("caught signal that caused a premature abort.");
need_refetch = 1;
}
skip_servers = 1; /* in this case, jump the while ... loop */
} else {
canjump = 1;
}
}
/* remove groups that haven't been read in a long time */
if (!noexpire)
expire_interesting();
mgetaline_settimeout(timeout_fetchnews);
/* main server loop */
for (current_server = servers;
!skip_servers && current_server;
current_server = current_server->next) {
if (verbose) {
if (current_server->port)
printf("%s: connecting to port %u...\n",
current_server->name, current_server->port);
else
printf("%s: connecting to port nntp...\n",
current_server->name);
}
fflush(stdout);
reply = nntpconnect(current_server);
if (reply) {
int r2;
if (verbose) {
int namlen = strlen(current_server->name);
printf("%s: connected.\n", current_server->name);
if (stat_is_evil)
printf("%s: server software does not implement\n"
"%*s STAT <message-ID> properly,\n"
"%*s using workaround with HEAD instead,\n"
"%*s at the expense of bandwidth.\n",
current_server->name, namlen, "", namlen, "", namlen, "");
else
printf("%s: using STAT <message-ID> command.\n",
current_server->name);
}
if (current_server->username)
if (!authenticate(current_server) && current_server->password)
ln_log(LNLOG_SERR, LNLOG_CTOP,
"error: may have been caused by premature authentication and be rather harmless.");
/* Get INN's nnrpd on the phone */
xsnprintf(lineout, SIZE_lineout, "MODE READER\r\n");
putaline();
r2 = nntpreply(current_server);
if (r2 < 400) reply = r2;
if (reply == 498) {
ln_log(LNLOG_SERR, LNLOG_CTOP, "%s: protocol error after sending mode reader",
current_server->name);
} else {
if (reply == 200 && current_server->nopost == 0) {
anypost = 1;
if (-1 == postarticles(current_server))
{
nntpquit();
goto connfail;
}
} else if (verbose) {
printf("Not posting to %s: ", current_server->name);
if (reply != 200)
printf("non-permission ");
if (current_server->nopost)
printf("nopost-set ");
printf("\n");
}
if (!date_is_evil)
check_date(current_server);
if (!postonly && !current_server->noread) {
time_t stamp = lastrun;
/* get list of newsgroups or new newsgroups */
if (current_server->updateactive) {
int r;
if ((r = nntpactive(current_server, &stamp))) {
if (forceactive || r == -2) {
error_refetch("obtaining the active file failed.");
need_refetch = 1;
}
rc = 1;
}
} else {
if (verbose)
printf("%s: not attempting to update newsgroups list\n",
current_server->name);
}
processupstream(current_server, stamp);
}
}
nntpquit();
if (verbose)
printf("%s: conversation completed, disconnected.\n", current_server->name);
} else { /* reply = nntpconnect */
if (verbose)
printf("%s: connection failed.\n", current_server->name);
connfail:
if (forceactive && current_server->updateactive && !postonly && !current_server->noread) {
error_refetch("needed to fetch the active list from the server but couldn't connect.");
need_refetch = 1;
}
rc = 2;
}
if (!usesupplement)
break;
}
mergegroups(); /* just in case we were interrupted while downloading
the list. */
(void)mysigact(SIGINT, 0, SIG_IGN, 0); /* do not siglongjmp any more */
(void)mysigact(SIGTERM, 0, SIG_IGN, 0); /* do not siglongjmp any more */
(void)mysigact(SIGPIPE, 0, SIG_IGN, 0); /* SIGPIPE should not happen below */
if (rc != 0 && oldactive) {
/* restore old active data to keep low/high marks */
unsigned long i;
for (i = 0; i < oldactivesize; i++) {
insertgroup(oldactive[i].name, 0, 0, 0);
}
mergegroups();
killactiveread();
}
freeactive(oldactive);
oldactive = NULL;
oldactivesize = 0;
if (anypost == 0 && skip_servers == 0 && rc != 2) {
const char *e = "WARNING: found no server with posting permission!";
if (!quiet)
fprintf(stderr, "%s\n", e);
syslog(LOG_WARNING, "%s", e);
}
if (rc == 2) {
const char *e = "WARNING: some servers have not been queried!";
if (!quiet)
fprintf(stderr, "%s\n", e);
syslog(LOG_WARNING, "%s", e);
}
if (fflush(stdout)) {
/* to avoid double logging of stuff */
ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot flush standard output: %m");
}
{
/* OK, we do a pipe trick to hold the child until the parent
* handed over the lock file (this must happen so it does not
* inadvertently get removed as stale). The child reads from the
* pipe and is blocked until the parent has written to the pipe.
* The parent writes to the pipe after it has handed over the
* lock.
*/
int pfd[2], pipeok;
pipeok = pipe(pfd);
if (!postonly) {
/* only update active.read when have read active from all servers */
if (writeactive()) {
ln_log(LNLOG_SERR, LNLOG_CTOP, "Error writing groupinfo.");
error_refetch("cannot write groupinfo file.");
rc = 1;
/* mark for refetch */
killactiveread();
} else {
/* active written successfully */
if (updateactive()) {
error_refetch("cannot update active.read file.");
need_refetch = 1;
rc = 1;
}
if (need_refetch) {
/* mark for refetch */
killactiveread();
}
}
#ifdef HAVE_WORKING_FORK
pid = waitchild ? -1 : fork();
#else
pid = -1; waitchild = 1;
#endif
switch (pid) {
case -1:
if (!waitchild)
syslog(LOG_NOTICE, "fork: %m, running on parent schedule.");
if (verbose)
printf("updating overview data in the foreground...\n");
fixxover();
unlink(lockfile);
if (verbose)
printf("done.\n");
break;
case 0:
(void)setsid();
if (debugmode)
syslog(LOG_DEBUG, "Process forked.");
if (pipeok == 0) {
/* wait for parent to hand over the lock */
char buf[4];
(void)close(pfd[1]);
/* we don't REALLY check for errors here, the worst thing
* that could happen (in case of a kernal bug...) to
* us is that we start early and delete the parent's
* lock file before the parent handed it over -- but
* at that time, we'll then exit and everything is
* in order, with just one bogus message in the log.
*/
while (read(pfd[0], buf, sizeof(buf)) < 0) {
if (errno != EAGAIN && errno != EINTR)
break;
}
(void)close(pfd[0]); }
fixxover();
freeactive(active);
if (unlink(lockfile))
ln_log(LNLOG_SERR, LNLOG_CTOP,
"unlink(\"%s\"): %m", lockfile);
if (debugmode)
syslog(LOG_DEBUG, "Process done.");
freeconfig();
_exit(0);
break;
default:
{
int lock_ok = handover_lock(pid);
if (verbose) puts("Started process to update overview data in the background.\nNetwork activity has finished.");
if (pipeok == 0) {
/* tell child it has the lock */
(void)close(pfd[0]);
(void)writes(pfd[1], "GO");
(void)close(pfd[1]);
}
if (lock_ok) {
/* could not hand over lock file to child, so wait until it
dies */
syslog(LOG_NOTICE, "could not hand over lockfile to child %lu: %m, "
"waiting until child is done.",
(unsigned long)pid);
(void)waitpid(pid, NULL, 0);
} else {
syslog(LOG_INFO, "child has process ID %lu",
(unsigned long)pid);
}
break;
}
}
} else { /* if (postonly) */
unlink(lockfile);
}
}
freeactive(active);
freeconfig();
exit(rc);
}
syntax highlighted by Code2HTML, v. 0.9.1