/* fetchnews -- post articles to and get news from upstream server(s) Written by Arnt Gulbrandsen and copyright 1995 Troll Tech AS, Postboks 6133 Etterstad, 0602 Oslo, Norway, fax +47 22646949. Modified by Cornelius Krasel and Randolf Skerka . Copyright of the modifications 1997. Modified by Kent Robotti . Copyright of the modifications 1998. Modified by Markus Enzenberger . Copyright of the modifications 1998. Modified by Cornelius Krasel . Copyright of the modifications 1998, 1999. Modified by Ralf Wildenhues . Copyright of the modifications 2002. Modified by Jonathan Larmour . Copyright of the modifications 2002. Modified by Richard van der Hoff Copyright of the modifications 2002. Enhanced and modified by Matthias Andree . Copyright of the modifications 2000 - 2006. See file COPYING for restrictions on the use of this software. */ #include "leafnode.h" #include "fetchnews.h" #include "mastring.h" #include "ln_log.h" #include "mysigact.h" #include #include #include "system.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "groupselect.h" int verbose = 0; int debug = 0; static time_t now; /* Variables set by command-line options which are specific for fetch */ static unsigned long extraarticles = 0; static int usesupplement = 1; static int postonly = 0; /* if 1, don't read files from upstream */ static int noexpire = 0; /* if 1, don't automatically unsubscribe */ static int forceactive = 0; /* if 1, reread complete active file */ static sigjmp_buf jmpbuffer; static volatile sig_atomic_t canjump; static int age( /*@null@*/ const char *date); static int postarticles(const struct server *current_server); static void ignore_answer(FILE * f) { char *l; while (((l = mgetaline(f)) != NULL) && strcmp(l, ".")); } static RETSIGTYPE sig_int(int signo) { if (canjump == 0) return; /* ignore unexpected signals */ if (signo == SIGINT || signo == SIGTERM) { canjump = 0; alarm(0); siglongjmp(jmpbuffer, signo); } } static void usage(void) { fprintf(stderr, "Usage: fetchnews [-q] [-v] [-x #] [-l] [-n] [-f] [-P] [-w]\n" " -q: quiet, suppress some warnings, cancels -v\n" " -v: more verbose (may be repeated), cancels -q\n" " -x: check for # extra articles in each group\n" " -l: do not use supplementary servers\n" " -n: do not automatically expire unread groups\n" " -f: force reload of groupinfo file\n" " -P: only post outgoing articles, don't fetch any\n" " -w: wait, run XOVER updater in foreground\n"); } /** * check whether any of the newsgroups is on server * return TRUE if yes, FALSE otherwise */ static int isgrouponserver(const struct server *current_server, char *newsgroups /** string will be destroyed! */) { char *p, *q; int retval; if (!newsgroups) return FALSE; retval = FALSE; p = newsgroups; do { q = strchr(p, ','); if (q) *q++ = '\0'; switch (gs_match(current_server->group_pcre, p)) { case 1: xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", p); putaline(); if (nntpreply(current_server) == 211) { if (debug > 1) syslog(LOG_DEBUG, "%s is matched by only_groups_pcre and is on server", p); retval = TRUE; } break; case 0: if (debug > 1) syslog(LOG_DEBUG, "%s not matched by only_group_pcre", p); if (current_server->only_groups_match_all) { if (debug > 1) syslog(LOG_DEBUG, "not posting article to this server, " "only_groups_match_all is set"); return FALSE; } break; default: break; } p = q; if (p) while (*p && isspace((unsigned char)*p)) p++; } while (q); return retval; } /* * check whether message-id is on server - msgid is expected * to contain the angle brackets, i. e. * return 1 if yes, 0 if no, -1 for error */ static int ismsgidonserver(const struct server *current_server, char *msgid) { int r; if (!msgid) return 0; xsnprintf(lineout, SIZE_lineout, "%s %s\r\n", stat_is_evil ? "HEAD" : "STAT", msgid); putaline(); r = nntpreply(current_server); if (r == 498 && lastreply() == NULL) return -1; if (r >= 220 && r <= 223) { if (stat_is_evil) ignore_answer(nntpin); return 1; } else return 0; } int age( /*@null@*/ const char *date) { char monthname[4]; /* RATS: ignore */ int month; int year; int day; const char *d; time_t tmp; struct tm time_struct; if (!date) return 1000; /* large number: OLD */ d = date; if (!(strncasecmp(d, "date:", 5))) d += 5; while (isspace((unsigned char)*d)) d++; if (isalpha((unsigned char)*d)) { while (*d && !isspace((unsigned char)*d)) /* skip "Mon" or "Tuesday," */ d++; } /* RFC 822 says we have 1*LWSP-char between tokens */ while (isspace((unsigned char)*d)) d++; /* parsing with sscanf leads to crashes */ day = atoi(d); while (isdigit((unsigned char)*d) || isspace((unsigned char)*d)) d++; if (!isalpha((unsigned char)*d)) { syslog(LOG_INFO, "Unable to parse %s", date); return 1003; } monthname[0] = *d++; monthname[1] = *d++; monthname[2] = *d++; monthname[3] = '\0'; if (strlen(monthname) != 3) { syslog(LOG_INFO, "Unable to parse month in %s", date); return 1004; } while (isalpha((unsigned char)*d)) d++; while (isspace((unsigned char)*d)) d++; year = atoi(d); if ((year < 1970) && (year > 99)) { syslog(LOG_INFO, "Unable to parse year in %s", date); return 1005; } else if (!(day > 0 && day < 32)) { syslog(LOG_INFO, "Unable to parse day in %s", date); return 1006; } else { if (!strcasecmp(monthname, "jan")) month = 0; else if (!strcasecmp(monthname, "feb")) month = 1; else if (!strcasecmp(monthname, "mar")) month = 2; else if (!strcasecmp(monthname, "apr")) month = 3; else if (!strcasecmp(monthname, "may")) month = 4; else if (!strcasecmp(monthname, "jun")) month = 5; else if (!strcasecmp(monthname, "jul")) month = 6; else if (!strcasecmp(monthname, "aug")) month = 7; else if (!strcasecmp(monthname, "sep")) month = 8; else if (!strcasecmp(monthname, "oct")) month = 9; else if (!strcasecmp(monthname, "nov")) month = 10; else if (!strcasecmp(monthname, "dec")) month = 11; else { syslog(LOG_INFO, "Unable to parse %s", date); return 1001; } if (year < 70) /* years 2000-2069 in two-digit form */ year += 100; else if (year > 1970) /* years > 1970 in four-digit form */ year -= 1900; memset(&time_struct, 0, sizeof(time_struct)); time_struct.tm_sec = 0; time_struct.tm_min = 0; time_struct.tm_hour = 0; time_struct.tm_mday = day; time_struct.tm_mon = month; time_struct.tm_year = year; time_struct.tm_isdst = 0; tmp = mktime(&time_struct); if (tmp == -1) return 1002; return ((now - tmp) / SECONDS_PER_DAY); } } /* * Get body of a single message of which the header has already been * downloaded and append it to the file with the header. * Returns 0 if file could not be retrieved, 1 otherwise. */ static int getbody_insitu(const struct server *current_server, struct newsgroup *group, unsigned long id) { const char *c; int rc = 0; char *l; char *messageid; FILE *f; char s[SIZE_s+1]; off_t pos; if (!chdirgroup(group->name, FALSE)) return 0; /* extract message-id: header */ xsnprintf(s, SIZE_s, "%lu", id); messageid = getheader(s, "Message-ID:"); if (!messageid) return 0; /* check whether we can retrieve the body */ if (verbose > 2) printf("%s: BODY %s\n", group->name, messageid); xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid); putaline(); if (nntpreply(current_server) != 222) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. No response", group->name, messageid); rc = 0; goto getbody_bail; } xsnprintf(s, SIZE_s, "%lu", id); c = lookup(messageid); if (!(f = fopen(c, "a"))) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: cannot open %s for appending", group->name, c); rc = 0; goto getbody_bail; } pos = ftell(f); fputc('\n', f); /* blank line -- separate header and body */ debug--; while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) { if (*l == '.') ++l; fputs(l, f); fputc('\n', f); } debug = debugmode; if (!l) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. " "Transmission interrupted.", group->name, messageid); ftruncate(fileno(f), pos); fclose(f); rc = 0; goto getbody_bail; } /* abort when disk is full */ if (fclose(f) && errno == ENOSPC) { truncate(s, pos); raise(SIGINT); return 0; } rc = 1; getbody_bail: if (messageid) free(messageid); return rc; } static int getbody_newno(const struct server *current_server, struct newsgroup *group, unsigned long id) { const char *c; int rc = 0; char *l; char *p, *q; char *messageid; char *newsgroups; /* I hope this is enough */ char *xref; FILE *f, *g; char s[SIZE_s+1]; if (!chdirgroup(group->name, FALSE)) return 0; /* extract message-id: and xref: headers */ xsnprintf(s, SIZE_s, "%lu", id); if (!(f = fopen(s, "r"))) { syslog(LOG_INFO, "%s: cannot open %s for reading -- possibly expired", group->name, s); return 1; /* pretend to have read file successfully so that it is purged from the list */ } messageid = NULL; newsgroups = NULL; xref = NULL; debug--; while ((l = getaline(f)) != NULL) { if (!newsgroups && !strncmp(l, "Newsgroups:", 11)) { p = l+11; SKIPLWS(p); newsgroups = critstrdup(p, "getbody"); } if (!messageid && !strncmp(l, "Message-ID:", 11)) { p = l+11; SKIPLWS(p); messageid = critstrdup(p, "getbody"); } if (!xref && !strncmp(l, "Xref:", 5)) { p = l + 5; SKIPLWS(p); xref = critstrdup(p, "getbody"); } } debug = debugmode; fclose(f); /* check whether we can retrieve the body */ if (verbose > 2) printf("%s: BODY %s\n", group->name, messageid); xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid); putaline(); if (nntpreply(current_server) != 222) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. No response", group->name, messageid); rc = 0; goto getbody_bail; } xsnprintf(s, SIZE_s, "%lu", id); c = lookup(messageid); log_unlink(c, 0); /* make space for new file */ if (!(f = fopen(c, "w"))) { ln_log(LNLOG_SERR, LNLOG_CGROUP, "%s: cannot open %s for writing", group->name, c); link(s, c); /* if we can't open new file restore old one */ rc = 0; goto getbody_bail; } /* copy all headers except Xref: into new file */ g = fopen(s, "r"); if (!g) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: open %s failed", group->name, s); rc = 0; goto getbody_bail; } debug--; while ((l = getaline(g)) != NULL) { /* skip xref: headers */ if (strncasecmp(l, "Xref:", 5) != 0) fprintf(f, "%s\n", l); } debug = debugmode; fclose(g); /* create a whole bunch of new hardlinks */ store(c, f, newsgroups, messageid); /* retrieve body */ fprintf(f, "\n"); /* Empty line between header and body. */ debug--; while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) { if (*l == '.') ++l; fputs(l, f); fputc('\n', f); } debug = debugmode; if (!l) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. " "Transmission interrupted.", group->name, messageid); fprintf(f, "\n\t[ Leafnode: ]\n" "\t[ An error occured while " "retrieving this message. ]\n"); fclose(f); rc = 0; goto getbody_bail; } /* abort when disk is full */ if (fclose(f) && errno == ENOSPC) raise(SIGINT); /* Remove old article files since we don't need them anymore. This is done by evaluating the old Xref: header. */ if (!xref) { /* no Xref: header --> make a fake one */ xref = critmalloc(50 + strlen(fqdn) + strlen(group->name), "getbody"); sprintf(xref, "%s %s:%lu", fqdn, group->name, id); /* RATS: ignore */ } if (debugmode) syslog(LOG_DEBUG, "xref: %s", xref); c = strchr(xref, ' '); #ifdef __LCLINT__ assert(c != NULL); /* we know c != NULL */ #endif /* __LCLINT__ */ while ((c++) && (*c) && (q = strchr(c, ':')) != NULL) { *q++ = '\0'; if ((p = strchr(q, ' ')) != NULL) *p = '\0'; /* c points to the newsgroup, q to the article number */ if (!chdirgroup(c, FALSE)) { return 0; } if (unlink(q)) syslog(LOG_NOTICE, "retrieved body, but unlinking headers-only file %s/%s failed", c, q); else if (debugmode) syslog(LOG_DEBUG, "retrieved body, now unlinking headers-only file %s/%s", c, q); c = strchr(q, ' '); } rc = 1; getbody_bail: if (xref) free(xref); if (messageid) free(messageid); if (newsgroups) free(newsgroups); return rc; } static int getbody(const struct server *cs, struct newsgroup *group, unsigned long id) { static int (*func)(const struct server *, struct newsgroup *, unsigned long); if (!func) { func = db_situ ? getbody_insitu : getbody_newno; } return func(cs, group, id); } /* * Get bodies of messages that have marked for download. * The group must already be selected at the remote server and * the current directory must be the one of the group. */ static void getmarked(const struct server *current_server, struct newsgroup *group) { int n, i; int had_bodies = 0; FILE *f; mastr *filename = mastr_new(PATH_MAX); unsigned long id[BODY_DOWNLOAD_LIMIT]; /* RATS: ignore */ char *t; /* #1 read interesting.groups file */ n = 0; mastr_vcat(filename, spooldir, "/interesting.groups/", group->name, NULL); if (!(f = fopen(mastr_str(filename), "r"))) ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for reading", mastr_str(filename)); else { struct stat st; if (fstat(fileno(f), &st) == 0 && st.st_size > 0) { had_bodies = 1; if (verbose) printf("%s: getting bodies of marked messages...\n", group->name); while ((t = getaline(f)) && n < BODY_DOWNLOAD_LIMIT) { if (sscanf(t, "%lu", &id[n]) == 1) ++n; } } fclose(f); } /* #2 get bodies */ if (delaybody || had_bodies) { syslog(LOG_INFO, "%s: marked bodies %d", group->name, n); if (verbose > 1) printf("%s: marked bodies %d\n", group->name, n); } for (i = 0; i < n; ++i) if (getbody(current_server, group, id[i])) id[i] = 0; /* #3 write back ids of all articles which could not be retrieved */ if (had_bodies) { if (!(f = fopen(mastr_str(filename), "w"))) ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for writing", mastr_str(filename)); else { for (i = 0; i < n; ++i) if (id[i] != 0) fprintf(f, "%lu\n", id[i]); fclose(f); } } if (delaybody || had_bodies) { if (verbose) printf("%s: Done getting article bodies.\n", group->name); } mastr_delete(filename); } /** count number of colons in the string s_in. */ static int count_colons(const char *s_in) { int ngs; const char *t; ngs = 0; t = s_in; for(;;) { t += strcspn(t, ":"); if (!*t) break; t++; ngs++; if (ngs < 0) { /* overflow */ return INT_MAX; } } return ngs; } /* * get newsgroup from a server. "server" is the last article that * was previously read from this group on that server */ static unsigned long getgroup(const struct server *current_server, /*@null@*/ struct newsgroup *g, unsigned long server) { #define HD_MAX 10 static char *hd[HD_MAX]; const char *hnames[HD_MAX] = { "Path: ", "Message-ID: ", "From: ", "Newsgroups: ", "Subject: ", "Date: ", "References: ", "Lines: ", "Xref: ", "" }; /* order of headers in XOVER */ enum enames { /* 0: art. no. */ h_sub = 1, h_fro, h_dat, h_mid, h_ref, h_byt, h_lin, h_xref }; /* XOVER fields: 0 Subject, 1 From, 2 Date, 3 Message-ID, 4 * References, 5 Bytes, 6 Lines, 7 Xref:full (optional) */ unsigned long fetched, killed; unsigned long h; long n; unsigned long last; unsigned long window; /* last ARTICLE n command sent */ char *l; FILE *f; const char *c; struct stat st; long outstanding = 0, j; unsigned long i; unsigned long *stufftoget; int localmaxage = maxage; int maxagelimit = -1; const char *limitfrom = ""; int expdays; if (!g) abort(); if (g->first > g->last && g->first - g->last > 1) g->last = g->first - 1; if ((expdays = lookup_expiredays(g->name)) > 0) { if (localmaxage > expdays) { maxagelimit = expdays; limitfrom = "groupexpire"; } } else { if (localmaxage > expiredays) { maxagelimit = expiredays; limitfrom = "global expire"; } } if (*limitfrom && localmaxage > maxagelimit) { if (clamp_maxage) { syslog(LOG_NOTICE, "clamping maxage for %s to %s %d", g->name, limitfrom, maxagelimit); localmaxage = maxagelimit; } else { fprintf(stderr, "warning: group %s: maxage of %d is inappropriate for your " "applicable %s of %d. This can cause excessive downloads of " "articles that were previously downloaded and expired. " "Fix your configuration.\n", g->name, localmaxage, limitfrom, maxagelimit); syslog(LOG_WARNING, "warning: group %s: maxage of %d is inappropriate for your " "applicable %s of %d. This can cause excessive downloads of " "articles that were previously downloaded and expired. " "Fix your configuration.", g->name, localmaxage, limitfrom, maxagelimit); } } if (server == 0ul) server = 1ul; /* skip */ if (gs_match(current_server->group_pcre, g->name) != 1) { if (verbose > 1) { printf("%s: skipped %s, not in only_groups_pcre\n", current_server->name, g->name); } return server; } /* skip */ if ((l = getenv("LN_SKIP_GROUPS"))) { char *x, *y = critstrdup(l, "getgroup"); for (x = strtok(y, ","); x; x = strtok(NULL, ",")) { if (wildmat(g->name, x)) { if (verbose) { printf("%s: skipped %s, LN_SKIP_GROUPS=%s\n", current_server->name, g->name, l); syslog(LOG_INFO, "%s: skipped %s, LN_SKIP_GROUPS=%s", current_server->name, g->name, l); } free(y); return server; } } free(y); } xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", g->name); putaline(); n = nntpreply(current_server); if (n == 498) { return 0; } l = lastreply(); if (n == 411) { /* group not available on server */ if (verbose > 1) printf("%s: no such group\n", g->name); return server; } if (sscanf(l, "%3ld %lu %lu %lu ", &n, &h, &window, &last) < 4 || n != 211) { fprintf(stderr, "Warning: %s: cannot parse server reply \"%s\"\n", g->name, l); syslog(LOG_WARNING, "Warning: %s: cannot parse server reply \"%s\"", g->name, l); return 0; } if (h == 0) { if (verbose > 1) printf("%s: upstream group is empty\n", g->name); return server; } if (extraarticles) { if (server > extraarticles) i = server - extraarticles; else i = 1; if (i < window) i = window; if (verbose > 1 && server > 1) printf("%s: backing up from %lu to %lu\n", g->name, server, i); server = i; } if (server > last + 1) { syslog(LOG_INFO, "%s: last seen article was %lu, server now has %lu-%lu", g->name, server, window, last); if (server > last + 5) { if (verbose) printf("%s: switched upstream servers? %lu > %lu\n", g->name, server - 1, last); server = window; /* insane - recover thoroughly */ } else { if (verbose) printf("%s: rampant spam cancel? %lu > %lu\n", g->name, server - 1, last); server = last - 5; /* a little bit too much */ } } if (initiallimit && server == 1 && last > server && last - server > initiallimit) { if (verbose > 1) printf("%s: skipping articles %lu-%lu inclusive (initial limit)\n", g->name, server, last - initiallimit); syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (initial limit)", g->name, server, last - initiallimit); server = last - initiallimit + 1; } if (artlimit && last > server && last - server > artlimit) { if (verbose > 1) printf("%s: skipping articles %lu-%lu inclusive (article limit)\n", g->name, server, last - artlimit - 1); syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (article limit)", g->name, server, last - artlimit - 1); server = last - artlimit; } getmarked(current_server, g); if (window < server) window = server; if (window < 1) window = 1; server = window; if (server > last) { if (verbose > 1) printf("%s: no new articles\n", g->name); syslog(LOG_INFO, "%s: no new articles\n", g->name); return server; } if (verbose > 1) printf("%s: considering articles %lu - %lu\n", g->name, server, last); syslog(LOG_INFO, "%s: considering articles %lu - %lu\n", g->name, server, last); fetched = 0; killed = 0; stufftoget = (unsigned long *)malloc(sizeof(stufftoget[0]) * (last + 1 - server)); if (!stufftoget) { ln_log(LNLOG_SERR, LNLOG_CGROUP, "not enough memory for XHDRs"); return server; } memset(stufftoget, 0, sizeof(stufftoget[0]) * (last + 1 - server)); if (!current_server->noxover) { int tmp; /* Try XOVER */ xsnprintf(lineout, SIZE_lineout, "XOVER %lu-%lu\r\n", server, last); putaline(); if (nntpreply(current_server) == 224) { mastr *ol = mastr_new(1024); debug--; while ((l = mgetaline(nntpin)) && strcmp(l, ".")) { /*@dependent@*/ char *fields[HD_MAX]; unsigned long art; const char *t; char *q; mastr_cpy(ol, l); /* split xover */ /*@+loopexec@*/ for (i = 0; l && l[0] && i < HD_MAX; i++) { char *y; fields[i] = l; if ((y = strchr(l, '\t'))) { y[0] = '\0'; l = y + 1; } else { l = NULL; }; }; /*@=loopexec@*/ /* short line -- log and skip */ if (i < 7) { ln_log(LNLOG_SWARNING, LNLOG_CTOP, "%s: %s: Warning: got unparsable XOVER line from server, " "too few fields (%lu): \"%s\"", current_server->name, g->name, i, mastr_str(ol)); continue; } for (; i < HD_MAX; i++) fields[i] = NULL; art = strtoul(fields[0], &q, 10); if (q && art >= server && art <= last) { long artlines; /* invalid: -1 */ unsigned long artbytes; /* invalid: 0 */ if (fields[h_lin]) artlines = strtol(fields[h_lin], NULL, 10); else artlines = -1; if (fields[h_byt]) artbytes = strtoul(fields[h_byt], NULL, 10); else artbytes = 0; if (maxbytes && fields[h_byt] && (strtoul(fields[h_byt], NULL, 10) > maxbytes)) { killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too many bytes (%lu > %lu)", g->name, art, fields[h_mid], strtoul(fields[h_byt], NULL, 10), maxbytes); } else if (maxbytes && artbytes != 0 && artbytes > maxbytes) { killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too large (%lu > %lu)", g->name, art, fields[h_mid], artbytes, maxbytes); } else if (maxlines && artlines != -1 && artlines > maxlines) { killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too many lines (%ld > %ld)", g->name, art, fields[h_mid], artlines, maxlines); } else if (minlines && artlines != -1 && artlines < minlines) { killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too few lines (%ld < %ld)", g->name, art, fields[h_mid], artlines, minlines); } else if (localmaxage && fields[h_dat] && (age(fields[h_dat]) > localmaxage)) { killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too old (%d > %d) days", g->name, art, fields[h_mid], age(fields[h_dat]), localmaxage); } else if (crosspostlimit && fields[h_xref] && (tmp = count_colons(fields[h_xref]) - 1) > crosspostlimit) { /* -1 to skip over the header's name, "Xref:" */ killed++; ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), " "too many groups in Xref: header (%d > %ld)", g->name, art, fields[h_mid], tmp, crosspostlimit); } else if (lstat(t = lookup(fields[h_mid]), &st) == 0) { killed++; ln_log(LNLOG_SDEBUG, 4, "%s: killed %lu (%s), already fetched before", g->name, art, fields[h_mid]); } else { stufftoget[outstanding] = art; outstanding++; if (verbose > 2) printf("%s: will fetch %lu (%s)\n", g->name, art, t); } } } mastr_delete(ol); debug = debugmode; goto have_outstanding; } ln_log(LNLOG_SINFO, 2, "XOVER failed, trying XHDR"); } xsnprintf(lineout, SIZE_lineout, "XHDR Message-ID %lu-%lu\r\n", server, last); putaline(); if (nntpreply(current_server) == 221) { debug--; while ((l = mgetaline(nntpin)) && strcmp(l, ".")) { unsigned long art; char *t; art = strtoul(l, &t, 10); if (t && isspace((unsigned char)*t)) { while (isspace((unsigned char)*t)) t++; if (art >= server && art <= last && stat(lookup(t), &st) != 0) { stufftoget[outstanding] = art; outstanding++; if (verbose > 2) printf("%s: will fetch %lu (%s)\n", g->name, art, t); } } } debug = debugmode; if (!l) { free(stufftoget); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout after XHDR", current_server->name, g->name); return 0; } } else { free(stufftoget); return server; } if (outstanding == 0) { free(stufftoget); return last + 1; } syslog(LOG_INFO, "%s: will fetch %ld article%s", g->name, outstanding, PLURAL(outstanding)); if (verbose > 1) printf("%s: will fetch %ld article%s\n", g->name, outstanding, PLURAL(outstanding)); if (minlines || maxlines) { xsnprintf(lineout, SIZE_lineout, "XHDR Lines %lu-%lu\r\n", server, last); putaline(); if (nntpreply(current_server) == 221) { while ((l = mgetaline(nntpin)) && strcmp(l, ".")) { unsigned long art; long lines = 0; char *t; art = strtoul(l, &t, 10); if (t) lines = strtol(t, NULL, 10); for (j = 0; j < outstanding; j++) { if (art == stufftoget[j]) break; } if (j < outstanding) if ((minlines && lines < minlines) || (maxlines && lines > maxlines)) { stufftoget[j] = 0; syslog(LOG_INFO, "%s: Killed article %lu: %ld line%s", g->name, art, lines, PLURAL(lines)); if (verbose > 2) printf("%s: Killed article %lu: %ld line%s.\n", g->name, art, lines, PLURAL(lines)); killed++; } } if (!l) { /* timeout */ free(stufftoget); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout after XHDR Lines", current_server->name, g->name); return 0; } } } if (maxbytes) { xsnprintf(lineout, SIZE_lineout, "XHDR Bytes %lu-%lu\r\n", server, last); putaline(); if (nntpreply(current_server) == 221) { while ((l = mgetaline(nntpin)) && strcmp(l, ".")) { unsigned long art, bytes = 0; char *t; art = strtoul(l, &t, 10); if (t) bytes = strtoul(t, NULL, 10); for (j = 0; j < outstanding; j++) { if (art == stufftoget[j]) break; } if (j < outstanding && (bytes > maxbytes)) { stufftoget[j] = 0; syslog(LOG_INFO, "%s: Killed article %lu (%lu > %lu bytes)", g->name, art, bytes, maxbytes); if (verbose > 2) printf("%s: Killed article %lu (%lu > %lu bytes).\n", g->name, art, bytes, maxbytes); killed++; } } if (!l) { /* timeout */ free(stufftoget); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout after XHDR Bytes", current_server->name, g->name); return 0; } } } if (localmaxage) { xsnprintf(lineout, SIZE_lineout, "XHDR Date %lu-%lu\r\n", server, last); putaline(); if (nntpreply(current_server) == 221) { debug--; while ((l = mgetaline(nntpin)) && strcmp(l, ".")) { unsigned long art; long aage = 0; char *t; art = strtoul(l, &t, 10); if (t) aage = age(t); for (j = 0; j < outstanding; j++) { if (art == stufftoget[j]) break; } if (j < outstanding && (aage > localmaxage)) { stufftoget[j] = 0; syslog(LOG_INFO, "%s: Killed article %lu (%ld > %d = localmaxage)", g->name, art, aage, localmaxage); if (verbose > 2) printf("%s: Killed article %lu (%ld > %d = localmaxage).\n", g->name, art, aage, localmaxage); killed++; } } if (!l) { /* timeout */ free(stufftoget); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout after XHDR Date", current_server->name, g->name); return 0; } debug = debugmode; } } /* now we have a list of articles in stufftoget[] */ /* let's get the header and possibly bodies of these */ have_outstanding: for (i = 0; outstanding > 0; i++) { int takethis = 1; int requested_body; const char *cmd; outstanding--; if (!stufftoget[i]) continue; if (stufftoget[i] < server) { if (verbose > 2) printf("%s: skipping %lu - not available or too old\n", g->name, stufftoget[i]); syslog(LOG_INFO, "%s: skipping %lu - not available or too old", g->name, stufftoget[i]); continue; } debug = debugmode; requested_body = ((!filterfile || article_despite_filter) && !delaybody); cmd = requested_body ? "ARTICLE" : "HEAD"; xsnprintf(lineout, SIZE_lineout, "%s %lu\r\n", cmd, stufftoget[i]); putaline(); l = mgetaline(nntpin); /* timeout */ if (!l) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "Server disconnection or timeout before article " "could be retrieved #1"); free(stufftoget); return 0; } /* check proper reply code */ if (sscanf(l, "%3ld %lu", &n, &h) < 2 || ((n / 10) != 22)) { if (verbose > 2) printf("%s %s %lu: reply %s (%ld more up in the air)\n", g->name, cmd, stufftoget[i], l, outstanding); syslog(LOG_INFO, "%s %s %lu: reply %s (%ld more up in the air)", g->name, cmd, stufftoget[i], l, outstanding); continue; } /* anything below this line will have to make sure that data is * drained properly in case */ debug--; if (verbose > 2) printf("%s: receiving article %lu (%ld more up in the air)\n", g->name, stufftoget[i], outstanding); for (h = 0; h < 10; h++) { if (hd[h]) free(hd[h]); hd[h] = critstrdup("", "getgroup"); } c = NULL; n = 9; /* "other" header */ while ((l = getfoldedline(nntpin, mgetaline)) && *l && strcmp(l, ".")) { /* regexp pattern matching */ if (filterfile && dofilter(l)) { killed++; if (verbose > 2) printf(".filtered article %lu: match on \"%s\"\n", stufftoget[i], l); syslog(LOG_INFO, "filtered article %lu: match on \"%s\"", stufftoget[i], l); takethis = 0; free(l); l = NULL; continue; } n = 0; while (strncasecmp(l, hnames[n], strlen(hnames[n]))) n++; if (n < 9 && hd[n] && *(hd[n])) /* second occurance of the same recognized header * is treated as if it was not listed * in hnames (as "other header") */ n = 9; hd[n] = critrealloc(hd[n], strlen(hd[n]) + strlen(l) + 2, "Fetching article header"); if (strlen(hd[n])) strcat(hd[n], "\n"); /* RATS: ignore */ strcat(hd[n], l); /* RATS: ignore */ if (debugmode > 1 && verbose > 3 && hnames[n] && *hnames[n]) printf("...saw header %s\n", hnames[n]); free(l); l = NULL; } /* end while */ /* if server ended the fetch prematurely, assume we didn't * request a body to ignore - ignoring would cause timeout * waiting for data that is never sent. * * SourceForge bug 873149, reported 2004-01-08 by Toni Viemerö, * sourceforge user "skithund" */ if (l == NULL) { /* timeout - don't flush body */ requested_body = FALSE; } else if (strcmp(l, ".") == 0 && requested_body) { ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "%s: %s:%lu: article without blank line after header, format violation", current_server->name, g->name, stufftoget[i]); requested_body = FALSE; } if (l) free(l); if (!takethis) { if (requested_body) ignore_answer(nntpin); continue; /* filtered article */ } debug = debugmode; if (!l) { /* timeout */ free(stufftoget); ln_log(LNLOG_SERR, LNLOG_CARTICLE, "Server disconnection or timeout before article " "could be retrieved #2"); return 0; } /* check headers */ for (h = 0; h < 6; h++) { if (!hd[h] || !*(hd[h])) { if (verbose) printf("Discarding article %lu - no %s found\n", stufftoget[i], hnames[h]); syslog(LOG_NOTICE, "Discarding article %lu - no %s found", stufftoget[i], hnames[h]); killed++; if (requested_body) ignore_answer(nntpin); takethis = 0; break; } } /* mandatory header missing */ if (!takethis) continue; if (localmaxage && age(hd[5]) > localmaxage) { if (verbose > 2) printf("Discarding article %lu - older than %d day%s\n", stufftoget[i], localmaxage, PLURAL(localmaxage)); syslog(LOG_INFO, "Discarding article %lu %s - older than %d day%s", stufftoget[i], hd[4], localmaxage, PLURAL(localmaxage)); killed++; if (requested_body) ignore_answer(nntpin); continue; } if (minlines || maxlines) { char *t; t = strchr(hd[7], ' '); if (t) { n = strtol(t, NULL, 10); if (minlines && n < minlines) { if (verbose > 2) printf("Discarding article %lu - %ld < minlines\n", stufftoget[i], n); syslog(LOG_INFO, "Discarding article %lu %s -- %ld < minlines", stufftoget[i], hd[4], n); killed++; if (requested_body) ignore_answer(nntpin); continue; } if (maxlines && n > maxlines) { if (verbose > 2) printf("Discarding article %lu - %ld > maxlines\n", stufftoget[i], n); syslog(LOG_INFO, "Discarding article %lu %s -- %ld > maxlines", stufftoget[i], hd[4], n); killed++; if (requested_body) ignore_answer(nntpin); continue; } } } if (crosspostlimit) { char *t; t = hd[3]; n = 1; /* number of groups the article is posted to */ while ((t = strchr(t, ',')) != NULL) { t++; n++; } if (crosspostlimit < n) { if (verbose > 2) printf("Discarding article %lu - posted to %ld groups " "(max. %ld)\n", stufftoget[i], n, crosspostlimit); syslog(LOG_INFO, "Discarding article %lu %s - posted to %ld groups " "(max. %ld)", stufftoget[i], hd[4], n, crosspostlimit); killed++; if (requested_body) ignore_answer(nntpin); continue; } } /* store articles */ f = NULL; c = lookup(strchr(hd[1], '<')); /* lookup also replaces '/' with '@' */ if (!c) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "lookup of %s failed", hd[1]); if (requested_body) ignore_answer(nntpin); continue; } if (!stat(c, &st)) { syslog(LOG_INFO, "article %s already stored", c); if (requested_body) ignore_answer(nntpin); continue; /* for some reasons, article is already there */ } else if (errno == ENOENT) { f = fopen(c, "w"); if (!f) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to create article %s: %m", c); if (requested_body) ignore_answer(nntpin); continue; } } else { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to store article %s: %m", c); if (requested_body) ignore_answer(nntpin); continue; } for (h = 0; h < 10; h++) if (h != 8 && hd[h] && *(hd[h])) fprintf(f, "%s\n", hd[h]); h = 0; /* replace tabs and other odd signs with spaces */ while (h < 8) { char *p1; char *p2; p1 = p2 = hd[h]; while (p1 && *p1) { if (isspace((unsigned char)*p1)) { *p2 = ' '; do { p1++; } while (isspace((unsigned char)*p1)); } else { *p2 = *p1++; } p2++; } *p2 = '\0'; h++; } if (fflush(f)) { (void)fclose(f); (void)unlink(c); if (requested_body) ignore_answer(nntpin); continue; } /* generate hardlinks; this procedure also increments g->last */ store(c, f, *hd[3] ? hd[3] + strlen(hnames[3]) : "", *hd[1] ? hd[1] + strlen(hnames[1]) : ""); if (delaybody) { if (fclose(f)) { int e = errno; (void)truncate(c, 0); (void)unlink(c); if (e == ENOSPC) raise(SIGINT); } else { fetched++; } continue; } if (!requested_body) { xsnprintf(lineout, SIZE_lineout, "BODY %lu\r\n", stufftoget[i]); putaline(); l = mgetaline(nntpin); if (!l) { /* timeout */ (void)fflush(f); (void)ftruncate(fileno(f), 0); (void)fclose(f); unlink(c); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout after BODY %lu", current_server->name, g->name, stufftoget[i]); free(stufftoget); return 0; } if (sscanf(l, "%3ld", &n) != 1 || (n / 10 != 22)) { if (verbose > 2) printf("BODY %lu: reply %s\n", stufftoget[i], l); syslog(LOG_NOTICE, "BODY %lu: reply %s", stufftoget[i], l); (void)fflush(f); (void)ftruncate(fileno(f), 0); (void)fclose(f); unlink(c); continue; } } debug--; fputs("\n", f); /* empty line between header and body */ while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".")) { if (*l == '.') l++; clearerr(f); fputs(l, f); fputc('\n', f); if (feof(f)) { l = NULL; break; } } debug = debugmode; fetched++; if (fflush(f)) { l = NULL; } if (fclose(f)) { l = NULL; } if (l == NULL) { /* article didn't terminate with a .: error */ (void)truncate(c, 0); (void)unlink(c); ln_log(LNLOG_SWARNING, LNLOG_CARTICLE, "warning: %s: %s: server disconnect or timeout retrieving article %lu", current_server->name, g->name, stufftoget[i]); free(stufftoget); return 0; } } syslog(LOG_INFO, "%s: %lu article%s fetched (to %lu), %lu killed", g->name, fetched, PLURAL(fetched), g->last, killed); if (verbose > 1) printf("%s: %lu article%s fetched, %lu killed\n", g->name, fetched, PLURAL(fetched), killed); free(stufftoget); return last + 1; } /* return 1 == success, 0 == failure */ static int expire_interesting(void) { DIR *d; struct dirent *de; char s[SIZE_s+1]; xsnprintf(s, SIZE_s, "%s/interesting.groups/", spooldir); d = opendir(s); if (d == NULL) { ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot open %s for reading: %m", s); return 0; } while ((de = readdir(d))) { struct stat st; xsnprintf(s, SIZE_s, "%s/interesting.groups/%s", spooldir, de->d_name); if (stat(s, &st) < 0) continue; /* reading a newsgroup changes the ctime; if the newsgroup is newly created, the mtime is changed as well */ if (((st.st_mtime == st.st_ctime) && (now - st.st_ctime > (timeout_short * SECONDS_PER_DAY))) || (now - st.st_ctime > (timeout_long * SECONDS_PER_DAY))) { if (verbose > 1) printf("unsubscribing from %s\n", de->d_name); syslog(LOG_INFO, "unsubscribing from %s (current time: %ld): " "ctime age %ld, mtime age %ld", de->d_name, (long)now, (long)now - st.st_ctime, (long)now - st.st_mtime); unlink(s); } } (void)closedir(d); return 1; } static void formatserver(char *d, size_t len, const struct server *serv, const char *suffix) { if (serv->port == 0 || serv->port == 119) xsnprintf(d, len, "%s/leaf.node/%s%s", spooldir, serv->name, suffix); else xsnprintf(d, len, "%s/leaf.node/%s:%u%s", spooldir, serv->name, serv->port, suffix); } /** get active file from current_server. * \returns 0 for success, non-zero for error. */ static int nntpactive(struct server *current_server, time_t *stamp) { struct stat st; char *l, *p; struct stringlist *groups = NULL; struct stringlist *helpptr = NULL; char timestr[64]; /* RATS: ignore */ long reply = 0l; int error, merge = 0; char s[SIZE_s+1]; int try_xgtitle = 1; static time_t cur_date; static int cur_date_init = 0; time_t t; if (!cur_date_init) { cur_date = time(NULL); cur_date_init = 1; } formatserver(s, SIZE_s, current_server, ""); t = time(NULL); if (active && !forceactive && (stat(s, &st) == 0)) { if (verbose) printf("%s: getting new newsgroups\n", current_server->name); /* to avoid a compiler warning we print out a four-digit year; * but since we need only the last two digits, we skip them * in the next line */ *stamp = st.st_mtime; (void)strftime(timestr, sizeof(timestr), "%Y%m%d %H%M%S", gmtime(&st.st_mtime)); xsnprintf(lineout, SIZE_lineout, "NEWGROUPS %s GMT\r\n", timestr + 2); putaline(); /* we used to expect 231 here, but some broken servers (MC-link * Custom News-server V1.06) return 215 instead. * Just accept any 2XX code as success. */ if ((reply = nntpreply(current_server)) < 200 || reply >= 300) { char *e = lastreply(); if (!e) e = "server disconnect or timeout"; ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading new newsgroups failed, reason \"%s\"", current_server->name, e); return -1; } while ((l = mgetaline(nntpin)) && (strcmp (l, "."))) { p = l; while (*p && !isspace((unsigned char)*p)) p++; if (*p) *p = '\0'; if (gs_match(current_server->group_pcre, l)) { merge++; insertgroup(l, 1, 0, cur_date); prependtolist(&groups, l); } } if (!l) { /* timeout */ ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading new newsgroups failed, server disconnect or timeout.", current_server->name); return -1; } ln_log(LNLOG_SINFO, LNLOG_CSERVER, "%s: got %d new newsgroups.", current_server->name, merge); if (merge) { mergegroups(); /* merge groups into active */ merge = 0; } helpptr = groups; if (verbose && helpptr && current_server->descriptions) printf("%s: getting newsgroup descriptions\n", current_server->name); while (helpptr != NULL) { if (current_server->descriptions) { error = 0; if (try_xgtitle) { xsnprintf(lineout, SIZE_lineout, "XGTITLE %s\r\n", helpptr->string); putaline(); reply = nntpreply(current_server); } if (!try_xgtitle || reply != 282) { try_xgtitle = 0; xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS %s\r\n", helpptr->string); putaline(); reply = nntpreply(current_server); if (reply && (reply != 215)) error = 1; } if (!error) { l = mgetaline(nntpin); if (l && *l && strcmp(l, ".")) { p = l; while (*p && !isspace((unsigned char)*p)) p++; while (isspace((unsigned char)*p)) { *p = '\0'; p++; } if (reply == 215 || reply == 282) changegroupdesc(l, *p ? p : NULL); do { l = mgetaline(nntpin); error++; } while (l && *l && strcmp(l, ".")); if (error > 1) { current_server->descriptions = 0; syslog(LOG_WARNING, "warning: %s does not process " "LIST NEWSGROUPS %s correctly: use nodesc\n", current_server->name, helpptr->string); fprintf(stderr, "warning: %s does not process LIST " "NEWSGROUPS %s correctly: use nodesc\n", current_server->name, helpptr->string); } } } } /* if ( current_server->descriptions ) */ helpptr = helpptr->next; } freelist(groups); } else { ln_log(LNLOG_SINFO, LNLOG_CSERVER, "%s: getting all newsgroups (debug: active: %s, forceactive: %s)", current_server->name, active ? "set" : "nil", forceactive ? "true" : "false"); xsnprintf(lineout, SIZE_lineout, "LIST\r\n"); putaline(); if (nntpreply(current_server) != 215) { char *e = lastreply(); if (!e) e = "server disconnect or timeout"; ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading all newsgroups failed, reason \"%s\".", current_server->name, e); return -2; } debug--; while ((l = mgetaline(nntpin)) && (strcmp(l, "."))) { p = l; while (*p && !isspace((unsigned char)*p)) p++; while (isspace((unsigned char)*p)) { *p = '\0'; p++; } if (gs_match(current_server->group_pcre, l)) { insertgroup(l, 1, 0, cur_date); } } mergegroups(); if (!l) { /* timeout */ ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading all newsgroups failed, server disconnect or timeout.", current_server->name); return -2; } if (current_server->descriptions) { if (verbose) printf("%s: getting newsgroup descriptions\n", current_server->name); xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS\r\n"); putaline(); l = mgetaline(nntpin); /* correct reply starts with "215". However, INN 1.5.1 is broken and immediately returns the list of groups */ if (l) { if (debug) syslog(LOG_DEBUG, "<%s", l); reply = strtol(l, &p, 10); if ((reply == 215) && (*p == ' ' || *p == '\0')) { l = mgetaline(nntpin); /* get first description */ } else if (*p != ' ' && *p != '\0') { int dummy = 0; /* INN 1.5.1: line already contains description */ (void)dummy; } else { ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading newsgroups descriptions failed: %s", current_server->name, l); ln_log(LNLOG_SERR, LNLOG_CSERVER, "Workaround: Add \"nodesc = 1\" (without quotes) below the server = %s line.", current_server->name); return -2; } } else { ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading newsgroups descriptions failed: server disconnect or timeout.", current_server->name); return -2; } while (l && (strcmp(l, "."))) { p = l; while (*p && !isspace((unsigned char)*p)) p++; while (isspace((unsigned char)*p)) { *p = '\0'; p++; } changegroupdesc(l, *p ? p : NULL); l = mgetaline(nntpin); } if (!l) { /* timeout */ ln_log(LNLOG_SERR, LNLOG_CSERVER, "%s: reading newsgroup descriptions failed, server disconnect or timeout.", current_server->name); return -2; } } debug = debugmode; /* mark active for /this/ server fetched */ { FILE *f = fopen(s, "a"); if (!f) { ln_log(LNLOG_SERR, LNLOG_CGROUP, "cannot open \"%s\": %m", s); } else { if (fclose(f)) ln_log(LNLOG_SERR, LNLOG_CGROUP, "cannot close \"%s\": %m", s); } } } *stamp = t; return 0; } static int movetofailed(const char *name) { char s[SIZE_s + 1]; xsnprintf(s, SIZE_s, "%s/failed.postings/%s", spooldir, name); ln_log(LNLOG_SERR, LNLOG_CARTICLE, "moving file %s to failed.postings", name); if (rename(name, s)) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to move failed posting to %s: %m", s); return -1; } else { return 0; } } /* * post all spooled articles * * if all postings succeed, returns 1 * if there are no postings to post, returns 1 * if a posting is strange for some reason, returns 0 * returns -1 if server should be skipped */ static int postarticles(const struct server *current_server) { struct stat st; char *line; DIR *d; struct dirent *de; FILE *f; int r, haveid, n; char *p, *q; int savedir; n = 0; savedir = open(".", O_RDONLY); if (savedir < 0) { ln_log(LNLOG_SERR, LNLOG_CTOP, "postarticles: Unable to save current working directory: %m"); return 0; } if (chdir(spooldir) || chdir("out.going")) { ln_log(LNLOG_SERR, LNLOG_CTOP, "postarticles: Unable to cd to %s/out.going: %m", spooldir); fchdir(savedir); close(savedir); return 0; } d = opendir("."); if (!d) { ln_log(LNLOG_SERR, LNLOG_CTOP, "postarticles: Unable to opendir %s/out.going: %m", spooldir); fchdir(savedir); close(savedir); return 0; } while ((de = readdir(d)) != NULL) { haveid = 0; f = NULL; if (!strcmp(".", de->d_name) || !strcmp("..", de->d_name)) { continue; } p = q = NULL; if ((lstat(de->d_name, &st) != 0)) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "postarticles: cannot stat %s: %m", de->d_name); continue; } if (!S_ISREG(st.st_mode)) { ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE, "postarticles: %s is not a regular file", de->d_name); movetofailed(de->d_name); continue; } if (!(st.st_mode & S_IRUSR)) { ln_log(LNLOG_SINFO, LNLOG_CARTICLE, "postarticles: skipping %s, not complete", de->d_name); continue; } f = fopen(de->d_name, "r"); if (f == NULL) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "postarticles: cannot open file %s for reading: %m", de->d_name); movetofailed(de->d_name); continue; } p = fgetheader(f, "Newsgroups:"); q = fgetheader(f, "Message-ID:"); if (p == NULL || q == NULL) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "postarticles: file %s lacks Newsgroups " "and/or Message-ID header (this cannot happen)", de->d_name); movetofailed(de->d_name); goto free_cont; } if (!current_server->post_anygroup) { char *pp = critstrdup(p, "postarticles"); if (!isgrouponserver(current_server, pp)) { ln_log(LNLOG_SINFO, LNLOG_CARTICLE, "%s: postarticles: file %s: only_groups_pcre excluded " "or server does not carry newsgroups %s", current_server->name, de->d_name, p); free(pp); goto free_cont; } free(pp); } haveid = ismsgidonserver(current_server, q); switch(haveid) { case 0: ln_log(LNLOG_SINFO, LNLOG_CARTICLE, "%s: postarticles: trying to post file %s Message-ID %s", current_server->name, de->d_name, q); xsnprintf(lineout, SIZE_lineout, "POST\r\n"); putaline(); r = nntpreply(current_server); if (r != 340) { char *e = lastreply(); if (e == NULL) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: postarticles: server disconnect or timeout" " while trying to post file %s)", current_server->name, de->d_name); goto free_ret; } ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: postarticles: server replied \"%s\" to our POST command, skipping server", current_server->name, e); goto free_ret; } else { /* server replied 340, it is willing to accept our article */ int postok = 0; debug--; while ((line = getaline(f)) != NULL) { /* can't use putaline() here because line length of lineout is restricted */ if (line[0] == '.') fputc('.', nntpout); fputs(line, nntpout); fputs("\r\n", nntpout); }; fflush(nntpout); debug = debugmode; xsnprintf(lineout, SIZE_lineout, ".\r\n"); putaline(); line = mgetaline(nntpin); if (!line) { /* timeout: posting failed */ ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: postarticles: server disconnect or timeout" " after sending article %s)", current_server->name, de->d_name); goto free_ret; } line = critstrdup(line, "postarticles"); if (strncmp(line, "240", 3) == 0) { postok = 1; } else if (ismsgidonserver(current_server, q) == 1) { syslog(LOG_NOTICE, "%s: postarticles: posting resulted in \"%s\", " "but article is available upstream, assuming OK.", current_server->name, line); printf("%s: postarticles: posting resulted in \"%s\",\n" "but article is available upstream, assuming OK.", current_server->name, line); postok = 1; } if (postok) { if (verbose > 2) printf("%s: postarticles: POST of article %s OK\n", current_server->name, de->d_name); n++; if (unlink(de->d_name)) { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "postarticles: unable to unlink posted article %s: %m", de->d_name); } } else { ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: postarticles: Article file %s Message-ID %s" " was rejected: \"%s\"", current_server->name, de->d_name, q, line); movetofailed(de->d_name); } free(line); } haveid = 0; break; case 1: syslog(LOG_INFO, "%s: postarticles: Message-ID of %s already in use" " upstream -- article discarded\n", current_server->name, de->d_name); if (verbose > 2) printf("%s: postarticles: %s already available upstream\n", current_server->name, de->d_name); unlink(de->d_name); break; case -1: ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: postarticles: skipping server", current_server->name); goto free_ret; } free_cont: fclose(f); if (p) free(p); if (q) free(q); continue; free_ret: if (p) free(p); if (q) free(q); fchdir(savedir); close(savedir); closedir(d); return -1; } /* while de = readdir */ closedir(d); if (verbose) printf("%s: %d article%s posted.\n", current_server->name, n, PLURAL(n)); syslog(LOG_INFO, "%s: %d article%s posted.", current_server->name, n, PLURAL(n)); fchdir(savedir); close(savedir); return 1; } static int processupstream(const struct server *serv, time_t stamp) { FILE *f; DIR *d; struct dirent *de; struct newsgroup *g; int havefile; unsigned long newserver = 0; char *l; char *oldfile; struct stat st1, st2; int have_st1 = 0; char s[SIZE_s+1]; int aborting = 0; struct stringlist *ngs = NULL, *a, *b = NULL; /* read server info */ formatserver(s, SIZE_s, serv, ""); oldfile = critstrdup(s, "processupstream"); havefile = 0; if ((f = fopen(s, "r")) != NULL) { if (fstat(fileno(f), &st1)) { int e = errno; ln_log(LNLOG_SERR, LNLOG_CSERVER, "Cannot stat %s: %s", s, strerror(e)); } else { have_st1 = 1; } /* a sorted array or a tree would be better than a list */ ngs = NULL; debug--; if (verbose > 1) printf("%s: reading server info from %s\n", serv->name, s); syslog(LOG_INFO, "%s: reading server info from %s", serv->name, s); while (((l = getaline(f)) != NULL) && (strlen(l))) { a = (struct stringlist *)critmalloc(sizeof(struct stringlist) + strlen(l), "Reading server info"); strcpy(a->string, l); /* RATS: ignore */ a->next = NULL; if (ngs == NULL) ngs = a; else b->next = a; b = a; } havefile = 1; debug = debugmode; fclose(f); } xsnprintf(s, SIZE_s, "%s/interesting.groups", spooldir); d = opendir(s); if (!d) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "opendir %s: %m", s); free(oldfile); return -1; } formatserver(s, SIZE_s, serv, "~"); if (stat(s, &st2) == 0 && (!have_st1 || st2.st_mtime > st1.st_mtime) && (f = fopen(s, "r"))) { int e = 0; /* roll in changes of a previous crash */ syslog(LOG_INFO, "Merging in %s from previous run", s); if (verbose > 1) printf("Merging in %s from previous run\n", s); while (((l = getaline(f)) != NULL)) { char *p, *t = strchr(l, ' '); if (!t || !*++t) continue; (void)strtoul(t, &p, 10); if (p && !*p) replaceinlist(&ngs, l, (size_t)(t-l)); } (void)fclose(f); /* read-only file, assume no error */ xsnprintf(s, SIZE_s, "%s/leaf.node/%s.new", spooldir, serv->name); f = fopen(s, "w"); if (!f) { e = 1; } else { a = ngs; while (a && a->string && !ferror(f)) { (void)fputs(a->string, f); (void)fputc('\n', f); a = a->next; } if (fclose(f)) e = 1; } if (e) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "open %s: %m", s); (void)unlink(s); return -1; } if (rename(s, oldfile)) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "rename %s -> %s: %m", s, oldfile); (void)unlink(s); return -1; } } formatserver(s, SIZE_s, serv, "~"); (void)unlink(s); f = fopen(s, "w"); if (f == NULL) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "Could not open %s for writing: %s", s, strerror(errno)); } else { /* make sure that at least SERVERINFO~ is complete */ if (mysetvbuf(f, NULL, _IOLBF, 4096)) { /* try to at least use unbuffered then */ mysetvbuf(f, NULL, _IONBF, 0); } } while ((de = readdir(d))) { if (isalnum((unsigned char)*(de->d_name))) { g = findgroup(de->d_name); if (g != NULL) { unsigned long newhigh; xsnprintf(s, SIZE_s, "%s ", g->name); newhigh = 1ul; l = havefile ? findinlist(ngs, s) : NULL; if (l && *l) { char *t; l = strchr(l, ' '); if (l) { newhigh = strtoul(l, &t, 10); if (t == l || *t) newhigh = 1ul; } } newserver = getgroup(serv, g, newhigh); /* run this independent of delaybody mode, because * the admin may have switched delaybody off recently, * and we still want users to be able to retrieve * articles. */ if (newserver) newhigh = newserver; if (f != NULL && newhigh > 0) { fprintf(f, "%s %lu\n", g->name, newhigh); } if (!newserver) { fprintf(stderr, "Warning: aborting fetch from %s due to previous condition.\n", serv->name); syslog(LOG_WARNING, "Warning: aborting fetch from %s due to previous condition.", serv->name); aborting = 1; break; } } else { if (verbose > 1) printf("%s not found in groupinfo file\n", de->d_name); syslog(LOG_NOTICE, "%s not found in groupinfo file", de->d_name); } } /* if isalnum */ } /* while readdir */ closedir(d); if (f != NULL) { int ren = 1; formatserver(s, SIZE_s, serv, "~"); if (ferror(f)) ren = 0; if (fflush(f)) ren = 0; if (fclose(f)) ren = 0; if (!aborting) { if (ren) { struct utimbuf ut; if (rename(s, oldfile)) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot rename %s to %s: %m", s, oldfile); } ut.modtime = ut.actime = stamp; if (utime(oldfile, &ut)) { ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot set proper time for %s to %lu: %m", s, (unsigned long)stamp); } } else { ln_log(LNLOG_SERR, LNLOG_CSERVER, "write error on %s, old version of %s kept", s, oldfile); } } } free(oldfile); freelist(ngs); return 0; } /* * checks whether all newsgroups have to be retrieved anew * returns 0 if yes, time of last update if not * mtime is the time when active was fetched fully * atime is the time when active was last updated */ static time_t checkactive(void) { struct stat st; char *s = activeread(); if (stat(s, &st)) { free(s); return 0; } if ((now - st.st_mtime) < (timeout_active * SECONDS_PER_DAY)) { if (debugmode) syslog(LOG_DEBUG, "Last LIST done %d seconds ago: NEWGROUPS\n", (int)(now - st.st_mtime)); free(s); return st.st_atime; } else { if (debugmode) syslog(LOG_DEBUG, "Last LIST done %d seconds ago: LIST\n", (int)(now - st.st_mtime)); free(s); return 0; } } static int updateactive(void) { struct stat st; struct utimbuf buf; FILE *f; char *s = activeread(); int rc = 0; if (stat(s, &st)) { /* active.read probably doesn't exist */ (void)unlink(s); /* delete it in case it's junk */ if ((f = fopen(s, "w")) != NULL) { if (fsync(fileno(f))) rc = -1; if (fclose(f)) rc = -1; } else { /* f == NULL, open error */ rc = -1; } } else { buf.actime = (now < st.st_atime) ? st.st_atime : now; /* now < update may happen through HW failures */ buf.modtime = st.st_mtime; if (utime(s, &buf)) { rc = -1; } } if (rc) ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot update or create %s: %m", s); free(s); return rc; } static void error_refetch(const char *e) { ln_log(LNLOG_SERR, LNLOG_CTOP, "ERROR: FETCHNEWS MUST REFETCH THE WHOLE ACTIVE FILE NEXT RUN."); ln_log(LNLOG_SERR, LNLOG_CTOP, "REASON: %s", e); } /** re-add non-expiring groups to active */ static void addnonexpiring(void) { struct stringlist *t, *l = get_grouplist(); int expdays; for (t=l; t != NULL; t = t->next) { char *x = t->string; if ((expdays = lookup_expiredays(x)) >= 0) { if (expdays == 0 || !(expdays = lookup_expire(x))) expdays = expire; } else { expdays = -1; } if (expdays == -1) { insertgroup(x, 0, 0, 0); } } freelist(l); } int main(int argc, char **argv) { /* the volatile keyboard avoids clobbering by siglongjmp */ volatile time_t lastrun; volatile int rc = 0, skip_servers = 0; volatile int anypost = 0, waitchild = 0, quiet; struct server *current_server; volatile int need_refetch = 0; int option, reply; pid_t pid; verbose = quiet = 0; postonly = waitchild = 0; myopenlog("fetchnews"); if (!initvars(argv[0])) exit(1); while ((option = getopt(argc, argv, "Pfhlnvx:qw")) != -1) { if (option == 'v') { verbose++; quiet = 0; } else if (option == 'h') { usage(); exit(EXIT_SUCCESS); } else if (option == 'x') { char *nptr, *endptr; nptr = optarg; endptr = NULL; extraarticles = strtoul(nptr, &endptr, 0); if (!nptr || !*nptr || !endptr || *endptr || !extraarticles) { usage(); exit(1); } syslog(LOG_NOTICE, "fetchnews: run with option -x %lu", extraarticles); } else if (option == 'l') { usesupplement = 0; /* don't use supplementary servers */ } else if (option == 'n') { noexpire = 1; } else if (option == 'f') { forceactive = 1; } else if (option == 'P') { postonly = 1; } else if (option == 'q') { verbose = 0; quiet = 1; } else if (option == 'w') { waitchild = 1; } else { usage(); exit(1); } } /* Set line buffering to ensure that logging gets displayed promptly. */ if (mysetvbuf(stdout, NULL, _IOLBF, 4096)) { /* Try to at least use unbuffered then */ mysetvbuf(stdout, NULL, _IONBF, 0); } now = time(NULL); umask(2); if (!readconfig(0)) { fprintf(stderr, "Reading configuration failed, exiting " "(see syslog for more information).\n"); freeconfig(); exit(1); } if (debugmode) syslog(LOG_DEBUG, "leafnode %s: verbosity level is %d, debugmode is %d", version, verbose, debugmode); if (verbose || debugmode) { printf("leafnode %s: verbosity level is %d, debugmode is %d\n", version, verbose, debugmode); if (verbose > 1 && noexpire) { printf("Don't automatically unsubscribe unread newsgroups.\n"); } } if (forceactive) { ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Forced active fetch requested from command-line (option -f)."); } if (try_lock(timeout_lock)) { ln_log(LNLOG_SERR, LNLOG_CTOP, "Cannot obtain lock file, aborting."); freeconfig(); exit(1); } if (!postonly) { readactive(); if (!active) { addnonexpiring(); fakeactive(); /* we need proper lowwater/highwater marks for the groups that exist */ ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Forced active fetch after trouble reading active file."); forceactive = 1; } readfilter(filterfile); } lastrun = 0; if (!postonly && !forceactive) { lastrun = checkactive(); if (!lastrun) { ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Active has not been fetched completely in previous run"); ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "or has never been fetched, forcing active fetch."); forceactive = 1; } } if (!postonly && !forceactive) { int staterror; struct stat st; char s[SIZE_s+1]; xsnprintf(s, SIZE_s, "%s/leaf.node/groupinfo", spooldir); if ((staterror = stat(s, &st)) && errno != ENOENT) { /* this should happen only when the problem occurs after * readactive() above, but needs to be checked nonetheless */ ln_log(LNLOG_SERR, LNLOG_CTOP, "Cannot open %s: %m", s); unlink(lockfile); freeconfig(); exit(EXIT_FAILURE); } if (staterror || st.st_size < 7) { ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Groupinfo file %s not present or too small, " "forcing active fetch.", s); forceactive = 1; } } if (forceactive) { oldactive = active; oldactivesize = activesize; active = NULL; activesize = 0; if (killactiveread()) exit(1); addnonexpiring(); } if (mysigact(SIGINT, 0, sig_int, SIGALRM) != 0) fprintf(stderr, "Can't catch SIGINT.\n"); if (mysigact(SIGTERM, 0, sig_int, SIGALRM) != 0) fprintf(stderr, "Can't catch SIGTERM.\n"); if (mysigact(SIGPIPE, 0, sig_int, SIGALRM) != 0) fprintf(stderr, "Can't catch SIGPIPE.\n"); { int sig = sigsetjmp(jmpbuffer, 1); if (sig != 0) { ln_log(LNLOG_SWARNING, LNLOG_CTOP, "fetchnews: caught signal %d, shutting down.", sig); nntpquit(); if (!rc) rc = 2; if (forceactive) { error_refetch("caught signal that caused a premature abort."); need_refetch = 1; } skip_servers = 1; /* in this case, jump the while ... loop */ } else { canjump = 1; } } /* remove groups that haven't been read in a long time */ if (!noexpire) expire_interesting(); mgetaline_settimeout(timeout_fetchnews); /* main server loop */ for (current_server = servers; !skip_servers && current_server; current_server = current_server->next) { if (verbose) { if (current_server->port) printf("%s: connecting to port %u...\n", current_server->name, current_server->port); else printf("%s: connecting to port nntp...\n", current_server->name); } fflush(stdout); reply = nntpconnect(current_server); if (reply) { int r2; if (verbose) { int namlen = strlen(current_server->name); printf("%s: connected.\n", current_server->name); if (stat_is_evil) printf("%s: server software does not implement\n" "%*s STAT properly,\n" "%*s using workaround with HEAD instead,\n" "%*s at the expense of bandwidth.\n", current_server->name, namlen, "", namlen, "", namlen, ""); else printf("%s: using STAT command.\n", current_server->name); } if (current_server->username) if (!authenticate(current_server) && current_server->password) ln_log(LNLOG_SERR, LNLOG_CTOP, "error: may have been caused by premature authentication and be rather harmless."); /* Get INN's nnrpd on the phone */ xsnprintf(lineout, SIZE_lineout, "MODE READER\r\n"); putaline(); r2 = nntpreply(current_server); if (r2 < 400) reply = r2; if (reply == 498) { ln_log(LNLOG_SERR, LNLOG_CTOP, "%s: protocol error after sending mode reader", current_server->name); } else { if (reply == 200 && current_server->nopost == 0) { anypost = 1; if (-1 == postarticles(current_server)) { nntpquit(); goto connfail; } } else if (verbose) { printf("Not posting to %s: ", current_server->name); if (reply != 200) printf("non-permission "); if (current_server->nopost) printf("nopost-set "); printf("\n"); } if (!date_is_evil) check_date(current_server); if (!postonly && !current_server->noread) { time_t stamp = lastrun; /* get list of newsgroups or new newsgroups */ if (current_server->updateactive) { int r; if ((r = nntpactive(current_server, &stamp))) { if (forceactive || r == -2) { error_refetch("obtaining the active file failed."); need_refetch = 1; } rc = 1; } } else { if (verbose) printf("%s: not attempting to update newsgroups list\n", current_server->name); } processupstream(current_server, stamp); } } nntpquit(); if (verbose) printf("%s: conversation completed, disconnected.\n", current_server->name); } else { /* reply = nntpconnect */ if (verbose) printf("%s: connection failed.\n", current_server->name); connfail: if (forceactive && current_server->updateactive && !postonly && !current_server->noread) { error_refetch("needed to fetch the active list from the server but couldn't connect."); need_refetch = 1; } rc = 2; } if (!usesupplement) break; } mergegroups(); /* just in case we were interrupted while downloading the list. */ (void)mysigact(SIGINT, 0, SIG_IGN, 0); /* do not siglongjmp any more */ (void)mysigact(SIGTERM, 0, SIG_IGN, 0); /* do not siglongjmp any more */ (void)mysigact(SIGPIPE, 0, SIG_IGN, 0); /* SIGPIPE should not happen below */ if (rc != 0 && oldactive) { /* restore old active data to keep low/high marks */ unsigned long i; for (i = 0; i < oldactivesize; i++) { insertgroup(oldactive[i].name, 0, 0, 0); } mergegroups(); killactiveread(); } freeactive(oldactive); oldactive = NULL; oldactivesize = 0; if (anypost == 0 && skip_servers == 0 && rc != 2) { const char *e = "WARNING: found no server with posting permission!"; if (!quiet) fprintf(stderr, "%s\n", e); syslog(LOG_WARNING, "%s", e); } if (rc == 2) { const char *e = "WARNING: some servers have not been queried!"; if (!quiet) fprintf(stderr, "%s\n", e); syslog(LOG_WARNING, "%s", e); } if (fflush(stdout)) { /* to avoid double logging of stuff */ ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot flush standard output: %m"); } { /* OK, we do a pipe trick to hold the child until the parent * handed over the lock file (this must happen so it does not * inadvertently get removed as stale). The child reads from the * pipe and is blocked until the parent has written to the pipe. * The parent writes to the pipe after it has handed over the * lock. */ int pfd[2], pipeok; pipeok = pipe(pfd); if (!postonly) { /* only update active.read when have read active from all servers */ if (writeactive()) { ln_log(LNLOG_SERR, LNLOG_CTOP, "Error writing groupinfo."); error_refetch("cannot write groupinfo file."); rc = 1; /* mark for refetch */ killactiveread(); } else { /* active written successfully */ if (updateactive()) { error_refetch("cannot update active.read file."); need_refetch = 1; rc = 1; } if (need_refetch) { /* mark for refetch */ killactiveread(); } } #ifdef HAVE_WORKING_FORK pid = waitchild ? -1 : fork(); #else pid = -1; waitchild = 1; #endif switch (pid) { case -1: if (!waitchild) syslog(LOG_NOTICE, "fork: %m, running on parent schedule."); if (verbose) printf("updating overview data in the foreground...\n"); fixxover(); unlink(lockfile); if (verbose) printf("done.\n"); break; case 0: (void)setsid(); if (debugmode) syslog(LOG_DEBUG, "Process forked."); if (pipeok == 0) { /* wait for parent to hand over the lock */ char buf[4]; (void)close(pfd[1]); /* we don't REALLY check for errors here, the worst thing * that could happen (in case of a kernal bug...) to * us is that we start early and delete the parent's * lock file before the parent handed it over -- but * at that time, we'll then exit and everything is * in order, with just one bogus message in the log. */ while (read(pfd[0], buf, sizeof(buf)) < 0) { if (errno != EAGAIN && errno != EINTR) break; } (void)close(pfd[0]); } fixxover(); freeactive(active); if (unlink(lockfile)) ln_log(LNLOG_SERR, LNLOG_CTOP, "unlink(\"%s\"): %m", lockfile); if (debugmode) syslog(LOG_DEBUG, "Process done."); freeconfig(); _exit(0); break; default: { int lock_ok = handover_lock(pid); if (verbose) puts("Started process to update overview data in the background.\nNetwork activity has finished."); if (pipeok == 0) { /* tell child it has the lock */ (void)close(pfd[0]); (void)writes(pfd[1], "GO"); (void)close(pfd[1]); } if (lock_ok) { /* could not hand over lock file to child, so wait until it dies */ syslog(LOG_NOTICE, "could not hand over lockfile to child %lu: %m, " "waiting until child is done.", (unsigned long)pid); (void)waitpid(pid, NULL, 0); } else { syslog(LOG_INFO, "child has process ID %lu", (unsigned long)pid); } break; } } } else { /* if (postonly) */ unlink(lockfile); } } freeactive(active); freeconfig(); exit(rc); }