"""Options Abstract: Options.options is a globally shared options object. This object is initialised when the module is loaded: the envar BAYESCUSTOMIZE is checked for a list of names, if nothing is found then the local directory and the home directory are checked for a file called bayescustomize.ini or .spambayesrc (respectively) and the initial values are loaded from this. The Option class is defined in OptionsClass.py - this module is responsible only for instantiating and loading the globally shared instance. To Do: o Suggestions? """ import sys, os try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 __all__ = ['options'] # Grab the stuff from the core options class. from OptionsClass import * # Format: # defaults is a dictionary, where the keys are the section names # each key maps to a tuple consisting of: # option name, display name, default, # doc string, possible values, restore on restore-to-defaults defaults = { "Tokenizer" : ( ("basic_header_tokenize", "Basic header tokenising", False, """If true, tokenizer.Tokenizer.tokenize_headers() will tokenize the contents of each header field just like the text of the message body, using the name of the header as a tag. Tokens look like "header:word". The basic approach is simple and effective, but also very sensitive to biases in the ham and spam collections. For example, if the ham and spam were collected at different times, several headers with date/time information will become the best discriminators. (Not just Date, but Received and X-From_.)""", BOOLEAN, RESTORE), ("basic_header_tokenize_only", "Only basic header tokenising", False, """If true and basic_header_tokenize is also true, then basic_header_tokenize is the only action performed.""", BOOLEAN, RESTORE), ("basic_header_skip", "Basic headers to skip", ("received date x-.*",), """If basic_header_tokenize is true, then basic_header_skip is a set of headers that should be skipped.""", HEADER_NAME, RESTORE), ("check_octets", "Check application/octet-stream sections", False, """If true, the first few characters of application/octet-stream sections are used, undecoded. What 'few' means is decided by octet_prefix_size.""", BOOLEAN, RESTORE), ("octet_prefix_size", "Number of characters of octet stream to process", 5, """The number of characters of the application/octet-stream sections to use, if check_octets is set to true.""", INTEGER, RESTORE), ("count_all_header_lines", "Count all header lines", False, """Generate tokens just counting the number of instances of each kind of header line, in a case-sensitive way. Depending on data collection, some headers are not safe to count. For example, if ham is collected from a mailing list but spam from your regular inbox traffic, the presence of a header like List-Info will be a very strong ham clue, but a bogus one. In that case, set count_all_header_lines to False, and adjust safe_headers instead.""", BOOLEAN, RESTORE), ("record_header_absence", "Record header absence", False, """When True, generate a "noheader:HEADERNAME" token for each header in safe_headers (below) that *doesn't* appear in the headers. This helped in various of Tim's python.org tests, but appeared to hurt a little in Anthony Baxter's tests.""", BOOLEAN, RESTORE), ("safe_headers", "Safe headers", ("abuse-reports-to", "date", "errors-to", "from", "importance", "in-reply-to", "message-id", "mime-version", "organization", "received", "reply-to", "return-path", "subject", "to", "user-agent", "x-abuse-info", "x-complaints-to", "x-face"), """Like count_all_header_lines, but restricted to headers in this list. safe_headers is ignored when count_all_header_lines is true, unless record_header_absence is also true.""", HEADER_NAME, RESTORE), ("mine_received_headers", "Mine the received headers", False, """A lot of clues can be gotten from IP addresses and names in Received: headers. This can give spectacular results for bogus reasons if your corpora are from different sources.""", BOOLEAN, RESTORE), ("address_headers", "Address headers to mine", ("from", "to", "cc", "sender", "reply-to"), """Mine the following address headers. If you have mixed source corpuses (as opposed to a mixed sauce walrus, which is delicious!) then you probably don't want to use 'to' or 'cc') Address headers will be decoded, and will generate charset tokens as well as the real address. Others to consider: to, cc, reply-to, errors-to, sender, ...""", HEADER_NAME, RESTORE), ("generate_long_skips", "Generate long skips", True, """If legitimate mail contains things that look like text to the tokenizer and turning turning off this option helps (perhaps binary attachments get 'defanged' by something upstream from this operation and thus look like text), this may help, and should be an alert that perhaps the tokenizer is broken.""", BOOLEAN, RESTORE), ("summarize_email_prefixes", "Summarise email prefixes", False, """Try to capitalize on mail sent to multiple similar addresses.""", BOOLEAN, RESTORE), ("summarize_email_suffixes", "Summarise email suffixes", False, """Try to capitalize on mail sent to multiple similar addresses.""", BOOLEAN, RESTORE), ("skip_max_word_size", "Long skip trigger length", 12, """Length of words that triggers 'long skips'. Longer than this triggers a skip.""", INTEGER, RESTORE), ("x-generate_time_buckets", "Generate time buckets", False, """(DEPRECATED) Generate tokens which resemble the posting time in 10-minute buckets: 'time:' hour ':' minute//10""", BOOLEAN, RESTORE), ("x-extract_dow", "Extract day-of-week", False, """(DEPRECATED) Extract day of the week tokens from the Date: header.""", BOOLEAN, RESTORE), ("x-pick_apart_urls", "Extract clues about url structure", False, """(EXPERIMENTAL) Note whether url contains non-standard port or user/password elements.""", BOOLEAN, RESTORE), ("x-fancy_url_recognition", "Extract URLs without http:// prefix", False, """(EXPERIMENTAL) Recognize 'www.python.org' or ftp.python.org as URLs instead of just long words.""", BOOLEAN, RESTORE), ("replace_nonascii_chars", "Replace non-ascii characters", False, """If true, replace high-bit characters (ord(c) >= 128) and control characters with question marks. This allows non-ASCII character strings to be identified with little training and small database burden. It's appropriate only if your ham is plain 7-bit ASCII, or nearly so, so that the mere presence of non-ASCII character strings is known in advance to be a strong spam indicator.""", BOOLEAN, RESTORE), ("x-search_for_habeas_headers", "Search for Habeas Headers", False, """(EXPERIMENTAL) If true, search for the habeas headers (see http://www.habeas.com). If they are present and correct, this should be a strong ham sign, if they are present and incorrect, this should be a strong spam sign.""", BOOLEAN, RESTORE), ("x-reduce_habeas_headers", "Reduce Habeas Header Tokens to Single", False, """(EXPERIMENTAL) If SpamBayes is set to search for the Habeas headers, nine tokens are generated for messages with habeas headers. This should be fine, since messages with the headers should either be ham, or result in FN so that we can send them to habeas so they can be sued. However, to reduce the strength of habeas headers, we offer the ability to reduce the nine tokens to one. (This option has no effect if search_for_habeas_headers is False)""", BOOLEAN, RESTORE), ), # These options are all experimental; it seemed better to put them into # their own category than have several interdependant experimental options. # If this capability is removed, the entire section can go. "URLRetriever" : ( ("x-slurp_urls", "Tokenize text content at the end of URLs", False, """(EXPERIMENTAL) If this option is enabled, when a message normally scores in the 'unsure' range, and has fewer tokens than the maximum looked at, and contains URLs, then the text at those URLs is obtained and tokenized. If those tokens result in the message moving to a score outside the 'unsure' range, then they are added to the tokens for the message. This should be particularly effective for messages that contain only a single URL and no other text.""", BOOLEAN, RESTORE), ("x-cache_expiry_days", "Number of days to store URLs in cache", 7, """(EXPERIMENTAL) This is the number of days that local cached copies of the text at the URLs will be stored for.""", INTEGER, RESTORE), ("x-cache_directory", "URL Cache Directory", "url-cache", """(EXPERIMENTAL) So that SpamBayes doesn't need to retrieve the same URL over and over again, it stores local copies of the text at the end of the URL. This is the directory that will be used for those copies.""", PATH, RESTORE), ("x-only_slurp_base", "Retrieve base url", False, """(EXPERIMENTAL) To try and speed things up, and to avoid following unique URLS, if this option is enabled, SpamBayes will convert the URL to as basic a form it we can. All directory information is removed and the domain is reduced to the two (or three for those with a country TLD) top-most elements. For example, http://www.massey.ac.nz/~tameyer/index.html?you=me would become http://massey.ac.nz and http://id.example.com would become http://example.com This should have two beneficial effects: o It's unlikely that any information could be contained in this 'base' url that could identify the user (unless they have a *lot* of domains). o Many urls (both spam and ham) will strip down into the same 'base' url. Since we have a limited form of caching, this means that a lot fewer urls will have to be retrieved. However, this does mean that if the 'base' url is hammy and the full is spammy, or vice-versa, that the slurp will give back the wrong information. Whether or not this is the case would have to be determined by testing. """, BOOLEAN, RESTORE), ("x-web_prefix", "Prefix for tokens from web pages", "", """(EXPERIMENTAL) It may be that what is hammy/spammy for you in email isn't from webpages. You can then set this option (to "web:", for example), and effectively create an independent (sub)database for tokens derived from parsing web pages.""", r"[\S]+", RESTORE), ), # These options control how a message is categorized "Categorization" : ( # spam_cutoff and ham_cutoff are used in Python slice sense: # A msg is considered ham if its score is in 0:ham_cutoff # A msg is considered unsure if its score is in ham_cutoff:spam_cutoff # A msg is considered spam if its score is in spam_cutoff: # # So it's unsure iff ham_cutoff <= score < spam_cutoff. # For a binary classifier, make ham_cutoff == spam_cutoff. # ham_cutoff > spam_cutoff doesn't make sense. # # The defaults here (.2 and .9) may be appropriate for the default chi- # combining scheme. Cutoffs for chi-combining typically aren't touchy, # provided you're willing to settle for "really good" instead of "optimal". # Tim found that .3 and .8 worked very well for well-trained systems on # his personal email, and his large comp.lang.python test. If just # beginning training, or extremely fearful of mistakes, 0.05 and 0.95 may # be more appropriate for you. # # Picking good values for gary-combining is much harder, and appears to be # corpus-dependent, and within a single corpus dependent on how much # training has been done. Values from 0.50 thru the low 0.60's have been # reported to work best by various testers on their data. ("ham_cutoff", "Ham cutoff", 0.20, """Spambayes gives each email message a spam probability between 0 and 1. Emails below the Ham Cutoff probability are classified as Ham. Larger values will result in more messages being classified as ham, but with less certainty that all of them actually are ham. This value should be between 0 and 1, and should be smaller than the Spam Cutoff.""", REAL, RESTORE), ("spam_cutoff", "Spam cutoff", 0.90, """Emails with a spam probability above the Spam Cutoff are classified as Spam - just like the Ham Cutoff but at the other end of the scale. Messages that fall between the two values are classified as Unsure.""", REAL, RESTORE), ), # These control various displays in class TestDriver.Driver, and # Tester.Test. "TestDriver" : ( ("nbuckets", "Number of buckets", 200, """Number of buckets in histograms.""", INTEGER, RESTORE), ("show_histograms", "Show histograms", True, """""", BOOLEAN, RESTORE), ("compute_best_cutoffs_from_histograms", "Compute best cutoffs from histograms", True, """After the display of a ham+spam histogram pair, you can get a listing of all the cutoff values (coinciding with histogram bucket boundaries) that minimize: best_cutoff_fp_weight * (# false positives) + best_cutoff_fn_weight * (# false negatives) + best_cutoff_unsure_weight * (# unsure msgs) This displays two cutoffs: hamc and spamc, where 0.0 <= hamc <= spamc <= 1.0 The idea is that if something scores < hamc, it's called ham; if something scores >= spamc, it's called spam; and everything else is called 'I am not sure' -- the middle ground. Note: You may wish to increase nbuckets, to give this scheme more cutoff values to analyze.""", BOOLEAN, RESTORE), ("best_cutoff_fp_weight", "Best cutoff false positive weight", 10.00, """""", REAL, RESTORE), ("best_cutoff_fn_weight", "Best cutoff false negative weight", 1.00, """""", REAL, RESTORE), ("best_cutoff_unsure_weight", "Best cutoff unsure weight", 0.20, """""", REAL, RESTORE), ("percentiles", "Percentiles", (5, 25, 75, 95), """Histogram analysis also displays percentiles. For each percentile p in the list, the score S such that p% of all scores are <= S is given. Note that percentile 50 is the median, and is displayed (along with the min score and max score) independent of this option.""", INTEGER, RESTORE), ("show_spam_lo", "", 1.0, """Display spam when show_spam_lo <= spamprob <= show_spam_hi and likewise for ham. The defaults here do not show anything.""", REAL, RESTORE), ("show_spam_hi", "", 0.0, """Display spam when show_spam_lo <= spamprob <= show_spam_hi and likewise for ham. The defaults here do not show anything.""", REAL, RESTORE), ("show_ham_lo", "", 1.0, """Display spam when show_spam_lo <= spamprob <= show_spam_hi and likewise for ham. The defaults here do not show anything.""", REAL, RESTORE), ("show_ham_hi", "", 0.0, """Display spam when show_spam_lo <= spamprob <= show_spam_hi and likewise for ham. The defaults here do not show anything.""", REAL, RESTORE), ("show_false_positives", "Show false positives", True, """""", BOOLEAN, RESTORE), ("show_false_negatives", "Show false negatives", False, """""", BOOLEAN, RESTORE), ("show_unsure", "Show unsure", False, """""", BOOLEAN, RESTORE), ("show_charlimit", "Show character limit", 3000, """The maximum # of characters to display for a msg displayed due to the show_xyz options above.""", INTEGER, RESTORE), ("save_trained_pickles", "Save trained pickles", False, """If save_trained_pickles is true, Driver.train() saves a binary pickle of the classifier after training. The file basename is given by pickle_basename, the extension is .pik, and increasing integers are appended to pickle_basename. By default (if save_trained_pickles is true), the filenames are class1.pik, class2.pik, ... If a file of that name already exists, it is overwritten. pickle_basename is ignored when save_trained_pickles is false.""", BOOLEAN, RESTORE), ("pickle_basename", "Pickle basename", "class", """""", r"[\w]+", RESTORE), ("save_histogram_pickles", "Save histogram pickles", False, """If save_histogram_pickles is true, Driver.train() saves a binary pickle of the spam and ham histogram for "all test runs". The file basename is given by pickle_basename, the suffix _spamhist.pik or _hamhist.pik is appended to the basename.""", BOOLEAN, RESTORE), ("spam_directories", "Spam directories", "Data/Spam/Set%d", """default locations for timcv and timtest - these get the set number interpolated.""", VARIABLE_PATH, RESTORE), ("ham_directories", "Ham directories", "Data/Ham/Set%d", """default locations for timcv and timtest - these get the set number interpolated.""", VARIABLE_PATH, RESTORE), ), "CV Driver": ( ("build_each_classifier_from_scratch", "Build each classifier from scratch", False, """A cross-validation driver takes N ham+spam sets, and builds N classifiers, training each on N-1 sets, and the predicting against the set not trained on. By default, it does this in a clever way, learning *and* unlearning sets as it goes along, so that it never needs to train on N-1 sets in one gulp after the first time. Setting this option true forces ''one gulp from-scratch'' training every time. There used to be a set of combining schemes that needed this, but now it is just in case you are paranoid .""", BOOLEAN, RESTORE), ), "Classifier": ( ("max_discriminators", "Maximum number of extreme words", 150, """The maximum number of extreme words to look at in a message, where "extreme" means with spam probability farthest away from 0.5. 150 appears to work well across all corpora tested.""", INTEGER, RESTORE), ("unknown_word_prob", "Unknown word probability", 0.5, """These two control the prior assumption about word probabilities. unknown_word_prob is essentially the probability given to a word that has never been seen before. Nobody has reported an improvement via moving it away from 1/2, although Tim has measured a mean spamprob of a bit over 0.5 (0.51-0.55) in 3 well-trained classifiers.""", REAL, RESTORE), ("unknown_word_strength", "Unknown word strength", 0.45, """This adjusts how much weight to give the prior assumption relative to the probabilities estimated by counting. At 0, the counting estimates are believed 100%, even to the extent of assigning certainty (0 or 1) to a word that has appeared in only ham or only spam. This is a disaster. As unknown_word_strength tends toward infintity, all probabilities tend toward unknown_word_prob. All reports were that a value near 0.4 worked best, so this does not seem to be corpus-dependent.""", REAL, RESTORE), ("minimum_prob_strength", "Minimum probability strength", 0.1, """When scoring a message, ignore all words with abs(word.spamprob - 0.5) < minimum_prob_strength. This may be a hack, but it has proved to reduce error rates in many tests. 0.1 appeared to work well across all corpora.""", REAL, RESTORE), ("use_chi_squared_combining", "Use chi-squared combining", True, """For vectors of random, uniformly distributed probabilities, -2*sum(ln(p_i)) follows the chi-squared distribution with 2*n degrees of freedom. This is the "provably most-sensitive" test the original scheme was monotonic with. Getting closer to the theoretical basis appears to give an excellent combining method, usually very extreme in its judgment, yet finding a tiny (in # of msgs, spread across a huge range of scores) middle ground where lots of the mistakes live. This is the best method so far. One systematic benefit is is immunity to "cancellation disease". One systematic drawback is sensitivity to *any* deviation from a uniform distribution, regardless of whether actually evidence of ham or spam. Rob Hooft alleviated that by combining the final S and H measures via (S-H+1)/2 instead of via S/(S+H)). In practice, it appears that setting ham_cutoff=0.05, and spam_cutoff=0.95, does well across test sets; while these cutoffs are rarely optimal, they get close to optimal. With more training data, Tim has had good luck with ham_cutoff=0.30 and spam_cutoff=0.80 across three test data sets (original c.l.p data, his own email, and newer general python.org traffic).""", BOOLEAN, RESTORE), # If the # of ham and spam in training data are out of balance, the # spamprob guesses can get stronger in the direction of the category # with more training msgs. In one sense this must be so, since the more # data we have of one flavor, the more we know about that flavor. But # that allows the accidental appearance of a strong word of that flavor # in a msg of the other flavor much more power than an accident in the # other direction. Enable experimental_ham_spam_imbalance_adjustment if # you have more ham than spam training data (or more spam than ham), and # the Bayesian probability adjustment won't 'believe' raw counts more # than min(# ham trained on, # spam trained on) justifies. I *expect* # this option will go away (and become the default), but people *with* # strong imbalance need to test it first.\ # LATER: this option sucked, creating more problems than it solved. # It's deprecated, and the support code has gone away. ("x-experimental_ham_spam_imbalance_adjustment", "Compensate for unequal numbers of spam and ham", False, """(DEPRECATED) If your training database has significantly more ham than spam, or vice versa, you may start seeing an increase in incorrect classifications (messages put in the wrong category, not just marked as unsure). If so, this option allows you to compensate for this, at the cost of increasing the number of messages classified as "unsure". Note that the effect is subtle, and you should experiment with both settings to choose the option that suits you best. You do not have to retrain your database if you change this option.""", BOOLEAN, RESTORE), ("x-use_bigrams", "Use mixed uni/bi-grams scheme", False, """(EXPERIMENTAL) Generate both unigrams (words) and bigrams (pairs of words). However, extending an idea originally from Gary Robinson, the message is 'tiled' into non-overlapping unigrams and bigrams, approximating the strongest outcome over all possible tilings. Note that to really test this option you need to retrain with it on, so that your database includes the bigrams - if you subsequently turn it off, these tokens will have no effect. This option will at least double your database size given the same training data, and will probably at least triple it. You may also wish to increase the max_discriminators (maximum number of extreme words) option if you enable this option, perhaps doubling or quadrupling it. It's not yet clear. Bigrams create many more hapaxes, and that seems to increase the brittleness of minimalist training regimes; increasing max_discriminators may help to soften that effect. OTOH, max_discriminators defaults to 150 in part because that makes it easy to prove that the chi-squared math is immune from numeric problems. Increase it too much, and insane results will eventually result (including fatal floating-point exceptions on some boxes). This option is experimental, and may be removed in a future release. We would appreciate feedback about it if you use it - email spambayes@python.org with your comments and results. """, BOOLEAN, RESTORE), ), "Hammie": ( ("train_on_filter", "Train when filtering", False, """Train when filtering? After filtering a message, hammie can then train itself on the judgement (ham or spam). This can speed things up with a procmail-based solution. If you do enable this, please make sure to retrain any mistakes. Otherwise, your word database will slowly become useless.""", BOOLEAN, RESTORE), ), # These options control where Spambayes data will be stored, and in # what form. They are used by many Spambayes applications (including # pop3proxy, smtpproxy, imapfilter and hammie), and mean that data # (such as the message database) is shared between the applications. # If this is not the desired behaviour, you must have a different # value for each of these options in a configuration file that gets # loaded by the appropriate application only. "Storage" : ( ("persistent_use_database", "Use database for storage", "dbm", """SpamBayes can use either a database (quick to score one message) or a pickle (quick to train on huge amounts of messages). There is also (currently experimental) the ability to use a mySQL or PostgrepSQL database. For historical reasons, if you set this to "True" you are selecting "dbm" and if you set this to "False" you are selecting "pickle". We recommend explicitly selecting the type, (i.e. changing "True" to "dbm" and "False" to "pickle", or sticking with the default.""", # True == "dbm", False == "pickle", "True" == "dbm", "False" == "pickle" ("mysql", "pgsql", "dbm", "pickle", "True", "False", True, False), RESTORE), ("persistent_storage_file", "Storage file name", "hammie.db", """Spambayes builds a database of information that it gathers from incoming emails and from you, the user, to get better and better at classifying your email. This option specifies the name of the database file. If you don't give a full pathname, the name will be taken to be relative to the location of the most recent configuration file loaded.""", FILE_WITH_PATH, DO_NOT_RESTORE), ("messageinfo_storage_file", "Message information file name", "spambayes.messageinfo.db", """Spambayes builds a database of information about messages that it has already seen and trained or classified. This database is used to ensure that these messages are not retrained or reclassified (unless specifically requested to). This option specifies the name of the database file. If you don't give a full pathname, the name will be taken to be relative to the location of the most recent configuration file loaded.""", FILE_WITH_PATH, DO_NOT_RESTORE), ("cache_use_gzip", "Use gzip", False, """Use gzip to compress the cache.""", BOOLEAN, RESTORE), ("cache_expiry_days", "Days before cached messages expire", 7, """Messages will be expired from the cache after this many days. After this time, you will no longer be able to train on these messages (note this does not affect the copy of the message that you have in your mail client).""", INTEGER, RESTORE), ("spam_cache", "Spam cache directory", "pop3proxy-spam-cache", """Directory that SpamBayes should cache spam in. If this does not exist, it will be created.""", PATH, DO_NOT_RESTORE), ("ham_cache", "Ham cache directory", "pop3proxy-ham-cache", """Directory that SpamBayes should cache ham in. If this does not exist, it will be created.""", PATH, DO_NOT_RESTORE), ("unknown_cache", "Unknown cache directory", "pop3proxy-unknown-cache", """Directory that SpamBayes should cache unclassified messages in. If this does not exist, it will be created.""", PATH, DO_NOT_RESTORE), ("cache_messages", "Cache messages", True, """You can disable the pop3proxy caching of messages. This will make the proxy a bit faster, and make it use less space on your hard drive. The proxy uses its cache for reviewing and training of messages, so if you disable caching you won't be able to do further training unless you re-enable it. Thus, you should only turn caching off when you are satisfied with the filtering that Spambayes is doing for you.""", BOOLEAN, RESTORE), ("no_cache_bulk_ham", "Suppress caching of bulk ham", False, """Where message caching is enabled, this option suppresses caching of messages which are classified as ham and marked as 'Precedence: bulk' or 'Precedence: list'. If you subscribe to a high-volume mailing list then your 'Review messages' page can be overwhelmed with list messages, making training a pain. Once you've trained Spambayes on enough list traffic, you can use this option to prevent that traffic showing up in 'Review messages'.""", BOOLEAN, RESTORE), ("no_cache_large_messages", "Maximum size of cached messages", 0, """Where message caching is enabled, this option suppresses caching of messages which are larger than this value (measured in bytes). If you receive a lot of messages that include large attachments (and are correctly classified), you may not wish to cache these. If you set this to zero (0), then this option will have no effect.""", INTEGER, RESTORE), ), # These options control the various headers that some Spambayes # applications add to incoming mail, including imapfilter, pop3proxy, # and hammie. "Headers" : ( # The name of the header that hammie, pop3proxy, and any other spambayes # software, adds to emails in filter mode. This will definately contain # the "classification" of the mail, and may also (i.e. with hammie) # contain the score ("classification_header_name", "Classification header name", "X-Spambayes-Classification", """Spambayes classifies each message by inserting a new header into the message. This header can then be used by your email client (provided your client supports filtering) to move spam into a separate folder (recommended), delete it (not recommended), etc. This option specifies the name of the header that Spambayes inserts. The default value should work just fine, but you may change it to anything that you wish.""", HEADER_NAME, RESTORE), # The three disposition names are added to the header as the following # three words: ("header_spam_string", "Spam disposition name", "spam", """The header that Spambayes inserts into each email has a name, (Classification eader name, above), and a value. If the classifier determines that this email is probably spam, it places a header named as above with a value as specified by this string. The default value should work just fine, but you may change it to anything that you wish.""", HEADER_VALUE, RESTORE), ("header_ham_string", "Ham disposition name", "ham", """As for Spam Designation, but for emails classified as Ham.""", HEADER_VALUE, RESTORE), ("header_unsure_string", "Unsure disposition name", "unsure", """As for Spam/Ham Designation, but for emails which the classifer wasn't sure about (ie. the spam probability fell between the Ham and Spam Cutoffs). Emails that have this classification should always be the subject of training.""", HEADER_VALUE, RESTORE), ("header_score_digits", "Accuracy of reported score", 2, """Accuracy of the score in the header in decimal digits.""", INTEGER, RESTORE), ("header_score_logarithm", "Augment score with logarithm", False, """Set this option to augment scores of 1.00 or 0.00 by a logarithmic "one-ness" or "zero-ness" score (basically it shows the "number of zeros" or "number of nines" next to the score value).""", BOOLEAN, RESTORE), ("include_score", "Add probability (score) header", False, """You can have Spambayes insert a header with the calculated spam probability into each mail. If you can view headers with your mailer, then you can see this information, which can be interesting and even instructive if you're a serious SpamBayes junkie.""", BOOLEAN, RESTORE), ("score_header_name", "Probability (score) header name", "X-Spambayes-Spam-Probability", """""", HEADER_NAME, RESTORE), ("include_thermostat", "Add level header", False, """You can have spambayes insert a header with the calculated spam probability, expressed as a number of '*'s, into each mail (the more '*'s, the higher the probability it is spam). If your mailer supports it, you can use this information to fine tune your classification of ham/spam, ignoring the classification given.""", BOOLEAN, RESTORE), ("thermostat_header_name", "Level header name", "X-Spambayes-Level", """""", HEADER_NAME, RESTORE), ("include_evidence", "Add evidence header", False, """You can have spambayes insert a header into mail, with the evidence that it used to classify that message (a collection of words with ham and spam probabilities). If you can view headers with your mailer, then this may give you some insight as to why a particular message was scored in a particular way.""", BOOLEAN, RESTORE), ("evidence_header_name", "Evidence header name", "X-Spambayes-Evidence", """""", HEADER_NAME, RESTORE), ("mailid_header_name", "Spambayes id header name", "X-Spambayes-MailId", """""", HEADER_NAME, RESTORE), ("include_trained", "Add trained header", True, """sb_mboxtrain.py can add a header that details how a message was trained, which lets you keep track of it, and appropriately re-train messages. However, if you would rather mboxtrain didn't rewrite the message files, you can disable this option.""", BOOLEAN, RESTORE), ("trained_header_name", "Trained header name", "X-Spambayes-Trained", """When training on a message, the name of the header to add with how it was trained""", HEADER_NAME, RESTORE), ("clue_mailheader_cutoff", "Debug header cutoff", 0.5, """The range of clues that are added to the "debug" header in the E-mail. All clues that have their probability smaller than this number, or larger than one minus this number are added to the header such that you can see why spambayes thinks this is ham/spam or why it is unsure. The default is to show all clues, but you can reduce that by setting showclue to a lower value, such as 0.1""", REAL, RESTORE), ("add_unique_id", "Add unique spambayes id", True, """If you wish to be able to find a specific message (via the 'find' box on the home page), or use the SMTP proxy to train using cached messages, you will need to know the unique id of each message. This option adds this information to a header added to each message.""", BOOLEAN, RESTORE), ("notate_to", "Notate to", (), """Some email clients (Outlook Express, for example) can only set up filtering rules on a limited set of headers. These clients cannot test for the existence/value of an arbitrary header and filter mail based on that information. To accommodate these kind of mail clients, you can add "spam", "ham", or "unsure" to the recipient list. A filter rule can then use this to see if one of these words (followed by a comma) is in the recipient list, and route the mail to an appropriate folder, or take whatever other action is supported and appropriate for the mail classification. As it interferes with replying, you may only wish to do this for spam messages; simply tick the boxes of the classifications take should be identified in this fashion.""", ("ham", "spam", "unsure"), RESTORE), ("notate_subject", "Classify in subject: header", (), """This option will add the same information as 'Notate To', but to the start of the mail subject line.""", ("ham", "spam", "unsure"), RESTORE), ), # pop3proxy settings: The only mandatory option is pop3proxy_servers, eg. # "pop3.my-isp.com:110", or a comma-separated list of those. The ":110" # is optional. If you specify more than one server in pop3proxy_servers, # you must specify the same number of ports in pop3proxy_ports. "pop3proxy" : ( ("remote_servers", "Remote Servers", (), """The SpamBayes POP3 proxy intercepts incoming email and classifies it before sending it on to your email client. You need to specify which POP3 server(s) you wish it to intercept - a POP3 server address typically looks like "pop3.myisp.net". If you use more than one server, simply separate their names with commas. You can get these server names from your existing email configuration, or from your ISP or system administrator. If you are using Web-based email, you can't use the SpamBayes POP3 proxy (sorry!). In your email client's configuration, where you would normally put your POP3 server address, you should now put the address of the machine running SpamBayes.""", SERVER, DO_NOT_RESTORE), ("listen_ports", "SpamBayes Ports", (), """Each POP3 server that is being monitored must be assigned to a 'port' in the SpamBayes POP3 proxy. This port must be different for each monitored server, and there must be a port for each monitored server. Again, you need to configure your email client to use this port. If there are multiple servers, you must specify the same number of ports as servers, separated by commas. If you don't know what to use here, and you only have one server, try 110, or if that doesn't work, try 8110.""", SERVER, DO_NOT_RESTORE), ("allow_remote_connections", "Allowed remote POP3 connections", "localhost", """Enter a list of trusted IPs, separated by commas. Remote POP connections from any of them will be allowed. You can trust any IP using a single '*' as field value. You can also trust ranges of IPs using the '*' character as a wildcard (for instance 192.168.0.*). The localhost IP will always be trusted. Type 'localhost' in the field to trust this only address.""", IP_LIST, RESTORE), ), "smtpproxy" : ( ("remote_servers", "Remote Servers", (), """Use of the SMTP proxy is optional - if you would rather just train via the web interface, or the pop3dnd or mboxtrain scripts, then you can safely leave this option blank. The Spambayes SMTP proxy intercepts outgoing email - if you forward mail to one of the addresses below, it is examined for an id and the message corresponding to that id is trained as ham/spam. All other mail is sent along to your outgoing mail server. You need to specify which SMTP server(s) you wish it to intercept - a SMTP server address typically looks like "smtp.myisp.net". If you use more than one server, simply separate their names with commas. You can get these server names from your existing email configuration, or from your ISP or system administrator. If you are using Web-based email, you can't use the Spambayes SMTP proxy (sorry!). In your email client's configuration, where you would normally put your SMTP server address, you should now put the address of the machine running SpamBayes.""", SERVER, DO_NOT_RESTORE), ("listen_ports", "SpamBayes Ports", (), """Each SMTP server that is being monitored must be assigned to a 'port' in the Spambayes SMTP proxy. This port must be different for each monitored server, and there must be a port for each monitored server. Again, you need to configure your email client to use this port. If there are multiple servers, you must specify the same number of ports as servers, separated by commas.""", SERVER, DO_NOT_RESTORE), ("allow_remote_connections", "Allowed remote SMTP connections", "localhost", """Enter a list of trusted IPs, separated by commas. Remote SMTP connections from any of them will be allowed. You can trust any IP using a single '*' as field value. You can also trust ranges of IPs using the '*' character as a wildcard (for instance 192.168.0.*). The localhost IP will always be trusted. Type 'localhost' in the field to trust this only address. Note that you can unwittingly turn a SMTP server into an open proxy if you open this up, as connections to the server will appear to be from your machine, even if they are from a remote machine *through* your machine, to the server. We do not recommend opening this up fully (i.e. using '*'). """, IP_LIST, RESTORE), ("ham_address", "Train as ham address", "spambayes_ham@localhost", """When a message is received that you wish to train on (for example, one that was incorrectly classified), you need to forward or bounce it to one of two special addresses so that the SMTP proxy can identify it. If you wish to train it as ham, forward or bounce it to this address. You will want to use an address that is not a valid email address, like ham@nowhere.nothing.""", EMAIL_ADDRESS, RESTORE), ("spam_address", "Train as spam address", "spambayes_spam@localhost", """As with Ham Address above, but the address that you need to forward or bounce mail that you wish to train as spam. You will want to use an address that is not a valid email address, like spam@nowhere.nothing.""", EMAIL_ADDRESS, RESTORE), ("use_cached_message", "Lookup message in cache", False, """If this option is set, then the smtpproxy will attempt to look up the messages sent to it (for training) in the POP3 proxy cache or IMAP filter folders, and use that message as the training data. This avoids any problems where your mail client might change the message when forwarding, contaminating your training data. If you can be sure that this won't occur, then the id-lookup can be avoided. Note that Outlook Express users cannot use the lookup option (because of the way messages are forwarded), and so if they wish to use the SMTP proxy they must enable this option (but as messages are altered, may not get the best results, and this is not recommended).""", BOOLEAN, RESTORE), ), "html_ui" : ( ("port", "Port", 8880, """""", PORT, RESTORE), ("launch_browser", "Launch browser", False, """If this option is set, then whenever sb_server or sb_imapfilter is started the default web browser will be opened to the main web interface page. Use of the -b switch when starting from the command line overrides this option.""", BOOLEAN, RESTORE), ("allow_remote_connections", "Allowed remote UI connections", "localhost", """Enter a list of trusted IPs, separated by commas. Remote connections from any of them will be allowed. You can trust any IP using a single '*' as field value. You can also trust ranges of IPs using the '*' character as a wildcard (for instance 192.168.0.*). The localhost IP will always be trusted. Type 'localhost' in the field to trust this only address.""", IP_LIST, RESTORE), ("display_headers", "Headers to display in message review", ("Subject", "From"), """When reviewing messages via the web user interface, you are presented with various information about the message. By default, you are shown the subject and who the message is from. You can add other message headers to display, however, such as the address the message is to, or the date that the message was sent.""", HEADER_NAME, RESTORE), ("display_received_time", "Display date received in message review", False, """When reviewing messages via the web user interface, you are presented with various information about the message. If you set this option, you will be shown the date that the message was received. """, BOOLEAN, RESTORE), ("display_score", "Display score in message review", False, """When reviewing messages via the web user interface, you are presented with various information about the message. If you set this option, this information will include the score that the message received when it was classified. You might wish to see this purely out of curiousity, or you might wish to only train on messages that score towards the boundaries of the classification areas. Note that in order to use this option, you must also enable the option to include the score in the message headers.""", BOOLEAN, RESTORE), ("display_adv_find", "Display the advanced find query", False, """Present advanced options in the 'Word Query' box on the front page, including wildcard and regular expression searching.""", BOOLEAN, RESTORE), ("default_ham_action", "Default training for ham", "ham", """When presented with the review list in the web interface, which button would you like checked by default when the message is classified as ham?""", ("ham", "spam", "discard", "defer"), RESTORE), ("default_spam_action", "Default training for spam", "spam", """When presented with the review list in the web interface, which button would you like checked by default when the message is classified as spam?""", ("ham", "spam", "discard", "defer"), RESTORE), ("default_unsure_action", "Default training for unsure", "defer", """When presented with the review list in the web interface, which button would you like checked by default when the message is classified as unsure?""", ("ham", "spam", "discard", "defer"), RESTORE), ("ham_discard_level", "Ham Discard Level", 0.0, """Hams scoring less than this percentage will default to being discarded in the training interface (they won't be trained). You'll need to turn off the 'Train when filtering' option, above, for this to have any effect""", REAL, RESTORE), ("spam_discard_level", "Spam Discard Level", 100.0, """Spams scoring more than this percentage will default to being discarded in the training interface (they won't be trained). You'll need to turn off the 'Train when filtering' option, above, for this to have any effect""", REAL, RESTORE), ("http_authentication", "HTTP Authentication", "None", """This option lets you choose the security level of the web interface. When selecting Basic or Digest, the user will be prompted a login and a password to access the web interface. The Basic option is faster, but transmits the password in clear on the network. The Digest option encrypts the password before transmission.""", ("None", "Basic", "Digest"), RESTORE), ("http_user_name", "User name", "admin", """If you activated the HTTP authentication option, you can modify the authorized user name here.""", r"[\w]+", RESTORE), ("http_password", "Password", "admin", """If you activated the HTTP authentication option, you can modify the authorized user password here.""", r"[\w]+", RESTORE), ("rows_per_section", "Rows per section", 10000, """Number of rows to display per ham/spam/unsure section.""", INTEGER, RESTORE), ), "imap" : ( ("server", "Server", (), """This is the name and port of the imap server that stores your mail, and which the imap filter will connect to - for example: mail.example.com or imap.example.com:143. The default IMAP port is 143, or 993 if using SSL; if you connect via one of those ports, you can leave this blank. If you use more than one server, then things are a bit more complicated for you at the moment, sorry. You will need to have multiple instances of the imap filter running, each with a different server (and possibly username and password) value. You can do this if you have a different configuration file for each instance, but you'll have to do it by hand for the moment. Please let the mailing list know if you are in this situation so that we can consider coming up with a better solution.""", SERVER, DO_NOT_RESTORE), ("username", "Username", (), """This is the id that you use to log into your imap server. If your address is funkyguy@example.com, then your username is probably funkyguy. If you are using multiple imap servers, or multiple accounts on the same server, please see the comments regarding the server value.""", IMAP_ASTRING, DO_NOT_RESTORE), ("password", "Password", (), """That is that password that you use to log into your imap server. This will be stored in plain text in your configuration file, and if you have set the web user interface to allow remote connections, then it will be available for the whole world to see in plain text. If I've just freaked you out, don't panic . You can leave this blank and use the -p command line option to imapfilter.py and you will be prompted for your password.""", IMAP_ASTRING, DO_NOT_RESTORE), ("expunge", "Purge//Expunge", False, """Permanently remove *all* messages flagged with //Deleted on logout. If you do not know what this means, then please leave this as False.""", BOOLEAN, RESTORE), ("use_ssl", "Connect via a secure socket layer", False, """Use SSL to connect to the server. This allows spambayes to connect without sending the password in plain text. Note that this does not check the server certificate at this point in time.""", BOOLEAN, DO_NOT_RESTORE), ("filter_folders", "Folders to filter", ("INBOX",), """Comma delimited list of folders to be filtered""", IMAP_FOLDER, DO_NOT_RESTORE), ("unsure_folder", "Folder for unsure messages", "", """""", IMAP_FOLDER, DO_NOT_RESTORE), ("spam_folder", "Folder for suspected spam", "", """""", IMAP_FOLDER, DO_NOT_RESTORE), ("ham_train_folders", "Folders with mail to be trained as ham", (), """Comma delimited list of folders that will be examined for messages to train as ham.""", IMAP_FOLDER, DO_NOT_RESTORE), ("spam_train_folders", "Folders with mail to be trained as spam", (), """Comma delimited list of folders that will be examined for messages to train as spam.""", IMAP_FOLDER, DO_NOT_RESTORE), ("move_trained_spam_to_folder", "Folder to move trained spam to", "", """When training, all messages in the spam training folder(s) (above) are examined - if they are new, they are used to train, if not, they are ignored. This examination does take time, however, so if speed is an issue for you, you may wish to move messages out of this folder once they have been trained (either to delete them or to a storage folder). If a folder name is specified here, this will happen automatically. Note that the filter is not yet clever enough to move the mail to different folders depending on which folder it was originally in - *all* messages will be moved to the same folder.""", IMAP_FOLDER, DO_NOT_RESTORE), ("move_trained_ham_to_folder", "Folder to move trained ham to", "", """When training, all messages in the ham training folder(s) (above) are examined - if they are new, they are used to train, if not, they are ignored. This examination does take time, however, so if speed is an issue for you, you may wish to move messages out of this folder once they have been trained (either to delete them or to a storage folder). If a folder name is specified here, this will happen automatically. Note that the filter is not yet clever enough to move the mail to different folders depending on which folder it was originally in - *all* messages will be moved to the same folder.""", IMAP_FOLDER, DO_NOT_RESTORE), ), "ZODB" : ( ("zeo_addr", "", "", """""", IMAP_ASTRING, DO_NOT_RESTORE), ("event_log_file", "", "", """""", IMAP_ASTRING, RESTORE), ("folder_dir", "", "", """""", PATH, DO_NOT_RESTORE), ("ham_folders", "", "", """""", PATH, DO_NOT_RESTORE), ("spam_folders", "", "", """""", PATH, DO_NOT_RESTORE), ("event_log_severity", "", 0, """""", INTEGER, RESTORE), ("cache_size", "", 2000, """""", INTEGER, RESTORE), ), "imapserver" : ( ("username", "Username", "", """The username to use when logging into the SpamBayes IMAP server.""", IMAP_ASTRING, DO_NOT_RESTORE), ("password", "Password", "", """The password to use when logging into the SpamBayes IMAP server.""", IMAP_ASTRING, DO_NOT_RESTORE), ("port", "IMAP Listen Port", 143, """The port to serve the SpamBayes IMAP server on.""", PORT, RESTORE), ), "globals" : ( ("verbose", "Verbose", False, """""", BOOLEAN, RESTORE), ("dbm_type", "Database storage type", "best", """What DBM storage type should we use? Must be best, db3hash, dbhash or gdbm. Windows folk should steer clear of dbhash. Default is "best", which will pick the best DBM type available on your platform.""", ("best", "db3hash", "dbhash", "gdbm"), RESTORE), ("proxy_username", "HTTP Proxy Username", "", """The username to give to the HTTP proxy when required. If a username is not necessary, simply leave blank.""", r"[\w]+", DO_NOT_RESTORE), ("proxy_password", "HTTP Proxy Password", "", """The password to give to the HTTP proxy when required. This is stored in clear text in your configuration file, so if that bothers you then don't do this. You'll need to use a proxy that doesn't need authentication, or do without any SpamBayes HTTP activity.""", r"[\w]+", DO_NOT_RESTORE), ("proxy_server", "HTTP Proxy Server", "", """If a spambayes application needs to use HTTP, it will try to do so through this proxy server. The port defaults to 8080, or can be entered with the server:port form.""", SERVER, DO_NOT_RESTORE), ), } # `optionsPathname` is the pathname of the last ini file in the list. # This is where the web-based configuration page will write its changes. # If no ini files are found, it defaults to bayescustomize.ini in the # current working directory. optionsPathname = None # The global options object - created by load_options options = None def load_options(): global optionsPathname, options options = OptionsClass() options.load_defaults(defaults) # Maybe we are reloading. if optionsPathname: options.merge_file(optionsPathname) alternate = None if hasattr(os, 'getenv'): alternate = os.getenv('BAYESCUSTOMIZE') if alternate: filenames = alternate.split(os.pathsep) options.merge_files(filenames) optionsPathname = os.path.abspath(filenames[-1]) else: alts = [] for path in ['bayescustomize.ini', '~/.spambayesrc']: epath = os.path.expanduser(path) if os.path.exists(epath): alts.append(epath) if alts: options.merge_files(alts) optionsPathname = os.path.abspath(alts[-1]) if not optionsPathname: optionsPathname = os.path.abspath('bayescustomize.ini') if sys.platform.startswith("win") and \ not os.path.isfile(optionsPathname): # If we are on Windows and still don't have an INI, default to the # 'per-user' directory. try: from win32com.shell import shell, shellcon except ImportError: # We are on Windows, with no BAYESCUSTOMIZE set, no ini file # in the current directory, and no win32 extensions installed # to locate the "user" directory - seeing things are so lamely # setup, it is worth printing a warning print >>sys.stderr, "NOTE: We can not locate an INI file " \ "for SpamBayes, and the Python for Windows extensions " \ "are not installed, meaning we can't locate your " \ "'user' directory. An empty configuration file at " \ "'%s' will be used." % optionsPathname.encode('mbcs') else: windowsUserDirectory = os.path.join( shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0), "SpamBayes", "Proxy") try: if not os.path.isdir(windowsUserDirectory): os.makedirs(windowsUserDirectory) except os.error: # unable to make the directory - stick to default. pass else: optionsPathname = os.path.join(windowsUserDirectory, 'bayescustomize.ini') # Not everyone is unicode aware - keep it a string. optionsPathname = optionsPathname.encode("mbcs") # If the file exists, then load it. if os.path.exists(optionsPathname): options.merge_file(optionsPathname) # Annoyingly, we have a special case. The notate_to and notate_subject # allowed values have to be set to the same values as the header_x_ # options, but this can't be done (AFAIK) dynmaically. If this isn't # the case, then if the header_x_string values are changed, the # notate_ options don't work. Outlook Express users like both of # these options...so we fix it here. See also sf #944109. header_strings = (options["Headers", "header_ham_string"], options["Headers", "header_spam_string"], options["Headers", "header_unsure_string"]) notate_to = options.get_option("Headers", "notate_to") notate_subject = options.get_option("Headers", "notate_subject") notate_to.allowed_values = header_strings notate_subject.allowed_values = header_strings def get_pathname_option(section, option): """Return the option relative to the path specified in the gloabl optionsPathname, unless it is already an absolute path.""" filename = os.path.expanduser(options.get(section, option)) if os.path.isabs(filename): return filename return os.path.join(os.path.dirname(optionsPathname), filename) # Ideally, we should not create the objects at import time - but we have # done it this way forever! # We avoid having the options loading code at the module level, as then # the only way to re-read is to reload this module, and as at 2.3, that # doesn't work in a .zip file. load_options()