//@copyright_begin
// ================================================================
// Copyright Notice
// Copyright (C) 1998-2004 by Joe Linoff
// 
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
// 
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL JOE LINOFF BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
// 
// Comments and suggestions are always welcome.
// Please report bugs to http://ccdoc.sourceforge.net/ccdoc
// ================================================================
//@copyright_end
#include "log.h"
#include "phase1_scanner.h"
#include <cstdio>

// ================================================================
// This variable allows the header version
// to be queried at runtime.
// ================================================================
namespace {
  char ccdoc_rcsid[] = "$Id: phase1_scanner_doc.cc,v 1.5 2004/09/30 04:16:07 jlinoff Exp $";
}

// ================================================================
// Constructor.
// ================================================================
ccdoc::phase1::scanner_doc::scanner_doc(scanner& scan,switches& sw)
  : m_scanner(scan),
    m_mode(SHORT),
    m_sw(sw)
{
}
// ================================================================
// Destructor.
// ================================================================
ccdoc::phase1::scanner_doc::~scanner_doc()
{
}
// ================================================================
// Empty?
// ================================================================
bool ccdoc::phase1::scanner_doc::empty() const
{
  return m_comment.empty();
}
// ================================================================
// Format
// ================================================================
const char* ccdoc::phase1::scanner_doc::format(char* token,int max)
{
  char lineno[64];
  sprintf(lineno,"%d",m_scanner.get_lineno() - 1);
  vector<string> tokens;
  m_comment.add_file(m_scanner.get_file());
  m_comment.add_lineno(lineno);
  m_comment.get(tokens);
  vector<string>::reverse_iterator itr = tokens.rbegin();
  for(;itr!=tokens.rend();++itr) {
    m_scanner.put_token(*itr);
  }
  m_scanner.get_token();
  token[0] = '@';
  token[1] = '{';
  token[2] = 0;
  return token;
}
// ================================================================
// Scan a single argument.
// ================================================================
bool ccdoc::phase1::scanner_doc::scan_1arg(const char* pline,
					   string& arg,
					   const char* directive,
					   const char* argid,
					   bool report_errors)
{
  // Skip white space.
  for(;*pline!=0 && (*pline==' ' || *pline=='\t');++pline)
    ;
  if( 0 != *pline ) {
    arg = pline;
    return true;
  }

  if(report_errors) {
    s_log.warning()
      << "Ignored the "
      << directive
      << " directive because argument "
      << argid
      << " is missing at line "
      << m_scanner.get_lineno()-1
      << " in "
      << m_scanner.get_file()
      << ".\n"
      << s_log.enable();
  }
  return false;
}
// ================================================================
// Parse the ccdoc directive with 2 arguments, like @param
// and @exception.
// ================================================================
bool ccdoc::phase1::scanner_doc::scan_2args(char* pstr,
					    string& arg1,
					    string& arg2,
					    const char* directive,
					    const char* arg1id,
					    const char* arg2id,
					    bool arg2_required)
{
  // Skip white space.
  for(;*pstr!=0 && (*pstr==' ' || *pstr=='\t');++pstr);
  if( 0 == *pstr ) {
    s_log.warning()
      << "Ignored the "
      << directive
      << " directive because argument "
      << arg1id
      << " is missing at line "
      << m_scanner.get_lineno()-1
      << " in "
      << m_scanner.get_file()
      << ".\n\tThe correct specification is "
      << directive
      << " "
      << arg1id
      << " "
      << arg2id
      << ".\n"
      << s_log.enable();
    return false;
  }
  char* arg1_beg = pstr;

  // Skip to next w/s unless the user specified an HTML
  // directive explicitly, this is determined by looking
  // at the first character. If it is a '<', pass everything
  // back as one argument.
  if( '<' == *pstr ) {
    for(;*pstr!=0;++pstr);
  }
  else {
    for(;*pstr!=0 && *pstr!=' ' && *pstr!='\t';++pstr);
  }
  if( 0 == *pstr ) {
    if( arg2_required ) {
      s_log.warning()
	<< "Ignored the "
	<< directive
	<< " directive because argument "
	<< arg2id
	<< " is missing at line "
	<< m_scanner.get_lineno()-1
	<< " in "
	<< m_scanner.get_file()
	<< ".\n\tThe correct specification is "
	<< directive
	<< " "
	<< arg1id
	<< " "
	<< arg2id
	<< ".\n"
	<< s_log.enable();
      return false;
    }
    arg1 = arg1_beg;
    arg2 = "";
    return true;
  }
  char* arg1_end = pstr;
  *arg1_end = 0;

  // Skip white space.
  for(++pstr;*pstr!=0 && (*pstr==' ' || *pstr=='\t');++pstr);
  if( 0 == *pstr) {
    // Trailing w/s but no name.
    // Assume that there is no arg.
    arg1 = arg1_beg;
    arg2 = "";
    return true;
  }

  // Set the end of the second argument.
  // Trim the trailing w/s.
  char* arg2_beg = pstr;
  char* arg2_end = arg2_beg;
  for(;*arg2_end;++arg2_end);
  for(--arg2_end;*arg2_end<=' ';--arg2_end);
  ++arg2_end;
  *arg2_end = 0;

  // Set the arguments.
  arg1 = arg1_beg;
  arg2 = arg2_beg;

  return true;
}
// ================================================================
// Is this a directive?
// Derived from code submitted by bzoe 10/18/01.
// ================================================================
bool ccdoc::phase1::scanner_doc::is_directive(const char* pstr1,
					      const char* pstr2,
					      bool null_term) const
{
  for(;*pstr1;++pstr1,++pstr2) {
    if( *pstr1 != *pstr2 ) {
      return false;
    }
  }
  if( *pstr2 == ' ' || *pstr2 == '\t' )
    return true;
  // Accept a terminating NULL?
  if( null_term && *pstr2 == 0 )
    return true;
  return false;
}
// ================================================================
// Parse line.
// ================================================================
void ccdoc::phase1::scanner_doc::parse_line(char* line)
{
  // ================================================
  // Check for ccdoc directives.
  // Skip the leading whitespace to look for a '@'
  // character.
  // ================================================
  string pkgdoctid; // Used for the @pkgdoctid directive.
  char* pline = line;
  for(;*pline && (*pline==' ' || *pline=='\t');++pline)
    ;
  if('@' == pline[0]) {
    // ================================================
    // ccdoc directive: '@@'
    // ================================================
    if('@' == pline[1]) {
      // '@@' directive.
      // Convert < -> &lt;
      // Convert > -> &gt;
      // Convert & -> &amp;
      const char* src = &pline[2];
      static char line1[65536]; // maximum line length
      char* dst = line1;
      for(;*src;++src) {
	if( '<' == *src ) {
	  *dst++ = '&';
	  *dst++ = 'l';
	  *dst++ = 't';
	  *dst++ = ';';
	}
	else if( '>' == *src ) {
	  *dst++ = '&';
	  *dst++ = 'g';
	  *dst++ = 't';
	  *dst++ = ';';
	}
	else if( '&' == *src ) {
	  *dst++ = '&';
	  *dst++ = 'a';
	  *dst++ = 'm';
	  *dst++ = 'p';
	  *dst++ = ';';
	}
	else {
	  *dst++ = *src;
	}
      }
      *dst = 0;
      add_line(line1);
    }
    // ================================================
    // ccdoc directives: '@{' or '@}'
    // ================================================
    else if('{' == pline[1] || '}' == pline[1]) {
      // Silently ignored, it is used the '//' processing.
    }
    // ================================================
    // ccdoc directive: '@$' - same as @link
    // ================================================
    else if('$' == pline[1] &&
	    ( ' ' == pline[2] || '\t' == pline[2] ) ) {
      pline += 2;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@$","<entity>","<name>",false)) {
	if( arg2.size() == 0 )
	  arg2 = arg1;
	add_line("@link");
	add_line(arg1.c_str());
	add_line(arg2.c_str());
      }
    }
    // ================================================
    // ccdoc directive: '@author'
    // ================================================
    else if(is_directive("author",&pline[1])) {
      pline+=7;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@author","<name>") )
	m_comment.add_author(arg);
    }
    // ================================================
    // ccdoc directive: '@deprecated'
    // ================================================
    else if(is_directive("deprecated",&pline[1],true)) {
      pline += 11;
      m_mode = DEPRECATED;
      string arg;
      if( scan_1arg(pline,arg,"@deprecated","<description>",false) )
        parse_comment_line(arg.c_str()); // Issue 0123
      //m_comment.add_deprecated(arg);
    }
    // ================================================
    // ccdoc directive: '@exception'
    // ================================================
    else if(is_directive("exception",&pline[1])) {
      pline += 10;
      m_mode = EXCEPTION;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@exception","<name>","<desc>",false)) {
	if(arg2.size()) {
	  m_comment.add_new_exception(arg1,arg2);
	}
	else {
	  m_comment.add_new_exception(arg1);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@link'
    // ================================================
    // This is now handled in parse_comment_line.
    else if(is_directive("link",&pline[1])) {
      pline += 5;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@link","<entity>","<name>",false)) {
	if( arg2.size() == 0 )
	  arg2 = arg1;

        // Search for an embedded @endlink directive.
        // If found, split it out.
        string arg3;
        if( arg2 == "@endlink" ) {
          // The simple case.
          arg2 = arg1;
        }
        else {
          string::size_type f = arg2.find("@endlink");
          if( f && f != string::npos) {
            arg3 = arg2;
            arg2 = arg3.substr(0,f-1);
            size_t last = arg2.find_last_not_of(" \t\r\n");
            if( string::npos != last ) {
              arg2 = arg2.substr(0,last+1);
            }
          }
          // Look for suffix characters.
          // @link a1 @endlink suffix characters.
          //          ^       ^
          //          |       +--- f+8
          //          +----------- f
          if( (f+8) < arg3.size() ) {
            string::size_type sz = arg3.size()-8;
            arg3 = arg3.substr(f+8,sz);
          }
          else {
            arg3 = "";
          }
        }
        add_line("@link");
        add_line(arg1.c_str());
        add_line(arg2.c_str());
        if( arg3.size() )
          add_line(arg3.c_str());
      }
    }
    // ================================================
    // ccdoc directive: '@param'
    // ================================================
    else if(is_directive("param",&pline[1])) {
      pline += 6;
      m_mode = PARAM;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@param","<name>","<desc>",false)) {
	if(arg2.size()) {
	  m_comment.add_new_param(arg1,arg2);
	}
	else {
	  m_comment.add_new_param(arg1);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@pkg'
    // ================================================
    else if(is_directive("pkg",&pline[1])) {
      pline += 4;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@pkg","<name>") ) {
	// Break the argument down into '.' separated
	// package entries.
	if( m_comment.get_pkg().size() ) {
	  report_multiply_defined_error("@pkg");
	}
	else if( m_comment.get_pkgdoc().size() ) {
	  // pkg and pkgdoc cannot simultaneously exist
	  // in the same ccdoc comment.
	  s_log.warning()
	    << "Illegal declaration, @pkgdoc and @pkg are "
	    << "mutually exclusive, @pkg will be ignored at line "
	    << m_scanner.get_lineno()-1
	    << " in "
	    << m_scanner.get_file()
	    << ".\n"
	    << s_log.enable();
	}
	else {
	  vector<string> vec;
	  parse_pkg_path(arg,"@pkg",vec);
	  vector<string>::iterator i = vec.begin();
	  for(;i!=vec.end();++i) {
	    m_comment.add_pkg(*i);
	  }
	}
      }
    }
    // ================================================
    // ccdoc directive: '@pkgdoc'
    // ================================================
    else if(is_directive("pkgdoc",&pline[1])) {
      pline += 7;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg1;
      string arg2;
      if( scan_2args(pline,arg1,arg2,"@pkgdoc","<name>","[<url>]",false) ) {
	// Break the argument down into '.' separated
	// package entries.
	if( m_comment.get_pkgdoc().size() ) {
	  report_multiply_defined_error("@pkgdoc");
	}
	else if( m_comment.get_pkg().size() ) {
	  // pkg and pkgdoc cannot simultaneously exist
	  // in the same ccdoc comment.
	  s_log.warning()
	    << "Illegal declaration, @pkgdoc and @pkg are "
	    << "mutually exclusive, @pkgdoc will be ignored at line "
	    << m_scanner.get_lineno()-1
	    << " in "
	    << m_scanner.get_file()
	    << ".\n"
	    << s_log.enable();
	}
	else if( arg1.size() ) {
	  // If the user specified a URL for the @pkgdoc directive,
	  // insert it first so that it is easy to find.
	  if( arg2.size() ) {
	    // Issue 0025
	    // There is a special form of the @pkgdoc directive
	    // that allows the user to specify their own URL
	    // for the package page. This must be used very carefully
	    // because the links to child entities may be lost.
	    string urlid = "@url";
	    m_comment.add_pkgdoc( urlid );
	    m_comment.add_pkgdoc( arg2 );
	  }
	  vector<string> vec;
	  parse_pkg_path(arg1,"@pkgdoc",vec);
	  vector<string>::iterator i = vec.begin();
	  for(;i!=vec.end();++i) {
	    m_comment.add_pkgdoc(*i);
	  }
	}
      }
    }
    // ================================================
    // ccdoc directive: '@pkgdoctid'
    // This must appear after an @pkgdoc directive.
    // ================================================
    else if(is_directive("pkgdoctid",&pline[1])) {
      pline += 10;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@pkgdoctid","<name>") ) {
	if( pkgdoctid.size() ) {
	  report_multiply_defined_error("@pkgdoctid");
	}
	else if( m_comment.get_pkgdoc().size() == 0 ) {
	  // @pkgdoc must be specified first.
	  s_log.warning()
	    << "Illegal declaration, @pkgdoc must be specified before "
	    << "@pkgdoctid at line "
	    << m_scanner.get_lineno()-1
	    << " in "
	    << m_scanner.get_file()
	    << ".\n"
	    << s_log.enable();
	}
	else {
	  pkgdoctid = arg;
	  m_comment.add_pkgdoc_tid(arg);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@return'
    // ================================================
    else if(is_directive("return",&pline[1],true)) {
      pline += 7;
      m_mode = RETURNS;
      string arg;
      if( scan_1arg(pline,arg,"@returns","<description>",false) )
        parse_comment_line(arg.c_str()); // Issue 0123
      //m_comment.add_returns(arg);
    }
    // ================================================
    // ccdoc directive: '@returns'
    // ================================================
    else if(is_directive("returns",&pline[1],true)) {
      pline += 8;
      m_mode = RETURNS;
      string arg;
      if( scan_1arg(pline,arg,"@returns","<description>",false) )
        parse_comment_line(arg.c_str()); // Issue 0123
    }
    // ================================================
    // ccdoc directive: '@see'
    // ================================================
    else if(is_directive("see",&pline[1])) {
      pline += 4;
      m_mode = SEE;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@see","<entity>","[<number>]",false)) {
	// Make sure that arg2 is numeric.
	if( arg2.size() ) {
	  if( arg2 != "*" ) {
	    // Make sure that the argument is numeric
	    // unless the user specified the special
	    // case of '*' which means all.
	    const char* p = arg2.c_str();
	    for(;*p;++p) {
	      if( *p < '0' || *p > '9' ) {
		s_log.warning()
		  << "Invalid number "
		  << arg2.c_str()
		  << " in @see directive at line "
		  << m_scanner.get_lineno()-1
		  << " in "
		  << m_scanner.get_file()
		  << " was ignored.\n"
		  << s_log.enable();
		arg2 = "0";
		break;
	      }
	    }
	  }
	}
	else {
	  // The user did not specify an index, set the default
	  // value of zero.
	  arg2 = "0";
	}
	m_comment.add_new_see(arg1,arg2);
      }
    }
    // ================================================
    // ccdoc directive: '@source'
    // ================================================
    else if(is_directive("source",&pline[1])) {
      pline += 7;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@source","<file>") ) {
	if( m_comment.get_source().size() ) {
	  report_multiply_defined_error("@source");
	}
	else {
	  m_comment.add_source(arg);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@since'
    // ================================================
    else if(is_directive("since",&pline[1])) {
      pline += 6;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@since","<version>") ) {
	if( m_comment.get_source().size() ) {
	  report_multiply_defined_error("@since");
	}
	else {
	  m_comment.add_since(arg);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@suffix'
    // ================================================
    else if(is_directive("suffix",&pline[1],true)) {
      pline += 7;
      m_comment.add_suffix(true);
    }
    // ================================================
    // Issue 0084.
    // ccdoc directive: '@throws'
    // Derived from code submitted by bzoe 10/18/01.
    // ================================================
    else if(is_directive("throws",&pline[1])) {
      pline += 7;
      m_mode = EXCEPTION;
      string arg1;
      string arg2;
      if(scan_2args(pline,arg1,arg2,"@throws","<name>","<desc>",false)) {
	if(arg2.size()) {
	  m_comment.add_new_exception(arg1,arg2);
	}
	else {
	  m_comment.add_new_exception(arg1);
	}
      }
    }
    // ================================================
    // ccdoc directive: '@todo' (Issue 0120)
    // ================================================
    else if(is_directive("todo",&pline[1],true)) {
      pline += 5;
      m_mode = TODO;
      string arg;
      if( scan_1arg(pline,arg,"@todo","<description>",false) )
        parse_comment_line(arg.c_str()); // Issue 0123
      //m_comment.add_returns(arg);
    }
    // ================================================
    // ccdoc directive: '@version'
    // ================================================
    else if(is_directive("version",&pline[1])) {
      pline += 8;
      if( m_mode == SHORT ) m_mode = LONG;
      string arg;
      if( scan_1arg(pline,arg,"@version","<id>") ) {
	if( m_comment.get_version().size() ) {
	  report_multiply_defined_error("@version");
	}
	else {
	  m_comment.add_version(arg);
	}
      }
    }
    else {
      // ================================================
      // Although this line looks like a ccdoc directive,
      // it isn't. Add it the the current mode collection.
      // ================================================
      add_line(line);
    }
  }
  else {
    parse_comment_line(line);
  }
}
// ================================================================
// Issue 0090:
// This line is not ccdoc directive, add it the the current mode
// collection unless it has embedded {@link ... } or @link/@endlink
// directives.
// ================================================================
void ccdoc::phase1::scanner_doc::parse_comment_line(const char* line)
{
  vector<string> new_lines;
  const char* ps = line;
  const char* pend = ps;

  // Now process the expected stuff.
  for(ps=line;*ps;++ps) {
    // "{@link ... }"
    if( '{' == ps[0] &&
	'@' == ps[1] &&
	'l' == ps[2] &&
	'i' == ps[3] &&
	'n' == ps[4] &&
	'k' == ps[5] &&
	( ' ' == ps[6] || '\t' == ps[6] ) ) {
      // It looks we have an embedded link, now scan ahead to
      // the terminating right brace.
      unsigned num_ws = 1;
      const char* pbeg = &ps[1]; // point to '@link'
      const char* ps1 = pbeg;
      size_t len2 = 0;
      for(;*ps1 && *ps1 != '{' && *ps1 != '}' ;++ps1,++len2) {
	if( *ps1 == ' ' || *ps1 == '\t' )
	  ++num_ws;
      }
      if( *ps1 == '}' && num_ws>1 ) {
	// We definitely have an embedded link.
	// Create a new line from the end of the previous link
	// and the beginning of this one.
	string new_line;

	// Figure out the length of the prefix string.
	size_t len1 = 0;
	for(const char* ps2=pend;ps2!=ps;++ps2,++len1)
	  ;
	if( len1 ) {
	  // Special case handling to work around
	  // paragraph recognition.
	  if ( len1 == 1 && *pend == ' ' )
	    new_line = "  "; // convert one space to two spaces
	  else
	    new_line.assign(pend,len1);
	  new_lines.push_back(new_line); // prefix text
	}
	new_line.assign(pbeg,len2); // trailing w/s is handled later
	new_lines.push_back(new_line); // @link directive

	// Add in the new @link line.
	len1 = 0;
	ps = ps1; // The for() increment will fix this.
	pend = ps+1; // point just past the trailing right brace
      }
    }
    // "@link ... @endlink"
    else if( '{' != ps[0] &&
             '@' == ps[1] &&
             'l' == ps[2] &&
             'i' == ps[3] &&
             'n' == ps[4] &&
             'k' == ps[5] &&
             ( ' ' == ps[6] || '\t' == ps[6] ) ) {
      // It looks we have an embedded link, now scan ahead to
      // the terminating @endlink or new line. The new line
      // is for backward compatibility with ccdoc. The @endlink
      // is for forward compatibility with doxygen.
      unsigned num_ws = 1;
      const char* pbeg = &ps[1]; // point to '@link'
      const char* ps1 = pbeg;
      const char* pend1 = pbeg;
      size_t len2 = 0;
      for(;*ps1;++ps1,++len2,++pend1) {
	if( *ps1 == ' ' || *ps1 == '\t' )
	  ++num_ws;
        else if( '@' == ps1[0] &&
                 'e' == ps1[1] &&
                 'n' == ps1[2] &&
                 'd' == ps1[3] &&
                 'l' == ps1[4] &&
                 'i' == ps1[5] &&
                 'n' == ps1[6] &&
                 'k' == ps1[7] &&
                 ( ' ' == ps1[8] || '\t' == ps1[8] || 0 == ps1[8] ) ) {
          pend1 = &ps1[7];
          break;
        }
      }
      if( num_ws>1 ) {
	// We definitely have an embedded link.
	// Create a new line from the end of the previous link
	// and the beginning of this one.
	string new_line;

	// Figure out the length of the prefix string.
	size_t len1 = 0;
	for(const char* ps2=pend;ps2!=ps;++ps2,++len1)
	  ;
	if( len1 ) {
	  // Special case handling to work around
	  // paragraph recognition.
	  if ( len1 == 1 && *pend == ' ' )
	    new_line = "  "; // convert one space to two spaces
	  else
	    new_line.assign(pend,len1);
	  new_lines.push_back(new_line); // prefix text
	}
	new_line.assign(pbeg,len2); // trailing w/s is handled later
	new_lines.push_back(new_line); // @link directive

	// Add in the new @link line.
	len1 = 0;
	ps = pend1; // The for() increment will fix this.
	pend = ps+1; // point just past the trailing right brace
      }
    }
  }

  // Handle the suffix characters.
  if( new_lines.size() ) {
    // Get the trailing characters:
    //  * Link {@link foo::bar xx} These are the suffix characters.
    //                             ^
    if( pend != ps ) {
      string new_line;
      size_t len1 = 0;
      for(const char* ps2=pend;ps2!=ps;++ps2,++len1)
	;
      if( len1 ) {
	// Special case handling to work around
	// paragraph recognition.
	if ( len1 == 1 && *pend == ' ' )
	  new_line = "  ";
	else
	  new_line.assign(pend,len1);
	new_lines.push_back(new_line); // suffix text
      }
    }
    static char s_line2[65536]; // maximum line length
    vector<string>::iterator itr = new_lines.begin();
    for(;itr!=new_lines.end();++itr) {
      strcpy(s_line2,itr->c_str());
      parse_line(s_line2);
    }
  }
  else {
    add_line(line);
  }
}
// ================================================================
// Add line.
// ================================================================
void ccdoc::phase1::scanner_doc::add_line(const char* line)
{
  // If a line is NULL, it means that the user
  // wants a paragraph separator. It is stored
  // as a space.
  string desc;
  if( !line || *line == 0 ) {
    desc = " "; // This is where blank lines are defined.
  }
  else {
    desc = line;
    if( m_sw.jdsds() && m_mode == SHORT ) {
      // Issue 0089: Extract short comments the javadoc way.
      size_t sz = 0;
      for(const char* p=line;*p;++p,++sz) {
	if( *p == '.' ) {
	  // In javadoc, a short description is terminated by
	  // a period that is followed by a SPACE, TAB, NEWLINE.
          const char* pnext = p+1; // Issue 0167
	  if( *pnext == 0 || // same as new line
	      *pnext == ' ' ||
	      *pnext == '\t' ||
	      *pnext == '\r' ||
	      *pnext == '\n' ) {
	    ++sz; // include the period.
	    string short_desc;
	    short_desc = desc.substr(0,sz);
	    m_comment.add_short_desc(short_desc);
	    if( *pnext ) {
	      // Handle this case:
	      //  Short description. Start of long description.
	      size_t first = sz+1;
	      for(sz=0;*pnext;++pnext) sz++;
	      string long_desc;
	      long_desc.assign(desc,first,sz);
	      if( long_desc.size() ) {
		m_comment.add_long_desc(long_desc);
	      }
	    }
	    m_mode = LONG;
	    return;
	  }
	}
      }
    }
  }
  switch(m_mode) {
  case DEPRECATED: m_comment.add_deprecated     (desc); break;
  case EXCEPTION:  m_comment.add_exception_desc (desc); break;
  case LONG:       m_comment.add_long_desc      (desc); break;
  case PARAM:      m_comment.add_param_desc     (desc); break;
  case RETURNS:    m_comment.add_returns        (desc); break;
  case SHORT:      m_comment.add_short_desc     (desc); break;
  case SEE:        m_comment.add_see_desc       (desc); break;
  case TODO:       m_comment.add_todo           (desc); break;
  }
}
// ================================================================
// Add line.
// ================================================================
void ccdoc::phase1::scanner_doc::add_line(const char* line,
					  vector<string>& vec)
{
  // This is where we handle embedded blank lines.
  if(*line)
    vec.push_back(line);
  else
    vec.push_back(" ");
}
// ================================================================
// Report multiply defined entities.
// ================================================================
void ccdoc::phase1::scanner_doc::report_multiply_defined_error(const char* directive) const
{
  s_log.warning()
    << "Directive " << directive << " is multiply defined at line "
    << m_scanner.get_lineno()-1
    << " in "
    << m_scanner.get_file()
    << ", the first definition will be used.\n"
    << s_log.enable();
}
// ================================================================
// Parse a pkg or pkgdoc path.
// ================================================================
void ccdoc::phase1::scanner_doc::parse_pkg_path(string& arg,
						const char* directive,
						vector<string>& vec)
{
  // Look for '::' and '.' separators.
  // The following are legal:
  //   A.B.C
  //   A::B::C
  //   A::B.C   <-- not recommended
  static char token[65536];
  ::strcpy(token,arg.c_str());
  vec.clear();
  char* b = token;
  char* e = b;
  for(;*e;++e) {
    if( ':' == *e ) {
      ++e;
      if( ':' == *e ) {
	--e; // back up to the first colon.
	*e = 0; // set the end of the previous token
	++e; // point to the start of the next token
	if(!parse_pkg_path_entry(b,directive)) {
	  vec.clear();
	  return;
	}
	vec.push_back(b);
	*e = '.';
	b = e+1;
      }
    }
    else if( '.' == *e ) {
      *e = 0;
      if(!parse_pkg_path_entry(b,directive)) {
	vec.clear();
	return;
      }
      vec.push_back(b);
      *e = '.';
      b = e+1;
    }
  }
  if(!parse_pkg_path_entry(b,directive)) {
    vec.clear();
    return;
  }
  vec.push_back(b);
}
// ================================================================
// Parse a pkg or pkgdoc path.
// ================================================================
bool ccdoc::phase1::scanner_doc::parse_pkg_path_entry(char* token,
						      const char* directive)
{
  // Trim off trailing w/s.
  char* ptr = token;
  for(;*ptr;++ptr); // First get to the end of the string.
  for(ptr--;ptr>token;--ptr) {
    if( ' ' != *ptr &&
	'\t' != *ptr &&
	'\n' != *ptr ) {
      ptr++;
      *ptr = 0;
      break;
    }
  }
  if(*token == 0) {
    // We found a zero length entry.
    // Report it as a warning and ignore it.
    s_log.warning()
      << "Illegal zero length subpkg name found in the "
      << directive
      << " directive at line "
      << m_scanner.get_lineno()-1
      << " in "
      << m_scanner.get_file()
      << ", the directive was ignored.\n"
      << s_log.enable();
    return false;
  }
  return true;
}