# --------------------------------------------------
# Copyright The IETF Trust 2011, All Rights Reserved
# --------------------------------------------------

import calendar
import codecs
import copy
import datetime
import textwrap
import lxml
import os
import re
import xml2rfc.log
import xml2rfc.util
import xml2rfc.utils

from lxml import etree
from optparse import Values

try:
    from xml2rfc import debug
    debug.debug = True
except ImportError:
    pass

from xml2rfc import strings, log
from xml2rfc.uniscripts import is_script
from xml2rfc.util.date import extract_date, augment_date, format_date, get_expiry_date
from xml2rfc.util.name import short_author_ascii_name_parts, full_author_name_expansion, short_author_name_parts
from xml2rfc.util.unicode import ( punctuation, unicode_replacements, unicode_content_tags, bare_unicode_tags,
    bare_latin_tags, unicode_attributes, downcode, downcode_punctuation)
from xml2rfc.utils import namespaces, find_duplicate_ids, slugify

default_silenced_messages = [
#    ".*[Pp]ostal address",
]

default_options = Values(defaults={
        'accept_prepped': None,
        'add_xinclude': None,
        'basename': None,
        'bom': False,
        'cache': None,
        'clear_cache': False,
        'css': None,
        'date': datetime.date.today(),
        'datestring': None,
        'debug': False,
        'docfile': False,
        'doc_template': None,
        'doi_base_url': 'https://doi.org/',
        'dtd': None,
        'expand': False,
        'external_css': False,
        'external_js': True,            # Change to False in 3.x
        'filename': None,
        'first_page_author_org': True,
        'html': False,
        'id_base_url': 'https://tools.ietf.org/html/',
        'id_reference_base_url': 'https://tools.ietf.org/html/',
        'id_is_work_in_progress': True,
        'image_svg': False,
        'indent': 2,
        'info': False,
        'legacy': True,
        'legacy_date_format': False,    # Change to True in 3.x
        'legacy_list_symbols': False,
        'list_symbols': ('*', '-', 'o', '+'),
        'manpage': False,
        'metadata_js_url': 'metadata.min.js',
        'no_css': False,
        'no_dtd': None,
        'no_network': False,
        'nroff': False,
        'omit_headers': None,
        'orphans': 2,
        'output_filename': None,
        'output_path': None,
        'pagination': True,
        'pdf': False,
        'preptool': False,
        'quiet': False,
        'remove_pis': False,
        'raw': False,
        'rfc': None,
        'rfc_base_url': 'https://www.rfc-editor.org/rfc/',
        'rfc_reference_base_url': 'https://rfc-editor.org/rfc/',
        'silence': default_silenced_messages,
        'strict': False,
        'table_hyphen_breaks': False,
        'table_borders': 'full',
        'template_dir': os.path.join(os.path.dirname(os.path.dirname(__file__)), 'templates'),
        'text': True,
        'unprep': False,
        'utf8': False,
        'verbose': False,
        'version': False,
        'v2v3': False,
        'vocabulary': 'v2',
        'widows': 2,
    })

class _RfcItem:
    """ A unique ID object for an anchored RFC element.
    
        Anchored elements are the following: Automatic sections, user (middle)
        sections, paragraphs, references, appendices, figures, and tables.
        
        RfcItems are collected into an index list in the base writer
    """
    def __init__(self, autoName, autoAnchor, counter='', title='', anchor='',
                 toc=True, level=1, appendix=False):
        self.counter = str(counter)
        self.autoName = autoName
        self.autoAnchor = autoAnchor
        self.title = title
        self.anchor = anchor
        self.toc = toc
        self.level = level
        self.appendix = appendix
        self.page = 0    # This will be set after buffers are complete!
        self.used = False


class _IrefItem:
    """ A unique ID object for an iref element """
    def __init__(self, anchor=None):
        self.pages = []
        self.subitems = {}
        self.anchor = anchor


class RfcWriterError(Exception):
    """ Exception class for errors during document writing """
    def __init__(self, msg):
        self.msg = msg


class BaseRfcWriter:
    """ Base class for all writers

        All public methods need to be overridden for a writer implementation.
    """

    # -------------------------------------------------------------------------
    # Attribute default values
    #
    # These will mainly come into play if DTD validation was disabled, and
    # processing that happens to rely on DTD populated attributes require a lookup
    defaults = {
        'section_toc':              'default',
        'xref_pageno':              'false',
        'xref_format':              'default',
        'iref_primary':             'false',
        'spanx_style':              'emph',
        'figure_suppress-title':    'false',
        'figure_title':             '',
        'figure_align':             'left',
        'vspace_blanklines':        0,
        'table_suppress-title':     'false',
        'table_title':              '',
        'table_align':              'center',
        'table_style':              'full',
        'ttcol_align':              'left',
        'ipr':                      'trust200902',
        'submissionType':           'IETF',
        'consensus':                'no',
    }

    # -------------------------------------------------------------------------
    # Boilerplate text
    boilerplate = {}

    # The RFC Editor's switchover date for https URLs
    boilerplate_https_date = datetime.date(year=2017, month=8, day=21)

    # Document stream names
    boilerplate['document_stream'] = {}
    boilerplate['document_stream']['IETF'] = \
        'Internet Engineering Task Force (IETF)'
    boilerplate['document_stream']['IAB'] = \
        'Internet Architecture Board (IAB)'
    boilerplate['document_stream']['IRTF'] = \
        'Internet Research Task Force (IRTF)'
    boilerplate['document_stream']['independent'] = \
        'Independent Submission'

    # Draft workgroup name
    boilerplate['draft_workgroup'] = 'Network Working Group'

    # Category names
    boilerplate['std'] = 'Standards Track'
    boilerplate['bcp'] = 'Best Current Practice'
    boilerplate['exp'] = 'Experimental'
    boilerplate['info'] = 'Informational'
    boilerplate['historic'] = 'Historic'

    # Series type
    boilerplate['series_name'] = {}
    boilerplate['series_name']['std'] = 'STD'
    boilerplate['series_name']['bcp'] = 'BCP'
    boilerplate['series_name']['info'] = 'FYI'

    # ISSN
    boilerplate['issn'] = '2070-1721'

    # 'Status of this Memo' boilerplate for RFCs
    boilerplate['status'] = {
        'std': {},
        'bcp': {},
        'exp': {},
        'info': {},
        'historic': {},
    }
    # 'Status of this Memo' boilerplate for RFCs
    boilerplate['status_5741'] = {
    }

    # Paragraph 1
    boilerplate['status']['std']['p1'] = \
        'This is an Internet Standards Track document.'
    boilerplate['status']['bcp']['p1'] = \
        'This memo documents an Internet Best Current Practice.'
    boilerplate['status']['exp']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for examination, experimental implementation, and ' \
        'evaluation.'
    boilerplate['status']['info']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for informational purposes.'
    boilerplate['status']['historic']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for the historical record.'

    # Paragraph 2 header
    boilerplate['status']['exp']['p2'] = \
        'This document defines an Experimental Protocol for the Internet ' \
        'community.'
    boilerplate['status']['historic']['p2'] = \
        'This document defines a Historic Document for the Internet ' \
        'community.'

    # Paragraph 2 body
    boilerplate['status']['IETF'] = \
        'This document is a product of the Internet Engineering Task Force ' \
        '(IETF).  It has been approved for publication by the Internet ' \
        'Engineering Steering Group (IESG).'
    boilerplate['status']['IETF_consensus'] = \
        'This document is a product of the Internet Engineering Task Force ' \
        '(IETF).  It represents the consensus of the IETF community.  It has ' \
        'received public review and has been approved for publication by ' \
        'the Internet Engineering Steering Group (IESG).'
    boilerplate['status']['IRTF'] = \
        'This document is a product of the Internet Research Task Force ' \
        '(IRTF).  The IRTF publishes the results of Internet-related ' \
        'research and development activities.  These results might not be ' \
        'suitable for deployment.'
    boilerplate['status']['IRTF_workgroup'] = \
        'This RFC represents the individual opinion(s) ' \
        'of one or more members of the %s Research Group of the Internet ' \
        'Research Task Force (IRTF).'
    boilerplate['status']['IRTF_workgroup_consensus'] = \
        'This RFC represents the consensus of the ' \
        '%s Research Group of the Internet Research Task Force (IRTF).'
    boilerplate['status']['IAB'] = \
        'This document is a product of the Internet Architecture Board ' \
        '(IAB) and represents information that the IAB has deemed valuable ' \
        'to provide for permanent record.'
    boilerplate['status']['IAB_consensus'] = (
        'This document is a product of the Internet Architecture Board ' 
        '(IAB) and represents information that the IAB has deemed valuable '
        'to provide for permanent record.  It represents the consensus of '
        'the Internet Architecture Board (IAB).')
    boilerplate['status']['independent'] = \
        'This is a contribution to the RFC Series, independently of any ' \
        'other RFC stream.  The RFC Editor has chosen to publish this ' \
        'document at its discretion and makes no statement about its value ' \
        'for implementation or deployment.'

    # Paragraph 2 last sentence
    boilerplate['status']['p2end_ietf_std'] = \
        'Further information on Internet Standards is available ' \
        'in Section 2 of RFC 7841.'
    boilerplate['status']['p2end_ietf_bcp'] = \
        'Further information on BCPs is available in Section 2 of RFC 7841.'
    boilerplate['status']['p2end_ietf_other'] = \
        'Not all documents approved by the IESG are candidates for any ' \
        'level of Internet Standard; see Section 2 of RFC 7841.'
    boilerplate['status']['p2end_other'] = \
        'Documents approved for publication by the %s are not ' \
        'candidates for any level of Internet Standard; see Section 2 of RFC ' \
        '7841.'

    boilerplate['status_5741']['p2end_ietf_std'] = \
        'Further information on Internet Standards is available ' \
        'in Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_ietf_bcp'] = \
        'Further information on BCPs is available in Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_ietf_other'] = \
        'Not all documents approved by the IESG are candidates for any ' \
        'level of Internet Standard; see Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_other'] = \
        'Documents approved for publication by the %s are not ' \
        'candidates for any level of Internet Standard; see Section 2 of RFC ' \
        '5741.'

    # Paragraph 3
    boilerplate['status']['p3'] = (
        'Information about the current status of this document, any errata, '
        'and how to provide feedback on it may be obtained at '
        'http://www.rfc-editor.org/info/rfc%s.')
    boilerplate['status']['p3_s'] = (
        'Information about the current status of this document, any errata, '
        'and how to provide feedback on it may be obtained at '
        'https://www.rfc-editor.org/info/rfc%s.')

    # 'Status of this Memo' boilerplate for drafts
    boilerplate['status']['draft'] = [
       'Internet-Drafts are working documents of the Internet Engineering '
       'Task Force (IETF).  Note that other groups may also distribute '
       'working documents as Internet-Drafts.  The list of current Internet-'
       'Drafts is at http://datatracker.ietf.org/drafts/current/.',
       #
       'Internet-Drafts are draft documents valid for a maximum of six months '
       'and may be updated, replaced, or obsoleted by other documents at any '
       'time.  It is inappropriate to use Internet-Drafts as reference '
       'material or to cite them other than as "work in progress."']
    boilerplate['status']['draft_s'] = [
       'Internet-Drafts are working documents of the Internet Engineering '
       'Task Force (IETF).  Note that other groups may also distribute '
       'working documents as Internet-Drafts.  The list of current Internet-'
       'Drafts is at https://datatracker.ietf.org/drafts/current/.',
       #
       'Internet-Drafts are draft documents valid for a maximum of six months '
       'and may be updated, replaced, or obsoleted by other documents at any '
       'time.  It is inappropriate to use Internet-Drafts as reference '
       'material or to cite them other than as "work in progress."']

    boilerplate['draft_expire'] = \
       'This Internet-Draft will expire on %s.'

    # IPR status boilerplate
    boilerplate['ipr_200902_status'] = \
        'This Internet-Draft is submitted in full conformance ' \
        'with the provisions of BCP 78 and BCP 79.'
    boilerplate['ipr_200811_status'] = \
        'This Internet-Draft is submitted to IETF in full conformance ' \
        'with the provisions of BCP 78 and BCP 79.'
    
    # Copyright boilerplate
    boilerplate['base_copyright_header'] = \
        'Copyright (c) %s IETF Trust and the persons identified as the ' \
        'document authors.  All rights reserved.'
    boilerplate['base_copyright_body'] = (
        'This document is subject to BCP 78 and the IETF Trust\'s Legal '
        'Provisions Relating to IETF Documents '
        '(http://trustee.ietf.org/license-info) in effect on the date of '
        'publication of this document.  Please review these documents '
        'carefully, as they describe your rights and restrictions with respect '
        'to this document.')
    boilerplate['base_copyright_body_s'] = (
        'This document is subject to BCP 78 and the IETF Trust\'s Legal '
        'Provisions Relating to IETF Documents '
        '(https://trustee.ietf.org/license-info) in effect on the date of '
        'publication of this document.  Please review these documents '
        'carefully, as they describe your rights and restrictions with respect '
        'to this document.')

    # IPR values which append things to copyright
    boilerplate['ipr_200902_copyright_ietfbody'] = \
        'Code Components extracted from this document must ' \
        'include Simplified BSD License text as described in Section 4.e of ' \
        'the Trust Legal Provisions and are provided without warranty as ' \
        'described in the Simplified BSD License.'
    boilerplate['ipr_noModification_copyright'] = \
        'This document may not be modified, and derivative works of it may ' \
        'not be created, except to format it for publication as an RFC or ' \
        'to translate it into languages other than English.'
    boilerplate['ipr_noDerivatives_copyright'] = \
        'This document may not be modified, and derivative works of it may ' \
        'not be created, and it may not be published except as an ' \
        'Internet-Draft.'
    boilerplate['ipr_pre5378Trust200902_copyright'] = \
        'This document may contain material from IETF Documents or IETF ' \
        'Contributions published or made publicly available before ' \
        'November 10, 2008.  The person(s) controlling the copyright in some ' \
        'of this material may not have granted the IETF Trust the right to ' \
        'allow modifications of such material outside the IETF Standards ' \
        'Process. Without obtaining an adequate license from the person(s) ' \
        'controlling the copyright in such materials, this document may not ' \
        'be modified outside the IETF Standards Process, and derivative ' \
        'works of it may not be created outside the IETF Standards Process, ' \
        'except to format it for publication as an RFC or to translate it ' \
        'into languages other than English.'

    # Any extra boilerplate
    # Disabled. See issue #123, http://trac.tools.ietf.org/tools/xml2rfc/trac/ticket/123
    ## boilerplate['iprnotified'] = \
    ##     'The IETF has been notified of intellectual property rights ' \
    ##     'claimed in regard to some or all of the specification contained ' \
    ##     'in this document.  For more information consult the online list ' \
    ##     'of claimed rights.'
    
    # Stream approvers
    approvers = {
        'IAB': 'IAB',
        'IRTF': 'IRSG',
        'independent': 'RFC Editor',
    }

    # Valid IPR attributes
    supported_ipr = [
        'trust200902',
        'noModificationTrust200902',
        'noDerivativesTrust200902',
        'pre5378Trust200902',
        'trust200811',
        'noModificationTrust200811',
        'noDerivativesTrust200811',
        'none',
    ]

    # -------------------------------------------------------------------------

    def __init__(self, xmlrfc, quiet=None, options=default_options, date=datetime.date.today()):
        if not quiet is None:
            options.quiet = quiet
        self.options = options
        self.date = date
        self.expire_string = ''
        self.ascii = False
        self.nbws_cond = u'\u00A0'
        self.eref_list = []
        self.xmlrfc = xmlrfc
        self.pis = self.xmlrfc.getpis()

        # We will refer to the XmlRfc document root as 'r'
        self.xmlrfc = xmlrfc
        self.r = xmlrfc.getroot()

        # Document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0

        # Set RFC number and draft flag
        self.rfcnumber = self.r.attrib.get('number', '')
        self.draft = bool(not self.rfcnumber)
        
        # Used for two-pass indexing
        self.indexmode = False

        # Item Indicies
        self._index = []
        self._iref_index = {}

    def _make_iref(self, item, subitem=None, anchor=None):
        """ Create an iref ID object if it doesnt exist yet """
        last = None
        if item not in self._iref_index:
            self._iref_index[item] = _IrefItem()
            last = self._iref_index[item]
        if subitem and subitem not in self._iref_index[item].subitems:
            self._iref_index[item].subitems[subitem] = _IrefItem()
            last = self._iref_index[item].subitems[subitem]
        if last and anchor:
            last.anchor = anchor

    def _add_iref_to_index(self, element):
        item = element.attrib.get('item', None)
        if item:
            subitem = element.attrib.get('subitem', None)
            self._make_iref(item, subitem)
            # Store the buffer position for pagination data later
            pos = len(self.buf)
            if not self.indexmode:
                if pos not in self.iref_marks:
                    self.iref_marks[pos] = []
                self.iref_marks[pos].append((item, subitem))

    def _indexParagraph(self, counter, p_counter, anchor=None, toc=False):
        counter = str(counter)  # This is the section counter
        p_counter = str(p_counter)  # This is the paragraph counter
        autoName = 'Section ' + counter + ', Paragraph ' + p_counter
        autoAnchor = 'rfc.section.' + counter + '.p.' + p_counter
        item = _RfcItem(autoName, autoAnchor, anchor=anchor, toc=toc, counter=p_counter)
        self._index.append(item)
        return item

    def _indexListParagraph(self, p_counter, anchor, toc=False):
        p_counter = str(p_counter)
        autoName = 'Paragraph ' + p_counter
        item = _RfcItem(autoName, '', counter=p_counter, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexSection(self, counter, title=None, anchor=None, toc=True, \
                      level=1, appendix=False, numbered=True):
        counter = str(counter)
        if numbered:
            if appendix:
                autoName = 'Appendix' + self.nbws_cond + counter
                autoAnchor = 'rfc.appendix.' + counter
            else:
                autoName = 'Section' + self.nbws_cond + counter
                autoAnchor = 'rfc.section.' + counter
            item = _RfcItem(autoName, autoAnchor, counter=counter, title=title,
                           anchor=anchor, toc=toc, level=level, appendix=appendix)
        else:
            if not title:
                raise RfcWriterError("No title available when trying to insert index item for anchor %s" % anchor)
            autoAnchor = 'rfc.' + re.sub('[^A-Za-z0-9]+', '_', title).lower()
            item = _RfcItem(title, autoAnchor, title=title)
        self._index.append(item)
        return item

    def _indexReferences(self, counter, title=None, anchor=None, toc=True, \
                         subCounter=0, level=1):
        if subCounter < 1:
            autoName = 'References'
            autoAnchor = 'rfc.references'
        else:
            subCounter = str(subCounter)
            autoName = 'References' + self.nbws_cond + subCounter
            autoAnchor = 'rfc.references.' + subCounter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, \
                       anchor=anchor, toc=toc, level=level)
        self._index.append(item)
        return item
    
    def _indexFigure(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        autoName = 'Figure' + self.nbws_cond + counter
        autoAnchor = 'rfc.figure.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, \
                       toc=toc)
        self._index.append(item)
        return item
        
    def _indexTable(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        autoName = 'Table' + self.nbws_cond + counter
        autoAnchor = 'rfc.table.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexRef(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        if self.pis['symrefs'] == 'yes':
            autoName = '[' + (anchor or counter ) +']'
        else:
            autoName = '['+counter+']'
        autoAnchor = 'rfc.ref.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexCref(self, counter, anchor):
        counter = str(counter)
        autoName = 'Comment' + self.nbws_cond + anchor
        autoAnchor = 'rfc.comment.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, anchor=anchor, toc=False)
        self._index.append(item)
        return item

    def _indexAuthor(self, counter, anchor, name):
        autoName = name
        autoAnchor = 'rfc.author.%s' % counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, anchor=anchor, toc=False)
        self._index.append(item)
        return item

    def get_initials(self, author):
        """author is an rfc2629 author element.  Return the author initials,
        fixed up according to current flavour and policy."""
        initials = author.attrib.get('initials', '')
        multiple = author.pis["multiple-initials"] == "yes"
        initials_list = re.split("[. ]+", initials)
        try:
            initials_list.remove('')
        except:
            pass
        if len(initials_list) > 0:
            if multiple:
                # preserve spacing, but make sure all parts have a trailing
                # period
                initials = initials.strip()
                initials += '.' if not initials.endswith('.') else ''
                initials = re.sub('([^.]) ', '\g<1>. ', initials)
            else:
                initials = initials_list[0] + "."
        return initials

    def parse_pi(self, pi):
        return xml2rfc.utils.parse_pi(pi, self.pis)

    def get_numeric_pi(self, key, default):
        num = self.pis.get(key, None)
        if num is None:
            return default
        if not num.isdigit():
            xml2rfc.log.warn('Expected a numeric value for the %s PI, found "%s"' % (key, num))
            return default
        return int(num)

    def _getTocIndex(self):
        return [item for item in self._index if item.toc]
        
    def _getItemByAnchor(self, anchor):
        for item in self._index:
            if item.autoAnchor == anchor or item.anchor == anchor:
                return item
        return None
    
    def _validate_ipr(self):
        """ Ensure the application has boilerplate for the ipr attribute given """
        ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
        if not ipr in self.supported_ipr:
            raise RfcWriterError('No boilerplate text available for '
            'ipr: \'%s\'.  Acceptable values are: ' % ipr + \
            ', '.join(self.supported_ipr))
        
    def is_numbered(self, node):
        attr = node.attrib.get('numbered', 'true')
        return attr in ['yes', 'true', ]
    
    def _format_date(self):
        """ Fix the date data """
        today = self.date
        date = self.r.find('front/date')
        assert date is not None, "Bug in schema validation: no date element in document"
        year = date.attrib.get('year')
        if not year:
            year = str(self.date.year)
            date.set('year', year)
        if not year.isdigit():
            xml2rfc.log.error("Expected a numeric year, found '%s'" % (year, ))
        year = int(year)
        #
        month = date.attrib.get('month')
        if not month:
            if year != today.year:
                xml2rfc.log.error("Cannot handle a <date> with year different than this year, and no month.  Using today's date.")
                year = today.year
            month = today.month
            date.set('month', str(month))
        else:
            if not month.isdigit():
                month = xml2rfc.util.date.normalize_month(month)
            month = int(month)
        #
        day = date.attrib.get('day')
        if day is None:
            temp_date = datetime.date(year=year, month=month, day=1)
            if today.year == year and today.month == month:
                day = today.day
            elif abs(today - temp_date) < datetime.timedelta(days=34):
                if datetime.date(year=year, month=month, day=1) < today:
                    # wrong month, and the first day of that month is earlier
                    # than today.  Use the last day of the month
                    day = calendar.monthrange(year, month)[1]
                else:
                    # wrong month, later than this month.  Use the first day.
                    day = 1
            else:
                day = 1
        else:
            day = int(day)

        self.date = datetime.date(year=year, month=month, day=day)

        # Setup the expiration string for drafts as published date + six months
        if self.draft:
            if not date.get('day'):
                date.set('day', str(day))
            if not date.get('month'):
                date.set('month', str(month))
            expire_date = self.date + datetime.timedelta(185)
            self.expire_string = expire_date.strftime('%B %d, %Y').replace(' 0', ' ')

    def _format_counter(self, text, count, list_length=1):
        """ Return a proper string for a formatted list bullet.  Allowed types:
                %c: Lowercase chars
                %C: Uppercase chars
                %d: Digits
                %i: Lowercase roman numerals
                %I: Uppercase roman numerals
                %o: octal
                %x: Lowercase hex
                %X: Uppercase hex
        """
        import math
        roman_widths = {        1:1,  2:2,  3:3,  4:2,  5:1,  6:2,  7:3,  8:4,  9:2,
                        10:1, 11:2, 12:3, 13:4, 14:3, 15:2, 16:3, 17:4, 18:5, 19:3,
                        20:2, 21:3, 22:4, 23:5, 24:4, 25:3, 26:4, 27:5, 28:6, 29:4, }
        #
        decimal_width = int(math.log(list_length, 10))
        roman_width = roman_widths.get(list_length, 6)
        letter_width = int(math.log(list_length, 26))
        hex_width = int(math.log(list_length, 16))
        octal_width = int(math.log(list_length, 8))
        extra_width = len(text)+1
        if '%d' in text:
            text = text.replace(r'%d', str(count)).ljust(decimal_width+extra_width)
        elif '%c' in text:
            text = text.replace(r'%c', xml2rfc.util.num.int2letter(count)).ljust(letter_width+extra_width)
        elif '%C' in text:
            text = text.replace(r'%C', xml2rfc.util.num.int2letter(count).upper()).ljust(letter_width+extra_width)
        elif '%i' in text:
            text = text.replace(r'%i', xml2rfc.util.num.int2roman(count)).ljust(roman_width+extra_width)
        elif '%I' in text:
            text = text.replace(r'%I', xml2rfc.util.num.int2roman(count).upper()).ljust(roman_width+extra_width)
        elif '%o' in text:
            text = text.replace(r'%o', oct(count).replace("0","",1)).replace("o","",1).ljust(octal_width+extra_width)
        elif '%x' in text:
            text = text.replace(r'%x', hex(count).replace("0x","",1)).ljust(hex_width+extra_width)
        elif '%X' in text:
            text = text.replace(r'%X', hex(count).replace("0x","",1).upper()).ljust(hex_width+extra_width)
        return text

    def _format_author_string(self, authors):
        """ Given a list of <author> elements, return a readable string of names """
        buf = []
        for i, author in enumerate(authors):
            organization = author.find('organization')
            initials, surname = short_author_name_parts(author)
            if i == len(authors) - 1 and len(authors) > 1:
                buf.append('and ')
            if surname:
                initials = self.get_initials(author) or initials or ''
                if i == len(authors) - 1 and len(authors) > 1:
                    # Last author is rendered in reverse
                    if len(initials) > 0:
                        buf.append(initials + ' ' + \
                                     surname)
                    else:
                        buf.append(surname)
                elif len(initials) > 0:
                    buf.append(surname + ', ' + initials)
                else:
                    buf.append(surname)
                if author.attrib.get('role', '') == 'editor':
                    buf.append(', Ed.')
            elif organization is not None and organization.text:
                # Use organization instead of name
                buf.append(organization.text.strip())
            else:
                continue
            if len(authors) == 2 and i == 0:
                buf.append(' ')
            elif i < len(authors) - 1:
                buf.append(', ')
        return ''.join(buf)

    def _prepare_top_left(self):
        """ Returns a lines of lines for the top left header """
        lines = []
        # Document stream / workgroup
        if not self.pis['private']:
            if self.draft:
                workgroup = self.r.find('front/workgroup')
                if workgroup is not None and workgroup.text:
                    lines.append(workgroup.text)
                else:
                    lines.append(self.boilerplate['draft_workgroup'])
            else:
                # Determine 'workgroup' from submissionType
                subtype = self.r.attrib.get('submissionType', 
                                            self.defaults['submissionType'])
                docstream = self.boilerplate['document_stream'].get(subtype)
                lines.append(docstream)

            # RFC number
            if not self.draft:
                lines.append('Request for Comments: ' + self.rfcnumber)
            elif not self.pis['private']:
                lines.append('Internet-Draft')

            # Series number
            category = self.r.attrib.get('category', '')
            seriesNo = self.r.attrib.get('seriesNo')
            if seriesNo is not None and category in self.boilerplate['series_name']:
                lines.append('%s: %s' % (self.boilerplate['series_name'][category], 
                                         seriesNo))

            # RFC relation notice
            approved_text = self.draft and '(if approved)' or ''
            obsoletes = self.r.attrib.get('obsoletes')
            if obsoletes:
                wrapper = textwrap.TextWrapper(width=40, subsequent_indent=' '*len('Obsoletes: '))
                line = 'Obsoletes: %s %s' % (obsoletes, approved_text)
                lines += wrapper.wrap(line)
            updates = self.r.attrib.get('updates')
            if updates:
                wrapper = textwrap.TextWrapper(width=40, subsequent_indent=' '*len('Updates: '))
                line = 'Updates: %s %s' % (updates, approved_text)
                lines += wrapper.wrap(line)

            # Category
            if category:
                cat_text = self.boilerplate[category]
                if self.draft:
                    lines.append('Intended status: ' + cat_text)
                else:
                    lines.append('Category: ' + cat_text)
            else:
                xml2rfc.log.warn('No category specified for document.')

            # Expiration notice for drafts
            if self.expire_string and not self.pis['private']:
                lines.append('Expires: ' + self.expire_string)

            # ISSN identifier
            if not self.draft:
                lines.append('ISSN: %s' % self.boilerplate['issn'])

        # Strip any whitespace from XML to make header as neat as possible
        lines = [ l.rstrip() for l in lines ]
        return lines

    def _prepare_top_right(self):
        """ Returns a list of lines for the top right header """
        lines = []
        # Keep track of previous organization and remove if redundant.
        last_org = None
        last_pos = None
        authors = self.r.findall('front/author')
        authors = [ a for a in authors if a.get('role') != 'contributor' ]
        for author in authors:
            role = author.attrib.get('role', '')
            if role == 'editor':
                role = ', Ed.'
            initials = self.get_initials(author)
            lines.append(initials + ' ' + author.attrib.\
                         get('surname', '') + role)
            organization = author.find('organization')
            org_name = ''
            if self.options.first_page_author_org:
                if organization is not None:
                    abbrev = organization.attrib.get("abbrev", None)
                    if  abbrev != None and abbrev.strip() != '':
                        org_name = abbrev.strip()
                    elif organization.text and organization.text.strip() != '':
                        org_name = organization.text.strip()
                if org_name == '':
                    lines.append('')
                else:
                    if org_name == last_org:
                        # Remove redundant organization
                        del lines[last_pos]
                    lines.append(org_name)
            last_org = org_name
            last_pos = len(lines)-1
        # remove blank lines between authors and date
        if lines[last_pos] == '':
            del lines[last_pos]
            last_pos = len(lines)-1

        date = self.r.find('front/date')
        if date is not None:
            year = date.attrib.get('year', '')
            month = date.attrib.get('month', '')
            day = date.attrib.get('day', '')
            if month:
                if month.isdigit():
                    month = calendar.month_name[int(month)]
                month = month + ' '
            if day:
                day = day + ', '
            lines.append(month + day + year)
            # Strip any whitespace from XML to make header as neat as possible
            lines = [ l.strip() for l in lines ]
        return lines

    def write_figure(self, figure):
        """ Writes <figure> elements """
        figure_align = figure.attrib.get('align', self.defaults['figure_align'])
        anchor = figure.attrib.get('anchor')
        title = figure.attrib.get('title', self.defaults['figure_title'])
        suppress_title = figure.attrib.get('suppress-title', 'false')

        # Keep track of count if there is an anchor, or PI was enabled
        if anchor or self.pis['figurecount'] == 'yes':
            self.figure_count += 1
        
        if anchor:
            # Insert anchor(s) for the figure
            self.insert_anchor('rfc.figure.' + str(self.figure_count))
            self.insert_anchor(anchor)
            if self.indexmode:
                # Add figure to the index, inserting any anchors necessary
                self._indexFigure(self.figure_count, anchor=anchor, title=title)

        # Write preamble
        preamble = figure.find('preamble')
        if preamble is not None:
            self.write_t_rec(preamble, align=figure_align)

        # iref
        for element in figure:
            if element.tag == 'iref':
                self._add_iref_to_index(element)

        # Write figure with optional delimiter
        delimiter = figure.pis['artworkdelimiter']
        artwork = figure.find('artwork')
        artwork_align = artwork.attrib.get('align', figure_align)
        blanklines = int(figure.pis['artworklines'])
        self.write_raw(artwork.text, align=artwork_align,
                       blanklines=blanklines, delimiter=delimiter,
                       source_line=figure.sourceline)

        # Write postamble
        postamble = figure.find('postamble')
        if postamble is not None:
            self.write_t_rec(postamble, align=figure_align)

        # Write label
        title = figure.attrib.get('title', '')
        if anchor or self.pis['figurecount'] == 'yes':
            if suppress_title == 'false':
                if title:
                    title = 'Figure ' + str(self.figure_count) + ': ' + title
                else:
                    title = 'Figure ' + str(self.figure_count)
        if title:
            self.write_label(title, type='figure', source_line=figure.sourceline)

    def write_table(self, table):
        """ Writes <texttable> elements """
        align = table.attrib.get('align', self.defaults['table_align'])
        anchor = table.attrib.get('anchor')
        title = table.attrib.get('title', self.defaults['table_title'])
        suppress_title = table.attrib.get('suppress-title', 'false')

        # Keep track of count if there is an anchor, or PI was enabled
        if anchor or self.pis['tablecount'] == 'yes':
            self.table_count += 1

        if anchor:
            # Insert anchor(s) for the table
            self.insert_anchor('rfc.table.' + str(self.table_count))
            self.insert_anchor(anchor)
            if self.indexmode:
                # Add table to the index, inserting any anchors necessary
                self._indexTable(self.table_count, anchor=anchor, title=title)

        # Write preamble
        preamble = table.find('preamble')
        if preamble is not None:
            self.write_t_rec(preamble, align=align)

        # Write table
        self.draw_table(table, table_num=self.table_count)

        # Write postamble
        postamble = table.find('postamble')
        if postamble is not None:
            self.write_t_rec(postamble, align=align)

        # Write label if anchor is set or PI figurecount = yes
        if anchor or self.pis['tablecount'] == 'yes':
            title = table.attrib.get('title', '')
            if suppress_title == 'false':
                if title:
                    title = 'Table ' + str(self.table_count) + ': ' + title
                else:
                    title = 'Table ' + str(self.table_count)
        if title:
            self.write_label(title, type='table', source_line=table.sourceline)

    def _index_t_rec(self, element):
        """ Traverse a <t> element only performing indexing operations """
        pass
        

    def numToAlpha(self, n):
        str = ""
        az = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
        while n > 0:
            r = (n-1) % 26
            n = int((n-1)/26)
            str = az[r] + str
        return str

    def write_section_rec(self, section, count_str="1.", appendix=False,
                           level=0, numbered=True, emit_numbered="all", s_count=1):
        """ Recursively writes <section> elements 
        
            We use the self.indexmode flag to determine whether or not we
            render actual text at this point or just lookup header information
            to construct the index
        """
        if level > 0:
            anchor = section.attrib.get('anchor')
            title = section.attrib.get('title')
            include_toc = section.attrib.get('toc', self.defaults['section_toc']) != 'exclude' \
                          and (not appendix or self.pis['tocappendix'] == 'yes')
            if level == 1:
                numbered = numbered and self.is_numbered(section)
            else:
                numbered = self.is_numbered(section)
            if level > 1 and not numbered:
                if not self.indexmode:
                    xml2rfc.log.warn('Unnumbered subsections are not permitted: found section "%s" with attribute numbered="no" or "false"' % (title, ))
                numbered = True
            if self.indexmode:
                # Add section to the index
                self._indexSection(count_str, title=title, anchor=anchor,
                                   toc=include_toc, level=level,
                                   appendix=appendix, numbered=numbered)
            else:
                # Write the section heading
                if numbered:
                    autoAnchor = 'rfc.' + ('section.' if not appendix else 'appendix.') + count_str
                else:
                    autoAnchor = 'rfc.' + re.sub('[^A-Za-z0-9]+', '_', title).lower()
                bullet = appendix and level == 1 and 'Appendix %s' % count_str or count_str
                self.write_heading(title, bullet=bullet + '.' if numbered else "",
                                   autoAnchor=autoAnchor,
                                   anchor=anchor, level=level)
        else:
            # Must be <middle> or <back> element -- no title or index.
            count_str = ''
            numbered = True

        p_count = 1  # Paragraph counter
        for element in section:
            # Check for a PI
            if element.tag is lxml.etree.PI:
                pidict = self.parse_pi(element)
                if pidict and "needLines" in pidict:
                    self.needLines(pidict["needLines"])
            # Write elements in XML document order
            if element.tag == 't':
                anchor = element.attrib.get('anchor')
                if self.indexmode:
                    self._indexParagraph(count_str, p_count, anchor=anchor)
                autoAnchor = 'rfc.section.' + count_str + '.p.' + str(p_count)
                self.write_t_rec(element, autoAnchor=autoAnchor)
                p_count += 1
            elif element.tag == 'figure':
                self.write_figure(element)
            elif element.tag == 'texttable':
                self.write_table(element)
            elif element.tag == 'iref':
                self._add_iref_to_index(element)

        # Append a dot to separate sub counters
        if count_str:
            count_str += '.'

        # Recurse on sections
        for child_sec in section.findall('section'):
            if level == 0:
                if numbered:
                    numbered = self.is_numbered(child_sec)
                elif self.is_numbered(child_sec):
                    title = child_sec.attrib.get('title')
                    if not self.indexmode:
                        xml2rfc.log.warn('Numbered sections are not permitted after unnumbered sections: found section "%s" without attribute numbered="no"' % (title,))
                    numbered = False
                if emit_numbered=="only" and not numbered:
                    continue
                if emit_numbered=="no" and numbered:
                    continue
            if appendix == True and not count_str:
                if s_count == 1 and self.pis["rfcedstyle"] == "yes":
                   self.needLines(-1)
                # Use an alphabetic counter for first-level appendix
                self.write_section_rec(child_sec, self.numToAlpha(s_count),
                                        level=level + 1, appendix=True, numbered=numbered)
            else:
                # Use a numeric counter
                self.write_section_rec(child_sec, count_str + str(s_count), 
                                        level=level + 1, appendix=appendix, numbered=numbered)

            s_count += 1

        # Set the ending index number so we know where to begin references
        if count_str == '' and appendix == False:
            self.refs_start = s_count

        return s_count

    def write_status_section(self):
        """ Writes the 'Status of this Memo' section """

        self.write_heading('Status of This Memo', autoAnchor='rfc.status')
        if not self.draft:  #RFC
            # Status boilerplate is determined by category/submissionType/consensus
            category = self.r.attrib.get('category', 'none')
            stream = self.r.attrib.get('submissionType', 
                                       self.defaults['submissionType'])
            # Consensus is assumed 'yes' for Standards Track documents and BCPs
            consensus = category in ['std', 'bcp'] and 'yes' or \
                        self.r.attrib.get('consensus', self.defaults['consensus'])
            workgroup = ''
            wg_element = self.r.find('front/workgroup')
            if wg_element is not None and wg_element.text:
                workgroup = wg_element.text

            # Write first paragraph
            if category in self.boilerplate['status']:
                self.write_paragraph(self.boilerplate['status'][category].get('p1', ''))
            
            # Build second paragraph
            p2 = []
            if category in self.boilerplate['status']:
                p2.append(self.boilerplate['status'][category].get('p2', ''))
            if stream == 'IETF':
                if consensus == 'yes':
                    p2.append(self.boilerplate['status']['IETF_consensus'])
                else:
                    p2.append(self.boilerplate['status']['IETF'])
            elif stream == 'IRTF':
                p2.append(self.boilerplate['status']['IRTF'])
                if workgroup:
                    if consensus == 'yes':
                        p2.append(self.boilerplate['status']['IRTF_workgroup_consensus'] \
                                  % workgroup)
                    else:
                        p2.append(self.boilerplate['status']['IRTF_workgroup'] \
                                  % workgroup)
            elif stream == 'IAB':
                if consensus == 'yes':
                    p2.append(self.boilerplate['status']['IAB_consensus'])
                else:
                    p2.append(self.boilerplate['status']['IAB'])
            else:
                p2.append(self.boilerplate['status'].get(stream, ''))

            # Last sentence of p2
            if self.date < datetime.date(year=2016, month=6, day=1):
                status="status_5741"
            else:
                status="status"         # Current boilerplate
            if stream == 'IETF' and category == 'std':
                p2.append(self.boilerplate[status]['p2end_ietf_std'])
            elif stream == 'IETF' and category == 'bcp':
                p2.append(self.boilerplate[status]['p2end_ietf_bcp'])
            elif stream == 'IETF':
                p2.append(self.boilerplate[status]['p2end_ietf_other'])
            else:
                p2.append(self.boilerplate[status]['p2end_other'] \
                          % self.approvers.get(stream, ''))


            # Write second paragraph
            self.write_paragraph('  '.join(p2))
            
            # Write third paragraph
            key = 'p3' if self.date < self.boilerplate_https_date else 'p3_s'
            self.write_paragraph(self.boilerplate['status'][key] % self.rfcnumber)

        else:  # Draft
            # Start by checking for an ipr header
            ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
            if '200902' in ipr:
                self.write_paragraph(self.boilerplate['ipr_200902_status'])
            elif '200811' in ipr:
                self.write_paragraph(self.boilerplate['ipr_200811_status'])

            # Write the standard draft status
            key = 'draft' if self.date < self.boilerplate_https_date else 'draft_s'
            for par in self.boilerplate['status'][key]:
                self.write_paragraph(par)

            # Write expiration string, if it was generated
            if self.expire_string:
                self.write_paragraph( \
                    self.boilerplate['draft_expire'] % self.expire_string)
    
    def write_copyright(self):
        """ Writes the 'Copyright' section """
        self.write_heading('Copyright Notice', autoAnchor='rfc.copyrightnotice')

        # Write header line with year
        date = self.r.find('front/date')
        year = ''
        if date is not None:
            year = date.attrib.get('year', self.date.year)
        self.write_paragraph(self.boilerplate['base_copyright_header'] % year)

        # Write next paragraph which may be modified by ipr
        ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
        key = 'base_copyright_body' if self.date < self.boilerplate_https_date else 'base_copyright_body_s'
        body = self.boilerplate[key]
        if '200902' in ipr and self.r.attrib.get('submissionType', 
                                   self.defaults['submissionType']) == 'IETF':
            body += '  ' + self.boilerplate['ipr_200902_copyright_ietfbody']
        self.write_paragraph(body)
        
        # Write any additional paragraphs
        if 'noModification' in ipr:
            self.write_paragraph(self.boilerplate['ipr_noModification_copyright'])
        elif 'noDerivatives' in ipr:
            self.write_paragraph(self.boilerplate['ipr_noDerivatives_copyright'])
        elif ipr == 'pre5378Trust200902':
            self.write_paragraph(self.boilerplate['ipr_pre5378Trust200902_copyright'])

    def _build_index(self):
        self.indexmode = True
        # Reset document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0
        self.pis = self.xmlrfc.getpis()

        # Abstract
        abstract = self.r.find('front/abstract')
        if abstract is not None:
            self.write_heading('Abstract', autoAnchor='rfc.abstract')
            for t in abstract.findall('t'):
                self.write_t_rec(t)

        # Authors
        count = 0
        authors = self.r.findall('front/author')
        for author in authors:
            count += 1
            autoName = full_author_name_expansion(author)
            self._indexAuthor(count, author.get('anchor'), autoName)

        # Optional notes
        for note in self.r.findall('front/note'):
            self.write_heading(note.attrib.get('title', 'Note'))
            for t in note.findall('t'):
                self.write_t_rec(t)

        # Middle sections
        middle = self.r.find('middle')
        if middle is not None:
            self.write_section_rec(middle, None)

        # References sections
        # Treat references as nested only if there is more than one
        ref_counter = 0
        refs_counter = str(self.refs_start)
        references = self.r.findall('back/references')
        # Write root level references header
        refs_title = self.pis['refparent']
        if len(references) == 1 and not self.eref_list:
            refs_title = references[0].attrib.get('title', refs_title)

        if len(references) > 0:
            self._indexReferences(refs_counter, title=refs_title)

        if len(references) > 1 or self.eref_list:
            for i, reference_list in enumerate(references):
                refs_newcounter = refs_counter + '.' + str(i + 1)
                refs_title = reference_list.attrib.get('title', self.pis["refparent"])
                self._indexReferences(refs_newcounter, title=refs_title, \
                                      subCounter=i+1, level=2)
            if self.eref_list:
                refs_newcounter = refs_counter + '.' + str(len(references)+1)
                self._indexReferences(refs_newcounter, title="URIs", level=2, subCounter=len(references)+1)


        for ref in self.r.xpath('.//references//reference'):
            if len(ref):
                ref_counter += 1
                title = ref.find("front/title")
                if title != None:
                    if 'anchor' in ref.attrib:
                        self._indexRef(ref_counter, title=title.text, anchor=ref.attrib["anchor"])
                    else:
                        raise RfcWriterError("Reference is missing an anchor: %s" % lxml.etree.tostring(ref))
                        
        # Appendix sections
        back = self.r.find('back')
        if back is not None:
            s_count = self.write_section_rec(back, None, appendix=True, emit_numbered="only")

        # Index section, disable if there are no irefs
        if len(self._iref_index) > 0:
            # Add explicitly to index
            title = 'Index'
            autoAnchor = 'rfc.index'
            item = _RfcItem(title, autoAnchor, title=title)
            self._index.append(item)

        if back is not None:
            self.write_section_rec(back, None, appendix=True, emit_numbered="no", s_count=s_count)

        # Authors addresses section

        if self.pis['authorship'] == 'yes':
            authors = self.r.findall('front/author')
            authors = [ a for a in authors if a.get('role') != 'contributor' ]
            autoAnchor = 'rfc.authors'
            if len(authors) > 1:
                title = "Authors' Addresses"
            else:
                title = "Author's Address"
            # Add explicitly to index
            item = _RfcItem(title, autoAnchor, title=title)
            self._index.append(item)

    def _build_document(self):
        self.indexmode = False
        # Reset document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0
        self.pis = self.xmlrfc.getpis()

        # Block header
        topblock = self.pis['topblock']
        if topblock == 'yes':
            self.write_top(self._prepare_top_left(), \
                               self._prepare_top_right())

        # Title & Optional docname
        title = self.r.find('front/title')
        if title is not None:
            docName = self.r.attrib.get('docName', None)
            rfcnum = self.r.attrib.get('number', None)
            if (not docName or not docName.strip()) and not rfcnum:
                xml2rfc.log.warn("No (or empty) 'docName' attribute in the <rfc/> element -- can't insert draft name on first page.")
            if docName and '.' in docName:
                xml2rfc.log.warn("The 'docName' attribute of the <rfc/> element should not contain any filename extension: docName=\"draft-foo-bar-02\".")
            if docName and not rfcnum and not re.search('-\d\d$', docName):
                xml2rfc.log.warn("The 'docName' attribute of the <rfc/> element should have a revision number as the last component: docName=\"draft-foo-bar-02\".")
            self.write_title(title.text, docName, title.sourceline)

        # Abstract
        abstract = self.r.find('front/abstract')
        if abstract is not None:
            self.write_heading('Abstract', autoAnchor='rfc.abstract')
            for t in abstract.findall('t'):
                self.write_t_rec(t)

        # Optional notified boilerplate
        # Disabled. See issue #123, http://trac.tools.ietf.org/tools/xml2rfc/trac/ticket/123
        ## if self.pis['iprnotified'] == 'yes':
        ##    self.write_paragraph(BaseRfcWriter.boilerplate['iprnotified'])

        # Optional notes
        for note in self.r.findall('front/note'):
            self.write_heading(note.attrib.get('title', 'Note'))
            for t in note.findall('t'):
                self.write_t_rec(t)

        if not self.pis['private']:
            # Verify that 'ipr' attribute is valid before continuing
            self._validate_ipr()

            # "Status of this Memo" section
            self.write_status_section()

            # Copyright section
            self.write_copyright()

        # Insert the table of contents marker at this position
        toc_enabled = self.pis['toc']
        if toc_enabled == 'yes':
            self.insert_toc()

        # Middle sections
        middle = self.r.find('middle')
        if middle is not None:
            self.write_section_rec(middle, None)

        # References sections
        # Treat references as nested only if there is more than one
        refs_counter = str(self.refs_start)
        references = self.r.findall('back/references')
        # Write root level references header
        refs_title = self.pis['refparent']
        if len(references) == 1 and not self.eref_list:
            refs_title = references[0].attrib.get('title', refs_title)

        if len(references) > 0 or self.eref_list:
            self.write_heading(refs_title, bullet=refs_counter + '.', \
                               autoAnchor='rfc.references')
        if len(references) > 1:
            for i, reference_list in enumerate(references):
                refs_newcounter = refs_counter + '.' + str(i + 1)
                refs_title = reference_list.attrib.get('title', self.pis['refparent'])
                autoAnchor = 'rfc.references.' + str(i + 1)
                self.write_heading(refs_title, bullet=refs_newcounter + '.',\
                                   autoAnchor=autoAnchor, level=2)
                self.write_reference_list(reference_list)
        elif len(references) == 1:
            if self.eref_list:
                refs_newcounter = refs_counter + '.1'
                refs_title = references[0].attrib.get('title', self.pis['refparent'])
                autoAnchor = 'rfc.references.1'
                self.write_heading(refs_title, bullet=refs_newcounter + '.',\
                                   autoAnchor=autoAnchor, level=2)
            self.write_reference_list(references[0])

        if self.eref_list:
            self.write_erefs(refs_counter, len(references)+1)

        # Appendix sections
        back = self.r.find('back')
        if back is not None:
            s_count = self.write_section_rec(back, None, appendix=True, emit_numbered="only")

        # Index section, disable if there are no irefs
        if len(self._iref_index) > 0:
            self.insert_iref_index()

        if back is not None:
            self.write_section_rec(back, None, appendix=True, emit_numbered="no", s_count=s_count)

        self.write_crefs()

        # Authors addresses section
        if self.pis['authorship'] == 'yes':
            authors = self.r.findall('front/author')
            authors = [ a for a in authors if a.get('role') != 'contributor' ]
            autoAnchor = 'rfc.authors'
            if len(authors) > 1:
                title = "Authors' Addresses"
            else:
                title = "Author's Address"
            self.write_heading(title, autoAnchor=autoAnchor)
            for author in authors:
                self.write_address_card(author)

        self.check_for_unused_references()
        
    def write(self, filename, tmpfile=None):
        """ Public method to write the RFC document to a file. """
        # If the ascii flag is enabled, replace unicode with ascii in the tree
        if self.ascii:
            xml2rfc.utils.safeReplaceUnicode(self.r)

        # Protect any words with slashes that are on the preserve list
        xml2rfc.utils.safeTagSlashedWords(self.r)

        # Do any pre processing necessary, such as inserting metadata
        self.pre_indexing()
        # Make two passes over the document, the first pass we run in
        # 'index mode' to construct the internal index and other things, 
        # the second pass will assemble a buffer and render the actual text
        self._build_index()
        # Format the date properly
        self._format_date()
        # Do any pre-build processing necessary, such as inserting metadata
        self.pre_rendering()
        # Build the document
        self._build_document()
        # Primary buffer is finished -- apply any post processing
        self.post_rendering()

        # Finished processing, write to file
        # Override file with keyword argument if passed in, ignoring filename.
        # Warning -- make sure file is open and ready for writing!
        if not tmpfile:
            if self.ascii:
                file = open(filename, 'w')
            else:
                # Open as unicode
                file = codecs.open(filename, 'w', encoding='utf-8')
            self.write_to_file(file)
            file.close()
        else:
            self.write_to_file(tmpfile)

        if not self.options.quiet and filename:
            xml2rfc.log.write(' Created file', filename)

    def write_erefs(self, refs_counter, refs_subsection):
        """ Only text versions do this so provide a default that does nothing
        """
        pass

    def write_crefs(self):
        """ Only text versions do this so provide a default that does nothing
        """
        pass

    def check_for_unused_references(self):
        """ If this is a reference and it is not used - then warn me
        """
        if not self.indexmode:
            for item in self._index:
                if item.autoAnchor.startswith("rfc.ref.") and not item.used:
                    xml2rfc.log.warn("No <xref> in <rfc> targets <reference anchor='%s'>" % item.anchor)

    def needLines(self, count):
        """ Deal with the needLines PI """
        pass
                
    # -----------------------------------------
    # Base writer interface methods to override
    # -----------------------------------------

    def insert_toc(self):
        """ Marks the current buffer position to insert ToC at """
        raise NotImplementedError('insert_toc() needs to be overridden')
    
    def insert_iref_index(self):
        """ Marks the current buffer position to insert the index at """
        raise NotImplementedError('insert_iref_index() needs to be '
                                  'overridden')

    def write_raw(self, text, indent=3, align='left', blanklines=0,
                  delimiter=None, source_line=None):
        """ Writes a block of text that preserves all whitespace """
        raise NotImplementedError('write_raw() needs to be overridden')

    def write_label(self, text, type='figure', source_line=None):
        """ Writes a table or figure label """
        raise NotImplementedError('write_label() needs to be overridden')

    def write_title(self, title, docName=None, source_line=None):
        """ Writes the document title """
        raise NotImplementedError('write_title() needs to be overridden')

    def write_heading(self, text, bullet='', autoAnchor=None, anchor=None,
                      level=1, breakNeed=0):
        """ Writes a section heading """
        raise NotImplementedError('write_heading() needs to be overridden')

    def write_paragraph(self, text, align='left', autoAnchor=None):
        """ Writes a paragraph of text """
        raise NotImplementedError('write_paragraph() needs to be overridden')

    def write_t_rec(self, t, align='left', autoAnchor=None):
        """ Recursively writes <t> elements """
        raise NotImplementedError('write_t_rec() needs to be overridden')

    def write_top(self, left_header, right_header):
        """ Writes the main document header

            Takes two list arguments, one for each margin, and combines them
            so that they exist on the same lines of text
        """
        raise NotImplementedError('write_top() needs to be overridden')

    def write_address_card(self, author):
        """ Writes the address information for an <author> element """
        raise NotImplementedError('write_address_card() needs to be ' \
                                  'overridden')

    def write_reference_list(self, list):
        """ Writes a <references> element """
        raise NotImplementedError('write_reference_list() needs to be ' \
                                  'overridden')

    def insert_anchor(self, text):
        """ Inserts a document anchor for internal links """
        raise NotImplementedError('insert_anchor() needs to be overridden')

    def draw_table(self, table, table_num=None):
        """ Draws a formatted table from a <texttable> element

            For HTML nothing is really 'drawn' since we can use <table>
        """
        raise NotImplementedError('draw_table() needs to be overridden')

    def pre_indexing(self):
        """ First method that is called before traversing the XML RFC tree for indexing"""
        raise NotImplementedError('pre_indexing() needs to be overridden')

    def pre_rendering(self):
        """ First method that is called before traversing the XML RFC tree for rendering"""
        raise NotImplementedError('pre_rendering() needs to be overridden')

    def post_rendering(self):
        """ Last method that is called after traversing the XML RFC tree """
        raise NotImplementedError('post_rendering() needs to be overridden')

    def write_to_file(self, file):
        """ Writes the finished buffer to a file """
        raise NotImplementedError('write_to_file() needs to be overridden')

# --------------------------------------------------------------------------------------------------
# Sets of various kinds of tags in the v3 schema
        
deprecated_element_tags = set([
    'c',
    'facsimile',
    'format',
    'list',
    'postamble',
    'preamble',
    'relref',
    'spanx',
    'texttable',
    'ttcol',
    'vspace',
])

deprecated_attributes = [
    # element, attrbute
    ('artwork', 'height'),
    ('artwork', 'width'),
    ('artwork', 'xml:space'),
    ('figure', 'align'),
    ('figure', 'alt'),
    ('figure', 'height'),
    ('figure', 'src'),
    ('figure', 'suppress-title'),
    ('figure', 'title'),
    ('figure', 'width'),
    ('note', 'title'),
    ('references', 'title'),
    ('section', 'title'),
    ('seriesInfo', 'status'),
    ('seriesInfo', 'stream'),
    ('t', 'hangText'),
    ('texttable', 'title'),
    ('xref', 'pageno'),
]


v3_rnc_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data', 'v3.rnc')
v3_rng_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data', 'v3.rng')
v3_schema = lxml.etree.ElementTree(file=v3_rng_file)

def get_element_tags():
    tags = set()
    elements = v3_schema.xpath("/x:grammar/x:define/x:element", namespaces=namespaces)
    for element in elements:
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_meta_tags():
    "Get tags that can have text content but don't occur in regular text"
    tags = set()
    for elem in ['author', 'address', 'front', 'postal', 'reference' ]:
        refs = v3_schema.xpath("/x:grammar/x:define[@name='%s']//x:ref"%elem, namespaces=namespaces)
        for r in refs:
            name = r.get('name')
            if not name in tags:
                tags.add(name)
    return tags

def get_text_tags():
    "Get tags that can have text content from the schema"
    tags = set()
    texts = v3_schema.xpath("/x:grammar/x:define/x:element//x:text", namespaces=namespaces)
    for t in texts:
        element = list(t.iterancestors('{*}element'))[0]
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_text_or_block_tags():
    "Get tags that can have either text or block contents from the schema"
    tags = set()
    texts = v3_schema.xpath("/x:grammar/x:define/x:element/x:choice/*/x:choice//x:text", namespaces=namespaces)
    for t in texts:
        element = list(t.iterancestors('{*}element'))[0]
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_inline_tags():
    "Get tags that can occur within text from the schema"
    tags = set()
    referenced = v3_schema.xpath("/x:grammar/x:define/x:element//x:ref", namespaces=namespaces)
    for ref in referenced:
        name = ref.get('name')
        if not name in tags:
            p = ref.getparent()
            text = p.find('x:text', namespaces=namespaces)
            if text != None:
                tags.add(name)
    return tags

def get_xref_tags():
    "Get tags that can occur as direct children of <xref>"
    tags = set()
    refs = v3_schema.xpath("/x:grammar/x:define/x:element[@name='xref']//x:ref", namespaces=namespaces)
    tags = set([ c.get('name') for c in refs ])
    return tags


element_tags= get_element_tags()
meta_tags   = get_meta_tags() - deprecated_element_tags
text_tags   = get_text_tags() - deprecated_element_tags
inline_tags = get_inline_tags() - deprecated_element_tags
block_tags  = element_tags - inline_tags - deprecated_element_tags
mixed_tags  = get_text_or_block_tags() - deprecated_element_tags
xref_tags   = get_xref_tags()

# --------------------------------------------------------------------------------------------------

class BaseV3Writer(object):

    def __init__(self, xmlrfc, quiet=None, options=default_options, date=datetime.date.today()):
        global v3_rnc_file, v3_rng_file, v3_schema
        self.xmlrfc = xmlrfc
        self.tree = xmlrfc.tree if xmlrfc else None
        self.root = self.tree.getroot() if xmlrfc else None
        self.options = options
        self.date = date
        self.v3_rnc_file = v3_rnc_file
        self.v3_rng_file = v3_rng_file
        self.v3_rng = lxml.etree.RelaxNG(file=self.v3_rng_file)
        self.v3_schema = v3_schema
        self.schema = v3_schema
        self.index_items = []
        self.meta_tags = set(meta_tags)
        self.text_tags = set(text_tags)
        self.inline_tags = set(inline_tags)
        self.mixed_tags = set(mixed_tags)
        self.xref_tags = set(xref_tags)
        self.attribute_defaults = self.get_all_attribute_defaults()
        #
        #
        self.errors = []
#         if options.debug:
#             found_handlers = []
#             missing_handlers = []
#             for tag in (self.get_element_tags() - deprecated_element_tags):
#                 func_name = "render_%s" % (tag.lower(),)
#                 if getattr(self, func_name, False):
#                     found_handlers.append(func_name)
#                 else:
#                     missing_handlers.append(func_name)
#             debug.pprint('found_handlers')
#             debug.show('len(found_handlers)')
#             debug.pprint('missing_handlers')
#             debug.show('len(missing_handlers)')

    def log(self, msg):
        xml2rfc.log.write(msg)

    def msg(self, e, label, text):
        if e != None:
            lnum = getattr(e, 'sourceline', None)
            file = getattr(e, 'base', None)
            if lnum:
                msg = "%s(%s): %s %s" % (file or self.xmlrfc.source, lnum, label, text, )
            else:
                msg = "(No source line available): %s %s" % (label, text, )
        else:
            msg = "%s %s" % (label, text)
        return msg

    def get_relevant_pis(self, e):
        pis = []
        if e != None:
            # directly inside element
            for c in e.getchildren():
                if c.tag == lxml.etree.PI and c.target == xml2rfc.V3_PI_TARGET:
                    pis.append(c)
            # siblings before element
            for s in e.itersiblings(preceding=True):
                if s.tag == lxml.etree.PI and s.target == xml2rfc.V3_PI_TARGET:
                    pis.append(s)
            # ancestor's earlier siblings
            for a in e.iterancestors():
                for s in a.itersiblings(preceding=True):
                    if s.tag == lxml.etree.PI and s.target == xml2rfc.V3_PI_TARGET:
                        pis.append(s)
            # before root elements
            p = self.root.getprevious()
            while p != None:
                if p.tag == lxml.etree.PI and p.target == xml2rfc.V3_PI_TARGET:
                    pis.append(p)
                p = p.getprevious()
        return pis

    def get_relevant_pi(self, e, name):
        pis = self.get_relevant_pis(e)
        pi_list = list(filter(None, [ pi.get(name) for pi in pis ]))
        return pi_list[-1] if pi_list else None

    def silenced(self, e, text):
        text = text.strip()
        pis = self.get_relevant_pis(e)
        silenced = filter(None, self.options.silence[:] + [ pi.get('silence') for pi in pis ])
        for s in silenced:
            for label in ['Note: ', 'Warning: ']:
                if s.startswith(label):
                    s = s[len(label):]
            if text.startswith(s):
                return True
            try:
                if re.match(s, text):
                    return True
            except re.error:
                pass
        return False

    def note(self, e, text, label='Note:'):
        if self.options.verbose:
            if not self.silenced(e, text):
                self.log(self.msg(e, label, text))

    def warn(self, e, text, label='Warning:'):
        if self.options.verbose or not self.silenced(e, text):
            self.log(self.msg(e, label, text))

    def err(self, e, text, trace=False):
        msg = self.msg(e, 'Error:', text)
        self.errors.append(msg)
        if trace or self.options.debug:
            raise RuntimeError(msg)
        else:
            self.log(msg)

    def die(self, e, text, trace=False):
        msg = self.msg(e, 'Error:', text)
        self.errors.append(msg)
        raise RfcWriterError(msg)

    def xinclude(self):
        ## From RFC7998:
        ##
        # 5.1.1.  XInclude Processing
        # 
        #    Process all <x:include> elements.  Note: XML <x:include> elements may
        #    include more <x:include> elements (with relative references resolved
        #    against the base URI potentially modified by a previously inserted
        #    xml:base attribute).  The tool may be configurable with a limit on
        #    the depth of recursion.
        try:
            self.tree.xinclude()
        except etree.XIncludeError as e:
            self.die(None, "XInclude processing failed: %s" % e)

    def remove_dtd(self):
        # 
        # 5.1.2.  DTD Removal
        # 
        #    Fully process any Document Type Definitions (DTDs) in the input
        #    document, then remove the DTD.  At a minimum, this entails processing
        #    the entity references and includes for external files.

        ## Entities has been resolved as part of the initial parsing.  Remove
        ## docinfo and PIs outside the <rfc/> element by copying the root
        ## element and creating a new tree.
        root = copy.deepcopy(self.root)
        self.tree = root.getroottree()
        self.root = root

    def get_refname_mapping(self):
        reflist = self.root.xpath('.//references/reference|.//references/referencegroup')
        if self.root.get('symRefs', 'true') == 'true':
            refname_mapping = dict( (e.get('anchor'), e.get('anchor')) for e in reflist )
        else:
            refname_mapping = (dict( (e.get('anchor'), str(i+1)) for i,e in enumerate(reflist) ))
        refname_mapping.update(dict( (e.get('target'), e.get('to')) for e in self.root.xpath('.//displayreference') ))
        return refname_mapping

    def dispatch(self, selectors):
        """
        Process selectors, extracting an XPath selector and generating a method name
        from each entry in self.selectors, and calling the method with all elements
        matching the XPath expression, in order to process self.tree.
        """
        # Setup
        selector_visits = dict( (s, 0) for s in selectors)
        # Check for duplicate <displayreference> 'to' values:
        seen = {}
        for e in self.root.xpath('.//displayreference'):
            to = e.get('to')
            if to in set(seen.keys()):
                self.die(e, 'Found duplicate displayreference value: "%s" has already been used in %s' % (to, etree.tostring(seen[to]).strip()))
            else:
                seen[to] = e
        del seen
        ## Do remaining processing by xpath selectors (listed above)
        for s in selectors:
            slug = slugify(s.replace('self::', '').replace(' or ','_').replace(';','_'))
            if '@' in s:
                func_name = 'attribute_%s' % slug
            elif "()" in s:
                func_name = slug
            else:
                if not slug:
                    slug = 'rfc'
                func_name = 'element_%s' % slug
            # get rid of selector annotation
            ss = s.split(';')[0]
            func = getattr(self, func_name, None)
            if func:
                if self.options.debug:
                    self.note(None, "Calling %s()" % func_name)
                for e in self.tree.xpath(ss):
                    func(e, e.getparent())
                    selector_visits[s] += 1
            else:
                self.warn(None, "No handler %s() found" % (func_name, ))
        if self.options.debug:
            for s in selectors:
                if selector_visits[s] == 0:
                    self.note(None, "Selector '%s' has not matched" % (s))
        if self.errors:
            raise RfcWriterError("Not creating output file due to errors (see above)")
        return self.tree

    def get_all_attribute_defaults(self):
        defaults = {}
        elements = self.schema.xpath("/x:grammar/x:define/x:element", namespaces=namespaces)
        for tag in [ e.get('name') for e in elements ]:
            attr = self.schema.xpath("/x:grammar/x:define/x:element[@name='%s']//x:attribute" % tag, namespaces=namespaces)
            defaults[tag] = dict( (a.get('name'), a.get("{%s}defaultValue"%namespaces['a'], None)) for a in attr )
        return defaults

    def check_refs_numbered(self):
        # see if references should be numbered.  This is True unless the last
        # top-level section of <middle/> had numbered='false'.
        if not hasattr(self, '_refs_numbered'):
            last_middle_section = ([None, ] + list(self.root.find('middle').iterchildren('section')))[-1]
            self._refs_numbered = last_middle_section.get('numbered', 'true') == 'true'
        return self._refs_numbered

    # methods operating on the xml tree

    def get_element_from_id(self, id):
        for a in 'anchor', 'pn', 'slugifiedName':
            elem = self.root.find('.//*[@%s="%s"]'%(a, id, ))
            if elem != None:
                break
        return elem

    def get_element_page(self, e):
        page = getattr(e, 'page', None)
        if not page:
            for a in e.iterancestors():
                page = getattr(a, 'page', None)
                if page != None:
                    break
        return page

#     def get_valid_child_tags(self, tag):
#         refs = self.schema.xpath("/x:grammar/x:define/x:element[@name='%s']//x:ref" % tag, namespaces=utils.namespaces)
#         names = set([ r.get('name') for r in refs ])
#         return names

    def write(self, filename):
        raise NotImplementedError()

    # methods for use in setting up page headers and footers

    def page_top_left(self):
        if self.root.get('ipr') == 'none':
            return ''
        number = self.root.get('number')
        if number != None:
            return 'RFC %s' % number
        info = self.root.find('./front/seriesInfo[@name="RFC"]')
        if info != None:
            return 'RFC %s' % info.get('value')
        return 'Internet-Draft'

    def page_top_center(self):
        title = self.root.find('./front/title')
        text = title.get('abbrev') or ' '.join(title.itertext())
        return text

    def page_top_right(self):
        date = self.root.find('./front/date')
        year, month, day = extract_date(date, self.options.date)
        year, month, day = augment_date(year, month, day, self.options.date)
        text = format_date(year, month, None, legacy=True)
        return text

    def page_bottom_left(self):
        authors = self.root.findall('front/author')
        authors = [ a for a in authors if a.get('role') != 'contributor' ]
        surnames = [ n for n in [ short_author_ascii_name_parts(a)[1] for a in authors ] if n ]
        if len(surnames) == 1:
            text = surnames[0]
        elif len(surnames) == 2:
            text = '%s & %s' % (surnames[0], surnames[1])
        elif len(surnames) > 2:
            text = '%s, et al.' % surnames[0]
        return text

    def page_bottom_center(self):
        # Either expiry date or category
        if self.options.rfc or self.root.get('ipr') == 'none':
            cat = self.root.get('category')
            text = strings.category_name[cat]
        else:
            date = get_expiry_date(self.tree, self.options.date)
            parts = date.year, date.month, date.day
            text = 'Expires %s' % format_date(*parts, legacy=self.options.legacy_date_format)
        return text

    def downcode_punctuation(self):
        self.downcode(replacements=punctuation)
        self.downcode_reference_punctuation()

    def downcode(self, replacements=unicode_replacements):
        """

        Traverses an lxml.etree and replaces unicode characters with the proper
        equivalents specified in rfc2629-xhtml.ent, resulting in no non-ascii
        characters except in elements that explicitly permit non-ascii content.

        """
        for e in self.tree.iter():
            if e.text:
                if not e.tag in unicode_content_tags:
                    try:
                        e.text.encode('ascii')
                    except UnicodeEncodeError:
                        e.text = downcode(e.text, replacements=replacements)
                elif e.tag in bare_unicode_tags:
                    pass
                elif e.tag in bare_latin_tags and is_script(e.text, 'Latin'):
                    pass
                elif not e.get('ascii'):
                    try:
                        e.text.encode('ascii')
                    except UnicodeEncodeError:
                        e.text = downcode(e.text, replacements=replacements)
            if e.tail:
                if not e.getparent().tag in unicode_content_tags:
                    try:
                        e.tail.encode('ascii')
                    except UnicodeEncodeError:
                        e.tail = downcode(e.tail, replacements=replacements)

    def downcode_attributes(self, replacements=unicode_replacements):
        for e in self.tree.iter():
            for key in e.attrib.keys():
                if not (e.tag, key) in unicode_attributes:
                    try:
                        e.get(key).encode('ascii')
                    except UnicodeEncodeError:
                        e.set(key, downcode(e.get(key), replacements=replacements))

    def downcode_reference_punctuation(self):
        for r in self.tree.xpath('.//references'):
            for e in r.iter():
                for key in e.attrib.keys():
                    try:
                        e.get(key).encode('ascii')
                    except UnicodeEncodeError:
                        e.set(key, downcode_punctuation(e.get(key)))

    def pretty_print_prep(self, e, p):
        ind = self.options.indent
        ## The actual printing is done in self.write()
        def indent(e, i):
            if e.tag in (lxml.etree.CDATA, ):
                return
            if e.tag in (lxml.etree.Comment, lxml.etree.PI, ):
                if not e.tail:
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
                return
            #
            if e.tag not in text_tags:
                if len(e) and (e.text is None or e.text.strip()==''):
                    e.text = '\n'+' '*(i+ind)
            elif e.tag in ['blockquote', 'li', 'dd', 'td', 'th' ]: # mixed content
                pass
                if len(e) and e[0] not in inline_tags and (e.text is None or e.text.strip()==''):
                    e.text = '\n'+' '*(i+ind)
            elif e.tag in ['artwork', 'sourcecode', ]:
                pass
            else:
                # inline tag
                if e.tail == '':
                    e.tail = None
                if len(e):
                    z = e[-1]
                    if z.tail and re.search(r'\n[ \t]+$', z.tail):
                        z.tail = re.sub(r'\n[ \t]+$', '\n'+' '*(i), z.tail)
                else:
                    if e.text and re.search(r'\n[ \t]+$', e.text):
                        e.text = re.sub(r'\n[ \t]+$', '\n'+' '*(i), e.text)
                    
            #
            for c in e:
                indent(c, i+ind)
            #
            if e.tag not in inline_tags:# and e.tag not in ['artwork', 'sourcecode', ]:
                if e.tail is None or e.tail.strip()=='':
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
                else:
                    self.warn(e, 'Unexpected tail: <%s>%s' % (e.tag, e.tail))
            else:
                if e.tail != None and e.tail.strip()=='' and '\n' in e.tail:
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
        indent(e, 0)
        e.tail = None

    def validate(self, when='', warn=False):
        # Note: Our schema doesn't permit xi:include elements, so the document
        # must have had XInclude processing done before calling validate()

        # The lxml Relax NG validator checks that xsd:ID values are unique,
        # but unfortunately the error messages are completely unhelpful (lxml
        # 4.1.1, libxml 2.9.1): "Element li has extra content: t" when 't' has
        # a duplicate xsd:ID attribute.  So we check all attributes with
        # content specified as xsd:ID first, and give better messages:

        # Get the attributes we need to check
        if when and not when.startswith(' '):
            when = ' '+when
        dups = find_duplicate_ids(self.schema, self.tree)
        for attr, id, e in dups:
            self.warn(e, 'Duplicate xsd:ID attribute %s="%s" found.  This will cause validation failure.' % (attr, id, ))

        try:
            self.v3_rng.assertValid(self.tree)
            return True
        except Exception as e:
            lxmlver = lxml.etree.LXML_VERSION[:3]
            if lxmlver < (3, 8, 0):
                self.warn(None, "The available version of the lxml library (%s) does not provide xpath "
                          "information as part of validation errors.  Upgrade to version 3.8.0 or "
                          "higher for better error messages." % ('.'.join(str(v) for v in lxmlver), ))
            # These warnings are occasionally incorrect -- disable this
            # output for now:
            if hasattr(e, 'error_log'):
                for error in e.error_log:
                    path = getattr(error, 'path', '')
                    msg = "%s(%s): %s: %s, at %s" % (self.xmlrfc.source, error.line, error.level_name.title(), error.message, path)
                    self.log(msg)
                    if error.message.startswith("Did not expect text"):
                        items = self.tree.xpath(error.path + '/text()')
                        for item in items:
                            if item.strip():
                                self.log("  Unexpected text:\n    %s" % item.strip())

            else:
                log.warn('\nInvalid document: %s' % (e,))
            if warn:
                self.warn(self.root, 'Invalid document%s.' % (when, ))
                return False
            else:
                self.die(self.root, 'Invalid document%s.' % (when, ))

    def validate_before(self, e, p):
        version = self.root.get('version', '3')
        if version not in ['3', ]:
            self.die(self.root, 'Expected <rfc> version="3", but found "%s"' % version)
        if not self.validate('before'):
            self.note(None, "Schema validation failed for input document")

    def validate_after(self, e, p):
        # XXX: There is an issue with exponential increase in validation time
        # as a function of the number of attributes on the root element, on
        # the order of minutes for the full set of possble attribute values.
        # See https://bugzilla.gnome.org/show_bug.cgi?id=133736 .  In our
        # schema, there is no dependency in the underlying schema on the root
        # element attributes.  In order to avoid very long validation times, we
        # strip the root attributes before validation, and put them back
        # afterwards.  
        for k in e.keys():
            if not e.get(k):
                del e.attrib[k]
        attrib = copy.deepcopy(e.attrib) 
        for k in attrib.keys():
            del e.attrib[k]
        #
        if not self.validate('after', warn=True):
            self.note(None, "Schema validation failed for input document")
        else:
            self.root.set('version', '3')
        #
        keys = list(attrib.keys())
        keys.sort()
        for k in keys:
            e.set(k, attrib[k])

    def remove(self, p, e):
        # Element.remove(child) removes both the child and its tail, so in
        # order not to loose text when removing comments or other elements,
        # we need to handle that case:
        if e.tail:
            s = e.getprevious()
            if s != None:
                s.tail = (s.tail or '') + e.tail
            else:
                p.text = (p.text or '') + e.tail
        p.remove(e)

