# Copyright 2018-2026 The Khronos Group Inc.
# Copyright (c) 2018-2019 Collabora, Ltd.
#
# SPDX-License-Identifier: Apache-2.0
#
# Author(s):    Rylie Pavlik <rylie.pavlik@collabora.com>
"""Provides MacroCheckerFile, a subclassable type that validates a single file in the spec."""

import logging
import re
from collections import OrderedDict, namedtuple
from enum import Enum
from inspect import currentframe
from typing import Set, Union

from .shared import (AUTO_FIX_STRING, CATEGORIES_WITH_VALIDITY,
                     EXTENSION_CATEGORY, NON_EXISTENT_MACROS, EntityData,
                     Message, MessageContext, MessageId, MessageType,
                     generateInclude, toNameAndLine)

# Code blocks may start and end with any number of ----
CODE_BLOCK_DELIM = '----'

# Mostly for ref page blocks, but also used elsewhere?
REF_PAGE_LIKE_BLOCK_DELIM = '--'

# For insets/blocks like the implicit valid usage
# TODO think it must start with this - does it have to be exactly this?
BOX_BLOCK_DELIM = '****'


INTERNAL_PLACEHOLDER = re.compile(
    r'(?P<delim>__+)([a-zA-Z]+)(?P=delim)'
)

# Matches any include line.
# Used to check for a leading path attribute.
INCLUDE_PATH_ATTRIBUTE = re.compile(
    r'include::(\{(?P<path_attribute>[a-z]+)\})?.*\[\]')

allowed_path_attributes = {
    '{appendices}',
    '{chapters}',
    '{config}',
    '{generated}',
    '{promoted}',
    '{style}',
}

# Matches a generated (api or validity) include line.
INCLUDE = re.compile(
    r'include::(?P<directory_traverse>((../){1,4}|\{generated\}/)(generated/)?)(?P<generated_type>(api|validity))/(?P<category>\w+)/(?P<entity_name>[^./]+).adoc[\[][\]]')  # noqa

# Matches an [[AnchorLikeThis]]
ANCHOR = re.compile(r'\[\[(?P<entity_name>[^\]]+)\]\]')

# Looks for flink:foo:: or slink:foo:: at the end of string:
# used to detect explicit pname context.
PRECEDING_MEMBER_REFERENCE = re.compile(
    r'\b(?P<macro>[fs](text|link)):(?P<entity_name>[\w*]+)::$')

# Matches something like slink:foo::pname:bar as well as
# the under-marked-up slink:foo::bar.
MEMBER_REFERENCE = re.compile(
    r'\b(?P<first_part>(?P<scope_macro>[fs](text|link)):(?P<scope>[\w*]+))(?P<double_colons>::)(?P<second_part>(?P<member_macro>pname:?)(?P<entity_name>[\w]+))\b'  # noqa
)

# Matches if a string ends while a link is still "open".
# (first half of a link being broken across two lines,
# or containing our interested area when matched against the text preceding).
# Used to skip checking in some places.
OPEN_LINK = re.compile(
    r'.*(?<!`)(<<[^>]*|xref:[^\[]*)$'
)

# Matches if a string begins and is followed by a link "close" without a matching open.
# (second half of a link being broken across two lines)
# Used to skip checking in some places.
CLOSE_LINK = re.compile(
    r'[^<]*>>.*$'
)

# Matches if a line should be skipped without further considering.
# Matches lines starting with:
# - `ifdef:`
# - `endif:`
# - `todo` (followed by something matching \b, like : or (. capitalization ignored)
# - `[#` (anchor for an image)
SKIP_LINE = re.compile(
    r'^(ifdef:)|(endif:)|([tT][oO][dD][oO]\b)|(\[#).*'
)

# Matches the whole inside of a refpage tag.
BRACKETS = re.compile(r'\[(?P<tags>.*)\]')

# Matches a key='value' pair from a ref page tag.
REF_PAGE_ATTRIB = re.compile(
    r"(?P<key>[a-z]+)='(?P<value>[^'\\]*(?:\\.[^'\\]*)*)'")

ARRAY_DIMENSION = re.compile(r"\[(?P<dim>[^\]]+)\]$")

class Attrib(Enum):
    """Attributes of a ref page."""

    REFPAGE = 'refpage'
    DESC = 'desc'
    TYPE = 'type'
    ALIAS = 'alias'
    XREFS = 'xrefs'
    ANCHOR = 'anchor'


VALID_REF_PAGE_ATTRIBS = set(
    (e.value for e in Attrib))

AttribData = namedtuple('AttribData', ['match', 'key', 'value'])


def makeAttribFromMatch(match):
    """Turn a match of REF_PAGE_ATTRIB into an AttribData value."""
    return AttribData(match=match, key=match.group(
        'key'), value=match.group('value'))


def parseRefPageAttribs(line):
    """Parse a ref page tag into a dictionary of attribute_name: AttribData."""
    return {m.group('key'): makeAttribFromMatch(m)
            for m in REF_PAGE_ATTRIB.finditer(line)}


def regenerateIncludeFromMatch(match, generated_type):
    """Create an include directive from an INCLUDE match and a (new or replacement) generated_type."""
    return generateInclude(
        match.group('directory_traverse'),
        generated_type,
        match.group('category'),
        match.group('entity_name'))


BlockEntry = namedtuple(
    'BlockEntry', ['delimiter', 'context', 'block_type', 'refpage'])


class BlockType(Enum):
    """Enumeration of the various distinct block types known."""
    CODE = 'code'
    REF_PAGE_LIKE = 'ref-page-like'  # with or without a ref page tag before
    BOX = 'box'

    @classmethod
    def lineToBlockType(self, line):
        """Return a BlockType if the given line is a block delimiter.

        Returns None otherwise.
        """
        if line == REF_PAGE_LIKE_BLOCK_DELIM:
            return BlockType.REF_PAGE_LIKE
        if line.startswith(CODE_BLOCK_DELIM):
            return BlockType.CODE
        if line.startswith(BOX_BLOCK_DELIM):
            return BlockType.BOX

        return None


def _pluralize(word, num):
    if num == 1:
        return word
    if word.endswith('y'):
        return f"{word[:-1]}ies"
    return f"{word}s"


def _s_suffix(num):
    """Simplify pluralization."""
    if num > 1:
        return 's'
    return ''


def shouldEntityBeText(entity, subscript):
    """Determine if an entity name appears to use placeholders, wildcards, etc. and thus merits use of a *text macro.

    Call with the entity and subscript groups from a match of MacroChecker.macro_re.
    """
    entity_only = entity
    if subscript:
        if subscript == '[]' or subscript == '[i]' or subscript.startswith(
                '[_') or subscript.endswith('_]'):
            return True
        entity_only = entity[:-len(subscript)]

    if ('*' in entity) or entity.startswith('_') or entity_only.endswith('_'):
        return True

    if INTERNAL_PLACEHOLDER.search(entity):
        return True
    return False


class MacroCheckerFile:
    """Object performing processing of a single AsciiDoctor file from a specification.

    For testing purposes, may also process a string as if it were a file.
    """

    def __init__(self, checker, filename: str, enabled_messages: Set[MessageId], stream_maker):
        """Construct a MacroCheckerFile object.

        Typically called by MacroChecker.processFile or MacroChecker.processString().

        Arguments:
        checker -- A MacroChecker object.
        filename -- A string to use in messages to refer to this checker, typically the file name.
        enabled_messages -- A set() of MessageId values that should be considered "enabled" and thus stored.
        stream_maker -- An object with a makeStream() method that returns a stream.
        """
        self.checker = checker
        self.filename = filename
        self.stream_maker = stream_maker
        self.enabled_messages = enabled_messages
        self.missing_validity_suppressions = set(
            self.getMissingValiditySuppressions())

        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(logging.NullHandler())

        self.fixes = set()
        self.messages = []

        self.pname_data = None
        self.pname_mentions = {}

        self.refpage_includes = {}

        self.lines = []

        # For both of these:
        # keys: entity name
        # values: MessageContext
        self.fs_api_includes = {}
        self.validity_includes = {}

        self.in_code_block = False
        self.in_ref_page = False
        self.prev_line_ref_page_tag = None
        self.current_ref_page: Union[None, str, EntityData] = None

        # Stack of block-starting delimiters.
        self.block_stack = []

        # Regexes that are members because they depend on the name prefix.
        self.suspected_missing_macro_re = self.checker.suspected_missing_macro_re
        self.heading_command_re = self.checker.heading_command_re

    ###
    # Main process/checking methods, arranged roughly from largest scope to smallest scope.
    ###

    def process(self):
        """Check the stream (file, string) created by the streammaker supplied to the constructor.

        This is the top-level method for checking a spec file.
        """
        self.logger.info("processing file %s", self.filename)

        # File content checks - performed line-by-line
        with self.stream_maker.make_stream() as f:
            # Iterate through lines, calling processLine on each.
            for lineIndex, line in enumerate(f):
                trimmedLine = line.rstrip()
                self.lines.append(trimmedLine)
                self.processLine(lineIndex + 1, trimmedLine)

        # End of file checks follow:

        # Check "state" at end of file: should have blocks closed.
        if self.prev_line_ref_page_tag:
            self.error(MessageId.REFPAGE_BLOCK,
                       "Reference page tag seen, but block not opened before end of file.",
                       context=self.storeMessageContext(match=None))

        if self.block_stack:
            locations = (x.context for x in self.block_stack)
            formatted_locations = [f'{x.delimiter} opened at {self.getBriefLocation(x.context)}'
                                   for x in self.block_stack]
            self.logger.warning("Unclosed blocks: %s",
                                ', '.join(formatted_locations))

            self.error(MessageId.UNCLOSED_BLOCK,
                       ["Reached end of page, with these unclosed blocks remaining:"] +
                       formatted_locations,
                       context=self.storeMessageContext(match=None),
                       see_also=locations)

        # Check that every include of an /api/ file in the protos or structs category
        # had a matching /validity/ include
        for entity, includeContext in self.fs_api_includes.items():
            if not self.checker.entity_db.entityHasValidity(entity):
                continue

            if entity in self.missing_validity_suppressions:
                continue

            if entity not in self.validity_includes:
                self.warning(MessageId.MISSING_VALIDITY_INCLUDE,
                             [f'Saw /api/ include for {entity}, but no matching /validity/ include',
                              f"Expected a line with {regenerateIncludeFromMatch(includeContext.match, 'validity')}"],
                             context=includeContext)

        # Check that we never include a /validity/ file
        # without a matching /api/ include
        for entity, includeContext in self.validity_includes.items():
            if entity not in self.fs_api_includes:
                self.error(MessageId.MISSING_API_INCLUDE,
                           [f'Saw /validity/ include for {entity}, but no matching /api/ include',
                            f"Expected a line with {regenerateIncludeFromMatch(includeContext.match, 'api')}"],
                           context=includeContext)

        if not self.numDiagnostics():
            # no problems, exit quietly
            return

        print(f'\nFor file {self.filename}:')

        self.printMessageCounts()
        numFixes = len(self.fixes)
        if numFixes > 0:
            fixes = ', '.join((f'{search} -> {replace}'
                               for search, replace in self.fixes))

            print(f"{numFixes} unique auto-fix {_pluralize('pattern', numFixes)} recorded: {fixes}")

    def processLine(self, lineNum, line):
        """Check the contents of a single line from a file.

        Eventually populates self.match, self.entity, self.macro,
        before calling processMatch.
        """
        self.lineNum = lineNum
        self.line = line
        self.match = None
        self.entity = None
        self.macro = None

        self.logger.debug("processing line %d", lineNum)

        if self.processPossibleBlockDelimiter():
            # This is a block delimiter - proceed to next line.
            # Block-type-specific stuff goes in processBlockOpen and processBlockClosed.
            return

        if self.in_code_block:
            # We do no processing in a code block.
            return

        ###
        # Detect if the previous line was [open,...] starting a refpage
        # but this line isn't --
        # If the line is some other block delimiter,
        # the related code in self.processPossibleBlockDelimiter()
        # would have handled it.
        # (because execution would never get to here for that line)
        if self.prev_line_ref_page_tag:
            self.handleExpectedRefpageBlock()

        ###
        # Detect headings
        if line.startswith('=='):
            # Headings cause us to clear our pname_context
            self.pname_data = None

            command = self.heading_command_re.match(line)
            if command:
                data = self.checker.findEntity(command)
                if data:
                    self.pname_data = data
            return

        ###
        # Detect [open, lines for manpages
        if line.startswith('[open,'):
            self.checkRefPage()
            return

        ###
        # Skip comments
        if line.lstrip().startswith('//'):
            return

        ###
        # Skip ifdef/endif
        if SKIP_LINE.match(line):
            return

        ###
        # Detect any include:: lines
        match = INCLUDE_PATH_ATTRIBUTE.match(line)
        if match:
            path_attribute = match.group(1)
            if path_attribute is None:
                self.error(MessageId.MISSING_INCLUDE_PATH_ATTRIBUTE,
                           '(no path attribute is present)')
                return
            if path_attribute not in allowed_path_attributes:
                self.error(MessageId.MISSING_INCLUDE_PATH_ATTRIBUTE,
                           f'(path attribute {{path_attribute}} is not one of {sorted(allowed_path_attributes)})')
                return

        ###
        # Detect include:::....[] lines for generated API fragments
        match = INCLUDE.match(line)
        if match:
            self.match = match
            entity = match.group('entity_name')

            data = self.checker.findEntity(entity)
            if not data:
                self.error(MessageId.UNKNOWN_INCLUDE,
                           f'Saw include for {entity}, but that entity is unknown.')
                self.pname_data = None
                return

            self.pname_data = data

            if match.group('generated_type') == 'api':
                self.recordInclude(self.checker.apiIncludes)

                # Set mentions to None. The first time we see something like `* pname:paramHere`,
                # we will set it to an empty set
                self.pname_mentions[entity] = None

                if match.group('category') in CATEGORIES_WITH_VALIDITY:
                    self.fs_api_includes[entity] = self.storeMessageContext()

                if entity in self.validity_includes:
                    name_and_line = toNameAndLine(
                        self.validity_includes[entity], root_path=self.checker.root_path)
                    self.error(MessageId.API_VALIDITY_ORDER,
                               [f'/api/ include found for {entity} after a corresponding /validity/ include',
                                f'Validity include located at {name_and_line}'])

            elif match.group('generated_type') == 'validity':
                self.recordInclude(self.checker.validityIncludes)
                self.validity_includes[entity] = self.storeMessageContext()

                if entity not in self.pname_mentions:
                    self.error(MessageId.API_VALIDITY_ORDER,
                               f'/validity/ include found for {entity} without a preceding /api/ include')
                    return

                if self.pname_mentions[entity]:
                    # Got a validity include and we have seen at least one * pname: line
                    # since we got the API include
                    # so we can warn if we haven't seen a reference to every
                    # parameter/member.
                    members = self.checker.getMemberNames(entity)
                    missing = [member for member in members
                               if member not in self.pname_mentions[entity]]
                    if missing:
                        self.error(MessageId.UNDOCUMENTED_MEMBER,
                                   [f'Validity include found for {entity}, but not all members/params apparently documented',
                                    f"Members/params not mentioned with pname: {', '.join(missing)}"])

            # If we found an include line, we're done with this line.
            return

        if self.pname_data is not None and '* pname:' in line:
            context_entity = self.pname_data.entity
            if self.pname_mentions[context_entity] is None:
                # First time seeing * pname: after an api include, prepare the set that
                # tracks
                self.pname_mentions[context_entity] = set()

        ###
        # Detect [[Entity]] anchors
        for match in ANCHOR.finditer(line):
            entity = match.group('entity_name')
            if self.checker.findEntity(entity):
                # We found an anchor with the same name as an entity:
                # treat it (mostly) like an API include
                self.match = match
                self.recordInclude(self.checker.apiIncludes,
                                   generated_type='api (manual anchor)')

        ###
        # Detect :: without pname
        for match in MEMBER_REFERENCE.finditer(line):
            if not match.group('member_macro'):
                self.match = match
                # Got :: but not followed by pname

                search = match.group()
                replacement = f"{match.group('first_part')}::pname:{match.group('second_part')}"
                self.error(MessageId.MEMBER_PNAME_MISSING,
                           'Found a function parameter or struct member reference with :: but missing pname:',
                           group='double_colons',
                           replacement='::pname:',
                           fix=(search, replacement))

                # check pname here because it won't come up in normal iteration below
                # because of the missing macro
                self.entity = match.group('entity_name')
                self.checkPname(match.group('scope'), fully_qualified=True)

        ###
        # Look for things that seem like a missing macro.
        for match in self.suspected_missing_macro_re.finditer(line):
            if OPEN_LINK.match(line, endpos=match.start()):
                # this is in a link, skip it.
                continue
            if CLOSE_LINK.match(line[match.end():]):
                # this is in a link, skip it.
                continue

            entity = match.group('entity_name')
            self.match = match
            self.entity = entity
            data = self.checker.findEntity(entity)
            if data:

                if data.category == EXTENSION_CATEGORY:
                    # Ah, this is an extension
                    self.warning(MessageId.EXTENSION, "Seems like this is an extension name that was not linked.",
                                 group='entity_name', replacement=self.makeExtensionLink())
                else:
                    macro = data.macro
                    category = data.category
                    self.warning(MessageId.MISSING_MACRO,
                                 [f'Seems like a "{macro}" macro was omitted for this reference to a known entity in category "{category}".',
                                  'Wrap in ` ` to silence this if you do not want a verified macro here.'],
                                 group='entity_name',
                                 replacement=self.makeMacroMarkup(data.macro))
            else:

                dataArray = self.checker.findEntityCaseInsensitive(entity)
                # We might have found the goof...

                if dataArray:
                    if len(dataArray) == 1:
                        # Yep, found the goof:
                        # incorrect macro and entity capitalization
                        data = dataArray[0]
                        if data.category == EXTENSION_CATEGORY:
                            # Ah, this is an extension
                            self.warning(MessageId.EXTENSION,
                                         "Seems like this is an extension name that was not linked.",
                                         group='entity_name', replacement=self.makeExtensionLink(data.entity))
                        else:
                            self.warning(MessageId.MISSING_MACRO,
                                         'Seems like a macro was omitted for this reference to a known entity in category "{}"'
                                         ', found by searching case-insensitively.'.format(
                                             data.category),
                                         replacement=self.makeMacroMarkup(data=data))

                    else:
                        # Ugh, more than one resolution

                        self.warning(MessageId.MISSING_MACRO,
                                     ['Seems like a macro was omitted for this reference to a known entity, found by searching case-insensitively.',
                                      'More than one apparent match.'],
                                     group='entity_name', see_also=dataArray[:])

        ###
        # Main operations: detect markup macros
        for match in self.checker.macro_re.finditer(line):
            self.match = match
            self.macro = match.group('macro')
            self.entity = match.group('entity_name')
            self.subscript = match.group('subscript')
            self.processMatch()

    def processPossibleBlockDelimiter(self):
        """Look at the current line, and if it's a delimiter, update the block stack.

        Calls self.processBlockDelimiter() as required.

        Returns True if a delimiter was processed, False otherwise.
        """
        line = self.line
        new_block_type = BlockType.lineToBlockType(line)
        if not new_block_type:
            return False

        ###
        # Detect if the previous line was [open,...] starting a refpage
        # but this line is some block delimiter other than --
        # Must do this here because if we get a different block open instead of the one we want,
        # the order of block opening will be wrong.
        if new_block_type != BlockType.REF_PAGE_LIKE and self.prev_line_ref_page_tag:
            self.handleExpectedRefpageBlock()

        # Delegate to the main process for delimiters.
        self.processBlockDelimiter(line, new_block_type)

        return True

    def processBlockDelimiter(self, line, new_block_type, context=None):
        """Update the block stack based on the current or supplied line.

        Calls self.processBlockOpen() or self.processBlockClosed() as required.

        Called by self.processPossibleBlockDelimiter() both in normal operation, as well as
        when "faking" a ref page block open.

        Returns BlockProcessResult.
        """
        if not context:
            context = self.storeMessageContext()

        location = self.getBriefLocation(context)

        top = self.getInnermostBlockEntry()
        top_delim = self.getInnermostBlockDelimiter()
        if top_delim == line:
            self.processBlockClosed()
            return

        if top and top.block_type == new_block_type:
            # Same block type, but not matching - might be an error?
            # TODO maybe create a diagnostic here?
            self.logger.warning(
                "processPossibleBlockDelimiter: %s: Matched delimiter type %s, but did not exactly match current delim %s to top of stack %s, may be a typo?",
                location, new_block_type, line, top_delim)

        # Empty stack, or top doesn't match us.
        self.processBlockOpen(new_block_type, delimiter=line)

    def processBlockOpen(self, block_type, context=None, delimiter=None):
        """Do any block-type-specific processing and push the new block.

        Must call self.pushBlock().
        May be overridden (carefully) or extended.

        Called by self.processBlockDelimiter().
        """
        if block_type == BlockType.REF_PAGE_LIKE:
            if self.prev_line_ref_page_tag:
                if self.current_ref_page:
                    refpage = self.current_ref_page
                else:
                    refpage = '?refpage-with-invalid-tag?'

                self.logger.info(
                    'processBlockOpen: Opening refpage for %s', refpage)
                # Opening of refpage block "consumes" the preceding ref
                # page context
                self.prev_line_ref_page_tag = None
                self.pushBlock(block_type, refpage=refpage,
                               context=context, delimiter=delimiter)
                self.in_ref_page = True
                return

        if block_type == BlockType.CODE:
            self.in_code_block = True

        self.pushBlock(block_type, context=context, delimiter=delimiter)

    def processBlockClosed(self):
        """Do any block-type-specific processing and pop the top block.

        Must call self.popBlock().
        May be overridden (carefully) or extended.

        Called by self.processPossibleBlockDelimiter().
        """
        old_top = self.popBlock()

        if old_top.block_type == BlockType.CODE:
            self.in_code_block = False

        elif old_top.block_type == BlockType.REF_PAGE_LIKE and old_top.refpage:
            self.logger.info(
                'processBlockClosed: Closing refpage for %s', old_top.refpage)
            # leaving a ref page so reset associated state.
            self.current_ref_page = None
            self.prev_line_ref_page_tag = None
            self.in_ref_page = False

    def processMatch(self):
        """Process a match of the macro:entity regex for correctness."""
        match = self.match
        entity = self.entity
        macro = self.macro

        ###
        # Track entities that we're actually linking to.
        ###
        if self.checker.entity_db.isLinkedMacro(macro):
            self.checker.addLinkToEntity(entity, self.storeMessageContext())

        ###
        # Link everything that should be, and nothing that shouldn't be
        ###
        if self.checkRecognizedEntity():
            # if this returns true,
            # then there is no need to do the remaining checks on this match
            return

        ###
        # Non-existent macros
        if macro in NON_EXISTENT_MACROS:
            self.error(MessageId.BAD_MACRO, '{} is not a macro provided in the specification, despite resembling other macros.'.format(
                macro), group='macro')

        ###
        # Wildcards (or leading underscore, or square brackets)
        # if and only if a 'text' macro
        self.checkText()

        # Do some validation of pname references.
        if macro == 'pname':
            # there is only a macro if there is a match
            assert match
            # See if there's an immediately-preceding entity
            preceding = self.line[:match.start()]
            scope = PRECEDING_MEMBER_REFERENCE.search(preceding)
            if scope:
                # Yes there is, check it out.
                self.checkPname(scope.group('entity_name'), fully_qualified=True)
            elif self.current_ref_page is not None and not isinstance(self.current_ref_page, str):
                # No, but there is a current ref page: very reliable
                self.checkPnameImpliedContext(self.current_ref_page)
            elif self.pname_data is not None:
                # No, but there is a pname_context - better than nothing.
                self.checkPnameImpliedContext(self.pname_data)
            else:
                # no, and no existing context we can imply:
                # can't check this.
                pass

    def shouldSkipUnrecognizedEntity(self, macro,  entity_name):
        """Return True if we should not warn about not recognizing a macro invocation for entity_name."""
        return False

    def checkRecognizedEntity(self):
        """Check the current macro:entity match to see if it is recognized.

        Returns True if there is no need to perform further checks on this match.

        Helps avoid duplicate warnings/errors: typically each macro should have at most
        one of this class of errors.
        """
        entity = self.entity
        macro = self.macro
        assert macro
        if self.checker.findMacroAndEntity(macro, entity) is not None:
            # We know this macro-entity combo
            return True

        # We don't know this macro-entity combo.
        possibleCats = self.checker.entity_db.getCategoriesForMacro(macro)
        if possibleCats is None:
            possibleCats = ['???']
        msg = ['Definition of link target {} with macro {} (used for {} {}) does not exist.'.format(
            entity,
            macro,
            _pluralize('category', len(possibleCats)),
            ', '.join(possibleCats))]

        if self.shouldSkipUnrecognizedEntity(macro, entity):
            return False

        data = self.checker.findEntity(entity)
        if data:
            # We found the goof: incorrect macro
            msg.append(f'Apparently matching entity in category {data.category} found.')
            self.handleWrongMacro(msg, data)
            return True

        see_also = []
        dataArray = self.checker.findEntityCaseInsensitive(entity)
        if dataArray:
            # We might have found the goof...

            if len(dataArray) == 1:
                # Yep, found the goof:
                # incorrect macro and entity capitalization
                data = dataArray[0]
                msg.append('Apparently matching entity in category {} found by searching case-insensitively.'.format(
                    data.category))
                self.handleWrongMacro(msg, data)
                return True
            else:
                # Ugh, more than one resolution
                msg.append(
                    'More than one apparent match found by searching case-insensitively, cannot auto-fix.')
                see_also = dataArray[:]

        # OK, so we don't recognize this entity (and couldn't auto-fix it).

        if self.checker.entity_db.shouldBeRecognized(macro, entity):
            # We should know the target - it's a link macro,
            # or there's some reason the entity DB thinks we should know it.
            if self.checker.likelyRecognizedEntity(entity):
                # Should be linked and it matches our pattern,
                # so probably not wrong macro.
                # Human brains required.
                if not self.checkText():
                    self.error(MessageId.BAD_ENTITY, msg + ['Might be a misspelling, or, less likely, the wrong macro.'],
                               see_also=see_also)
            else:
                # Doesn't match our pattern,
                # so probably should be name instead of link.
                newMacro = f"{macro[0]}name"
                if self.checker.entity_db.isValidMacro(newMacro):
                    self.error(MessageId.BAD_ENTITY, msg +
                               ['Entity name does not fit the pattern for this API, which would mean it should be a "name" macro instead of a "link" macro'],
                               group='macro', replacement=newMacro, fix=self.makeFix(newMacro=newMacro), see_also=see_also)
                else:
                    self.error(MessageId.BAD_ENTITY, msg +
                               ['Entity name does not fit the pattern for this API, which would mean it should be a "name" macro instead of a "link" macro',
                                f'However, {newMacro} is not a known macro so cannot auto-fix.'], see_also=see_also)

        elif macro == 'ename':
            # TODO This might be an ambiguity in the style guide - ename might be a known enumerant value,
            # or it might be an enumerant value in an external library, etc. that we don't know about - so
            # hard to check this.
            if self.checker.likelyRecognizedEntity(entity):
                if not self.checkText():
                    self.warning(MessageId.BAD_ENUMERANT, msg +
                                 [f'Unrecognized ename:{entity} that we would expect to recognize since it fits the pattern for this API.'],
                                 see_also=see_also)
        else:
            # This is fine:
            # it doesn't need to be recognized since it's not linked.
            pass
        # Don't skip other tests.
        return False

    def checkText(self):
        """Evaluate the usage (or non-usage) of a *text macro.

        Wildcards (or leading or trailing underscore, or square brackets with
        nothing or a placeholder) if and only if a 'text' macro.

        Called by checkRecognizedEntity() when appropriate.
        """
        macro = self.macro
        entity = self.entity
        shouldBeText = shouldEntityBeText(entity, self.subscript)
        assert macro
        if shouldBeText and not macro.endswith('text') and not macro == 'code':
            newMacro = f"{macro[0]}text"
            if self.checker.entity_db.getCategoriesForMacro(newMacro):
                self.error(MessageId.MISSING_TEXT,
                           [f'Asterisk/leading or trailing underscore/bracket found - macro should end with "text:", probably {newMacro}:',
                            AUTO_FIX_STRING],
                           group='macro', replacement=newMacro, fix=self.makeFix(newMacro=newMacro))
            else:
                self.error(MessageId.MISSING_TEXT,
                           ['Asterisk/leading or trailing underscore/bracket found, so macro should end with "text:".',
                            f'However {newMacro}: is not a known macro so cannot auto-fix.'],
                           group='macro')
            return True
        elif macro.endswith('text') and not shouldBeText:
            msg = [
                f"No asterisk/leading or trailing underscore/bracket in the entity, so this might be a mistaken use of the 'text' macro {macro}:"]
            data = self.checker.findEntity(entity)
            if data:
                if self.shouldSkipUnrecognizedEntity(macro, entity):
                    return False

                # We found the goof: incorrect macro
                msg.append(f'Apparently matching entity in category {data.category} found.')
                msg.append(AUTO_FIX_STRING)
                replacement = self.makeFix(data=data)
                if data.category == EXTENSION_CATEGORY:
                    self.error(MessageId.EXTENSION, msg,
                               replacement=replacement, fix=replacement)
                else:
                    self.error(MessageId.WRONG_MACRO, msg,
                               group='macro', replacement=data.macro, fix=replacement)
            else:
                if self.checker.likelyRecognizedEntity(entity):
                    # This is a use of *text: for something that fits the pattern but isn't in the spec.
                    # This is OK.
                    return False
                msg.append('Entity not found in spec, either.')
                if macro[0] != 'e':
                    # Only suggest a macro if we aren't in elink/ename/etext,
                    # since ename and elink are not related in an equivalent way
                    # to the relationship between flink and fname.
                    newMacro = f"{macro[0]}name"
                    if self.checker.entity_db.getCategoriesForMacro(newMacro):
                        msg.append(
                            f'Consider if {newMacro}: might be the correct macro to use here.')
                    else:
                        msg.append(
                            f'Cannot suggest a new macro because {newMacro}: is not a known macro.')
                self.warning(MessageId.MISUSED_TEXT, msg)
            return True
        return False

    def checkPnameImpliedContext(self, pname_context):
        """Handle pname: macros not immediately preceded by something like flink:entity or slink:entity.

        Also records pname: mentions of members/parameters for completeness checking in doc blocks.

        Contains call to self.checkPname().
        Called by self.processMatch()
        """
        self.checkPname(pname_context.entity)
        if pname_context.entity in self.pname_mentions and \
                self.pname_mentions[pname_context.entity] is not None:
            # Record this mention,
            # in case we're in the documentation block.
            self.pname_mentions[pname_context.entity].add(self.entity)

    def checkPname(self, pname_context, fully_qualified=False):
        """Check the current match (as a pname: usage) with the given entity as its 'pname context', if possible.

        e.g. slink:foo::pname:bar, pname_context would be 'foo', while self.entity would be 'bar', etc.

        Called by self.processLine(), self.processMatch(), as well as from self.checkPnameImpliedContext().
        """
        if '*' in pname_context:
            # This context has a placeholder, can't verify it.
            return

        entity = self.entity

        context_data = self.checker.findEntity(pname_context)
        members = self.checker.getMemberNames(pname_context)

        if context_data and not members:
            # This is a recognized parent entity that doesn't have detectable member names,
            # skip validation
            # TODO: Annotate parameters of function pointer types with <name>
            # and <param>?
            return
        if not members:
            self.warning(MessageId.UNRECOGNIZED_CONTEXT,
                         f'pname context entity was un-recognized {pname_context}')
            return

        assert entity

        # Strip trailing array dimension
        if ARRAY_DIMENSION.search(entity):
            entity = ARRAY_DIMENSION.sub("", entity)

        if entity not in members:
            message_id = MessageId.UNKNOWN_MEMBER_FULLY_QUALIFIED if fully_qualified else MessageId.UNKNOWN_MEMBER
            self.warning(message_id, ["Could not find member/param named '{}' in {}".format(entity, pname_context),
                                                    'Known {} member/param names are: {}'.format(
                pname_context, ', '.join(members))], group='entity_name')

    def checkIncludeRefPageRelation(self, entity, generated_type):
        """Identify if our current ref page (or lack thereof) is appropriate for an include just recorded.

        Called by self.recordInclude().
        """
        if not self.in_ref_page:
            # Not in a ref page block: This probably means this entity needs a
            # ref-page block added.
            self.handleIncludeMissingRefPage(entity, generated_type)
            return

        if not isinstance(self.current_ref_page, EntityData):
            # This isn't a fully-valid ref page, so can't check the includes any better.
            return

        ref_page_entity = self.current_ref_page.entity
        if ref_page_entity not in self.refpage_includes:
            self.refpage_includes[ref_page_entity] = set()
        expected_ref_page_entity = self.computeExpectedRefPageFromInclude(
            entity)
        self.refpage_includes[ref_page_entity].add((generated_type, entity))

        if ref_page_entity == expected_ref_page_entity:
            # OK, this is a total match.
            pass
        elif self.checker.entity_db.areAliases(expected_ref_page_entity, ref_page_entity):
            # This appears to be a promoted synonym which is OK.
            pass
        else:
            # OK, we are in a ref page block that doesn't match
            self.handleIncludeMismatchRefPage(entity, generated_type)

    def perform_entity_check(self, type):
        """Returns True if an entity check should be performed on this
           refpage type.

           May override."""

        return True

    def checkRefPage(self):
        """Check if the current line (a refpage tag) meets requirements.

        Called by self.processLine().
        """
        line = self.line

        # Should always be found
        self.match = BRACKETS.match(line)

        data = None
        directory = None
        if self.in_ref_page:
            msg = ["Found reference page markup, but we are already in a refpage block.",
                   "The block before the first message of this type is most likely not closed.", ]
            # Fake-close the previous ref page, if it's trivial to do so.
            innermost_block = self.getInnermostBlockEntry()
            # self.in_ref_page is true only when the innermost block is something
            assert innermost_block
            if innermost_block.block_type == BlockType.REF_PAGE_LIKE:
                msg.append(
                    "Pretending that there was a line with `--` immediately above to close that ref page, for more readable messages.")
                self.processBlockDelimiter(
                    REF_PAGE_LIKE_BLOCK_DELIM, BlockType.REF_PAGE_LIKE)
            else:
                msg.append(
                    "Ref page wasn't the last block opened, so not pretending to auto-close it for more readable messages.")

            self.error(MessageId.REFPAGE_BLOCK, msg)

        attribs = parseRefPageAttribs(line)

        unknown_attribs = set(attribs.keys()).difference(
            VALID_REF_PAGE_ATTRIBS)
        if unknown_attribs:
            self.error(MessageId.REFPAGE_UNKNOWN_ATTRIB,
                       f"Found unknown attrib(s) in reference page markup: {','.join(unknown_attribs)}")

        # Required field: refpage='xrValidEntityHere'
        if Attrib.REFPAGE.value in attribs:
            attrib = attribs[Attrib.REFPAGE.value]
            text = attrib.value
            self.entity = text

            context = self.storeMessageContext(
                group='value', match=attrib.match)
            if self.checker.seenRefPage(text):
                self.error(MessageId.REFPAGE_DUPLICATE,
                           [f"Found reference page markup when we already saw refpage='{text}' elsewhere.",
                            "This (or the other mention) may be a copy-paste error."],
                           context=context)
            self.checker.addRefPage(text)

            # Entity check can be skipped depending on the refpage type
            # Determine page type for use in several places
            type_text = ''
            if Attrib.TYPE.value in attribs:
                type_text = attribs[Attrib.TYPE.value].value

            if self.perform_entity_check(type_text):
                data = self.checker.findEntity(text)
                if data:
                    # OK, this is a known entity that we're seeing a refpage for.
                    directory = data.directory
                    self.current_ref_page = data
                else:
                    # TODO suggest fixes here if applicable
                    self.error(MessageId.REFPAGE_NAME,
                               ["Found reference page markup, but refpage='{}' type='{}' does not refer to a recognized entity".format(
                                   text, type_text),
                                'If this is intentional, add the entity to EXTRA_DEFINES or EXTRA_REFPAGES in check_spec_links.py.'],
                               context=context)
        else:
            self.error(MessageId.REFPAGE_TAG,
                       "Found apparent reference page markup, but missing refpage='...'",
                       group=None)

        # Required field: desc='preferably non-empty'
        if Attrib.DESC.value in attribs:
            attrib = attribs[Attrib.DESC.value]
            text = attrib.value
            if not text:
                context = self.storeMessageContext(
                    group=None, match=attrib.match)
                self.warning(MessageId.REFPAGE_MISSING_DESC,
                             "Found reference page markup, but desc='' is empty",
                             context=context)
        else:
            self.error(MessageId.REFPAGE_TAG,
                       "Found apparent reference page markup, but missing desc='...'",
                       group=None)

        # Required field: type='protos' for example
        # (used by genRef.py to compute the macro to use)
        if Attrib.TYPE.value in attribs:
            attrib = attribs[Attrib.TYPE.value]
            text = attrib.value
            if directory and not text == directory:
                context = self.storeMessageContext(
                    group='value', match=attrib.match)
                self.error(MessageId.REFPAGE_TYPE,
                           f"Found reference page markup, but type='{text}' is not the expected value '{directory}'",
                           context=context)
        else:
            self.error(MessageId.REFPAGE_TAG,
                       "Found apparent reference page markup, but missing type='...'",
                       group=None)

        # Optional field: alias='spaceDelimited validEntities'
        # Currently does nothing. Could modify checkRefPageXrefs to also
        # check alias= attribute value
        # if Attrib.ALIAS.value in attribs:
        #    # This field is optional
        #    self.checkRefPageXrefs(attribs[Attrib.XREFS.value])

        # Optional field: xrefs='spaceDelimited validEntities'
        if Attrib.XREFS.value in attribs:
            # This field is optional
            self.checkRefPageXrefs(attribs[Attrib.XREFS.value])
        self.prev_line_ref_page_tag = self.storeMessageContext()

    def checkRefPageXrefs(self, xrefs_attrib):
        """Check all cross-refs indicated in an xrefs attribute for a ref page.

        Called by self.checkRefPage().

        Argument:
        xrefs_attrib -- A match of REF_PAGE_ATTRIB where the group 'key' is 'xrefs'.
        """
        text = xrefs_attrib.value
        context = self.storeMessageContext(
            group='value', match=xrefs_attrib.match)

        def splitRefs(s):
            """Split the string on whitespace, into individual references."""
            return s.split()  # [x for x in s.split() if x]

        def remakeRefs(refs):
            """Re-create a xrefs string from something list-shaped."""
            return ' '.join(refs)

        refs = splitRefs(text)

        # Pre-checking if messages are enabled, so that we can correctly determine
        # the current string following any auto-fixes:
        # the fixes for messages directly in this method would interact,
        # and thus must be in the order specified here.

        if self.messageEnabled(MessageId.REFPAGE_XREFS_COMMA) and ',' in text:
            old_text = text
            # Re-split after replacing commas.
            refs = splitRefs(text.replace(',', ' '))
            # Re-create the space-delimited text.
            text = remakeRefs(refs)
            self.error(MessageId.REFPAGE_XREFS_COMMA,
                       "Found reference page markup, with an unexpected comma in the (space-delimited) xrefs attribute",
                       context=context,
                       replacement=text,
                       fix=(old_text, text))

        # We could conditionally perform this creation, but the code complexity would increase substantially,
        # for presumably minimal runtime improvement.
        unique_refs = OrderedDict.fromkeys(refs)
        if self.messageEnabled(MessageId.REFPAGE_XREF_DUPE) and len(unique_refs) != len(refs):
            # TODO is it safe to auto-fix here?
            old_text = text
            text = remakeRefs(unique_refs.keys())
            self.warning(MessageId.REFPAGE_XREF_DUPE,
                         [f"Reference page for {self.entity} contains at least one duplicate in its cross-references.",
                             "Look carefully to see if this is a copy and paste error and should be changed to a different but related entity:",
                             "auto-fix simply removes the duplicate."],
                         context=context,
                         replacement=text,
                         fix=(old_text, text))

        if self.messageEnabled(MessageId.REFPAGE_SELF_XREF) and self.entity and self.entity in unique_refs:
            # Not modifying unique_refs here because that would accidentally affect the whitespace auto-fix.
            new_text = remakeRefs(
                [x for x in unique_refs.keys() if x != self.entity])

            # DON'T AUTOFIX HERE because these are likely copy-paste between related entities:
            # e.g. a Create function and the associated CreateInfo struct.
            self.warning(MessageId.REFPAGE_SELF_XREF,
                         [f"Reference page for {self.entity} included itself in its cross-references.",
                          "This is typically a copy and paste error, and the dupe should likely be changed to a different but related entity.",
                          "Not auto-fixing for this reason."],
                         context=context,
                         replacement=new_text,)

        # We didn't have another reason to replace the whole attribute value,
        # so let's make sure it doesn't have any extra spaces
        if self.messageEnabled(MessageId.REFPAGE_WHITESPACE) and xrefs_attrib.value == text:
            old_text = text
            text = remakeRefs(unique_refs.keys())
            if old_text != text:
                self.warning(MessageId.REFPAGE_WHITESPACE,
                             [f"Cross-references for reference page for {self.entity} had non-minimal whitespace,",
                              "and no other enabled message has re-constructed this value already."],
                             context=context,
                             replacement=text,
                             fix=(old_text, text))

        for entity in unique_refs.keys():
            self.checkRefPageXref(entity, context)

    @property
    def allowEnumXrefs(self):
        """Returns True if enums can be specified in the 'xrefs' attribute
        of a refpage.

        May override.
        """
        return False

    def checkRefPageXref(self, referenced_entity, line_context):
        """Check a single cross-reference entry for a refpage.

        Called by self.checkRefPageXrefs().

        Arguments:
        referenced_entity -- The individual entity under consideration from the xrefs='...' string.
        line_context -- A MessageContext referring to the entire line.
        """
        data = self.checker.findEntity(referenced_entity)
        context = line_context
        match = re.search(r'\b{}\b'.format(referenced_entity), self.line)
        if match:
            context = self.storeMessageContext(
                group=None, match=match)

        if data and data.category == "enumvalues" and not self.allowEnumXrefs:
            msg = ["Found reference page markup, with an enum value listed: {}".format(
                referenced_entity)]
            self.error(MessageId.REFPAGE_XREFS,
                       msg,
                       context=context)
            return

        if data:
            # This is OK: we found it, and it's not an enum value
            return

        msg = ["Found reference page markup, with an unrecognized entity listed: {}".format(
            referenced_entity)]

        see_also = None
        dataArray = self.checker.findEntityCaseInsensitive(
            referenced_entity)

        if dataArray:
            # We might have found the goof...

            if len(dataArray) == 1:
                # Yep, found the goof - incorrect entity capitalization
                data = dataArray[0]
                new_entity = data.entity
                self.error(MessageId.REFPAGE_XREFS, msg + [
                    f'Apparently matching entity in category {data.category} found by searching case-insensitively.',
                    AUTO_FIX_STRING],
                    replacement=new_entity,
                    fix=(referenced_entity, new_entity),
                    context=context)
                return

            # Ugh, more than one resolution
            msg.append(
                'More than one apparent match found by searching case-insensitively, cannot auto-fix.')
            see_also = dataArray[:]
        else:
            # Probably not just a typo
            msg.append(
                'If this is intentional, add the entity to EXTRA_DEFINES or EXTRA_REFPAGES in check_spec_links.py.')

        # Multiple or no resolutions found
        self.error(MessageId.REFPAGE_XREFS,
                   msg,
                   see_also=see_also,
                   context=context)

    ###
    # Message-related methods.
    ###

    def warning(self, message_id, messageLines, context=None, group=None,
                replacement=None, fix=None, see_also=None, frame=None):
        """Log a warning for the file, if the message ID is enabled.

        Wrapper around self.diag() that automatically sets severity as well as frame.

        Arguments:
        message_id -- A MessageId value.
        messageLines -- A string or list of strings containing a human-readable error description.

        Optional, named arguments:
        context -- A MessageContext. If None, will be constructed from self.match and group.
        group -- The name of the regex group in self.match that contains the problem. Only used if context is None.
          If needed and is None, self.group is used instead.
        replacement -- The string, if any, that should be suggested as a replacement for the group in question.
          Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough
          (or can't easily phrase a regex) to do it automatically.
        fix -- A (old text, new text) pair if this error is auto-fixable safely.
        see_also -- An optional array of other MessageContext locations relevant to this message.
        frame -- The 'inspect' stack frame corresponding to the location that raised this message.
          If None, will assume it is the direct caller of self.warning().
        """
        if not frame:
            f = currentframe()
            if f:
                frame = f.f_back
        self.diag(MessageType.WARNING, message_id, messageLines, group=group,
                  replacement=replacement, context=context, fix=fix, see_also=see_also, frame=frame)

    def error(self, message_id, messageLines, group=None, replacement=None,
              context=None, fix=None, see_also=None, frame=None):
        """Log an error for the file, if the message ID is enabled.

        Wrapper around self.diag() that automatically sets severity as well as frame.

        Arguments:
        message_id -- A MessageId value.
        messageLines -- A string or list of strings containing a human-readable error description.

        Optional, named arguments:
        context -- A MessageContext. If None, will be constructed from self.match and group.
        group -- The name of the regex group in self.match that contains the problem. Only used if context is None.
          If needed and is None, self.group is used instead.
        replacement -- The string, if any, that should be suggested as a replacement for the group in question.
          Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough
          (or can't easily phrase a regex) to do it automatically.
        fix -- A (old text, new text) pair if this error is auto-fixable safely.
        see_also -- An optional array of other MessageContext locations relevant to this message.
        frame -- The 'inspect' stack frame corresponding to the location that raised this message.
          If None, will assume it is the direct caller of self.error().
        """
        if not frame:
            f = currentframe()
            if f:
                frame = f.f_back
        self.diag(MessageType.ERROR, message_id, messageLines, group=group,
                  replacement=replacement, context=context, fix=fix, see_also=see_also, frame=frame)

    def diag(self, severity, message_id, messageLines, context=None, group=None,
             replacement=None, fix=None, see_also=None, frame=None):
        """Log a diagnostic for the file, if the message ID is enabled.

        Also records the auto-fix, if applicable.

        Arguments:
        severity -- A MessageType value.
        message_id -- A MessageId value.
        messageLines -- A string or list of strings containing a human-readable error description.

        Optional, named arguments:
        context -- A MessageContext. If None, will be constructed from self.match and group.
        group -- The name of the regex group in self.match that contains the problem. Only used if context is None.
          If needed and is None, self.group is used instead.
        replacement -- The string, if any, that should be suggested as a replacement for the group in question.
          Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough
          (or can't easily phrase a regex) to do it automatically.
        fix -- A (old text, new text) pair if this error is auto-fixable safely.
        see_also -- An optional array of other MessageContext locations relevant to this message.
        frame -- The 'inspect' stack frame corresponding to the location that raised this message.
          If None, will assume it is the direct caller of self.diag().
        """
        if not self.messageEnabled(message_id):
            self.logger.debug(
                'Discarding a %s message because it is disabled.', message_id)
            return

        if isinstance(messageLines, str):
            messageLines = [messageLines]

        self.logger.info('Recording a %s message: %s',
                         message_id, ' '.join(messageLines))

        # Ensure all auto-fixes are marked as such.
        if fix is not None and AUTO_FIX_STRING not in messageLines:
            messageLines.append(AUTO_FIX_STRING)

        if not frame:
            f = currentframe()
            if f:
                frame = f.f_back
        if context is None:
            message = Message(message_id=message_id,
                              message_type=severity,
                              message=messageLines,
                              context=self.storeMessageContext(group=group),
                              replacement=replacement,
                              see_also=see_also,
                              fix=fix,
                              frame=frame)
        else:
            message = Message(message_id=message_id,
                              message_type=severity,
                              message=messageLines,
                              context=context,
                              replacement=replacement,
                              see_also=see_also,
                              fix=fix,
                              frame=frame)
        if fix is not None:
            self.fixes.add(fix)
        self.messages.append(message)

    def messageEnabled(self, message_id):
        """Return true if the given message ID is enabled."""
        return message_id in self.enabled_messages

    ###
    # Accessors for externally-interesting information

    def numDiagnostics(self):
        """Count the total number of diagnostics (errors or warnings) for this file."""
        return len(self.messages)

    def numErrors(self):
        """Count the total number of errors for this file."""
        return self.numMessagesOfType(MessageType.ERROR)

    def numMessagesOfType(self, message_type):
        """Count the number of messages of a particular type (severity)."""
        return len(
            [msg for msg in self.messages if msg.message_type == message_type])

    def hasFixes(self):
        """Return True if any messages included auto-fix patterns."""
        return len(self.fixes) > 0

    ###
    # Assorted internal methods.
    def printMessageCounts(self):
        """Print a simple count of each MessageType of diagnostics."""
        for message_type in [MessageType.ERROR, MessageType.WARNING]:
            count = self.numMessagesOfType(message_type)
            if count > 0:
                print(f'{count} {message_type}{_s_suffix(count)} generated.')

    def dumpInternals(self):
        """Dump internal variables to screen, for debugging."""
        print('self.lineNum: ', self.lineNum)
        print('self.line:', self.line)
        print('self.prev_line_ref_page_tag: ', self.prev_line_ref_page_tag)
        print('self.current_ref_page:', self.current_ref_page)

    def getMissingValiditySuppressions(self):
        """Return an enumerable of entity names that we shouldn't warn about missing validity.

        May override.
        """
        return []

    def recordInclude(self, include_dict, generated_type=None):
        """Store the current line as being the location of an include directive or equivalent.

        Reports duplicate include errors, as well as include/ref-page mismatch or missing ref-page,
        by calling self.checkIncludeRefPageRelation() for "actual" includes (where generated_type is None).

        Arguments:
        include_dict -- The include dictionary to update: one of self.apiIncludes or self.validityIncludes.
        generated_type -- The type of include (e.g. 'api', 'valid', etc). By default, extracted from self.match.
        """
        assert self.match
        entity = self.match.group('entity_name')
        if generated_type is None:
            generated_type = self.match.group('generated_type')

            # Only checking the ref page relation if it's retrieved from regex.
            # Otherwise it might be a manual anchor recorded as an include,
            # etc.
            self.checkIncludeRefPageRelation(entity, generated_type)

        if entity in include_dict:
            self.error(MessageId.DUPLICATE_INCLUDE,
                       "Included {} docs for {} when they were already included.".format(generated_type,
                                                                                         entity), see_also=include_dict[entity])
            include_dict[entity].append(self.storeMessageContext())
        else:
            include_dict[entity] = [self.storeMessageContext()]

    def getInnermostBlockEntry(self):
        """Get the BlockEntry for the top block delim on our stack."""
        if not self.block_stack:
            return None
        return self.block_stack[-1]

    def getInnermostBlockDelimiter(self):
        """Get the delimiter for the top block on our stack."""
        top = self.getInnermostBlockEntry()
        if not top:
            return None
        return top.delimiter

    def pushBlock(self, block_type, refpage=None, context=None, delimiter=None):
        """Push a new entry on the block stack."""
        if not delimiter:
            self.logger.info("pushBlock: not given delimiter")
            delimiter = self.line
        if not context:
            context = self.storeMessageContext()

        old_top_delim = self.getInnermostBlockDelimiter()

        self.block_stack.append(BlockEntry(
            delimiter=delimiter,
            context=context,
            refpage=refpage,
            block_type=block_type))

        location = self.getBriefLocation(context)
        self.logger.info(
            "pushBlock: %s: Pushed %s delimiter %s, previous top was %s, now %d elements on the stack",
            location, block_type.value, delimiter, old_top_delim, len(self.block_stack))

        self.dumpBlockStack()

    def popBlock(self):
        """Pop and return the top entry from the block stack."""
        old_top = self.block_stack.pop()
        location = self.getBriefLocation(old_top.context)
        self.logger.info(
            "popBlock: %s: popping %s delimiter %s, now %d elements on the stack",
            location, old_top.block_type.value, old_top.delimiter, len(self.block_stack))

        self.dumpBlockStack()

        return old_top

    def dumpBlockStack(self):
        self.logger.debug('Block stack, top first:')
        for distFromTop, x in enumerate(reversed(self.block_stack)):
            self.logger.debug(' - block_stack[%d]: Line %d: "%s" refpage=%s',
                              -1 - distFromTop,
                              x.context.lineNum, x.delimiter, x.refpage)

    def getBriefLocation(self, context):
        """Format a context briefly - omitting the filename if it has newlines in it."""
        if '\n' in context.filename:
            return f'input string line {context.lineNum}'
        return f'{context.filename}:{context.lineNum}'

    ###
    # Handlers for a variety of diagnostic-meriting conditions
    #
    # Split out for clarity and for allowing fine-grained override on a per-project basis.
    ###

    def handleIncludeMissingRefPage(self, entity, generated_type):
        """Report a message about an include outside of a ref-page block."""
        msg = [f"Found {generated_type} include for {entity} outside of a reference page block.",
               "This is probably a missing reference page block."]
        refpage = self.computeExpectedRefPageFromInclude(entity)
        data = self.checker.findEntity(refpage)
        if data:
            msg.append('Expected ref page block might start like:')
            msg.append(self.makeRefPageTag(refpage, data=data))
        else:
            msg.append(
                f"But, expected ref page entity name {refpage} isn't recognized...")
        self.warning(MessageId.REFPAGE_MISSING, msg)

    def handleIncludeMismatchRefPage(self, entity, generated_type):
        """Report a message about an include not matching its containing ref-page block."""
        assert self.current_ref_page
        self.warning(MessageId.REFPAGE_MISMATCH, "Found {} include for {}, inside the reference page block of {}".format(
            generated_type, entity, self.current_ref_page.entity))

    def handleWrongMacro(self, msg, data):
        """Report an appropriate message when we found that the macro used is incorrect.

        May be overridden depending on each API's behavior regarding macro misuse:
        e.g. in some cases, it may be considered a MessageId.LEGACY warning rather than
        a MessageId.WRONG_MACRO or MessageId.EXTENSION.
        """
        message_type = MessageType.WARNING
        message_id = MessageId.WRONG_MACRO
        group = 'macro'

        if data.category == EXTENSION_CATEGORY:
            # Ah, this is an extension
            msg.append(
                'This is apparently an extension name, which should be marked up as a link.')
            message_id = MessageId.EXTENSION
            group = None  # replace the whole thing
        else:
            # Non-extension, we found the macro though.
            message_type = MessageType.ERROR
        msg.append(AUTO_FIX_STRING)
        self.diag(message_type, message_id, msg,
                  group=group, replacement=self.makeMacroMarkup(data=data), fix=self.makeFix(data=data))

    def handleExpectedRefpageBlock(self):
        """Handle expecting to see -- to start a refpage block, but not seeing that at all."""
        self.error(MessageId.REFPAGE_BLOCK,
                   ["Expected, but did not find, a line containing only -- following a reference page tag,",
                    "Pretending to insert one, for more readable messages."],
                   see_also=[self.prev_line_ref_page_tag])
        # Fake "in ref page" regardless, to avoid spurious extra errors.
        self.processBlockDelimiter('--', BlockType.REF_PAGE_LIKE,
                                   context=self.prev_line_ref_page_tag)

    ###
    # Construct related values (typically named tuples) based on object state and supplied arguments.
    #
    # Results are typically supplied to another method call.
    ###

    def storeMessageContext(self, group=None, match=None):
        """Create message context from corresponding instance variables.

        Arguments:
        group -- The regex group name, if any, identifying the part of the match to highlight.
        match -- The regex match. If None, will use self.match.
        """
        if match is None:
            match = self.match
        return MessageContext(filename=self.filename,
                              lineNum=self.lineNum,
                              line=self.line,
                              match=match,
                              group=group)

    def makeFix(self, newMacro=None, newEntity=None, data=None):
        """Construct a fix pair for replacing the old macro:entity with new.

        Wrapper around self.makeSearch() and self.makeMacroMarkup().
        """
        return (self.makeSearch(), self.makeMacroMarkup(
            newMacro, newEntity, data))

    def makeSearch(self):
        """Construct the string self.macro:self.entity, for use in the old text part of a fix pair."""
        return f'{self.macro}:{self.entity}'

    def makeMacroMarkup(self, newMacro=None, newEntity=None, data=None):
        """Construct appropriate markup for referring to an entity.

        Typically constructs macro:entity, but can construct `<<EXTENSION_NAME>>`
        (or equivalent) if the supplied entity is identified as an extension.

        Arguments:
        newMacro -- The macro to use. Defaults to data.macro (if available), otherwise self.macro.
        newEntity -- The entity to use. Defaults to data.entity (if available), otherwise self.entity.
        data -- An EntityData value corresponding to this entity. If not provided, will be looked up by newEntity.
        """
        if not newEntity:
            if data:
                newEntity = data.entity
            else:
                newEntity = self.entity
        if not newMacro:
            if data:
                newMacro = data.macro
            else:
                newMacro = self.macro
        if not data:
            data = self.checker.findEntity(newEntity)
        if data and data.category == EXTENSION_CATEGORY:
            return self.makeExtensionLink(newEntity)
        return f'{newMacro}:{newEntity}'

    def makeExtensionLink(self, newEntity=None):
        """Create a correctly-formatted link to an extension.

        Result takes the form `<<EXTENSION_NAME>>` or whatever the Conventions define.

        Argument:
        newEntity -- The extension name to link to. Defaults to self.entity.
        """
        if not newEntity:
            newEntity = self.entity
        return self.checker.conventions().formatExtension(newEntity)

    def computeExpectedRefPageFromInclude(self, entity):
        """Compute the expected ref page entity based on an include entity name."""
        # No-op in general.
        return entity

    def makeRefPageTag(self, entity, data=None,
                       ref_type=None, desc='', xrefs=None):
        """Construct a ref page tag string from attribute values."""
        if ref_type is None and data is not None:
            ref_type = data.directory
        if ref_type is None:
            ref_type = "????"
        return "[open,refpage='{}',type='{}',desc='{}',xrefs='{}']".format(
            entity, ref_type, desc, ' '.join(xrefs or []))
