#!/usr/bin/env python3
"""
There are four types of CCPP metadata tables, scheme, module, ddt, and host.
A metadata file contains one or more metadata tables.
A metadata file SHOULD NOT mix metadata table types. The exception is a
  metadata file which contains one or more ddt tables followed by a module
  or host table.

Each metadata table begins with a 'ccpp-table-properties' section followed by
  one or more 'ccpp-arg-table' sections. These sections are described below.
A 'ccpp-arg-table' section is followed by one or more variable declaration
  sections, also described below.

Metadata headers are in config file format.

A 'ccpp-table-properties' section entries are:
name = <name> : the name of the following ccpp-arg-table entries (required).
                It is one of the following possibilities:
   - SchemeName: the name of a scheme (i.e., the name of
                  a scheme interface (related to SubroutineName below).
   - DerivedTypeName: a derived type name for a type which will be used
                      somewhere in the CCPP interface.
   - ModuleName: the name of the module whose module variables will be
                 used somewhere in the CCPP interface
   - HostName: the name of the host model. Variables in this section become
                 part of the CCPP UI, the CCPP routines called by the
                 host model (e.g., <HostName>_ccpp_physics_run).
type = <type> : The type of header (required), one of:
    - scheme: A CCPP subroutine
    - ddt: A header for a derived data type
    - module: A header on some module data
    - host: A header on data which will be part of the CCPP UI

The ccpp-arg-table section entries in this section are:
name = <name> : the name of the file object which immediately follows the
                argument table (required).
                It is one of the following possibilities:
   - SubroutineName: the name of a subroutine (i.e., the name of
                     a scheme interface function such as SchemeName_run)
   - DerivedTypeName: a derived type name for a type which will be used
                      somewhere in the CCPP interface.
   - ModuleName: the name of the module whose module variables will be
                 used somewhere in the CCPP interface
   - HostName: the name of the host model. Variables in this section become
                 part of the CCPP UI, the CCPP routines called by the
                 host model (e.g., <HostName>_ccpp_physics_run).
type = <type> : The type of header (required). It must match the type of the
                associated ccpp-table-properties section (see above).

A variable declaration section begins with a variable name line (a local
variable name enclosed in square brackets) followed by one or more
variable attribute statements.
A variable attribute statement is an attribute name and the value for
that attribute separated by an equal sign. Whitespace is not
significant except inside of strings.
Variable attribute statements may be combined on a line if separated by
a vertical bar.

An example argument table is shown below.

[ccpp-table-properties]
  name = <name>
  type = scheme
  relative_path = <relative path>
  dependencies = <dependencies>

[ccpp-arg-table]
  name = <name>
  type = scheme
[ im ]
  standard_name = horizontal_loop_extent
  long_name = horizontal loop extent, start at 1
  units = index
  type = integer
  dimensions = ()
  intent = in
[ ix ]
  standard_name = horizontal_loop_dimension
  long_name = horizontal dimension
  units = index | type = integer | dimensions = ()
  intent = in
  ...
[ errmsg]
  standard_name = ccpp_error_message
  long_name = error message for error handling in CCPP
  units = none
  type = character
  len = *
  dimensions = ()
  intent = out
[ ierr ]
  standard_name = ccpp_error_code
  long_name = error flag for error handling in CCPP
  type = integer
  units = 1
  dimensions = ()
  intent=out

Notes on the input format:
- SubroutineName must match the name of the subroutine that the argument
  table describes
- DerivedTypeName must match the name of the derived type that the argument
  table describes
- ModuleName must match the name of the module whose variables the argument
  table describes
- for variable type definitions and module variables, the intent keyword
  is not functional and should be omitted
- each argument table (and its subroutine) must accept the following two arguments for error handling (the local name can vary):
[ errmsg ]
  standard_name = ccpp_error_message
  long_name = Error message for error handling in CCPP
  units = none
  dimensions = ()
  type = character
  kind = len=512
  intent = out
[ errflg ]
  standard_name = ccpp_error_code
  long_name = Error flag for error handling in CCPP
  units = 1
  dimensions = ()
  type = integer
  intent = out
"""

# Python library imports
import difflib
import logging
import os.path
import re
# CCPP framework imports
from ccpp_state_machine  import CCPP_STATE_MACH
from metavar     import Var, VarDictionary, CCPP_CONSTANT_VARS
from parse_tools import ParseObject, ParseSource, ParseContext, context_string
from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError
from parse_tools import FORTRAN_ID, FORTRAN_SCALAR_REF, FORTRAN_SCALAR_REF_RE
from parse_tools import check_fortran_ref, check_fortran_id
from parse_tools import check_fortran_intrinsic
from parse_tools import register_fortran_ddt_name, unique_standard_name

########################################################################

SCHEME_HEADER_TYPE = 'scheme'
_SINGLETON_TABLE_TYPES = ['ddt', 'host', 'module'] # Only one section per table
TABLE_TYPES = _SINGLETON_TABLE_TYPES + [SCHEME_HEADER_TYPE]
HEADER_TYPES = TABLE_TYPES + ['local']
UNKNOWN_PROCESS_TYPE = 'UNKNOWN'

_BLANK_LINE = re.compile(r"\s*[#;]")

def blank_metadata_line(line):
    """Return True if <line> is a valid config format blank or comment
    line. Also return True if we have reached the end of the file
    (no line)"""
    return (not line) or (_BLANK_LINE.match(line) is not None)

########################################################################

def _parse_config_line(line, context):
    """Parse a config line and return a list of keyword value pairs."""
    parse_items = list()
    if line is None:
        pass # No properties on this line
    elif blank_metadata_line(line):
        pass # No properties on this line
    else:
        properties = line.strip().split('|')
        for prop in properties:
            pitems = prop.split('=', 1)
            if len(pitems) >= 2:
                parse_items.append(pitems)
            else:
                raise ParseSyntaxError("variable property syntax",
                                       token=prop, context=context)
            # end if
        # end for
    # end if
    return parse_items

########################################################################

def parse_metadata_file(filename, known_ddts, run_env):
    """Parse <filename> and return list of parsed metadata tables"""
    # Read all lines of the file at once
    meta_tables = list()
    table_titles = list() # Keep track of names in file
    with open(filename, 'r') as infile:
        fin_lines = infile.readlines()
        for index, fin_line in enumerate(fin_lines):
            fin_lines[index] = fin_line.rstrip('\n')
        # end for
    # end with
    # Look for a header start
    parse_obj = ParseObject(filename, fin_lines)
    curr_line, curr_line_num = parse_obj.curr_line()
    while curr_line is not None:
        if MetadataTable.table_start(curr_line):
            new_table = MetadataTable(run_env, parse_object=parse_obj,
                                      known_ddts=known_ddts)
            ntitle = new_table.table_name
            if ntitle not in table_titles:
                meta_tables.append(new_table)
                table_titles.append(ntitle)
                if new_table.table_type == 'ddt':
                    known_ddts.append(ntitle)
                # end if
            else:
                errmsg = 'Duplicate metadata table, {}, at {}:{}'
                ctx = curr_line_num + 1
                raise CCPPError(errmsg.format(ntitle, filename, ctx))
            # end if
            curr_line, curr_line_num = parse_obj.curr_line()
        elif blank_metadata_line(curr_line):
            curr_line, curr_line_num = parse_obj.next_line()
        else:
            raise ParseSyntaxError('CCPP metadata line', token=curr_line,
                                   context=parse_obj)
        # end if
    # end while
    return meta_tables

########################################################################

def find_scheme_names(filename):
    """Find and return a list of all the physics scheme names in
    <filename>. A scheme is identified by its ccpp-table-properties name.
    """
    scheme_names = list()
    with open(filename, 'r') as infile:
        fin_lines = infile.readlines()
    # end with
    num_lines = len(fin_lines)
    context = ParseContext(linenum=1, filename=filename)
    while context.line_num <= num_lines:
        if MetadataTable.table_start(fin_lines[context.line_num - 1]):
            found_start = False
            while not found_start:
                line = fin_lines[context.line_num].strip()
                context.line_num += 1
                if line and (line[0] == '['):
                    found_start = True
                elif line:
                    props = _parse_config_line(line, context)
                    for prop in props:
                        # Look for name property
                        key = prop[0].strip().lower()
                        value = prop[1].strip()
                        if key == 'name':
                            scheme_names.append(value)
                        # end if
                    # end for
                # end if
                if context.line_num > num_lines:
                    break
                # end if
            # end while
        else:
            context.line_num += 1
        # end if
    # end while
    return scheme_names

########################################################################

class MetadataTable():
    """Class to hold a CCPP Metadata table including the table header
    (ccpp-table-properties section) and all of the associated table
    sections (ccpp-arg-table sections)."""

    __table_start = re.compile(r"(?i)\s*\[\s*ccpp-table-properties\s*\]")

    def __init__(self, run_env, table_name_in=None, table_type_in=None,
                 dependencies=None, relative_path=None, known_ddts=None,
                 var_dict=None, module=None, parse_object=None):
        """Initialize a MetadataTable, either with a name, <table_name_in>, and
        type, <table_type_in>, or with information from a file (<parse_object>).
        if <parse_object> is None, <dependencies> and <relative_path> are
          also stored.
        If <var_dict> and / or module are passed (not allowed with
          <parse_object), then a single MetadataSection is added with
          that information.
        """
        self.__pobj = parse_object
        self.__dependencies = dependencies
        self.__relative_path = relative_path
        self.__sections = list()
        self.__run_env = run_env
        if parse_object is None:
            if table_name_in is not None:
                self.__table_name = table_name_in
            else:
                raise ParseInternalError('MetadataTable requires a name')
            # end if
            if table_type_in is None:
                perr = 'MetadataTable requires a table type'
                raise ParseInternalError(perr)
            # end if
            if table_type_in in HEADER_TYPES:
                self.__table_type = table_type_in
            else:
                raise ParseSyntaxError("metadata arg table type",
                                       token=table_type_in,
                                       context=self.__pobj)
            # end if
            self.__start_context = None
            if (var_dict is not None) or module:
                if var_dict is None:
                    var_dict = VarDictionary(self.table_name, run_env)
                # end if
                # This interface is used by the Fortran parser so strip off
                #    the phase from the table name but not the section title
                stitle = self.table_name
                if self.table_type == SCHEME_HEADER_TYPE:
                    fnam, _, _ = CCPP_STATE_MACH.function_match(self.table_name)
                    self.__table_name = fnam or stitle
                # end if
                sect = MetadataSection(self.table_name, self.table_type,
                                       run_env, title=stitle,
                                       type_in=self.table_type, module=module,
                                       var_dict=var_dict, known_ddts=known_ddts)
                self.__sections.append(sect)
            # end if
        else:
            self.__table_name = None
            self.__table_type = None
            if dependencies is not None:
                perr = "dependencies not allowed as argument when reading file"
                raise ParseInternalError(perr)
            # end if
            if relative_path:
                perr = "relative_path not allowed as argument when reading file"
                raise ParseInternalError(perr)
            # end if
            if var_dict is not None: # i.e., not even an empty dict
                perr = "var_dict not allowed as argument when reading file"
                raise ParseInternalError(perr)
            if module:
                perr = "module not allowed as argument when reading file"
                raise ParseInternalError(perr)
            # end if
            if known_ddts is None:
                known_ddts = list()
            # end if
            self.__start_context = ParseContext(context=self.__pobj)
            self.__init_from_file(known_ddts, self.__run_env)
        # end if

    def __init_from_file(self, known_ddts, run_env):
        """ Read the table preamble, assume the caller already figured out
        the first line of the header using the header_start method."""
        curr_line, _ = self.__pobj.next_line()
        in_properties_header = True
        skip_rest_of_section = False
        self.__dependencies = list() # Default is no dependencies
        # Process lines until the end of the file or start of the next table.
        while ((curr_line is not None) and
               (not MetadataTable.table_start(curr_line))):
            if MetadataSection.header_start(curr_line):
                in_properties_header = False
            # end if
            if blank_metadata_line(curr_line):
                curr_line, _ = self.__pobj.next_line()
            elif in_properties_header:
                # Process the properties in this table header line
                for prop in _parse_config_line(curr_line, self.__pobj):
                    # Manually parse name, type, and table properties
                    key = prop[0].strip().lower()
                    value = prop[1].strip()
                    if key == 'name':
                        self.__table_name = value
                    elif key == 'type':
                        if value not in TABLE_TYPES:
                            self.__pobj.add_syntax_err("metadata table type",
                                                       token=value)
                        # end if
                        self.__table_type = value
                    elif key == 'dependencies':
                        if value.lower() == "none":
                            if self.__dependencies:
                                emsg = "dependencies = {} is ".format(value)
                                depends = ", ".join(self.__dependencies)
                                emsg += "incompatible with {}".format(depends)
                                self.__pobj.add_syntax_err(emsg)
                            else:
                                self.__dependencies = None
                            # end if
                        elif self.__dependencies is None:
                            emsg = "Cannot add dependencies, they have "
                            emsg += "already been declared as None"
                            self.__pobj.add_syntax_err(emsg)
                        else:
                            depends = [x.strip() for x in value.split(',')
                                       if x.strip()]
                            self.__dependencies.extend(depends)
                        # end if
                    elif key == 'relative_path':
                        self.__relative_path = value
                    else:
                        tok_type = "metadata table start property"
                        self.__pobj.add_syntax_err(tok_type, token=value)
                    # end if
                # end for
                curr_line, _ = self.__pobj.next_line()
            else:
                # Process a metadata section
                if MetadataSection.header_start(curr_line):
                    skip_rest_of_section = False
                    section = MetadataSection(self.table_name, self.table_type,
                                              run_env, parse_object=self.__pobj,
                                              known_ddts=known_ddts)
                    # Some table types only allow for one associated section
                    if ((len(self.__sections) == 1) and
                        (self.table_type in _SINGLETON_TABLE_TYPES)):
                        prev_title = self.__sections[0].title
                        emsg = "{}, '{}', table already contains '{}'"
                        self.__pobj.add_syntax_err(emsg.format(self.table_type,
                                                               section.title,
                                                               prev_title))
                    # end if
                    self.__sections.append(section)
                    # Note: Do not read next line, we are already on it.
                    curr_line, _ = self.__pobj.curr_line()
                elif not blank_metadata_line(curr_line):
                    if not skip_rest_of_section:
                        self.__pobj.add_syntax_err("metadata file line",
                                                   token=curr_line)
                        skip_rest_of_section = True
                    # end if
                    curr_line, _ = self.__pobj.next_line()
                else:
                    curr_line, _ = self.__pobj.next_line()
                # end if
            # end if
        # end while
        if self.__pobj.error_message:
            # Time to dump out error messages
            raise CCPPError(self.__pobj.error_message)
        # end if
        if self.table_type == "ddt":
            known_ddts.append(self.table_name)
        # end if
        if self.__dependencies is None:
            self.__dependencies = list()
        # end if

    def start_context(self, with_comma=True, nodir=True):
        """Return a context string for the beginning of the table"""
        return context_string(self.__start_context,
                              with_comma=with_comma, nodir=nodir)

    def sections(self):
        """Return the metadata header sections for this table"""
        if self.__sections:
            # Return a copy so it cannot be modified
            return list(self.__sections)
        return self.__sections

    @property
    def table_name(self):
        'Return the name of the metadata table'
        return self.__table_name

    @property
    def table_type(self):
        'Return the type of structure this header documents'
        return self.__table_type

    @property
    def dependencies(self):
        """Return the dependencies for this table"""
        return self.__dependencies

    @property
    def relative_path(self):
        """Return the relative path for the table's dependencies"""
        return self.__relative_path

    @property
    def run_env(self):
        """Return this table's CCPPFrameworkEnv object"""
        return self.__run_env

    def __repr__(self):
        '''Print representation for MetadataTable objects'''
        return "<{} {} @ 0X{:X}>".format(self.__class__.__name__,
                                         self.table_name, id(self))

    def __str__(self):
        '''Print string for MetadataTable objects'''
        return "<{} {}>".format(self.__class__.__name__, self.table_name)

    @classmethod
    def table_start(cls, line):
        """Return True iff <line> is a ccpp-table-properties header statement.
        """
        if (line is None) or blank_metadata_line(line):
            match = None
        else:
            match = cls.__table_start.match(line)
        # end if
        return match is not None

########################################################################

class MetadataSection(ParseSource):
    """Class to hold all information from a metadata header
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type = scheme", "module = foo",    \
                       "[ im ]", "standard_name = horizontal_loop_extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in"])) #doctest: +ELLIPSIS
    <__main__.MetadataSection foo / footable at 0x...>
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type = scheme", "module = foobar", \
                       "[ im ]", "standard_name = horizontal_loop_extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS
    <metavar.Var horizontal_loop_extent: im at 0x...>
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type = scheme", "module = foobar", \
                       "process = microphysics", "[ im ]",                    \
                       "standard_name = horizontal_loop_extent",              \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS
    <metavar.Var horizontal_loop_extent: im at 0x...>
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type=scheme", "module = foo",      \
                       "[ im ]", "standard_name = horizontal_loop_extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       "  subroutine foo()"])).find_variable('horizontal_loop_extent') #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
    parse_source.ParseSyntaxError: Invalid variable property syntax, 'subroutine foo()', at foobar.txt:9
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type = scheme", "module=foobar",   \
                       "[ im ]", "standard_name = horizontal_loop_extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       ""], line_start=0)).find_variable('horizontal_loop_extent').get_prop_value('local_name')
    'im'
    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = footable", "type = scheme"                     \
                       "[ im ]", "standard_name = horizontalloop extent",     \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       ""], line_start=0)).find_variable('horizontal_loop_extent')

    >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV,                 \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["[ccpp-arg-table]", "name = foobar", "type = scheme"   \
                       "[ im ]", "standard_name = horizontal loop extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       ""], line_start=0)).find_variable('horizontal_loop_extent')

    >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV,                   \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = foobar", "module = foo"                        \
                       "[ im ]", "standard_name = horizontal loop extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       ""], line_start=0)).find_variable('horizontal_loop_extent')

    >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV,                   \
                      parse_object=ParseObject("foobar.txt",                  \
                      ["name = foobar", "foo = bar"                           \
                       "[ im ]", "standard_name = horizontal loop extent",    \
                       "long_name = horizontal loop extent, start at 1",      \
                       "units = index | type = integer",                      \
                       "dimensions = () |  intent = in",                      \
                       ""], line_start=0)).find_variable('horizontal_loop_extent')

    >>> MetadataSection.header_start('[ ccpp-arg-table ]')
    True
    >>> MetadataSection.header_start('[ qval ]')
    False
    >>> MetadataSection.header_start('  local_name = foo')
    False
    >>> MetadataSection.variable_start('[ qval ]', ParseObject('foo.meta', []))
    'qval'
    >>> MetadataSection.variable_start('[ qval(hi_mom) ]', ParseObject('foo.meta', []))
    'qval(hi_mom)'
    >>> MetadataSection.variable_start('  local_name = foo', ParseContext(filename='foo.meta', linenum=1))

"""

    __header_start = re.compile(r"(?i)\s*\[\s*ccpp-arg-table\s*\]")

    __var_start = re.compile(r"^\[\s*"+FORTRAN_ID+r"\s*\]$")

    __vref_start = re.compile(r"^\[\s*"+FORTRAN_SCALAR_REF+r"\s*\]$")

    __html_template__ = """
<html>
<head>
<title>{title}</title>
<meta charset="UTF-8">
</head>
<body>
<table border="1">
{header}{contents}</table>
</body>
</html>
"""

    def __init__(self, table_name, table_type, run_env, parse_object=None,
                 title=None, type_in=None, module=None, process_type=None,
                 var_dict=None, known_ddts=None):
        """Initialize a new MetadataSection object.
        If <parse_object> is not None, initialize from the current file and
        location in <parse_object>.
        If <parse_object> is None, initialize from <title>, <type>, <module>,
        and <var_dict>. Note that if <parse_object> is not None, <title>,
        <type>, <module>, and <var_dict> are ignored.
        <table_name> and <table_type> are the name and type of the
        metadata header of which this section is a part. They must match
        the type and name of this section (once the name action has been
        removed, e.g., name = foo_init matches type foo).
        """
        self.__pobj = parse_object
        self.__variables = None # In case __init__ crashes
        self.__section_title = None
        self.__header_type = None
        self.__module_name = None
        self.__process_type = UNKNOWN_PROCESS_TYPE
        self.__section_valid = True
        self.__run_env = run_env
        if parse_object is None:
            if title is not None:
                self.__section_title = title
            else:
                raise ParseInternalError('MetadataSection requires a title')
            # end if
            if type_in is None:
                perr = 'MetadataSection requires a header type'
                raise ParseInternalError(perr)
            # end if
            if type_in in HEADER_TYPES:
                self.__header_type = type_in
            else:
                self.__pobj.add_syntax_err("metadata arg table type",
                                           token=type_in)
                self.__section_valid = False
            # end if
            mismatch = self.section_table_mismatch(table_name, table_type)
            if mismatch:
                self.__pobj.add_syntax_err(mismatch)
                self.__section_valid = False
            # end if
            mismatch = self.section_table_mismatch(table_name, table_type)
            if mismatch:
                raise CCPPError(mismatch)
            # end if
            if module is not None:
                self.__module_name = module
            else:
                perr = "MetadataSection requires a module name"
                self.__pobj.add_syntax_err(perr)
                self.__section_valid = False
            # end if
            if process_type is None:
                self.__process_type = UNKNOWN_PROCESS_TYPE
            else:
                self.__process_type = process_type
            # end if
            #  Initialize our ParseSource parent
            super().__init__(self.title, self.header_type, self.__pobj)
            self.__variables = VarDictionary(self.title, run_env)
            for var in var_dict.variable_list(): # Let this crash if no dict
                self.__variables.add_variable(var, run_env)
            # end for
            self.__start_context = None
        else:
            if known_ddts is None:
                known_ddts = list()
            # end if
            self.__start_context = ParseContext(context=self.__pobj)
            self.__init_from_file(table_name, table_type, known_ddts, run_env)
        # end if
        # Register this header if it is a DDT
        if self.header_type == 'ddt':
            register_fortran_ddt_name(self.title)
        # end if
        # Categorize the variables
        self._var_intents = {'in' : list(), 'out' : list(), 'inout' : list()}
        for var in self.variable_list():
            intent = var.get_prop_value('intent')
            if intent is not None:
                self._var_intents[intent].append(var)
            # end if
        # end for

    def _default_module(self):
        """Set a default module for this header"""
        mfile = self.__pobj.file_name
        if mfile[-5:] == '.meta':
            # Default value is a Fortran module that matches the filename
            def_mod = os.path.basename(mfile)[:-5]
        else:
            def_mod = os.path.basename(mfile)
            last_dot = def_mod.rfind('.')
            if last_dot >= 0:
                ldef = len(def_mod)
                def_mod = def_mod[:last_dot-ldef]
            # end if
        # end if
        return def_mod

    def __init_from_file(self, table_name, table_type, known_ddts, run_env):
        """ Read the section preamble, assume the caller already figured out
        the first line of the header using the header_start method."""
        start_ctx = context_string(self.__pobj)
        curr_line, _ = self.__pobj.next_line() # Skip past [ccpp-arg-table]
        while ((curr_line is not None) and
               (not MetadataSection.variable_start(curr_line, self.__pobj)) and
               (not MetadataSection.header_start(curr_line)) and
               (not MetadataTable.table_start(curr_line))):
            for prop in _parse_config_line(curr_line, self.__pobj):
                # Manually parse name, type, and module properties
                key = prop[0].strip().lower()
                value = prop[1].strip()
                if key == 'name':
                    self.__section_title = value
                elif key == 'type':
                    if value not in HEADER_TYPES:
                        self.__pobj.add_syntax_err("metadata table type",
                                                   token=value)
                        self.__section_valid = False
                        close = difflib.get_close_matches(value, HEADER_TYPES)
                        if close:
                            self.__header_type = close[0] # Allow error continue
                        # end if
                    # end if
                    # Set value even if error so future error msgs make sense
                    self.__header_type = value
                elif key == 'module':
                    if value != "None":
                        self.__module_name = value
                    else:
                        self.__pobj.add_syntax_err("metadata table, no module")
                        self.__module_name = 'INVALID' # Allow error continue
                        self.__section_valid = False
                    # end if
                elif key == 'process':
                    self.__process_type = value
                else:
                    self.__pobj.add_syntax_err("metadata table start property",
                                               token=value)
                    self.__process_type = 'INVALID' # Allow error continue
                    self.__section_valid = False
                # end if
            # end for
            curr_line, _ = self.__pobj.next_line()
        # end while
        if self.title is None:
            self.__pobj.add_syntax_err("metadata header start, no table name",
                                       token=curr_line)
            self.__section_valid = False
        # end if
        if self.header_type is None:
            self.__pobj.add_syntax_err("metadata header start, no table type",
                                       token=curr_line)
            self.__section_valid = False
        # end if
        if ((self.header_type != SCHEME_HEADER_TYPE) and
            (self.process_type != UNKNOWN_PROCESS_TYPE)):
            emsg = "process keyword only allowed for a scheme"
            self.__pobj.add_syntax_err(emsg, token=curr_line)
            self.__process_type = UNKNOWN_PROCESS_TYPE # Allow error continue
            self.__section_valid = False
        # end if
        mismatch = self.section_table_mismatch(table_name, table_type)
        if mismatch:
            self.__pobj.add_syntax_err(mismatch)
            self.__section_valid = False
        # end if
        if run_env.logger and run_env.logger.isEnabledFor(logging.INFO):
            run_env.logger.info("Parsing {} {}{}".format(self.header_type,
                                                         self.title, start_ctx))
        # end if
        if self.header_type == "ddt":
            known_ddts.append(self.title)
        # end if
        # We need a default module if none was listed
        if self.module is None:
            self.__module_name = self._default_module()
        # end if
        #  Initialize our ParseSource parent
        super().__init__(self.title, self.header_type, self.__pobj)
        # Read the variables
        valid_lines = True
        self.__variables = VarDictionary(self.title, run_env)
        while valid_lines:
            newvar, curr_line = self.parse_variable(curr_line, known_ddts)
            valid_lines = newvar is not None
            if valid_lines:
                if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
                    dmsg = 'Adding {} to {}'
                    lname = newvar.get_prop_value('local_name')
                    run_env.logger.debug(dmsg.format(lname, self.title))
                # end if
                self.__variables.add_variable(newvar, run_env)
                # Check to see if we hit the end of the table
                valid_lines = not MetadataSection.header_start(curr_line)
            else:
                # We have a bad variable, see if we have more variables
                lname = MetadataSection.variable_start(curr_line, self.__pobj)
                valid_lines = lname is not None
                # end while
            # end if
        # end while

    def parse_variable(self, curr_line, known_ddts):
        """Parse a new metadata variable beginning on <curr_line>.
        The header line has the format [ <valid_fortran_symbol> ].
        """
        newvar = None
        var_ok = True # Set to False if an error is detected
        valid_line = ((curr_line is not None) and
                      (not MetadataSection.header_start(curr_line)) and
                      (not MetadataTable.table_start(curr_line)))
        if valid_line:
             # variable_start handles exception
            local_name = MetadataSection.variable_start(curr_line, self.__pobj)
        else:
            local_name = None
        # end if
        if local_name is None:
            # This is not a valid variable line, punt (should be end of table)
            valid_line = False
        # end if
        # Parse lines until invalid line is found
        # NB: Header variables cannot have embedded blank lines
        if valid_line:
            var_props = {}
            var_props['local_name'] = local_name
            # Grab context that points at beginning of definition
            context = ParseContext(context=self.__pobj)
        else:
            var_props = None
        # end if
        while valid_line:
            curr_line, _ = self.__pobj.next_line()
            valid_line = ((curr_line is not None) and
                          (not MetadataSection.header_start(curr_line)) and
                          (not MetadataTable.table_start(curr_line)) and
                          (MetadataSection.variable_start(curr_line,
                                                          self.__pobj) is None))
            # A valid line may have multiple properties (separated by '|')
            if valid_line:
                properties = _parse_config_line(curr_line, self.__pobj)
                for prop in properties:
                    pname = prop[0].strip().lower()
                    pval_str = prop[1].strip()
                    if ((pname == 'type') and
                        (not check_fortran_intrinsic(pval_str, error=False))):
                        if pval_str in known_ddts:
                            pval = pval_str
                            pname = 'ddt_type'
                        else:
                            errmsg = "Unknown DDT type, {}".format(pval_str)
                            self.__pobj.add_syntax_err(errmsg)
                            self.__section_valid = False
                            var_ok = False
                        # end if
                    else:
                        # Make sure this is a match
                        check_prop = Var.get_prop(pname)
                        if check_prop is not None:
                            pval = check_prop.valid_value(pval_str)
                        else:
                            emsg = "variable property name"
                            self.__pobj.add_syntax_err(emsg, token=pname)
                            self.__section_valid = False
                            var_ok = False
                        # end if
                        if pval is None:
                            errmsg = "'{}' property value"
                            self.__pobj.add_syntax_err(errmsg.format(pname),
                                                       token=pval_str)
                            self.__section_valid = False
                            var_ok = False
                        # end if
                    # end if
                    if var_ok:
                        # If we get this far, we have a valid property.
                        # Special case for dimensions, turn them into ranges
                        if pname == 'dimensions':
                            porig = pval
                            pval = list()
                            for dim in porig:
                                if ':' in dim:
                                    pval.append(dim)
                                else:
                                    cone_str = 'ccpp_constant_one:{}'
                                    pval.append(cone_str.format(dim))
                                # end if
                            # end for
                        # end if
                        # Add the property to our Var dictionary
                        var_props[pname] = pval
                    # end if
                # end for
            # end if
        # end while
        if var_ok and (var_props is not None):
            # Check for array reference
            sub_name = MetadataSection.check_array_reference(local_name,
                                                             var_props, context)
            if sub_name:
                var_props['local_name'] = sub_name
            # end if (else just leave the local name alone)
            try:
                newvar = Var(var_props, self, self.run_env, context=context)
            except CCPPError as verr:
                self.__pobj.add_syntax_err(verr, skip_context=True)
                var_ok = False
                self.__section_valid = False
            # end try
        # No else, will return None for newvar
        # end if
        return newvar, curr_line

    @staticmethod
    def check_array_reference(local_name, var_dict, context):
        """If <local_name> is an array reference, check it against
        the 'dimensions' property in <var_dict>. If <local_name> is an
        array reference, return it with the colons filled in with the
        dictionary dimensions, otherwise, return None.
        >>> MetadataSection.check_array_reference('foo', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta'))

        >>> MetadataSection.check_array_reference('foo', {}, ParseContext(filename='foo.meta'))

        >>> MetadataSection.check_array_reference('foo(qux', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
        ParseInternalError: Invalid scalar reference, foo(qux, in foo.meta
        >>> MetadataSection.check_array_reference('foo(qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
        ParseInternalError: foo has rank 2 but foo(qux) has 0, in foo.meta
        >>> MetadataSection.check_array_reference('foo(:,qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
        ParseInternalError: foo has rank 2 but foo(:,qux) has 1, in foo.meta
        >>> MetadataSection.check_array_reference('foo(:,qux)', {'foo':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
        ParseInternalError: Missing variable dimensions, foo(:,qux), in foo.meta
        >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
        ParseInternalError: foo has rank 1 but foo(:,:,qux) has 2, in foo.meta
        >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar','ccpp_constant_one:baz']}, ParseContext(filename='foo.meta'))
        'foo(:, :, qux)'
        """
        retval = None
        if check_fortran_id(local_name, var_dict, False) is None:
            rmatch = FORTRAN_SCALAR_REF_RE.match(local_name)
            if rmatch is None:
                errmsg = 'Invalid scalar reference, {}{}'
                ctx = context_string(context)
                raise ParseInternalError(errmsg.format(local_name, ctx))
            # end if
            rname = rmatch.group(1)
            rdims = [x.strip() for x in rmatch.group(2).split(',')]
            if 'dimensions' in var_dict:
                vdims = [x.strip() for x in var_dict['dimensions']]
            else:
                errmsg = 'Missing variable dimensions, {}{}'
                ctx = context_string(context)
                raise ParseInternalError(errmsg.format(local_name, ctx))
            # end if
            colon_rank = len([x for x in rdims if x == ':'])
            if colon_rank != len(vdims):
                errmsg = '{} has rank {} but {} has {}{}'
                ctx = context_string(context)
                raise ParseInternalError(errmsg.format(rname, len(vdims),
                                                       local_name, colon_rank,
                                                       ctx))
            # end if
            sub_dims = list()
            sindex = 0
            for rind in rdims:
                if rind == ':':
                    sub_dims.append(':')
                    sindex += 1
                else:
                    sub_dims.append(rind)
                # end if
            # end for
            retval = '{}({})'.format(rname, ', '.join(sub_dims))
        # end if
        return retval

    def variable_list(self, std_vars=True, loop_vars=True, consts=True):
        """Return an ordered list of the header's variables"""
        return self.__variables.variable_list(recursive=False,
                                              std_vars=std_vars,
                                              loop_vars=loop_vars,
                                              consts=consts)

    def find_variable(self, std_name, use_local_name=False):
        """Find a variable in this header's dictionary"""
        var = None
        if use_local_name:
            var = self.__variables.find_local_name(std_name)
        else:
            var = self.__variables.find_variable(std_name, any_scope=False)
        # end if
        return var

    def convert_dims_to_standard_names(self, var, logger=None, context=None):
        """Convert the dimension elements in <var> to standard names by
        by using other variables in this header.
        """
        std_dims = list()
        vdims = var.get_dimensions()
        # Check for bad dimensions
        if vdims is None:
            vdim_prop = var.get_prop_value('dimensions').strip()
            if vdim_prop[0] == '(':
                vdim_prop = vdim_prop[1:]
            # end if
            if vdim_prop[-1] == ')':
                vdim_prop = vdim_prop[0:-1]
            # end if
            vdim_strs = [x.strip() for x in vdim_prop.split(',')]
            lname = var.get_prop_value('local_name')
            ctx = context_string(var.context)
            sep = ''
            errstr = "{}{}: Invalid dimension, '{}'{}"
            errmsg = ''
            for vdim in vdim_strs:
                if not check_fortran_id(vdim, None, False):
                    errmsg += errstr.format(sep, lname, vdim, ctx)
                    sep = '\n'
                # end if
            # end for
            raise CCPPError("{}".format(errmsg))
        # end if
        for dim in vdims:
            std_dim = list()
            if ':' not in dim:
                # Metadata dimensions always have an explicit start
                var_one = CCPP_CONSTANT_VARS.find_local_name('1')
                if var_one is not None:
                    std = var_one.get_prop_value('standard_name')
                    std_dim.append(std)
                # end if
            # end if
            for item in dim.split(':'):
                try:
                    _ = int(item)
                    dvar = CCPP_CONSTANT_VARS.find_local_name(item)
                    if dvar is not None:
                        # If this integer value is a CCPP standard int, use that
                        dname = dvar.get_prop_value('standard_name')
                    else:
                        # Some non-standard integer value
                        dname = item
                    # end if
                except ValueError:
                    # Not an integer, try to find the standard_name
                    if not item:
                        # Naked colons are okay
                        dname = ''
                    else:
                        dvar = self.find_variable(item, use_local_name=True)
                        if dvar is not None:
                            dname = dvar.get_prop_value('standard_name')
                        else:
                            dname = None
                        # end if
                    # end if
                    if dname is None:
                        errmsg = "Unknown dimension element, {}, in {}{}"
                        std = var.get_prop_value('local_name')
                        ctx = context_string(context)
                        if logger is not None:
                            errmsg = "ERROR: " + errmsg
                            logger.error(errmsg.format(item, std, ctx))
                            dname = unique_standard_name()
                        else:
                            raise CCPPError(errmsg.format(item, std, ctx))
                        # end if
                    # end if
                # end try
                if dname is not None:
                    std_dim.append(dname)
                else:
                    std_dim = None
                    break
                # end if
            # end for
            if std_dim is not None:
                std_dims.append(':'.join(std_dim))
            else:
                break
            # end if
        # end for

        return std_dims

    def prop_list(self, prop_name):
        """Return list of <prop_name> values for this scheme's arguments"""
        return self.__variables.prop_list(prop_name)

    def section_table_mismatch(self, table_title, table_type):
        """Return  an error string if this arg table does not match its
        metadata table parent. If they match , return an empty string."""
        mismatch = ""
        # The header type must match its table's type
        if self.header_type is None:
            mstr = "Invalid section type, 'None'"
            mismatch += mstr.format(self.header_type, table_type)
        elif table_type != self.header_type:
            mstr = "Section type, '{}', does not match table type, '{}'"
            mismatch += mstr.format(self.header_type, table_type)
        # end if
        if self.header_type == SCHEME_HEADER_TYPE:
            # For schemes, strip off the scheme function phase (e.g., _init)
            sect_func, _, _ = CCPP_STATE_MACH.function_match(self.title)
        else:
            sect_func = self.title
        # end if
        # The Fortran parser cannot tell a scheme from a host subroutine
        # Detect this and adjust
        if sect_func is None:
            sect_func = self.title
        # end if
        # The header name (minus phase) must match its table's name
        if table_title != sect_func:
            if mismatch:
                mismatch += '\n'
            # end if
            mstr = "Section name, '{}', does not match table title, '{}'"
            mismatch += mstr.format(self.title, table_title)
        # end if
        if mismatch:
            mismatch += context_string(self.__pobj)
        # end if
        return mismatch

    @staticmethod
    def variable_start(line, pobj):
        """Return variable name if <line> is an interface metadata table header
        """
        if line is None:
            match = None
        else:
            match = MetadataSection.__var_start.match(line)
            if match is None:
                match = MetadataSection.__vref_start.match(line)
                if match is not None:
                    name = match.group(1)+'('+match.group(2)+')'
                # end if
            else:
                name = match.group(1)
            # end if
        # end if
        if match is not None:
            if not MetadataSection.is_scalar_reference(name):
                pobj.add_syntax_err("local variable name", token=name)
                name = None
            # end if
        else:
            name = None
        # end if
        return name

    def write_to_file(self, filename, append=False):
        """Write this metadata table to <filename>. If <append> is True,
        append this table to the end of <filename>, otherwise, create
        or truncate the file."""
        if append:
            oflag = 'a'
        else:
            oflag = 'w'
        # end if
        with open(filename, oflag) as mfile:
            mfile.write("[ccpp-arg-table]")
            mfile.write("  name = {}".format(self.title))
            mfile.write("  type = {}".format(self.header_type))
            for var in self.variable_list():
                var.write_metadata(mfile)
            # end for
        # end with

    def to_html(self, outdir, props):
        """Write html file for metadata section and return filename.
        Skip metadata sections without variables"""
        if not self.__variables.variable_list():
            return None
        # Write table header
        header = f"<tr>"
        for prop in props:
            header += f"<th>{prop}</th>".format(prop=prop)
        header += f"</tr>\n"
        # Write table contents, one row per variable
        contents = ""
        for var in self.__variables.variable_list():
            row = f"<tr>"
            for prop in props:
                value = var.get_prop_value(prop)
                # Pretty-print for dimensions
                if prop == 'dimensions':
                    value = '(' + ', '.join(value) + ')'
                elif value is None:
                    value = f"n/a"
                row += f"<td>{value}</td>".format(value=value)
            row += f"</tr>\n"
            contents += row
        filename = os.path.join(outdir, self.title + '.html')
        with open(filename,"w") as f:
            f.writelines(self.__html_template__.format(title=self.title + ' argument table',
                                                       header=header, contents=contents))
        return filename

    def __repr__(self):
        base = super().__repr__()
        pind = base.find(' object ')
        if pind >= 0:
            pre = base[0:pind]
        else:
            pre = '<MetadataSection'
        # end if
        bind = base.find('at 0x')
        if bind >= 0:
            post = base[bind:]
        else:
            post = '>'
        # end if
        return '{} {} / {} {}'.format(pre, self.module, self.title, post)

    def __del__(self):
        try:
            del self.__variables
        except AttributeError:
            pass

    def start_context(self, with_comma=True, nodir=True):
        """Return a context string for the beginning of the table"""
        return context_string(self.__start_context,
                              with_comma=with_comma, nodir=nodir)

    @property
    def title(self):
        """Return the name of the metadata arg_table"""
        return self.__section_title

    @property
    def module(self):
        """Return the module name for this header (if it exists)"""
        return self.__module_name

    @property
    def header_type(self):
        """Return the type of structure this header documents"""
        return self.__header_type

    @property
    def process_type(self):
        """Return the type of physical process this header documents"""
        return self.__process_type

    @property
    def has_variables(self):
        """Convenience function for finding empty headers"""
        return self.__variables

    @property
    def run_env(self):
        """Return this section's CCPPFrameworkEnv object"""
        return self.__run_env

    @property
    def valid(self):
        """Return True iff we did not encounter an error creating
        this section"""
        return self.__section_valid

    def __str__(self):
        '''Print string for MetadataSection objects'''
        return "<{} {}>".format(self.__class__.__name__, self.title)

    @classmethod
    def header_start(cls, line):
        """Return True iff <line> is a Metadata section header (ccpp-arg-table).
        """
        if (line is None) or blank_metadata_line(line):
            match = None
        else:
            match = cls.__header_start.match(line)
        # end if
        return match is not None

    @staticmethod
    def is_scalar_reference(test_val):
        """Return True iff <test_val> refers to a Fortran scalar."""
        return check_fortran_ref(test_val, None, False) is not None

########################################################################

if __name__ == "__main__":
# pylint: enable=ungrouped-imports
    import doctest
    import sys
# pylint: disable=ungrouped-imports
    from framework_env import CCPPFrameworkEnv
    _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, {'host_files':'',
                                             'scheme_files':'',
                                             'suites':''})
    fail, _ = doctest.testmod()
    sys.exit(fail)