# EasyScout XML Backup Parser

# Copyright (c) 2004 by Ryan Kirkpatrick (scouts@rkirkpat.net). 
#   All rights reserved.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING.  If not, write to
# the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

# Documentation.
"""
This is a parser for the XML format of the EasyScout backup files. Given a file
handle, it reads in XML data and parses it into a dictionary data structure. All
tags are keys at the different levels in the dictionary, while the values can
be another dictionary at a lower level, or a array of dictionaries if the tag
can appear multiple times at the same level. Functions:
  Parse(file)       -- XML EasyScout backup file parser.
  ReqPreParse(file) -- Pre-filters XML requirements files for Parse().

The structure of the dictionary for the EasyScout data is as follows, where key
values have been omitted where they only contain data (and not more keys):
  {'troop': {'created', 'email', 'name', 'owner', 'password', 'readonly',
             'leader': [ { 'address', 'birthdate', 'email', 'firstname',
                           'lastname', 'middlename', 'notes', 'phone',
                           'position', 'password' } ]
             'scout': [ { 'address', 'birthdate', 'contact', 'currentrank',
                          'email', 'emergency', 'firstname', 'lastname', 'patrol',
                          'medicalnotes', 'middlename', 'notes', 'password', 'phone'
                          'activity':[{'date', 'description', 'name'}, ...],
                          'leadership':[{'start', 'end', 'name'}, ...],
                          'meritbadges':[{'conselor','earned','name','required'}, ...],
                          'rank':[{'awarded', 'name',
                                   'requirement':[{'index','name','passed'}, ...]
                                  }, ...]
                        }, ...]
            }
  }

The structure of the dictionary for EasyScout rank/requirements data is as follows,
where key values have been omitted where they only contain data (and not more keys):
  {'requirements': {'rank': [ { 'name', 'meritbadgecount', 'requiredcount',
                                'requirement':[{'index','name'}, ...],
                              }, ... ]
                   }
  }

Note: This conversion is not simple and takes a moderate amount of time. Effort
has been made to optimize the parsing, but XML is bulky and there is a certain
amount of effort required that can not be optimized out. On a PII-333MHz system,
the parsing speed is about 2-3 seconds per scout.

Note: EasyScout data values exported, but appear to have no meaning:
  Leader password field.
"""
__credits__ = 'Copyright 2004 (c) by Ryan Kirkpatrick. All Rights Reserved'
__author__ = 'Ryan Kirkpatrick <scouts@rkirkpat.net>'
__date__ = '2004/01/05'
__version__ = 'ESPyUtils 1.0'

# Needed modules.
import re
import utils
from tempfile import TemporaryFile

# Pre-parse requirement files into XML that Parse() can stomach better.
def ReqPreParse(file):
    """
    Pre-parses an EasyScout requirements XML lookup file so that Parse() can
    stomach it better.
      file -- A file open for reading that comtains the XML data.

    Returns a file open for reading that contains the pre-parsed XML data.
    """
    
    # Open a temp file to write pre-parsed data to.
    new = TemporaryFile()
    utils.Verbose("  Pre-parsing XML rank data...")

    # Read and parse the given data.
    line = file.readline()
    while line:
        # Break rank name out to its own tag.
        rank_name = re.search('<Rank name="([^"]+)">',line)
        if rank_name:
            line = re.sub('<Rank[^>]+>','<Rank><Name>%s</Name>' % rank_name.groups()[0],line)

        # Break requirement index out to its own tag.
        req_index = re.search('<Requirement index="([^"]+)">',line)
        if req_index:
            line = re.sub('<Requirement[^>]+>','<Requirement><Index>%s</Index>' % req_index.groups()[0],line)

        # Wrap the requirement name in its own tag as well.
        line = re.sub('</Index>','</Index><Name>',line)
        line = re.sub('</Requirement>','</Name></Requirement>',line)

        # Write out the line and get the next.
        new.write(line)
        line = file.readline()

    # We are done, seek to the start of the new file and return.
    utils.Verbose()
    new.seek(0)
    return new

# Parse XML into a dictionary.
def Parse(file):
    """
    Parses an EasyScout XML backup file into a dictionary data structure. Parameters:
      file -- A file open for reading that contains the XML data.
    
    Returns a dictionary data structure of the XML data.
    """

    # Tag format, must start with a letter and contain only alphanumerics (plus a few
    #   others).
    re_tag = re.compile('[A-Za-z][-_.:a-zA-Z0-9]*')
    re_tag_close = re.compile('</[^>]*>')

    # Tags that appear more than once at the same level, and are never found
    #   at the bottom level (i.e. always have tags within them).
    multi_tags = ['scout','meritbadge','rank','requirement','activity','leadership','leader']

    # Function to build dictionary references.
    def DictRef(multi_tags, tag_list):
        # Step through each level of tags.
        dict_entry = ''
        for tag in tag_list[:-1]:
            if tag in multi_tags:
                # When dealing with arrays of tag values, always use the last one.
                dict_entry += "['%s'][-1]" % tag
            else:
                # No array present, just address directly.
                dict_entry += "['%s']" % tag

        # Return the final dictionary reference.
        return dict_entry

    # Function to read XML data from file.
    def Read(re_tag_close,file,text):
        # Read data from the file until there is at least one closing tag present
        #   in the string of read data, or there is no data to read.
        while not re_tag_close.search(text):
            new = file.readline()
            if new == "": break
            text += new

        # Got enough data for now, return.
        return text        

    # Now, parse the XML data.
    utils.Verbose("  Parsing XML data...\n",0)
    dict = {}
    tag_list = []
    text = ""; text = Read(re_tag_close,file,text)
    while text:
        if text[0].isspace():
            # Ignore white space.
            text = text[1:]
            continue

        elif text.startswith('</'):
            # End off an old tag.

            # Build a reference into the dictionary where the current tag's data should be.
            dict_entry = DictRef(multi_tags,tag_list)
            dict_ref = eval ("dict%s" % dict_entry)

            # Check that the tag's value is meaningful, otherwise make it so.
            if dict_ref[tag_list[-1]] == {}:
                # An empty dictionary, then this was an empty tag, which means empty string.
                dict_ref[tag_list[-1]] = ""

            # We are now done with the tag.
            if len(tag_list) == 3 and tag_list[-1] in ("name","lastname"):
                utils.Verbose("    Parsing data for '%s'...\n" % dict_ref[tag_list[-1]],0)
            tag_list.pop()
            
            # Remove the tag from the XML data to process.
            text = text[text.find('>')+1:]
            
        elif text.startswith('<'):
            # Begin a new tag.
            tagLoc = re_tag.search(text)
            if not tagLoc:
                # No valid tag found, skip this tag looking item.
                text = text[text.find('>')+1:]
            elif text[1:tagLoc.start()] and not text[1:tagLoc.start()].isspace():
                # Tags can't start with spaces, skip this tag looking item.
                text = text[text.find('>')+1:]
            else:
                # Add the tag to the list of tag we are current working with.
                tag_list.append(text[tagLoc.start():tagLoc.end()])

                # Ensure the tag is in lower case.
                tag_list[-1] = tag_list[-1].lower()

                # Remove the tag from the XML data to process.
                text = text[text.find('>')+1:]

                # Build a reference into the dictionary where this tag's data should go.
                dict_entry = DictRef(multi_tags,tag_list)
                dict_ref = eval ("dict%s" % dict_entry)

                # Does this tag already exist in the dictionary?
                stat = eval("dict%s.has_key('%s')" % (dict_entry,tag_list[-1]))
                if not stat:
                    # No, create it as appropriate.
                    if tag_list[-1] in multi_tags:
                        # Tag appears multiple times at the same level, its value is an
                        #   array of dictionaries.
                        dict_ref[tag_list[-1]] = [{}]
                    else:
                        # Tag appears once at this level, its value is a dictionary.
                        dict_ref[tag_list[-1]] = {}
                elif stat and tag_list[-1] in multi_tags:
                    # Yes, and it appears multiple times at the same level, so append
                    #   another dictionary to the array.
                    dict_ref[tag_list[-1]].append({})
        else:
            # Gather tag values. First build a reference into the dictionary to our
            #   current tag.
            dict_entry = DictRef(multi_tags,tag_list)
            dict_ref = eval ("dict%s" % dict_entry)

            # Find the end of data for the current tag, and save it as the value of
            #  our current tag.
            pos = text.lower().find('</%s>' % tag_list[-1])
            dict_ref[tag_list[-1]] = "%s" % text[:pos]

            # Remove the value and the closing tag from the XML data to process.
            text = text[pos:]

        # Read more data if needed from the XML file.
        text = Read(re_tag_close,file,text)

    # We are done, return the dictionary we built.
    return dict

# Default behavior.
if __name__ == '__main__':
    # Get program name, and set defaults.
    import sys
    name = sys.argv.pop(0)
    printit = 0
    req = 0
    filename = ""

    # What the command should look like.
    def Usage():
        print "Usage: xml.py [-v] [-p] [-r] {file}"
        sys.exit(1)

    # Were there options? Handle them.
    for argv in sys.argv:
        # Verbosity? Set if so.
        if argv == "-v": utils.verbose = 1

        # Print output? Otherwise we only validate.
        elif argv == "-p": printit = 1

        # Is this requirements data? If so, pre-parsing needed.
        elif argv == "-r": req = 1

        # Otherwise, this has to be a filename.
        else: filename = argv

    # Did we get a valid filename? Give help if not.
    if not filename: Usage()
    try: file = open(filename,"r")
    except: Usage()

    # Pre-parse the file as needed.
    utils.Verbose("Validating EasyScout backup file '%s'...\n" % filename,0)
    if req: file = ReqPreParse(file)

    # Parse the file.
    data = Parse(file)
    utils.Verbose("Parsing complete and okay.\n",0)

    # Are we to print the output?
    if printit:
        # Yes, pretty print it.
        import pprint
        pprint.pprint(data)
