#!/usr/bin/env python2.7

import sys

import json as json

import ast

import jsontree

import asttokens

from xml.sax.saxutils import quoteattr

def PrintUsage():

sys.stderr.write("""

Usage:

parse_python.py <file>

""")

exit(1)

def read_file_to_string(filename):

f = open(filename, 'rt')

s = f.read()

f.close()

return s

def parse_file(filename):

tree = asttokens.ASTTokens(read_file_to_string(filename), parse=True).tree

json_tree = []

def gen_identifier(identifier, node_type = 'identifier', node=None):

pos = len(json_tree)

json_node = {}

json_tree.append(json_node)

json_node['type'] = node_type

json_node['value'] = identifier

json_pos_and_length = extract_pos_and_length(node, node)

json_node['pos'] = str(json_pos_and_length[0])

json_node['length'] = str(json_pos_and_length[1])

return pos

def traverse_list(l, node_type = 'list', node = None):

pos = len(json_tree)

json_node = {}

json_tree.append(json_node)

json_node['type'] = node_type

if (len(l) > 0):

json_pos_and_length = extract_pos_and_length(l[0], l[-1])

else:

json_pos_and_length = extract_pos_and_length(node, node)

json_node['pos'] = str(json_pos_and_length[0])

json_node['length'] = str(json_pos_and_length[1])

children = []

for item in l:

children.append(traverse(item))

if (len(children) != 0):

json_node['children'] = children

return pos

def traverse(node):

pos = len(json_tree)

json_node = {}

json_tree.append(json_node)

json_node['type'] = type(node).__name__

json_pos_and_length = extract_pos_and_length(node, node)

json_node['pos'] = str(json_pos_and_length[0])

json_node['length'] = str(json_pos_and_length[1])

children = []

if isinstance(node, ast.Name):

json_node['value'] = node.id

elif isinstance(node, ast.Num):

json_node['value'] = unicode(node.n)

elif isinstance(node, ast.Str):

json_node['value'] = node.s.decode('utf-8')

elif isinstance(node, ast.alias):

json_node['value'] = unicode(node.name)

if node.asname:

children.append(gen_identifier(node.asname, node = node))

elif isinstance(node, ast.FunctionDef):

json_node['value'] = unicode(node.name)

elif isinstance(node, ast.ClassDef):

json_node['value'] = unicode(node.name)

elif isinstance(node, ast.ImportFrom):

if node.module:

json_node['value'] = unicode(node.module)

elif isinstance(node, ast.Global):

for n in node.names:

children.append(gen_identifier(n, node = node))

elif isinstance(node, ast.keyword):

json_node['value'] = unicode(node.arg)

# Process children.

if isinstance(node, ast.For):

children.append(traverse(node.target))

children.append(traverse(node.iter))

children.append(traverse_list(node.body, 'body', node))

if node.orelse:

children.append(traverse_list(node.orelse, 'orelse', node))

elif isinstance(node, ast.If) or isinstance(node, ast.While):

children.append(traverse(node.test))

children.append(traverse_list(node.body, 'body', node))

if node.orelse:

children.append(traverse_list(node.orelse, 'orelse', node))

elif isinstance(node, ast.With):

children.append(traverse(node.context_expr))

if node.optional_vars:

children.append(traverse(node.optional_vars))

children.append(traverse_list(node.body, 'body', node))

elif isinstance(node, ast.TryExcept):

children.append(traverse_list(node.body, 'body', node))

children.append(traverse_list(node.handlers, 'handlers', node))

if node.orelse:

children.append(traverse_list(node.orelse, 'orelse', node))

elif isinstance(node, ast.TryFinally):

children.append(traverse_list(node.body, 'body', node))

children.append(traverse_list(node.finalbody, 'finalbody', node))

elif isinstance(node, ast.arguments):

children.append(traverse_list(node.args, 'args', node))

children.append(traverse_list(node.defaults, 'defaults', node))

if node.vararg:

children.append(gen_identifier(node.vararg, 'vararg', node))

if node.kwarg:

children.append(gen_identifier(node.kwarg, 'kwarg', node))

elif isinstance(node, ast.ExceptHandler):

if node.type:

children.append(traverse_list([node.type], 'type', node))

if node.name:

children.append(traverse_list([node.name], 'name', node))

children.append(traverse_list(node.body, 'body', node))

elif isinstance(node, ast.ClassDef):

children.append(traverse_list(node.bases, 'bases', node))

children.append(traverse_list(node.body, 'body', node))

children.append(traverse_list(node.decorator_list, 'decorator_list', node))

elif isinstance(node, ast.FunctionDef):

children.append(traverse(node.args))

children.append(traverse_list(node.body, 'body', node))

children.append(traverse_list(node.decorator_list, 'decorator_list', node))

else:

# Default handling: iterate over children.

for child in ast.iter_child_nodes(node):

if isinstance(child, ast.expr_context) or isinstance(child, ast.operator) or isinstance(child, ast.boolop) or isinstance(child, ast.unaryop) or isinstance(child, ast.cmpop):

# Directly include expr_context, and operators into the type instead of creating a child.

json_node['type'] = json_node['type'] + type(child).__name__

else:

children.append(traverse(child))

if isinstance(node, ast.Attribute):

children.append(gen_identifier(node.attr, 'attr', node))

if (len(children) != 0):

json_node['children'] = children

return pos

def extract_pos_and_length(node, other_node):

try:

return [node.startpos, other_node.endpos - node.startpos]

except:

try:

return [node.first_token.startpos, other_node.last_token.endpos - node.first_token.startpos]

except:

pass

return [-1, -1]

traverse(tree)

return json.dumps(json_tree, separators=(',', ':'), ensure_ascii=False)

def write(i, indent_level = 0):

global lines

indent_string = ' '

indent = indent_string * indent_level

node = tree[i]

label_attr = ' label=' + quoteEscape(node["value"]) if node['value'] else ''

lines.append(indent + '<tree type="' + node['type'] + '"' + label_attr + ' pos="' + str(node['pos']) + '" length="' + str(node['length']) + '">')

for child in node["children"]:

write(int(child), indent_level + 1)

lines.append(indent + '</tree>')

def quoteEscape(x):

return quoteattr(x);

if __name__ == "__main__":

try:

text = open(sys.argv[1], "r+").read()

json_file = parse_file(sys.argv[1])

tree = jsontree.JSONTreeDecoder().decode(json_file)

x = tree[0]

x['length'] = len(text)

lines = []

lines.append("<root>")

lines.append("<context></context>")

write(0)

lines.append("</root>")

print('\n'.join(lines))

except (UnicodeEncodeError, UnicodeDecodeError):

pass