shithub: openh264

ref: 542fd232cc05f86b4de26516d71a5c9ec7d0e55a
dir: /docs/doxygen2rst.py/

View raw version
from __future__ import print_function
import re, sys, os, time, glob, errno, tempfile, binascii, subprocess, shutil
from lxml import etree
from optparse import OptionParser
import textwrap
import string

VERSION = '0.1'
__all__ = ['DoxyGen2RST']
LINE_BREAKER = "\n"
MAX_COLUMN = 80

def is_valid_uuid(uuid_string):
    uuid4hex = re.compile('[0-9a-f]{32}\Z', re.I)
    return uuid4hex.match(uuid_string) != None

def get_page(refid):
    fields = refid.split("_")
    if(is_valid_uuid(fields[-1][-32:])):
        return ["_".join(fields[0:-1]), fields[-1]]
    return [refid, None]

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def _glob(path, *exts):
    path = os.path.join(path, "*") if os.path.isdir(path) else path + "*"
    return [f for files in [glob.glob(path + ext) for ext in exts] for f in files]

class DoxyGen2RST(object):
    """
    Customize the Doxygen XML output into RST format, then it can
    be translated into all formats with the unified user interface.
    The Doxygen output itself is too verbose and not hard to be
    organized for a good documentation.
    """

    def __init__(self,
                 src,
                 dst,
                 missing_filename = "missing.rst",
                 is_github = False,
                 enable_uml = True,
                 github_ext = ""):
        self.doxy_output_dir = os.path.join(src, "_doxygen", "xml")
        self.output_dir = dst
        self.rst_dir = src
        self.enable_uml = enable_uml
        mkdir_p(dst)
        self.is_github = is_github
        if(is_github):
            self.page_ext = github_ext
            self.anchor_prefix = "wiki-"
        else:
            self.anchor_prefix = ""
            self.page_ext = ".html"
        self.filter = ["*.rst", "*.rest"]
        self.re_doxy = "<doxygen2rst\s(\S*)=(\S*)>(.*?)</doxygen2rst>"
        self.index_root = etree.parse(os.path.join(self.doxy_output_dir, "index.xml")).getroot()
        self.references = {}
        self.missed_types_structs = {}
        self.name_refid_map = {}
        self.build_references()
        self.page_references = {}
        self.missing_filename = missing_filename
        self.temp_uml_path = os.path.join(tempfile.gettempdir(), "uml_" + binascii.b2a_hex(os.urandom(15)))
        if os.path.exists(self.temp_uml_path):
            shutil.rmtree(self.temp_uml_path)
        os.mkdir(self.temp_uml_path)

    def _find_ref_id(self, kind, name):
        #print("_find_ref_id, %s - %s" %(kind, name))
        if(kind == "function"):
            for comp in self.index_root.iter("member"):
                if(comp.attrib["kind"].lower() == kind.lower() and
                   comp.findtext("name").lower() == name.lower()):
                    return (comp.attrib["refid"])
            pass
        else:
            for comp in self.index_root.iter("compound"):
                if(comp.attrib["kind"].lower() == kind.lower() and
                   comp.findtext("name").lower() == name.lower()):
                   return comp.attrib["refid"]
        return None

    def strip_title_ref(self, text):
        table = string.maketrans("","")
        retstr = text.translate(table, string.punctuation)
        words = retstr.split()
        retstr = "-".join(words)
        return retstr.lower()

    def build_references(self):
        for file in _glob(self.rst_dir, *self.filter):
            filename = os.path.basename(file)
            fin = open(file,'r')
            content = fin.read()
            it = re.finditer(self.re_doxy, content, re.DOTALL)
            for m in it:
                ref_id = self._find_ref_id(m.groups()[0], m.groups()[1])
                if(ref_id is None):
                    #print("Reference is NOT found for: %s=%s" % (m.groups()[0], m.groups()[1]))
                    continue
                page_name = os.path.splitext(filename)[0]
                title_ref = self.strip_title_ref(m.groups()[2])
                self.references[ref_id] = [m.groups()[0], m.groups()[1], page_name, filename, title_ref]
                self.name_refid_map[m.groups()[1]] = ref_id
            fin.close()
        #print(self.references)

    def call_plantuml(self):
        if(not self.enable_uml):
            return

        java_bin = os.path.join(os.environ['JAVA_HOME'], "bin", "java")
        output_path = os.path.abspath(os.path.join(self.output_dir, "images"))
        cmds = ["\"" + java_bin + "\"", "-jar", "plantuml.jar", self.temp_uml_path + "/", "-o", output_path]
        print(" ".join(cmds))
        os.system(" ".join(cmds))
        shutil.rmtree(self.temp_uml_path)

    def _build_uml(self, uml_name, content):
        uml_path = os.path.join(self.temp_uml_path, uml_name + ".txt")
        fuml = open(uml_path, "w+")
        fuml.write("@startuml\n")
        fuml.write(content)
        fuml.write("\n@enduml\n")
        fuml.close()
        return ".. image:: images/" + uml_name + ".png" + LINE_BREAKER

    def _build(self, m):
        retstr = ""
        if(m.groups()[0] == "uml"):
            retstr = self._build_uml(m.groups()[1], m.groups()[2])
        elif(m.groups()[0] == "link"):
            link = m.groups()[1] + self.page_ext
            retstr = ("`%s <%s>`_" % (m.groups()[2], link))
        else:
            if(m.groups()[0] != "function"):
                retstr +=  self._build_title(m.groups()[2])
            retstr += self.convert_doxy(m.groups()[0], m.groups()[1])

        return retstr

    def generate(self):
        for file in _glob(self.rst_dir, *self.filter):
            filename = os.path.basename(file)
            fin = open(file,'r')
            input_txt = fin.read()
            fin.close()

            output_txt = re.sub(self.re_doxy, self._build, input_txt, 0, re.DOTALL)
            output_txt  += self._build_page_ref_notes()

            fout = open(os.path.join(self.output_dir, filename), 'w+')
            fout.write(output_txt)
            fout.close()
            #print("%s --- %s" %( file, os.path.join(self.output_dir, filename)))

        self._build_missed_types_and_structs()
        self.call_plantuml()

    def make_para_title(self, title,  indent = 4):
        retstr = LINE_BREAKER
        if(title):
            retstr += "".ljust(indent, " ") + "| **" + title + "**" +  LINE_BREAKER
        return retstr

    def _build_title(self, title, flag = '=', ref = None):
        retstr = LINE_BREAKER
        if(ref):
            retstr += ".. _ref-" + ref + ":" + LINE_BREAKER + LINE_BREAKER
        retstr += title + LINE_BREAKER
        retstr += "".ljust(20, flag) + LINE_BREAKER
        retstr += LINE_BREAKER
        return retstr

    def _build_ref(self, node):
        text = node.text.strip()
        retstr = ""
        target = '`' + text + '`'
        retstr += target + "_ "
        if target in self.page_references:
            reflink = self.page_references[target]
            print("Link already added: %s == %s" % (reflink[0], node.attrib["refid"]))
            assert(reflink[0] == node.attrib["refid"])
            pass
        else:
            self.page_references[target] = (node.attrib["refid"], node.attrib["kindref"], text)

        return retstr

    def _build_code_block(self, node):
        retstr = "::" + LINE_BREAKER + LINE_BREAKER
        for codeline in node.iter("codeline"):
            retstr += "  "
            for phrases in codeline.iter("highlight"):
                if(phrases.text):
                    retstr += phrases.text.strip()
                for child in phrases:
                    if(child.text):
                        retstr += child.text.strip()
                    if(child.tag == "sp"):
                        retstr += " "
                    if(child.tag == "ref" and child.text):
                        #escape the reference in the code block
                        retstr += "" # self._build_ref(child)
                    if(child.tail):
                        retstr += child.tail.strip()
            retstr += LINE_BREAKER
        return retstr

    def _build_itemlist(self, node):
        retstr = ""
        for para in node:
            if(para.tag != "para"):
                continue
            if(para.text):
                retstr += para.text.strip()
            for child in para:
                if(child.tag == "ref" and child.text):
                    retstr += self._build_ref(child)
                if(child.tail):
                    retstr += child.tail.strip()

        return retstr

    def _build_itemizedlist(self, node):
        retstr = LINE_BREAKER
        if(node == None):
            return ""
        for item in node:
            if(item.tag != "listitem"):
                continue
            retstr += "    - " + self._build_itemlist(item)
            retstr += LINE_BREAKER
        return retstr

    def _build_verbatim(self, node):
        retstr = LINE_BREAKER
        if(node.text):
            lines = node.text.splitlines()
            print(lines[0])
            m = re.search("{plantuml}\s(\S*)", lines[0])
            if(m):
                uml_name = "uml_" + m.groups()[0]
                retstr += self._build_uml(uml_name, "\n".join(lines[1:]))
            else:
                retstr += "::" + LINE_BREAKER + LINE_BREAKER
                retstr += node.text

        return retstr

    def _build_para(self, para):
        retstr = ""
        no_new_line = False
        if(para.text):
            retstr += textwrap.fill(para.text.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
        for child in para:
            no_new_line = False
            if(child.tag == "simplesect"):
                for child_para in child:
                    if(child.attrib["kind"] == "return"):
                        return_str = self._build_para(child_para)
                        retstr += "".ljust(4, " ") + "| Return:" + LINE_BREAKER
                        for line in return_str.splitlines():
                            retstr += "".ljust(4, " ") + "| " + line + LINE_BREAKER
                    elif(child_para.tag == "title" and child_para.text):
                        lf.make_para_title(child_para.text.strip(), 4)
                    elif(child_para.tag == "para"): #for @see
                        retstr += self._build_para(child_para)
                    elif(child_para.text):
                        retstr += "".ljust(4, " ") + "| " + child_para.text.strip() + LINE_BREAKER
            if(child.tag == "preformatted"):
                retstr += "::" + LINE_BREAKER + LINE_BREAKER
                if(child.text):
                    for line in child.text.splitlines():
                        retstr += "  " + line + LINE_BREAKER
            if(child.tag == "ref" and child.text):
                retstr = retstr.rstrip('\n')
                retstr += " " + self._build_ref(child)
                no_new_line = True
            if(child.tag == "programlisting"):
                retstr += self._build_code_block(child)
            if(child.tag == "itemizedlist"):
                retstr += self._build_itemizedlist(child)
            if(child.tag == "verbatim"):
                retstr += self._build_verbatim(child)
            if(not no_new_line):
                retstr += LINE_BREAKER
            if(child.tail):
                retstr += textwrap.fill(child.tail.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
        return retstr

    def get_text(self, node):
        retstr = ""
        if(node == None):
            return ""
        for para in node:
            if(para.tag != "para"):
                continue
            retstr += self._build_para(para)

        return retstr

    def _find_text_ref(self, node):
        retstr = ""
        if(node.text):
            retstr += node.text.strip()
        for child in node:
            if(child.tag == "ref"):
                retstr += " " + self._build_ref(child) + " "
            if(child.tail):
                retstr += child.tail.strip()
        return retstr

    def _build_row_breaker(self, columns):
        retstr = "+"
        for column in columns:
            retstr += "".ljust(column, "-") + "+"
        return retstr + LINE_BREAKER

    def _wrap_cell(self, text, length = 30):
        newlines = []
        for line in text.splitlines():
            newlines.extend(textwrap.wrap(line, length))
        return newlines

    def _build_row(self, row, columns):
        retstr = ""
        row_lines = []
        max_line = 0
        for i in range(3):
            row_lines.append(row[i].splitlines())
            if(max_line < len(row_lines[i])):
                max_line = len(row_lines[i])

        for i in range(max_line):
            for j in range(3):
                retstr += "|"
                if(len(row_lines[j]) > i):
                    retstr += row_lines[j][i]
                    retstr += "".ljust(columns[j] - len(row_lines[j][i]), " ")
                else:
                    retstr += "".ljust(columns[j], " ")
            retstr += "|" + LINE_BREAKER
        return retstr

    def _build_table(self, rows):
        retstr = ""
        columns = [0, 0, 0]
        for row in rows:
            for i in range(3):
                for rowline in row[i].splitlines():
                    if(columns[i] < len(rowline) + 2):
                        columns[i] = len(rowline) + 2

        #columns[0] = 40 if(columns[0] > 40) else columns[0]
        #columns[1] = 40 if(columns[1] > 40) else columns[1]
        #columns[2] = MAX_COLUMN - columns[0] - columns[1]

        retstr += self._build_row_breaker(columns)
        for row in rows:
            retstr += self._build_row(row, columns)
            retstr += self._build_row_breaker(columns)
        return retstr;

    def build_param_list(self, params, paramdescs):
        retstr = ""
        param_descriptions = []
        for desc in paramdescs:
            param_descriptions.append(desc)

        rows = []
        rows.append(("Name", "Type", "Descritpion"))
        for param in params:
            declname = param.findtext("declname")
            paramdesc = None
            for desc in param_descriptions:
                paramname = desc.findtext("parameternamelist/parametername")
                if(paramname.lower() == declname.lower()):
                    paramdesc = desc.find("parameterdescription")
                    break
            decltype = self._find_text_ref(param.find("type"))
            rows.append((declname, decltype, self.get_text(paramdesc)))

        if(len(rows) > 1):
            retstr += self._build_table(rows)
        return retstr

    def _build_enum(self, member):
        enum_id = member.attrib["id"]
        file, tag = get_page(enum_id)
        retstr = self._build_title(member.findtext("name"), ref = tag)
        detail_node = self.get_desc_node(member)
        if(detail_node is not None):
            retstr += LINE_BREAKER
            retstr += self.get_text(detail_node)

        rows = []
        rows.append(("Name", "Initializer", "Descritpion"))
        for enumvalue in member.iter("enumvalue"):
            name = enumvalue.findtext("name")
            initializer = enumvalue.findtext("initializer")
            if(not initializer):
                initializer = ""
            desc = self.get_text(enumvalue.find("briefdescription"))
            desc += self.get_text(enumvalue.find("detaileddescription"))
            if(not desc):
                desc = ""
            rows.append((name, initializer, desc))

        if(len(rows) > 1):
            retstr += self._build_table(rows)
        return retstr


    def _build_struct(self, node):
        retstr = ""
        detail_node = self.get_desc_node(node)
        if(detail_node is not None):
            retstr += self.get_text(detail_node) + LINE_BREAKER
        rows = []
        rows.append(("Name", "Type", "Descritpion"))
        for member in node.iter("memberdef"):
            if(member.attrib["kind"] == "variable"):
                name = member.findtext("name")
                type = self._find_text_ref(member.find("type"))
                desc = self.get_text(member.find("briefdescription"))
                desc += self.get_text(member.find("detaileddescription"))
                desc += self.get_text(member.find("inbodydescription"))
                if(not desc):
                    desc = ""
                rows.append((name, type, desc))

        if(len(rows) > 1):
            retstr += self._build_table(rows)
        return retstr

    def _build_class(self, node):
        retstr = ""

        for member in node.iter("memberdef"):
            if(member.attrib["kind"] == "function"):
                retstr += self.build_function(member)
        return retstr

    def get_desc_node(self, member):
        detail_node = member.find("detaileddescription")
        brief_node = member.find("briefdescription")
        detail_txt = ""
        if(detail_node == None and brief_node == None):
            return None

        if(detail_node is not None):
            detail_txt = detail_node.findtext("para")

        if(not detail_txt and brief_node != None):
            detail_txt = brief_node.findtext("para")
            detail_node = brief_node

        return detail_node

    def build_function(self, member):
        retstr = ""

        desc_node = self.get_desc_node(member)
        if(desc_node is None):
            return ""
        detail_txt = desc_node.findtext("para")
        if(not detail_txt or detail_txt.strip() == "{ignore}"):
            return ""

        func_id = member.attrib["id"]
        page_id, ref_id = get_page(func_id)
        retstr += self._build_title(member.findtext("name"), '-', ref = ref_id)
        retstr += self.get_text(desc_node)
        retstr += LINE_BREAKER
        detail_node = member.find("detaileddescription")
        if(desc_node != detail_node):
            retstr += self.get_text(detail_node)
        retstr += self.build_param_list(member.iter("param"), detail_node.iter("parameteritem"))
        return retstr

    def _build_missed_types_and_structs(self):
        fout = open(os.path.join(self.output_dir, self.missing_filename), 'w+')
        fout.write(".. contents:: " + LINE_BREAKER)
        fout.write("    :local:"  + LINE_BREAKER)
        fout.write("    :depth: 2" + LINE_BREAKER + LINE_BREAKER)

        footnote = ""
        while (len(self.missed_types_structs) > 0):
            for key, value in self.missed_types_structs.iteritems():
                fout.write(self.covert_item(value[0], key, value[1]))
                #print(value)
            self.missed_types_structs = {}
            footnote += self._build_page_ref_notes()

        fout.write(footnote)

        fout.close()

    def _build_page_ref_notes(self):
        retstr = LINE_BREAKER
        #TODO
        for key, value in self.page_references.iteritems():
            page, tag = get_page(value[0])
            m = re.search("_8h_", page)
            if(m):
                continue;

            rstname = None
            anchor = value[2].lower()
            if not page in self.references:
                self.missed_types_structs[value[0]] = (page, tag)
                rstname = os.path.splitext(self.missing_filename)[0]
            else:
                rstname = self.references[page][2]
                anchor = self.references[page][4]
            #if(tag and not self.is_github):
            #    anchor = self.anchor_prefix + "ref-" + tag
            retstr += ".. _" + key + ": " + rstname + self.page_ext + "#" + anchor
            retstr += LINE_BREAKER + LINE_BREAKER
        self.page_references = {}
        return retstr

    def _build_item_by_id(self, node, id):
        retstr = ""
        for member in node.iter("memberdef"):
            if(member.attrib["id"] != id):
                continue
            if(member.attrib["kind"] == "enum"):
                retstr += self._build_enum(member)
        return retstr

    def covert_item(self, compound, id, tag):
        xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % compound)
        print("covert_item: id=%s, name=%s" % (id, xml_path))
        obj_root = etree.parse(xml_path).getroot()
        retstr = ""
        compound = obj_root.find("compounddef")
        compound_kind = compound.attrib["kind"]
        if(not tag):
            retstr += self._build_title(compound.findtext("compoundname"))
            if(compound_kind == "class"):
                retstr += self._build_class(compound)
            elif(compound_kind == "struct"):
                retstr += self._build_struct(compound)
        else:
            retstr += self._build_item_by_id(compound, id)

        return retstr

    def _build_page(self, compound):
        retstr = ""
        retstr += self.get_text(compound.find("detaileddescription"))
        return retstr

    def _build_file(self, compound, type, ref_id, name):
        retstr = ""
        for member in compound.iter("memberdef"):
            if(member.attrib["kind"] == "function" and member.attrib["id"] == ref_id):
                retstr += self.build_function(member)
        return retstr

    def convert_doxy(self, type, name):
        #print(name)
        file = ref_id = self.name_refid_map[name]
        dst_kind = type
        if(type == "function"):
            file, tag = get_page(ref_id)
            dst_kind = "file"
        xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % file)
        print("convert_doxy: type=%s, name=%s" % (type, xml_path))
        obj_root = etree.parse(xml_path).getroot()
        compound = obj_root.find("compounddef")
        compound_kind = compound.attrib["kind"]
        assert(dst_kind == compound_kind)
        retstr = ""
        if(compound_kind == "class"):
            retstr += self._build_class(compound)
        elif(compound_kind == "struct"):
            retstr += self._build_struct(compound)
        elif(compound_kind == "page"):
            retstr += self._build_page(compound)
        elif(compound_kind == "group"):
            retstr += self._build_page(compound)
        elif(compound_kind == "file"):
            retstr += self._build_file(compound, type, ref_id, name)
        return retstr


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-g", "--github", action="store_true", help="Render the link in format of github wiki.")
    parser.add_argument("-e", "--ext", default="", help="extension for github wiki")
    parser.add_argument("-i", "--input", default="doxygen", help="Input file path of doxygen output and source rst file.")
    parser.add_argument("-o", "--output", default="wikipage", help="Output converted restructured text files to path.")
    parser.add_argument("-s", "--struct", default="TypesAndStructures.rest", help="Output of auto generated enum and structures.")
    parser.add_argument("-u", "--uml", action="store_true", help="Enable UML, you need to download plantuml.jar from Plantuml and put it to here. http://plantuml.sourceforge.net/")

    args = parser.parse_args()
    ext = ""
    if(len(args.ext) > 0):
        ext = ("." + args.ext)
    agent = DoxyGen2RST(args.input,
                        args.output,
                        args.struct,
                        is_github = True,
                        enable_uml = args.uml,
                        github_ext = ext)
    agent.generate()