python/helpers/pycharm_generator_utils/module_redeclarator.py

   1 from pycharm_generator_utils.util_methods import *
   2 from pycharm_generator_utils.constants import *
   3 import keyword, re
   4
   5
   6 class emptylistdict(dict):
   7     """defaultdict not available before 2.5; simplest reimplementation using [] as default"""
   8
   9     def __getitem__(self, item):
  10         if item in self:
  11             return dict.__getitem__(self, item)
  12         else:
  13             it = []
  14             self.__setitem__(item, it)
  15             return it
  16
  17 class Buf(object):
  18     """Buffers data in a list, can write to a file. Indentation is provided externally."""
  19
  20     def __init__(self, indenter):
  21         self.data = []
  22         self.indenter = indenter
  23
  24     def put(self, data):
  25         if data:
  26             self.data.append(ensureUnicode(data))
  27
  28     def out(self, indent, *what):
  29         """Output the arguments, indenting as needed, and adding an eol"""
  30         self.put(self.indenter.indent(indent))
  31         for item in what:
  32             self.put(item)
  33         self.put("\n")
  34
  35     def flush_bytes(self, outfile):
  36         for data in self.data:
  37             outfile.write(data.encode(OUT_ENCODING, "replace"))
  38
  39     def flush_str(self, outfile):
  40         for data in self.data:
  41             outfile.write(data)
  42
  43     if version[0] < 3:
  44         flush = flush_bytes
  45     else:
  46         flush = flush_str
  47
  48     def isEmpty(self):
  49         return len(self.data) == 0
  50
  51
  52 #noinspection PyUnresolvedReferences,PyBroadException
  53 class ModuleRedeclarator(object):
  54     def __init__(self, module, outfile, mod_filename, indent_size=4, doing_builtins=False):
  55         """
  56         Create new instance.
  57         @param module module to restore.
  58         @param outfile output file, must be open and writable.
  59         @param mod_filename filename of binary module (the .dll or .so)
  60         @param indent_size amount of space characters per indent
  61         """
  62         self.module = module
  63         self.outfile = outfile # where we finally write
  64         self.mod_filename = mod_filename
  65         # we write things into buffers out-of-order
  66         self.header_buf = Buf(self)
  67         self.imports_buf = Buf(self)
  68         self.functions_buf = Buf(self)
  69         self.classes_buf = Buf(self)
  70         self.footer_buf = Buf(self)
  71         self.indent_size = indent_size
  72         self._indent_step = " " * self.indent_size
  73         #
  74         self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module}
  75         self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported
  76         # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome.
  77         self._defined = {} # stores True for every name defined so far, to break circular refs in values
  78         self.doing_builtins = doing_builtins
  79         self.ret_type_cache = {}
  80         self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names
  81
  82     def _initializeQApp(self):
  83         try:  # QtGui should be imported _before_ QtCore package.
  84             # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+
  85             # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type"
  86             import PyQt4.QtGui
  87         except ImportError:
  88             pass
  89
  90         # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection)
  91         # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS
  92         try:
  93             from PyQt4.QtCore import QCoreApplication
  94             self.app = QCoreApplication([])
  95             return
  96         except ImportError:
  97             pass
  98         try:
  99             from PyQt5.QtCore import QCoreApplication
 100             self.app = QCoreApplication([])
 101         except ImportError:
 102             pass
 103
 104     def indent(self, level):
 105         """Return indentation whitespace for given level."""
 106         return self._indent_step * level
 107
 108     def flush(self):
 109         for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf, self.footer_buf):
 110             buf.flush(self.outfile)
 111
 112     # Some builtin classes effectively change __init__ signature without overriding it.
 113     # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS
 114     def fake_builtin_init(self):
 115         pass # just a callable, sig doesn't matter
 116
 117     fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead
 118
 119
 120     def find_imported_name(self, item):
 121         """
 122         Finds out how the item is represented in imported modules.
 123         @param item what to check
 124         @return qualified name (like "sys.stdin") or None
 125         """
 126         # TODO: return a pair, not a glued string
 127         if not isinstance(item, SIMPLEST_TYPES):
 128             for mname in self.imported_modules:
 129                 m = self.imported_modules[mname]
 130                 for inner_name in m.__dict__:
 131                     suspect = getattr(m, inner_name)
 132                     if suspect is item:
 133                         if mname:
 134                             mname += "."
 135                         elif self.module is the_builtins: # don't short-circuit builtins
 136                             return None
 137                         return mname + inner_name
 138         return None
 139
 140     _initializers = (
 141         (dict, "{}"),
 142         (tuple, "()"),
 143         (list, "[]"),
 144     )
 145
 146     def invent_initializer(self, a_type):
 147         """
 148         Returns an innocuous initializer expression for a_type, or "None"
 149         """
 150         for initializer_type, r in self._initializers:
 151             if initializer_type == a_type:
 152                 return r
 153             # NOTE: here we could handle things like defaultdict, sets, etc if we wanted
 154         return "None"
 155
 156
 157     def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None):
 158         """
 159         Formats and outputs value (it occupies an entire line or several lines).
 160         @param out function that does output (a Buf.out)
 161         @param p_value the value.
 162         @param indent indent level.
 163         @param prefix text to print before the value
 164         @param postfix text to print after the value
 165         @param as_name hints which name are we trying to print; helps with circular refs.
 166         @param seen_values a list of keys we've seen if we're processing a dict
 167         """
 168         SELF_VALUE = "<value is a self-reference, replaced by this string>"
 169         ERR_VALUE = "<failed to retrieve the value>"
 170         if isinstance(p_value, SIMPLEST_TYPES):
 171             out(indent, prefix, reliable_repr(p_value), postfix)
 172         else:
 173             if sys.platform == "cli":
 174                 imported_name = None
 175             else:
 176                 imported_name = self.find_imported_name(p_value)
 177             if imported_name:
 178                 out(indent, prefix, imported_name, postfix)
 179                 # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name
 180                 # else we could potentially return smth we did not otherwise import. but not likely.
 181             else:
 182                 if isinstance(p_value, (list, tuple)):
 183                     if not seen_values:
 184                         seen_values = [p_value]
 185                     if len(p_value) == 0:
 186                         out(indent, prefix, repr(p_value), postfix)
 187                     else:
 188                         if isinstance(p_value, list):
 189                             lpar, rpar = "[", "]"
 190                         else:
 191                             lpar, rpar = "(", ")"
 192                         out(indent, prefix, lpar)
 193                         for value in p_value:
 194                             if value in seen_values:
 195                                 value = SELF_VALUE
 196                             elif not isinstance(value, SIMPLEST_TYPES):
 197                                 seen_values.append(value)
 198                             self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values)
 199                         out(indent, rpar, postfix)
 200                 elif isinstance(p_value, dict):
 201                     if len(p_value) == 0:
 202                         out(indent, prefix, repr(p_value), postfix)
 203                     else:
 204                         if not seen_values:
 205                             seen_values = [p_value]
 206                         out(indent, prefix, "{")
 207                         keys = list(p_value.keys())
 208                         try:
 209                             keys.sort()
 210                         except TypeError:
 211                             pass # unsortable keys happen, e,g, in py3k _ctypes
 212                         for k in keys:
 213                             value = p_value[k]
 214
 215                             try:
 216                                 is_seen = value in seen_values
 217                             except:
 218                                 is_seen = False
 219                                 value = ERR_VALUE
 220
 221                             if is_seen:
 222                                 value = SELF_VALUE
 223                             elif not isinstance(value, SIMPLEST_TYPES):
 224                                 seen_values.append(value)
 225                             if isinstance(k, SIMPLEST_TYPES):
 226                                 self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",",
 227                                               seen_values=seen_values)
 228                             else:
 229                                 # both key and value need fancy formatting
 230                                 self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values)
 231                                 self.fmt_value(out, value, indent + 2, seen_values=seen_values)
 232                                 out(indent + 1, ",")
 233                         out(indent, "}", postfix)
 234                 else: # something else, maybe representable
 235                     # look up this value in the module.
 236                     if sys.platform == "cli":
 237                         out(indent, prefix, "None", postfix)
 238                         return
 239                     found_name = ""
 240                     for inner_name in self.module.__dict__:
 241                         if self.module.__dict__[inner_name] is p_value:
 242                             found_name = inner_name
 243                             break
 244                     if self._defined.get(found_name, False):
 245                         out(indent, prefix, found_name, postfix)
 246                     else:
 247                     # a forward / circular declaration happens
 248                         notice = ""
 249                         real_value = cleanup(repr(p_value))
 250                         if found_name:
 251                             if found_name == as_name:
 252                                 notice = " # (!) real value is %r" % real_value
 253                                 real_value = "None"
 254                             else:
 255                                 notice = " # (!) forward: %s, real value is %r" % (found_name, real_value)
 256                         if SANE_REPR_RE.match(real_value):
 257                             out(indent, prefix, real_value, postfix, notice)
 258                         else:
 259                             if not found_name:
 260                                 notice = " # (!) real value is %r" % real_value
 261                             out(indent, prefix, "None", postfix, notice)
 262
 263     def get_ret_type(self, attr):
 264         """
 265         Returns a return type string as given by T_RETURN in tokens, or None
 266         """
 267         if attr:
 268             ret_type = RET_TYPE.get(attr, None)
 269             if ret_type:
 270                 return ret_type
 271             thing = getattr(self.module, attr, None)
 272             if thing:
 273                 if not isinstance(thing, type) and is_callable(thing): # a function
 274                     return None # TODO: maybe divinate a return type; see pygame.mixer.Channel
 275                 return attr
 276                 # adds no noticeable slowdown, I did measure. dch.
 277             for im_name, im_module in self.imported_modules.items():
 278                 cache_key = (im_name, attr)
 279                 cached = self.ret_type_cache.get(cache_key, None)
 280                 if cached:
 281                     return cached
 282                 ret_type = getattr(im_module, attr, None)
 283                 if ret_type:
 284                     if isinstance(ret_type, type):
 285                         # detect a constructor
 286                         constr_args = detect_constructor(ret_type)
 287                         if constr_args is None:
 288                             constr_args = "*(), **{}" # a silly catch-all constructor
 289                         reference = "%s(%s)" % (attr, constr_args)
 290                     elif is_callable(ret_type): # a function, classes are ruled out above
 291                         return None
 292                     else:
 293                         reference = attr
 294                     if im_name:
 295                         result = "%s.%s" % (im_name, reference)
 296                     else: # built-in
 297                         result = reference
 298                     self.ret_type_cache[cache_key] = result
 299                     return result
 300                     # TODO: handle things like "[a, b,..] and (foo,..)"
 301         return None
 302
 303
 304     SIG_DOC_NOTE = "restored from __doc__"
 305     SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ "
 306
 307     def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None):
 308         """
 309         @param signature_string: parameter list extracted from the doc string.
 310         @param class_name: name of the containing class, or None
 311         @param deco: decorator to use
 312         @param ret_hint: return type hint, if available
 313         @return (reconstructed_spec, return_type, note) or (None, _, _) if failed.
 314         """
 315         action("restoring func %r of class %r", signature_string, class_name)
 316         # parse
 317         parsing_failed = False
 318         ret_type = None
 319         try:
 320             # strict parsing
 321             tokens = paramSeqAndRest.parseString(signature_string, True)
 322             ret_name = None
 323             if tokens:
 324                 ret_t = tokens[-1]
 325                 if ret_t[0] is T_RETURN:
 326                     ret_name = ret_t[1]
 327             ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint)
 328         except ParseException:
 329             # it did not parse completely; scavenge what we can
 330             parsing_failed = True
 331             tokens = []
 332             try:
 333                 # most unrestrictive parsing
 334                 tokens = paramSeq.parseString(signature_string, False)
 335             except ParseException:
 336                 pass
 337                 #
 338         seq = transform_seq(tokens)
 339
 340         # add safe defaults for unparsed
 341         if parsing_failed:
 342             doc_node = self.SIG_DOC_UNRELIABLY
 343             starred = None
 344             double_starred = None
 345             for one in seq:
 346                 if type(one) is str:
 347                     if one.startswith("**"):
 348                         double_starred = one
 349                     elif one.startswith("*"):
 350                         starred = one
 351             if not starred:
 352                 seq.append("*args")
 353             if not double_starred:
 354                 seq.append("**kwargs")
 355         else:
 356             doc_node = self.SIG_DOC_NOTE
 357
 358         # add 'self' if needed YYY
 359         if class_name and (not seq or seq[0] != 'self'):
 360             first_param = propose_first_param(deco)
 361             if first_param:
 362                 seq.insert(0, first_param)
 363         seq = make_names_unique(seq)
 364         return (seq, ret_type, doc_node)
 365
 366     def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False):
 367         """
 368         @param func_doc: __doc__ of the function.
 369         @param func_id: name to look for as identifier of the function in docstring
 370         @param func_name: name of the function.
 371         @param class_name: name of the containing class, or None
 372         @param deco: decorator to use
 373         @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed.
 374         """
 375         if sip_generated:
 376             overloads = []
 377             for part in func_doc.split('\n'):
 378                 signature = func_id + '('
 379                 i = part.find(signature)
 380                 if i >= 0:
 381                     overloads.append(part[i + len(signature):])
 382             if len(overloads) > 1:
 383                 docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads]
 384                 ret_types = []
 385                 for result in docstring_results:
 386                     rt = result[1]
 387                     if rt and rt not in ret_types:
 388                         ret_types.append(rt)
 389                 if ret_types:
 390                     ret_literal = " or ".join(ret_types)
 391                 else:
 392                     ret_literal = None
 393                 param_lists = [result[0] for result in docstring_results]
 394                 spec = build_signature(func_name, restore_parameters_for_overloads(param_lists))
 395                 return (spec, ret_literal, "restored from __doc__ with multiple overloads")
 396
 397         # find the first thing to look like a definition
 398         prefix_re = re.compile("\s*(?:(\w+)[ \\t]+)?" + func_id + "\s*\(") # "foo(..." or "int foo(..."
 399         match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time
 400         # parse the part that looks right
 401         if match:
 402             ret_hint = match.group(1)
 403             params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint)
 404             spec = func_name + flatten(params)
 405             return (spec, ret_literal, doc_note)
 406         else:
 407             return (None, None, None)
 408
 409
 410     def is_predefined_builtin(self, module_name, class_name, func_name):
 411         return self.doing_builtins and module_name == BUILTIN_MOD_NAME and (
 412         class_name, func_name) in PREDEFINED_BUILTIN_SIGS
 413
 414
 415     def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None):
 416         """
 417         Restore function argument list as best we can.
 418         @param out output function of a Buf
 419         @param p_func function or method object
 420         @param p_name function name as known to owner
 421         @param indent indentation level
 422         @param p_class the class that contains this function as a method
 423         @param p_modname module name
 424         @param seen {id(func): name} map of functions already seen in the same namespace;
 425                id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7)
 426         """
 427         action("redoing func %r of class %r", p_name, p_class)
 428         if seen is not None:
 429             other_func = seen.get(id(p_func), None)
 430             if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None):
 431                 # _bisect.bisect == _bisect.bisect_right in py31, but docs differ
 432                 out(indent, p_name, " = ", seen[id(p_func)])
 433                 out(indent, "")
 434                 return
 435             else:
 436                 seen[id(p_func)] = p_name
 437             # real work
 438         if classname is None:
 439             classname = p_class and p_class.__name__ or None
 440         if p_class and hasattr(p_class, '__mro__'):
 441             sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)]
 442         else:
 443             sip_generated = False
 444         deco = None
 445         deco_comment = ""
 446         mod_class_method_tuple = (p_modname, classname, p_name)
 447         ret_literal = None
 448         is_init = False
 449         # any decorators?
 450         action("redoing decos of func %r of class %r", p_name, p_class)
 451         if self.doing_builtins and p_modname == BUILTIN_MOD_NAME:
 452             deco = KNOWN_DECORATORS.get((classname, p_name), None)
 453             if deco:
 454                 deco_comment = " # known case"
 455         elif p_class and p_name in p_class.__dict__:
 456             # detect native methods declared with METH_CLASS flag
 457             descriptor = p_class.__dict__[p_name]
 458             if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'):
 459                 # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython
 460                 deco = "classmethod"
 461             elif type(p_func).__name__.startswith('staticmethod'):
 462                 deco = "staticmethod"
 463         if p_name == "__new__":
 464             deco = "staticmethod"
 465             deco_comment = " # known case of __new__"
 466
 467         action("redoing innards of func %r of class %r", p_name, p_class)
 468         if deco and HAS_DECORATORS:
 469             out(indent, "@", deco, deco_comment)
 470         if inspect and inspect.isfunction(p_func):
 471             out(indent, "def ", p_name, restore_by_inspect(p_func), ": # reliably restored by inspect", )
 472             out_doc_attr(out, p_func, indent + 1, p_class)
 473         elif self.is_predefined_builtin(*mod_class_method_tuple):
 474             spec, sig_note = restore_predefined_builtin(classname, p_name)
 475             out(indent, "def ", spec, ": # ", sig_note)
 476             out_doc_attr(out, p_func, indent + 1, p_class)
 477         elif sys.platform == 'cli' and is_clr_type(p_class):
 478             spec, sig_note = restore_clr(p_name, p_class)
 479             if not spec: return
 480             if sig_note:
 481                 out(indent, "def ", spec, ": #", sig_note)
 482             else:
 483                 out(indent, "def ", spec, ":")
 484             if not p_name in ['__gt__', '__ge__', '__lt__', '__le__', '__ne__', '__reduce_ex__', '__str__']:
 485                 out_doc_attr(out, p_func, indent + 1, p_class)
 486         elif mod_class_method_tuple in PREDEFINED_MOD_CLASS_SIGS:
 487             sig, ret_literal = PREDEFINED_MOD_CLASS_SIGS[mod_class_method_tuple]
 488             if classname:
 489                 ofwhat = "%s.%s.%s" % mod_class_method_tuple
 490             else:
 491                 ofwhat = "%s.%s" % (p_modname, p_name)
 492             out(indent, "def ", p_name, sig, ": # known case of ", ofwhat)
 493             out_doc_attr(out, p_func, indent + 1, p_class)
 494         else:
 495         # __doc__ is our best source of arglist
 496             sig_note = "real signature unknown"
 497             spec = ""
 498             is_init = (p_name == "__init__" and p_class is not None)
 499             funcdoc = None
 500             if is_init and hasattr(p_class, "__doc__"):
 501                 if hasattr(p_func, "__doc__"):
 502                     funcdoc = p_func.__doc__
 503                 if funcdoc == object.__init__.__doc__:
 504                     funcdoc = p_class.__doc__
 505             elif hasattr(p_func, "__doc__"):
 506                 funcdoc = p_func.__doc__
 507             sig_restored = False
 508             action("parsing doc of func %r of class %r", p_name, p_class)
 509             if isinstance(funcdoc, STR_TYPES):
 510                 (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, p_name, p_name, classname, deco,
 511                                                                     sip_generated)
 512                 if spec is None and p_name == '__init__' and classname:
 513                     (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, classname, p_name, classname, deco,
 514                                                                         sip_generated)
 515                 sig_restored = spec is not None
 516                 if more_notes:
 517                     if sig_note:
 518                         sig_note += "; "
 519                     sig_note += more_notes
 520             if not sig_restored:
 521             # use an allow-all declaration
 522                 decl = []
 523                 if p_class:
 524                     first_param = propose_first_param(deco)
 525                     if first_param:
 526                         decl.append(first_param)
 527                 decl.append("*args")
 528                 decl.append("**kwargs")
 529                 spec = p_name + "(" + ", ".join(decl) + ")"
 530             out(indent, "def ", spec, ": # ", sig_note)
 531             # to reduce size of stubs, don't output same docstring twice for class and its __init__ method
 532             if not is_init or funcdoc != p_class.__doc__:
 533                 out_docstring(out, funcdoc, indent + 1)
 534             # body
 535         if ret_literal and not is_init:
 536             out(indent + 1, "return ", ret_literal)
 537         else:
 538             out(indent + 1, "pass")
 539         if deco and not HAS_DECORATORS:
 540             out(indent, p_name, " = ", deco, "(", p_name, ")", deco_comment)
 541         out(0, "") # empty line after each item
 542
 543
 544     def redo_class(self, out, p_class, p_name, indent, p_modname=None, seen=None, inspect_dir=False):
 545         """
 546         Restores a class definition.
 547         @param out output function of a relevant buf
 548         @param p_class the class object
 549         @param p_name class name as known to owner
 550         @param indent indentation level
 551         @param p_modname name of module
 552         @param seen {class: name} map of classes already seen in the same namespace
 553         """
 554         action("redoing class %r of module %r", p_name, p_modname)
 555         if seen is not None:
 556             if p_class in seen:
 557                 out(indent, p_name, " = ", seen[p_class])
 558                 out(indent, "")
 559                 return
 560             else:
 561                 seen[p_class] = p_name
 562         bases = get_bases(p_class)
 563         base_def = ""
 564         skipped_bases = []
 565         if bases:
 566             skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions']
 567             skip_qualifiers.extend(KNOWN_FAKE_REEXPORTERS.get(p_modname, ()))
 568             bases_list = [] # what we'll render in the class decl
 569             for base in bases:
 570                 if [1 for (cls, mdl) in KNOWN_FAKE_BASES if cls == base and mdl != self.module]:
 571                     # our base is a wrapper and our module is not its defining module
 572                     skipped_bases.append(str(base))
 573                     continue
 574                     # somehow import every base class
 575                 base_name = base.__name__
 576                 qual_module_name = qualifier_of(base, skip_qualifiers)
 577                 got_existing_import = False
 578                 if qual_module_name:
 579                     if qual_module_name in self.used_imports:
 580                         import_list = self.used_imports[qual_module_name]
 581                         if base in import_list:
 582                             bases_list.append(base_name) # unqualified: already set to import
 583                             got_existing_import = True
 584                     if not got_existing_import:
 585                         mangled_qualifier = "__" + qual_module_name.replace('.', '_') # foo.bar -> __foo_bar
 586                         bases_list.append(mangled_qualifier + "." + base_name)
 587                         self.hidden_imports[qual_module_name] = mangled_qualifier
 588                 else:
 589                     bases_list.append(base_name)
 590             base_def = "(" + ", ".join(bases_list) + ")"
 591         out(indent, "class ", p_name, base_def, ":",
 592             skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "")
 593         out_doc_attr(out, p_class, indent + 1)
 594         # inner parts
 595         methods = {}
 596         properties = {}
 597         others = {}
 598         we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object"
 599         field_source = {}
 600         try:
 601             if hasattr(p_class, "__dict__") and not inspect_dir:
 602                 field_source = p_class.__dict__
 603                 field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here
 604             else:
 605                 field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare
 606         except:
 607             field_keys = ()
 608         for item_name in field_keys:
 609             if item_name in ("__doc__", "__module__"):
 610                 if we_are_the_base_class:
 611                     item = "" # must be declared in base types
 612                 else:
 613                     continue # in all other cases must be skipped
 614             elif keyword.iskeyword(item_name):  # for example, PyQt4 contains definitions of methods named 'exec'
 615                 continue
 616             else:
 617                 try:
 618                     item = getattr(p_class, item_name) # let getters do the magic
 619                 except AttributeError:
 620                     item = field_source[item_name] # have it raw
 621                 except Exception:
 622                     continue
 623             if is_callable(item) and not isinstance(item, type):
 624                 methods[item_name] = item
 625             elif is_property(item):
 626                 properties[item_name] = item
 627             else:
 628                 others[item_name] = item
 629                 #
 630         if we_are_the_base_class:
 631             others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :)
 632             # add fake __init__s to have the right sig
 633         if p_class in FAKE_BUILTIN_INITS:
 634             methods["__init__"] = self.fake_builtin_init
 635             note("Faking init of %s", p_name)
 636         elif '__init__' not in methods:
 637             init_method = getattr(p_class, '__init__', None)
 638             if init_method:
 639                 methods['__init__'] = init_method
 640
 641         #
 642         seen_funcs = {}
 643         for item_name in sorted_no_case(methods.keys()):
 644             item = methods[item_name]
 645             try:
 646                 self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs)
 647             except:
 648                 handle_error_func(item_name, out)
 649             #
 650         known_props = KNOWN_PROPS.get(p_modname, {})
 651         a_setter = "lambda self, v: None"
 652         a_deleter = "lambda self: None"
 653         for item_name in sorted_no_case(properties.keys()):
 654             item = properties[item_name]
 655             prop_docstring = getattr(item, '__doc__', None)
 656             prop_key = (p_name, item_name)
 657             if prop_key in known_props:
 658                 prop_descr = known_props.get(prop_key, None)
 659                 if prop_descr is None:
 660                     continue # explicitly omitted
 661                 acc_line, getter_and_type = prop_descr
 662                 if getter_and_type:
 663                     getter, prop_type = getter_and_type
 664                 else:
 665                     getter, prop_type = None, None
 666                 out(indent + 1, item_name,
 667                     " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")"
 668                 )
 669                 if prop_type:
 670                     if prop_docstring:
 671                         out(indent + 1, '"""', prop_docstring)
 672                         out(0, "")
 673                         out(indent + 1, ':type: ', prop_type)
 674                         out(indent + 1, '"""')
 675                     else:
 676                         out(indent + 1, '""":type: ', prop_type, '"""')
 677                     out(0, "")
 678             else:
 679                 out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None)  # default")
 680                 if prop_docstring:
 681                     out(indent + 1, '"""', prop_docstring, '"""')
 682                 out(0, "")
 683         if properties:
 684             out(0, "") # empty line after the block
 685             #
 686         for item_name in sorted_no_case(others.keys()):
 687             item = others[item_name]
 688             self.fmt_value(out, item, indent + 1, prefix=item_name + " = ")
 689         if p_name == "object":
 690             out(indent + 1, "__module__ = ''")
 691         if others:
 692             out(0, "") # empty line after the block
 693             #
 694         if not methods and not properties and not others:
 695             out(indent + 1, "pass")
 696
 697
 698
 699     def redo_simple_header(self, p_name):
 700         """Puts boilerplate code on the top"""
 701         out = self.header_buf.out # 1st class methods rule :)
 702         out(0, "# encoding: %s" % OUT_ENCODING) # line 1
 703         # NOTE: maybe encoding should be selectable
 704         if hasattr(self.module, "__name__"):
 705             self_name = self.module.__name__
 706             if self_name != p_name:
 707                 mod_name = " calls itself " + self_name
 708             else:
 709                 mod_name = ""
 710         else:
 711             mod_name = " does not know its name"
 712         out(0, "# module ", p_name, mod_name) # line 2
 713
 714         BUILT_IN_HEADER = "(built-in)"
 715         if self.mod_filename:
 716             filename = self.mod_filename
 717         elif p_name in sys.builtin_module_names:
 718             filename = BUILT_IN_HEADER
 719         else:
 720             filename = getattr(self.module, "__file__", BUILT_IN_HEADER)
 721
 722         out(0, "# from %s" % filename)  # line 3
 723         out(0, "# by generator %s" % VERSION) # line 4
 724         if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6:
 725             out(0, "from __future__ import print_function")
 726         out_doc_attr(out, self.module, 0)
 727
 728
 729     def redo_imports(self):
 730         module_type = type(sys)
 731         for item_name in self.module.__dict__.keys():
 732             try:
 733                 item = self.module.__dict__[item_name]
 734             except:
 735                 continue
 736             if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here
 737                 self.imported_modules[item_name] = item
 738                 self.add_import_header_if_needed()
 739                 ref_notice = getattr(item, "__file__", str(item))
 740                 if hasattr(item, "__name__"):
 741                     self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice)
 742                 else:
 743                     self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice)
 744
 745     def add_import_header_if_needed(self):
 746         if self.imports_buf.isEmpty():
 747             self.imports_buf.out(0, "")
 748             self.imports_buf.out(0, "# imports")
 749
 750
 751     def redo(self, p_name, inspect_dir):
 752         """
 753         Restores module declarations.
 754         Intended for built-in modules and thus does not handle import statements.
 755         @param p_name name of module
 756         """
 757         action("redoing header of module %r %r", p_name, str(self.module))
 758
 759         if "pyqt" in p_name.lower():   # qt specific patch
 760             self._initializeQApp()
 761
 762         self.redo_simple_header(p_name)
 763
 764         # find whatever other self.imported_modules the module knows; effectively these are imports
 765         action("redoing imports of module %r %r", p_name, str(self.module))
 766         try:
 767             self.redo_imports()
 768         except:
 769             pass
 770
 771         action("redoing innards of module %r %r", p_name, str(self.module))
 772
 773         module_type = type(sys)
 774         # group what we have into buckets
 775         vars_simple = {}
 776         vars_complex = {}
 777         funcs = {}
 778         classes = {}
 779         module_dict = self.module.__dict__
 780         if inspect_dir:
 781             module_dict = dir(self.module)
 782         for item_name in module_dict:
 783             note("looking at %s", item_name)
 784             if item_name in (
 785             "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"):
 786                 continue # handled otherwise
 787             try:
 788                 item = getattr(self.module, item_name) # let getters do the magic
 789             except AttributeError:
 790                 if not item_name in self.module.__dict__: continue
 791                 item = self.module.__dict__[item_name] # have it raw
 792                 # check if it has percolated from an imported module
 793             except NotImplementedError:
 794                 if not item_name in self.module.__dict__: continue
 795                 item = self.module.__dict__[item_name] # have it raw
 796
 797             # unless we're adamantly positive that the name was imported, we assume it is defined here
 798             mod_name = None # module from which p_name might have been imported
 799             # IronPython has non-trivial reexports in System module, but not in others:
 800             skip_modname = sys.platform == "cli" and p_name != "System"
 801             surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ())
 802             ## can't figure weirdness in some modules, assume no reexports:
 803             #skip_modname =  skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS
 804             if not skip_modname:
 805                 try:
 806                     mod_name = getattr(item, '__module__', None)
 807                 except:
 808                     pass
 809                 # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect)
 810             maybe_import_mod_name = mod_name or ""
 811             import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name)
 812             note("mod_name = %s, prospective = %s,  from top = %s", mod_name, maybe_import_mod_name, import_is_from_top)
 813             want_to_import = False
 814             if (mod_name
 815                 and mod_name != BUILTIN_MOD_NAME
 816                 and mod_name != p_name
 817                 and mod_name not in surely_not_imported_mods
 818                 and not import_is_from_top
 819             ):
 820                 # import looks valid, but maybe it's a .py file? we're certain not to import from .py
 821                 # e.g. this rules out _collections import collections and builtins import site.
 822                 try:
 823                     imported = __import__(mod_name) # ok to repeat, Python caches for us
 824                     if imported:
 825                         qualifiers = mod_name.split(".")[1:]
 826                         for qual in qualifiers:
 827                             imported = getattr(imported, qual, None)
 828                             if not imported:
 829                                 break
 830                         imported_path = (getattr(imported, '__file__', False) or "").lower()
 831                         want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc'))
 832                         note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import)
 833                 except ImportError:
 834                     want_to_import = False
 835                     # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all
 836                 if want_to_import:
 837                     import_list = self.used_imports[mod_name]
 838                     if item_name not in import_list:
 839                         import_list.append(item_name)
 840             if not want_to_import:
 841                 if isinstance(item, type) or type(item).__name__ == 'classobj':
 842                     classes[item_name] = item
 843                 elif is_callable(item): # some classes are callable, check them before functions
 844                     funcs[item_name] = item
 845                 elif isinstance(item, module_type):
 846                     continue # self.imported_modules handled above already
 847                 else:
 848                     if isinstance(item, SIMPLEST_TYPES):
 849                         vars_simple[item_name] = item
 850                     else:
 851                         vars_complex[item_name] = item
 852
 853         # sort and output every bucket
 854         action("outputting innards of module %r %r", p_name, str(self.module))
 855         #
 856         omitted_names = OMIT_NAME_IN_MODULE.get(p_name, [])
 857         if vars_simple:
 858             out = self.functions_buf.out
 859             prefix = "" # try to group variables by common prefix
 860             PREFIX_LEN = 2 # default prefix length if we can't guess better
 861             out(0, "# Variables with simple values")
 862             for item_name in sorted_no_case(vars_simple.keys()):
 863                 if item_name in omitted_names:
 864                     out(0, "# definition of " + item_name + " omitted")
 865                     continue
 866                 item = vars_simple[item_name]
 867                 # track the prefix
 868                 if len(item_name) >= PREFIX_LEN:
 869                     prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore
 870                     if prefix_pos < 1:
 871                         prefix_pos = PREFIX_LEN
 872                     beg = item_name[0:prefix_pos]
 873                     if prefix != beg:
 874                         out(0, "") # space out from other prefix
 875                         prefix = beg
 876                 else:
 877                     prefix = ""
 878                     # output
 879                 replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
 880                 if replacement is not None:
 881                     out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced")
 882                 elif is_skipped_in_module(p_name, item_name):
 883                     t_item = type(item)
 884                     out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item),
 885                         " skipped")
 886                 else:
 887                     self.fmt_value(out, item, 0, prefix=item_name + " = ")
 888                 self._defined[item_name] = True
 889             out(0, "") # empty line after vars
 890             #
 891         if funcs:
 892             out = self.functions_buf.out
 893             out(0, "# functions")
 894             out(0, "")
 895             seen_funcs = {}
 896             for item_name in sorted_no_case(funcs.keys()):
 897                 if item_name in omitted_names:
 898                     out(0, "# definition of ", item_name, " omitted")
 899                     continue
 900                 item = funcs[item_name]
 901                 try:
 902                     self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs)
 903                 except:
 904                     handle_error_func(item_name, out)
 905         else:
 906             self.functions_buf.out(0, "# no functions")
 907             #
 908         if classes:
 909             out = self.functions_buf.out
 910             out(0, "# classes")
 911             out(0, "")
 912             seen_classes = {}
 913             # sort classes so that inheritance order is preserved
 914             cls_list = [] # items are (class_name, mro_tuple)
 915             for cls_name in sorted_no_case(classes.keys()):
 916                 cls = classes[cls_name]
 917                 ins_index = len(cls_list)
 918                 for i in range(ins_index):
 919                     maybe_child_bases = cls_list[i][1]
 920                     if cls in maybe_child_bases:
 921                         ins_index = i # we could not go farther than current ins_index
 922                         break         # ...and need not go fartehr than first known child
 923                 cls_list.insert(ins_index, (cls_name, get_mro(cls)))
 924             for item_name in [cls_item[0] for cls_item in cls_list]:
 925                 if item_name in omitted_names:
 926                     out(0, "# definition of ", item_name, " omitted")
 927                     continue
 928                 item = classes[item_name]
 929                 self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir)
 930                 self._defined[item_name] = True
 931                 out(0, "") # empty line after each item
 932
 933             if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3:
 934                 # classobj still supported
 935                 txt = classobj_txt
 936                 self.classes_buf.out(0, txt)
 937
 938             if self.doing_builtins and p_name == BUILTIN_MOD_NAME:
 939                 txt = create_generator()
 940                 self.classes_buf.out(0, txt)
 941
 942                 # Fake <type 'namedtuple'>
 943                 if version[0] >= 3 or (version[0] == 2 and version[1] >= 6):
 944                     namedtuple_text = create_named_tuple()
 945                     self.classes_buf.out(0, namedtuple_text)
 946
 947         else:
 948             self.classes_buf.out(0, "# no classes")
 949             #
 950         if vars_complex:
 951             out = self.footer_buf.out
 952             out(0, "# variables with complex values")
 953             out(0, "")
 954             for item_name in sorted_no_case(vars_complex.keys()):
 955                 if item_name in omitted_names:
 956                     out(0, "# definition of " + item_name + " omitted")
 957                     continue
 958                 item = vars_complex[item_name]
 959                 if str(type(item)) == "<type 'namespace#'>":
 960                     continue            # this is an IronPython submodule, we mustn't generate a reference for it in the base module
 961                 replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
 962                 if replacement is not None:
 963                     out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced")
 964                 elif is_skipped_in_module(p_name, item_name):
 965                     t_item = type(item)
 966                     out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str(
 967                         t_item) + " skipped")
 968                 else:
 969                     self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name)
 970                 self._defined[item_name] = True
 971                 out(0, "") # empty line after each item
 972         values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None)
 973         if values_to_add:
 974             self.footer_buf.out(0, "# intermittent names")
 975             for value in values_to_add:
 976                 self.footer_buf.out(0, value)
 977             # imports: last, because previous parts could alter used_imports or hidden_imports
 978         self.output_import_froms()
 979         if self.imports_buf.isEmpty():
 980             self.imports_buf.out(0, "# no imports")
 981         self.imports_buf.out(0, "") # empty line after imports
 982
 983     def output_import_froms(self):
 984         """Mention all imported names known within the module, wrapping as per PEP."""
 985         out = self.imports_buf.out
 986         if self.used_imports:
 987             self.add_import_header_if_needed()
 988             for mod_name in sorted_no_case(self.used_imports.keys()):
 989                 import_names = self.used_imports[mod_name]
 990                 if import_names:
 991                     self._defined[mod_name] = True
 992                     right_pos = 0 # tracks width of list to fold it at right margin
 993                     import_heading = "from % s import (" % mod_name
 994                     right_pos += len(import_heading)
 995                     names_pack = [import_heading]
 996                     indent_level = 0
 997                     import_names = list(import_names)
 998                     import_names.sort()
 999                     for n in import_names:
1000                         self._defined[n] = True
1001                         len_n = len(n)
1002                         if right_pos + len_n >= 78:
1003                             out(indent_level, *names_pack)
1004                             names_pack = [n, ", "]
1005                             if indent_level == 0:
1006                                 indent_level = 1 # all but first line is indented
1007                             right_pos = self.indent_size + len_n + 2
1008                         else:
1009                             names_pack.append(n)
1010                             names_pack.append(", ")
1011                             right_pos += (len_n + 2)
1012                         # last line is...
1013                     if indent_level == 0: # one line
1014                         names_pack[0] = names_pack[0][:-1] # cut off lpar
1015                         names_pack[-1] = "" # cut last comma
1016                     else: # last line of multiline
1017                         names_pack[-1] = ")" # last comma -> rpar
1018                     out(indent_level, *names_pack)
1019
1020                     out(0, "") # empty line after group
1021
1022         if self.hidden_imports:
1023             self.add_import_header_if_needed()
1024             for mod_name in sorted_no_case(self.hidden_imports.keys()):
1025                 out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name])
1026             out(0, "") # empty line after group