performance for generator: Split big generated modules (like _Gtk, PyQt) into smaller...
[idea/community.git] / python / helpers / pycharm_generator_utils / module_redeclarator.py
1 import keyword
2
3 from pycharm_generator_utils.util_methods import *
4 from pycharm_generator_utils.constants import *
5
6
7 class emptylistdict(dict):
8     """defaultdict not available before 2.5; simplest reimplementation using [] as default"""
9
10     def __getitem__(self, item):
11         if item in self:
12             return dict.__getitem__(self, item)
13         else:
14             it = []
15             self.__setitem__(item, it)
16             return it
17
18 class Buf(object):
19     """Buffers data in a list, can write to a file. Indentation is provided externally."""
20
21     def __init__(self, indenter):
22         self.data = []
23         self.indenter = indenter
24
25     def put(self, data):
26         if data:
27             self.data.append(ensureUnicode(data))
28
29     def out(self, indent, *what):
30         """Output the arguments, indenting as needed, and adding an eol"""
31         self.put(self.indenter.indent(indent))
32         for item in what:
33             self.put(item)
34         self.put("\n")
35
36     def flush_bytes(self, outfile):
37         for data in self.data:
38             outfile.write(data.encode(OUT_ENCODING, "replace"))
39
40     def flush_str(self, outfile):
41         for data in self.data:
42             outfile.write(data)
43
44     if version[0] < 3:
45         flush = flush_bytes
46     else:
47         flush = flush_str
48
49     def isEmpty(self):
50         return len(self.data) == 0
51
52
53 class ClassBuf(Buf):
54     def __init__(self, name, indenter):
55         super(ClassBuf, self).__init__(indenter)
56         self.name = name
57
58 #noinspection PyUnresolvedReferences,PyBroadException
59 class ModuleRedeclarator(object):
60     def __init__(self, module, outfile, mod_filename, indent_size=4, doing_builtins=False):
61         """
62         Create new instance.
63         @param module module to restore.
64         @param outfile output file, must be open and writable.
65         @param mod_filename filename of binary module (the .dll or .so)
66         @param indent_size amount of space characters per indent
67         """
68         self.module = module
69         self.outfile = outfile # where we finally write
70         self.mod_filename = mod_filename
71         # we write things into buffers out-of-order
72         self.header_buf = Buf(self)
73         self.imports_buf = Buf(self)
74         self.functions_buf = Buf(self)
75         self.classes_buf = Buf(self)
76         self.classes_buffs = list()
77         self.footer_buf = Buf(self)
78         self.indent_size = indent_size
79         self._indent_step = " " * self.indent_size
80         #
81         self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module}
82         self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported
83         # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome.
84         self._defined = {} # stores True for every name defined so far, to break circular refs in values
85         self.doing_builtins = doing_builtins
86         self.ret_type_cache = {}
87         self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names
88
89     def _initializeQApp(self):
90         try:  # QtGui should be imported _before_ QtCore package.
91             # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+
92             # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type"
93             import PyQt4.QtGui
94         except ImportError:
95             pass
96
97         # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection)
98         # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS
99         try:
100             from PyQt4.QtCore import QCoreApplication
101             self.app = QCoreApplication([])
102             return
103         except ImportError:
104             pass
105         try:
106             from PyQt5.QtCore import QCoreApplication
107             self.app = QCoreApplication([])
108         except ImportError:
109             pass
110
111     def indent(self, level):
112         """Return indentation whitespace for given level."""
113         return self._indent_step * level
114
115     def flush(self):
116         init = None
117         try:
118             if self.mod_filename and len(self.classes_buffs) >= 30:
119                 mod_path = self.outfile.strip(".py")
120
121                 fname = build_output_name(mod_path, "__init__")
122                 init = fopen(fname, "w")
123                 for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf):
124                     buf.flush(init)
125
126                 data = ""
127                 for buf in self.classes_buffs:
128                     fname = build_output_name(mod_path, buf.name)
129                     dummy = fopen(fname, "w")
130                     buf.flush(dummy)
131                     data += "from " + buf.name + " import " + buf.name + "\n"
132                     dummy.close()
133
134                 init.write(data)
135                 self.footer_buf.flush(init)
136             else:
137                 init = fopen(self.outfile, "w")
138                 for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf):
139                     buf.flush(init)
140
141                 for buf in self.classes_buffs:
142                     buf.flush(init)
143
144                 self.footer_buf.flush(init)
145
146         finally:
147             if init is not None and not init.closed:
148                 init.close()
149
150     # Some builtin classes effectively change __init__ signature without overriding it.
151     # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS
152     def fake_builtin_init(self):
153         pass # just a callable, sig doesn't matter
154
155     fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead
156
157
158     def find_imported_name(self, item):
159         """
160         Finds out how the item is represented in imported modules.
161         @param item what to check
162         @return qualified name (like "sys.stdin") or None
163         """
164         # TODO: return a pair, not a glued string
165         if not isinstance(item, SIMPLEST_TYPES):
166             for mname in self.imported_modules:
167                 m = self.imported_modules[mname]
168                 for inner_name in m.__dict__:
169                     suspect = getattr(m, inner_name)
170                     if suspect is item:
171                         if mname:
172                             mname += "."
173                         elif self.module is the_builtins: # don't short-circuit builtins
174                             return None
175                         return mname + inner_name
176         return None
177
178     _initializers = (
179         (dict, "{}"),
180         (tuple, "()"),
181         (list, "[]"),
182     )
183
184     def invent_initializer(self, a_type):
185         """
186         Returns an innocuous initializer expression for a_type, or "None"
187         """
188         for initializer_type, r in self._initializers:
189             if initializer_type == a_type:
190                 return r
191                 # NOTE: here we could handle things like defaultdict, sets, etc if we wanted
192         return "None"
193
194
195     def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None):
196         """
197         Formats and outputs value (it occupies an entire line or several lines).
198         @param out function that does output (a Buf.out)
199         @param p_value the value.
200         @param indent indent level.
201         @param prefix text to print before the value
202         @param postfix text to print after the value
203         @param as_name hints which name are we trying to print; helps with circular refs.
204         @param seen_values a list of keys we've seen if we're processing a dict
205         """
206         SELF_VALUE = "<value is a self-reference, replaced by this string>"
207         ERR_VALUE = "<failed to retrieve the value>"
208         if isinstance(p_value, SIMPLEST_TYPES):
209             out(indent, prefix, reliable_repr(p_value), postfix)
210         else:
211             if sys.platform == "cli":
212                 imported_name = None
213             else:
214                 imported_name = self.find_imported_name(p_value)
215             if imported_name:
216                 out(indent, prefix, imported_name, postfix)
217                 # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name
218                 # else we could potentially return smth we did not otherwise import. but not likely.
219             else:
220                 if isinstance(p_value, (list, tuple)):
221                     if not seen_values:
222                         seen_values = [p_value]
223                     if len(p_value) == 0:
224                         out(indent, prefix, repr(p_value), postfix)
225                     else:
226                         if isinstance(p_value, list):
227                             lpar, rpar = "[", "]"
228                         else:
229                             lpar, rpar = "(", ")"
230                         out(indent, prefix, lpar)
231                         for value in p_value:
232                             if value in seen_values:
233                                 value = SELF_VALUE
234                             elif not isinstance(value, SIMPLEST_TYPES):
235                                 seen_values.append(value)
236                             self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values)
237                         out(indent, rpar, postfix)
238                 elif isinstance(p_value, dict):
239                     if len(p_value) == 0:
240                         out(indent, prefix, repr(p_value), postfix)
241                     else:
242                         if not seen_values:
243                             seen_values = [p_value]
244                         out(indent, prefix, "{")
245                         keys = list(p_value.keys())
246                         try:
247                             keys.sort()
248                         except TypeError:
249                             pass # unsortable keys happen, e,g, in py3k _ctypes
250                         for k in keys:
251                             value = p_value[k]
252
253                             try:
254                                 is_seen = value in seen_values
255                             except:
256                                 is_seen = False
257                                 value = ERR_VALUE
258
259                             if is_seen:
260                                 value = SELF_VALUE
261                             elif not isinstance(value, SIMPLEST_TYPES):
262                                 seen_values.append(value)
263                             if isinstance(k, SIMPLEST_TYPES):
264                                 self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",",
265                                                seen_values=seen_values)
266                             else:
267                                 # both key and value need fancy formatting
268                                 self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values)
269                                 self.fmt_value(out, value, indent + 2, seen_values=seen_values)
270                                 out(indent + 1, ",")
271                         out(indent, "}", postfix)
272                 else: # something else, maybe representable
273                     # look up this value in the module.
274                     if sys.platform == "cli":
275                         out(indent, prefix, "None", postfix)
276                         return
277                     found_name = ""
278                     for inner_name in self.module.__dict__:
279                         if self.module.__dict__[inner_name] is p_value:
280                             found_name = inner_name
281                             break
282                     if self._defined.get(found_name, False):
283                         out(indent, prefix, found_name, postfix)
284                     else:
285                         # a forward / circular declaration happens
286                         notice = ""
287                         real_value = cleanup(repr(p_value))
288                         if found_name:
289                             if found_name == as_name:
290                                 notice = " # (!) real value is %r" % real_value
291                                 real_value = "None"
292                             else:
293                                 notice = " # (!) forward: %s, real value is %r" % (found_name, real_value)
294                         if SANE_REPR_RE.match(real_value):
295                             out(indent, prefix, real_value, postfix, notice)
296                         else:
297                             if not found_name:
298                                 notice = " # (!) real value is %r" % real_value
299                             out(indent, prefix, "None", postfix, notice)
300
301     def get_ret_type(self, attr):
302         """
303         Returns a return type string as given by T_RETURN in tokens, or None
304         """
305         if attr:
306             ret_type = RET_TYPE.get(attr, None)
307             if ret_type:
308                 return ret_type
309             thing = getattr(self.module, attr, None)
310             if thing:
311                 if not isinstance(thing, type) and is_callable(thing): # a function
312                     return None # TODO: maybe divinate a return type; see pygame.mixer.Channel
313                 return attr
314                 # adds no noticeable slowdown, I did measure. dch.
315             for im_name, im_module in self.imported_modules.items():
316                 cache_key = (im_name, attr)
317                 cached = self.ret_type_cache.get(cache_key, None)
318                 if cached:
319                     return cached
320                 ret_type = getattr(im_module, attr, None)
321                 if ret_type:
322                     if isinstance(ret_type, type):
323                         # detect a constructor
324                         constr_args = detect_constructor(ret_type)
325                         if constr_args is None:
326                             constr_args = "*(), **{}" # a silly catch-all constructor
327                         reference = "%s(%s)" % (attr, constr_args)
328                     elif is_callable(ret_type): # a function, classes are ruled out above
329                         return None
330                     else:
331                         reference = attr
332                     if im_name:
333                         result = "%s.%s" % (im_name, reference)
334                     else: # built-in
335                         result = reference
336                     self.ret_type_cache[cache_key] = result
337                     return result
338                     # TODO: handle things like "[a, b,..] and (foo,..)"
339         return None
340
341
342     SIG_DOC_NOTE = "restored from __doc__"
343     SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ "
344
345     def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None):
346         """
347         @param signature_string: parameter list extracted from the doc string.
348         @param class_name: name of the containing class, or None
349         @param deco: decorator to use
350         @param ret_hint: return type hint, if available
351         @return (reconstructed_spec, return_type, note) or (None, _, _) if failed.
352         """
353         action("restoring func %r of class %r", signature_string, class_name)
354         # parse
355         parsing_failed = False
356         ret_type = None
357         try:
358             # strict parsing
359             tokens = paramSeqAndRest.parseString(signature_string, True)
360             ret_name = None
361             if tokens:
362                 ret_t = tokens[-1]
363                 if ret_t[0] is T_RETURN:
364                     ret_name = ret_t[1]
365             ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint)
366         except ParseException:
367             # it did not parse completely; scavenge what we can
368             parsing_failed = True
369             tokens = []
370             try:
371                 # most unrestrictive parsing
372                 tokens = paramSeq.parseString(signature_string, False)
373             except ParseException:
374                 pass
375                 #
376         seq = transform_seq(tokens)
377
378         # add safe defaults for unparsed
379         if parsing_failed:
380             doc_node = self.SIG_DOC_UNRELIABLY
381             starred = None
382             double_starred = None
383             for one in seq:
384                 if type(one) is str:
385                     if one.startswith("**"):
386                         double_starred = one
387                     elif one.startswith("*"):
388                         starred = one
389             if not starred:
390                 seq.append("*args")
391             if not double_starred:
392                 seq.append("**kwargs")
393         else:
394             doc_node = self.SIG_DOC_NOTE
395
396         # add 'self' if needed YYY
397         if class_name and (not seq or seq[0] != 'self'):
398             first_param = propose_first_param(deco)
399             if first_param:
400                 seq.insert(0, first_param)
401         seq = make_names_unique(seq)
402         return (seq, ret_type, doc_node)
403
404     def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False):
405         """
406         @param func_doc: __doc__ of the function.
407         @param func_id: name to look for as identifier of the function in docstring
408         @param func_name: name of the function.
409         @param class_name: name of the containing class, or None
410         @param deco: decorator to use
411         @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed.
412         """
413         if sip_generated:
414             overloads = []
415             for part in func_doc.split('\n'):
416                 signature = func_id + '('
417                 i = part.find(signature)
418                 if i >= 0:
419                     overloads.append(part[i + len(signature):])
420             if len(overloads) > 1:
421                 docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads]
422                 ret_types = []
423                 for result in docstring_results:
424                     rt = result[1]
425                     if rt and rt not in ret_types:
426                         ret_types.append(rt)
427                 if ret_types:
428                     ret_literal = " or ".join(ret_types)
429                 else:
430                     ret_literal = None
431                 param_lists = [result[0] for result in docstring_results]
432                 spec = build_signature(func_name, restore_parameters_for_overloads(param_lists))
433                 return (spec, ret_literal, "restored from __doc__ with multiple overloads")
434
435         # find the first thing to look like a definition
436         prefix_re = re.compile("\s*(?:(\w+)[ \\t]+)?" + func_id + "\s*\(") # "foo(..." or "int foo(..."
437         match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time
438         # parse the part that looks right
439         if match:
440             ret_hint = match.group(1)
441             params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint)
442             spec = func_name + flatten(params)
443             return (spec, ret_literal, doc_note)
444         else:
445             return (None, None, None)
446
447
448     def is_predefined_builtin(self, module_name, class_name, func_name):
449         return self.doing_builtins and module_name == BUILTIN_MOD_NAME and (
450             class_name, func_name) in PREDEFINED_BUILTIN_SIGS
451
452
453     def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None):
454         """
455         Restore function argument list as best we can.
456         @param out output function of a Buf
457         @param p_func function or method object
458         @param p_name function name as known to owner
459         @param indent indentation level
460         @param p_class the class that contains this function as a method
461         @param p_modname module name
462         @param seen {id(func): name} map of functions already seen in the same namespace;
463                id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7)
464         """
465         action("redoing func %r of class %r", p_name, p_class)
466         if seen is not None:
467             other_func = seen.get(id(p_func), None)
468             if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None):
469                 # _bisect.bisect == _bisect.bisect_right in py31, but docs differ
470                 out(indent, p_name, " = ", seen[id(p_func)])
471                 out(indent, "")
472                 return
473             else:
474                 seen[id(p_func)] = p_name
475                 # real work
476         if classname is None:
477             classname = p_class and p_class.__name__ or None
478         if p_class and hasattr(p_class, '__mro__'):
479             sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)]
480         else:
481             sip_generated = False
482         deco = None
483         deco_comment = ""
484         mod_class_method_tuple = (p_modname, classname, p_name)
485         ret_literal = None
486         is_init = False
487         # any decorators?
488         action("redoing decos of func %r of class %r", p_name, p_class)
489         if self.doing_builtins and p_modname == BUILTIN_MOD_NAME:
490             deco = KNOWN_DECORATORS.get((classname, p_name), None)
491             if deco:
492                 deco_comment = " # known case"
493         elif p_class and p_name in p_class.__dict__:
494             # detect native methods declared with METH_CLASS flag
495             descriptor = p_class.__dict__[p_name]
496             if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'):
497                 # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython
498                 deco = "classmethod"
499             elif type(p_func).__name__.startswith('staticmethod'):
500                 deco = "staticmethod"
501         if p_name == "__new__":
502             deco = "staticmethod"
503             deco_comment = " # known case of __new__"
504
505         action("redoing innards of func %r of class %r", p_name, p_class)
506         if deco and HAS_DECORATORS:
507             out(indent, "@", deco, deco_comment)
508         if inspect and inspect.isfunction(p_func):
509             out(indent, "def ", p_name, restore_by_inspect(p_func), ": # reliably restored by inspect", )
510             out_doc_attr(out, p_func, indent + 1, p_class)
511         elif self.is_predefined_builtin(*mod_class_method_tuple):
512             spec, sig_note = restore_predefined_builtin(classname, p_name)
513             out(indent, "def ", spec, ": # ", sig_note)
514             out_doc_attr(out, p_func, indent + 1, p_class)
515         elif sys.platform == 'cli' and is_clr_type(p_class):
516             spec, sig_note = restore_clr(p_name, p_class)
517             if not spec: return
518             if sig_note:
519                 out(indent, "def ", spec, ": #", sig_note)
520             else:
521                 out(indent, "def ", spec, ":")
522             if not p_name in ['__gt__', '__ge__', '__lt__', '__le__', '__ne__', '__reduce_ex__', '__str__']:
523                 out_doc_attr(out, p_func, indent + 1, p_class)
524         elif mod_class_method_tuple in PREDEFINED_MOD_CLASS_SIGS:
525             sig, ret_literal = PREDEFINED_MOD_CLASS_SIGS[mod_class_method_tuple]
526             if classname:
527                 ofwhat = "%s.%s.%s" % mod_class_method_tuple
528             else:
529                 ofwhat = "%s.%s" % (p_modname, p_name)
530             out(indent, "def ", p_name, sig, ": # known case of ", ofwhat)
531             out_doc_attr(out, p_func, indent + 1, p_class)
532         else:
533             # __doc__ is our best source of arglist
534             sig_note = "real signature unknown"
535             spec = ""
536             is_init = (p_name == "__init__" and p_class is not None)
537             funcdoc = None
538             if is_init and hasattr(p_class, "__doc__"):
539                 if hasattr(p_func, "__doc__"):
540                     funcdoc = p_func.__doc__
541                 if funcdoc == object.__init__.__doc__:
542                     funcdoc = p_class.__doc__
543             elif hasattr(p_func, "__doc__"):
544                 funcdoc = p_func.__doc__
545             sig_restored = False
546             action("parsing doc of func %r of class %r", p_name, p_class)
547             if isinstance(funcdoc, STR_TYPES):
548                 (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, p_name, p_name, classname, deco,
549                                                                       sip_generated)
550                 if spec is None and p_name == '__init__' and classname:
551                     (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, classname, p_name, classname, deco,
552                                                                           sip_generated)
553                 sig_restored = spec is not None
554                 if more_notes:
555                     if sig_note:
556                         sig_note += "; "
557                     sig_note += more_notes
558             if not sig_restored:
559                 # use an allow-all declaration
560                 decl = []
561                 if p_class:
562                     first_param = propose_first_param(deco)
563                     if first_param:
564                         decl.append(first_param)
565                 decl.append("*args")
566                 decl.append("**kwargs")
567                 spec = p_name + "(" + ", ".join(decl) + ")"
568             out(indent, "def ", spec, ": # ", sig_note)
569             # to reduce size of stubs, don't output same docstring twice for class and its __init__ method
570             if not is_init or funcdoc != p_class.__doc__:
571                 out_docstring(out, funcdoc, indent + 1)
572                 # body
573         if ret_literal and not is_init:
574             out(indent + 1, "return ", ret_literal)
575         else:
576             out(indent + 1, "pass")
577         if deco and not HAS_DECORATORS:
578             out(indent, p_name, " = ", deco, "(", p_name, ")", deco_comment)
579         out(0, "") # empty line after each item
580
581
582     def redo_class(self, out, p_class, p_name, indent, p_modname=None, seen=None, inspect_dir=False):
583         """
584         Restores a class definition.
585         @param out output function of a relevant buf
586         @param p_class the class object
587         @param p_name class name as known to owner
588         @param indent indentation level
589         @param p_modname name of module
590         @param seen {class: name} map of classes already seen in the same namespace
591         """
592         action("redoing class %r of module %r", p_name, p_modname)
593         if seen is not None:
594             if p_class in seen:
595                 out(indent, p_name, " = ", seen[p_class])
596                 out(indent, "")
597                 return
598             else:
599                 seen[p_class] = p_name
600         bases = get_bases(p_class)
601         base_def = ""
602         skipped_bases = []
603         if bases:
604             skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions']
605             skip_qualifiers.extend(KNOWN_FAKE_REEXPORTERS.get(p_modname, ()))
606             bases_list = [] # what we'll render in the class decl
607             for base in bases:
608                 if [1 for (cls, mdl) in KNOWN_FAKE_BASES if cls == base and mdl != self.module]:
609                     # our base is a wrapper and our module is not its defining module
610                     skipped_bases.append(str(base))
611                     continue
612                     # somehow import every base class
613                 base_name = base.__name__
614                 qual_module_name = qualifier_of(base, skip_qualifiers)
615                 got_existing_import = False
616                 if qual_module_name:
617                     if qual_module_name in self.used_imports:
618                         import_list = self.used_imports[qual_module_name]
619                         if base in import_list:
620                             bases_list.append(base_name) # unqualified: already set to import
621                             got_existing_import = True
622                     if not got_existing_import:
623                         mangled_qualifier = "__" + qual_module_name.replace('.', '_') # foo.bar -> __foo_bar
624                         bases_list.append(mangled_qualifier + "." + base_name)
625                         self.hidden_imports[qual_module_name] = mangled_qualifier
626                 else:
627                     bases_list.append(base_name)
628             base_def = "(" + ", ".join(bases_list) + ")"
629         out(indent, "class ", p_name, base_def, ":",
630             skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "")
631         out_doc_attr(out, p_class, indent + 1)
632         # inner parts
633         methods = {}
634         properties = {}
635         others = {}
636         we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object"
637         field_source = {}
638         try:
639             if hasattr(p_class, "__dict__") and not inspect_dir:
640                 field_source = p_class.__dict__
641                 field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here
642             else:
643                 field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare
644         except:
645             field_keys = ()
646         for item_name in field_keys:
647             if item_name in ("__doc__", "__module__"):
648                 if we_are_the_base_class:
649                     item = "" # must be declared in base types
650                 else:
651                     continue # in all other cases must be skipped
652             elif keyword.iskeyword(item_name):  # for example, PyQt4 contains definitions of methods named 'exec'
653                 continue
654             else:
655                 try:
656                     item = getattr(p_class, item_name) # let getters do the magic
657                 except AttributeError:
658                     item = field_source[item_name] # have it raw
659                 except Exception:
660                     continue
661             if is_callable(item) and not isinstance(item, type):
662                 methods[item_name] = item
663             elif is_property(item):
664                 properties[item_name] = item
665             else:
666                 others[item_name] = item
667                 #
668         if we_are_the_base_class:
669             others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :)
670             # add fake __init__s to have the right sig
671         if p_class in FAKE_BUILTIN_INITS:
672             methods["__init__"] = self.fake_builtin_init
673             note("Faking init of %s", p_name)
674         elif '__init__' not in methods:
675             init_method = getattr(p_class, '__init__', None)
676             if init_method:
677                 methods['__init__'] = init_method
678
679         #
680         seen_funcs = {}
681         for item_name in sorted_no_case(methods.keys()):
682             item = methods[item_name]
683             try:
684                 self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs)
685             except:
686                 handle_error_func(item_name, out)
687                 #
688         known_props = KNOWN_PROPS.get(p_modname, {})
689         a_setter = "lambda self, v: None"
690         a_deleter = "lambda self: None"
691         for item_name in sorted_no_case(properties.keys()):
692             item = properties[item_name]
693             prop_docstring = getattr(item, '__doc__', None)
694             prop_key = (p_name, item_name)
695             if prop_key in known_props:
696                 prop_descr = known_props.get(prop_key, None)
697                 if prop_descr is None:
698                     continue # explicitly omitted
699                 acc_line, getter_and_type = prop_descr
700                 if getter_and_type:
701                     getter, prop_type = getter_and_type
702                 else:
703                     getter, prop_type = None, None
704                 out(indent + 1, item_name,
705                     " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")"
706                 )
707                 if prop_type:
708                     if prop_docstring:
709                         out(indent + 1, '"""', prop_docstring)
710                         out(0, "")
711                         out(indent + 1, ':type: ', prop_type)
712                         out(indent + 1, '"""')
713                     else:
714                         out(indent + 1, '""":type: ', prop_type, '"""')
715                     out(0, "")
716             else:
717                 out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None)  # default")
718                 if prop_docstring:
719                     out(indent + 1, '"""', prop_docstring, '"""')
720                 out(0, "")
721         if properties:
722             out(0, "") # empty line after the block
723             #
724         for item_name in sorted_no_case(others.keys()):
725             item = others[item_name]
726             self.fmt_value(out, item, indent + 1, prefix=item_name + " = ")
727         if p_name == "object":
728             out(indent + 1, "__module__ = ''")
729         if others:
730             out(0, "") # empty line after the block
731             #
732         if not methods and not properties and not others:
733             out(indent + 1, "pass")
734
735
736
737     def redo_simple_header(self, p_name):
738         """Puts boilerplate code on the top"""
739         out = self.header_buf.out # 1st class methods rule :)
740         out(0, "# encoding: %s" % OUT_ENCODING) # line 1
741         # NOTE: maybe encoding should be selectable
742         if hasattr(self.module, "__name__"):
743             self_name = self.module.__name__
744             if self_name != p_name:
745                 mod_name = " calls itself " + self_name
746             else:
747                 mod_name = ""
748         else:
749             mod_name = " does not know its name"
750         out(0, "# module ", p_name, mod_name) # line 2
751
752         BUILT_IN_HEADER = "(built-in)"
753         if self.mod_filename:
754             filename = self.mod_filename
755         elif p_name in sys.builtin_module_names:
756             filename = BUILT_IN_HEADER
757         else:
758             filename = getattr(self.module, "__file__", BUILT_IN_HEADER)
759
760         out(0, "# from %s" % filename)  # line 3
761         out(0, "# by generator %s" % VERSION) # line 4
762         if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6:
763             out(0, "from __future__ import print_function")
764         out_doc_attr(out, self.module, 0)
765
766
767     def redo_imports(self):
768         module_type = type(sys)
769         for item_name in self.module.__dict__.keys():
770             try:
771                 item = self.module.__dict__[item_name]
772             except:
773                 continue
774             if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here
775                 self.imported_modules[item_name] = item
776                 self.add_import_header_if_needed()
777                 ref_notice = getattr(item, "__file__", str(item))
778                 if hasattr(item, "__name__"):
779                     self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice)
780                 else:
781                     self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice)
782
783     def add_import_header_if_needed(self):
784         if self.imports_buf.isEmpty():
785             self.imports_buf.out(0, "")
786             self.imports_buf.out(0, "# imports")
787
788
789     def redo(self, p_name, inspect_dir):
790         """
791         Restores module declarations.
792         Intended for built-in modules and thus does not handle import statements.
793         @param p_name name of module
794         """
795         action("redoing header of module %r %r", p_name, str(self.module))
796
797         if "pyqt" in p_name.lower():   # qt specific patch
798             self._initializeQApp()
799
800         self.redo_simple_header(p_name)
801
802         # find whatever other self.imported_modules the module knows; effectively these are imports
803         action("redoing imports of module %r %r", p_name, str(self.module))
804         try:
805             self.redo_imports()
806         except:
807             pass
808
809         action("redoing innards of module %r %r", p_name, str(self.module))
810
811         module_type = type(sys)
812         # group what we have into buckets
813         vars_simple = {}
814         vars_complex = {}
815         funcs = {}
816         classes = {}
817         module_dict = self.module.__dict__
818         if inspect_dir:
819             module_dict = dir(self.module)
820         for item_name in module_dict:
821             note("looking at %s", item_name)
822             if item_name in (
823                 "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"):
824                 continue # handled otherwise
825             try:
826                 item = getattr(self.module, item_name) # let getters do the magic
827             except AttributeError:
828                 if not item_name in self.module.__dict__: continue
829                 item = self.module.__dict__[item_name] # have it raw
830                 # check if it has percolated from an imported module
831             except NotImplementedError:
832                 if not item_name in self.module.__dict__: continue
833                 item = self.module.__dict__[item_name] # have it raw
834
835             # unless we're adamantly positive that the name was imported, we assume it is defined here
836             mod_name = None # module from which p_name might have been imported
837             # IronPython has non-trivial reexports in System module, but not in others:
838             skip_modname = sys.platform == "cli" and p_name != "System"
839             surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ())
840             ## can't figure weirdness in some modules, assume no reexports:
841             #skip_modname =  skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS
842             if not skip_modname:
843                 try:
844                     mod_name = getattr(item, '__module__', None)
845                 except:
846                     pass
847                     # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect)
848             maybe_import_mod_name = mod_name or ""
849             import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name)
850             note("mod_name = %s, prospective = %s,  from top = %s", mod_name, maybe_import_mod_name, import_is_from_top)
851             want_to_import = False
852             if (mod_name
853                 and mod_name != BUILTIN_MOD_NAME
854                 and mod_name != p_name
855                 and mod_name not in surely_not_imported_mods
856                 and not import_is_from_top
857             ):
858                 # import looks valid, but maybe it's a .py file? we're certain not to import from .py
859                 # e.g. this rules out _collections import collections and builtins import site.
860                 try:
861                     imported = __import__(mod_name) # ok to repeat, Python caches for us
862                     if imported:
863                         qualifiers = mod_name.split(".")[1:]
864                         for qual in qualifiers:
865                             imported = getattr(imported, qual, None)
866                             if not imported:
867                                 break
868                         imported_path = (getattr(imported, '__file__', False) or "").lower()
869                         want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc'))
870                         note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import)
871                 except ImportError:
872                     want_to_import = False
873                     # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all
874                 if want_to_import:
875                     import_list = self.used_imports[mod_name]
876                     if item_name not in import_list:
877                         import_list.append(item_name)
878             if not want_to_import:
879                 if isinstance(item, type) or type(item).__name__ == 'classobj':
880                     classes[item_name] = item
881                 elif is_callable(item): # some classes are callable, check them before functions
882                     funcs[item_name] = item
883                 elif isinstance(item, module_type):
884                     continue # self.imported_modules handled above already
885                 else:
886                     if isinstance(item, SIMPLEST_TYPES):
887                         vars_simple[item_name] = item
888                     else:
889                         vars_complex[item_name] = item
890
891         # sort and output every bucket
892         action("outputting innards of module %r %r", p_name, str(self.module))
893         #
894         omitted_names = OMIT_NAME_IN_MODULE.get(p_name, [])
895         if vars_simple:
896             out = self.functions_buf.out
897             prefix = "" # try to group variables by common prefix
898             PREFIX_LEN = 2 # default prefix length if we can't guess better
899             out(0, "# Variables with simple values")
900             for item_name in sorted_no_case(vars_simple.keys()):
901                 if item_name in omitted_names:
902                     out(0, "# definition of " + item_name + " omitted")
903                     continue
904                 item = vars_simple[item_name]
905                 # track the prefix
906                 if len(item_name) >= PREFIX_LEN:
907                     prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore
908                     if prefix_pos < 1:
909                         prefix_pos = PREFIX_LEN
910                     beg = item_name[0:prefix_pos]
911                     if prefix != beg:
912                         out(0, "") # space out from other prefix
913                         prefix = beg
914                 else:
915                     prefix = ""
916                     # output
917                 replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
918                 if replacement is not None:
919                     out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced")
920                 elif is_skipped_in_module(p_name, item_name):
921                     t_item = type(item)
922                     out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item),
923                         " skipped")
924                 else:
925                     self.fmt_value(out, item, 0, prefix=item_name + " = ")
926                 self._defined[item_name] = True
927             out(0, "") # empty line after vars
928             #
929         if funcs:
930             out = self.functions_buf.out
931             out(0, "# functions")
932             out(0, "")
933             seen_funcs = {}
934             for item_name in sorted_no_case(funcs.keys()):
935                 if item_name in omitted_names:
936                     out(0, "# definition of ", item_name, " omitted")
937                     continue
938                 item = funcs[item_name]
939                 try:
940                     self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs)
941                 except:
942                     handle_error_func(item_name, out)
943         else:
944             self.functions_buf.out(0, "# no functions")
945             #
946         if classes:
947             self.classes_buf.out(0, "# classes")
948             self.classes_buf.out(0, "")
949             seen_classes = {}
950             # sort classes so that inheritance order is preserved
951             cls_list = [] # items are (class_name, mro_tuple)
952             for cls_name in sorted_no_case(classes.keys()):
953                 cls = classes[cls_name]
954                 ins_index = len(cls_list)
955                 for i in range(ins_index):
956                     maybe_child_bases = cls_list[i][1]
957                     if cls in maybe_child_bases:
958                         ins_index = i # we could not go farther than current ins_index
959                         break         # ...and need not go fartehr than first known child
960                 cls_list.insert(ins_index, (cls_name, get_mro(cls)))
961             for item_name in [cls_item[0] for cls_item in cls_list]:
962                 buf = ClassBuf(item_name, self)
963                 self.classes_buffs.append(buf)
964                 out = buf.out
965                 if item_name in omitted_names:
966                     out(0, "# definition of ", item_name, " omitted")
967                     continue
968                 item = classes[item_name]
969                 self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir)
970                 self._defined[item_name] = True
971                 out(0, "") # empty line after each item
972
973             if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3:
974                 # classobj still supported
975                 txt = classobj_txt
976                 self.classes_buf.out(0, txt)
977
978             if self.doing_builtins and p_name == BUILTIN_MOD_NAME:
979                 txt = create_generator()
980                 self.classes_buf.out(0, txt)
981
982                 # Fake <type 'namedtuple'>
983                 if version[0] >= 3 or (version[0] == 2 and version[1] >= 6):
984                     namedtuple_text = create_named_tuple()
985                     self.classes_buf.out(0, namedtuple_text)
986
987         else:
988             self.classes_buf.out(0, "# no classes")
989             #
990         if vars_complex:
991             out = self.footer_buf.out
992             out(0, "# variables with complex values")
993             out(0, "")
994             for item_name in sorted_no_case(vars_complex.keys()):
995                 if item_name in omitted_names:
996                     out(0, "# definition of " + item_name + " omitted")
997                     continue
998                 item = vars_complex[item_name]
999                 if str(type(item)) == "<type 'namespace#'>":
1000                     continue            # this is an IronPython submodule, we mustn't generate a reference for it in the base module
1001                 replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None)
1002                 if replacement is not None:
1003                     out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced")
1004                 elif is_skipped_in_module(p_name, item_name):
1005                     t_item = type(item)
1006                     out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str(
1007                         t_item) + " skipped")
1008                 else:
1009                     self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name)
1010                 self._defined[item_name] = True
1011                 out(0, "") # empty line after each item
1012         values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None)
1013         if values_to_add:
1014             self.footer_buf.out(0, "# intermittent names")
1015             for value in values_to_add:
1016                 self.footer_buf.out(0, value)
1017                 # imports: last, because previous parts could alter used_imports or hidden_imports
1018         self.output_import_froms()
1019         if self.imports_buf.isEmpty():
1020             self.imports_buf.out(0, "# no imports")
1021         self.imports_buf.out(0, "") # empty line after imports
1022
1023     def output_import_froms(self):
1024         """Mention all imported names known within the module, wrapping as per PEP."""
1025         out = self.imports_buf.out
1026         if self.used_imports:
1027             self.add_import_header_if_needed()
1028             for mod_name in sorted_no_case(self.used_imports.keys()):
1029                 import_names = self.used_imports[mod_name]
1030                 if import_names:
1031                     self._defined[mod_name] = True
1032                     right_pos = 0 # tracks width of list to fold it at right margin
1033                     import_heading = "from % s import (" % mod_name
1034                     right_pos += len(import_heading)
1035                     names_pack = [import_heading]
1036                     indent_level = 0
1037                     import_names = list(import_names)
1038                     import_names.sort()
1039                     for n in import_names:
1040                         self._defined[n] = True
1041                         len_n = len(n)
1042                         if right_pos + len_n >= 78:
1043                             out(indent_level, *names_pack)
1044                             names_pack = [n, ", "]
1045                             if indent_level == 0:
1046                                 indent_level = 1 # all but first line is indented
1047                             right_pos = self.indent_size + len_n + 2
1048                         else:
1049                             names_pack.append(n)
1050                             names_pack.append(", ")
1051                             right_pos += (len_n + 2)
1052                             # last line is...
1053                     if indent_level == 0: # one line
1054                         names_pack[0] = names_pack[0][:-1] # cut off lpar
1055                         names_pack[-1] = "" # cut last comma
1056                     else: # last line of multiline
1057                         names_pack[-1] = ")" # last comma -> rpar
1058                     out(indent_level, *names_pack)
1059
1060                     out(0, "") # empty line after group
1061
1062         if self.hidden_imports:
1063             self.add_import_header_if_needed()
1064             for mod_name in sorted_no_case(self.hidden_imports.keys()):
1065                 out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name])
1066             out(0, "") # empty line after group