#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2016 California Institute of Technology. # Copyright (c) 2016-2023 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE # # inspired by inspect.py from Python-2.7.6 # inspect.py author: 'Ka-Ping Yee <ping@lfw.org>' # inspect.py merged into original dill.source by Mike McKerns 4/13/14 """ Extensions to python's 'inspect' module, which can be used to retrieve information from live python objects. The methods defined in this module are augmented to facilitate access to source code of interactively defined functions and classes, as well as provide access to source code for objects defined in a file. """ __all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \ '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \ '_importable', 'importable','isdynamic', 'isfrommain'] import linecache import re from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize, isbuiltin, isclass, iscode, isframe, isfunction, ismethod, ismodule, istraceback) from tokenize import TokenError from ._dill import IS_IPYTHON def isfrommain(obj): "check if object was built in __main__" module = getmodule(obj) if module and module.__name__ == '__main__': return True return False def isdynamic(obj): "check if object was built in the interpreter" try: file = getfile(obj) except TypeError: file = None if file == '<stdin>' and isfrommain(obj): return True return False def _matchlambda(func, line): """check if lambda object 'func' matches raw line of code 'line'""" from .detect import code as getcode from .detect import freevars, globalvars, varnames dummy = lambda : '__this_is_a_big_dummy_function__' # process the line (removing leading whitespace, etc) lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) except Exception: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure _f = [line.count(i) for i in freevars(func).keys()] if not _f: # not in closure # check if code matches if _ == code: return True return False # weak check on freevars if not all(_f): return False #XXX: VERY WEAK # weak check on varnames and globalvars _f = varnames(func) _f = [line.count(i) for i in _f[0]+_f[1]] if _f and not all(_f): return False #XXX: VERY WEAK _f = [line.count(i) for i in globalvars(func).keys()] if _f and not all(_f): return False #XXX: VERY WEAK # check if func is a double lambda if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) except Exception: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False #NOTE: should be same code same order, but except for 't' and '\x88' _ = set((i,j) for (i,j) in zip(_,code) if i != j) if len(_) != 1: return False #('t','\x88') return True # check indentsize if not indentsize(line): return False #FIXME: is this a good check??? # check if code 'pattern' matches #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?) _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same? _f = code.split(code[0]) # '\x88' #NOTE: should be same code different order, with different first element _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_))) _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f))) if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())): return True return False def findsource(object): """Return the entire source file and starting line number for an object. For interactively-defined objects, the 'file' is the interpreter's history. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of all the lines in the file and the line number indexes a line in that list. An IOError is raised if the source code cannot be retrieved, while a TypeError is raised for objects where the source code is unavailable (e.g. builtins).""" module = getmodule(object) try: file = getfile(module) except TypeError: file = None is_module_main = (module and module.__name__ == '__main__' and not file) if IS_IPYTHON and is_module_main: #FIXME: quick fix for functions and classes in IPython interpreter try: file = getfile(object) sourcefile = getsourcefile(object) except TypeError: if isclass(object): for object_method in filter(isfunction, object.__dict__.values()): # look for a method of the class file_candidate = getfile(object_method) if not file_candidate.startswith('<ipython-input-'): continue file = file_candidate sourcefile = getsourcefile(object_method) break if file: lines = linecache.getlines(file) else: # fallback to use history history = '\n'.join(get_ipython().history_manager.input_hist_parsed) lines = [line + '\n' for line in history.splitlines()] # use readline when working in interpreter (i.e. __main__ and not file) elif is_module_main: try: import readline err = '' except ImportError: import sys err = sys.exc_info()[1].args[0] if sys.platform[:3] == 'win': err += ", please install 'pyreadline'" if err: raise IOError(err) lbuf = readline.get_current_history_length() lines = [readline.get_history_item(i)+'\n' for i in range(1,lbuf)] else: try: # special handling for class instances if not isclass(object) and isclass(type(object)): # __class__ file = getfile(module) sourcefile = getsourcefile(module) else: # builtins fail with a TypeError file = getfile(object) sourcefile = getsourcefile(object) except (TypeError, AttributeError): # fail with better error file = getfile(object) sourcefile = getsourcefile(object) if not sourcefile and file[:1] + file[-1:] != '<>': raise IOError('source code not available') file = sourcefile if sourcefile else file module = getmodule(object, file) if module: lines = linecache.getlines(file, module.__dict__) else: lines = linecache.getlines(file) if not lines: raise IOError('could not extract source code') #FIXME: all below may fail if exec used (i.e. exec('f = lambda x:x') ) if ismodule(object): return lines, 0 #NOTE: beneficial if search goes from end to start of buffer history name = pat1 = obj = '' pat2 = r'^(\s*@)' # pat1b = r'^(\s*%s\W*=)' % name #FIXME: finds 'f = decorate(f)', not exec if ismethod(object): name = object.__name__ if name == '<lambda>': pat1 = r'(.*(?<!\w)lambda(:|\s))' else: pat1 = r'^(\s*def\s)' object = object.__func__ if isfunction(object): name = object.__name__ if name == '<lambda>': pat1 = r'(.*(?<!\w)lambda(:|\s))' obj = object #XXX: better a copy? else: pat1 = r'^(\s*def\s)' object = object.__code__ if istraceback(object): object = object.tb_frame if isframe(object): object = object.f_code if iscode(object): if not hasattr(object, 'co_firstlineno'): raise IOError('could not find function definition') stdin = object.co_filename == '<stdin>' if stdin: lnum = len(lines) - 1 # can't get lnum easily, so leverage pat if not pat1: pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)' else: lnum = object.co_firstlineno - 1 pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)' pat1 = re.compile(pat1); pat2 = re.compile(pat2) #XXX: candidate_lnum = [n for n in range(lnum) if pat1.match(lines[n])] while lnum > 0: #XXX: won't find decorators in <stdin> ? line = lines[lnum] if pat1.match(line): if not stdin: break # co_firstlineno does the job if name == '<lambda>': # hackery needed to confirm a match if _matchlambda(obj, line): break else: # not a lambda, just look for the name if name in line: # need to check for decorator... hats = 0 for _lnum in range(lnum-1,-1,-1): if pat2.match(lines[_lnum]): hats += 1 else: break lnum = lnum - hats break lnum = lnum - 1 return lines, lnum try: # turn instances into classes if not isclass(object) and isclass(type(object)): # __class__ object = object.__class__ #XXX: sometimes type(class) is better? #XXX: we don't find how the instance was built except AttributeError: pass if isclass(object): name = object.__name__ pat = re.compile(r'^(\s*)class\s*' + name + r'\b') # make some effort to find the best matching class definition: # use the one with the least indentation, which is the one # that's most probably not inside a function definition. candidates = [] for i in range(len(lines)-1,-1,-1): match = pat.match(lines[i]) if match: # if it's at toplevel, it's already the best one if lines[i][0] == 'c': return lines, i # else add whitespace to candidate list candidates.append((match.group(1), i)) if candidates: # this will sort by whitespace, and by line number, # less whitespace first #XXX: should sort high lnum before low candidates.sort() return lines, candidates[0][1] else: raise IOError('could not find class definition') raise IOError('could not find code object') def getblocks(object, lstrip=False, enclosing=False, locate=False): """Return a list of source lines and starting line number for an object. Interactively-defined objects refer to lines in the interpreter's history. If enclosing=True, then also return any enclosing code. If lstrip=True, ensure there is no indentation in the first line of code. If locate=True, then also return the line number for the block of code. DEPRECATED: use 'getsourcelines' instead """ lines, lnum = findsource(object) if ismodule(object): if lstrip: lines = _outdent(lines) return ([lines], [0]) if locate is True else [lines] #XXX: 'enclosing' means: closures only? or classes and files? indent = indentsize(lines[lnum]) block = getblock(lines[lnum:]) #XXX: catch any TokenError here? if not enclosing or not indent: if lstrip: block = _outdent(block) return ([block], [lnum]) if locate is True else [block] pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))'; pat1 = re.compile(pat1) pat2 = r'^(\s*@)'; pat2 = re.compile(pat2) #pat3 = r'^(\s*class\s)'; pat3 = re.compile(pat3) #XXX: enclosing class? #FIXME: bound methods need enclosing class (and then instantiation) # *or* somehow apply a partial using the instance skip = 0 line = 0 blocks = []; _lnum = [] target = ''.join(block) while line <= lnum: #XXX: repeat lnum? or until line < lnum? # see if starts with ('def','lambda') and contains our target block if pat1.match(lines[line]): if not skip: try: code = getblock(lines[line:]) except TokenError: code = [lines[line]] if indentsize(lines[line]) > indent: #XXX: should be >= ? line += len(code) - skip elif target in ''.join(code): blocks.append(code) # save code block as the potential winner _lnum.append(line - skip) # save the line number for the match line += len(code) - skip else: line += 1 skip = 0 # find skip: the number of consecutive decorators elif pat2.match(lines[line]): try: code = getblock(lines[line:]) except TokenError: code = [lines[line]] skip = 1 for _line in code[1:]: # skip lines that are decorators if not pat2.match(_line): break skip += 1 line += skip # no match: reset skip and go to the next line else: line +=1 skip = 0 if not blocks: blocks = [block] _lnum = [lnum] if lstrip: blocks = [_outdent(block) for block in blocks] # return last match return (blocks, _lnum) if locate is True else blocks def getsourcelines(object, lstrip=False, enclosing=False): """Return a list of source lines and starting line number for an object. Interactively-defined objects refer to lines in the interpreter's history. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of the lines corresponding to the object and the line number indicates where in the original source file the first line of code was found. An IOError is raised if the source code cannot be retrieved, while a TypeError is raised for objects where the source code is unavailable (e.g. builtins). If lstrip=True, ensure there is no indentation in the first line of code. If enclosing=True, then also return any enclosing code.""" code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True) return code[-1], n[-1] #NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True) def getsource(object, alias='', lstrip=False, enclosing=False, \ force=False, builtin=False): """Return the text of the source code for an object. The source code for interactively-defined objects are extracted from the interpreter's history. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a single string. An IOError is raised if the source code cannot be retrieved, while a TypeError is raised for objects where the source code is unavailable (e.g. builtins). If alias is provided, then add a line of code that renames the object. If lstrip=True, ensure there is no indentation in the first line of code. If enclosing=True, then also return any enclosing code. If force=True, catch (TypeError,IOError) and try to use import hooks. If builtin=True, force an import for any builtins """ # hascode denotes a callable hascode = _hascode(object) # is a class instance type (and not in builtins) instance = _isinstance(object) # get source lines; if fail, try to 'force' an import try: # fails for builtins, and other assorted object types lines, lnum = getsourcelines(object, enclosing=enclosing) except (TypeError, IOError): # failed to get source, resort to import hooks if not force: # don't try to get types that findsource can't get raise if not getmodule(object): # get things like 'None' and '1' if not instance: return getimport(object, alias, builtin=builtin) # special handling (numpy arrays, ...) _import = getimport(object, builtin=builtin) name = getname(object, force=True) _alias = "%s = " % alias if alias else "" if alias == name: _alias = "" return _import+_alias+"%s\n" % name else: #FIXME: could use a good bit of cleanup, since using getimport... if not instance: return getimport(object, alias, builtin=builtin) # now we are dealing with an instance... name = object.__class__.__name__ module = object.__module__ if module in ['builtins','__builtin__']: return getimport(object, alias, builtin=builtin) else: #FIXME: leverage getimport? use 'from module import name'? lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0 obj = eval(lines[0].lstrip(name + ' = ')) lines, lnum = getsourcelines(obj, enclosing=enclosing) # strip leading indent (helps ensure can be imported) if lstrip or alias: lines = _outdent(lines) # instantiate, if there's a nice repr #XXX: BAD IDEA??? if instance: #and force: #XXX: move into findsource or getsourcelines ? if '(' in repr(object): lines.append('%r\n' % object) #else: #XXX: better to somehow to leverage __reduce__ ? # reconstructor,args = object.__reduce__() # _ = reconstructor(*args) else: # fall back to serialization #XXX: bad idea? #XXX: better not duplicate work? #XXX: better new/enclose=True? lines = dumpsource(object, alias='', new=force, enclose=False) lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0 #else: object.__code__ # raise AttributeError # add an alias to the source code if alias: if hascode: skip = 0 for line in lines: # skip lines that are decorators if not line.startswith('@'): break skip += 1 #XXX: use regex from findsource / getsourcelines ? if lines[skip].lstrip().startswith('def '): # we have a function if alias != object.__name__: lines.append('\n%s = %s\n' % (alias, object.__name__)) elif 'lambda ' in lines[skip]: # we have a lambda if alias != lines[skip].split('=')[0].strip(): lines[skip] = '%s = %s' % (alias, lines[skip]) else: # ...try to use the object's name if alias != object.__name__: lines.append('\n%s = %s\n' % (alias, object.__name__)) else: # class or class instance if instance: if alias != lines[-1].split('=')[0].strip(): lines[-1] = ('%s = ' % alias) + lines[-1] else: name = getname(object, force=True) or object.__name__ if alias != name: lines.append('\n%s = %s\n' % (alias, name)) return ''.join(lines) def _hascode(object): '''True if object has an attribute that stores it's __code__''' return getattr(object,'__code__',None) or getattr(object,'func_code',None) def _isinstance(object): '''True if object is a class instance type (and is not a builtin)''' if _hascode(object) or isclass(object) or ismodule(object): return False if istraceback(object) or isframe(object) or iscode(object): return False # special handling (numpy arrays, ...) if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']: return True # # check if is instance of a builtin # if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']: # return False _types = ('<class ',"<type 'instance'>") if not repr(type(object)).startswith(_types): #FIXME: weak hack return False if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']: return False return True # by process of elimination... it's what we want def _intypes(object): '''check if object is in the 'types' module''' import types # allow user to pass in object or object.__name__ if type(object) is not type(''): object = getname(object, force=True) if object == 'ellipsis': object = 'EllipsisType' return True if hasattr(types, object) else False def _isstring(object): #XXX: isstringlike better? '''check if object is a string-like type''' return isinstance(object, (str, bytes)) def indent(code, spaces=4): '''indent a block of code with whitespace (default is 4 spaces)''' indent = indentsize(code) if type(spaces) is int: spaces = ' '*spaces # if '\t' is provided, will indent with a tab nspaces = indentsize(spaces) # blank lines (etc) need to be ignored lines = code.split('\n') ## stq = "'''"; dtq = '"""' ## in_stq = in_dtq = False for i in range(len(lines)): #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc _indent = indentsize(lines[i]) if indent > _indent: continue lines[i] = spaces+lines[i] ## #FIXME: may fail when stq and dtq in same line (depends on ordering) ## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq) ## if not in_dtq and not in_stq: ## lines[i] = spaces+lines[i] # we indent ## # entering a comment block ## if nstq%2: in_stq = not in_stq ## if ndtq%2: in_dtq = not in_dtq ## # leaving a comment block ## elif in_dtq and ndtq%2: in_dtq = not in_dtq ## elif in_stq and nstq%2: in_stq = not in_stq ## else: pass if lines[-1].strip() == '': lines[-1] = '' return '\n'.join(lines) def _outdent(lines, spaces=None, all=True): '''outdent lines of code, accounting for docs and line continuations''' indent = indentsize(lines[0]) if spaces is None or spaces > indent or spaces < 0: spaces = indent for i in range(len(lines) if all else 1): #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc _indent = indentsize(lines[i]) if spaces > _indent: _spaces = _indent else: _spaces = spaces lines[i] = lines[i][_spaces:] return lines def outdent(code, spaces=None, all=True): '''outdent a block of code (default is to strip all leading whitespace)''' indent = indentsize(code) if spaces is None or spaces > indent or spaces < 0: spaces = indent #XXX: will this delete '\n' in some cases? if not all: return code[spaces:] return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all)) #XXX: not sure what the point of _wrap is... __globals__ = globals() __locals__ = locals() def _wrap(f): """ encapsulate a function and it's __import__ """ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust exec(getimportable(f, alias='_'), __globals__, __locals__) except Exception: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ func.__doc__ = f.__doc__ return func def _enclose(object, alias=''): #FIXME: needs alias to hold returned object """create a function enclosure around the source of some object""" #XXX: dummy and stub should append a random string dummy = '__this_is_a_big_dummy_enclosing_function__' stub = '__this_is_a_stub_variable__' code = 'def %s():\n' % dummy code += indent(getsource(object, alias=stub, lstrip=True, force=True)) code += indent('return %s\n' % stub) if alias: code += '%s = ' % alias code += '%s(); del %s\n' % (dummy, dummy) #code += "globals().pop('%s',lambda :None)()\n" % dummy return code def dumpsource(object, alias='', new=False, enclose=True): """'dump to source', where the code includes a pickled object. If new=True and object is a class instance, then create a new instance using the unpacked class source code. If enclose, then create the object inside a function enclosure (thus minimizing any global namespace pollution). """ from dill import dumps pik = repr(dumps(object)) code = 'import dill\n' if enclose: stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub pre = '%s = ' % stub new = False #FIXME: new=True doesn't work with enclose=True else: stub = alias pre = '%s = ' % stub if alias else alias # if a 'new' instance is not needed, then just dump and load if not new or not _isinstance(object): code += pre + 'dill.loads(%s)\n' % pik else: #XXX: other cases where source code is needed??? code += getsource(object.__class__, alias='', lstrip=True, force=True) mod = repr(object.__module__) # should have a module (no builtins here) code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! if enclose: # generation of the 'enclosure' dummy = '__this_is_a_big_dummy_object__' dummy = _enclose(dummy, alias=alias) # hack to replace the 'dummy' with the 'real' code dummy = dummy.split('\n') code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:]) return code #XXX: better 'dumpsourcelines', returning list of lines? def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail? """get the name of the object. for lambdas, get the name of the pointer """ if fqn: return '.'.join(_namespace(obj)) module = getmodule(obj) if not module: # things like "None" and "1" if not force: return None return repr(obj) try: #XXX: 'wrong' for decorators and curried functions ? # if obj.func_closure: ...use logic from getimportable, etc ? name = obj.__name__ if name == '<lambda>': return getsource(obj).split('=',1)[0].strip() # handle some special cases if module.__name__ in ['builtins','__builtin__']: if name == 'ellipsis': name = 'EllipsisType' return name except AttributeError: #XXX: better to just throw AttributeError ? if not force: return None name = repr(obj) if name.startswith('<'): # or name.split('('): return None return name def _namespace(obj): """_namespace(obj); return namespace hierarchy (as a list of names) for the given object. For an instance, find the class hierarchy. For example: >>> from functools import partial >>> p = partial(int, base=2) >>> _namespace(p) [\'functools\', \'partial\'] """ # mostly for functions and modules and such #FIXME: 'wrong' for decorators and curried functions try: #XXX: needs some work and testing on different types module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'") qual = qual.split('.') if ismodule(obj): return qual # get name of a lambda, function, etc name = getname(obj) or obj.__name__ # failing, raise AttributeError # check special cases (NoneType, ...) if module in ['builtins','__builtin__']: # BuiltinFunctionType if _intypes(name): return ['types'] + [name] return qual + [name] #XXX: can be wrong for some aliased objects except Exception: pass # special case: numpy.inf and numpy.nan (we don't want them as floats) if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? return ['numpy'] + [str(obj)] # mostly for classes and class instances and such module = getattr(obj.__class__, '__module__', None) qual = str(obj.__class__) try: qual = qual[qual.index("'")+1:-2] except ValueError: pass # str(obj.__class__) made the 'try' unnecessary qual = qual.split(".") if module in ['builtins','__builtin__']: # check special cases (NoneType, Ellipsis, ...) if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType' if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType qual = [module] + qual return qual #NOTE: 05/25/14 broke backward compatibility: added 'alias' as 3rd argument def _getimport(head, tail, alias='', verify=True, builtin=False): """helper to build a likely import string from head and tail of namespace. ('head','tail') are used in the following context: "from head import tail" If verify=True, then test the import string before returning it. If builtin=True, then force an import for builtins where possible. If alias is provided, then rename the object on import. """ # special handling for a few common types if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']: head = len.__module__ elif tail in ['None'] and head in ['types']: _alias = '%s = ' % alias if alias else '' if alias == tail: _alias = '' return _alias+'%s\n' % tail # we don't need to import from builtins, so return '' # elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ? if head in ['builtins','__builtin__']: # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType if tail == 'ellipsis': tail = 'EllipsisType' if _intypes(tail): head = 'types' elif not builtin: _alias = '%s = ' % alias if alias else '' if alias == tail: _alias = '' return _alias+'%s\n' % tail else: pass # handle builtins below # get likely import string if not head: _str = "import %s" % tail else: _str = "from %s import %s" % (head, tail) _alias = " as %s\n" % alias if alias else "\n" if alias == tail: _alias = "\n" _str += _alias # FIXME: fails on most decorators, currying, and such... # (could look for magic __wrapped__ or __func__ attr) # (could fix in 'namespace' to check obj for closure) if verify and not head.startswith('dill.'):# weird behavior for dill #print(_str) try: exec(_str) #XXX: check if == obj? (name collision) except ImportError: #XXX: better top-down or bottom-up recursion? _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?) if not _head: raise if _head != head: _str = _getimport(_head, tail, alias, verify) return _str #XXX: rename builtin to force? vice versa? verify to force? (as in getsource) #NOTE: 05/25/14 broke backward compatibility: added 'alias' as 2nd argument def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): """get the likely import string for the given object obj is the object to inspect If verify=True, then test the import string before returning it. If builtin=True, then force an import for builtins where possible. If enclosing=True, get the import for the outermost enclosing callable. If alias is provided, then rename the object on import. """ if enclosing: from .detect import outermost _obj = outermost(obj) obj = _obj if _obj else obj # get the namespace qual = _namespace(obj) head = '.'.join(qual[:-1]) tail = qual[-1] # for named things... with a nice repr #XXX: move into _namespace? try: # look for '<...>' and be mindful it might be in lists, dicts, etc... name = repr(obj).split('<',1)[1].split('>',1)[1] name = None # we have a 'object'-style repr except Exception: # it's probably something 'importable' if head in ['builtins','__builtin__']: name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? else: name = repr(obj).split('(')[0] #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0] #else: name = None if name: # try using name instead of tail try: return _getimport(head, name, alias, verify, builtin) except ImportError: pass except SyntaxError: if head in ['builtins','__builtin__']: _alias = '%s = ' % alias if alias else '' if alias == name: _alias = '' return _alias+'%s\n' % name else: pass try: #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType #else: _builtin = False return _getimport(head, tail, alias, verify, builtin) except ImportError: raise # could do some checking against obj except SyntaxError: if head in ['builtins','__builtin__']: _alias = '%s = ' % alias if alias else '' if alias == tail: _alias = '' return _alias+'%s\n' % tail raise # could do some checking against obj def _importable(obj, alias='', source=None, enclosing=False, force=True, \ builtin=True, lstrip=True): """get an import string (or the source code) for the given object This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, to attempt to force the use of an import, use source=False; otherwise an import will be sought for objects not defined in __main__. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If source=True, use these options: If enclosing=True, then also return any enclosing code. If force=True, catch (TypeError,IOError) and try to use import hooks. If lstrip=True, ensure there is no indentation in the first line of code. If source=False, use these options: If enclosing=True, get the import for the outermost enclosing callable. If force=True, then don't test the import string before returning it. If builtin=True, then force an import for builtins where possible. """ if source is None: source = True if isfrommain(obj) else False if source: # first try to get the source try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) except Exception: pass try: if not _isinstance(obj): return getimport(obj, alias, enclosing=enclosing, \ verify=(not force), builtin=builtin) # first 'get the import', then 'get the instance' _import = getimport(obj, enclosing=enclosing, \ verify=(not force), builtin=builtin) name = getname(obj, force=True) if not name: raise AttributeError("object has no atribute '__name__'") _alias = "%s = " % alias if alias else "" if alias == name: _alias = "" return _import+_alias+"%s\n" % name except Exception: pass if not source: # try getsource, only if it hasn't been tried yet try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) except Exception: pass # get the name (of functions, lambdas, and classes) # or hope that obj can be built from the __repr__ #XXX: what to do about class instances and such? obj = getname(obj, force=force) # we either have __repr__ or __name__ (or None) if not obj or obj.startswith('<'): raise AttributeError("object has no atribute '__name__'") _alias = '%s = ' % alias if alias else '' if alias == obj: _alias = '' return _alias+'%s\n' % obj #XXX: possible failsafe... (for example, for instances when source=False) # "import dill; result = dill.loads(<pickled_object>); # repr(<object>)" def _closuredimport(func, alias='', builtin=False): """get import for closured objects; return a dict of 'name' and 'import'""" import re from .detect import freevars, outermost free_vars = freevars(func) func_vars = {} # split into 'funcs' and 'non-funcs' for name,obj in list(free_vars.items()): if not isfunction(obj): continue # get import for 'funcs' fobj = free_vars.pop(name) src = getsource(fobj) if src.lstrip().startswith('@'): # we have a decorator src = getimport(fobj, alias=alias, builtin=builtin) else: # we have to "hack" a bit... and maybe be lucky encl = outermost(func) # pattern: 'func = enclosing(fobj' pat = r'.*[\w\s]=\s*'+getname(encl)+r'\('+getname(fobj) mod = getname(getmodule(encl)) #HACK: get file containing 'outer' function; is func there? lines,_ = findsource(encl) candidate = [line for line in lines if getname(encl) in line and \ re.match(pat, line)] if not candidate: mod = getname(getmodule(fobj)) #HACK: get file containing 'inner' function; is func there? lines,_ = findsource(fobj) candidate = [line for line in lines \ if getname(fobj) in line and re.match(pat, line)] if not len(candidate): raise TypeError('import could not be found') candidate = candidate[-1] name = candidate.split('=',1)[0].split()[-1].strip() src = _getimport(mod, name, alias=alias, builtin=builtin) func_vars[name] = src if not func_vars: name = outermost(func) mod = getname(getmodule(name)) if not mod or name is func: # then it can be handled by getimport name = getname(func, force=True) #XXX: better key? src = getimport(func, alias=alias, builtin=builtin) else: lines,_ = findsource(name) # pattern: 'func = enclosing(' candidate = [line for line in lines if getname(name) in line and \ re.match(r'.*[\w\s]=\s*'+getname(name)+r'\(', line)] if not len(candidate): raise TypeError('import could not be found') candidate = candidate[-1] name = candidate.split('=',1)[0].split()[-1].strip() src = _getimport(mod, name, alias=alias, builtin=builtin) func_vars[name] = src return func_vars #XXX: should be able to use __qualname__ def _closuredsource(func, alias=''): """get source code for closured objects; return a dict of 'name' and 'code blocks'""" #FIXME: this entire function is a messy messy HACK # - pollutes global namespace # - fails if name of freevars are reused # - can unnecessarily duplicate function code from .detect import freevars free_vars = freevars(func) func_vars = {} # split into 'funcs' and 'non-funcs' for name,obj in list(free_vars.items()): if not isfunction(obj): # get source for 'non-funcs' free_vars[name] = getsource(obj, force=True, alias=name) continue # get source for 'funcs' fobj = free_vars.pop(name) src = getsource(fobj, alias) # DO NOT include dependencies # if source doesn't start with '@', use name as the alias if not src.lstrip().startswith('@'): #FIXME: 'enclose' in dummy; src = importable(fobj,alias=name)# wrong ref 'name' org = getsource(func, alias, enclosing=False, lstrip=True) src = (src, org) # undecorated first, then target else: #NOTE: reproduces the code! org = getsource(func, enclosing=True, lstrip=False) src = importable(fobj, alias, source=True) # include dependencies src = (org, src) # target first, then decorated func_vars[name] = src src = ''.join(free_vars.values()) if not func_vars: #FIXME: 'enclose' in dummy; wrong ref 'name' org = getsource(func, alias, force=True, enclosing=False, lstrip=True) src = (src, org) # variables first, then target else: src = (src, None) # just variables (better '' instead of None?) func_vars[None] = src # FIXME: remove duplicates (however, order is important...) return func_vars def importable(obj, alias='', source=None, builtin=True): """get an importable string (i.e. source code or the import string) for the given object, including any required objects from the enclosing and global scope This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, to attempt to force the use of an import, use source=False; otherwise an import will be sought for objects not defined in __main__. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If builtin=True, then force an import for builtins where possible. """ #NOTE: we always 'force', and 'lstrip' as necessary #NOTE: for 'enclosing', use importable(outermost(obj)) if source is None: source = True if isfrommain(obj) else False elif builtin and isbuiltin(obj): source = False tried_source = tried_import = False while True: if not source: # we want an import try: if _isinstance(obj): # for instances, punt to _importable return _importable(obj, alias, source=False, builtin=builtin) src = _closuredimport(obj, alias=alias, builtin=builtin) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] except Exception: if tried_source: raise tried_import = True # we want the source try: src = _closuredsource(obj, alias=alias) if len(src) == 0: raise NotImplementedError('not implemented') # groan... an inline code stitcher def _code_stitcher(block): "stitch together the strings in tuple 'block'" if block[0] and block[-1]: block = '\n'.join(block) elif block[0]: block = block[0] elif block[-1]: block = block[-1] else: block = '' return block # get free_vars first _src = _code_stitcher(src.pop(None)) _src = [_src] if _src else [] # get func_vars for xxx in src.values(): xxx = _code_stitcher(xxx) if xxx: _src.append(xxx) # make a single source string if not len(_src): src = '' elif len(_src) == 1: src = _src[0] else: src = '\n'.join(_src) # get source code of objects referred to by obj in global scope from .detect import globalvars obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc? obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj)) obj = '\n'.join(obj) if obj else '' # combine all referred-to source (global then enclosing) if not obj: return src if not src: return obj return obj + src except Exception: if tried_import: raise tried_source = True source = not source # should never get here return # backward compatibility def getimportable(obj, alias='', byname=True, explicit=False): return importable(obj,alias,source=(not byname),builtin=explicit) #return outdent(_importable(obj,alias,source=(not byname),builtin=explicit)) def likely_import(obj, passive=False, explicit=False): return getimport(obj, verify=(not passive), builtin=explicit) def _likely_import(first, last, passive=False, explicit=True): return _getimport(first, last, verify=(not passive), builtin=explicit) _get_name = getname getblocks_from_history = getblocks # EOF