/IPython/core/magics/osm.py
Python | 849 lines | 824 code | 15 blank | 10 comment | 16 complexity | 79f2463ed4002bdfd32b80ff9dd6c9cf MD5 | raw file
Possible License(s): BSD-3-Clause, MIT, Apache-2.0
- """Implementation of magic functions for interaction with the OS.
- Note: this module is named 'osm' instead of 'os' to avoid a collision with the
- builtin.
- """
- # Copyright (c) IPython Development Team.
- # Distributed under the terms of the Modified BSD License.
- import io
- import os
- import re
- import sys
- from pprint import pformat
- from IPython.core import magic_arguments
- from IPython.core import oinspect
- from IPython.core import page
- from IPython.core.alias import AliasError, Alias
- from IPython.core.error import UsageError
- from IPython.core.magic import (
- Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
- )
- from IPython.testing.skipdoctest import skip_doctest
- from IPython.utils.openpy import source_to_unicode
- from IPython.utils.process import abbrev_cwd
- from IPython.utils.terminal import set_term_title
- from traitlets import Bool
- @magics_class
- class OSMagics(Magics):
- """Magics to interact with the underlying OS (shell-type functionality).
- """
- cd_force_quiet = Bool(False,
- help="Force %cd magic to be quiet even if -q is not passed."
- ).tag(config=True)
- def __init__(self, shell=None, **kwargs):
- # Now define isexec in a cross platform manner.
- self.is_posix = False
- self.execre = None
- if os.name == 'posix':
- self.is_posix = True
- else:
- try:
- winext = os.environ['pathext'].replace(';','|').replace('.','')
- except KeyError:
- winext = 'exe|com|bat|py'
-
- self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
- # call up the chain
- super().__init__(shell=shell, **kwargs)
- @skip_doctest
- def _isexec_POSIX(self, file):
- """
- Test for executable on a POSIX system
- """
- if os.access(file.path, os.X_OK):
- # will fail on maxOS if access is not X_OK
- return file.is_file()
- return False
-
- @skip_doctest
- def _isexec_WIN(self, file):
- """
- Test for executable file on non POSIX system
- """
- return file.is_file() and self.execre.match(file.name) is not None
- @skip_doctest
- def isexec(self, file):
- """
- Test for executable file on non POSIX system
- """
- if self.is_posix:
- return self._isexec_POSIX(file)
- else:
- return self._isexec_WIN(file)
- @skip_doctest
- @line_magic
- def alias(self, parameter_s=''):
- """Define an alias for a system command.
- '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
- Then, typing 'alias_name params' will execute the system command 'cmd
- params' (from your underlying operating system).
- Aliases have lower precedence than magic functions and Python normal
- variables, so if 'foo' is both a Python variable and an alias, the
- alias can not be executed until 'del foo' removes the Python variable.
- You can use the %l specifier in an alias definition to represent the
- whole line when the alias is called. For example::
- In [2]: alias bracket echo "Input in brackets: <%l>"
- In [3]: bracket hello world
- Input in brackets: <hello world>
- You can also define aliases with parameters using %s specifiers (one
- per parameter)::
- In [1]: alias parts echo first %s second %s
- In [2]: %parts A B
- first A second B
- In [3]: %parts A
- Incorrect number of arguments: 2 expected.
- parts is an alias to: 'echo first %s second %s'
- Note that %l and %s are mutually exclusive. You can only use one or
- the other in your aliases.
- Aliases expand Python variables just like system calls using ! or !!
- do: all expressions prefixed with '$' get expanded. For details of
- the semantic rules, see PEP-215:
- http://www.python.org/peps/pep-0215.html. This is the library used by
- IPython for variable expansion. If you want to access a true shell
- variable, an extra $ is necessary to prevent its expansion by
- IPython::
- In [6]: alias show echo
- In [7]: PATH='A Python string'
- In [8]: show $PATH
- A Python string
- In [9]: show $$PATH
- /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
- You can use the alias facility to access all of $PATH. See the %rehashx
- function, which automatically creates aliases for the contents of your
- $PATH.
- If called with no parameters, %alias prints the current alias table
- for your system. For posix systems, the default aliases are 'cat',
- 'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
- aliases are added. For windows-based systems, the default aliases are
- 'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
- You can see the definition of alias by adding a question mark in the
- end::
- In [1]: cat?
- Repr: <alias cat for 'cat'>"""
- par = parameter_s.strip()
- if not par:
- aliases = sorted(self.shell.alias_manager.aliases)
- # stored = self.shell.db.get('stored_aliases', {} )
- # for k, v in stored:
- # atab.append(k, v[0])
- print("Total number of aliases:", len(aliases))
- sys.stdout.flush()
- return aliases
- # Now try to define a new one
- try:
- alias,cmd = par.split(None, 1)
- except TypeError:
- print(oinspect.getdoc(self.alias))
- return
-
- try:
- self.shell.alias_manager.define_alias(alias, cmd)
- except AliasError as e:
- print(e)
- # end magic_alias
- @line_magic
- def unalias(self, parameter_s=''):
- """Remove an alias"""
- aname = parameter_s.strip()
- try:
- self.shell.alias_manager.undefine_alias(aname)
- except ValueError as e:
- print(e)
- return
-
- stored = self.shell.db.get('stored_aliases', {} )
- if aname in stored:
- print("Removing %stored alias",aname)
- del stored[aname]
- self.shell.db['stored_aliases'] = stored
- @line_magic
- def rehashx(self, parameter_s=''):
- """Update the alias table with all executable files in $PATH.
- rehashx explicitly checks that every entry in $PATH is a file
- with execute access (os.X_OK).
- Under Windows, it checks executability as a match against a
- '|'-separated string of extensions, stored in the IPython config
- variable win_exec_ext. This defaults to 'exe|com|bat'.
- This function also resets the root module cache of module completer,
- used on slow filesystems.
- """
- from IPython.core.alias import InvalidAliasError
- # for the benefit of module completer in ipy_completers.py
- del self.shell.db['rootmodules_cache']
- path = [os.path.abspath(os.path.expanduser(p)) for p in
- os.environ.get('PATH','').split(os.pathsep)]
- syscmdlist = []
- savedir = os.getcwd()
- # Now walk the paths looking for executables to alias.
- try:
- # write the whole loop for posix/Windows so we don't have an if in
- # the innermost part
- if self.is_posix:
- for pdir in path:
- try:
- os.chdir(pdir)
- except OSError:
- continue
- # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
- dirlist = os.scandir(path=pdir)
- for ff in dirlist:
- if self.isexec(ff):
- fname = ff.name
- try:
- # Removes dots from the name since ipython
- # will assume names with dots to be python.
- if not self.shell.alias_manager.is_alias(fname):
- self.shell.alias_manager.define_alias(
- fname.replace('.',''), fname)
- except InvalidAliasError:
- pass
- else:
- syscmdlist.append(fname)
- else:
- no_alias = Alias.blacklist
- for pdir in path:
- try:
- os.chdir(pdir)
- except OSError:
- continue
- # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
- dirlist = os.scandir(pdir)
- for ff in dirlist:
- fname = ff.name
- base, ext = os.path.splitext(fname)
- if self.isexec(ff) and base.lower() not in no_alias:
- if ext.lower() == '.exe':
- fname = base
- try:
- # Removes dots from the name since ipython
- # will assume names with dots to be python.
- self.shell.alias_manager.define_alias(
- base.lower().replace('.',''), fname)
- except InvalidAliasError:
- pass
- syscmdlist.append(fname)
- self.shell.db['syscmdlist'] = syscmdlist
- finally:
- os.chdir(savedir)
- @skip_doctest
- @line_magic
- def pwd(self, parameter_s=''):
- """Return the current working directory path.
- Examples
- --------
- ::
- In [9]: pwd
- Out[9]: '/home/tsuser/sprint/ipython'
- """
- try:
- return os.getcwd()
- except FileNotFoundError:
- raise UsageError("CWD no longer exists - please use %cd to change directory.")
- @skip_doctest
- @line_magic
- def cd(self, parameter_s=''):
- """Change the current working directory.
- This command automatically maintains an internal list of directories
- you visit during your IPython session, in the variable _dh. The
- command %dhist shows this history nicely formatted. You can also
- do 'cd -<tab>' to see directory history conveniently.
- Usage:
- cd 'dir': changes to directory 'dir'.
- cd -: changes to the last visited directory.
- cd -<n>: changes to the n-th directory in the directory history.
- cd --foo: change to directory that matches 'foo' in history
- cd -b <bookmark_name>: jump to a bookmark set by %bookmark
- (note: cd <bookmark_name> is enough if there is no
- directory <bookmark_name>, but a bookmark with the name exists.)
- 'cd -b <tab>' allows you to tab-complete bookmark names.
- Options:
- -q: quiet. Do not print the working directory after the cd command is
- executed. By default IPython's cd command does print this directory,
- since the default prompts do not display path information.
- Note that !cd doesn't work for this purpose because the shell where
- !command runs is immediately discarded after executing 'command'.
- Examples
- --------
- ::
- In [10]: cd parent/child
- /home/tsuser/parent/child
- """
- try:
- oldcwd = os.getcwd()
- except FileNotFoundError:
- # Happens if the CWD has been deleted.
- oldcwd = None
- numcd = re.match(r'(-)(\d+)$',parameter_s)
- # jump in directory history by number
- if numcd:
- nn = int(numcd.group(2))
- try:
- ps = self.shell.user_ns['_dh'][nn]
- except IndexError:
- print('The requested directory does not exist in history.')
- return
- else:
- opts = {}
- elif parameter_s.startswith('--'):
- ps = None
- fallback = None
- pat = parameter_s[2:]
- dh = self.shell.user_ns['_dh']
- # first search only by basename (last component)
- for ent in reversed(dh):
- if pat in os.path.basename(ent) and os.path.isdir(ent):
- ps = ent
- break
- if fallback is None and pat in ent and os.path.isdir(ent):
- fallback = ent
- # if we have no last part match, pick the first full path match
- if ps is None:
- ps = fallback
- if ps is None:
- print("No matching entry in directory history")
- return
- else:
- opts = {}
- else:
- opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
- # jump to previous
- if ps == '-':
- try:
- ps = self.shell.user_ns['_dh'][-2]
- except IndexError:
- raise UsageError('%cd -: No previous directory to change to.')
- # jump to bookmark if needed
- else:
- if not os.path.isdir(ps) or 'b' in opts:
- bkms = self.shell.db.get('bookmarks', {})
- if ps in bkms:
- target = bkms[ps]
- print('(bookmark:%s) -> %s' % (ps, target))
- ps = target
- else:
- if 'b' in opts:
- raise UsageError("Bookmark '%s' not found. "
- "Use '%%bookmark -l' to see your bookmarks." % ps)
- # at this point ps should point to the target dir
- if ps:
- try:
- os.chdir(os.path.expanduser(ps))
- if hasattr(self.shell, 'term_title') and self.shell.term_title:
- set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
- except OSError:
- print(sys.exc_info()[1])
- else:
- cwd = os.getcwd()
- dhist = self.shell.user_ns['_dh']
- if oldcwd != cwd:
- dhist.append(cwd)
- self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
- else:
- os.chdir(self.shell.home_dir)
- if hasattr(self.shell, 'term_title') and self.shell.term_title:
- set_term_title(self.shell.term_title_format.format(cwd="~"))
- cwd = os.getcwd()
- dhist = self.shell.user_ns['_dh']
- if oldcwd != cwd:
- dhist.append(cwd)
- self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
- if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
- print(self.shell.user_ns['_dh'][-1])
- @line_magic
- def env(self, parameter_s=''):
- """Get, set, or list environment variables.
- Usage:\\
- %env: lists all environment variables/values
- %env var: get value for var
- %env var val: set value for var
- %env var=val: set value for var
- %env var=$val: set value for var, using python expansion if possible
- """
- if parameter_s.strip():
- split = '=' if '=' in parameter_s else ' '
- bits = parameter_s.split(split)
- if len(bits) == 1:
- key = parameter_s.strip()
- if key in os.environ:
- return os.environ[key]
- else:
- err = "Environment does not have key: {0}".format(key)
- raise UsageError(err)
- if len(bits) > 1:
- return self.set_env(parameter_s)
- env = dict(os.environ)
- # hide likely secrets when printing the whole environment
- for key in list(env):
- if any(s in key.lower() for s in ('key', 'token', 'secret')):
- env[key] = '<hidden>'
- return env
- @line_magic
- def set_env(self, parameter_s):
- """Set environment variables. Assumptions are that either "val" is a
- name in the user namespace, or val is something that evaluates to a
- string.
- Usage:\\
- %set_env var val: set value for var
- %set_env var=val: set value for var
- %set_env var=$val: set value for var, using python expansion if possible
- """
- split = '=' if '=' in parameter_s else ' '
- bits = parameter_s.split(split, 1)
- if not parameter_s.strip() or len(bits)<2:
- raise UsageError("usage is 'set_env var=val'")
- var = bits[0].strip()
- val = bits[1].strip()
- if re.match(r'.*\s.*', var):
- # an environment variable with whitespace is almost certainly
- # not what the user intended. what's more likely is the wrong
- # split was chosen, ie for "set_env cmd_args A=B", we chose
- # '=' for the split and should have chosen ' '. to get around
- # this, users should just assign directly to os.environ or use
- # standard magic {var} expansion.
- err = "refusing to set env var with whitespace: '{0}'"
- err = err.format(val)
- raise UsageError(err)
- os.environ[var] = val
- print('env: {0}={1}'.format(var,val))
- @line_magic
- def pushd(self, parameter_s=''):
- """Place the current dir on stack and change directory.
- Usage:\\
- %pushd ['dirname']
- """
- dir_s = self.shell.dir_stack
- tgt = os.path.expanduser(parameter_s)
- cwd = os.getcwd().replace(self.shell.home_dir,'~')
- if tgt:
- self.cd(parameter_s)
- dir_s.insert(0,cwd)
- return self.shell.magic('dirs')
- @line_magic
- def popd(self, parameter_s=''):
- """Change to directory popped off the top of the stack.
- """
- if not self.shell.dir_stack:
- raise UsageError("%popd on empty stack")
- top = self.shell.dir_stack.pop(0)
- self.cd(top)
- print("popd ->",top)
- @line_magic
- def dirs(self, parameter_s=''):
- """Return the current directory stack."""
- return self.shell.dir_stack
- @line_magic
- def dhist(self, parameter_s=''):
- """Print your history of visited directories.
- %dhist -> print full history\\
- %dhist n -> print last n entries only\\
- %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
- This history is automatically maintained by the %cd command, and
- always available as the global list variable _dh. You can use %cd -<n>
- to go to directory number <n>.
- Note that most of time, you should view directory history by entering
- cd -<TAB>.
- """
- dh = self.shell.user_ns['_dh']
- if parameter_s:
- try:
- args = map(int,parameter_s.split())
- except:
- self.arg_err(self.dhist)
- return
- if len(args) == 1:
- ini,fin = max(len(dh)-(args[0]),0),len(dh)
- elif len(args) == 2:
- ini,fin = args
- fin = min(fin, len(dh))
- else:
- self.arg_err(self.dhist)
- return
- else:
- ini,fin = 0,len(dh)
- print('Directory history (kept in _dh)')
- for i in range(ini, fin):
- print("%d: %s" % (i, dh[i]))
- @skip_doctest
- @line_magic
- def sc(self, parameter_s=''):
- """Shell capture - run shell command and capture output (DEPRECATED use !).
- DEPRECATED. Suboptimal, retained for backwards compatibility.
- You should use the form 'var = !command' instead. Example:
- "%sc -l myfiles = ls ~" should now be written as
- "myfiles = !ls ~"
- myfiles.s, myfiles.l and myfiles.n still apply as documented
- below.
- --
- %sc [options] varname=command
- IPython will run the given command using commands.getoutput(), and
- will then update the user's interactive namespace with a variable
- called varname, containing the value of the call. Your command can
- contain shell wildcards, pipes, etc.
- The '=' sign in the syntax is mandatory, and the variable name you
- supply must follow Python's standard conventions for valid names.
- (A special format without variable name exists for internal use)
- Options:
- -l: list output. Split the output on newlines into a list before
- assigning it to the given variable. By default the output is stored
- as a single string.
- -v: verbose. Print the contents of the variable.
- In most cases you should not need to split as a list, because the
- returned value is a special type of string which can automatically
- provide its contents either as a list (split on newlines) or as a
- space-separated string. These are convenient, respectively, either
- for sequential processing or to be passed to a shell command.
- For example::
- # Capture into variable a
- In [1]: sc a=ls *py
- # a is a string with embedded newlines
- In [2]: a
- Out[2]: 'setup.py\\nwin32_manual_post_install.py'
- # which can be seen as a list:
- In [3]: a.l
- Out[3]: ['setup.py', 'win32_manual_post_install.py']
- # or as a whitespace-separated string:
- In [4]: a.s
- Out[4]: 'setup.py win32_manual_post_install.py'
- # a.s is useful to pass as a single command line:
- In [5]: !wc -l $a.s
- 146 setup.py
- 130 win32_manual_post_install.py
- 276 total
- # while the list form is useful to loop over:
- In [6]: for f in a.l:
- ...: !wc -l $f
- ...:
- 146 setup.py
- 130 win32_manual_post_install.py
- Similarly, the lists returned by the -l option are also special, in
- the sense that you can equally invoke the .s attribute on them to
- automatically get a whitespace-separated string from their contents::
- In [7]: sc -l b=ls *py
- In [8]: b
- Out[8]: ['setup.py', 'win32_manual_post_install.py']
- In [9]: b.s
- Out[9]: 'setup.py win32_manual_post_install.py'
- In summary, both the lists and strings used for output capture have
- the following special attributes::
- .l (or .list) : value as list.
- .n (or .nlstr): value as newline-separated string.
- .s (or .spstr): value as space-separated string.
- """
- opts,args = self.parse_options(parameter_s, 'lv')
- # Try to get a variable name and command to run
- try:
- # the variable name must be obtained from the parse_options
- # output, which uses shlex.split to strip options out.
- var,_ = args.split('=', 1)
- var = var.strip()
- # But the command has to be extracted from the original input
- # parameter_s, not on what parse_options returns, to avoid the
- # quote stripping which shlex.split performs on it.
- _,cmd = parameter_s.split('=', 1)
- except ValueError:
- var,cmd = '',''
- # If all looks ok, proceed
- split = 'l' in opts
- out = self.shell.getoutput(cmd, split=split)
- if 'v' in opts:
- print('%s ==\n%s' % (var, pformat(out)))
- if var:
- self.shell.user_ns.update({var:out})
- else:
- return out
- @line_cell_magic
- def sx(self, line='', cell=None):
- """Shell execute - run shell command and capture output (!! is short-hand).
- %sx command
- IPython will run the given command using commands.getoutput(), and
- return the result formatted as a list (split on '\\n'). Since the
- output is _returned_, it will be stored in ipython's regular output
- cache Out[N] and in the '_N' automatic variables.
- Notes:
- 1) If an input line begins with '!!', then %sx is automatically
- invoked. That is, while::
- !ls
- causes ipython to simply issue system('ls'), typing::
- !!ls
- is a shorthand equivalent to::
- %sx ls
- 2) %sx differs from %sc in that %sx automatically splits into a list,
- like '%sc -l'. The reason for this is to make it as easy as possible
- to process line-oriented shell output via further python commands.
- %sc is meant to provide much finer control, but requires more
- typing.
- 3) Just like %sc -l, this is a list with special attributes:
- ::
- .l (or .list) : value as list.
- .n (or .nlstr): value as newline-separated string.
- .s (or .spstr): value as whitespace-separated string.
- This is very useful when trying to use such lists as arguments to
- system commands."""
-
- if cell is None:
- # line magic
- return self.shell.getoutput(line)
- else:
- opts,args = self.parse_options(line, '', 'out=')
- output = self.shell.getoutput(cell)
- out_name = opts.get('out', opts.get('o'))
- if out_name:
- self.shell.user_ns[out_name] = output
- else:
- return output
- system = line_cell_magic('system')(sx)
- bang = cell_magic('!')(sx)
- @line_magic
- def bookmark(self, parameter_s=''):
- """Manage IPython's bookmark system.
- %bookmark <name> - set bookmark to current dir
- %bookmark <name> <dir> - set bookmark to <dir>
- %bookmark -l - list all bookmarks
- %bookmark -d <name> - remove bookmark
- %bookmark -r - remove all bookmarks
- You can later on access a bookmarked folder with::
- %cd -b <name>
- or simply '%cd <name>' if there is no directory called <name> AND
- there is such a bookmark defined.
- Your bookmarks persist through IPython sessions, but they are
- associated with each profile."""
- opts,args = self.parse_options(parameter_s,'drl',mode='list')
- if len(args) > 2:
- raise UsageError("%bookmark: too many arguments")
- bkms = self.shell.db.get('bookmarks',{})
- if 'd' in opts:
- try:
- todel = args[0]
- except IndexError:
- raise UsageError(
- "%bookmark -d: must provide a bookmark to delete")
- else:
- try:
- del bkms[todel]
- except KeyError:
- raise UsageError(
- "%%bookmark -d: Can't delete bookmark '%s'" % todel)
- elif 'r' in opts:
- bkms = {}
- elif 'l' in opts:
- bks = sorted(bkms)
- if bks:
- size = max(map(len, bks))
- else:
- size = 0
- fmt = '%-'+str(size)+'s -> %s'
- print('Current bookmarks:')
- for bk in bks:
- print(fmt % (bk, bkms[bk]))
- else:
- if not args:
- raise UsageError("%bookmark: You must specify the bookmark name")
- elif len(args)==1:
- bkms[args[0]] = os.getcwd()
- elif len(args)==2:
- bkms[args[0]] = args[1]
- self.shell.db['bookmarks'] = bkms
- @line_magic
- def pycat(self, parameter_s=''):
- """Show a syntax-highlighted file through a pager.
- This magic is similar to the cat utility, but it will assume the file
- to be Python source and will show it with syntax highlighting.
- This magic command can either take a local filename, an url,
- an history range (see %history) or a macro as argument ::
- %pycat myscript.py
- %pycat 7-27
- %pycat myMacro
- %pycat http://www.example.com/myscript.py
- """
- if not parameter_s:
- raise UsageError('Missing filename, URL, input history range, '
- 'or macro.')
- try :
- cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
- except (ValueError, IOError):
- print("Error: no such file, variable, URL, history range or macro")
- return
- page.page(self.shell.pycolorize(source_to_unicode(cont)))
- @magic_arguments.magic_arguments()
- @magic_arguments.argument(
- '-a', '--append', action='store_true', default=False,
- help='Append contents of the cell to an existing file. '
- 'The file will be created if it does not exist.'
- )
- @magic_arguments.argument(
- 'filename', type=str,
- help='file to write'
- )
- @cell_magic
- def writefile(self, line, cell):
- """Write the contents of the cell to a file.
-
- The file will be overwritten unless the -a (--append) flag is specified.
- """
- args = magic_arguments.parse_argstring(self.writefile, line)
- if re.match(r'^(\'.*\')|(".*")$', args.filename):
- filename = os.path.expanduser(args.filename[1:-1])
- else:
- filename = os.path.expanduser(args.filename)
-
- if os.path.exists(filename):
- if args.append:
- print("Appending to %s" % filename)
- else:
- print("Overwriting %s" % filename)
- else:
- print("Writing %s" % filename)
-
- mode = 'a' if args.append else 'w'
- with io.open(filename, mode, encoding='utf-8') as f:
- f.write(cell)