From 5b25c5623dbd30ad364cca3f95799fb7fffc29bd Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 15:03:35 -0800 Subject: [PATCH 1/7] cli: Add a command system As a debugger library, drgn can be a little bit verbose when running interactively. There has been general concensus that it would be nice to have a way to define shorthands for common operations, or define user scripts that are easy to run within the shell. However we have yet to settle upon a solution. Implementing a full CLI seems out of the scope of drgn. Leveraging the power of a system like IPython's "magic" commands is promising, but adds heavy dependencies. One example CLI to draw inspiration from is SQLite, which uses the "." prefix to define meta-commands that aren't part of the SQL language. Python, like SQL, cannot begin a statement with a ".", so it could be used to unambiguously signal that a command is being entered, and thanks to the code.InteractiveConsole, we can easily extend the console to handle these commands. So, let's add a command system. Commands are simply callables which take a program, the full line of text (which may contain arguments), and the CLI locals. They may return a Python object which is stored to `builtins._`. They can be registered using a decorator, and an API is available to retrieve the default list. The run_interactive() function allows the user to customize the available command list. Signed-off-by: Stephen Brennan --- drgn/cli.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/drgn/cli.py b/drgn/cli.py index 06c3c197a..cb0b18255 100644 --- a/drgn/cli.py +++ b/drgn/cli.py @@ -16,6 +16,8 @@ import runpy import shutil import sys +import textwrap +import traceback from typing import Any, Callable, Dict, Optional import drgn @@ -26,6 +28,52 @@ logger = logging.getLogger("drgn") +Command = Callable[[drgn.Program, str, Dict[str, Any]], Any] + +_COMMANDS: Dict[str, Command] = {} + + +def command(name: str) -> Callable[[Command], Command]: + """ + A decorator for registering a command function + + Example usage: + + >>> @drgn.cli.command("hello") + ... def hello(prog, line, locals_): + ... print("hello world") + + The decorator will be added to Drgn's default command set. Please keep in + mind that the decorator is evaluated when your module is imported. If you'd + like to extend drgn's default set of commands, then you should ensure your + module is imported before the CLI starts. + """ + + def decorator(cmd: Command) -> Command: + _COMMANDS[name] = cmd + return cmd + + return decorator + + +def all_commands() -> Dict[str, Command]: + """ + Returns all registered drgn CLI commands + """ + return _COMMANDS.copy() + + +def help_command(commands: Dict[str, Command]) -> Command: + def help(prog: drgn.Program, line: str, locals_: Dict[str, Any]) -> None: + try: + width = os.get_terminal_size().columns + except OSError: + width = 80 + print("Drgn CLI commands:\n") + print(textwrap.fill(" ".join(commands.keys()), width=width)) + + return help + class _LogFormatter(logging.Formatter): _LEVELS = ( @@ -156,6 +204,32 @@ def _displayhook(value: Any) -> None: setattr(builtins, "_", value) +class _InteractiveConsoleWithCommand(code.InteractiveConsole): + def __init__(self, commands: Dict[str, Command], *args: Any, **kwargs: Any): + self.__in_multi_line = False + self.__commands = commands + super().__init__(*args, **kwargs) + + def __run_command(self, line: str) -> None: + cmd_name = line.split(maxsplit=1)[0][1:] + if cmd_name not in self.__commands: + print(f"{cmd_name}: drgn command not found") + return + cmd = self.__commands[cmd_name] + prog = self.locals["prog"] + try: + setattr(builtins, "_", cmd(prog, line, self.locals)) # type: ignore + except Exception: + traceback.print_exception(*sys.exc_info()) + + def push(self, line: str) -> bool: + if not self.__in_multi_line and line and line.lstrip()[0] == ".": + self.__run_command(line) + return False + self.__in_multi_line = super().push(line) + return self.__in_multi_line + + def _main() -> None: handler = logging.StreamHandler() handler.setFormatter( @@ -307,6 +381,7 @@ def run_interactive( prog: drgn.Program, banner_func: Optional[Callable[[str], str]] = None, globals_func: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None, + commands_func: Optional[Callable[[Dict[str, Command]], Dict[str, Command]]] = None, quiet: bool = False, ) -> None: """ @@ -324,6 +399,9 @@ def run_interactive( :param globals_func: Optional function to modify globals provided to the session. Called with a dictionary of default globals, and must return a dictionary to use instead. + :param commands_func: Optional function to modify the command list which is + used for the session. Called with a dictionary of commands, and must + return a dictionary to use instead. :param quiet: Ignored. Will be removed in the future. .. note:: @@ -378,6 +456,11 @@ def run_interactive( if globals_func: init_globals = globals_func(init_globals) + commands = all_commands() + if commands_func: + commands = commands_func(commands) + commands["help"] = help_command(commands) + old_path = list(sys.path) old_displayhook = sys.displayhook old_history_length = readline.get_history_length() @@ -405,7 +488,9 @@ def run_interactive( drgn.set_default_prog(prog) try: - code.interact(banner=banner, exitmsg="", local=init_globals) + console = _InteractiveConsoleWithCommand(commands=commands) + console.locals = init_globals + console.interact(banner=banner, exitmsg="") finally: try: readline.write_history_file(histfile) From 16450b2f8ae521e7c14cfb8a7003c5cc55bbd8fd Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 16:47:33 -0800 Subject: [PATCH 2/7] execscript: Allow setting the globals The execscript() helper is intended to run from the CLI and access the calling scope's global variables. However, for CLI commands to use it, execscript() must optionally accept a global variable dict directly. Signed-off-by: Stephen Brennan --- drgn/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drgn/__init__.py b/drgn/__init__.py index 1df95b5fd..e44e5e21d 100644 --- a/drgn/__init__.py +++ b/drgn/__init__.py @@ -44,7 +44,7 @@ import pkgutil import sys import types -from typing import Union +from typing import Any, BinaryIO, Dict, Optional, Union from _drgn import ( NULL, @@ -152,7 +152,6 @@ if sys.version_info >= (3, 8): _open_code = io.open_code # novermin else: - from typing import BinaryIO def _open_code(path: str) -> BinaryIO: return open(path, "rb") @@ -172,7 +171,7 @@ def _open_code(path: str) -> BinaryIO: ) -def execscript(path: str, *args: str) -> None: +def execscript(path: str, *args: str, globals: Optional[Dict[str, Any]] = None) -> None: """ Execute a script. @@ -214,6 +213,7 @@ def task_exe_path(task): :param path: File path of the script. :param args: Zero or more additional arguments to pass to the script. This is a :ref:`variable argument list `. + :param globals: If provided, globals to use instead of the caller's. """ # This is based on runpy.run_path(), which we can't use because we want to # update globals even if the script throws an exception. @@ -238,7 +238,10 @@ def task_exe_path(task): module.__file__ = path module.__cached__ = None # type: ignore[attr-defined] - caller_globals = sys._getframe(1).f_globals + if globals is not None: + caller_globals = globals + else: + caller_globals = sys._getframe(1).f_globals caller_special_globals = { name: caller_globals[name] for name in _special_globals From a859b392af8a4f90d7b6ea736224ddd0bd30becc Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 16:48:59 -0800 Subject: [PATCH 3/7] Add eval_typed_expression() This helper allows users to evaluate very basic C expressions of the form "(type)int_literal". This enables users to more easily copy and paste printed results back into the console to get the corresponding object. Signed-off-by: Stephen Brennan --- drgn/helpers/common/type.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/drgn/helpers/common/type.py b/drgn/helpers/common/type.py index 26bac600f..77b332230 100644 --- a/drgn/helpers/common/type.py +++ b/drgn/helpers/common/type.py @@ -10,10 +10,11 @@ """ import enum +import re import typing from typing import Container -from drgn import Type +from drgn import Object, Program, Type __all__ = ("enum_type_to_class",) @@ -38,3 +39,25 @@ def enum_type_to_class( if name not in exclude ] return enum.IntEnum(name, enumerators) # type: ignore # python/mypy#4865 + + +def eval_typed_expression(prog: Program, expr: str) -> Object: + """ + Evaluate a C typed expression and return the resulting Object + + In many cases, drgn may format an Object value like this: + + (type)integer_value + + For example, ``(struct task_struct *)0xffff9ea240dd5640``. This function is + able to parse strings like these and return an equivalent object. + + :param expr: Expression of the form "(type)int_literal" + :returns: The equivalent Object + """ + match = re.fullmatch(r"\s*\(([^\)]+)\)\s*(0x[0-9a-zA-Z]+|[0-9]+)\s*", expr) + if not match: + raise ValueError("only expressions of the form (type)integer are allowed") + type_str, val_str = match.groups() + val = int(val_str, 16) if val_str.startswith("0x") else int(val_str) + return Object(prog, type_str, val) From b8b7f5ee4d8af26e85da2c2cad9986c0b1a73a88 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 16:53:07 -0800 Subject: [PATCH 4/7] Add some built-in commands: x, contrib, and let The "x" command is short for "eXecute" or execscript(). The "contrib" command is similar to "x", but runs scripts from the "contrib/" directory if it is present. The "let" command sets variables based on the typed expression. Signed-off-by: Stephen Brennan --- drgn/cli.py | 2 + drgn/helpers/common/commands.py | 97 +++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 drgn/helpers/common/commands.py diff --git a/drgn/cli.py b/drgn/cli.py index cb0b18255..c6d2588c9 100644 --- a/drgn/cli.py +++ b/drgn/cli.py @@ -60,6 +60,8 @@ def all_commands() -> Dict[str, Command]: """ Returns all registered drgn CLI commands """ + import drgn.helpers.common.commands # noqa + return _COMMANDS.copy() diff --git a/drgn/helpers/common/commands.py b/drgn/helpers/common/commands.py new file mode 100644 index 000000000..c4bdc2ee2 --- /dev/null +++ b/drgn/helpers/common/commands.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023, Oracle and/or its affiliates. +# SPDX-License-Identifier: LGPL-2.1-or-later +""" +Commands +-------- + +The ``drgn.helpers.common.commands`` module contains several useful built-in CLI +commands. +""" +from pathlib import Path +import re +import shlex +from typing import Any, Dict + +from drgn import Object, Program, execscript +from drgn.cli import command +from drgn.helpers.common.type import eval_typed_expression + + +@command("x") +def execscript_command(prog: Program, line: str, locals_: Dict[str, Any]) -> None: + """ + The ``x`` command in the drgn CLI: executes a script + + Usage: ``.x SCRIPT_PATH [ARG1 [ARG2 [...]]]`` + """ + args = shlex.split(line) + if len(args) < 2: + print("error: too few arguments") + return + if not Path(args[1]).exists(): + print(f"error: script '{args[1]}' not found") + return + execscript(args[1], *args[2:], globals=locals_) + + +@command("let") +def let_command(prog: Program, line: str, locals_: Dict[str, Any]) -> Object: + """ + The ``let`` command in the drgn CLI: sets a variable to an Object expression + + Usage: ``.let VARNAME = EXPRESSION`` + + For example: + + >>> .let secret = (unsigned int) 42 + >>> secret + (unsigned int)42 + >>> .let ptr = (struct task_struct *)0xffff9ea240dd5640 + >>> ptr.format_(dereference=False) + '(struct task_struct *)0xffff9ea240dd5640' + """ + cmd, stmt = line.split(maxsplit=1) + var, expr = stmt.split("=", maxsplit=1) + var = var.strip() + if not re.fullmatch(r"[a-zA-Z_]\w*", var, re.ASCII): + raise ValueError(f"error: not a valid variable name: {var}") + obj = eval_typed_expression(prog, expr) + locals_[var] = obj + return obj + + +@command("contrib") +def contrib_command(prog: Program, line: str, locals_: Dict[str, Any]) -> None: + """ + The ``contrib`` command in the drgn CLI: executes a contrib script + + Usage: ``.contrib SCRIPT_NAME [ARG1 [ARG2 [...]]]`` + + The script will be searched for in the ``contrib/`` directory: or at least, + where we expect to find it. Please note that contrib scripts are not + included in distributions on PyPI, so this will only work if you are running + from a Git checkout. + + Scripts can be referenced with or without their ".py" extension. + """ + args = shlex.split(line) + if len(args) < 2: + print("error: too few arguments") + return + try: + contrib_dir = Path(__file__).parents[3] / "contrib" + except IndexError: + print("error: could not find contrib directory") + return + if not contrib_dir.is_dir(): + print("error: could not find contrib directory") + return + if args[1].endswith(".py"): + script = contrib_dir / args[1] + else: + script = contrib_dir / f"{args[1]}.py" + if not script.exists(): + print(f"error: contrib script '{script.name}' not found") + return + execscript(str(script), *args[2:], globals=locals_) From 1338a08565b20c5e31e2a0bf2aa23ca4b0053c17 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 23:07:30 -0800 Subject: [PATCH 5/7] doc: Properly document the CLI command feature Signed-off-by: Stephen Brennan --- docs/user_guide.rst | 25 +++++++++++++++++++++++++ drgn/cli.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 0ebf60b4d..f3ada48f3 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -375,6 +375,7 @@ adds a few nice features, including: * Tab completion * Automatic import of relevant helpers * Pretty printing of objects and types +* Helper commands The default behavior of the Python `REPL `_ is to @@ -409,6 +410,30 @@ explicitly:: int counter; } atomic_t +Interactive Commands +^^^^^^^^^^^^^^^^^^^^ + +When running in interactive mode, the drgn CLI accepts an extensible set of +commands. Any input which starts with a ``.`` will be interpreted as a command, +rather than as Python code. For example, you can use the ``.x`` command to +execute a script, similar to the :func:`execscript()` function:: + + $ cat myscript.py + import sys + print("Executing myscript.py with arguments: " + str(sys.argv[1:])) + $ sudo drgn + >>> .x myscript.py argument + Executing myscript.py with arguments ['argument'] + >>> execscript("myscript.py", "argument") + Executing myscript.py with arguments ['argument'] + +Not only are the interactive commands less verbose than their equivalent Python +code, but they are also user-extensible. You can define a +:class:`Command ` function and either register it with the +:func:`@command ` decorator, or provide it to +:func:`run_interactive ` using the argument +``commands_func``. + Next Steps ---------- diff --git a/drgn/cli.py b/drgn/cli.py index c6d2588c9..099c4344e 100644 --- a/drgn/cli.py +++ b/drgn/cli.py @@ -29,6 +29,36 @@ logger = logging.getLogger("drgn") Command = Callable[[drgn.Program, str, Dict[str, Any]], Any] +""" +A command which can be executed in the drgn CLI + +The drgn CLI allows for shell-like commands to be executed. Any input to the CLI +which begins with a ``.`` is interpreted as a command rather than a Python +statement. Commands are simply callables which take three arguments: + +- a :class:`drgn.Program` +- a ``str`` which contains the command line, and +- a dictionary of local variables in the CLI (``Dict[str, Any]``) + +For example, the following is a command function:: + + def hello_world(prog, cmdline, locals_): + print("hello world!") + print(f"your command: {cmdline}") + print(f"kernel command line: {prog['saved_command_line']}") + locals_["secret"] = 42 + +The command, if registered with the drgn CLI, might be used like so: + + >>> .hello + hello world! + your command: .hello + kernel command line: (char *)0xffff9ea9cf7c3600 = "quiet splash" + +User-defined commands may be provided to :func:`run_interactive()` via the +``commands_func`` argument. Commands can also be registered so that they are +included in drgn's default command set using the :func:`command` decorator. +""" _COMMANDS: Dict[str, Command] = {} From 8b313502ee49d01a0cdc50f06338f690a6fc8af9 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 8 Nov 2023 23:38:00 -0800 Subject: [PATCH 6/7] cli: load drgn commands using drgn.command.v1 entry point Signed-off-by: Stephen Brennan --- .pre-commit-config.yaml | 1 + drgn/cli.py | 46 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1588bd9a0..3ca485a84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,7 @@ repos: - id: mypy args: [--show-error-codes, --strict, --no-warn-return-any, --no-warn-unused-ignores] files: ^drgn/.*\.py|_drgn.pyi$ + additional_dependencies: ["types-setuptools"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/drgn/cli.py b/drgn/cli.py index 099c4344e..d01fabd56 100644 --- a/drgn/cli.py +++ b/drgn/cli.py @@ -18,7 +18,7 @@ import sys import textwrap import traceback -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Iterable, Optional import drgn from drgn.internal.rlcompleter import Completer @@ -89,9 +89,53 @@ def decorator(cmd: Command) -> Command: def all_commands() -> Dict[str, Command]: """ Returns all registered drgn CLI commands + + By default, only the commands which are built-in to drgn, or registered via + :func:`command`, are returned. Since decorators are evaluated at module load + time, any command defined in a module which is not imported prior to the + drgn CLI being run, will not be loaded. + + However, drgn can allow user command modules to be loaded and registered by + using the ``drgn.command.v1`` `entry point + `_. + Third-party packages can define a module as an entry point which should be + imported, typically in the setup.py:: + + setup( + ..., + entry_points={ + "drgn.command.v1": { + "my_module = fully_qualified.module_path", + }, + } + ) + + In the above example, a function defined within the module + ``fully_qualified.module_path`` and registered with :func:`command`, would + always be included in the drgn CLI and this functon if the relevant package + is installed. """ import drgn.helpers.common.commands # noqa + # The importlib.metadata API is included in Python 3.8+. Normally, one might + # simply try to import it, catching the ImportError and falling back to the + # older API. However, the API was _transitional_ in 3.8 and 3.9, and it is + # different enough to break callers compared to the non-transitional API. So + # here we are, using sys.version_info like heathens. + if sys.version_info >= (3, 10): + from importlib.metadata import entry_points # novermin + else: + import pkg_resources + + def entry_points(group: str) -> Iterable[pkg_resources.EntryPoint]: + return pkg_resources.iter_entry_points(group) + + # Drgn command "entry points" are simply modules. The act of loading / + # importing them will result in their @command decorators being executed, + # and _COMMANDS will be updated properly. + for entry_point in entry_points(group="drgn.command.v1"): # type: ignore + entry_point.load() # type: ignore + return _COMMANDS.copy() From 3809cd24774ec0d018bc69f5b477ea3528efe5e8 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Thu, 9 Nov 2023 01:11:02 -0800 Subject: [PATCH 7/7] contrib: Add command support to ptdrgn.py Signed-off-by: Stephen Brennan --- contrib/ptdrgn.py | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/contrib/ptdrgn.py b/contrib/ptdrgn.py index d47c6895b..07cd82eb6 100644 --- a/contrib/ptdrgn.py +++ b/contrib/ptdrgn.py @@ -11,6 +11,7 @@ Requires: "pip install ptpython" which brings in pygments and prompt_toolkit """ +import builtins import functools import importlib import os @@ -18,16 +19,20 @@ import sys from typing import Any, Callable, Dict, Optional, Set +import ptpython.repl from prompt_toolkit.completion import Completion, Completer +from prompt_toolkit.document import Document from prompt_toolkit.formatted_text import PygmentsTokens from prompt_toolkit.formatted_text import fragment_list_to_text, to_formatted_text from ptpython import embed from ptpython.completer import DictionaryCompleter -from ptpython.repl import run_config +from ptpython.repl import PythonRepl, run_config +from ptpython.validator import PythonValidator from pygments.lexers.c_cpp import CLexer import drgn import drgn.cli +from drgn.cli import all_commands, Command, help_command class DummyForRepr: @@ -124,10 +129,40 @@ def _format_result_output(result: object): repl.completer = ReorderDrgnObjectCompleter(repl.completer) +class DrgnPythonValidator(PythonValidator): + + def validate(self, document: Document) -> None: + if document.text.lstrip().startswith("."): + return + return super().validate(document) + + +class DrgnPythonRepl(PythonRepl): + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs, _validator=DrgnPythonValidator()) + + def __run_command(self, line: str) -> object: + cmd_name = line.split(maxsplit=1)[0][1:] + if cmd_name not in self._commands: + print(f"{cmd_name}: drgn command not found") + return None + cmd = self._commands[cmd_name] + locals = self.get_locals() + prog = locals["prog"] + setattr(builtins, "_", cmd(prog, line, locals)) + + def eval(self, line: str) -> object: + if line.lstrip().startswith('.'): + return self.__run_command(line) + return super().eval(line) + + def run_interactive( prog: drgn.Program, banner_func: Optional[Callable[[str], str]] = None, globals_func: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None, + commands_func: Optional[Callable[[Dict[str, Command]], Dict[str, Command]]] = None, quiet: bool = False, ) -> None: """ @@ -184,6 +219,12 @@ def run_interactive( if globals_func: init_globals = globals_func(init_globals) + commands = all_commands() + if commands_func: + commands = commands_func(commands) + commands["help"] = help_command(commands) + DrgnPythonRepl._commands = commands + old_path = list(sys.path) try: old_default_prog = drgn.get_default_prog() @@ -212,6 +253,7 @@ def run_interactive( if __name__ == "__main__": + ptpython.repl.PythonRepl = DrgnPythonRepl # Muck around with the internals of drgn: swap out run_interactive() with our # ptpython version, and then call main as if nothing happened. drgn.cli.run_interactive = run_interactive