1"""Contains the class definitions outlining the schema of the test data. For LLDB conversion2from/into these types, see `./from_lldb.py`"""34import json5import os6from dataclasses import asdict, dataclass, field, fields, is_dataclass7from enum import Enum8from pprint import pformat9from typing import Any, Final, Optional, Union, _eval_type, get_origin1011char = str12Primitive = Union[int, float, bool, char]13ByteSize = int1415# see: default json decoder docs https://docs.python.org/3/library/json.html#json.JSONDecoder16# The types we're dealing with can only be: int, str, float, list, dict, bool, and None17JsonType = Union[int, str, float, list["JsonType"], bool, None, dict[str, "JsonType"]]181920class Result(Enum):21    Ok = True22    Mismatch = False2324    def __and__(self, other: "Result") -> "Result":25        return Result(self.value & other.value)2627    def __bool__(self) -> bool:28        return self.value293031ANSI_RED = "\033[91m"32ANSI_END = "\033[0m"333435def print_error(error_source: str, message: str):36    print(f"{ANSI_RED}  [repr error: {error_source}]{ANSI_END} {message}")373839def format_mismatch(label: str, got: Optional[Any], expected: Optional[Any]) -> str:40    if got is None and expected is not None:41        return f"{label} not found, expected: {expected}"42    elif expected is not None and got is None:43        return f"{label} '{got}' found when none was expected."44    else:45        return f"{label} does not match.\n    Expected: {expected}\n    Got: {got}"464748def print_mismatch(49    error_source: str, label: str, got: Optional[Any], expected: Optional[Any]50):51    print_error(error_source, format_mismatch(label, got, expected))525354class Target(Enum):55    """Due to the differences between PDB and DWARF debug info, we cannot guarantee their output56    will be identical. Since LLDB can handle both, we need to conditionally select the correct57    test data to use.5859    Additionally, since there are differences in the internals of some structs based on OS (e.g.60    `PathBuf`/`OsString`), we need to be aware of whether we're on Windows or not.6162    A global var `TARGET` is set to the current variant upon `lldb_test.py`'s instantiation using an63    env var passed from `compiletest` and is not expected to change afterwards."""6465    NonWindows = "non_windows"66    WindowsGnu = "windows_gnu"67    WindowsMsvc = "windows_msvc"686970def get_target() -> Target:71    # set by compiletest when launching LLDB72    t: str = os.environ["LLDB_BATCHMODE_TARGET_TRIPLE"]7374    if t.endswith("windows-msvc"):75        return Target.WindowsMsvc76    if t.endswith(("windows-gnu", "windows-gnullvm")):77        return Target.WindowsGnu7879    return Target.NonWindows808182BLESS: Final[bool] = os.environ["LLDB_BATCHMODE_BLESS_TEST_DATA"] == "1"83"""Global constant set by `compiletest` that determines whether or not we are blessing the test84data."""858687TARGET: Final[Target] = get_target()88"""Global constant set by `compiletest`. Determines which target the tests were run for, thus which89set of test input we check."""909192def from_dict(ty: type[Any], data: JsonType):93    """Translates a dictionary into an instance of the given dataclass type (with possibly nested94    dataclasses).9596    Relies on accurate type hints for the dataclass's fields, and the default `dataclass.__init__`97    definition."""9899    ty = _eval_type(ty, globals(), locals())100101    origin = get_origin(ty) or ty102103    # Optional isn't a constructor, so we have to "unwrap" it.104    if origin == Union and len(ty.__args__) == 2 and ty.__args__[1] == type(None):105        ty = ty.__args__[0]106107    # Special handling for array-like children space optimization108    if (109        ty == Union[list[Child], ArrayLikeChildren, type(None)]110        or ty == Union[list["Child"], ArrayLikeChildren, type(None)]111    ):112        if isinstance(data, dict):113            al_type = data["type"]114            vals: Union[list[dict], list[Primitive]] = data["vals"]115            # if we have a list of dict, we know we have array children116            # otherwise, we have a list of primitives that can be used117            # as-is118            if len(vals) != 0 and isinstance(vals[0], dict):119                vals = [from_dict(ArrayChild, v) for v in vals]120121            return ArrayLikeChildren(al_type, vals)122        elif isinstance(data, list):123            return [from_dict(Child, i) for i in data]124125    # recurse into lists126    if isinstance(data, list):127        # pulls the generic type from the list (e.g. `list[int]` -> `int`)128        inner = ty.__args__[0]129130        return [from_dict(inner, i) for i in data]131132    if origin == dict and ty.__args__[0] == str:133        assert isinstance(data, dict)134        val_ty = ty.__args__[1]135136        if val_ty in [Variable, Child, Type, Field, ArrayChild]:137            return {k: from_dict(val_ty, data[k]) for k in data}138139    # map dict -> dataclass, recursing for each field140    if is_dataclass(ty):141        assert isinstance(data, dict)142143        field_types = {f.name: f.type for f in fields(ty)}144145        try:146            field_map = {}147148            for f in data:149                f_type = field_types[f]150151                field_map[f] = from_dict(f_type, data[f])152153            # if you've never seen this before, `**` is the splat operator. It expands a mapping154            # type (in this case a dict) to keyword arguments. The ordering of the mapping does not155            # matter, only that the mapping's keys match the functions keyword args, and156            # `len(mapping)` == the number of keyword args.157            return ty(**field_map)158        except KeyError as e:159            print(160                f"Unable to convert dict to {ty}: Invalid field name {e}. If the test schema was \161changed intentionally, use the `--bless` option to update test data to the new schema."162            )163164    # for any other type, we don't need to do any processing165    return data166167168@dataclass(frozen=True)169class Field:170    name: str171    type: str172    """The fully qualified name of the field's type. Full type information should be looked up173    via `TargetData.types`"""174175    offset: ByteSize176177178@dataclass179class Type:180    size: ByteSize181    # When GDB support is added to the test framework, basic_type and type_class will probably be182    # converted to a wrapper IntEnum that converts GDB's equivalent information to183    type_class: int184    """The `lldb.eTypeClass` value associated with thjs type. Tested due to our use of it in type185    recognizer functions."""186187    basic_type: Optional[int] = None188    """The `lldb.eBasicType` value associated with this type. Tested due to our use of it in type189    recognizer functions."""190191    fields: Optional[list[Field]] = field(default_factory=list)192    """Stored as a list due to our reliance on `SBType.GetFieldAtIndex()`193194    Note: LLDB **does not** reorder the fields of a type based on their offset. For example,195    `GetFieldAtIndex(0).GetByteOffset()` may return `8`. Instead, the order of the fields is a196    direct reflection of their ordering in the debug info (which, as far as I know, is the same as197    their declaration order in the source code).198    """199200    generic_params: Optional[list[str]] = field(default_factory=list)201    """Stored as a list due to our reliance on `SBType.GetTemplateArgumentType()` and the sequential202    behavior of `lldb_providers.get_template_args`"""203    # FIXME the only way we can look up static fields is by name (as of lldb 22), so we need a way204    # to discover them. ATM only sum-type enums on MSVC use static fields, and those are fixed205    # values, so it's not super urgent.206    # static_fields: list[StaticField]207208    def matches(209        self, expected: "Type", type_name: str, provider_ok: bool = False210    ) -> Result:211        result = Result.Ok212        error_source = f"type '{type_name}'"213        # FIXME handle 32 bit targets214        if self.size != expected.size:215            result = Result.Mismatch216            print_mismatch(error_source, "size", self.size, expected.size)217218        if self.fields != expected.fields:219            result = Result.Mismatch220            self.print_field_errors(expected, error_source)221222        if self.generic_params != expected.generic_params:223            result = Result.Mismatch224            print_mismatch(225                error_source,226                "generic_params",227                self.generic_params,228                expected.generic_params,229            )230231        if result == Result.Mismatch and provider_ok:232            print_error(233                error_source,234                "It appears these changes do not affect the type's providers. Consider rerunning \235with the `--bless` option",236            )237238        return result239240    def print_field_errors(self, expected: "Type", error_source: str):241        """Extra processing for better error messages. The following common cases are covered:242        * New/Missing fields243        * Source code rearranged fields244        * Rustc rearranged fields245        * Renamed fields246247        If none of the common cases are encountered, we just generically print any mismatched248        fields.249        """250251        # FIXME these checks aren't exactly the most efficient they could be. Luckily, the happy252        # path skips this function entirely, so passing tests are still fast. These checks could253        # probably all be done in 2ish total iters over each list, but optimization isn't a huge254        # concern at the moment.255256        got_set = set(self.fields)257        expected_set = set(expected.fields)258259        if len(self.fields) != len(expected.fields):260            new_fields = got_set.difference(expected_set)261262            missing_fields = expected_set.difference(got_set)263264            if len(missing_fields) != 0:265                print_error(266                    error_source,267                    f"The following field(s) appear to have been removed from the type:\n\268{missing_fields}",269                )270271            if len(new_fields) != 0:272                print_error(273                    error_source,274                    f"The following field(s) appear to have been added to the type:\n\275{new_fields}",276                )277278        # are all of the same fields present, regardless of order? If so, they were rearranged279        # in the source code, but the compiler kept the same ordering.280        elif got_set == expected_set:281            print_error(282                error_source,283                f"Field(s) appear to have been rearranged:\n    Expected:\n\284{pformat(self.fields, indent=6)}\n    Got:\n{pformat(expected.fields, indent=6)}",285            )286        else:287            # we know for sure that both sets of fields are the same length, but some parts of one288            # or more fields don't match289            types_match = True290            offsets_match = True291            names_match = True292            mismatches: list[tuple[Field, Field]] = []293294            for g, e in zip(self.fields, expected.fields):295                if g.type != e.type:296                    types_match = False297                    mismatches.append((g, e))298                if g.offset != e.offset:299                    offsets_match = False300                    mismatches.append((g, e))301                if g.name != e.name:302                    names_match = False303                    mismatches.append((g, e))304305            # If the types and offsets are the same but the names aren't, we know fields have306            # been renamed.307            if types_match and offsets_match:308                renames = "\n    ".join(309                    f"{m[1].name} -> {m[0].name}" for m in mismatches310                )311                print_error(312                    error_source,313                    f"The following field(s) appear to have been renamed (expected -> got):\n\314    {renames}",315                )316317            # If the types and names are the same, but the offsets are different, we know that rustc318            # has decided to order the fields differently, despite the source code not changing319            elif types_match and names_match:320                reordered = "\n    ".join(321                    (322                        f"{m[1].name} offset: +{m[1].offset} -> {m[0].name} offset: \n\323+{m[0].offset}"324                    )325                    for m in mismatches326                )327328                print_error(329                    error_source,330                    f"The following field(s) appear to have been reordered by rustc (expected -> \331got):\n    {reordered}",332                )333334            else:335                mm_string = "\n    ".join(f"{m[1]} -> {m[0]}" for m in mismatches)336337                print_error(338                    error_source,339                    f"The following field(s) do not match (expected -> got):\n\340    {mm_string}",341                )342343344@dataclass345class ArrayChild:346    """A child in `ArrayLikeChildren`"""347348    value: Optional[Primitive] = None349    children: Optional[Union[list["Child"], "ArrayLikeChildren"]] = None350351352@dataclass353class ArrayLikeChildren:354    """Visualizers commonly output children in the form `[0]=<value>, [1]=<value>, ...` where the355    types of all elements are the same. This container allows for space-optimizing the output data356    by only storing the type once, and not storing the names of the children. During comparison,357    `ArrayLikeChildren` are converted to the equivalent `list[Child]` and compared as normal.358359    To detect if a set of children are array-like, use `is_arraylike(children)`. To convert an360    array-like list into `ArrayLikeChildren`, use `make_arraylike(children)`361    """362363    type: str364    vals: Union[list[ArrayChild], list[Primitive]] = field(default_factory=list)365366    def __len__(self):367        return len(self.vals)368369    def __getitem__(self, idx):370        return self.vals[idx]371372373@dataclass374class Child:375    """Similar to `Variable`, but carries less information since we primarily test top-level376    values (and assume values of these child types have been tested thoroughly elsewhere).377378    Note that if the type has a synthetic provider (lldb) or pretty printer (gdb), the child names379    and types can be set to anything at all, so we do need to test these separately from the380    parent's type's fields."""381382    name: str383    """The name used to access the child. If the parent object has a synthetic, the child name can384    be overridden."""385386    type: str387    """The fully qualified name of the child's type. Full type information should be looked up388    via `TargetData.types`"""389390    value: Optional[Primitive] = None391    children: Optional[Union[list["Child"], ArrayLikeChildren]] = field(392        default_factory=list393    )394    """Children are stored as a list because of our use of `GetChildAtIndex()`. Providers can also395    dictate the order that children populate, so it's important to ensure that stays consistent too.396    """397398399def is_arraylike(children: list[Child]) -> bool:400    """Returns true if every child is the same type, and if all of the child names are, in order,401    `[0]`, `[1]`, `[2]`, ...402    """403    if len(children) <= 0:404        return False405406    first_type = children[0].type407408    return all(409        x.name.startswith("[")410        and x.name.endswith("]")411        and int(x.name[1:-1]) == i412        and x.type == first_type413        for i, x in enumerate(children)414    )415416417def make_arraylike(children: list[Child]) -> ArrayLikeChildren:418    return ArrayLikeChildren(419        children[0].type,420        [421            ArrayChild(c.value, c.children) if len(c.children) != 0 else c.value422            for c in children423        ],424    )425426427@dataclass428class Variable:429    type: str430    """The fully qualified name of the variable's type. Full type information should be looked up431    via `TargetData.types`"""432433    pretty_type_name: Optional[str] = None434    """Type names can be overridden by `SyntehticProvider.get_type_name()` in LLDB and by435    `type_printer` in GDB"""436437    pretty_print: Optional[str] = None438    """The string-result of pretty printing the value (`SBValue.GetSummary` for LLDB,439    `pretty_printer.to_string` for GDB). `None` for aggregates with no summary provider."""440441    value: Optional[Primitive] = None442    """`None` if the object does not have a primitive representation."""443444    synthetic: Optional[str] = None445    """The class/function name of the synthetic provider (lldb) or pretty printer (gdb).446    `None` if the object does not have a synthetic provider"""447448    summary: Optional[str] = None449    """The function name of the summary provider. `None` if the object does not have a summary450    provider, or if the test data is for GDB"""451452    format: Optional[int] = None453    """The `lldb.eFormat` enum variant associated with this type (if applicable)."""454455    # Stored as a list instead of a dict because child order matters456    children: Optional[Union[list[Child], ArrayLikeChildren]] = field(457        default_factory=list458    )459    """A list of children provided by the object. If the object has a synthetic provider, the460    children are the result of the provider's `get_child_at_index` function"""461462    def has_visualizer(self) -> bool:463        return (464            self.synthetic is not None465            or self.summary is not None466            or self.format is not None467        )468469470@dataclass471class BlessMetadata:472    """473    Contains additional context about the tools at the time the test data was generated.474    """475476    python_version: str = ""477    debugger_version: str = ""478    feature_flags: str = ""479480481@dataclass482class TargetData:483    """484    Top-level container for all test data.485486    Due to the differences between PDB and DWARF debug info, we cannot guarantee their output487    will be identical. Since LLDB can handle both, we need to conditionally select the correct488    test data to use.489490    Additionally, since there are differences in the internals of some structs based on OS (e.g.491    `PathBuf`/`OsString`), we need to be aware of whether we're on Windows or not.492493    A global var `TARGET` is set to the current variant upon `lldb_batchmode`'s instantiation using494    an env var passed from `compiletest` and is not expected to change afterwards.495    """496497    bless_metadata: BlessMetadata = field(default_factory=BlessMetadata)498    """Miscellaneous data included to make diagnosing issues easier. This data is not intended to be499    tested against."""500501    # If we ever decide that it makes sense to check the same variable twice at the same breakpoint502    # this will need to be converted to a list503    breakpoints: list[dict[str, Variable]] = field(default_factory=list)504    """Each element corresponds to one stopping point in the test. The element itself is a505    dictionary mapping variable names to their respective test data."""506507    types: dict[str, Type] = field(default_factory=dict)508    """509    A map of type names to types. Contains all types present in the test's variables, including the510    types of fields and child objects.511    """512513    @staticmethod514    def initialize() -> "TargetData":515        result = TargetData()516        path = os.environ["LLDB_BATCHMODE_INPUT_DATA_PATH"]517        if not os.path.isfile(path):518            if BLESS:519                return result520            else:521                raise Exception(522                    f"Invalid input data path: '{path}'\nIf test data has not been \523generated for this test yet, consider using the `--bless` option."524                )525526        if BLESS:527            return result528529        with open(path, "r") as f:530            try:531                result = from_dict(TargetData, json.load(f))532            except json.decoder.JSONDecodeError:533                print("Warning: Malformed input data, reverting to default")534535        return result536537    def save_blessing(self, metadata: BlessMetadata):538        """Writes the entirety of `self` to the env var `LLDB_BATCHMODE_INPUT_DATA_PATH`, which is539        set by `compiletest` before running `lldb_batchmode. Used to finalize changes made by one or540        more `from_lldb.bless_variable` calls.541542        This function should be called exactly once, right before543        `lldb_batchmode.runner.main` exits if the following conditions are met:544545        1. No other exceptions or error states occurred546        2. `BLESS == True`547        3. At least one `repr` pseudo-command was processed548549        This prevents us from saving incomplete data or invalid data. It also prevents us from550        creating input data files for tests that do not need it.551        """552553        self.bless_metadata = metadata554        path = os.environ["LLDB_BATCHMODE_INPUT_DATA_PATH"]555        # dumping directly to a file is somewhat unsafe. If the `Variable`/`Type` data ends up in a556        # state that cannot be serialized correctly, the json ends up malformed, and we could end up557        # overwriting valid test data with a complete mess. Since the in-memory data is typically558        # completely valid, the testing logic will pass and make it seem like nothing is wrong.559560        # While we could rely on git to help revert the test file, it's better to just not allow it561        # to save malformed json in the first place. Thus, we dump the JSON, re-read it to check562        # for `JSONDecodeError`, and write it to the target file if no error occurred.563        x = json.dumps(clean_nones(asdict(self)), indent=" ")564        _ = json.loads(x)565566        # ensure the necessary directories exist first567        import pathlib568569        os.makedirs(pathlib.Path(path).parent, exist_ok=True)570571        with open(path, "w") as f:572            f.write(x)573            f.write("\n")574575576def clean_nones(value):577    """578    Recursively remove all None values from dictionaries and lists, and returns579    the result as a new dictionary or list.580    """581    if isinstance(value, list):582        x = [clean_nones(x) for x in value if x is not None]583        return x if len(x) != 0 else None584    elif isinstance(value, dict):585        x = {586            key: clean_nones(val)587            for key, val in value.items()588            if clean_nones(val) is not None589        }590        return x if len(x) != 0 else None591    else:592        return value593594595INPUT_DATA: TargetData = TargetData.initialize()