build / gn_ast / json_gn_editor.py [blame]

# Lint as: python3
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''Helper script to use GN's JSON interface to make changes.'''

from __future__ import annotations

import contextlib
import copy
import dataclasses
import json
import logging
import os
import pathlib
import re
import shutil
import subprocess
import sys

from typing import Dict, Iterator, List, Optional, Tuple

_SRC_PATH = pathlib.Path(__file__).resolve().parents[2]

_BUILD_ANDROID_GYP_PATH = _SRC_PATH / 'build/android/gyp'
if str(_BUILD_ANDROID_GYP_PATH) not in sys.path:
    sys.path.append(str(_BUILD_ANDROID_GYP_PATH))

from util import build_utils

# Refer to parse_tree.cc for GN AST implementation details:
# https://gn.googlesource.com/gn/+/refs/heads/main/src/gn/parse_tree.cc
# These constants should match corresponding entries in parse_tree.cc.
# TODO: Add high-level details for the expected data structure.
NODE_CHILD = 'child'
NODE_TYPE = 'type'
NODE_VALUE = 'value'
BEFORE_COMMENT = 'before_comment'
SUFFIX_COMMENT = 'suffix_comment'
AFTER_COMMENT = 'after_comment'


@contextlib.contextmanager
def _backup_and_restore_file_contents(path: str):
    with open(path) as f:
        contents = f.read()
    try:
        yield
    finally:
        # Ensure that the timestamp is updated since otherwise ninja will not
        # re-build relevant targets with the original file.
        with open(path, 'w') as f:
            f.write(contents)


def _build_targets_output(
        out_dir: str,
        targets: List[str],
        should_print: Optional[bool] = None) -> Optional[str]:
    env = os.environ.copy()
    if should_print is None:
        should_print = logging.getLogger().isEnabledFor(logging.DEBUG)
    # Ensuring ninja does not attempt to summarize the build results in slightly
    # faster builds. This script does many builds so this time can add up.
    if 'NINJA_SUMMARIZE_BUILD' in env:
        del env['NINJA_SUMMARIZE_BUILD']
    proc = subprocess.Popen(['autoninja', '-C', out_dir] + targets,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            env=env,
                            text=True)
    lines = []
    prev_line = ''
    width = shutil.get_terminal_size().columns
    while proc.poll() is None:
        line = proc.stdout.readline()
        lines.append(line)
        if should_print:
            if prev_line.startswith('[') and line.startswith('['):
                # Shrink the line according to terminal size.
                msg = line.rstrip()
                if len(msg) > width:
                    # 5 = 3 (Ellipsis) + 2 (header)
                    length_to_show = width - 5
                    msg = f'{msg[:2]}...{msg[-length_to_show:]}'
                # \r to return the carriage to the beginning of line, \033[K to
                # replace the normal \n to erase until the end of the line. This
                # allows ninja output for successful targets to overwrite each
                # other.
                msg = f'\r{msg}\033[K'
            elif prev_line.startswith('['):
                # Since the previous line likely did not include a newline, an
                # extra newline is needed to avoid the current line being
                # appended to the previous line.
                msg = f'\n{line}'
            else:
                msg = line
            print(msg, end='')
        prev_line = line
    if proc.returncode != 0:
        return None
    return ''.join(lines)


def _generate_project_json_content(out_dir: str) -> str:
    build_utils.CheckOutput(['gn', 'gen', '--ide=json', out_dir])
    with open(os.path.join(out_dir, 'project.json')) as f:
        return f.read()


@dataclasses.dataclass
class DepList:
    """Represents a dep list assignment in GN."""
    target_name: Optional[str]  # The name of the target containing the list.
    variable_name: str  # Left-hand side variable name the list is assigned to.
    child_nodes: List[dict]  # Right-hand side list of nodes.
    operation: str  # The assignment operation, whether += or =.


class BuildFile:
    """Represents the contents of a BUILD.gn file."""
    def __init__(self,
                 build_gn_path: str,
                 root_gn_path: pathlib.Path,
                 *,
                 dryrun: bool = False):
        self._root = root_gn_path
        self._rel_path = os.path.relpath(build_gn_path, root_gn_path)
        self._gn_rel_path = '//' + os.path.dirname(self._rel_path)
        self._full_path = os.path.abspath(build_gn_path)
        self._skip_write_content = dryrun

    def __enter__(self):
        output = build_utils.CheckOutput(
            ['gn', 'format', '--dump-tree=json', self._full_path])
        self._content = json.loads(output)
        self._original_content = json.dumps(self._content)
        return self

    def __exit__(self, exc, value, tb):
        if not self._skip_write_content:
            self.write_content_to_file()

    # See: https://gist.github.com/sgraham/bd9ffee312f307d5f417019a9c0f0777
    def _find_all(self, match_fn):
        results = []

        def get_target_name(node) -> Optional[str]:
            """Example format (with irrelevant fields omitted):
            {
                "child": [ {
                    "child": [ {
                        "type": "LITERAL",
                        "value": "\"hello_world_java\""
                    } ],
                    "type": "LIST"
                }, {
                    ...
                } ],
                "type": "FUNCTION",
                "value": "java_library"
            }

            Example return: hello_world_java
            """
            if node.get(NODE_TYPE) != 'FUNCTION':
                return None
            children = node.get(NODE_CHILD)
            if not children:
                return None
            first_child = children[0]
            if first_child.get(NODE_TYPE) != 'LIST':
                return None
            grand_children = first_child.get(NODE_CHILD)
            if not grand_children:
                return None
            grand_child = grand_children[0]
            if grand_child.get(NODE_TYPE) != 'LITERAL':
                return None
            name = grand_child.get(NODE_VALUE)
            if name.startswith('"'):
                return name[1:-1]
            return name

        def recursive_find(root, last_known_target=None):
            target_name = get_target_name(root) or last_known_target
            matched = match_fn(root)
            if matched is not None:
                results.append((target_name, matched))
                return
            children = root.get(NODE_CHILD)
            if children:
                for child in children:
                    recursive_find(child, last_known_target=target_name)

        recursive_find(self._content)
        return results

    def _normalize(self, name: Optional[str], abs_path: bool = True):
        """Returns the absolute GN path to the target with |name|.

        This method normalizes target names, assuming that relative targets are
        referenced based on the current file, allowing targets to be compared
        by name to determine whether they are the same or not.

        Given the current file is chrome/android/BUILD.gn:

        # Removes surrounding quotation marks.
        "//chrome/android:chrome_java" -> //chrome/android:chrome_java

        # Makes relative paths absolute.
        :chrome_java -> //chrome/android:chrome_java

        # Spells out GN shorthands for basenames.
        //chrome/android -> //chrome/android:android
        """
        if not name:
            return ''
        if name.startswith('"'):
            name = name[1:-1]
        if not name.startswith('//') and abs_path:
            name = self._gn_rel_path + name
        if not ':' in name:
            name += ':' + os.path.basename(name)
        return name

    def _find_all_list_assignments(self):
        def match_list_assignments(node):
            r"""Matches and returns the list being assigned.

            Binary node (with an operation such as = or +=)
             /       \
            /         \
            name      list of nodes

            Returns (name, list of nodes, op)
            """
            if node.get(NODE_TYPE) != 'BINARY':
                return None
            operation = node.get(NODE_VALUE)
            children = node.get(NODE_CHILD)
            assert len(children) == 2, (
                'Binary nodes should have two child nodes, but the node is: '
                f'{node}')
            left_child, right_child = children
            if left_child.get(NODE_TYPE) != 'IDENTIFIER':
                return None
            name = left_child.get(NODE_VALUE)
            if right_child.get(NODE_TYPE) != 'LIST':
                return None
            list_of_nodes = right_child.get(NODE_CHILD)
            return name, list_of_nodes, operation

        return self._find_all(match_list_assignments)

    def _find_all_deps_lists(self) -> Iterator[DepList]:
        list_tuples = self._find_all_list_assignments()
        for target_name, (var_name, node_list, operation) in list_tuples:
            if (var_name == 'deps' or var_name.startswith('deps_')
                    or var_name.endswith('_deps') or '_deps_' in var_name):
                yield DepList(target_name=target_name,
                              variable_name=var_name,
                              child_nodes=node_list,
                              operation=operation)

    def _new_literal_node(self, value: str, begin_line: int = 1):
        return {
            'location': {
                'begin_column': 1,
                'begin_line': begin_line,
                'end_column': 2,
                'end_line': begin_line,
            },
            'type': 'LITERAL',
            'value': f'"{value}"'
        }

    def _clone_replacing_value(self, node_to_copy: Dict, new_dep_name: str):
        """Clone the existing node to preserve line numbers and update name.

        It is easier to clone an existing node around the same location, as the
        actual dict looks like this:
        {
            'location': {
                'begin_column': 5,
                'begin_line': 137,
                'end_column': 27,
                'end_line': 137
            },
            'type': 'LITERAL',
            'value': '":anr_data_proto_java"'
        }

        Thus the new node to return should keep the same 'location' value (the
        parser is tolerant as long as it's roughly in the correct spot) but
        update the 'value' to the new dependency name.
        """
        new_dep = copy.deepcopy(node_to_copy)
        # Any comments associated with the previous dep would not apply.
        for comment_key in (BEFORE_COMMENT, AFTER_COMMENT, SUFFIX_COMMENT):
            new_dep.pop(comment_key, None)  # Remove if exists.
        new_dep[NODE_VALUE] = f'"{new_dep_name}"'
        return new_dep

    def add_deps(self, target: str, deps: List[str]) -> bool:
        added_new_dep = False
        normalized_target = self._normalize(target)
        for dep_list in self._find_all_deps_lists():
            if dep_list.target_name is None:
                continue
            # Only modify the first assignment operation to the deps variable,
            # otherwise if there are += operations, then the list of deps will
            # be added multiple times to the same target's deps.
            if dep_list.operation != '=':
                continue
            full_target_name = f'{self._gn_rel_path}:{dep_list.target_name}'
            # Support both the exact name and the absolute GN target names
            # starting with //.
            if (target != dep_list.target_name
                    and normalized_target != full_target_name):
                continue
            if dep_list.variable_name != 'deps':
                continue
            existing_dep_names = set(
                self._normalize(child.get(NODE_VALUE), abs_path=False)
                for child in dep_list.child_nodes)
            for new_dep_name in deps:
                if new_dep_name in existing_dep_names:
                    logging.info(
                        f'Skipping existing {new_dep_name} in {target}.deps')
                    continue
                logging.info(f'Adding {new_dep_name} to {target}.deps')
                # If there are no existing child nodes, then create a new one.
                # Otherwise clone an existing child node to ensure more accurate
                # line numbers and possible better preserve comments.
                if not dep_list.child_nodes:
                    new_dep = self._new_literal_node(new_dep_name)
                else:
                    new_dep = self._clone_replacing_value(
                        dep_list.child_nodes[0], new_dep_name)
                dep_list.child_nodes.append(new_dep)
                added_new_dep = True
        if not added_new_dep:
            # This should match the string in bytecode_processor.py.
            print(f'Unable to find {target}')
        return added_new_dep

    def search_deps(self, name_query: Optional[str],
                    path_query: Optional[str]) -> bool:
        if path_query:
            if not re.search(path_query, self._rel_path):
                return False
            elif not name_query:
                print(self._rel_path)
                return True
        for dep_list in self._find_all_deps_lists():
            for child in dep_list.child_nodes:
                # Typically searches run on non-absolute dep paths.
                dep_name = self._normalize(child.get(NODE_VALUE),
                                           abs_path=False)
                if name_query and re.search(name_query, dep_name):
                    print(f'{self._rel_path}: {dep_name} in '
                          f'{dep_list.target_name}.{dep_list.variable_name}')
                    return True
        return False

    def split_deps(self, original_dep_name: str,
                   new_dep_names: List[str]) -> bool:
        split = False
        for new_dep_name in new_dep_names:
            if self._split_dep(original_dep_name, new_dep_name):
                split = True
        return split

    def _split_dep(self, original_dep_name: str, new_dep_name: str) -> bool:
        """Add |new_dep_name| to GN deps that contains |original_dep_name|.

        Supports deps, public_deps, and other deps variables.

        Works for explicitly assigning a list to deps:
        deps = [ ..., "original_dep", ...]
        # Becomes
        deps = [ ..., "original_dep", "new_dep", ...]
        Also works for appending a list to deps:
        public_deps += [ ..., "original_dep", ...]
        # Becomes
        public_deps += [ ..., "original_dep", "new_dep", ...]

        Does not work for assigning or appending variables to deps:
        deps = other_list_of_deps # Does NOT check other_list_of_deps.
        # Becomes (no changes)
        deps = other_list_of_deps

        Does not work with parameter expansion, i.e. $variables.

        Returns whether the new dep was added one or more times.
        """
        for dep_name in (original_dep_name, new_dep_name):
            assert dep_name.startswith('//'), (
                f'Absolute GN path required, starting with //: {dep_name}')

        added_new_dep = False
        normalized_original_dep_name = self._normalize(original_dep_name)
        normalized_new_dep_name = self._normalize(new_dep_name)
        for dep_list in self._find_all_deps_lists():
            original_dep_idx = None
            new_dep_already_exists = False
            for idx, child in enumerate(dep_list.child_nodes):
                dep_name = self._normalize(child.get(NODE_VALUE))
                if dep_name == normalized_original_dep_name:
                    original_dep_idx = idx
                if dep_name == normalized_new_dep_name:
                    new_dep_already_exists = True
            if original_dep_idx is not None and not new_dep_already_exists:
                if dep_list.target_name is None:
                    target_str = self._gn_rel_path
                else:
                    target_str = f'{self._gn_rel_path}:{dep_list.target_name}'
                location = f"{target_str}'s {dep_list.variable_name} variable"
                logging.info(f'Adding {new_dep_name} to {location}')
                new_dep = self._clone_replacing_value(
                    dep_list.child_nodes[original_dep_idx], new_dep_name)
                # Add the new dep after the existing dep to preserve comments
                # before the existing dep.
                dep_list.child_nodes.insert(original_dep_idx + 1, new_dep)
                added_new_dep = True

        return added_new_dep

    def remove_deps(self,
                    dep_names: List[str],
                    out_dir: str,
                    targets: List[str],
                    target_name_filter: Optional[str],
                    inline_mode: bool = False) -> Tuple[bool, str]:
        if not inline_mode:
            deps_to_remove = dep_names
        else:
            # If the first dep cannot be removed (or is not found) then in the
            # case of inlining we can skip this file for the rest of the deps.
            first_dep = dep_names[0]
            if not self._remove_deps([first_dep], out_dir, targets,
                                     target_name_filter):
                return False
            deps_to_remove = dep_names[1:]
        return self._remove_deps(deps_to_remove, out_dir, targets,
                                 target_name_filter)

    def _remove_deps(self, dep_names: List[str], out_dir: str,
                     targets: List[str],
                     target_name_filter: Optional[str]) -> Tuple[bool, str]:
        """Remove |dep_names| if the target can still be built in |out_dir|.

        Supports deps, public_deps, and other deps variables.

        Works for explicitly assigning a list to deps:
        deps = [ ..., "original_dep", ...]
        # Becomes
        deps = [ ..., ...]

        Does not work with parameter expansion, i.e. $variables.

        Returns whether any deps were removed.
        """
        normalized_dep_names = set()
        for dep_name in dep_names:
            assert dep_name.startswith('//'), (
                f'Absolute GN path required, starting with //: {dep_name}')
            normalized_dep_names.add(self._normalize(dep_name))

        removed_dep = False
        for dep_list in self._find_all_deps_lists():
            child_deps_to_remove = [
                c for c in dep_list.child_nodes
                if self._normalize(c.get(NODE_VALUE)) in normalized_dep_names
            ]
            if not child_deps_to_remove:
                continue

            if dep_list.target_name is None:
                target_name_str = self._gn_rel_path
            else:
                target_name_str = f'{self._gn_rel_path}:{dep_list.target_name}'
            if (target_name_filter is not None and
                    re.search(target_name_filter, target_name_str) is None):
                logging.info(f'Skip: Since re.search("{target_name_filter}", '
                             f'"{target_name_str}") is None.')
                continue

            location = f"{target_name_str}'s {dep_list.variable_name} variable"
            expected_json = _generate_project_json_content(out_dir)
            num_to_remove = len(child_deps_to_remove)
            for remove_idx, child_dep in enumerate(child_deps_to_remove):
                child_dep_name = self._normalize(child_dep.get(NODE_VALUE))
                idx_to_remove = dep_list.child_nodes.index(child_dep)
                logging.info(f'({remove_idx + 1}/{num_to_remove}) Found '
                             f'{child_dep_name} in {location}.')
                child_to_remove = dep_list.child_nodes[idx_to_remove]
                can_remove_dep = False
                with _backup_and_restore_file_contents(self._full_path):
                    dep_list.child_nodes.remove(child_to_remove)
                    self.write_content_to_file()
                    # Immediately restore deps_list's original value in case the
                    # following build is interrupted. We don't want the
                    # intermediate untested value to be written as the final
                    # build file.
                    dep_list.child_nodes.insert(idx_to_remove, child_to_remove)
                    if expected_json is not None:
                        # If no changes to project.json was detected, this means
                        # the current target is not part of out_dir's build and
                        # cannot be removed even if the build succeeds.
                        after_json = _generate_project_json_content(out_dir)
                        if expected_json == after_json:
                            # If one change in this list isn't part of the
                            # build, no need to try any other in this list.
                            logging.info('Skip: No changes to project.json.')
                            break

                        # Avoids testing every dep removal for the same list.
                        expected_json = None
                    if self._can_still_build_everything(out_dir, targets):
                        can_remove_dep = True
                if not can_remove_dep:
                    continue

                dep_list.child_nodes.remove(child_to_remove)
                # Comments before a target can apply to the targets after.
                if (BEFORE_COMMENT in child_to_remove
                        and idx_to_remove < len(dep_list.child_nodes)):
                    child_after = dep_list.child_nodes[idx_to_remove]
                    if BEFORE_COMMENT not in child_after:
                        child_after[BEFORE_COMMENT] = []
                    child_after[BEFORE_COMMENT][:] = (
                        child_to_remove[BEFORE_COMMENT] +
                        child_after[BEFORE_COMMENT])
                # Comments after or behind a target don't make sense to re-
                # position, simply ignore AFTER_COMMENT and SUFFIX_COMMENT.
                removed_dep = True
                logging.info(f'Removed {child_dep_name} from {location}.')
        return removed_dep

    def _can_still_build_everything(self, out_dir: str,
                                    targets: List[str]) -> bool:
        output = _build_targets_output(out_dir, targets)
        if output is None:
            logging.info('Ninja failed to build all targets')
            return False
        # If ninja did not re-build anything, then the target changed is not
        # among the targets being built. Avoid this change as it's not been
        # tested/used.
        if 'ninja: no work to do.' in output:
            logging.info('Ninja did not find any targets to build')
            return False
        return True

    def write_content_to_file(self) -> None:
        current_content = json.dumps(self._content)
        if current_content != self._original_content:
            subprocess.run(
                ['gn', 'format', '--read-tree=json', self._full_path],
                text=True,
                check=True,
                input=current_content)