| # Lint as: python3 |
| # Copyright 2021 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| '''Helper script to use GN's JSON interface to make changes.''' |
| |
| from __future__ import annotations |
| |
| import contextlib |
| import copy |
| import dataclasses |
| import json |
| import logging |
| import os |
| import pathlib |
| import re |
| import shutil |
| import subprocess |
| import sys |
| |
| from typing import Dict, Iterator, List, Optional, Tuple |
| |
| _SRC_PATH = pathlib.Path(__file__).resolve().parents[2] |
| |
| _BUILD_ANDROID_GYP_PATH = _SRC_PATH / 'build/android/gyp' |
| if str(_BUILD_ANDROID_GYP_PATH) not in sys.path: |
| sys.path.append(str(_BUILD_ANDROID_GYP_PATH)) |
| |
| from util import build_utils |
| |
| # Refer to parse_tree.cc for GN AST implementation details: |
| # https://gn.googlesource.com/gn/+/refs/heads/main/src/gn/parse_tree.cc |
| # These constants should match corresponding entries in parse_tree.cc. |
| # TODO: Add high-level details for the expected data structure. |
| NODE_CHILD = 'child' |
| NODE_TYPE = 'type' |
| NODE_VALUE = 'value' |
| BEFORE_COMMENT = 'before_comment' |
| SUFFIX_COMMENT = 'suffix_comment' |
| AFTER_COMMENT = 'after_comment' |
| |
| |
| @contextlib.contextmanager |
| def _backup_and_restore_file_contents(path: str): |
| with open(path) as f: |
| contents = f.read() |
| try: |
| yield |
| finally: |
| # Ensure that the timestamp is updated since otherwise ninja will not |
| # re-build relevant targets with the original file. |
| with open(path, 'w') as f: |
| f.write(contents) |
| |
| |
| def _build_targets_output( |
| out_dir: str, |
| targets: List[str], |
| should_print: Optional[bool] = None) -> Optional[str]: |
| env = os.environ.copy() |
| if should_print is None: |
| should_print = logging.getLogger().isEnabledFor(logging.DEBUG) |
| # Ensuring ninja does not attempt to summarize the build results in slightly |
| # faster builds. This script does many builds so this time can add up. |
| if 'NINJA_SUMMARIZE_BUILD' in env: |
| del env['NINJA_SUMMARIZE_BUILD'] |
| proc = subprocess.Popen(['autoninja', '-C', out_dir] + targets, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, |
| env=env, |
| text=True) |
| lines = [] |
| prev_line = '' |
| width = shutil.get_terminal_size().columns |
| while proc.poll() is None: |
| line = proc.stdout.readline() |
| lines.append(line) |
| if should_print: |
| if prev_line.startswith('[') and line.startswith('['): |
| # Shrink the line according to terminal size. |
| msg = line.rstrip() |
| if len(msg) > width: |
| # 5 = 3 (Ellipsis) + 2 (header) |
| length_to_show = width - 5 |
| msg = f'{msg[:2]}...{msg[-length_to_show:]}' |
| # \r to return the carriage to the beginning of line, \033[K to |
| # replace the normal \n to erase until the end of the line. This |
| # allows ninja output for successful targets to overwrite each |
| # other. |
| msg = f'\r{msg}\033[K' |
| elif prev_line.startswith('['): |
| # Since the previous line likely did not include a newline, an |
| # extra newline is needed to avoid the current line being |
| # appended to the previous line. |
| msg = f'\n{line}' |
| else: |
| msg = line |
| print(msg, end='') |
| prev_line = line |
| if proc.returncode != 0: |
| return None |
| return ''.join(lines) |
| |
| |
| def _generate_project_json_content(out_dir: str) -> str: |
| build_utils.CheckOutput(['gn', 'gen', '--ide=json', out_dir]) |
| with open(os.path.join(out_dir, 'project.json')) as f: |
| return f.read() |
| |
| |
| @dataclasses.dataclass |
| class DepList: |
| """Represents a dep list assignment in GN.""" |
| target_name: Optional[str] # The name of the target containing the list. |
| variable_name: str # Left-hand side variable name the list is assigned to. |
| child_nodes: List[dict] # Right-hand side list of nodes. |
| operation: str # The assignment operation, whether += or =. |
| |
| |
| class BuildFile: |
| """Represents the contents of a BUILD.gn file.""" |
| def __init__(self, |
| build_gn_path: str, |
| root_gn_path: pathlib.Path, |
| *, |
| dryrun: bool = False): |
| self._root = root_gn_path |
| self._rel_path = os.path.relpath(build_gn_path, root_gn_path) |
| self._gn_rel_path = '//' + os.path.dirname(self._rel_path) |
| self._full_path = os.path.abspath(build_gn_path) |
| self._skip_write_content = dryrun |
| |
| def __enter__(self): |
| output = build_utils.CheckOutput( |
| ['gn', 'format', '--dump-tree=json', self._full_path]) |
| self._content = json.loads(output) |
| self._original_content = json.dumps(self._content) |
| return self |
| |
| def __exit__(self, exc, value, tb): |
| if not self._skip_write_content: |
| self.write_content_to_file() |
| |
| # See: https://gist.github.com/sgraham/bd9ffee312f307d5f417019a9c0f0777 |
| def _find_all(self, match_fn): |
| results = [] |
| |
| def get_target_name(node) -> Optional[str]: |
| """Example format (with irrelevant fields omitted): |
| { |
| "child": [ { |
| "child": [ { |
| "type": "LITERAL", |
| "value": "\"hello_world_java\"" |
| } ], |
| "type": "LIST" |
| }, { |
| ... |
| } ], |
| "type": "FUNCTION", |
| "value": "java_library" |
| } |
| |
| Example return: hello_world_java |
| """ |
| if node.get(NODE_TYPE) != 'FUNCTION': |
| return None |
| children = node.get(NODE_CHILD) |
| if not children: |
| return None |
| first_child = children[0] |
| if first_child.get(NODE_TYPE) != 'LIST': |
| return None |
| grand_children = first_child.get(NODE_CHILD) |
| if not grand_children: |
| return None |
| grand_child = grand_children[0] |
| if grand_child.get(NODE_TYPE) != 'LITERAL': |
| return None |
| name = grand_child.get(NODE_VALUE) |
| if name.startswith('"'): |
| return name[1:-1] |
| return name |
| |
| def recursive_find(root, last_known_target=None): |
| target_name = get_target_name(root) or last_known_target |
| matched = match_fn(root) |
| if matched is not None: |
| results.append((target_name, matched)) |
| return |
| children = root.get(NODE_CHILD) |
| if children: |
| for child in children: |
| recursive_find(child, last_known_target=target_name) |
| |
| recursive_find(self._content) |
| return results |
| |
| def _normalize(self, name: Optional[str], abs_path: bool = True): |
| """Returns the absolute GN path to the target with |name|. |
| |
| This method normalizes target names, assuming that relative targets are |
| referenced based on the current file, allowing targets to be compared |
| by name to determine whether they are the same or not. |
| |
| Given the current file is chrome/android/BUILD.gn: |
| |
| # Removes surrounding quotation marks. |
| "//chrome/android:chrome_java" -> //chrome/android:chrome_java |
| |
| # Makes relative paths absolute. |
| :chrome_java -> //chrome/android:chrome_java |
| |
| # Spells out GN shorthands for basenames. |
| //chrome/android -> //chrome/android:android |
| """ |
| if not name: |
| return '' |
| if name.startswith('"'): |
| name = name[1:-1] |
| if not name.startswith('//') and abs_path: |
| name = self._gn_rel_path + name |
| if not ':' in name: |
| name += ':' + os.path.basename(name) |
| return name |
| |
| def _find_all_list_assignments(self): |
| def match_list_assignments(node): |
| r"""Matches and returns the list being assigned. |
| |
| Binary node (with an operation such as = or +=) |
| / \ |
| / \ |
| name list of nodes |
| |
| Returns (name, list of nodes, op) |
| """ |
| if node.get(NODE_TYPE) != 'BINARY': |
| return None |
| operation = node.get(NODE_VALUE) |
| children = node.get(NODE_CHILD) |
| assert len(children) == 2, ( |
| 'Binary nodes should have two child nodes, but the node is: ' |
| f'{node}') |
| left_child, right_child = children |
| if left_child.get(NODE_TYPE) != 'IDENTIFIER': |
| return None |
| name = left_child.get(NODE_VALUE) |
| if right_child.get(NODE_TYPE) != 'LIST': |
| return None |
| list_of_nodes = right_child.get(NODE_CHILD) |
| return name, list_of_nodes, operation |
| |
| return self._find_all(match_list_assignments) |
| |
| def _find_all_deps_lists(self) -> Iterator[DepList]: |
| list_tuples = self._find_all_list_assignments() |
| for target_name, (var_name, node_list, operation) in list_tuples: |
| if (var_name == 'deps' or var_name.startswith('deps_') |
| or var_name.endswith('_deps') or '_deps_' in var_name): |
| yield DepList(target_name=target_name, |
| variable_name=var_name, |
| child_nodes=node_list, |
| operation=operation) |
| |
| def _new_literal_node(self, value: str, begin_line: int = 1): |
| return { |
| 'location': { |
| 'begin_column': 1, |
| 'begin_line': begin_line, |
| 'end_column': 2, |
| 'end_line': begin_line, |
| }, |
| 'type': 'LITERAL', |
| 'value': f'"{value}"' |
| } |
| |
| def _clone_replacing_value(self, node_to_copy: Dict, new_dep_name: str): |
| """Clone the existing node to preserve line numbers and update name. |
| |
| It is easier to clone an existing node around the same location, as the |
| actual dict looks like this: |
| { |
| 'location': { |
| 'begin_column': 5, |
| 'begin_line': 137, |
| 'end_column': 27, |
| 'end_line': 137 |
| }, |
| 'type': 'LITERAL', |
| 'value': '":anr_data_proto_java"' |
| } |
| |
| Thus the new node to return should keep the same 'location' value (the |
| parser is tolerant as long as it's roughly in the correct spot) but |
| update the 'value' to the new dependency name. |
| """ |
| new_dep = copy.deepcopy(node_to_copy) |
| # Any comments associated with the previous dep would not apply. |
| for comment_key in (BEFORE_COMMENT, AFTER_COMMENT, SUFFIX_COMMENT): |
| new_dep.pop(comment_key, None) # Remove if exists. |
| new_dep[NODE_VALUE] = f'"{new_dep_name}"' |
| return new_dep |
| |
| def add_deps(self, target: str, deps: List[str]) -> bool: |
| added_new_dep = False |
| normalized_target = self._normalize(target) |
| for dep_list in self._find_all_deps_lists(): |
| if dep_list.target_name is None: |
| continue |
| # Only modify the first assignment operation to the deps variable, |
| # otherwise if there are += operations, then the list of deps will |
| # be added multiple times to the same target's deps. |
| if dep_list.operation != '=': |
| continue |
| full_target_name = f'{self._gn_rel_path}:{dep_list.target_name}' |
| # Support both the exact name and the absolute GN target names |
| # starting with //. |
| if (target != dep_list.target_name |
| and normalized_target != full_target_name): |
| continue |
| if dep_list.variable_name != 'deps': |
| continue |
| existing_dep_names = set( |
| self._normalize(child.get(NODE_VALUE), abs_path=False) |
| for child in dep_list.child_nodes) |
| for new_dep_name in deps: |
| if new_dep_name in existing_dep_names: |
| logging.info( |
| f'Skipping existing {new_dep_name} in {target}.deps') |
| continue |
| logging.info(f'Adding {new_dep_name} to {target}.deps') |
| # If there are no existing child nodes, then create a new one. |
| # Otherwise clone an existing child node to ensure more accurate |
| # line numbers and possible better preserve comments. |
| if not dep_list.child_nodes: |
| new_dep = self._new_literal_node(new_dep_name) |
| else: |
| new_dep = self._clone_replacing_value( |
| dep_list.child_nodes[0], new_dep_name) |
| dep_list.child_nodes.append(new_dep) |
| added_new_dep = True |
| if not added_new_dep: |
| # This should match the string in bytecode_processor.py. |
| print(f'Unable to find {target}') |
| return added_new_dep |
| |
| def search_deps(self, name_query: Optional[str], |
| path_query: Optional[str]) -> bool: |
| if path_query: |
| if not re.search(path_query, self._rel_path): |
| return False |
| elif not name_query: |
| print(self._rel_path) |
| return True |
| for dep_list in self._find_all_deps_lists(): |
| for child in dep_list.child_nodes: |
| # Typically searches run on non-absolute dep paths. |
| dep_name = self._normalize(child.get(NODE_VALUE), |
| abs_path=False) |
| if name_query and re.search(name_query, dep_name): |
| print(f'{self._rel_path}: {dep_name} in ' |
| f'{dep_list.target_name}.{dep_list.variable_name}') |
| return True |
| return False |
| |
| def split_deps(self, original_dep_name: str, |
| new_dep_names: List[str]) -> bool: |
| split = False |
| for new_dep_name in new_dep_names: |
| if self._split_dep(original_dep_name, new_dep_name): |
| split = True |
| return split |
| |
| def _split_dep(self, original_dep_name: str, new_dep_name: str) -> bool: |
| """Add |new_dep_name| to GN deps that contains |original_dep_name|. |
| |
| Supports deps, public_deps, and other deps variables. |
| |
| Works for explicitly assigning a list to deps: |
| deps = [ ..., "original_dep", ...] |
| # Becomes |
| deps = [ ..., "original_dep", "new_dep", ...] |
| Also works for appending a list to deps: |
| public_deps += [ ..., "original_dep", ...] |
| # Becomes |
| public_deps += [ ..., "original_dep", "new_dep", ...] |
| |
| Does not work for assigning or appending variables to deps: |
| deps = other_list_of_deps # Does NOT check other_list_of_deps. |
| # Becomes (no changes) |
| deps = other_list_of_deps |
| |
| Does not work with parameter expansion, i.e. $variables. |
| |
| Returns whether the new dep was added one or more times. |
| """ |
| for dep_name in (original_dep_name, new_dep_name): |
| assert dep_name.startswith('//'), ( |
| f'Absolute GN path required, starting with //: {dep_name}') |
| |
| added_new_dep = False |
| normalized_original_dep_name = self._normalize(original_dep_name) |
| normalized_new_dep_name = self._normalize(new_dep_name) |
| for dep_list in self._find_all_deps_lists(): |
| original_dep_idx = None |
| new_dep_already_exists = False |
| for idx, child in enumerate(dep_list.child_nodes): |
| dep_name = self._normalize(child.get(NODE_VALUE)) |
| if dep_name == normalized_original_dep_name: |
| original_dep_idx = idx |
| if dep_name == normalized_new_dep_name: |
| new_dep_already_exists = True |
| if original_dep_idx is not None and not new_dep_already_exists: |
| if dep_list.target_name is None: |
| target_str = self._gn_rel_path |
| else: |
| target_str = f'{self._gn_rel_path}:{dep_list.target_name}' |
| location = f"{target_str}'s {dep_list.variable_name} variable" |
| logging.info(f'Adding {new_dep_name} to {location}') |
| new_dep = self._clone_replacing_value( |
| dep_list.child_nodes[original_dep_idx], new_dep_name) |
| # Add the new dep after the existing dep to preserve comments |
| # before the existing dep. |
| dep_list.child_nodes.insert(original_dep_idx + 1, new_dep) |
| added_new_dep = True |
| |
| return added_new_dep |
| |
| def remove_deps(self, |
| dep_names: List[str], |
| out_dir: str, |
| targets: List[str], |
| target_name_filter: Optional[str], |
| inline_mode: bool = False) -> Tuple[bool, str]: |
| if not inline_mode: |
| deps_to_remove = dep_names |
| else: |
| # If the first dep cannot be removed (or is not found) then in the |
| # case of inlining we can skip this file for the rest of the deps. |
| first_dep = dep_names[0] |
| if not self._remove_deps([first_dep], out_dir, targets, |
| target_name_filter): |
| return False |
| deps_to_remove = dep_names[1:] |
| return self._remove_deps(deps_to_remove, out_dir, targets, |
| target_name_filter) |
| |
| def _remove_deps(self, dep_names: List[str], out_dir: str, |
| targets: List[str], |
| target_name_filter: Optional[str]) -> Tuple[bool, str]: |
| """Remove |dep_names| if the target can still be built in |out_dir|. |
| |
| Supports deps, public_deps, and other deps variables. |
| |
| Works for explicitly assigning a list to deps: |
| deps = [ ..., "original_dep", ...] |
| # Becomes |
| deps = [ ..., ...] |
| |
| Does not work with parameter expansion, i.e. $variables. |
| |
| Returns whether any deps were removed. |
| """ |
| normalized_dep_names = set() |
| for dep_name in dep_names: |
| assert dep_name.startswith('//'), ( |
| f'Absolute GN path required, starting with //: {dep_name}') |
| normalized_dep_names.add(self._normalize(dep_name)) |
| |
| removed_dep = False |
| for dep_list in self._find_all_deps_lists(): |
| child_deps_to_remove = [ |
| c for c in dep_list.child_nodes |
| if self._normalize(c.get(NODE_VALUE)) in normalized_dep_names |
| ] |
| if not child_deps_to_remove: |
| continue |
| |
| if dep_list.target_name is None: |
| target_name_str = self._gn_rel_path |
| else: |
| target_name_str = f'{self._gn_rel_path}:{dep_list.target_name}' |
| if (target_name_filter is not None and |
| re.search(target_name_filter, target_name_str) is None): |
| logging.info(f'Skip: Since re.search("{target_name_filter}", ' |
| f'"{target_name_str}") is None.') |
| continue |
| |
| location = f"{target_name_str}'s {dep_list.variable_name} variable" |
| expected_json = _generate_project_json_content(out_dir) |
| num_to_remove = len(child_deps_to_remove) |
| for remove_idx, child_dep in enumerate(child_deps_to_remove): |
| child_dep_name = self._normalize(child_dep.get(NODE_VALUE)) |
| idx_to_remove = dep_list.child_nodes.index(child_dep) |
| logging.info(f'({remove_idx + 1}/{num_to_remove}) Found ' |
| f'{child_dep_name} in {location}.') |
| child_to_remove = dep_list.child_nodes[idx_to_remove] |
| can_remove_dep = False |
| with _backup_and_restore_file_contents(self._full_path): |
| dep_list.child_nodes.remove(child_to_remove) |
| self.write_content_to_file() |
| # Immediately restore deps_list's original value in case the |
| # following build is interrupted. We don't want the |
| # intermediate untested value to be written as the final |
| # build file. |
| dep_list.child_nodes.insert(idx_to_remove, child_to_remove) |
| if expected_json is not None: |
| # If no changes to project.json was detected, this means |
| # the current target is not part of out_dir's build and |
| # cannot be removed even if the build succeeds. |
| after_json = _generate_project_json_content(out_dir) |
| if expected_json == after_json: |
| # If one change in this list isn't part of the |
| # build, no need to try any other in this list. |
| logging.info('Skip: No changes to project.json.') |
| break |
| |
| # Avoids testing every dep removal for the same list. |
| expected_json = None |
| if self._can_still_build_everything(out_dir, targets): |
| can_remove_dep = True |
| if not can_remove_dep: |
| continue |
| |
| dep_list.child_nodes.remove(child_to_remove) |
| # Comments before a target can apply to the targets after. |
| if (BEFORE_COMMENT in child_to_remove |
| and idx_to_remove < len(dep_list.child_nodes)): |
| child_after = dep_list.child_nodes[idx_to_remove] |
| if BEFORE_COMMENT not in child_after: |
| child_after[BEFORE_COMMENT] = [] |
| child_after[BEFORE_COMMENT][:] = ( |
| child_to_remove[BEFORE_COMMENT] + |
| child_after[BEFORE_COMMENT]) |
| # Comments after or behind a target don't make sense to re- |
| # position, simply ignore AFTER_COMMENT and SUFFIX_COMMENT. |
| removed_dep = True |
| logging.info(f'Removed {child_dep_name} from {location}.') |
| return removed_dep |
| |
| def _can_still_build_everything(self, out_dir: str, |
| targets: List[str]) -> bool: |
| output = _build_targets_output(out_dir, targets) |
| if output is None: |
| logging.info('Ninja failed to build all targets') |
| return False |
| # If ninja did not re-build anything, then the target changed is not |
| # among the targets being built. Avoid this change as it's not been |
| # tested/used. |
| if 'ninja: no work to do.' in output: |
| logging.info('Ninja did not find any targets to build') |
| return False |
| return True |
| |
| def write_content_to_file(self) -> None: |
| current_content = json.dumps(self._content) |
| if current_content != self._original_content: |
| subprocess.run( |
| ['gn', 'format', '--read-tree=json', self._full_path], |
| text=True, |
| check=True, |
| input=current_content) |