build / android / unused_resources / UnusedResources.java [blame]

/*
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Modifications are owned by the Chromium Authors.
// Copyright 2021 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package build.android.unused_resources;

import static com.android.ide.common.symbols.SymbolIo.readFromAapt;
import static com.android.utils.SdkUtils.endsWithIgnoreCase;

import static com.google.common.base.Charsets.UTF_8;

import com.android.ide.common.resources.usage.ResourceUsageModel;
import com.android.ide.common.resources.usage.ResourceUsageModel.Resource;
import com.android.ide.common.symbols.Symbol;
import com.android.ide.common.symbols.SymbolTable;
import com.android.resources.ResourceFolderType;
import com.android.resources.ResourceType;
import com.android.tools.r8.CompilationFailedException;
import com.android.tools.r8.ProgramResource;
import com.android.tools.r8.ProgramResourceProvider;
import com.android.tools.r8.ResourceShrinker;
import com.android.tools.r8.ResourceShrinker.Command;
import com.android.tools.r8.ResourceShrinker.ReferenceChecker;
import com.android.tools.r8.origin.PathOrigin;
import com.android.utils.XmlUtils;

import com.google.common.base.Charsets;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import com.google.common.io.Files;

import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import javax.xml.parsers.ParserConfigurationException;

/**
  Copied with modifications from gradle core source
  https://cs.android.com/search?q=f:build-system.*ResourceUsageAnalyzer.java

  Modifications are mostly to:
    - Remove unused code paths to reduce complexity.
    - Reduce dependencies unless absolutely required.
*/

public class UnusedResources {
    private static final String ANDROID_RES = "android_res/";
    private static final String DOT_DEX = ".dex";
    private static final String DOT_CLASS = ".class";
    private static final String DOT_XML = ".xml";
    private static final String DOT_JAR = ".jar";
    private static final String FN_RESOURCE_TEXT = "R.txt";

    /* A source of resource classes to track, can be either a folder or a jar */
    private final Iterable<File> mRTxtFiles;
    private final File mProguardMapping;
    /** These can be class or dex files. */
    private final Iterable<File> mClasses;
    private final Iterable<File> mManifests;
    private final Iterable<File> mResourceDirs;

    private final File mReportFile;
    private final StringWriter mDebugOutput;
    private final PrintWriter mDebugPrinter;

    /** The computed set of unused resources */
    private List<Resource> mUnused;

    /**
     * Map from resource class owners (VM format class) to corresponding resource entries.
     * This lets us map back from code references (obfuscated class and possibly obfuscated field
     * reference) back to the corresponding resource type and name.
     */
    private Map<String, Pair<ResourceType, Map<String, String>>> mResourceObfuscation =
            Maps.newHashMapWithExpectedSize(30);

    /** Obfuscated name of android/support/v7/widget/SuggestionsAdapter.java */
    private String mSuggestionsAdapter;

    /** Obfuscated name of android/support/v7/internal/widget/ResourcesWrapper.java */
    private String mResourcesWrapper;

    /* A Pair class because java does not come with batteries included. */
    private static class Pair<U, V> {
        private U mFirst;
        private V mSecond;

        Pair(U first, V second) {
            this.mFirst = first;
            this.mSecond = second;
        }

        public U getFirst() {
            return mFirst;
        }

        public V getSecond() {
            return mSecond;
        }
    }

    public UnusedResources(Iterable<File> rTxtFiles, Iterable<File> classes,
            Iterable<File> manifests, File mapping, Iterable<File> resources, File reportFile) {
        mRTxtFiles = rTxtFiles;
        mProguardMapping = mapping;
        mClasses = classes;
        mManifests = manifests;
        mResourceDirs = resources;

        mReportFile = reportFile;
        if (reportFile != null) {
            mDebugOutput = new StringWriter(8 * 1024);
            mDebugPrinter = new PrintWriter(mDebugOutput);
        } else {
            mDebugOutput = null;
            mDebugPrinter = null;
        }
    }

    public void close() {
        if (mDebugOutput != null) {
            String output = mDebugOutput.toString();

            if (mReportFile != null) {
                File dir = mReportFile.getParentFile();
                if (dir != null) {
                    if ((dir.exists() || dir.mkdir()) && dir.canWrite()) {
                        try {
                            Files.asCharSink(mReportFile, Charsets.UTF_8).write(output);
                        } catch (IOException ignore) {
                        }
                    }
                }
            }
        }
    }

    public void analyze() throws IOException, ParserConfigurationException, SAXException {
        gatherResourceValues(mRTxtFiles);
        recordMapping(mProguardMapping);

        for (File jarOrDir : mClasses) {
            recordClassUsages(jarOrDir);
        }
        recordManifestUsages(mManifests);
        recordResources(mResourceDirs);
        dumpReferences();
        mModel.processToolsAttributes();
        mUnused = mModel.findUnused();
    }

    public void emitConfig(Path destination) throws IOException {
        File destinationFile = destination.toFile();
        if (!destinationFile.exists()) {
            destinationFile.getParentFile().mkdirs();
            boolean success = destinationFile.createNewFile();
            if (!success) {
                throw new IOException("Could not create " + destination);
            }
        }
        StringBuilder sb = new StringBuilder();
        Collections.sort(mUnused);
        for (Resource resource : mUnused) {
            if (resource.type.isSynthetic()) {
                // Ignore synthetic resources like overlayable or macro that are
                // not actually listed in the ResourceTable.
                continue;
            }
            sb.append(resource.type + "/" + resource.name + "#remove\n");
        }
        Files.asCharSink(destinationFile, UTF_8).write(sb.toString());
    }

    private void dumpReferences() {
        if (mDebugPrinter != null) {
            mDebugPrinter.print(mModel.dumpReferences());
        }
    }

    private void dumpModel() {
        if (mDebugPrinter != null) {
            mDebugPrinter.print(mModel.dumpResourceModel());
        }
    }

    private void recordResources(Iterable<File> resources)
            throws IOException, SAXException, ParserConfigurationException {
        for (File resDir : resources) {
            File[] resourceFolders = resDir.listFiles();
            assert resourceFolders != null : "Invalid resource directory " + resDir;
            for (File folder : resourceFolders) {
                ResourceFolderType folderType = ResourceFolderType.getFolderType(folder.getName());
                if (folderType != null) {
                    recordResources(folderType, folder);
                }
            }
        }
    }

    private void recordResources(ResourceFolderType folderType, File folder)
            throws ParserConfigurationException, SAXException, IOException {
        File[] files = folder.listFiles();
        if (files != null) {
            for (File file : files) {
                String path = file.getPath();
                mModel.file = file;
                try {
                    boolean isXml = endsWithIgnoreCase(path, DOT_XML);
                    if (isXml) {
                        String xml = Files.toString(file, UTF_8);
                        Document document = XmlUtils.parseDocument(xml, true);
                        mModel.visitXmlDocument(file, folderType, document);
                    } else {
                        mModel.visitBinaryResource(folderType, file);
                    }
                } finally {
                    mModel.file = null;
                }
            }
        }
    }

    void recordMapping(File mapping) throws IOException {
        if (mapping == null || !mapping.exists()) {
            return;
        }
        final String arrowString = " -> ";
        final String resourceString = ".R$";
        Map<String, String> nameMap = null;
        for (String line : Files.readLines(mapping, UTF_8)) {
            // Ignore R8's mapping comments.
            if (line.startsWith("#")) {
                continue;
            }
            if (line.startsWith(" ") || line.startsWith("\t")) {
                if (nameMap != null) {
                    // We're processing the members of a resource class: record names into the map
                    int n = line.length();
                    int i = 0;
                    for (; i < n; i++) {
                        if (!Character.isWhitespace(line.charAt(i))) {
                            break;
                        }
                    }
                    if (i < n && line.startsWith("int", i)) { // int or int[]
                        int start = line.indexOf(' ', i + 3) + 1;
                        int arrow = line.indexOf(arrowString);
                        if (start > 0 && arrow != -1) {
                            int end = line.indexOf(' ', start + 1);
                            if (end != -1) {
                                String oldName = line.substring(start, end);
                                String newName =
                                        line.substring(arrow + arrowString.length()).trim();
                                if (!newName.equals(oldName)) {
                                    nameMap.put(newName, oldName);
                                }
                            }
                        }
                    }
                }
                continue;
            } else {
                nameMap = null;
            }
            int index = line.indexOf(resourceString);
            if (index == -1) {
                // Record obfuscated names of a few known appcompat usages of
                // Resources#getIdentifier that are unlikely to be used for general
                // resource name reflection
                if (line.startsWith("android.support.v7.widget.SuggestionsAdapter ")) {
                    mSuggestionsAdapter =
                            line.substring(line.indexOf(arrowString) + arrowString.length(),
                                        line.indexOf(':') != -1 ? line.indexOf(':') : line.length())
                                    .trim()
                                    .replace('.', '/')
                            + DOT_CLASS;
                } else if (line.startsWith("android.support.v7.internal.widget.ResourcesWrapper ")
                        || line.startsWith("android.support.v7.widget.ResourcesWrapper ")
                        || (mResourcesWrapper == null // Recently wrapper moved
                                && line.startsWith(
                                        "android.support.v7.widget.TintContextWrapper$TintResources"
                                                + " "))) {
                    mResourcesWrapper =
                            line.substring(line.indexOf(arrowString) + arrowString.length(),
                                        line.indexOf(':') != -1 ? line.indexOf(':') : line.length())
                                    .trim()
                                    .replace('.', '/')
                            + DOT_CLASS;
                }
                continue;
            }
            int arrow = line.indexOf(arrowString, index + 3);
            if (arrow == -1) {
                continue;
            }
            String typeName = line.substring(index + resourceString.length(), arrow);
            ResourceType type = ResourceType.fromClassName(typeName);
            if (type == null) {
                continue;
            }
            int end = line.indexOf(':', arrow + arrowString.length());
            if (end == -1) {
                end = line.length();
            }
            String target = line.substring(arrow + arrowString.length(), end).trim();
            String ownerName = target.replace('.', '/');

            nameMap = Maps.newHashMap();
            Pair<ResourceType, Map<String, String>> pair = new Pair(type, nameMap);
            mResourceObfuscation.put(ownerName, pair);
            // For fast lookup in isResourceClass
            mResourceObfuscation.put(ownerName + DOT_CLASS, pair);
        }
    }

    private void recordManifestUsages(File manifest)
            throws IOException, ParserConfigurationException, SAXException {
        String xml = Files.toString(manifest, UTF_8);
        Document document = XmlUtils.parseDocument(xml, true);
        mModel.visitXmlDocument(manifest, null, document);
    }

    private void recordManifestUsages(Iterable<File> manifests)
            throws IOException, ParserConfigurationException, SAXException {
        for (File manifest : manifests) {
            recordManifestUsages(manifest);
        }
    }

    private void recordClassUsages(File file) throws IOException {
        assert file.isFile();
        if (file.getPath().endsWith(DOT_DEX)) {
            byte[] bytes = Files.toByteArray(file);
            recordClassUsages(file, file.getName(), bytes);
        } else if (file.getPath().endsWith(DOT_JAR)) {
            ZipInputStream zis = null;
            try {
                FileInputStream fis = new FileInputStream(file);
                try {
                    zis = new ZipInputStream(fis);
                    ZipEntry entry = zis.getNextEntry();
                    while (entry != null) {
                        String name = entry.getName();
                        if (name.endsWith(DOT_DEX)) {
                            byte[] bytes = ByteStreams.toByteArray(zis);
                            if (bytes != null) {
                                recordClassUsages(file, name, bytes);
                            }
                        }

                        entry = zis.getNextEntry();
                    }
                } finally {
                    Closeables.close(fis, true);
                }
            } finally {
                Closeables.close(zis, true);
            }
        }
    }

    private String stringifyResource(Resource resource) {
        return String.format("%s:%s:0x%08x", resource.type, resource.name, resource.value);
    }

    private void recordClassUsages(File file, String name, byte[] bytes) {
        assert name.endsWith(DOT_DEX);
        ReferenceChecker callback = new ReferenceChecker() {
            @Override
            public boolean shouldProcess(String internalName) {
                // We do not need to ignore R subclasses since R8 now removes
                // unused resource id fields in R subclasses thus their
                // remaining presence means real usage.
                return true;
            }

            @Override
            public void referencedInt(int value) {
                UnusedResources.this.referencedInt("dex", value, file, name);
            }

            @Override
            public void referencedString(String value) {
                // do nothing.
            }

            @Override
            public void referencedStaticField(String internalName, String fieldName) {
                Resource resource = getResourceFromCode(internalName, fieldName);
                if (resource != null) {
                    ResourceUsageModel.markReachable(resource);
                    if (mDebugPrinter != null) {
                        mDebugPrinter.println("Marking " + stringifyResource(resource)
                                + " reachable: referenced from dex"
                                + " in " + file + ":" + name + " (static field access "
                                + internalName + "." + fieldName + ")");
                    }
                }
            }

            @Override
            public void referencedMethod(
                    String internalName, String methodName, String methodDescriptor) {
                // Do nothing.
            }
        };
        ProgramResource resource = ProgramResource.fromBytes(
                new PathOrigin(file.toPath()), ProgramResource.Kind.DEX, bytes, null);
        ProgramResourceProvider provider = () -> Arrays.asList(resource);
        try {
            Command command =
                    (new ResourceShrinker.Builder()).addProgramResourceProvider(provider).build();
            ResourceShrinker.run(command, callback);
        } catch (CompilationFailedException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }
    }

    /** Returns whether the given class file name points to an aapt-generated compiled R class. */
    boolean isResourceClass(String name) {
        if (mResourceObfuscation.containsKey(name)) {
            return true;
        }
        int index = name.lastIndexOf('/');
        if (index != -1 && name.startsWith("R$", index + 1) && name.endsWith(DOT_CLASS)) {
            String typeName = name.substring(index + 3, name.length() - DOT_CLASS.length());
            return ResourceType.fromClassName(typeName) != null;
        }
        return false;
    }

    Resource getResourceFromCode(String owner, String name) {
        Pair<ResourceType, Map<String, String>> pair = mResourceObfuscation.get(owner);
        if (pair != null) {
            ResourceType type = pair.getFirst();
            Map<String, String> nameMap = pair.getSecond();
            String renamedField = nameMap.get(name);
            if (renamedField != null) {
                name = renamedField;
            }
            return mModel.getResource(type, name);
        }
        if (isValidResourceType(owner)) {
            ResourceType type =
                    ResourceType.fromClassName(owner.substring(owner.lastIndexOf('$') + 1));
            if (type != null) {
                return mModel.getResource(type, name);
            }
        }
        return null;
    }

    private Boolean isValidResourceType(String candidateString) {
        return candidateString.contains("/")
                && candidateString.substring(candidateString.lastIndexOf('/') + 1).contains("$");
    }

    private void gatherResourceValues(Iterable<File> rTxts) throws IOException {
        for (File rTxt : rTxts) {
            assert rTxt.isFile();
            assert rTxt.getName().endsWith(FN_RESOURCE_TEXT);
            addResourcesFromRTxtFile(rTxt);
        }
    }

    private void addResourcesFromRTxtFile(File file) {
        try {
            SymbolTable st = readFromAapt(file, null);
            for (Symbol symbol : st.getSymbols().values()) {
                String symbolValue = symbol.getValue();
                if (symbol.getResourceType() == ResourceType.STYLEABLE) {
                    if (symbolValue.trim().startsWith("{")) {
                        // Only add the styleable parent, styleable children are not yet supported.
                        mModel.addResource(symbol.getResourceType(), symbol.getName(), null);
                    }
                } else {
                    if (mDebugPrinter != null) {
                        mDebugPrinter.println("Extracted R.txt resource: "
                                + symbol.getResourceType() + ":" + symbol.getName() + ":"
                                + String.format(
                                        "0x%08x", Integer.parseInt(symbolValue.substring(2), 16)));
                    }
                    mModel.addResource(symbol.getResourceType(), symbol.getName(), symbolValue);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    ResourceUsageModel getModel() {
        return mModel;
    }

    private void referencedInt(String context, int value, File file, String currentClass) {
        Resource resource = mModel.getResource(value);
        if (ResourceUsageModel.markReachable(resource) && mDebugPrinter != null) {
            mDebugPrinter.println("Marking " + stringifyResource(resource)
                    + " reachable: referenced from " + context + " in " + file + ":"
                    + currentClass);
        }
    }

    private final ResourceShrinkerUsageModel mModel = new ResourceShrinkerUsageModel();

    private class ResourceShrinkerUsageModel extends ResourceUsageModel {
        public File file;

        /**
         * Whether we should ignore tools attribute resource references.
         * <p>
         * For example, for resource shrinking we want to ignore tools attributes,
         * whereas for resource refactoring on the source code we do not.
         *
         * @return whether tools attributes should be ignored
         */
        @Override
        protected boolean ignoreToolsAttributes() {
            return true;
        }

        @Override
        protected void onRootResourcesFound(List<Resource> roots) {
            if (mDebugPrinter != null) {
                mDebugPrinter.println("\nThe root reachable resources are:");
                for (Resource root : roots) {
                    mDebugPrinter.println("   " + stringifyResource(root) + ",");
                }
            }
        }

        @Override
        protected Resource declareResource(ResourceType type, String name, Node node) {
            Resource resource = super.declareResource(type, name, node);
            resource.addLocation(file);
            return resource;
        }

        @Override
        protected void referencedString(String string) {
            // Do nothing
        }
    }

    private static List<File> parsePathsFromFile(String path) throws IOException {
        return java.nio.file.Files.readAllLines(new File(path).toPath()).stream()
                .map(File::new)
                .collect(Collectors.toList());
    }

    public static void main(String[] args) throws Exception {
        List<File> rTxtFiles = null; // R.txt files
        List<File> classes = null; // Dex/jar w dex
        List<File> manifests = null; // manifests
        File mapping = null; // mapping
        List<File> resources = null; // resources dirs
        File log = null; // output log for debugging
        Path configPath = null; // output config
        for (int i = 0; i < args.length; i += 2) {
            switch (args[i]) {
                case "--rtxts":
                    rTxtFiles = Arrays.stream(args[i + 1].split(":"))
                                        .map(s -> new File(s))
                                        .collect(Collectors.toList());
                    break;
                case "--dexes":
                    classes = parsePathsFromFile(args[i + 1]);
                    break;
                case "--manifests":
                    manifests = parsePathsFromFile(args[i + 1]);
                    break;
                case "--mapping":
                    mapping = new File(args[i + 1]);
                    break;
                case "--resourceDirs":
                    resources = parsePathsFromFile(args[i + 1]);
                    break;
                case "--log":
                    log = new File(args[i + 1]);
                    break;
                case "--outputConfig":
                    configPath = Paths.get(args[i + 1]);
                    break;
                default:
                    throw new IllegalArgumentException(args[i] + " is not a valid arg.");
            }
        }
        UnusedResources unusedResources =
                new UnusedResources(rTxtFiles, classes, manifests, mapping, resources, log);
        unusedResources.analyze();
        unusedResources.close();
        unusedResources.emitConfig(configPath);
    }
}