001/*
002 * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.hotspot;
024
025import static jdk.internal.jvmci.compiler.Compiler.*;
026
027import java.io.*;
028import java.lang.annotation.*;
029import java.lang.reflect.*;
030import java.net.*;
031import java.util.*;
032import java.util.concurrent.*;
033import java.util.concurrent.atomic.*;
034import java.util.jar.*;
035import java.util.stream.*;
036
037import jdk.internal.jvmci.compiler.Compiler;
038import jdk.internal.jvmci.hotspot.*;
039import jdk.internal.jvmci.meta.*;
040import jdk.internal.jvmci.options.*;
041import jdk.internal.jvmci.options.OptionUtils.OptionConsumer;
042import jdk.internal.jvmci.options.OptionValue.OverrideScope;
043import jdk.internal.jvmci.runtime.*;
044
045import com.oracle.graal.compiler.*;
046import com.oracle.graal.compiler.CompilerThreadFactory.DebugConfigAccess;
047import com.oracle.graal.debug.*;
048import com.oracle.graal.debug.internal.*;
049
050import static com.oracle.graal.hotspot.CompileTheWorld.Options.*;
051
052/**
053 * This class implements compile-the-world functionality with JVMCI.
054 */
055public final class CompileTheWorld {
056
057    /**
058     * Magic token to trigger reading files from the boot class path.
059     */
060    public static final String SUN_BOOT_CLASS_PATH = "sun.boot.class.path";
061
062    public static class Options {
063        // @formatter:off
064        @Option(help = "Compile all methods in all classes on given class path", type = OptionType.Debug)
065        public static final OptionValue<String> CompileTheWorldClasspath = new OptionValue<>(SUN_BOOT_CLASS_PATH);
066        @Option(help = "Verbose CompileTheWorld operation", type = OptionType.Debug)
067        public static final OptionValue<Boolean> CompileTheWorldVerbose = new OptionValue<>(true);
068        @Option(help = "The number of CompileTheWorld iterations to perform", type = OptionType.Debug)
069        public static final OptionValue<Integer> CompileTheWorldIterations = new OptionValue<>(1);
070        @Option(help = "Only compile methods matching this filter", type = OptionType.Debug)
071        public static final OptionValue<String> CompileTheWorldMethodFilter = new OptionValue<>(null);
072        @Option(help = "Exclude methods matching this filter from compilation", type = OptionType.Debug)
073        public static final OptionValue<String> CompileTheWorldExcludeMethodFilter = new OptionValue<>(null);
074        @Option(help = "First class to consider when using -XX:+CompileTheWorld", type = OptionType.Debug)
075        public static final OptionValue<Integer> CompileTheWorldStartAt = new OptionValue<>(1);
076        @Option(help = "Last class to consider when using -XX:+CompileTheWorld", type = OptionType.Debug)
077        public static final OptionValue<Integer> CompileTheWorldStopAt = new OptionValue<>(Integer.MAX_VALUE);
078        @Option(help = "Option value overrides to use during compile the world. For example, " +
079                       "to disable inlining and partial escape analysis specify '-PartialEscapeAnalysis -Inline'. " +
080                       "The format for each option is the same as on the command line just without the '-G:' prefix.", type = OptionType.Debug)
081        public static final OptionValue<String> CompileTheWorldConfig = new OptionValue<>(null);
082
083        @Option(help = "Run CTW using as many threads as there are processors on the system", type = OptionType.Debug)
084        public static final OptionValue<Boolean> CompileTheWorldMultiThreaded = new OptionValue<>(false);
085        @Option(help = "Number of threads to use for multithreaded CTW.  Defaults to Runtime.getRuntime().availableProcessors()", type = OptionType.Debug)
086        public static final OptionValue<Integer> CompileTheWorldThreads = new OptionValue<>(0);
087        // @formatter:on
088
089        /**
090         * Overrides {@link #CompileTheWorldStartAt} and {@link #CompileTheWorldStopAt} from
091         * {@code -XX} HotSpot options of the same name if the latter have non-default values.
092         */
093        public static void overrideWithNativeOptions(HotSpotVMConfig c) {
094            if (c.compileTheWorldStartAt != 1) {
095                CompileTheWorldStartAt.setValue(c.compileTheWorldStartAt);
096            }
097            if (c.compileTheWorldStopAt != Integer.MAX_VALUE) {
098                CompileTheWorldStopAt.setValue(c.compileTheWorldStopAt);
099            }
100        }
101    }
102
103    /**
104     * A mechanism for overriding JVMCI options that affect compilation. A {@link Config} object
105     * should be used in a try-with-resources statement to ensure overriding of options is scoped
106     * properly. For example:
107     *
108     * <pre>
109     *     Config config = ...;
110     *     try (AutoCloseable s = config == null ? null : config.apply()) {
111     *         // perform a JVMCI compilation
112     *     }
113     * </pre>
114     */
115    @SuppressWarnings("serial")
116    public static class Config extends HashMap<OptionValue<?>, Object> implements OptionConsumer {
117        /**
118         * Creates a {@link Config} object by parsing a set of space separated override options.
119         *
120         * @param options a space separated set of option value settings with each option setting in
121         *            a format compatible with
122         *            {@link OptionUtils#parseOption(String, OptionConsumer)}. Ignored if null.
123         */
124        public Config(String options) {
125            if (options != null) {
126                for (String option : options.split("\\s+")) {
127                    OptionUtils.parseOption(option, this);
128                }
129            }
130        }
131
132        /**
133         * Applies the overrides represented by this object. The overrides are in effect until
134         * {@link OverrideScope#close()} is called on the returned object.
135         */
136        OverrideScope apply() {
137            return OptionValue.override(this);
138        }
139
140        public void set(OptionDescriptor desc, Object value) {
141            put(desc.getOptionValue(), value);
142        }
143    }
144
145    /** List of Zip/Jar files to compile (see {@link Options#CompileTheWorldClasspath}). */
146    private final String files;
147
148    /** Class index to start compilation at (see {@link Options#CompileTheWorldStartAt}). */
149    private final int startAt;
150
151    /** Class index to stop compilation at (see {@link Options#CompileTheWorldStopAt}). */
152    private final int stopAt;
153
154    /** Only compile methods matching one of the filters in this array if the array is non-null. */
155    private final MethodFilter[] methodFilters;
156
157    /** Exclude methods matching one of the filters in this array if the array is non-null. */
158    private final MethodFilter[] excludeMethodFilters;
159
160    // Counters
161    private int classFileCounter = 0;
162    private AtomicLong compiledMethodsCounter = new AtomicLong();
163    private AtomicLong compileTime = new AtomicLong();
164    private AtomicLong memoryUsed = new AtomicLong();
165
166    private boolean verbose;
167    private final Config config;
168
169    /**
170     * Signal that the threads should start compiling in multithreaded mode.
171     */
172    private boolean running;
173
174    private ThreadPoolExecutor threadPool;
175
176    /**
177     * Creates a compile-the-world instance.
178     *
179     * @param files {@link File#pathSeparator} separated list of Zip/Jar files to compile
180     * @param startAt index of the class file to start compilation at
181     * @param stopAt index of the class file to stop compilation at
182     * @param methodFilters
183     * @param excludeMethodFilters
184     */
185    public CompileTheWorld(String files, Config config, int startAt, int stopAt, String methodFilters, String excludeMethodFilters, boolean verbose) {
186        this.files = files;
187        this.startAt = startAt;
188        this.stopAt = stopAt;
189        this.methodFilters = methodFilters == null || methodFilters.isEmpty() ? null : MethodFilter.parse(methodFilters);
190        this.excludeMethodFilters = excludeMethodFilters == null || excludeMethodFilters.isEmpty() ? null : MethodFilter.parse(excludeMethodFilters);
191        this.verbose = verbose;
192        this.config = config;
193
194        // We don't want the VM to exit when a method fails to compile...
195        config.putIfAbsent(ExitVMOnException, false);
196
197        // ...but we want to see exceptions.
198        config.putIfAbsent(PrintBailout, true);
199        config.putIfAbsent(PrintStackTraceOnException, true);
200        config.putIfAbsent(HotSpotResolvedJavaMethodImpl.Options.UseProfilingInformation, false);
201    }
202
203    public CompileTheWorld() {
204        this(CompileTheWorldClasspath.getValue(), new Config(CompileTheWorldConfig.getValue()), CompileTheWorldStartAt.getValue(), CompileTheWorldStopAt.getValue(),
205                        CompileTheWorldMethodFilter.getValue(), CompileTheWorldExcludeMethodFilter.getValue(), CompileTheWorldVerbose.getValue());
206    }
207
208    /**
209     * Compiles all methods in all classes in the Zip/Jar archive files in
210     * {@link Options#CompileTheWorldClasspath}. If {@link Options#CompileTheWorldClasspath}
211     * contains the magic token {@link #SUN_BOOT_CLASS_PATH} passed up from HotSpot we take the
212     * files from the boot class path.
213     */
214    public void compile() throws Throwable {
215        // By default only report statistics for the CTW threads themselves
216        if (GraalDebugConfig.DebugValueThreadFilter.hasDefaultValue()) {
217            GraalDebugConfig.DebugValueThreadFilter.setValue("^CompileTheWorld");
218        }
219
220        if (SUN_BOOT_CLASS_PATH.equals(files)) {
221            final String[] entries = System.getProperty(SUN_BOOT_CLASS_PATH).split(File.pathSeparator);
222            String bcpFiles = "";
223            for (int i = 0; i < entries.length; i++) {
224                final String entry = entries[i];
225
226                // We stop at rt.jar, unless it is the first boot class path entry.
227                if (entry.endsWith("rt.jar") && (i > 0)) {
228                    break;
229                }
230                if (i > 0) {
231                    bcpFiles += File.pathSeparator;
232                }
233                bcpFiles += entry;
234            }
235            compile(bcpFiles);
236        } else {
237            compile(files);
238        }
239    }
240
241    public void println() {
242        println("");
243    }
244
245    public void println(String format, Object... args) {
246        println(String.format(format, args));
247    }
248
249    public void println(String s) {
250        if (verbose) {
251            TTY.println(s);
252        }
253    }
254
255    @SuppressWarnings("unused")
256    private static void dummy() {
257    }
258
259    /**
260     * Compiles all methods in all classes in the Zip/Jar files passed.
261     *
262     * @param fileList {@link File#pathSeparator} separated list of Zip/Jar files to compile
263     * @throws IOException
264     */
265    private void compile(String fileList) throws IOException {
266        final String[] entries = fileList.split(File.pathSeparator);
267        long start = System.currentTimeMillis();
268
269        CompilerThreadFactory factory = new CompilerThreadFactory("CompileTheWorld", new DebugConfigAccess() {
270            public GraalDebugConfig getDebugConfig() {
271                if (Debug.isEnabled() && DebugScope.getConfig() == null) {
272                    return DebugEnvironment.initialize(System.out);
273                }
274                return null;
275            }
276        });
277
278        try {
279            // compile dummy method to get compiler initilized outside of the config debug override.
280            HotSpotResolvedJavaMethod dummyMethod = (HotSpotResolvedJavaMethod) JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess().lookupJavaMethod(
281                            CompileTheWorld.class.getDeclaredMethod("dummy"));
282            CompilationTask task = new CompilationTask(dummyMethod, Compiler.INVOCATION_ENTRY_BCI, 0L, dummyMethod.allocateCompileId(Compiler.INVOCATION_ENTRY_BCI), false);
283            task.runCompilation();
284        } catch (NoSuchMethodException | SecurityException e1) {
285            e1.printStackTrace();
286        }
287
288        /*
289         * Always use a thread pool, even for single threaded mode since it simplifies the use of
290         * DebugValueThreadFilter to filter on the thread names.
291         */
292        int threadCount = 1;
293        if (Options.CompileTheWorldMultiThreaded.getValue()) {
294            threadCount = Options.CompileTheWorldThreads.getValue();
295            if (threadCount == 0) {
296                threadCount = Runtime.getRuntime().availableProcessors();
297            }
298        } else {
299            running = true;
300        }
301        threadPool = new ThreadPoolExecutor(threadCount, threadCount, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), factory);
302
303        try (OverrideScope s = config.apply()) {
304            for (int i = 0; i < entries.length; i++) {
305                final String entry = entries[i];
306
307                // For now we only compile all methods in all classes in zip/jar files.
308                if (!entry.endsWith(".zip") && !entry.endsWith(".jar")) {
309                    println("CompileTheWorld : Skipped classes in " + entry);
310                    println();
311                    continue;
312                }
313
314                if (methodFilters == null || methodFilters.length == 0) {
315                    println("CompileTheWorld : Compiling all classes in " + entry);
316                } else {
317                    String include = Arrays.asList(methodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
318                    println("CompileTheWorld : Compiling all methods in " + entry + " matching one of the following filters: " + include);
319                }
320                if (excludeMethodFilters != null && excludeMethodFilters.length > 0) {
321                    String exclude = Arrays.asList(excludeMethodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
322                    println("CompileTheWorld : Excluding all methods matching one of the following filters: " + exclude);
323                }
324                println();
325
326                URL url = new URL("jar", "", "file:" + entry + "!/");
327                ClassLoader loader = new URLClassLoader(new URL[]{url});
328
329                JarFile jarFile = new JarFile(entry);
330                Enumeration<JarEntry> e = jarFile.entries();
331
332                while (e.hasMoreElements()) {
333                    JarEntry je = e.nextElement();
334                    if (je.isDirectory() || !je.getName().endsWith(".class")) {
335                        continue;
336                    }
337
338                    // Are we done?
339                    if (classFileCounter >= stopAt) {
340                        break;
341                    }
342
343                    String className = je.getName().substring(0, je.getName().length() - ".class".length());
344                    String dottedClassName = className.replace('/', '.');
345                    classFileCounter++;
346
347                    if (methodFilters != null && !MethodFilter.matchesClassName(methodFilters, dottedClassName)) {
348                        continue;
349                    }
350                    if (excludeMethodFilters != null && MethodFilter.matchesClassName(excludeMethodFilters, dottedClassName)) {
351                        continue;
352                    }
353
354                    if (dottedClassName.startsWith("jdk.management.") || dottedClassName.startsWith("jdk.internal.cmm.*")) {
355                        continue;
356                    }
357
358                    try {
359                        // Load and initialize class
360                        Class<?> javaClass = Class.forName(dottedClassName, true, loader);
361
362                        // Pre-load all classes in the constant pool.
363                        try {
364                            HotSpotResolvedObjectType objectType = HotSpotResolvedObjectTypeImpl.fromObjectClass(javaClass);
365                            ConstantPool constantPool = objectType.constantPool();
366                            for (int cpi = 1; cpi < constantPool.length(); cpi++) {
367                                constantPool.loadReferencedType(cpi, HotSpotConstantPool.Bytecodes.LDC);
368                            }
369                        } catch (Throwable t) {
370                            // If something went wrong during pre-loading we just ignore it.
371                            println("Preloading failed for (%d) %s: %s", classFileCounter, className, t);
372                        }
373
374                        // Are we compiling this class?
375                        MetaAccessProvider metaAccess = JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess();
376                        if (classFileCounter >= startAt) {
377                            println("CompileTheWorld (%d) : %s", classFileCounter, className);
378
379                            // Compile each constructor/method in the class.
380                            for (Constructor<?> constructor : javaClass.getDeclaredConstructors()) {
381                                HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(constructor);
382                                if (canBeCompiled(javaMethod, constructor.getModifiers())) {
383                                    compileMethod(javaMethod);
384                                }
385                            }
386                            for (Method method : javaClass.getDeclaredMethods()) {
387                                HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(method);
388                                if (canBeCompiled(javaMethod, method.getModifiers())) {
389                                    compileMethod(javaMethod);
390                                }
391                            }
392                        }
393                    } catch (Throwable t) {
394                        println("CompileTheWorld (%d) : Skipping %s %s", classFileCounter, className, t.toString());
395                        t.printStackTrace();
396                    }
397                }
398                jarFile.close();
399            }
400        }
401
402        if (!running) {
403            startThreads();
404        }
405        int wakeups = 0;
406        while (threadPool.getCompletedTaskCount() != threadPool.getTaskCount()) {
407            if (wakeups % 15 == 0) {
408                TTY.println("CompileTheWorld : Waiting for " + (threadPool.getTaskCount() - threadPool.getCompletedTaskCount()) + " compiles");
409            }
410            try {
411                threadPool.awaitTermination(1, TimeUnit.SECONDS);
412                wakeups++;
413            } catch (InterruptedException e) {
414            }
415        }
416        threadPool = null;
417
418        long elapsedTime = System.currentTimeMillis() - start;
419
420        println();
421        if (Options.CompileTheWorldMultiThreaded.getValue()) {
422            TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms elapsed, %d ms compile time, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), elapsedTime,
423                            compileTime.get(), memoryUsed.get());
424        } else {
425            TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), compileTime.get(), memoryUsed.get());
426        }
427    }
428
429    private synchronized void startThreads() {
430        running = true;
431        // Wake up any waiting threads
432        notifyAll();
433    }
434
435    private synchronized void waitToRun() {
436        while (!running) {
437            try {
438                wait();
439            } catch (InterruptedException e) {
440            }
441        }
442    }
443
444    private void compileMethod(HotSpotResolvedJavaMethod method) throws InterruptedException, ExecutionException {
445        if (methodFilters != null && !MethodFilter.matches(methodFilters, method)) {
446            return;
447        }
448        if (excludeMethodFilters != null && MethodFilter.matches(excludeMethodFilters, method)) {
449            return;
450        }
451        Future<?> task = threadPool.submit(new Runnable() {
452            public void run() {
453                waitToRun();
454                try (OverrideScope s = config.apply()) {
455                    compileMethod(method, classFileCounter);
456                }
457            }
458        });
459        if (threadPool.getCorePoolSize() == 1) {
460            task.get();
461        }
462    }
463
464    /**
465     * Compiles a method and gathers some statistics.
466     */
467    private void compileMethod(HotSpotResolvedJavaMethod method, int counter) {
468        try {
469            long start = System.currentTimeMillis();
470            long allocatedAtStart = MemUseTrackerImpl.getCurrentThreadAllocatedBytes();
471
472            CompilationTask task = new CompilationTask(method, Compiler.INVOCATION_ENTRY_BCI, 0L, method.allocateCompileId(Compiler.INVOCATION_ENTRY_BCI), false);
473            task.runCompilation();
474
475            memoryUsed.getAndAdd(MemUseTrackerImpl.getCurrentThreadAllocatedBytes() - allocatedAtStart);
476            compileTime.getAndAdd(System.currentTimeMillis() - start);
477            compiledMethodsCounter.incrementAndGet();
478        } catch (Throwable t) {
479            // Catch everything and print a message
480            println("CompileTheWorld (%d) : Error compiling method: %s", counter, method.format("%H.%n(%p):%r"));
481            t.printStackTrace(TTY.out);
482        }
483    }
484
485    /**
486     * Determines if a method should be compiled (Cf. CompilationPolicy::can_be_compiled).
487     *
488     * @return true if it can be compiled, false otherwise
489     */
490    private static boolean canBeCompiled(HotSpotResolvedJavaMethod javaMethod, int modifiers) {
491        if (Modifier.isAbstract(modifiers) || Modifier.isNative(modifiers)) {
492            return false;
493        }
494        HotSpotVMConfig c = HotSpotJVMCIRuntime.runtime().getConfig();
495        if (c.dontCompileHugeMethods && javaMethod.getCodeSize() > c.hugeMethodLimit) {
496            return false;
497        }
498        // Allow use of -XX:CompileCommand=dontinline to exclude problematic methods
499        if (!javaMethod.canBeInlined()) {
500            return false;
501        }
502        // Skip @Snippets for now
503        for (Annotation annotation : javaMethod.getAnnotations()) {
504            if (annotation.annotationType().getName().equals("com.oracle.graal.replacements.Snippet")) {
505                return false;
506            }
507        }
508        return true;
509    }
510
511}