[platform] optimize setting watch roots
authorTomas Lundell <tomlu@google.com>
Thu, 10 Dec 2015 19:22:06 +0000 (20:22 +0100)
committerRoman Shevchenko <roman.shevchenko@jetbrains.com>
Thu, 10 Dec 2015 19:22:06 +0000 (20:22 +0100)
The file system operations in local file system and canonical path map can be slow when done in serial for a large number of roots. Run these in parallel on an executor.

On our benchmarks with about 8000 roots this saves ~700ms, or about 70%, when setting watch roots.

#331 (https://github.com/JetBrains/intellij-community/pull/331)

platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/CanonicalPathMap.java
platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java
platform/util/src/com/intellij/util/concurrency/Futures.java [new file with mode: 0644]

index 29914afe9ebc57cf47bdfd434ca532d207c13972..76d77207fe3667d68473ed05af4e019070414bf5 100644 (file)
@@ -20,14 +20,21 @@ import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.io.FileSystemUtil;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.util.Function;
+import com.intellij.util.concurrency.BoundedTaskExecutor;
+import com.intellij.util.concurrency.Futures;
 import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.containers.JBIterable;
 import com.intellij.util.containers.MultiMap;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.ide.PooledThreadExecutor;
 
 import java.io.File;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
 
 import static com.intellij.openapi.util.Pair.pair;
 
@@ -50,18 +57,41 @@ class CanonicalPathMap {
     myFlatWatchRoots = ContainerUtil.newArrayList(flat);
 
     List<Pair<String, String>> mapping = ContainerUtil.newSmartList();
-    myCanonicalRecursiveWatchRoots = mapPaths(recursive, mapping);
-    myCanonicalFlatWatchRoots = mapPaths(flat, mapping);
+    Map<String, String> resolvedPaths = resolvePaths(recursive, flat);
+    myCanonicalRecursiveWatchRoots = mapPaths(resolvedPaths, recursive, mapping);
+    myCanonicalFlatWatchRoots = mapPaths(resolvedPaths, flat, mapping);
 
     myPathMapping = MultiMap.createConcurrentSet();
     addMapping(mapping);
   }
 
-  private static List<String> mapPaths(List<String> paths, Collection<Pair<String, String>> mapping) {
+  private static Map<String, String> resolvePaths(Collection<String> recursiveRoots, Collection<String> flatRoots) {
+    final Map<String, String> resolvedPaths = ContainerUtil.newConcurrentMap();
+
+    final BoundedTaskExecutor executor = new BoundedTaskExecutor(PooledThreadExecutor.INSTANCE, Runtime.getRuntime().availableProcessors());
+    Futures.invokeAll(JBIterable.from(recursiveRoots).append(flatRoots).transform(new Function<String, Future<?>>() {
+      @Override
+      public Future<?> fun(final String root) {
+        return executor.submit(new Runnable() {
+          @Override
+          public void run() {
+            String canonicalPath = FileSystemUtil.resolveSymLink(root);
+            if (canonicalPath != null) {
+              resolvedPaths.put(root, canonicalPath);
+            }
+          }
+        });
+      }
+    }).toList());
+
+    return resolvedPaths;
+  }
+
+  private static List<String> mapPaths(Map<String, String> resolvedPaths, List<String> paths, Collection<Pair<String, String>> mapping) {
     List<String> canonicalPaths = ContainerUtil.newArrayList(paths);
     for (int i = 0; i < paths.size(); i++) {
       String path = paths.get(i);
-      String canonicalPath = FileSystemUtil.resolveSymLink(path);
+      String canonicalPath = resolvedPaths.get(path);
       if (canonicalPath != null && !path.equals(canonicalPath)) {
         canonicalPaths.set(i, canonicalPath);
         mapping.add(pair(canonicalPath, path));
@@ -178,4 +208,4 @@ class CanonicalPathMap {
 
     return results;
   }
-}
\ No newline at end of file
+}
index 75afd747bab63709d36872270d1a6123d96a078e..6fa10f8ce5f21897f20529c6b7ee7bdbc2f71dbc 100644 (file)
@@ -31,18 +31,24 @@ import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
 import com.intellij.openapi.vfs.newvfs.RefreshQueue;
 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
 import com.intellij.util.Consumer;
+import com.intellij.util.Function;
+import com.intellij.util.concurrency.BoundedTaskExecutor;
+import com.intellij.util.concurrency.Futures;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.containers.HashSet;
+import com.intellij.util.containers.JBIterable;
 import gnu.trove.THashMap;
 import gnu.trove.THashSet;
 import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
+import org.jetbrains.ide.PooledThreadExecutor;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.util.*;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 public final class LocalFileSystemImpl extends LocalFileSystemBase implements ApplicationComponent {
@@ -59,7 +65,7 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap
     private final boolean myWatchRecursively;
     private boolean myDominated;
 
-    public WatchRequestImpl(String rootPath, boolean watchRecursively) throws FileNotFoundException {
+    public WatchRequestImpl(String rootPath, boolean isDirectory, boolean watchRecursively) throws FileNotFoundException {
       int index = rootPath.indexOf(JarFileSystem.JAR_SEPARATOR);
       if (index >= 0) rootPath = rootPath.substring(0, index);
 
@@ -68,7 +74,7 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap
         throw new FileNotFoundException("Invalid path: " + rootPath);
       }
 
-      if (index > 0 || !(FileUtil.isRootPath(rootFile) || rootFile.isDirectory())) {
+      if (index > 0 || !(FileUtil.isRootPath(rootFile) || isDirectory)) {
         File parentFile = rootFile.getParentFile();
         if (parentFile == null) {
           throw new FileNotFoundException(rootPath);
@@ -421,8 +427,10 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap
                                     @NotNull final Set<VirtualFile> filesToSync) {
     boolean update = false;
 
+    Set<String> directories = findDirectories(recursiveRoots, flatRoots);
+
     for (String root : recursiveRoots) {
-      final WatchRequestImpl request = watch(root, true);
+      WatchRequestImpl request = watch(root, directories.contains(root), true);
       if (request == null) continue;
       final boolean alreadyWatched = isAlreadyWatched(request);
 
@@ -434,7 +442,7 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap
     }
 
     for (String root : flatRoots) {
-      final WatchRequestImpl request = watch(root, false);
+      WatchRequestImpl request = watch(root, directories.contains(root), false);
       if (request == null) continue;
       final boolean alreadyWatched = isAlreadyWatched(request);
 
@@ -455,10 +463,31 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap
     return update;
   }
 
+  private static Set<String> findDirectories(Collection<String> recursiveRoots, Collection<String> flatRoots) {
+    final Set<String> directories = ContainerUtil.newConcurrentSet();
+
+    final BoundedTaskExecutor executor = new BoundedTaskExecutor(PooledThreadExecutor.INSTANCE, Runtime.getRuntime().availableProcessors());
+    Futures.invokeAll(JBIterable.from(recursiveRoots).append(flatRoots).transform(new Function<String, Future<?>>() {
+      @Override
+      public Future<?> fun(final String root) {
+        return executor.submit(new Runnable() {
+          @Override
+          public void run() {
+            if (!root.contains(JarFileSystem.JAR_SEPARATOR) && new File(root).isDirectory()) {
+              directories.add(root);
+            }
+          }
+        });
+      }
+    }).toList());
+
+    return directories;
+  }
+
   @Nullable
-  private static WatchRequestImpl watch(final String root, final boolean recursively) {
+  private static WatchRequestImpl watch(final String root, boolean isDirectory, final boolean recursively) {
     try {
-      return new WatchRequestImpl(root, recursively);
+      return new WatchRequestImpl(root, isDirectory, recursively);
     }
     catch (FileNotFoundException e) {
       LOG.warn(e);
diff --git a/platform/util/src/com/intellij/util/concurrency/Futures.java b/platform/util/src/com/intellij/util/concurrency/Futures.java
new file mode 100644 (file)
index 0000000..0913480
--- /dev/null
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.concurrency;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+public class Futures {
+  /** A representation of a {@link Future} computation result. */
+  public static class Result<V> {
+    private final V myResult;
+    private final Throwable myError;
+
+    private Result(V result, @Nullable Throwable error) {
+      myResult = result;
+      myError = error;
+    }
+
+    public boolean isOK() {
+      return myError == null;
+    }
+
+    public V get() throws IllegalStateException {
+      if (myError != null) {
+        throw new IllegalStateException(myError);
+      }
+      else {
+        return myResult;
+      }
+    }
+
+    @Nullable
+    public Throwable getError() {
+      return myError;
+    }
+  }
+
+  @NotNull
+  public static <V> Collection<Result<V>> invokeAll(@NotNull Collection<? extends Future<? extends V>> futures) {
+    List<Result<V>> results = ContainerUtil.newArrayListWithCapacity(futures.size());
+
+    try {
+      for (Future<? extends V> future : futures) {
+        try {
+          results.add(new Result<V>(future.get(), null));
+        }
+        catch (ExecutionException e) {
+          results.add(new Result<V>(null, e));
+        }
+      }
+    }
+    catch (InterruptedException e) {
+      Logger.getInstance(Futures.class).error(e);
+      Thread.currentThread().interrupt();
+    }
+
+    return results;
+  }
+
+  private Futures() { }
+}
\ No newline at end of file