Indexing diagnostics: record statistics for cancelled indexing runs of individual...
authorSergey Patrikeev <Sergey.Patrikeev@jetbrains.com>
Mon, 10 Aug 2020 13:28:59 +0000 (16:28 +0300)
committerintellij-monorepo-bot <intellij-monorepo-bot-no-reply@jetbrains.com>
Mon, 10 Aug 2020 14:48:56 +0000 (14:48 +0000)
GitOrigin-RevId: ded3a30a57cc48e7ac66393bcb72f800dce78001

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java
platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesUpdater.java
platform/lang-impl/src/com/intellij/util/indexing/contentQueue/IndexUpdateRunner.java
platform/lang-impl/src/com/intellij/util/indexing/diagnostic/FileProviderIndexStatistics.kt [deleted file]
platform/lang-impl/src/com/intellij/util/indexing/diagnostic/ProjectIndexingHistory.kt
platform/lang-impl/src/com/intellij/util/indexing/diagnostic/dto/JsonConverter.kt

index 66aef2a7276706a0d09e33e7c2d3915e77a56bf4..ff117adef0eeb05910d209c96dd5a2343ed3f8ca 100644 (file)
@@ -26,6 +26,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileVisitor;
+import com.intellij.util.ExceptionUtil;
 import com.intellij.util.Processor;
 import com.intellij.util.indexing.contentQueue.IndexUpdateRunner;
 import org.jetbrains.annotations.ApiStatus;
@@ -158,7 +159,12 @@ public final class FileBasedIndexProjectHandler implements IndexableFileSet {
           PerformanceWatcher.Snapshot snapshot = PerformanceWatcher.takeSnapshot();
           int numberOfIndexingThreads = UnindexedFilesUpdater.getNumberOfIndexingThreads();
           LOG.info("Using " + numberOfIndexingThreads + " " + StringUtil.pluralize("thread", numberOfIndexingThreads) + " for indexing");
-          new IndexUpdateRunner(index, UnindexedFilesUpdater.GLOBAL_INDEXING_EXECUTOR, numberOfIndexingThreads).indexFiles(project, files, indicator);
+          IndexUpdateRunner indexUpdateRunner = new IndexUpdateRunner(index, UnindexedFilesUpdater.GLOBAL_INDEXING_EXECUTOR, numberOfIndexingThreads);
+          try {
+            indexUpdateRunner.indexFiles(project, files, indicator);
+          } catch (IndexUpdateRunner.IndexingInterruptedException e) {
+            ExceptionUtil.rethrow(e.getCause());
+          }
           snapshot.logResponsivenessSinceCreation("Reindexing refreshed files");
         }
       }
index 4ce3d69b2e83ac7e041bdc220536eb9deb740e93..3d22bda535f63fb8f2e1f7d8c659ecc402ed1efd 100644 (file)
@@ -23,13 +23,13 @@ import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileFilter;
 import com.intellij.openapi.vfs.VirtualFileManager;
 import com.intellij.openapi.vfs.newvfs.RefreshQueue;
+import com.intellij.util.ExceptionUtil;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.concurrency.AppExecutorUtil;
 import com.intellij.util.concurrency.NonUrgentExecutor;
 import com.intellij.util.containers.ConcurrentBitSet;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.contentQueue.IndexUpdateRunner;
-import com.intellij.util.indexing.diagnostic.FileProviderIndexStatistics;
 import com.intellij.util.indexing.diagnostic.IndexDiagnosticDumper;
 import com.intellij.util.indexing.diagnostic.IndexingJobStatistics;
 import com.intellij.util.indexing.diagnostic.ProjectIndexingHistory;
@@ -159,19 +159,28 @@ public final class UnindexedFilesUpdater extends DumbModeTask {
       concurrentTasksProgressManager.setText(provider.getIndexingProgressText());
       SubTaskProgressIndicator subTaskIndicator = concurrentTasksProgressManager.createSubTaskIndicator(providerFiles.size());
       try {
-        long startTime = System.nanoTime();
-        IndexingJobStatistics indexStatistics = indexUpdateRunner.indexFiles(myProject, providerFiles, subTaskIndicator);
-        long totalTime = System.nanoTime() - startTime;
+        long totalTime = System.nanoTime();
+        IndexingJobStatistics statistics;
+        IndexUpdateRunner.IndexingInterruptedException exception = null;
         try {
-          FileProviderIndexStatistics statistics = new FileProviderIndexStatistics(provider.getDebugName(),
-                                                                                   providerFiles.size(),
-                                                                                   totalTime,
-                                                                                   indexStatistics);
-          projectIndexingHistory.addProviderStatistics(statistics);
+          statistics = indexUpdateRunner.indexFiles(myProject, providerFiles, subTaskIndicator);
+        } catch (IndexUpdateRunner.IndexingInterruptedException e) {
+          exception = e;
+          statistics = e.myStatistics;
+        } finally {
+          totalTime = System.nanoTime() - totalTime;
+        }
+
+        try {
+          projectIndexingHistory.addProviderStatistics(provider.getDebugName(), providerFiles.size(), totalTime, statistics);
         }
         catch (Exception e) {
           LOG.error("Failed to add indexing statistics for " + provider.getDebugName(), e);
         }
+
+        if (exception != null) {
+          ExceptionUtil.rethrow(exception.getCause());
+        }
       } finally {
         subTaskIndicator.finished();
       }
index 4520ae470792e3ebe265529ff890132281bb3661..797fc24663b14e5b2281609fa3cc1fe836aea0ec 100644 (file)
@@ -76,17 +76,43 @@ public final class IndexUpdateRunner {
     myNumberOfIndexingThreads = numberOfIndexingThreads;
   }
 
+  /**
+   * This exception contains indexing statistics accumulated by the time of a thrown exception.
+   */
+  public static class IndexingInterruptedException extends Exception {
+    public final IndexingJobStatistics myStatistics;
+
+    public IndexingInterruptedException(@NotNull Throwable cause, IndexingJobStatistics statistics) {
+      super(cause);
+      myStatistics = statistics;
+    }
+  }
+
   @NotNull
   public IndexingJobStatistics indexFiles(@NotNull Project project,
                                           @NotNull Collection<VirtualFile> files,
-                                          @NotNull ProgressIndicator indicator) {
+                                          @NotNull ProgressIndicator indicator) throws IndexingInterruptedException {
+    IndexingJobStatistics statistics = new IndexingJobStatistics();
+    try {
+      doIndexFiles(project, files, indicator, statistics);
+    }
+    catch (RuntimeException e) {
+      throw new IndexingInterruptedException(e, statistics);
+    }
+    return statistics;
+  }
+
+  private void doIndexFiles(@NotNull Project project,
+                            @NotNull Collection<VirtualFile> files,
+                            @NotNull ProgressIndicator indicator,
+                            @NotNull IndexingJobStatistics statistics) {
     indicator.checkCanceled();
     indicator.setIndeterminate(false);
 
     CachedFileContentLoader contentLoader = new CurrentProjectHintedCachedFileContentLoader(project);
     ProgressIndicator originalIndicator = unwrapAll(indicator);
     ProgressSuspender originalSuspender = ProgressSuspender.getSuspender(originalIndicator);
-    IndexingJob indexingJob = new IndexingJob(project, indicator, contentLoader, files, originalIndicator, originalSuspender);
+    IndexingJob indexingJob = new IndexingJob(project, indicator, contentLoader, files, statistics, originalIndicator, originalSuspender);
     if (ApplicationManager.getApplication().isWriteAccessAllowed()) {
       // If the current thread has acquired the write lock, we can't grant it to worker threads, so we must do the work in the current thread.
       while (!indexingJob.areAllFilesProcessed()) {
@@ -138,7 +164,6 @@ public final class IndexUpdateRunner {
         ourIndexingJobs.remove(indexingJob);
       }
     }
-    return indexingJob.myStatistics;
   }
 
   // Index jobs one by one while there are some. Jobs may belong to different projects, and we index them fairly.
@@ -396,13 +421,14 @@ public final class IndexUpdateRunner {
     final CountDownLatch myAllFilesAreProcessedLatch;
     final ProgressIndicator myOriginalProgressIndicator;
     @Nullable final ProgressSuspender myOriginalProgressSuspender;
-    final IndexingJobStatistics myStatistics = new IndexingJobStatistics();
+    final IndexingJobStatistics myStatistics;
     final AtomicReference<Throwable> myError = new AtomicReference<>();
 
     IndexingJob(@NotNull Project project,
                 @NotNull ProgressIndicator indicator,
                 @NotNull CachedFileContentLoader contentLoader,
                 @NotNull Collection<VirtualFile> files,
+                @NotNull IndexingJobStatistics statistics,
                 @NotNull ProgressIndicator originalProgressIndicator,
                 @Nullable ProgressSuspender originalProgressSuspender) {
       myProject = project;
@@ -410,6 +436,7 @@ public final class IndexUpdateRunner {
       myTotalFiles = files.size();
       myContentLoader = contentLoader;
       myQueueOfFiles = new ArrayBlockingQueue<>(files.size(), false, files);
+      myStatistics = statistics;
       myAllFilesAreProcessedLatch = new CountDownLatch(files.size());
       myOriginalProgressIndicator = originalProgressIndicator;
       myOriginalProgressSuspender = originalProgressSuspender;
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/FileProviderIndexStatistics.kt b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/FileProviderIndexStatistics.kt
deleted file mode 100644 (file)
index ec8289e..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.intellij.util.indexing.diagnostic
-
-class FileProviderIndexStatistics(
-  val providerDebugName: String,
-  val numberOfFiles: Int,
-  val totalTime: TimeNano,
-  val indexingStatistics: IndexingJobStatistics
-) 
\ No newline at end of file
index b151a5cbc623650c9f6e403e3da36eac2499fbd3..de03d6c28cb5a94b4b9ec0edbfe98484f909f76d 100644 (file)
@@ -2,7 +2,7 @@
 package com.intellij.util.indexing.diagnostic
 
 import com.intellij.util.indexing.diagnostic.dto.JsonFileProviderIndexStatistics
-import com.intellij.util.indexing.diagnostic.dto.toJson
+import com.intellij.util.indexing.diagnostic.dto.createProviderJsonStatistics
 import java.time.Instant
 
 typealias TimeMillis = Long
@@ -25,11 +25,16 @@ data class ProjectIndexingHistory(val projectName: String) {
   var totalNumberOfTooLargeFiles: Int = 0
   val totalTooLargeFiles = LimitedPriorityQueue<TooLargeForIndexingFile>(5, compareBy { it.fileSize })
 
-  fun addProviderStatistics(statistics: FileProviderIndexStatistics) {
+  fun addProviderStatistics(
+    providerDebugName: String,
+    numberOfFiles: Int,
+    totalTime: TimeNano,
+    statistics: IndexingJobStatistics
+  ) {
     // Convert to Json to release memory occupied by statistic values.
-    providerStatistics += statistics.toJson()
+    providerStatistics += statistics.createProviderJsonStatistics(providerDebugName, numberOfFiles, totalTime)
 
-    for ((fileType, fileTypeStats) in statistics.indexingStatistics.statsPerFileType) {
+    for ((fileType, fileTypeStats) in statistics.statsPerFileType) {
       val totalStats = totalStatsPerFileType.getOrPut(fileType) {
         StatsPerFileType(0, 0, 0, 0, LimitedPriorityQueue(biggestContributorsPerFileTypeLimit, compareBy { it.indexingTimeInAllThreads }))
       }
@@ -39,7 +44,7 @@ data class ProjectIndexingHistory(val projectName: String) {
       totalStats.totalContentLoadingTimeInAllThreads += fileTypeStats.contentLoadingTime.sumTime
       totalStats.biggestFileTypeContributors.addElement(
         BiggestFileTypeContributor(
-          statistics.providerDebugName,
+          providerDebugName,
           fileTypeStats.numberOfFiles,
           fileTypeStats.totalBytes,
           fileTypeStats.indexingTime.sumTime
@@ -47,15 +52,15 @@ data class ProjectIndexingHistory(val projectName: String) {
       )
     }
 
-    for ((indexId, stats) in statistics.indexingStatistics.statsPerIndexer) {
+    for ((indexId, stats) in statistics.statsPerIndexer) {
       val totalStats = totalStatsPerIndexer.getOrPut(indexId) { StatsPerIndexer(0, 0, 0) }
       totalStats.totalNumberOfFiles += stats.numberOfFiles
       totalStats.totalBytes += stats.totalBytes
       totalStats.totalIndexingTimeInAllThreads += stats.indexingTime.sumTime
     }
 
-    totalNumberOfTooLargeFiles += statistics.indexingStatistics.numberOfTooLargeForIndexingFiles.get()
-    statistics.indexingStatistics.tooLargeForIndexingFiles.biggestElements.forEach { totalTooLargeFiles.addElement(it) }
+    totalNumberOfTooLargeFiles += statistics.numberOfTooLargeForIndexingFiles.get()
+    statistics.tooLargeForIndexingFiles.biggestElements.forEach { totalTooLargeFiles.addElement(it) }
   }
 
   data class StatsPerFileType(
index 7e14b0346a458c6fd2dfa5510615a516fe66cdf0..70433dfd051c888c35daba5505982bf5a6211eb0 100644 (file)
@@ -13,7 +13,11 @@ typealias PositiveInt = Int?
 
 fun Int.toPositiveInt() = takeIf { it > 0 }
 
-fun FileProviderIndexStatistics.toJson(): JsonFileProviderIndexStatistics {
+fun IndexingJobStatistics.createProviderJsonStatistics(
+  providerDebugName: String,
+  numberOfFiles: Int,
+  totalTime: TimeNano
+): JsonFileProviderIndexStatistics {
   val statsPerFileType = aggregateStatsPerFileType()
   val allStatsPerIndexer = aggregateStatsPerIndexer()
   val (statsPerIndexer, fastIndexers) = allStatsPerIndexer.partition { it.partOfTotalIndexingTime.percentages > 0.01 }
@@ -22,26 +26,26 @@ fun FileProviderIndexStatistics.toJson(): JsonFileProviderIndexStatistics {
     providerDebugName,
     numberOfFiles,
     JsonDuration(totalTime),
-    indexingStatistics.numberOfTooLargeForIndexingFiles.get().toPositiveInt(),
-    indexingStatistics.tooLargeForIndexingFiles.biggestElements.map { it.toJson() }.takeIf { it.isNotEmpty() },
+    numberOfTooLargeForIndexingFiles.get().toPositiveInt(),
+    tooLargeForIndexingFiles.biggestElements.map { it.toJson() }.takeIf { it.isNotEmpty() },
     statsPerFileType.sortedByDescending { it.partOfTotalIndexingTime.percentages },
     statsPerIndexer.sortedByDescending { it.partOfTotalIndexingTime.percentages },
     fastIndexers.map { it.indexId }.sorted()
   )
 }
 
-private fun FileProviderIndexStatistics.aggregateStatsPerFileType(): List<JsonFileProviderIndexStatistics.JsonStatsPerFileType> {
-  val totalIndexingTimePerFileType = indexingStatistics.statsPerFileType.values
+private fun IndexingJobStatistics.aggregateStatsPerFileType(): List<JsonFileProviderIndexStatistics.JsonStatsPerFileType> {
+  val totalIndexingTimePerFileType = statsPerFileType.values
     .filterNot { it.indexingTime.isEmpty }
     .map { it.indexingTime.sumTime }
     .sum()
 
-  val totalContentLoadingTimePerFileType = indexingStatistics.statsPerFileType.values
+  val totalContentLoadingTimePerFileType = statsPerFileType.values
     .filterNot { it.contentLoadingTime.isEmpty }
     .map { it.contentLoadingTime.sumTime }
     .sum()
 
-  return indexingStatistics.statsPerFileType
+  return statsPerFileType
     .mapNotNull { (fileTypeName, stats) ->
       JsonFileProviderIndexStatistics.JsonStatsPerFileType(
         fileTypeName,
@@ -53,13 +57,13 @@ private fun FileProviderIndexStatistics.aggregateStatsPerFileType(): List<JsonFi
     }
 }
 
-private fun FileProviderIndexStatistics.aggregateStatsPerIndexer(): List<JsonFileProviderIndexStatistics.JsonStatsPerIndexer> {
-  val totalIndexingTimePerIndexer = indexingStatistics.statsPerIndexer.values
+private fun IndexingJobStatistics.aggregateStatsPerIndexer(): List<JsonFileProviderIndexStatistics.JsonStatsPerIndexer> {
+  val totalIndexingTimePerIndexer = statsPerIndexer.values
     .filterNot { it.indexingTime.isEmpty }
     .map { it.indexingTime.sumTime }
     .sum()
 
-  return indexingStatistics.statsPerIndexer
+  return statsPerIndexer
     .mapNotNull { (indexId, stats) ->
       JsonFileProviderIndexStatistics.JsonStatsPerIndexer(
         indexId,