stubs: log PCE-s on index update
[idea/community.git] / platform / lang-impl / src / com / intellij / psi / stubs / StubIndexImpl.java
1 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
2 package com.intellij.psi.stubs;
3
4 import com.intellij.model.ModelBranchImpl;
5 import com.intellij.openapi.application.AppUIExecutor;
6 import com.intellij.openapi.application.ModalityState;
7 import com.intellij.openapi.diagnostic.Logger;
8 import com.intellij.openapi.extensions.ExtensionPointListener;
9 import com.intellij.openapi.extensions.PluginDescriptor;
10 import com.intellij.openapi.extensions.impl.ExtensionPointImpl;
11 import com.intellij.openapi.progress.ProgressManager;
12 import com.intellij.openapi.progress.util.ProgressIndicatorUtils;
13 import com.intellij.openapi.project.DumbService;
14 import com.intellij.openapi.project.Project;
15 import com.intellij.openapi.util.ModificationTracker;
16 import com.intellij.openapi.util.io.FileUtil;
17 import com.intellij.openapi.vfs.VirtualFile;
18 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
19 import com.intellij.psi.PsiElement;
20 import com.intellij.psi.search.EverythingGlobalScope;
21 import com.intellij.psi.search.GlobalSearchScope;
22 import com.intellij.psi.util.CachedValue;
23 import com.intellij.psi.util.CachedValueProvider;
24 import com.intellij.util.*;
25 import com.intellij.util.containers.FactoryMap;
26 import com.intellij.util.indexing.*;
27 import com.intellij.util.indexing.impl.AbstractUpdateData;
28 import com.intellij.util.indexing.impl.KeyValueUpdateProcessor;
29 import com.intellij.util.indexing.impl.RemovedKeyProcessor;
30 import com.intellij.util.indexing.impl.storage.TransientChangesIndexStorage;
31 import com.intellij.util.indexing.impl.storage.VfsAwareMapIndexStorage;
32 import com.intellij.util.indexing.impl.storage.VfsAwareMapReduceIndex;
33 import com.intellij.util.indexing.memory.InMemoryIndexStorage;
34 import com.intellij.util.io.DataExternalizer;
35 import com.intellij.util.io.KeyDescriptor;
36 import com.intellij.util.io.VoidDataExternalizer;
37 import it.unimi.dsi.fastutil.ints.IntArrayList;
38 import it.unimi.dsi.fastutil.objects.Object2IntMap;
39 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
40 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
41 import org.jetbrains.annotations.NotNull;
42 import org.jetbrains.annotations.Nullable;
43 import org.jetbrains.annotations.TestOnly;
44
45 import java.io.File;
46 import java.io.IOException;
47 import java.util.*;
48 import java.util.concurrent.CompletableFuture;
49 import java.util.concurrent.ConcurrentHashMap;
50 import java.util.concurrent.Future;
51 import java.util.concurrent.atomic.AtomicReference;
52 import java.util.concurrent.locks.Lock;
53 import java.util.concurrent.locks.ReadWriteLock;
54 import java.util.function.IntPredicate;
55
56 public final class StubIndexImpl extends StubIndexEx {
57   private static final AtomicReference<Boolean> ourForcedClean = new AtomicReference<>(null);
58   static final Logger LOG = Logger.getInstance(StubIndexImpl.class);
59
60   private static final class AsyncState {
61     private final Map<StubIndexKey<?, ?>, UpdatableIndex<?, Void, FileContent>> myIndices = new Object2ObjectOpenHashMap<>();
62     private final Object2IntMap<ID<?, ?>> myIndexIdToVersionMap = new Object2IntOpenHashMap<>();
63   }
64
65   private final Map<StubIndexKey<?, ?>, CachedValue<Map<CompositeKey<?>, StubIdList>>> myCachedStubIds = FactoryMap.createMap(k -> {
66     UpdatableIndex<Integer, SerializedStubTree, FileContent> index = getStubUpdatingIndex();
67     ModificationTracker tracker = index::getModificationStamp;
68     return new CachedValueImpl<>(() -> new CachedValueProvider.Result<>(new ConcurrentHashMap<>(), tracker));
69   }, ConcurrentHashMap::new);
70
71   private final StubProcessingHelper myStubProcessingHelper = new StubProcessingHelper();
72   private final IndexAccessValidator myAccessValidator = new IndexAccessValidator();
73   private volatile CompletableFuture<AsyncState> myStateFuture;
74   private volatile AsyncState myState;
75   private volatile boolean myInitialized;
76
77   public StubIndexImpl() {
78     StubIndexExtension.EP_NAME.addExtensionPointListener(new ExtensionPointListener<StubIndexExtension<?, ?>>() {
79       @Override
80       public void extensionRemoved(@NotNull StubIndexExtension<?, ?> extension, @NotNull PluginDescriptor pluginDescriptor) {
81         ID.unloadId(extension.getKey());
82       }
83     }, null);
84   }
85
86   static @Nullable StubIndexImpl getInstanceOrInvalidate() {
87     if (ourForcedClean.compareAndSet(null, Boolean.TRUE)) {
88       return null;
89     }
90     return (StubIndexImpl)getInstance();
91   }
92
93   private AsyncState getAsyncState() {
94     AsyncState state = myState; // memory barrier
95     if (state == null) {
96       if (myStateFuture == null) {
97         ((FileBasedIndexImpl)FileBasedIndex.getInstance()).waitUntilIndicesAreInitialized();
98       }
99       myState = state = ProgressIndicatorUtils.awaitWithCheckCanceled(myStateFuture);
100     }
101     return state;
102   }
103
104   public void initializationFailed(@NotNull Throwable error) {
105     myStateFuture = new CompletableFuture<>();
106     myStateFuture.completeExceptionally(error);
107   }
108
109   public static @NotNull <K> FileBasedIndexExtension<K, Void> wrapStubIndexExtension(StubIndexExtension<K, ?> extension) {
110     return new FileBasedIndexExtension<K, Void>() {
111       @Override
112       public @NotNull ID<K, Void> getName() {
113         @SuppressWarnings("unchecked") ID<K, Void> key = (ID<K, Void>)extension.getKey();
114         return key;
115       }
116
117       @Override
118       public @NotNull FileBasedIndex.InputFilter getInputFilter() {
119         return f -> {
120           throw new UnsupportedOperationException();
121         };
122       }
123
124       @Override
125       public boolean dependsOnFileContent() {
126         return true;
127       }
128
129       @Override
130       public boolean needsForwardIndexWhenSharing() {
131         return false;
132       }
133
134       @Override
135       public @NotNull DataIndexer<K, Void, FileContent> getIndexer() {
136         return i -> {
137           throw new AssertionError();
138         };
139       }
140
141       @Override
142       public @NotNull KeyDescriptor<K> getKeyDescriptor() {
143         return extension.getKeyDescriptor();
144       }
145
146       @Override
147       public @NotNull DataExternalizer<Void> getValueExternalizer() {
148         return VoidDataExternalizer.INSTANCE;
149       }
150
151       @Override
152       public int getVersion() {
153         return extension.getVersion();
154       }
155
156       @Override
157       public boolean traceKeyHashToVirtualFileMapping() {
158         return extension instanceof StringStubIndexExtension && ((StringStubIndexExtension<?>)extension).traceKeyHashToVirtualFileMapping();
159       }
160     };
161   }
162
163   @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
164   private static <K> void registerIndexer(final @NotNull StubIndexExtension<K, ?> extension, final boolean forceClean,
165                                           @NotNull AsyncState state, @NotNull IndexVersionRegistrationSink registrationResultSink)
166     throws IOException {
167     final StubIndexKey<K, ?> indexKey = extension.getKey();
168     final int version = extension.getVersion();
169     FileBasedIndexExtension<K, Void> wrappedExtension = wrapStubIndexExtension(extension);
170     synchronized (state) {
171       state.myIndexIdToVersionMap.put(indexKey, version);
172     }
173
174     final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey);
175
176     IndexingStamp.IndexVersionDiff versionDiff = forceClean
177                                                  ? new IndexingStamp.IndexVersionDiff.InitialBuild(version)
178                                                  : IndexingStamp.versionDiffers(indexKey, version);
179
180     registrationResultSink.setIndexVersionDiff(indexKey, versionDiff);
181     if (versionDiff != IndexingStamp.IndexVersionDiff.UP_TO_DATE) {
182       final File versionFile = IndexInfrastructure.getVersionFile(indexKey);
183       final boolean versionFileExisted = versionFile.exists();
184
185       final String[] children = indexRootDir.list();
186       // rebuild only if there exists what to rebuild
187       boolean indexRootHasChildren = children != null && children.length > 0;
188       boolean needRebuild = !forceClean && (versionFileExisted || indexRootHasChildren);
189
190       if (indexRootHasChildren) FileUtil.deleteWithRenaming(indexRootDir);
191       IndexingStamp.rewriteVersion(indexKey, version); // todo snapshots indices
192
193       try {
194         if (needRebuild) {
195           for (FileBasedIndexInfrastructureExtension ex : FileBasedIndexInfrastructureExtension.EP_NAME.getExtensionList()) {
196             ex.onStubIndexVersionChanged(indexKey);
197           }
198         }
199       } catch (Exception e) {
200         LOG.error(e);
201       }
202     }
203
204     UpdatableIndex<Integer, SerializedStubTree, FileContent> stubUpdatingIndex = getStubUpdatingIndex();
205     ReadWriteLock lock = stubUpdatingIndex.getLock();
206
207     for (int attempt = 0; attempt < 2; attempt++) {
208       try {
209         final VfsAwareIndexStorage<K, Void> storage = FileBasedIndex.USE_IN_MEMORY_INDEX
210                                                       ? new InMemoryIndexStorage<>()
211                                                       : new VfsAwareMapIndexStorage<>(
212           IndexInfrastructure.getStorageFile(indexKey).toPath(),
213           wrappedExtension.getKeyDescriptor(),
214           wrappedExtension.getValueExternalizer(),
215           wrappedExtension.getCacheSize(),
216           wrappedExtension.keyIsUniqueForIndexedFile(),
217           wrappedExtension.traceKeyHashToVirtualFileMapping()
218         );
219         final TransientChangesIndexStorage<K, Void> memStorage = new TransientChangesIndexStorage<>(storage, indexKey);
220         UpdatableIndex<K, Void, FileContent> index = new VfsAwareMapReduceIndex<>(wrappedExtension, memStorage, null, null, null, lock);
221
222         for (FileBasedIndexInfrastructureExtension infrastructureExtension : FileBasedIndexInfrastructureExtension.EP_NAME.getExtensionList()) {
223           UpdatableIndex<K, Void, FileContent> intermediateIndex = infrastructureExtension.combineIndex(wrappedExtension, index);
224           if (intermediateIndex != null) {
225             index = intermediateIndex;
226           }
227         }
228
229         synchronized (state) {
230           state.myIndices.put(indexKey, index);
231         }
232         break;
233       }
234       catch (IOException e) {
235         registrationResultSink.setIndexVersionDiff(indexKey, new IndexingStamp.IndexVersionDiff.CorruptedRebuild(version));
236         onExceptionInstantiatingIndex(indexKey, version, indexRootDir, e);
237       }
238       catch (RuntimeException e) {
239         Throwable cause = FileBasedIndexImpl.getCauseToRebuildIndex(e);
240         if (cause == null) throw e;
241         onExceptionInstantiatingIndex(indexKey, version, indexRootDir, e);
242       }
243     }
244   }
245
246   private static <K> void onExceptionInstantiatingIndex(@NotNull StubIndexKey<K, ?> indexKey,
247                                                         int version,
248                                                         @NotNull File indexRootDir,
249                                                         @NotNull Exception e) throws IOException {
250     LOG.info(e);
251     FileUtil.deleteWithRenaming(indexRootDir);
252     IndexingStamp.rewriteVersion(indexKey, version); // todo snapshots indices
253   }
254
255   public long getIndexModificationStamp(@NotNull StubIndexKey<?, ?> indexId, @NotNull Project project) {
256     UpdatableIndex<?, Void, FileContent> index = getAsyncState().myIndices.get(indexId);
257     if (index != null) {
258       FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, GlobalSearchScope.allScope(project));
259       return index.getModificationStamp();
260     }
261     return -1;
262   }
263
264   public void flush() throws StorageException {
265     if (!myInitialized) {
266       return;
267     }
268     for (UpdatableIndex<?, Void, FileContent> index : getAsyncState().myIndices.values()) {
269       index.flush();
270     }
271   }
272
273
274   @Override
275   public <Key, Psi extends PsiElement> boolean processElements(final @NotNull StubIndexKey<Key, Psi> indexKey,
276                                                                final @NotNull Key key,
277                                                                final @NotNull Project project,
278                                                                final @Nullable GlobalSearchScope scope,
279                                                                @Nullable IdFilter idFilter,
280                                                                final @NotNull Class<Psi> requiredClass,
281                                                                final @NotNull Processor<? super Psi> processor) {
282     boolean dumb = DumbService.isDumb(project);
283     if (dumb) {
284       DumbModeAccessType accessType = FileBasedIndex.getInstance().getCurrentDumbModeAccessType();
285       if (accessType == DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) {
286         throw new AssertionError("raw index data access is not available for StubIndex");
287       }
288     }
289
290     PairProcessor<VirtualFile, StubIdList> stubProcessor = (file, list) ->
291       myStubProcessingHelper.processStubsInFile(project, file, list, processor, scope, requiredClass);
292
293     if (!ModelBranchImpl.processModifiedFilesInScope(scope != null ? scope : new EverythingGlobalScope(project),
294                                                      file -> processInMemoryStubs(indexKey, key, project, stubProcessor, file))) {
295       return false;
296     }
297
298     IdIterator ids = getContainingIds(indexKey, key, project, idFilter, scope);
299     PersistentFS fs = PersistentFS.getInstance();
300     IntPredicate accessibleFileFilter = ((FileBasedIndexEx)FileBasedIndex.getInstance()).getAccessibleFileIdFilter(project);
301     // already ensured up-to-date in getContainingIds() method
302     try {
303       while (ids.hasNext()) {
304         int id = ids.next();
305         ProgressManager.checkCanceled();
306         if (!accessibleFileFilter.test(id)) {
307           continue;
308         }
309         VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id);
310         if (file == null) {
311           continue;
312         }
313
314         List<VirtualFile> filesInScope = scope != null ? FileBasedIndexEx.filesInScopeWithBranches(scope, file) : Collections.singletonList(file);
315         if (filesInScope.isEmpty()) {
316           continue;
317         }
318
319         StubIdList list = myCachedStubIds.get(indexKey).getValue().computeIfAbsent(new CompositeKey<>(key, id), __ ->
320           myStubProcessingHelper.retrieveStubIdList(indexKey, key, file, project)
321         );
322         if (list == null) {
323           LOG.error("StubUpdatingIndex & " + indexKey + " stub index mismatch. No stub index key is present");
324           continue;
325         }
326         for (VirtualFile eachFile : filesInScope) {
327           if (!stubProcessor.process(eachFile, list)) {
328             return false;
329           }
330         }
331       }
332     }
333     catch (RuntimeException e) {
334       final Throwable cause = FileBasedIndexImpl.getCauseToRebuildIndex(e);
335       if (cause != null) {
336         forceRebuild(cause);
337       }
338       else {
339         throw e;
340       }
341     } finally {
342       wipeProblematicFileIdsForParticularKeyAndStubIndex(indexKey, key);
343     }
344     return true;
345   }
346
347   private static <Key, Psi extends PsiElement> boolean processInMemoryStubs(StubIndexKey<Key, Psi> indexKey,
348                                                                             Key key,
349                                                                             Project project,
350                                                                             PairProcessor<VirtualFile, StubIdList> stubProcessor,
351                                                                             VirtualFile file) {
352     Map<Integer, SerializedStubTree> data = FileBasedIndex.getInstance().getFileData(StubUpdatingIndex.INDEX_ID, file, project);
353     if (data.size() == 1) {
354       try {
355         StubIdList list = data.values().iterator().next().restoreIndexedStubs(indexKey, key);
356         if (list != null) {
357           return stubProcessor.process(file, list);
358         }
359       }
360       catch (IOException e) {
361         throw new RuntimeException(e);
362       }
363     }
364     return true;
365   }
366
367   @SuppressWarnings("unchecked")
368   private <Key> UpdatableIndex<Key, Void, FileContent> getIndex(@NotNull StubIndexKey<Key, ?> indexKey) {
369     return (UpdatableIndex<Key, Void, FileContent>)getAsyncState().myIndices.get(indexKey);
370   }
371
372   // Self repair for IDEA-181227, caused by (yet) unknown file event processing problem in indices
373   // FileBasedIndex.requestReindex doesn't handle the situation properly because update requires old data that was lost
374   private <Key> void wipeProblematicFileIdsForParticularKeyAndStubIndex(@NotNull StubIndexKey<Key, ?> indexKey,
375                                                                         @NotNull Key key) {
376     Set<VirtualFile> filesWithProblems = myStubProcessingHelper.takeAccumulatedFilesWithIndexProblems();
377
378     if (filesWithProblems != null) {
379       LOG.info("data for " + indexKey.getName() + " will be wiped for a some files because of internal stub processing error");
380       ((FileBasedIndexImpl)FileBasedIndex.getInstance()).runCleanupAction(() -> {
381         Lock writeLock = getIndex(indexKey).getLock().writeLock();
382         boolean locked = writeLock.tryLock();
383         if (!locked) return; // nested indices invocation, can not cleanup without deadlock
384         try {
385           for (VirtualFile file : filesWithProblems) {
386             updateIndex(indexKey,
387                         FileBasedIndex.getFileId(file),
388                         Collections.singleton(key),
389                         Collections.emptySet());
390           }
391         }
392         finally {
393           writeLock.unlock();
394         }
395       });
396     }
397   }
398
399   @Override
400   public void forceRebuild(@NotNull Throwable e) {
401     FileBasedIndex.getInstance().scheduleRebuild(StubUpdatingIndex.INDEX_ID, e);
402   }
403
404   private static void requestRebuild() {
405     FileBasedIndex.getInstance().requestRebuild(StubUpdatingIndex.INDEX_ID);
406   }
407
408   @Override
409   public @NotNull <K> Collection<K> getAllKeys(@SuppressWarnings("BoundedWildcard") @NotNull StubIndexKey<K, ?> indexKey, @NotNull Project project) {
410     Set<K> allKeys = new HashSet<>();
411     processAllKeys(indexKey, project, Processors.cancelableCollectProcessor(allKeys));
412     return allKeys;
413   }
414
415   @Override
416   public <K> boolean processAllKeys(@NotNull StubIndexKey<K, ?> indexKey,
417                                     @NotNull Processor<? super K> processor,
418                                     @NotNull GlobalSearchScope scope,
419                                     @Nullable IdFilter idFilter) {
420     final UpdatableIndex<K, Void, FileContent> index = getIndex(indexKey); // wait for initialization to finish
421     if (index == null ||
422         !((FileBasedIndexEx)FileBasedIndex.getInstance()).ensureUpToDate(StubUpdatingIndex.INDEX_ID, scope.getProject(), scope, null)) {
423       return true;
424     }
425
426     try {
427       return myAccessValidator.validate(StubUpdatingIndex.INDEX_ID, ()->FileBasedIndexImpl.disableUpToDateCheckIn(()->
428         index.processAllKeys(processor, scope, idFilter)));
429     }
430     catch (StorageException e) {
431       forceRebuild(e);
432     }
433     catch (RuntimeException e) {
434       final Throwable cause = e.getCause();
435       if (cause instanceof IOException || cause instanceof StorageException) {
436         forceRebuild(e);
437       }
438       throw e;
439     }
440     return true;
441   }
442
443   @Override
444   public @NotNull <Key> IdIterator getContainingIds(@NotNull StubIndexKey<Key, ?> indexKey,
445                                                     @NotNull Key dataKey,
446                                                     final @NotNull Project project,
447                                                     final @Nullable GlobalSearchScope scope) {
448     return getContainingIds(indexKey, dataKey, project, null, scope);
449   }
450
451   private @NotNull <Key> IdIterator getContainingIds(@NotNull StubIndexKey<Key, ?> indexKey,
452                                                      @NotNull Key dataKey,
453                                                      final @NotNull Project project,
454                                                      @Nullable IdFilter idFilter,
455                                                      final @Nullable GlobalSearchScope scope) {
456     final FileBasedIndexEx fileBasedIndex = (FileBasedIndexEx)FileBasedIndex.getInstance();
457     ID<Integer, SerializedStubTree> stubUpdatingIndexId = StubUpdatingIndex.INDEX_ID;
458     final UpdatableIndex<Key, Void, FileContent> index = getIndex(indexKey);   // wait for initialization to finish
459     if (index == null || !fileBasedIndex.ensureUpToDate(stubUpdatingIndexId, project, scope, null)) return IdIterator.EMPTY;
460
461     if (idFilter == null) {
462       idFilter = ((FileBasedIndexEx)FileBasedIndex.getInstance()).projectIndexableFiles(project);
463     }
464
465     UpdatableIndex<Integer, SerializedStubTree, FileContent> stubUpdatingIndex = fileBasedIndex.getIndex(stubUpdatingIndexId);
466
467     try {
468       IntArrayList result = new IntArrayList();
469       IdFilter finalIdFilter = idFilter;
470       myAccessValidator.validate(stubUpdatingIndexId, ()-> {
471         // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index
472         //noinspection Convert2Lambda (workaround for JBR crash, JBR-2349)
473         return FileBasedIndexImpl.disableUpToDateCheckIn(() -> ConcurrencyUtil.withLock(stubUpdatingIndex.getLock().readLock(), () ->
474           index.getData(dataKey).forEach(new ValueContainer.ContainerAction<Void>() {
475             @Override
476             public boolean perform(int id, Void value) {
477               if (finalIdFilter == null || finalIdFilter.containsFileId(id)) {
478                 result.add(id);
479               }
480               return true;
481             }
482           })
483         ));
484       });
485       return new IdIterator() {
486         int cursor;
487
488         @Override
489         public boolean hasNext() {
490           return cursor < result.size();
491         }
492
493         @Override
494         public int next() {
495           return result.getInt(cursor++);
496         }
497
498         @Override
499         public int size() {
500           return result.size();
501         }
502       };
503     }
504     catch (StorageException e) {
505       forceRebuild(e);
506     }
507     catch (RuntimeException e) {
508       final Throwable cause = FileBasedIndexImpl.getCauseToRebuildIndex(e);
509       if (cause != null) {
510         forceRebuild(cause);
511       }
512       else {
513         throw e;
514       }
515     }
516
517     return IdIterator.EMPTY;
518   }
519
520   void initializeStubIndexes() {
521     assert !myInitialized;
522     // ensure that FileBasedIndex task "FileIndexDataInitialization" submitted first
523     FileBasedIndex.getInstance();
524     myStateFuture = new CompletableFuture<>();
525     Future<AsyncState> future = IndexInfrastructure.submitGenesisTask(new StubIndexInitialization());
526
527     if (!IndexInfrastructure.ourDoAsyncIndicesInitialization) {
528       try {
529         future.get();
530       }
531       catch (Throwable t) {
532         LOG.error(t);
533       }
534     }
535   }
536
537   public void dispose() {
538     try {
539       for (UpdatableIndex<?, ?, ?> index : getAsyncState().myIndices.values()) {
540         index.dispose();
541       }
542     } finally {
543       clearState();
544     }
545   }
546
547   private void clearState() {
548     StubIndexKeyDescriptorCache.INSTANCE.clear();
549     ((SerializationManagerImpl)SerializationManager.getInstance()).dropSerializerData();
550     myCachedStubIds.clear();
551     myStateFuture = null;
552     myState = null;
553     myInitialized = false;
554     LOG.info("StubIndexExtension-s were unloaded");
555   }
556
557   void setDataBufferingEnabled(final boolean enabled) {
558     for (UpdatableIndex<?, ?, ?> index : getAsyncState().myIndices.values()) {
559       index.setBufferingEnabled(enabled);
560     }
561   }
562
563   void cleanupMemoryStorage() {
564     UpdatableIndex<Integer, SerializedStubTree, FileContent> stubUpdatingIndex = getStubUpdatingIndex();
565     stubUpdatingIndex.getLock().writeLock().lock();
566
567     try {
568       for (UpdatableIndex<?, ?, ?> index : getAsyncState().myIndices.values()) {
569         index.cleanupMemoryStorage();
570       }
571     }
572     finally {
573       stubUpdatingIndex.getLock().writeLock().unlock();
574     }
575   }
576
577   void clearAllIndices() {
578     if (!myInitialized) return;
579     for (UpdatableIndex<?, ?, ?> index : getAsyncState().myIndices.values()) {
580       try {
581         index.clear();
582       }
583       catch (StorageException e) {
584         LOG.error(e);
585         throw new RuntimeException(e);
586       }
587     }
588   }
589
590   <K> void removeTransientDataForFile(@NotNull StubIndexKey<K, ?> key, int inputId, @NotNull Collection<? extends K> keys) {
591     UpdatableIndex<K, Void, FileContent> index = getIndex(key);
592     index.removeTransientDataForKeys(inputId, keys);
593   }
594
595   public <K> void updateIndex(@NotNull StubIndexKey<K, ?> stubIndexKey,
596                               int fileId,
597                               @NotNull Set<K> oldKeys,
598                               @NotNull Set<K> newKeys) {
599     ProgressManager.getInstance().executeNonCancelableSection(() -> {
600       try {
601         if (FileBasedIndexImpl.DO_TRACE_STUB_INDEX_UPDATE) {
602           LOG.info("stub index '" + stubIndexKey + "' update: " + fileId +
603                    " old = " + Arrays.toString(oldKeys.toArray()) +
604                    " new  = " + Arrays.toString(newKeys.toArray()) +
605                    " updated_id = " + System.identityHashCode(newKeys));
606         }
607         final UpdatableIndex<K, Void, FileContent> index = getIndex(stubIndexKey);
608         if (index == null) return;
609         index.updateWithMap(new AbstractUpdateData<>(fileId) {
610           @Override
611           protected boolean iterateKeys(@NotNull KeyValueUpdateProcessor<? super K, ? super Void> addProcessor,
612                                         @NotNull KeyValueUpdateProcessor<? super K, ? super Void> updateProcessor,
613                                         @NotNull RemovedKeyProcessor<? super K> removeProcessor) throws StorageException {
614
615             if (FileBasedIndexImpl.DO_TRACE_STUB_INDEX_UPDATE) {
616               LOG.info("iterating keys updated_id = " + System.identityHashCode(newKeys));
617             }
618
619             boolean modified = false;
620
621             for (K oldKey : oldKeys) {
622               if (!newKeys.contains(oldKey)) {
623                 removeProcessor.process(oldKey, fileId);
624                 if (!modified) modified = true;
625               }
626             }
627
628             for (K oldKey : newKeys) {
629               if (!oldKeys.contains(oldKey)) {
630                 addProcessor.process(oldKey, null, fileId);
631                 if (!modified) modified = true;
632               }
633             }
634
635             if (FileBasedIndexImpl.DO_TRACE_STUB_INDEX_UPDATE) {
636               LOG.info("keys iteration finished updated_id = " + System.identityHashCode(newKeys) + "; modified = " + modified);
637             }
638
639             return modified;
640           }
641         });
642       }
643       catch (StorageException e) {
644         LOG.info(e);
645         requestRebuild();
646       }
647     });
648
649   }
650
651   private class StubIndexInitialization extends IndexInfrastructure.DataInitialization<AsyncState> {
652     private final AsyncState state = new AsyncState();
653     private final IndexVersionRegistrationSink indicesRegistrationSink = new IndexVersionRegistrationSink();
654
655     @Override
656     protected void prepare() {
657       Iterator<StubIndexExtension<?, ?>> extensionsIterator =
658         IndexInfrastructure.hasIndices() ?
659           ((ExtensionPointImpl<StubIndexExtension<?, ?>>)StubIndexExtension.EP_NAME.getPoint()).iterator() :
660           Collections.emptyIterator();
661
662       boolean forceClean = Boolean.TRUE == ourForcedClean.getAndSet(Boolean.FALSE);
663       while(extensionsIterator.hasNext()) {
664         StubIndexExtension<?, ?> extension = extensionsIterator.next();
665         if (extension == null) break;
666         extension.getKey(); // initialize stub index keys
667
668         addNestedInitializationTask(() -> registerIndexer(extension, forceClean, state, indicesRegistrationSink));
669       }
670     }
671
672     @Override
673     protected void onThrowable(@NotNull Throwable t) {
674       LOG.error(t);
675     }
676
677     @Override
678     protected AsyncState finish() {
679       indicesRegistrationSink.logChangedAndFullyBuiltIndices(LOG, "Following stub indices will be updated:",
680                                                              "Following stub indices will be built:");
681
682       if (indicesRegistrationSink.hasChangedIndexes()) {
683         final Throwable e = new Throwable(indicesRegistrationSink.changedIndices());
684         // avoid direct forceRebuild as it produces dependency cycle (IDEA-105485)
685         AppUIExecutor.onWriteThread(ModalityState.NON_MODAL).later().submit(() -> forceRebuild(e));
686       }
687
688       myInitialized = true;
689       myStateFuture.complete(state);
690       return state;
691     }
692   }
693
694   static UpdatableIndex<Integer, SerializedStubTree, FileContent> getStubUpdatingIndex() {
695     return ((FileBasedIndexEx)FileBasedIndex.getInstance()).getIndex(StubUpdatingIndex.INDEX_ID);
696   }
697
698   private static final class CompositeKey<K> {
699     private final K key;
700     private final int fileId;
701
702     private CompositeKey(K key, int id) {
703       this.key = key;
704       fileId = id;
705     }
706
707     @Override
708     public boolean equals(Object o) {
709       if (this == o) return true;
710       if (o == null || getClass() != o.getClass()) return false;
711       CompositeKey<?> key1 = (CompositeKey<?>)o;
712       return fileId == key1.fileId && Objects.equals(key, key1.key);
713     }
714
715     @Override
716     public int hashCode() {
717       return Objects.hash(key, fileId);
718     }
719   }
720
721   @TestOnly
722   public boolean areAllProblemsProcessedInTheCurrentThread() {
723     return myStubProcessingHelper.areAllProblemsProcessedInTheCurrentThread();
724   }
725 }