ea2fb70f244946e9ee5c99f7583fc93299dfea37
[idea/community.git] / platform / lang-impl / src / com / intellij / psi / stubs / StubUpdatingIndex.java
1 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
2 package com.intellij.psi.stubs;
3
4 import com.intellij.index.PrebuiltIndexProvider;
5 import com.intellij.lang.Language;
6 import com.intellij.lang.LanguageParserDefinitions;
7 import com.intellij.lang.ParserDefinition;
8 import com.intellij.openapi.diagnostic.Attachment;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.diagnostic.RuntimeExceptionWithAttachments;
11 import com.intellij.openapi.extensions.ExtensionPoint;
12 import com.intellij.openapi.fileTypes.FileType;
13 import com.intellij.openapi.fileTypes.FileTypeRegistry;
14 import com.intellij.openapi.fileTypes.LanguageFileType;
15 import com.intellij.openapi.progress.ProcessCanceledException;
16 import com.intellij.openapi.progress.ProgressManager;
17 import com.intellij.openapi.project.Project;
18 import com.intellij.openapi.project.ProjectUtil;
19 import com.intellij.openapi.util.Computable;
20 import com.intellij.openapi.util.KeyedExtensionCollector;
21 import com.intellij.openapi.vfs.VirtualFile;
22 import com.intellij.openapi.vfs.newvfs.FileAttribute;
23 import com.intellij.openapi.vfs.newvfs.persistent.FSRecords;
24 import com.intellij.psi.tree.IFileElementType;
25 import com.intellij.psi.tree.IStubFileElementType;
26 import com.intellij.util.BitUtil;
27 import com.intellij.util.KeyedLazyInstance;
28 import com.intellij.util.SystemProperties;
29 import com.intellij.util.indexing.*;
30 import com.intellij.util.indexing.impl.IndexDebugProperties;
31 import com.intellij.util.indexing.impl.IndexStorage;
32 import com.intellij.util.indexing.impl.forward.EmptyForwardIndex;
33 import com.intellij.util.indexing.impl.forward.ForwardIndex;
34 import com.intellij.util.indexing.impl.forward.ForwardIndexAccessor;
35 import com.intellij.util.indexing.impl.forward.IntMapForwardIndex;
36 import com.intellij.util.indexing.impl.storage.TransientChangesIndexStorage;
37 import com.intellij.util.indexing.impl.storage.VfsAwareMapReduceIndex;
38 import com.intellij.util.indexing.snapshot.SnapshotInputMappings;
39 import com.intellij.util.io.*;
40 import org.jetbrains.annotations.NotNull;
41 import org.jetbrains.annotations.Nullable;
42
43 import java.io.DataInputStream;
44 import java.io.DataOutputStream;
45 import java.io.IOException;
46 import java.util.*;
47 import java.util.stream.Stream;
48
49 public final class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<SerializedStubTree>
50   implements CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree> {
51   static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class);
52   public static final boolean USE_SNAPSHOT_MAPPINGS = SystemProperties.is("stubs.use.snapshot.mappings");
53
54   private static final int VERSION = 45 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0);
55
56   // todo remove once we don't need this for stub-ast mismatch debug info
57   private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true);
58
59   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
60
61   private static final FileBasedIndex.InputFilter INPUT_FILTER = StubUpdatingIndex::canHaveStub;
62
63   @NotNull
64   private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
65
66   @NotNull
67   private final SerializationManagerEx mySerializationManager;
68
69   public StubUpdatingIndex() {
70     this(StubForwardIndexExternalizer.getIdeUsedExternalizer(), SerializationManagerEx.getInstanceEx());
71   }
72
73   public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer,
74                            @NotNull SerializationManagerEx serializationManager) {
75     myStubIndexesExternalizer = stubIndexesExternalizer;
76     mySerializationManager = serializationManager;
77   }
78
79   @Override
80   public boolean hasSnapshotMapping() {
81     return USE_SNAPSHOT_MAPPINGS;
82   }
83
84   public static boolean canHaveStub(@NotNull VirtualFile file) {
85     Project project = ProjectUtil.guessProjectForFile(file);
86     FileType fileType = SubstitutedFileType.substituteFileType(file, file.getFileType(), project);
87     if (fileType instanceof LanguageFileType) {
88       final Language l = ((LanguageFileType)fileType).getLanguage();
89       final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
90       if (parserDefinition == null) {
91         return false;
92       }
93
94       final IFileElementType elementType = parserDefinition.getFileNodeType();
95       if (elementType instanceof IStubFileElementType && ((IStubFileElementType)elementType).shouldBuildStubFor(file)) {
96         return true;
97       }
98     }
99     final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
100     return builder != null && builder.acceptsFile(file);
101   }
102
103   @NotNull
104   @Override
105   public ID<Integer, SerializedStubTree> getName() {
106     return INDEX_ID;
107   }
108
109   @NotNull
110   @Override
111   public SingleEntryIndexer<SerializedStubTree> getIndexer() {
112     return new SingleEntryCompositeIndexer<SerializedStubTree, StubBuilderType, String>(false) {
113       @Override
114       public boolean requiresContentForSubIndexerEvaluation(@NotNull IndexedFile file) {
115         return StubTreeBuilder.requiresContentToFindBuilder(file.getFileType());
116       }
117
118       @Nullable
119       @Override
120       public StubBuilderType calculateSubIndexer(@NotNull IndexedFile file) {
121         return StubTreeBuilder.getStubBuilderType(file, true);
122       }
123
124       @NotNull
125       @Override
126       public String getSubIndexerVersion(@NotNull StubBuilderType type) {
127         return type.getVersion();
128       }
129
130       @NotNull
131       @Override
132       public KeyDescriptor<String> getSubIndexerVersionDescriptor() {
133         return EnumeratorStringDescriptor.INSTANCE;
134       }
135
136       @Override
137       protected @Nullable SerializedStubTree computeValue(@NotNull FileContent inputData) {
138         StubBuilderType subIndexerType = calculateSubIndexer(inputData);
139         if (subIndexerType == null) {
140           if (LOG.isDebugEnabled()) {
141             LOG.debug("Stub builder not found for " + inputData.getFile() + ", " + calculateIndexingStamp(inputData));
142           }
143           return null;
144         }
145         return computeValue(inputData, Objects.requireNonNull(subIndexerType));
146       }
147
148       @Override
149       @Nullable
150       protected SerializedStubTree computeValue(@NotNull final FileContent inputData, @NotNull StubBuilderType type) {
151         try {
152           SerializedStubTree prebuiltTree = findPrebuiltSerializedStubTree(inputData);
153           if (prebuiltTree != null) {
154             prebuiltTree = prebuiltTree.reSerialize(mySerializationManager, myStubIndexesExternalizer);
155             if (PrebuiltIndexProvider.DEBUG_PREBUILT_INDICES) {
156               assertPrebuiltStubTreeMatchesActualTree(prebuiltTree, inputData, type);
157             }
158             return prebuiltTree;
159           }
160         } catch (ProcessCanceledException pce) {
161           throw pce;
162         } catch (Exception e) {
163           LOG.error("Error while indexing: " + inputData.getFileName() + " using prebuilt stub index", e);
164         }
165
166         try {
167           Stub stub = StubTreeBuilder.buildStubTree(inputData, type);
168           if (stub == null) {
169             if (LOG.isDebugEnabled()) {
170               LOG.debug("No stub present for " + inputData.getFile() + ", " + calculateIndexingStamp(inputData));
171             }
172             return null;
173           }
174           SerializedStubTree serializedStubTree = SerializedStubTree.serializeStub(stub, mySerializationManager, myStubIndexesExternalizer);
175           if (IndexDebugProperties.DEBUG) {
176             assertDeserializedStubMatchesOriginalStub(serializedStubTree, stub);
177           }
178           if (LOG.isDebugEnabled()) {
179             LOG.debug("Indexing stubs for " + inputData.getFile() + ", " + indexingStampInfo);
180           }
181           return serializedStubTree;
182         }
183         catch (ProcessCanceledException pce) {
184           throw pce;
185         }
186         catch (Throwable t) {
187           LOG.error("Error indexing:" + inputData.getFile(), t);
188           return null;
189         }
190       }
191     };
192   }
193
194   @Nullable
195   static SerializedStubTree findPrebuiltSerializedStubTree(@NotNull FileContent fileContent) {
196     if (!PrebuiltIndexProvider.USE_PREBUILT_INDEX) {
197       return null;
198     }
199     PrebuiltStubsProvider prebuiltStubsProvider = PrebuiltStubsKt.getPrebuiltStubsProvider().forFileType(fileContent.getFileType());
200     if (prebuiltStubsProvider == null) {
201       return null;
202     }
203     return prebuiltStubsProvider.findStub(fileContent);
204   }
205
206   private static void assertDeserializedStubMatchesOriginalStub(@NotNull SerializedStubTree stubTree,
207                                                                 @NotNull Stub originalStub) {
208     Stub deserializedStub;
209     try {
210       deserializedStub = stubTree.getStub();
211     }
212     catch (SerializerNotFoundException e) {
213       throw new RuntimeException("Failed to deserialize stub tree", e);
214     }
215     assertStubsAreSimilar(originalStub, deserializedStub);
216   }
217
218   private static void assertStubsAreSimilar(@NotNull Stub stub, @NotNull Stub stub2) {
219     assert stub.getStubType() == stub2.getStubType() : stub.getStubType() + "!=" + stub2.getStubType();
220     List<? extends Stub> stubs = stub.getChildrenStubs();
221     List<? extends Stub> stubs2 = stub2.getChildrenStubs();
222     assert stubs.size() == stubs2.size() : stub.getStubType() + ": " + stubs.size() + "!=" + stubs2.size();
223     for (int i = 0, len = stubs.size(); i < len; ++i) {
224       assertStubsAreSimilar(stubs.get(i), stubs2.get(i));
225     }
226   }
227
228   private void assertPrebuiltStubTreeMatchesActualTree(@NotNull SerializedStubTree prebuiltStubTree,
229                                                        @NotNull FileContent fileContent,
230                                                        @NotNull StubBuilderType type) {
231     try {
232       Stub stub = StubTreeBuilder.buildStubTree(fileContent, type);
233       if (stub == null) {
234         return;
235       }
236       SerializedStubTree actualTree = SerializedStubTree.serializeStub(stub, mySerializationManager, myStubIndexesExternalizer);
237       if (!IndexDataComparer.INSTANCE.areStubTreesTheSame(actualTree, prebuiltStubTree)) {
238         throw new RuntimeExceptionWithAttachments(
239           "Prebuilt stub tree does not match actual stub tree",
240           new Attachment("actual-stub-tree.txt", IndexDataPresenter.INSTANCE.getPresentableSerializedStubTree(actualTree)),
241           new Attachment("prebuilt-stub-tree.txt", IndexDataPresenter.INSTANCE.getPresentableSerializedStubTree(prebuiltStubTree))
242         );
243       }
244     }
245     catch (IOException e) {
246       throw new RuntimeException(e);
247     }
248   }
249
250   private static final byte IS_BINARY_MASK = 1;
251   private static final byte BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK = 1 << 1;
252   @NotNull
253   private static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) {
254     VirtualFile file = content.getFile();
255     boolean isBinary = file.getFileType().isBinary();
256     int contentLength = isBinary ? -1 : content.getPsiFile().getTextLength();
257     long byteLength = file.getLength();
258
259     return new IndexingStampInfo(file.getTimeStamp(), byteLength, contentLength, isBinary);
260   }
261
262   static void saveIndexingStampInfo(@Nullable IndexingStampInfo indexingStampInfo, int fileId) {
263     try (DataOutputStream stream = FSRecords.writeAttribute(fileId, INDEXED_STAMP)) {
264       if (indexingStampInfo == null) return;
265       DataInputOutputUtil.writeTIME(stream, indexingStampInfo.indexingFileStamp);
266       DataInputOutputUtil.writeLONG(stream, indexingStampInfo.indexingByteLength);
267
268       boolean lengthsAreTheSame = indexingStampInfo.indexingCharLength == indexingStampInfo.indexingByteLength;
269       byte flags = 0;
270       flags = BitUtil.set(flags, IS_BINARY_MASK, indexingStampInfo.isBinary);
271       flags = BitUtil.set(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK, lengthsAreTheSame);
272       stream.writeByte(flags);
273
274       if (!lengthsAreTheSame && !indexingStampInfo.isBinary) {
275         DataInputOutputUtil.writeINT(stream, indexingStampInfo.indexingCharLength);
276       }
277     }
278     catch (IOException e) {
279       LOG.error(e);
280     }
281   }
282
283   @Nullable
284   static IndexingStampInfo readSavedIndexingStampInfo(@NotNull VirtualFile file) {
285     try (DataInputStream stream = INDEXED_STAMP.readAttribute(file)) {
286       if (stream == null || stream.available() <= 0) {
287         return null;
288       }
289       long stamp = DataInputOutputUtil.readTIME(stream);
290       long byteLength = DataInputOutputUtil.readLONG(stream);
291
292       byte flags = stream.readByte();
293       boolean isBinary = BitUtil.isSet(flags, IS_BINARY_MASK);
294       boolean readOnlyOneLength = BitUtil.isSet(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK);
295
296       int charLength;
297       if (isBinary) {
298         charLength = -1;
299       }
300       else if (readOnlyOneLength) {
301         charLength = (int)byteLength;
302       }
303       else {
304         charLength = DataInputOutputUtil.readINT(stream);
305       }
306       return new IndexingStampInfo(stamp, byteLength, charLength, isBinary);
307     }
308     catch (IOException e) {
309       LOG.error(e);
310       return null;
311     }
312   }
313
314   @NotNull
315   @Override
316   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
317     ensureSerializationManagerInitialized(mySerializationManager);
318     return new SerializedStubTreeDataExternalizer(mySerializationManager, myStubIndexesExternalizer);
319   }
320
321   @NotNull
322   @Override
323   public FileBasedIndex.InputFilter getInputFilter() {
324     return INPUT_FILTER;
325   }
326
327   @Override
328   public int getVersion() {
329     return VERSION;
330   }
331
332   @Override
333   public void handleInitializationError(@NotNull Throwable e) {
334     ((StubIndexImpl)StubIndex.getInstance()).initializationFailed(e);
335   }
336
337   @NotNull
338   @Override
339   public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(@NotNull final FileBasedIndexExtension<Integer, SerializedStubTree> extension,
340                                                                                             @NotNull IndexStorage<Integer, SerializedStubTree> storage)
341     throws StorageException, IOException {
342     ((StubIndexImpl)StubIndex.getInstance()).initializeStubIndexes();
343     if (storage instanceof TransientChangesIndexStorage) {
344       final TransientChangesIndexStorage<Integer, SerializedStubTree>
345         memStorage = (TransientChangesIndexStorage<Integer, SerializedStubTree>)storage;
346       memStorage.addBufferingStateListener(new TransientChangesIndexStorage.BufferingStateListener() {
347         @Override
348         public void bufferingStateChanged(final boolean newState) {
349           ((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState);
350         }
351
352         @Override
353         public void memoryStorageCleared() {
354           ((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage();
355         }
356       });
357     }
358     checkNameStorage();
359
360     boolean hasSnapshotMapping = VfsAwareMapReduceIndex.hasSnapshotMapping(this);
361     StubUpdatingForwardIndexAccessor stubForwardIndexAccessor = new StubUpdatingForwardIndexAccessor(extension);
362
363     SnapshotInputMappings<Integer, SerializedStubTree> snapshotInputMappings =
364       hasSnapshotMapping
365       ? new SnapshotInputMappings<>(this, stubForwardIndexAccessor)
366       : null;
367
368     ForwardIndex forwardIndex =
369       hasSnapshotMapping
370       ? new IntMapForwardIndex(snapshotInputMappings.getInputIndexStorageFile(), true)
371       : new EmptyForwardIndex();
372
373     ForwardIndexAccessor<Integer, SerializedStubTree> accessor =
374       hasSnapshotMapping
375       ? snapshotInputMappings.getForwardIndexAccessor()
376       : stubForwardIndexAccessor;
377
378     return new MyIndex(extension, storage, forwardIndex, accessor, snapshotInputMappings);
379   }
380
381   private void checkNameStorage() throws StorageException {
382     if (mySerializationManager.isNameStorageCorrupted()) {
383       mySerializationManager.repairNameStorage();
384       throw new StorageException("NameStorage for stubs serialization has been corrupted");
385     }
386   }
387
388   private static class MyIndex extends VfsAwareMapReduceIndex<Integer, SerializedStubTree> {
389     private StubIndexImpl myStubIndex;
390     @Nullable
391     private final CompositeBinaryBuilderMap myCompositeBinaryBuilderMap = FileBasedIndex.USE_IN_MEMORY_INDEX ? null : new CompositeBinaryBuilderMap();
392
393     MyIndex(@NotNull FileBasedIndexExtension<Integer, SerializedStubTree> extension,
394             @NotNull IndexStorage<Integer, SerializedStubTree> storage,
395             @Nullable ForwardIndex forwardIndex,
396             @Nullable ForwardIndexAccessor<Integer, SerializedStubTree> forwardIndexAccessor,
397             @Nullable SnapshotInputMappings<Integer, SerializedStubTree> snapshotInputMappings) throws IOException {
398       super(extension, storage, forwardIndex, forwardIndexAccessor, snapshotInputMappings, null);
399     }
400
401     @Override
402     protected void doFlush() throws IOException, StorageException {
403       final StubIndexImpl stubIndex = getStubIndex();
404       try {
405         stubIndex.flush();
406       }
407       finally {
408         super.doFlush();
409       }
410     }
411
412     @NotNull
413     private StubIndexImpl getStubIndex() {
414       StubIndexImpl index = myStubIndex;
415       if (index == null) {
416         myStubIndex = index = (StubIndexImpl)StubIndex.getInstance();
417       }
418       return index;
419     }
420
421     @Override
422     public @NotNull Computable<Boolean> mapInputAndPrepareUpdate(int inputId, @Nullable FileContent content)
423       throws MapInputException, ProcessCanceledException {
424       Computable<Boolean> indexUpdateComputable = super.mapInputAndPrepareUpdate(inputId, content);
425       IndexingStampInfo indexingStampInfo = content == null ? null : calculateIndexingStamp(content);
426
427       return () -> {
428         try {
429           Boolean result = indexUpdateComputable.compute();
430           if (Boolean.TRUE.equals(result)) {
431             saveIndexingStampInfo(indexingStampInfo, inputId);
432           }
433           return result;
434         } catch (ProcessCanceledException e) {
435           LOG.error("ProcessCanceledException is not expected here", e);
436           throw e;
437         }
438       };
439     }
440
441     @Override
442     protected void removeTransientDataForInMemoryKeys(int inputId, @NotNull Map<? extends Integer, ? extends SerializedStubTree> map) {
443       super.removeTransientDataForInMemoryKeys(inputId, map);
444       removeStubIndexKeys(inputId, getStubIndexMaps(map));
445     }
446
447     @Override
448     public void removeTransientDataForKeys(int inputId, @NotNull Collection<? extends Integer> keys) {
449       Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> maps;
450       try {
451         Map<Integer, SerializedStubTree> data = getIndexedFileData(inputId);
452         maps = getStubIndexMaps(data);
453       }
454       catch (StorageException e) {
455         throw new RuntimeException(e);
456       }
457       super.removeTransientDataForKeys(inputId, keys);
458       removeStubIndexKeys(inputId, maps);
459     }
460
461     private static void removeStubIndexKeys(int inputId, @NotNull Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> indexedStubs) {
462       StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
463       for (StubIndexKey key : indexedStubs.keySet()) {
464         stubIndex.removeTransientDataForFile(key, inputId, indexedStubs.get(key).keySet());
465       }
466     }
467
468     @NotNull
469     private static Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> getStubIndexMaps(@NotNull Map<? extends Integer, ? extends SerializedStubTree> data) {
470       if (data.isEmpty()) {
471         return Collections.emptyMap();
472       }
473       SerializedStubTree tree = data.values().iterator().next();
474       return tree == null ? Collections.emptyMap() : tree.getStubIndicesValueMap();
475     }
476
477     @Override
478     protected void doClear() throws StorageException, IOException {
479       final StubIndexImpl stubIndex = StubIndexImpl.getInstanceOrInvalidate();
480       if (stubIndex != null) {
481         stubIndex.clearAllIndices();
482       }
483       super.doClear();
484     }
485
486     @Override
487     protected void doDispose() throws StorageException {
488       try {
489         super.doDispose();
490       }
491       finally {
492         getStubIndex().dispose();
493       }
494     }
495
496     @Override
497     public void setIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
498       super.setIndexedStateForFile(fileId, file);
499       setBinaryBuilderConfiguration(fileId, file);
500     }
501
502     @Override
503     protected boolean isIndexConfigurationUpToDate(int fileId, @NotNull IndexedFile file) {
504       if (myCompositeBinaryBuilderMap == null) return true;
505       try {
506         return myCompositeBinaryBuilderMap.isUpToDateState(fileId, file.getFile());
507       } catch (IOException e) {
508         LOG.error(e);
509         return false;
510       }
511     }
512
513     @Override
514     protected void setIndexConfigurationUpToDate(int fileId, @NotNull IndexedFile file) {
515       setBinaryBuilderConfiguration(fileId, file);
516     }
517
518     private void setBinaryBuilderConfiguration(int fileId, @NotNull IndexedFile file) {
519       if (myCompositeBinaryBuilderMap != null) {
520         try {
521           myCompositeBinaryBuilderMap.persistState(fileId, file.getFile());
522         }
523         catch (IOException e) {
524           LOG.error(e);
525         }
526       }
527     }
528   }
529
530   private static void ensureSerializationManagerInitialized(@NotNull SerializationManagerEx serializationManager) {
531     ProgressManager.getInstance().executeNonCancelableSection(() -> {
532       instantiateElementTypesFromFields();
533       StubIndexEx.initExtensions();
534       serializationManager.initSerializers();
535     });
536   }
537
538   private static void instantiateElementTypesFromFields() {
539     // load stub serializers before usage
540     FileTypeRegistry.getInstance().getRegisteredFileTypes();
541     getExtensions(BinaryFileStubBuilders.INSTANCE).forEach(builder -> {});
542     getExtensions(LanguageParserDefinitions.INSTANCE).forEach(ParserDefinition::getFileNodeType);
543   }
544
545   private static @NotNull <T> Stream<T> getExtensions(@NotNull KeyedExtensionCollector<T, ?> collector) {
546     ExtensionPoint<KeyedLazyInstance<T>> point = collector.getPoint();
547     return point == null ? Stream.empty() : point.extensions().map(KeyedLazyInstance::getInstance);
548   }
549 }