b9d9e96b303b79395f27444f724343887ed0f626
[idea/community.git] / platform / lang-impl / src / com / intellij / psi / stubs / StubUpdatingIndex.java
1 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
2 package com.intellij.psi.stubs;
3
4 import com.intellij.index.PrebuiltIndexProvider;
5 import com.intellij.lang.Language;
6 import com.intellij.lang.LanguageParserDefinitions;
7 import com.intellij.lang.ParserDefinition;
8 import com.intellij.openapi.diagnostic.Attachment;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.diagnostic.RuntimeExceptionWithAttachments;
11 import com.intellij.openapi.extensions.ExtensionPoint;
12 import com.intellij.openapi.fileTypes.FileType;
13 import com.intellij.openapi.fileTypes.FileTypeRegistry;
14 import com.intellij.openapi.fileTypes.LanguageFileType;
15 import com.intellij.openapi.progress.ProcessCanceledException;
16 import com.intellij.openapi.progress.ProgressManager;
17 import com.intellij.openapi.project.Project;
18 import com.intellij.openapi.project.ProjectUtil;
19 import com.intellij.openapi.util.Computable;
20 import com.intellij.openapi.util.KeyedExtensionCollector;
21 import com.intellij.openapi.vfs.VirtualFile;
22 import com.intellij.openapi.vfs.newvfs.FileAttribute;
23 import com.intellij.openapi.vfs.newvfs.persistent.FSRecords;
24 import com.intellij.psi.tree.IFileElementType;
25 import com.intellij.psi.tree.IStubFileElementType;
26 import com.intellij.util.BitUtil;
27 import com.intellij.util.KeyedLazyInstance;
28 import com.intellij.util.SystemProperties;
29 import com.intellij.util.indexing.*;
30 import com.intellij.util.indexing.impl.IndexDebugProperties;
31 import com.intellij.util.indexing.impl.IndexStorage;
32 import com.intellij.util.indexing.impl.forward.EmptyForwardIndex;
33 import com.intellij.util.indexing.impl.forward.ForwardIndex;
34 import com.intellij.util.indexing.impl.forward.ForwardIndexAccessor;
35 import com.intellij.util.indexing.impl.forward.IntMapForwardIndex;
36 import com.intellij.util.indexing.impl.storage.TransientChangesIndexStorage;
37 import com.intellij.util.indexing.impl.storage.VfsAwareMapReduceIndex;
38 import com.intellij.util.indexing.snapshot.SnapshotInputMappings;
39 import com.intellij.util.io.*;
40 import org.jetbrains.annotations.NotNull;
41 import org.jetbrains.annotations.Nullable;
42
43 import java.io.DataInputStream;
44 import java.io.DataOutputStream;
45 import java.io.IOException;
46 import java.util.Collection;
47 import java.util.Collections;
48 import java.util.List;
49 import java.util.Map;
50 import java.util.stream.Stream;
51
52 public final class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<SerializedStubTree>
53   implements CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree> {
54   static final Logger LOG = Logger.getInstance(StubUpdatingIndex.class);
55   public static final boolean USE_SNAPSHOT_MAPPINGS = SystemProperties.is("stubs.use.snapshot.mappings");
56
57   private static final int VERSION = 45 + (PersistentHashMapValueStorage.COMPRESSION_ENABLED ? 1 : 0);
58
59   // todo remove once we don't need this for stub-ast mismatch debug info
60   private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 3, true);
61
62   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
63
64   private static final FileBasedIndex.InputFilter INPUT_FILTER = StubUpdatingIndex::canHaveStub;
65
66   @NotNull
67   private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
68
69   @NotNull
70   private final SerializationManagerEx mySerializationManager;
71
72   public StubUpdatingIndex() {
73     this(StubForwardIndexExternalizer.getIdeUsedExternalizer(), SerializationManagerEx.getInstanceEx());
74   }
75
76   public StubUpdatingIndex(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer,
77                            @NotNull SerializationManagerEx serializationManager) {
78     myStubIndexesExternalizer = stubIndexesExternalizer;
79     mySerializationManager = serializationManager;
80   }
81
82   @Override
83   public boolean hasSnapshotMapping() {
84     return USE_SNAPSHOT_MAPPINGS;
85   }
86
87   public static boolean canHaveStub(@NotNull VirtualFile file) {
88     Project project = ProjectUtil.guessProjectForFile(file);
89     FileType fileType = SubstitutedFileType.substituteFileType(file, file.getFileType(), project);
90     if (fileType instanceof LanguageFileType) {
91       final Language l = ((LanguageFileType)fileType).getLanguage();
92       final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
93       if (parserDefinition == null) {
94         return false;
95       }
96
97       final IFileElementType elementType = parserDefinition.getFileNodeType();
98       if (elementType instanceof IStubFileElementType && ((IStubFileElementType)elementType).shouldBuildStubFor(file)) {
99         return true;
100       }
101     }
102     final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
103     return builder != null && builder.acceptsFile(file);
104   }
105
106   @NotNull
107   @Override
108   public ID<Integer, SerializedStubTree> getName() {
109     return INDEX_ID;
110   }
111
112   @NotNull
113   @Override
114   public SingleEntryIndexer<SerializedStubTree> getIndexer() {
115     return new SingleEntryCompositeIndexer<SerializedStubTree, StubBuilderType, String>(false) {
116       @Override
117       public boolean requiresContentForSubIndexerEvaluation(@NotNull IndexedFile file) {
118         return StubTreeBuilder.requiresContentToFindBuilder(file.getFileType());
119       }
120
121       @Nullable
122       @Override
123       public StubBuilderType calculateSubIndexer(@NotNull IndexedFile file) {
124         return StubTreeBuilder.getStubBuilderType(file, true);
125       }
126
127       @NotNull
128       @Override
129       public String getSubIndexerVersion(@NotNull StubBuilderType type) {
130         return type.getVersion();
131       }
132
133       @NotNull
134       @Override
135       public KeyDescriptor<String> getSubIndexerVersionDescriptor() {
136         return EnumeratorStringDescriptor.INSTANCE;
137       }
138
139       @Override
140       @Nullable
141       protected SerializedStubTree computeValue(@NotNull final FileContent inputData, @NotNull StubBuilderType type) {
142         try {
143           SerializedStubTree prebuiltTree = findPrebuiltSerializedStubTree(inputData);
144           if (prebuiltTree != null) {
145             prebuiltTree = prebuiltTree.reSerialize(mySerializationManager, myStubIndexesExternalizer);
146             if (PrebuiltIndexProvider.DEBUG_PREBUILT_INDICES) {
147               assertPrebuiltStubTreeMatchesActualTree(prebuiltTree, inputData, type);
148             }
149             return prebuiltTree;
150           }
151         } catch (ProcessCanceledException pce) {
152           throw pce;
153         } catch (Exception e) {
154           LOG.error("Error while indexing: " + inputData.getFileName() + " using prebuilt stub index", e);
155         }
156
157         try {
158           Stub stub = StubTreeBuilder.buildStubTree(inputData, type);
159           if (stub == null) {
160             return null;
161           }
162           SerializedStubTree serializedStubTree = SerializedStubTree.serializeStub(stub, mySerializationManager, myStubIndexesExternalizer);
163           if (IndexDebugProperties.DEBUG) {
164             assertDeserializedStubMatchesOriginalStub(serializedStubTree, stub);
165           }
166           return serializedStubTree;
167         }
168         catch (ProcessCanceledException pce) {
169           throw pce;
170         }
171         catch (Throwable t) {
172           LOG.error("Error indexing:" + inputData.getFile(), t);
173           return null;
174         }
175       }
176     };
177   }
178
179   @Nullable
180   static SerializedStubTree findPrebuiltSerializedStubTree(@NotNull FileContent fileContent) {
181     if (!PrebuiltIndexProvider.USE_PREBUILT_INDEX) {
182       return null;
183     }
184     PrebuiltStubsProvider prebuiltStubsProvider = PrebuiltStubsKt.getPrebuiltStubsProvider().forFileType(fileContent.getFileType());
185     if (prebuiltStubsProvider == null) {
186       return null;
187     }
188     return prebuiltStubsProvider.findStub(fileContent);
189   }
190
191   private static void assertDeserializedStubMatchesOriginalStub(@NotNull SerializedStubTree stubTree,
192                                                                 @NotNull Stub originalStub) {
193     Stub deserializedStub;
194     try {
195       deserializedStub = stubTree.getStub();
196     }
197     catch (SerializerNotFoundException e) {
198       throw new RuntimeException("Failed to deserialize stub tree", e);
199     }
200     assertStubsAreSimilar(originalStub, deserializedStub);
201   }
202
203   private static void assertStubsAreSimilar(@NotNull Stub stub, @NotNull Stub stub2) {
204     assert stub.getStubType() == stub2.getStubType() : stub.getStubType() + "!=" + stub2.getStubType();
205     List<? extends Stub> stubs = stub.getChildrenStubs();
206     List<? extends Stub> stubs2 = stub2.getChildrenStubs();
207     assert stubs.size() == stubs2.size() : stub.getStubType() + ": " + stubs.size() + "!=" + stubs2.size();
208     for (int i = 0, len = stubs.size(); i < len; ++i) {
209       assertStubsAreSimilar(stubs.get(i), stubs2.get(i));
210     }
211   }
212
213   private void assertPrebuiltStubTreeMatchesActualTree(@NotNull SerializedStubTree prebuiltStubTree,
214                                                        @NotNull FileContent fileContent,
215                                                        @NotNull StubBuilderType type) {
216     try {
217       Stub stub = StubTreeBuilder.buildStubTree(fileContent, type);
218       if (stub == null) {
219         return;
220       }
221       SerializedStubTree actualTree = SerializedStubTree.serializeStub(stub, mySerializationManager, myStubIndexesExternalizer);
222       if (!IndexDataComparer.INSTANCE.areStubTreesTheSame(actualTree, prebuiltStubTree)) {
223         throw new RuntimeExceptionWithAttachments(
224           "Prebuilt stub tree does not match actual stub tree",
225           new Attachment("actual-stub-tree.txt", IndexDataPresenter.INSTANCE.getPresentableSerializedStubTree(actualTree)),
226           new Attachment("prebuilt-stub-tree.txt", IndexDataPresenter.INSTANCE.getPresentableSerializedStubTree(prebuiltStubTree))
227         );
228       }
229     }
230     catch (IOException e) {
231       throw new RuntimeException(e);
232     }
233   }
234
235   private static final byte IS_BINARY_MASK = 1;
236   private static final byte BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK = 1 << 1;
237   @NotNull
238   private static IndexingStampInfo calculateIndexingStamp(@NotNull FileContent content) {
239     VirtualFile file = content.getFile();
240     boolean isBinary = file.getFileType().isBinary();
241     int contentLength = isBinary ? -1 : content.getPsiFile().getTextLength();
242     long byteLength = file.getLength();
243
244     IndexingStampInfo indexingStampInfo = new IndexingStampInfo(file.getTimeStamp(), byteLength, contentLength, isBinary);
245     if (LOG.isDebugEnabled()) {
246       LOG.debug("Indexing stubs for " + file + ", " + indexingStampInfo);
247     }
248     return indexingStampInfo;
249   }
250
251   static void saveIndexingStampInfo(@Nullable IndexingStampInfo indexingStampInfo, int fileId) {
252     try (DataOutputStream stream = FSRecords.writeAttribute(fileId, INDEXED_STAMP)) {
253       if (indexingStampInfo == null) return;
254       DataInputOutputUtil.writeTIME(stream, indexingStampInfo.indexingFileStamp);
255       DataInputOutputUtil.writeLONG(stream, indexingStampInfo.indexingByteLength);
256
257       boolean lengthsAreTheSame = indexingStampInfo.indexingCharLength == indexingStampInfo.indexingByteLength;
258       byte flags = 0;
259       flags = BitUtil.set(flags, IS_BINARY_MASK, indexingStampInfo.isBinary);
260       flags = BitUtil.set(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK, lengthsAreTheSame);
261       stream.writeByte(flags);
262
263       if (!lengthsAreTheSame && !indexingStampInfo.isBinary) {
264         DataInputOutputUtil.writeINT(stream, indexingStampInfo.indexingCharLength);
265       }
266     }
267     catch (IOException e) {
268       LOG.error(e);
269     }
270   }
271
272   @Nullable
273   static IndexingStampInfo readSavedIndexingStampInfo(@NotNull VirtualFile file) {
274     try (DataInputStream stream = INDEXED_STAMP.readAttribute(file)) {
275       if (stream == null || stream.available() <= 0) {
276         return null;
277       }
278       long stamp = DataInputOutputUtil.readTIME(stream);
279       long byteLength = DataInputOutputUtil.readLONG(stream);
280
281       byte flags = stream.readByte();
282       boolean isBinary = BitUtil.isSet(flags, IS_BINARY_MASK);
283       boolean readOnlyOneLength = BitUtil.isSet(flags, BYTE_AND_CHAR_LENGTHS_ARE_THE_SAME_MASK);
284
285       int charLength;
286       if (isBinary) {
287         charLength = -1;
288       }
289       else if (readOnlyOneLength) {
290         charLength = (int)byteLength;
291       }
292       else {
293         charLength = DataInputOutputUtil.readINT(stream);
294       }
295       return new IndexingStampInfo(stamp, byteLength, charLength, isBinary);
296     }
297     catch (IOException e) {
298       LOG.error(e);
299       return null;
300     }
301   }
302
303   @NotNull
304   @Override
305   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
306     ensureSerializationManagerInitialized(mySerializationManager);
307     return new SerializedStubTreeDataExternalizer(true, mySerializationManager, myStubIndexesExternalizer);
308   }
309
310   @NotNull
311   @Override
312   public FileBasedIndex.InputFilter getInputFilter() {
313     return INPUT_FILTER;
314   }
315
316   @Override
317   public int getVersion() {
318     return VERSION;
319   }
320
321   @Override
322   public void handleInitializationError(@NotNull Throwable e) {
323     ((StubIndexImpl)StubIndex.getInstance()).initializationFailed(e);
324   }
325
326   @NotNull
327   @Override
328   public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(@NotNull final FileBasedIndexExtension<Integer, SerializedStubTree> extension,
329                                                                                             @NotNull IndexStorage<Integer, SerializedStubTree> storage)
330     throws StorageException, IOException {
331     ((StubIndexImpl)StubIndex.getInstance()).initializeStubIndexes();
332     if (storage instanceof TransientChangesIndexStorage) {
333       final TransientChangesIndexStorage<Integer, SerializedStubTree>
334         memStorage = (TransientChangesIndexStorage<Integer, SerializedStubTree>)storage;
335       memStorage.addBufferingStateListener(new TransientChangesIndexStorage.BufferingStateListener() {
336         @Override
337         public void bufferingStateChanged(final boolean newState) {
338           ((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState);
339         }
340
341         @Override
342         public void memoryStorageCleared() {
343           ((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage();
344         }
345       });
346     }
347     checkNameStorage();
348
349     boolean hasSnapshotMapping = VfsAwareMapReduceIndex.hasSnapshotMapping(this);
350     StubUpdatingForwardIndexAccessor stubForwardIndexAccessor = new StubUpdatingForwardIndexAccessor(extension);
351
352     SnapshotInputMappings<Integer, SerializedStubTree> snapshotInputMappings =
353       hasSnapshotMapping
354       ? new SnapshotInputMappings<>(this, stubForwardIndexAccessor)
355       : null;
356
357     ForwardIndex forwardIndex =
358       hasSnapshotMapping
359       ? new IntMapForwardIndex(snapshotInputMappings.getInputIndexStorageFile(), true)
360       : new EmptyForwardIndex();
361
362     ForwardIndexAccessor<Integer, SerializedStubTree> accessor =
363       hasSnapshotMapping
364       ? snapshotInputMappings.getForwardIndexAccessor()
365       : stubForwardIndexAccessor;
366
367     return new MyIndex(extension, storage, forwardIndex, accessor, snapshotInputMappings);
368   }
369
370   private void checkNameStorage() throws StorageException {
371     if (mySerializationManager.isNameStorageCorrupted()) {
372       mySerializationManager.repairNameStorage();
373       throw new StorageException("NameStorage for stubs serialization has been corrupted");
374     }
375   }
376
377   private static class MyIndex extends VfsAwareMapReduceIndex<Integer, SerializedStubTree> {
378     private StubIndexImpl myStubIndex;
379     @Nullable
380     private final CompositeBinaryBuilderMap myCompositeBinaryBuilderMap = FileBasedIndex.USE_IN_MEMORY_INDEX ? null : new CompositeBinaryBuilderMap();
381
382     MyIndex(@NotNull FileBasedIndexExtension<Integer, SerializedStubTree> extension,
383             @NotNull IndexStorage<Integer, SerializedStubTree> storage,
384             @Nullable ForwardIndex forwardIndex,
385             @Nullable ForwardIndexAccessor<Integer, SerializedStubTree> forwardIndexAccessor,
386             @Nullable SnapshotInputMappings<Integer, SerializedStubTree> snapshotInputMappings) throws IOException {
387       super(extension, storage, forwardIndex, forwardIndexAccessor, snapshotInputMappings, null);
388     }
389
390     @Override
391     protected void doFlush() throws IOException, StorageException {
392       final StubIndexImpl stubIndex = getStubIndex();
393       try {
394         stubIndex.flush();
395       }
396       finally {
397         super.doFlush();
398       }
399     }
400
401     @NotNull
402     private StubIndexImpl getStubIndex() {
403       StubIndexImpl index = myStubIndex;
404       if (index == null) {
405         myStubIndex = index = (StubIndexImpl)StubIndex.getInstance();
406       }
407       return index;
408     }
409
410     @Override
411     public @NotNull Computable<Boolean> mapInputAndPrepareUpdate(int inputId, @Nullable FileContent content)
412       throws MapInputException, ProcessCanceledException {
413       Computable<Boolean> indexUpdateComputable = super.mapInputAndPrepareUpdate(inputId, content);
414       IndexingStampInfo indexingStampInfo = content == null ? null : calculateIndexingStamp(content);
415
416       return () -> {
417         Boolean result = indexUpdateComputable.compute();
418         if (Boolean.TRUE.equals(result)) {
419           saveIndexingStampInfo(indexingStampInfo, inputId);
420         }
421         return result;
422       };
423     }
424
425     @Override
426     protected void removeTransientDataForInMemoryKeys(int inputId, @NotNull Map<? extends Integer, ? extends SerializedStubTree> map) {
427       super.removeTransientDataForInMemoryKeys(inputId, map);
428       removeStubIndexKeys(inputId, getStubIndexMaps(map));
429     }
430
431     @Override
432     public void removeTransientDataForKeys(int inputId, @NotNull Collection<? extends Integer> keys) {
433       Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> maps;
434       try {
435         Map<Integer, SerializedStubTree> data = getIndexedFileData(inputId);
436         maps = getStubIndexMaps(data);
437       }
438       catch (StorageException e) {
439         throw new RuntimeException(e);
440       }
441       super.removeTransientDataForKeys(inputId, keys);
442       removeStubIndexKeys(inputId, maps);
443     }
444
445     private static void removeStubIndexKeys(int inputId, @NotNull Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> indexedStubs) {
446       StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
447       for (StubIndexKey key : indexedStubs.keySet()) {
448         stubIndex.removeTransientDataForFile(key, inputId, indexedStubs.get(key).keySet());
449       }
450     }
451
452     @NotNull
453     private static Map<StubIndexKey<?, ?>, Map<Object, StubIdList>> getStubIndexMaps(@NotNull Map<? extends Integer, ? extends SerializedStubTree> data) {
454       if (data.isEmpty()) {
455         return Collections.emptyMap();
456       }
457       SerializedStubTree tree = data.values().iterator().next();
458       return tree == null ? Collections.emptyMap() : tree.getStubIndicesValueMap();
459     }
460
461     @Override
462     protected void doClear() throws StorageException, IOException {
463       final StubIndexImpl stubIndex = StubIndexImpl.getInstanceOrInvalidate();
464       if (stubIndex != null) {
465         stubIndex.clearAllIndices();
466       }
467       super.doClear();
468     }
469
470     @Override
471     protected void doDispose() throws StorageException {
472       try {
473         super.doDispose();
474       }
475       finally {
476         getStubIndex().dispose();
477       }
478     }
479
480     @Override
481     public void setIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
482       super.setIndexedStateForFile(fileId, file);
483       setBinaryBuilderConfiguration(fileId, file);
484     }
485
486     @Override
487     protected boolean isIndexConfigurationUpToDate(int fileId, @NotNull IndexedFile file) {
488       if (myCompositeBinaryBuilderMap == null) return true;
489       try {
490         return myCompositeBinaryBuilderMap.isUpToDateState(fileId, file.getFile());
491       } catch (IOException e) {
492         LOG.error(e);
493         return false;
494       }
495     }
496
497     @Override
498     protected void setIndexConfigurationUpToDate(int fileId, @NotNull IndexedFile file) {
499       setBinaryBuilderConfiguration(fileId, file);
500     }
501
502     private void setBinaryBuilderConfiguration(int fileId, @NotNull IndexedFile file) {
503       if (myCompositeBinaryBuilderMap != null) {
504         try {
505           myCompositeBinaryBuilderMap.persistState(fileId, file.getFile());
506         }
507         catch (IOException e) {
508           LOG.error(e);
509         }
510       }
511     }
512   }
513
514   private static void ensureSerializationManagerInitialized(@NotNull SerializationManagerEx serializationManager) {
515     ProgressManager.getInstance().executeNonCancelableSection(() -> {
516       instantiateElementTypesFromFields();
517       StubIndexEx.initExtensions();
518       serializationManager.initSerializers();
519     });
520   }
521
522   private static void instantiateElementTypesFromFields() {
523     // load stub serializers before usage
524     FileTypeRegistry.getInstance().getRegisteredFileTypes();
525     getExtensions(BinaryFileStubBuilders.INSTANCE).forEach(builder -> {});
526     getExtensions(LanguageParserDefinitions.INSTANCE).forEach(ParserDefinition::getFileNodeType);
527   }
528
529   private static @NotNull <T> Stream<T> getExtensions(@NotNull KeyedExtensionCollector<T, ?> collector) {
530     ExtensionPoint<KeyedLazyInstance<T>> point = collector.getPoint();
531     return point == null ? Stream.empty() : point.extensions().map(KeyedLazyInstance::getInstance);
532   }
533 }