serialize keys before calling append
[idea/community.git] / platform / lang-impl / src / com / intellij / util / indexing / SharedIndicesData.java
1 /*
2  * Copyright 2000-2016 JetBrains s.r.o.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.intellij.util.indexing;
17
18 import com.intellij.openapi.application.PathManager;
19 import com.intellij.openapi.diagnostic.Logger;
20 import com.intellij.openapi.util.LowMemoryWatcher;
21 import com.intellij.openapi.util.Ref;
22 import com.intellij.openapi.util.ShutDownTracker;
23 import com.intellij.openapi.util.ThrowableComputable;
24 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
25 import com.intellij.util.SystemProperties;
26 import com.intellij.util.io.*;
27 import com.intellij.util.io.DataOutputStream;
28 import gnu.trove.TIntLongHashMap;
29 import gnu.trove.TIntLongProcedure;
30 import gnu.trove.TIntObjectHashMap;
31 import gnu.trove.TIntObjectProcedure;
32 import org.jetbrains.annotations.NotNull;
33 import org.jetbrains.annotations.Nullable;
34
35 import java.io.*;
36 import java.util.Arrays;
37
38 public class SharedIndicesData {
39   private static PersistentHashMap<Integer, byte[]> ourSharedFileInputsIndex;
40   private static PersistentHashMap<Integer, byte[]> ourSharedContentInputsIndex;
41   static final boolean ourFileSharedIndicesEnabled = SystemProperties.getBooleanProperty("idea.shared.input.index.enabled", true);
42   static final boolean DO_CHECKS = ourFileSharedIndicesEnabled && SystemProperties.getBooleanProperty("idea.shared.input.index.checked", true);
43
44   //private static ScheduledFuture<?> ourFlushingFuture;
45   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
46   @SuppressWarnings({"FieldCanBeLocal", "unused"}) private static LowMemoryWatcher myLowMemoryCallback;
47
48   private static final FileAccessorCache<Integer, IndexedState> ourFileIndexedStates = new FileAccessorCache<Integer, IndexedState>(200, 100) {
49     @Override
50     protected IndexedState createAccessor(Integer key) throws IOException {
51       return new IndexedState(key, ourSharedFileInputsIndex);
52     }
53
54     @Override
55     protected void disposeAccessor(IndexedState fileAccessor) throws IOException {
56       fileAccessor.flush();
57     }
58   };
59
60   private static final FileAccessorCache<Integer, IndexedState> ourContentIndexedStates = new FileAccessorCache<Integer, IndexedState>(200, 100) {
61     @Override
62     protected IndexedState createAccessor(Integer key) throws IOException {
63       return new IndexedState(key, ourSharedContentInputsIndex);
64     }
65
66     @Override
67     protected void disposeAccessor(IndexedState fileAccessor) throws IOException {
68       fileAccessor.flush();
69     }
70   };
71
72   static {
73     if (ourFileSharedIndicesEnabled) {
74       try {
75         myLowMemoryCallback = LowMemoryWatcher.register(new Runnable() {
76           @Override
77           public void run() {
78             ourFileIndexedStates.clear();
79             ourContentIndexedStates.clear();
80           }
81         });
82         ourSharedFileInputsIndex = createSharedMap(new File(PathManager.getIndexRoot(), "file_inputs.data"));
83         ourSharedContentInputsIndex = createSharedMap(new File(IndexInfrastructure.getPersistentIndexRoot(), "content_inputs.data"));
84
85         ShutDownTracker.getInstance().registerShutdownTask(new Runnable() {
86           @Override
87           public void run() {
88             try {
89               ourSharedFileInputsIndex.close();
90             } catch (IOException ex) {
91               LOG.error(ex);
92             }
93             try {
94               ourSharedContentInputsIndex.close();
95             } catch (IOException ex) {
96               LOG.error(ex);
97             }
98           }
99         });
100       } catch (IOException ex) {
101         throw new RuntimeException(ex);
102       }
103     }
104   }
105
106   private static PersistentHashMap<Integer, byte[]> createSharedMap(final File indexFile) throws IOException {
107     return IOUtil.openCleanOrResetBroken(
108       new ThrowableComputable<PersistentHashMap<Integer, byte[]>, IOException>() {
109         @Override
110         public PersistentHashMap<Integer, byte[]> compute() throws IOException {
111           return new PersistentHashMap<Integer, byte[]>(indexFile, EnumeratorIntegerDescriptor.INSTANCE,
112                                                         new DataExternalizer<byte[]>() {
113                                                           @Override
114                                                           public void save(@NotNull DataOutput out, byte[] value) throws IOException {
115                                                             out.write(value);
116                                                           }
117
118                                                           @Override
119                                                           public byte[] read(@NotNull DataInput in) throws IOException {
120                                                             int available = ((InputStream)in).available();
121                                                             byte[] result = new byte[available];
122                                                             in.readFully(result);
123                                                             return result;
124                                                           }
125                                                         });
126         }
127       }, indexFile);
128   }
129
130   public static void init() {
131
132   }
133
134   public static void flushData() {
135     if (!ourFileSharedIndicesEnabled) return;
136     ourFileIndexedStates.clear();
137     if (ourSharedFileInputsIndex != null && ourSharedFileInputsIndex.isDirty()) ourSharedFileInputsIndex.force();
138     ourContentIndexedStates.clear();
139     if (ourSharedContentInputsIndex != null && ourSharedContentInputsIndex.isDirty()) ourSharedContentInputsIndex.force();
140   }
141
142   public static void beforeSomeIndexVersionInvalidation() {
143     flushData();
144   }
145
146   static class IndexedState {
147     private final int fileOrContentId;
148     private final PersistentHashMap<Integer, byte[]> storage;
149
150     private byte[] values;
151     private TIntLongHashMap indexId2Offset;
152     private TIntObjectHashMap<byte[]> indexId2NewState;
153     private boolean compactNecessary;
154
155     IndexedState(int fileOrContentId, PersistentHashMap<Integer, byte[]> storage) throws IOException {
156       this.fileOrContentId = fileOrContentId;
157       this.storage = storage;
158       byte[] bytes = storage.get(fileOrContentId);
159       if (bytes == null) {
160         return;
161       }
162
163       DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes));
164       boolean compactNecessary = false;
165       TIntLongHashMap stateMap = null;
166
167       while(stream.available() > 0) {
168         int chunkSize = DataInputOutputUtil.readINT(stream);
169         int chunkIndexId = DataInputOutputUtil.readINT(stream);
170         long chunkIndexTimeStamp = DataInputOutputUtil.readTIME(stream);
171         int currentOffset = bytes.length - stream.available();
172
173         ID<?, ?> chunkIndexID;
174         if (((chunkIndexID = ID.findById(chunkIndexId)) != null &&
175              chunkIndexTimeStamp == IndexingStamp.getIndexCreationStamp(chunkIndexID))
176           ) {
177           if (chunkSize != 0) {
178             if (stateMap == null) stateMap = new TIntLongHashMap();
179             stateMap.put(chunkIndexId, (((long)currentOffset) << 32) | chunkSize);
180           } else if (stateMap != null) {
181             stateMap.remove(chunkIndexId);
182             compactNecessary = true;
183           }
184         } else {
185           compactNecessary = true;
186         }
187
188         stream.skipBytes(chunkSize);
189       }
190       values = bytes;
191       this.compactNecessary = compactNecessary;
192       indexId2Offset = stateMap;
193     }
194
195     synchronized void flush() throws IOException {
196       if (compactNecessary) {
197         //noinspection IOResourceOpenedButNotSafelyClosed
198         UnsyncByteArrayOutputStream compactedOutputStream = new UnsyncByteArrayOutputStream(values.length);
199         //noinspection IOResourceOpenedButNotSafelyClosed
200         DataOutput compactedOutput = new DataOutputStream(compactedOutputStream);
201
202         Ref<IOException> ioExceptionRef = new Ref<>();
203
204         boolean result = indexId2NewState == null || indexId2NewState.forEachEntry(new TIntObjectProcedure<byte[]>() {
205           @Override
206           public boolean execute(int indexUniqueId, byte[] indexValue) {
207             try {
208               long indexCreationStamp = IndexingStamp.getIndexCreationStamp(ID.findById(indexUniqueId));
209
210               writeIndexValue(indexUniqueId, indexCreationStamp, indexValue, 0, indexValue.length, compactedOutput);
211
212               return true;
213             }
214             catch (IOException ex) {
215               ioExceptionRef.set(ex);
216               return false;
217             }
218           }
219         });
220         if (!result) throw ioExceptionRef.get();
221
222         result = indexId2Offset == null || indexId2Offset.forEachEntry(new TIntLongProcedure() {
223           @Override
224           public boolean execute(int chunkIndexId, long chunkOffsetAndSize) {
225             try {
226               int chunkOffset = (int)(chunkOffsetAndSize >> 32);
227               int chunkSize = (int)chunkOffsetAndSize;
228
229               writeIndexValue(
230                 chunkIndexId,
231                 IndexingStamp.getIndexCreationStamp(ID.findById(chunkIndexId)),
232                 values,
233                 chunkOffset,
234                 chunkSize,
235                 compactedOutput
236               );
237
238               return true;
239             }
240             catch (IOException e) {
241               ioExceptionRef.set(e);
242               return false;
243             }
244           }
245         });
246         if (!result) throw ioExceptionRef.get();
247         if (compactedOutputStream.size() > 0) storage.put(fileOrContentId, compactedOutputStream.toByteArray());
248         else storage.remove(fileOrContentId);
249       }
250     }
251
252     // todo: what about handling changed indices' versions
253     synchronized void appendIndexedState(ID<?, ?> indexId, long timestamp, byte[] buffer, int size) {
254       int indexUniqueId = indexId.getUniqueId();
255
256       if (indexId2Offset != null) indexId2Offset.remove(indexUniqueId);
257       if (buffer == null) {
258         if (indexId2NewState != null) indexId2NewState.remove(indexUniqueId);
259       } else {
260         if (indexId2NewState == null) indexId2NewState = new TIntObjectHashMap<>();
261         indexId2NewState.put(indexUniqueId, Arrays.copyOf(buffer, size));
262       }
263     }
264
265     synchronized @Nullable DataInputStream readIndexedState(ID<?, ?> indexId) {
266       int indexUniqueId = indexId.getUniqueId();
267       int offset = 0;
268       int length = 0;
269       byte[] bytes = null;
270
271       if (indexId2NewState != null) { // newdata
272         bytes = indexId2NewState.get(indexUniqueId);
273         offset = 0;
274         length = bytes != null ? bytes.length : 0;
275       }
276
277       if (bytes == null) {
278         if (values == null || // empty
279             indexId2Offset == null ||
280             !indexId2Offset.contains(indexUniqueId) // no previous data
281           ) {
282           return null;
283         }
284         bytes = values;
285         long offsetAndSize = indexId2Offset.get(indexUniqueId);
286         offset = (int)(offsetAndSize >> 32);
287         length = (int)offsetAndSize;
288       }
289
290       return new DataInputStream(new UnsyncByteArrayInputStream(bytes, offset, offset + length));
291     }
292   }
293
294   private static void writeIndexValue(int indexUniqueId,
295                                       long indexCreationStamp,
296                                       byte[] indexValue,
297                                       int indexValueOffset, int indexValueLength,
298                                       DataOutput compactedOutput) throws IOException {
299     DataInputOutputUtil.writeINT(compactedOutput, indexValueLength);
300     DataInputOutputUtil.writeINT(compactedOutput, indexUniqueId);
301
302     DataInputOutputUtil.writeTIME(compactedOutput, indexCreationStamp);
303     if (indexValue != null) {
304       assert indexValueLength > 0;
305       compactedOutput.write(indexValue, indexValueOffset, indexValueLength);
306     }
307     else {
308       assert indexValueLength == 0;
309     }
310   }
311
312   // Record:  (<chunkSize> <indexId> <indexStamp> <SavedData>)*
313
314   public static @Nullable <Key, Value> Value recallFileData(int id, ID<Key, ?> indexId, DataExternalizer<Value> externalizer)
315     throws IOException {
316     return doRecallData(id, indexId, externalizer, ourFileIndexedStates);
317   }
318
319   public static @Nullable <Key, Value> Value recallContentData(int id, ID<Key, ?> indexId, DataExternalizer<Value> externalizer)
320     throws IOException {
321     return doRecallData(id, indexId, externalizer, ourContentIndexedStates);
322   }
323
324   @Nullable
325   private static <Key, Value> Value doRecallData(int id,
326                                                  ID<Key, ?> indexId,
327                                                  DataExternalizer<Value> externalizer,
328                                                  FileAccessorCache<Integer, IndexedState> states)
329     throws IOException {
330     FileAccessorCache.Handle<IndexedState> stateHandle = states.get(id);
331     IndexedState indexedState = stateHandle.get();
332
333     try {
334       DataInputStream in = indexedState.readIndexedState(indexId);
335       if (in == null) return null;
336       return externalizer.read(in);
337     } finally {
338       stateHandle.release();
339     }
340   }
341
342   public static <Key, Value> void associateFileData(int id, ID<Key, ?> indexId, Value keys, DataExternalizer<Value> externalizer)
343     throws IOException {
344     doAssociateData(id, indexId, keys, externalizer, ourFileIndexedStates, ourSharedFileInputsIndex);
345   }
346
347   public static <Key, Value> void associateContentData(int id, ID<Key, ?> indexId, Value keys, DataExternalizer<Value> externalizer)
348     throws IOException {
349     doAssociateData(id, indexId, keys, externalizer, ourContentIndexedStates, ourSharedContentInputsIndex);
350   }
351
352   private static <Key, Value> void doAssociateData(int id,
353                                                    final ID<Key, ?> indexId,
354                                                    Value keys,
355                                                    DataExternalizer<Value> externalizer,
356                                                    FileAccessorCache<Integer, IndexedState> states,
357                                                    PersistentHashMap<Integer, byte[]> index)
358     throws IOException {
359     final BufferExposingByteArrayOutputStream savedKeysData;
360     if (keys != null) {
361       //noinspection IOResourceOpenedButNotSafelyClosed
362       externalizer.save(new DataOutputStream(savedKeysData = new BufferExposingByteArrayOutputStream()), keys);
363     } else {
364       savedKeysData = null;
365     }
366
367     FileAccessorCache.Handle<IndexedState> stateHandle = states.getIfCached(id);
368
369     try {
370       index.appendData(id, new PersistentHashMap.ValueDataAppender() {
371         @Override
372         public void append(DataOutput out) throws IOException {
373           byte[] internalBuffer = null;
374           int size = 0;
375           if (savedKeysData != null) {
376             internalBuffer = savedKeysData.getInternalBuffer();
377             size = savedKeysData.size();
378           }
379
380           long indexCreationStamp = IndexingStamp.getIndexCreationStamp(indexId);
381           writeIndexValue(
382             indexId.getUniqueId(),
383             indexCreationStamp,
384             internalBuffer,
385             0,
386             size,
387             out
388           );
389
390           final IndexedState indexedState = stateHandle != null ? stateHandle.get() : null;
391           if (indexedState != null) {
392             indexedState.appendIndexedState(indexId, indexCreationStamp, internalBuffer, size);
393           }
394         }
395       });
396     } finally {
397       if (stateHandle != null) stateHandle.release();
398     }
399   }
400 }