platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java

   1 /*
   2  * Copyright 2000-2016 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.intellij.util.indexing;
  17
  18 import com.intellij.openapi.application.PathManager;
  19 import com.intellij.openapi.diagnostic.Logger;
  20 import com.intellij.openapi.util.LowMemoryWatcher;
  21 import com.intellij.openapi.util.Ref;
  22 import com.intellij.openapi.util.ShutDownTracker;
  23 import com.intellij.openapi.util.ThrowableComputable;
  24 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
  25 import com.intellij.util.SystemProperties;
  26 import com.intellij.util.io.*;
  27 import com.intellij.util.io.DataOutputStream;
  28 import gnu.trove.TIntLongHashMap;
  29 import gnu.trove.TIntLongProcedure;
  30 import gnu.trove.TIntObjectHashMap;
  31 import gnu.trove.TIntObjectProcedure;
  32 import org.jetbrains.annotations.NotNull;
  33 import org.jetbrains.annotations.Nullable;
  34
  35 import java.io.*;
  36 import java.util.Arrays;
  37
  38 public class SharedIndicesData {
  39   private static PersistentHashMap<Integer, byte[]> ourSharedFileInputsIndex;
  40   private static PersistentHashMap<Integer, byte[]> ourSharedContentInputsIndex;
  41   static final boolean ourFileSharedIndicesEnabled = SystemProperties.getBooleanProperty("idea.shared.input.index.enabled", true);
  42   static final boolean DO_CHECKS = ourFileSharedIndicesEnabled && SystemProperties.getBooleanProperty("idea.shared.input.index.checked", true);
  43
  44   //private static ScheduledFuture<?> ourFlushingFuture;
  45   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
  46   @SuppressWarnings({"FieldCanBeLocal", "unused"}) private static LowMemoryWatcher myLowMemoryCallback;
  47
  48   private static final FileAccessorCache<Integer, IndexedState> ourFileIndexedStates = new FileAccessorCache<Integer, IndexedState>(200, 100) {
  49     @Override
  50     protected IndexedState createAccessor(Integer key) throws IOException {
  51       return new IndexedState(key, ourSharedFileInputsIndex);
  52     }
  53
  54     @Override
  55     protected void disposeAccessor(IndexedState fileAccessor) throws IOException {
  56       fileAccessor.flush();
  57     }
  58   };
  59
  60   private static final FileAccessorCache<Integer, IndexedState> ourContentIndexedStates = new FileAccessorCache<Integer, IndexedState>(200, 100) {
  61     @Override
  62     protected IndexedState createAccessor(Integer key) throws IOException {
  63       return new IndexedState(key, ourSharedContentInputsIndex);
  64     }
  65
  66     @Override
  67     protected void disposeAccessor(IndexedState fileAccessor) throws IOException {
  68       fileAccessor.flush();
  69     }
  70   };
  71
  72   static {
  73     if (ourFileSharedIndicesEnabled) {
  74       try {
  75         myLowMemoryCallback = LowMemoryWatcher.register(new Runnable() {
  76           @Override
  77           public void run() {
  78             ourFileIndexedStates.clear();
  79             ourContentIndexedStates.clear();
  80           }
  81         });
  82         ourSharedFileInputsIndex = createSharedMap(new File(PathManager.getIndexRoot(), "file_inputs.data"));
  83         ourSharedContentInputsIndex = createSharedMap(new File(IndexInfrastructure.getPersistentIndexRoot(), "content_inputs.data"));
  84
  85         ShutDownTracker.getInstance().registerShutdownTask(new Runnable() {
  86           @Override
  87           public void run() {
  88             try {
  89               ourSharedFileInputsIndex.close();
  90             } catch (IOException ex) {
  91               LOG.error(ex);
  92             }
  93             try {
  94               ourSharedContentInputsIndex.close();
  95             } catch (IOException ex) {
  96               LOG.error(ex);
  97             }
  98           }
  99         });
 100       } catch (IOException ex) {
 101         throw new RuntimeException(ex);
 102       }
 103     }
 104   }
 105
 106   private static PersistentHashMap<Integer, byte[]> createSharedMap(final File indexFile) throws IOException {
 107     return IOUtil.openCleanOrResetBroken(
 108       new ThrowableComputable<PersistentHashMap<Integer, byte[]>, IOException>() {
 109         @Override
 110         public PersistentHashMap<Integer, byte[]> compute() throws IOException {
 111           return new PersistentHashMap<Integer, byte[]>(indexFile, EnumeratorIntegerDescriptor.INSTANCE,
 112                                                         new DataExternalizer<byte[]>() {
 113                                                           @Override
 114                                                           public void save(@NotNull DataOutput out, byte[] value) throws IOException {
 115                                                             out.write(value);
 116                                                           }
 117
 118                                                           @Override
 119                                                           public byte[] read(@NotNull DataInput in) throws IOException {
 120                                                             int available = ((InputStream)in).available();
 121                                                             byte[] result = new byte[available];
 122                                                             in.readFully(result);
 123                                                             return result;
 124                                                           }
 125                                                         });
 126         }
 127       }, indexFile);
 128   }
 129
 130   public static void init() {
 131
 132   }
 133
 134   public static void flushData() {
 135     if (!ourFileSharedIndicesEnabled) return;
 136     ourFileIndexedStates.clear();
 137     if (ourSharedFileInputsIndex != null && ourSharedFileInputsIndex.isDirty()) ourSharedFileInputsIndex.force();
 138     ourContentIndexedStates.clear();
 139     if (ourSharedContentInputsIndex != null && ourSharedContentInputsIndex.isDirty()) ourSharedContentInputsIndex.force();
 140   }
 141
 142   public static void beforeSomeIndexVersionInvalidation() {
 143     flushData();
 144   }
 145
 146   static class IndexedState {
 147     private final int fileOrContentId;
 148     private final PersistentHashMap<Integer, byte[]> storage;
 149
 150     private byte[] values;
 151     private TIntLongHashMap indexId2Offset;
 152     private TIntObjectHashMap<byte[]> indexId2NewState;
 153     private boolean compactNecessary;
 154
 155     IndexedState(int fileOrContentId, PersistentHashMap<Integer, byte[]> storage) throws IOException {
 156       this.fileOrContentId = fileOrContentId;
 157       this.storage = storage;
 158       byte[] bytes = storage.get(fileOrContentId);
 159       if (bytes == null) {
 160         return;
 161       }
 162
 163       DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes));
 164       boolean compactNecessary = false;
 165       TIntLongHashMap stateMap = null;
 166
 167       while(stream.available() > 0) {
 168         int chunkSize = DataInputOutputUtil.readINT(stream);
 169         int chunkIndexId = DataInputOutputUtil.readINT(stream);
 170         long chunkIndexTimeStamp = DataInputOutputUtil.readTIME(stream);
 171         int currentOffset = bytes.length - stream.available();
 172
 173         ID<?, ?> chunkIndexID;
 174         if (((chunkIndexID = ID.findById(chunkIndexId)) != null &&
 175              chunkIndexTimeStamp == IndexingStamp.getIndexCreationStamp(chunkIndexID))
 176           ) {
 177           if (chunkSize != 0) {
 178             if (stateMap == null) stateMap = new TIntLongHashMap();
 179             stateMap.put(chunkIndexId, (((long)currentOffset) << 32) | chunkSize);
 180           } else if (stateMap != null) {
 181             stateMap.remove(chunkIndexId);
 182             compactNecessary = true;
 183           }
 184         } else {
 185           compactNecessary = true;
 186         }
 187
 188         stream.skipBytes(chunkSize);
 189       }
 190       values = bytes;
 191       this.compactNecessary = compactNecessary;
 192       indexId2Offset = stateMap;
 193     }
 194
 195     synchronized void flush() throws IOException {
 196       if (compactNecessary) {
 197         //noinspection IOResourceOpenedButNotSafelyClosed
 198         UnsyncByteArrayOutputStream compactedOutputStream = new UnsyncByteArrayOutputStream(values.length);
 199         //noinspection IOResourceOpenedButNotSafelyClosed
 200         DataOutput compactedOutput = new DataOutputStream(compactedOutputStream);
 201
 202         Ref<IOException> ioExceptionRef = new Ref<>();
 203
 204         boolean result = indexId2NewState == null || indexId2NewState.forEachEntry(new TIntObjectProcedure<byte[]>() {
 205           @Override
 206           public boolean execute(int indexUniqueId, byte[] indexValue) {
 207             try {
 208               long indexCreationStamp = IndexingStamp.getIndexCreationStamp(ID.findById(indexUniqueId));
 209
 210               writeIndexValue(indexUniqueId, indexCreationStamp, indexValue, 0, indexValue.length, compactedOutput);
 211
 212               return true;
 213             }
 214             catch (IOException ex) {
 215               ioExceptionRef.set(ex);
 216               return false;
 217             }
 218           }
 219         });
 220         if (!result) throw ioExceptionRef.get();
 221
 222         result = indexId2Offset == null || indexId2Offset.forEachEntry(new TIntLongProcedure() {
 223           @Override
 224           public boolean execute(int chunkIndexId, long chunkOffsetAndSize) {
 225             try {
 226               int chunkOffset = (int)(chunkOffsetAndSize >> 32);
 227               int chunkSize = (int)chunkOffsetAndSize;
 228
 229               writeIndexValue(
 230                 chunkIndexId,
 231                 IndexingStamp.getIndexCreationStamp(ID.findById(chunkIndexId)),
 232                 values,
 233                 chunkOffset,
 234                 chunkSize,
 235                 compactedOutput
 236               );
 237
 238               return true;
 239             }
 240             catch (IOException e) {
 241               ioExceptionRef.set(e);
 242               return false;
 243             }
 244           }
 245         });
 246         if (!result) throw ioExceptionRef.get();
 247         if (compactedOutputStream.size() > 0) storage.put(fileOrContentId, compactedOutputStream.toByteArray());
 248         else storage.remove(fileOrContentId);
 249       }
 250     }
 251
 252     // todo: what about handling changed indices' versions
 253     synchronized void appendIndexedState(ID<?, ?> indexId, long timestamp, byte[] buffer, int size) {
 254       int indexUniqueId = indexId.getUniqueId();
 255
 256       if (indexId2Offset != null) indexId2Offset.remove(indexUniqueId);
 257       if (buffer == null) {
 258         if (indexId2NewState != null) indexId2NewState.remove(indexUniqueId);
 259       } else {
 260         if (indexId2NewState == null) indexId2NewState = new TIntObjectHashMap<>();
 261         indexId2NewState.put(indexUniqueId, Arrays.copyOf(buffer, size));
 262       }
 263     }
 264
 265     synchronized @Nullable DataInputStream readIndexedState(ID<?, ?> indexId) {
 266       int indexUniqueId = indexId.getUniqueId();
 267       int offset = 0;
 268       int length = 0;
 269       byte[] bytes = null;
 270
 271       if (indexId2NewState != null) { // newdata
 272         bytes = indexId2NewState.get(indexUniqueId);
 273         offset = 0;
 274         length = bytes != null ? bytes.length : 0;
 275       }
 276
 277       if (bytes == null) {
 278         if (values == null || // empty
 279             indexId2Offset == null ||
 280             !indexId2Offset.contains(indexUniqueId) // no previous data
 281           ) {
 282           return null;
 283         }
 284         bytes = values;
 285         long offsetAndSize = indexId2Offset.get(indexUniqueId);
 286         offset = (int)(offsetAndSize >> 32);
 287         length = (int)offsetAndSize;
 288       }
 289
 290       return new DataInputStream(new UnsyncByteArrayInputStream(bytes, offset, offset + length));
 291     }
 292   }
 293
 294   private static void writeIndexValue(int indexUniqueId,
 295                                       long indexCreationStamp,
 296                                       byte[] indexValue,
 297                                       int indexValueOffset, int indexValueLength,
 298                                       DataOutput compactedOutput) throws IOException {
 299     DataInputOutputUtil.writeINT(compactedOutput, indexValueLength);
 300     DataInputOutputUtil.writeINT(compactedOutput, indexUniqueId);
 301
 302     DataInputOutputUtil.writeTIME(compactedOutput, indexCreationStamp);
 303     if (indexValue != null) {
 304       assert indexValueLength > 0;
 305       compactedOutput.write(indexValue, indexValueOffset, indexValueLength);
 306     }
 307     else {
 308       assert indexValueLength == 0;
 309     }
 310   }
 311
 312   // Record:  (<chunkSize> <indexId> <indexStamp> <SavedData>)*
 313
 314   public static @Nullable <Key, Value> Value recallFileData(int id, ID<Key, ?> indexId, DataExternalizer<Value> externalizer)
 315     throws IOException {
 316     return doRecallData(id, indexId, externalizer, ourFileIndexedStates);
 317   }
 318
 319   public static @Nullable <Key, Value> Value recallContentData(int id, ID<Key, ?> indexId, DataExternalizer<Value> externalizer)
 320     throws IOException {
 321     return doRecallData(id, indexId, externalizer, ourContentIndexedStates);
 322   }
 323
 324   @Nullable
 325   private static <Key, Value> Value doRecallData(int id,
 326                                                  ID<Key, ?> indexId,
 327                                                  DataExternalizer<Value> externalizer,
 328                                                  FileAccessorCache<Integer, IndexedState> states)
 329     throws IOException {
 330     FileAccessorCache.Handle<IndexedState> stateHandle = states.get(id);
 331     IndexedState indexedState = stateHandle.get();
 332
 333     try {
 334       DataInputStream in = indexedState.readIndexedState(indexId);
 335       if (in == null) return null;
 336       return externalizer.read(in);
 337     } finally {
 338       stateHandle.release();
 339     }
 340   }
 341
 342   public static <Key, Value> void associateFileData(int id, ID<Key, ?> indexId, Value keys, DataExternalizer<Value> externalizer)
 343     throws IOException {
 344     doAssociateData(id, indexId, keys, externalizer, ourFileIndexedStates, ourSharedFileInputsIndex);
 345   }
 346
 347   public static <Key, Value> void associateContentData(int id, ID<Key, ?> indexId, Value keys, DataExternalizer<Value> externalizer)
 348     throws IOException {
 349     doAssociateData(id, indexId, keys, externalizer, ourContentIndexedStates, ourSharedContentInputsIndex);
 350   }
 351
 352   private static <Key, Value> void doAssociateData(int id,
 353                                                    final ID<Key, ?> indexId,
 354                                                    Value keys,
 355                                                    DataExternalizer<Value> externalizer,
 356                                                    FileAccessorCache<Integer, IndexedState> states,
 357                                                    PersistentHashMap<Integer, byte[]> index)
 358     throws IOException {
 359     final BufferExposingByteArrayOutputStream savedKeysData;
 360     if (keys != null) {
 361       //noinspection IOResourceOpenedButNotSafelyClosed
 362       externalizer.save(new DataOutputStream(savedKeysData = new BufferExposingByteArrayOutputStream()), keys);
 363     } else {
 364       savedKeysData = null;
 365     }
 366
 367     FileAccessorCache.Handle<IndexedState> stateHandle = states.getIfCached(id);
 368
 369     try {
 370       index.appendData(id, new PersistentHashMap.ValueDataAppender() {
 371         @Override
 372         public void append(DataOutput out) throws IOException {
 373           byte[] internalBuffer = null;
 374           int size = 0;
 375           if (savedKeysData != null) {
 376             internalBuffer = savedKeysData.getInternalBuffer();
 377             size = savedKeysData.size();
 378           }
 379
 380           long indexCreationStamp = IndexingStamp.getIndexCreationStamp(indexId);
 381           writeIndexValue(
 382             indexId.getUniqueId(),
 383             indexCreationStamp,
 384             internalBuffer,
 385             0,
 386             size,
 387             out
 388           );
 389
 390           final IndexedState indexedState = stateHandle != null ? stateHandle.get() : null;
 391           if (indexedState != null) {
 392             indexedState.appendIndexedState(indexId, indexCreationStamp, internalBuffer, size);
 393           }
 394         }
 395       });
 396     } finally {
 397       if (stateHandle != null) stateHandle.release();
 398     }
 399   }
 400 }