extract MapReduceIndexBase (doesn't depend on VFS & IDE's components) from MapReduceIndex appcode/171.1012 phpstorm/171.1014
authorDmitry Batkovich <dmitry.batkovich@jetbrains.com>
Wed, 16 Nov 2016 10:04:17 +0000 (13:04 +0300)
committerDmitry Batkovich <dmitry.batkovich@jetbrains.com>
Wed, 16 Nov 2016 10:04:17 +0000 (13:04 +0300)
33 files changed:
platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java
platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java
platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java
platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java
platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java [moved from platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java with 65% similarity]
platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java [new file with mode: 0644]
platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java [new file with mode: 0644]
platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java [moved from platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java with 97% similarity]
platform/util/src/com/intellij/util/indexing/DataIndexer.java [moved from platform/indexing-api/src/com/intellij/util/indexing/DataIndexer.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/DebugAssertions.java [moved from platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java with 80% similarity]
platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java [moved from platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java with 95% similarity]
platform/util/src/com/intellij/util/indexing/ID.java [moved from platform/core-api/src/com/intellij/util/indexing/ID.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/IndexExtension.java [moved from platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/IndexStorage.java [moved from platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java with 83% similarity]
platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java [moved from platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java with 95% similarity]
platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java [moved from platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java with 93% similarity]
platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java [new file with mode: 0644]
platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java [new file with mode: 0644]
platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java [new file with mode: 0644]
platform/util/src/com/intellij/util/indexing/StorageException.java [moved from platform/lang-impl/src/com/intellij/util/indexing/StorageException.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/UpdatableValueContainer.java [moved from platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/UpdateData.java [moved from platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/ValueContainer.java [moved from platform/lang-impl/src/com/intellij/util/indexing/ValueContainer.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java [moved from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java with 96% similarity]
platform/util/src/com/intellij/util/indexing/ValueContainerMap.java [moved from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java with 94% similarity]
platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java with 99% similarity]
platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java with 99% similarity]
platform/util/src/com/intellij/util/indexing/containers/IdSet.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java with 97% similarity]
platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java with 98% similarity]
platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java with 100% similarity]
platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java [moved from platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java with 100% similarity]
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java

index 7dc4c304040a45169f6a3c902c42fe36fe7d5fa5..0d8520ac5ad9fe58c951c850d06e5c41d867d95d 100644 (file)
@@ -25,12 +25,10 @@ import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.search.ProjectAndLibrariesScope;
 import com.intellij.psi.search.ProjectScopeImpl;
 import com.intellij.util.Processor;
-import com.intellij.util.Processors;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.ThrowableRunnable;
 import com.intellij.util.containers.ConcurrentIntObjectMap;
 import com.intellij.util.containers.ContainerUtil;
-import com.intellij.util.containers.SLRUCache;
 import com.intellij.util.io.*;
 import com.intellij.util.io.DataOutputStream;
 import gnu.trove.TIntHashSet;
@@ -39,31 +37,18 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.*;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 20, 2007
  */
-public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
+public final class MapIndexStorage<Key, Value> extends MapIndexStorageBase<Key, Value> implements VfsAwareIndexStorage<Key, Value> {
   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
   private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
   private final boolean myBuildKeyHashToVirtualFileMapping;
-  private PersistentMap<Key, ValueContainer<Value>> myMap;
   private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping;
-  private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
   private volatile int myLastScannedId;
-  private final File myBaseStorageFile;
-  private final KeyDescriptor<Key> myKeyDescriptor;
-  private final int myCacheSize;
 
-  private final Lock l = new ReentrantLock();
-  private final DataExternalizer<Value> myDataExternalizer;
-  private final boolean myKeyIsUniqueForIndexedFile;
   private static final ConcurrentIntObjectMap<Boolean> ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap();
 
   public MapIndexStorage(@NotNull File storageFile,
@@ -80,85 +65,18 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
                          final int cacheSize,
                          boolean keyIsUniqueForIndexedFile,
                          boolean buildKeyHashToVirtualFileMapping) throws IOException {
-    myBaseStorageFile = storageFile;
-    myKeyDescriptor = keyDescriptor;
-    myCacheSize = cacheSize;
-    myDataExternalizer = valueExternalizer;
-    myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
+    super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile);
     myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
     initMapAndCache();
   }
 
-  private static final PersistentHashMapValueStorage.ExceptionalIOCancellationCallback ourProgressManagerCheckCancelledIOCanceller =
-    new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
-      @Override
-      public void checkCancellation() {
-        ProgressManager.checkCanceled();
-      }
-  };
-  private void initMapAndCache() throws IOException {
-    final ValueContainerMap<Key, Value> map;
-    PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(ourProgressManagerCheckCancelledIOCanceller);
-    PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
-    try {
-      map = new ValueContainerMap<>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
-    } finally {
-      PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
-      PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
-    }
-    myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
-      @Override
-      @NotNull
-      public ChangeTrackingValueContainer<Value> createValue(final Key key) {
-        return new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer<Value>() {
-          @NotNull
-          @Override
-          public Object getLock() {
-            return map.getDataAccessLock();
-          }
-
-          @Nullable
-          @Override
-          public ValueContainer<Value> compute() {
-            ValueContainer<Value> value;
-            try {
-              value = map.get(key);
-              if (value == null) {
-                value = new ValueContainerImpl<>();
-              }
-            }
-            catch (IOException e) {
-              throw new RuntimeException(e);
-            }
-            return value;
-          }
-        });
-      }
-
-      @Override
-      protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
-        if (valueContainer.isDirty()) {
-          try {
-            map.put(key, valueContainer);
-          }
-          catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        }
-      }
-    };
-
-    myMap = map;
-
+  @Override
+  protected void initMapAndCache() throws IOException {
+    super.initMapAndCache();
     myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ?
                                     new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true) : null;
   }
 
-  @NotNull
-  private File getStorageFile() {
-    return new File(myBaseStorageFile.getPath() + ".storage");
-  }
-
   @NotNull
   private File getProjectFile() {
     return new File(myBaseStorageFile.getPath() + ".project");
@@ -172,14 +90,12 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
       myKeyHashToVirtualFileMapping.getPagedFileStorage().unlock();
     }
   }
+
   @Override
   public void flush() {
     l.lock();
     try {
-      if (!myMap.isClosed()) {
-        myCache.clear();
-        if (myMap.isDirty()) myMap.force();
-      }
+      super.flush();
       if (myKeyHashToVirtualFileMapping != null && myKeyHashToVirtualFileMapping.isDirty()) {
         withLock(() -> myKeyHashToVirtualFileMapping.force());
       }
@@ -191,57 +107,34 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
 
   @Override
   public void close() throws StorageException {
+    super.close();
     try {
-      flush();
       if (myKeyHashToVirtualFileMapping != null) {
         withLock(() -> myKeyHashToVirtualFileMapping.close());
       }
-      myMap.close();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      unwrapCauseAndRethrow(e);
     }
   }
 
   @Override
   public void clear() throws StorageException{
     try {
-      myMap.close();
       if (myKeyHashToVirtualFileMapping != null) {
         withLock(() -> myKeyHashToVirtualFileMapping.close());
       }
     }
-    catch (IOException|RuntimeException e) {
+    catch (RuntimeException e) {
       LOG.error(e);
     }
     try {
-      IOUtil.deleteAllFilesStartingWith(getStorageFile());
       if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
-      initMapAndCache();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      unwrapCauseAndRethrow(e);
     }
+    super.clear();
   }
 
   @Override
@@ -302,20 +195,13 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
           return processor.process(key);
         });
       }
-      return myMap.processKeys(processor);
+      return processKeys(processor);
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      return unwrapCauseAndRethrow(e);
     }
     finally {
       l.unlock();
@@ -406,36 +292,6 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
     return new File(getSessionDir(), getProjectFile().getName() + "." + project.hashCode() + "." + id + "." + scope.isSearchInLibraries());
   }
 
-  @NotNull
-  @Override
-  public Collection<Key> getKeys() throws StorageException {
-    List<Key> keys = new ArrayList<>();
-    processKeys(Processors.cancelableCollectProcessor(keys), null, null);
-    return keys;
-  }
-
-  @Override
-  @NotNull
-  public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
-    l.lock();
-    try {
-      return myCache.get(key);
-    }
-    catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
-    }
-    finally {
-      l.unlock();
-    }
-  }
-
   @Override
   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
     try {
@@ -447,41 +303,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
           myLastScannedId = 0;
         }
       }
-
-      myMap.markDirty();
-      if (!myKeyIsUniqueForIndexedFile) {
-        read(key).addValue(inputId, value);
-        return;
-      }
-
-      ChangeTrackingValueContainer<Value> cached;
-      try {
-        l.lock();
-        cached = myCache.getIfCached(key);
-      } finally {
-        l.unlock();
-      }
-
-      if (cached != null) {
-        cached.addValue(inputId, value);
-        return;
-      }
-      // do not pollute the cache with keys unique to indexed file
-      ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<>(null);
-      valueContainer.addValue(inputId, value);
-      myMap.put(key, valueContainer);
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
-    }
-  }
-
-  @Override
-  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
-    try {
-      myMap.markDirty();
-      // important: assuming the key exists in the index
-      read(key).removeAssociatedValue(inputId);
+      super.addValue(key, inputId, value);
     }
     catch (IOException e) {
       throw new StorageException(e);
index 44a53d3375be3475dbcd708a27c600940633a8bc..b1b92efe625bf79c15f3e903f34a33dec2a2a235 100644 (file)
@@ -24,7 +24,6 @@ import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.progress.ProcessCanceledException;
-import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.util.*;
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
 import com.intellij.openapi.util.io.ByteSequence;
@@ -46,72 +45,43 @@ import java.io.*;
 import java.nio.charset.Charset;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 10, 2007
  */
-public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Value, Input> {
+public class MapReduceIndex<Key, Value, Input> extends MapReduceIndexBase<Key,Value, Input> implements UpdatableIndex<Key,Value, Input>{
   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
   private static final int NULL_MAPPING = 0;
-  @NotNull private final ID<Key, Value> myIndexId;
-  private final DataIndexer<Key, Value, Input> myIndexer;
-  @NotNull protected final IndexStorage<Key, Value> myStorage;
   private final boolean myHasSnapshotMapping;
 
-  private final DataExternalizer<Value> myValueExternalizer;
   private final DataExternalizer<Collection<Key>> mySnapshotIndexExternalizer;
   private final boolean myIsPsiBackedIndex;
-  private final IndexExtension<Key, Value, Input> myExtension;
   private final AtomicBoolean myInMemoryMode = new AtomicBoolean();
-  private final AtomicLong myModificationStamp = new AtomicLong();
   private final TIntObjectHashMap<Collection<Key>> myInMemoryKeys = new TIntObjectHashMap<>();
 
   private PersistentHashMap<Integer, ByteSequence> myContents;
   private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
-  @Nullable protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
   private PersistentHashMap<Integer, String> myIndexingTrace;
 
-  private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
-
-  private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
-    @Override
-    public void run() {
-      try {
-        if (myStorage instanceof MemoryIndexStorage) {
-          Lock writeLock = getWriteLock();
-          if (writeLock.tryLock()) {
-            try {
-              ((MemoryIndexStorage<Key, Value>)myStorage).clearCaches();
-            } finally {
-              writeLock.unlock();
-            }
-          }
-        }
-        flush();
-      } catch (StorageException e) {
-        LOG.info(e);
-        requestRebuild(null);
-      }
+  static {
+    if (!DebugAssertions.DEBUG) {
+      final Application app = ApplicationManager.getApplication();
+      DebugAssertions.DEBUG = app.isEAP() || app.isInternal();
     }
-  });
+  }
 
   public MapReduceIndex(IndexExtension<Key, Value, Input> extension,
                         @NotNull IndexStorage<Key, Value> storage) throws IOException {
-    myIndexId = extension.getName();
-    myExtension = extension;
+    super(extension, storage, false);
+
     SharedIndicesData.registerIndex(myIndexId, extension);
-    myIndexer = extension.getIndexer();
-    myStorage = storage;
     myHasSnapshotMapping = extension instanceof FileBasedIndexExtension &&
                            ((FileBasedIndexExtension<Key, Value>)extension).hasSnapshotMapping() &&
                            IdIndex.ourSnapshotMappingsEnabled;
 
     mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor());
-    myValueExternalizer = extension.getValueExternalizer();
     myIsPsiBackedIndex = extension instanceof PsiDependentIndex;
 
     myContents = createContentsIndex(); // todo
@@ -146,21 +116,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     }
   }
 
-  private static <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
-                                                                           ID<K, ?> indexId,
-                                                                           KeyDescriptor<K> keyDescriptor) {
-    DataExternalizer<Collection<K>> externalizer;
-    if (extension instanceof CustomInputsIndexFileBasedIndexExtension) {
-      externalizer = ((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer();
-    } else {
-      externalizer = new InputIndexDataExternalizer<>(keyDescriptor, indexId);
-    }
-    return externalizer;
-  }
-
   @NotNull
-  private static <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension <K, ?, ?> extension,
-                                                                                      @NotNull MemoryIndexStorage<K, ?> storage)
+  private <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension <K, ?, ?> extension,
+                                                                                @NotNull MemoryIndexStorageBase<K, ?, ?> storage)
     throws IOException {
     ID<K, ?> indexId = extension.getName();
     KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
@@ -186,41 +144,19 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     }
   }
 
-  @NotNull
-  public IndexStorage<Key, Value> getStorage() {
-    return myStorage;
-  }
-
   @Override
-  public void clear() throws StorageException {
-    try {
-      getWriteLock().lock();
-      myStorage.clear();
-      if (myInputsIndex != null) {
-        cleanMapping(myInputsIndex);
-        myInputsIndex = createInputsIndex();
-      }
-      if (myInputsSnapshotMapping != null) {
-        cleanMapping(myInputsSnapshotMapping);
-        myInputsSnapshotMapping = createInputSnapshotMapping();
-      }
-      if (myIndexingTrace != null) {
-        cleanMapping(myIndexingTrace);
-        myIndexingTrace = createIndexingTrace();
-      }
-      if (myContents != null) {
-        cleanMapping(myContents);
-        myContents = createContentsIndex();
-      }
-    }
-    catch (StorageException e) {
-      LOG.error(e);
+  protected void cleanMappings() throws IOException {
+    if (myInputsSnapshotMapping != null) {
+      cleanMapping(myInputsSnapshotMapping);
+      myInputsSnapshotMapping = createInputSnapshotMapping();
     }
-    catch (IOException e) {
-      LOG.error(e);
+    if (myIndexingTrace != null) {
+      cleanMapping(myIndexingTrace);
+      myIndexingTrace = createIndexingTrace();
     }
-    finally {
-      getWriteLock().unlock();
+    if (myContents != null) {
+      cleanMapping(myContents);
+      myContents = createContentsIndex();
     }
   }
 
@@ -265,72 +201,18 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     }
   }
 
-  private static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
-    final File baseFile = index.getBaseFile();
-    try {
-      index.close();
-    }
-    catch (Throwable ignored) {
-    }
-
-    IOUtil.deleteAllFilesStartingWith(baseFile);
-  }
-
   @Override
-  public void flush() throws StorageException{
-    try {
-      getReadLock().lock();
-      doForce(myInputsIndex);
-      doForce(myInputsSnapshotMapping);
-      doForce(myIndexingTrace);
-      doForce(myContents);
-      myStorage.flush();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
-    }
-    catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof StorageException || cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      else {
-        throw e;
-      }
-    }
-    finally {
-      getReadLock().unlock();
-    }
-  }
-
-  private static void doForce(@Nullable PersistentHashMap<?, ?> inputsIndex) {
-    if (inputsIndex != null && inputsIndex.isDirty()) {
-      inputsIndex.force();
-    }
+  protected void doForce() {
+    doForce(myInputsSnapshotMapping);
+    doForce(myIndexingTrace);
+    doForce(myContents);
   }
 
   @Override
-  public void dispose() {
-    myLowMemoryFlusher.stop();
-    final Lock lock = getWriteLock();
-    try {
-      lock.lock();
-      try {
-        myStorage.close();
-      }
-      finally {
-        doClose(myInputsIndex);
-        doClose(myInputsSnapshotMapping);
-        doClose(myIndexingTrace);
-        doClose(myContents);
-      }
-    }
-    catch (StorageException e) {
-      LOG.error(e);
-    }
-    finally {
-      lock.unlock();
-    }
+  protected void doClose() {
+    doClose(myInputsSnapshotMapping);
+    doClose(myIndexingTrace);
+    doClose(myContents);
   }
 
   @Override
@@ -348,35 +230,12 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId);
   }
 
-  private static void doClose(@Nullable PersistentHashMap<?, ?> index) {
-    if (index != null) {
-      try {
-        index.close();
-      }
-      catch (IOException e) {
-        LOG.error(e);
-      }
-    }
-  }
-
-  @NotNull
-  @Override
-  public final Lock getReadLock() {
-    return myLock.readLock();
-  }
-
-  @NotNull
-  @Override
-  public final Lock getWriteLock() {
-    return myLock.writeLock();
-  }
-
   @Override
   public boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
     final Lock lock = getReadLock();
     try {
       lock.lock();
-      return myStorage.processKeys(processor, scope, idFilter);
+      return ((VfsAwareIndexStorage<Key, Value>)myStorage).processKeys(processor, scope, idFilter);
     }
     finally {
       lock.unlock();
@@ -384,22 +243,8 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
   }
 
   @Override
-  @NotNull
-  public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
-    final Lock lock = getReadLock();
-    try {
-      lock.lock();
-      ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
-      return myStorage.read(key);
-    }
-    finally {
-      ValueContainerImpl.ourDebugIndexInfo.set(null);
-      lock.unlock();
-    }
-  }
-
   protected PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
-    return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage<Key, ?>)myStorage);
+    return createIdToDataKeysIndex(myExtension, (MemoryIndexStorageBase<Key, ?, ?>)myStorage);
   }
 
   private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);
@@ -457,9 +302,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
 
     if (data == null) {
       data = content != null ? myIndexer.map(content) : Collections.emptyMap();
-      if (DebugAssertions.DEBUG) {
-        checkValuesHaveProperEqualsAndHashCode(data);
-      }
+      checkValuesHaveProperEqualsAndHashCode(data);
     }
 
     if (hashId != null && !havePersistentData) {
@@ -477,7 +320,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
         }
       }
     }
-    ProgressManager.checkCanceled();
+    myStorage.checkCanceled();
 
     UpdateData<Key, Value> optimizedUpdateData = null;
     final NotNullComputable<Collection<Key>> oldKeysGetter;
@@ -559,15 +402,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
         throw new RuntimeException(ex);
       }
     } else {
-      oldKeysGetter = () -> {
-        try {
-          Collection<Key> oldKeys = readInputKeys(inputId);
-          return oldKeys == null? Collections.<Key>emptyList() : oldKeys;
-        }
-        catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-      };
+      oldKeysGetter = createOldKeysGetterByInputIndex(inputId);
       savedInputId = inputId;
     }
 
@@ -580,13 +415,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
       }
       catch (StorageException|ProcessCanceledException ex) {
         LOG.info("Exception during updateWithMap:" + ex);
-        Application application = ApplicationManager.getApplication();
-        if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
-          // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
-          application.invokeLater(() -> requestRebuild(ex), ModalityState.any());
-        } else {
-          requestRebuild(ex);
-        }
+        requestRebuild(ex);
         return Boolean.FALSE;
       }
 
@@ -594,15 +423,35 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     };
   }
 
-  protected void requestRebuild(@Nullable Exception ex) {
-    if (ex == null) {
-      FileBasedIndex.getInstance().requestRebuild(myIndexId);
-    }
-    else {
-      FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
+  @Override
+  public void requestRebuild(@Nullable Exception ex) {
+    Runnable action = () -> {
+      if (ex == null) {
+        FileBasedIndex.getInstance().requestRebuild(myIndexId);
+      }
+      else {
+        FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
+      }
+    };
+    Application application = ApplicationManager.getApplication();
+    if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
+      // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
+      application.invokeLater(action, ModalityState.any());
+    } else {
+      action.run();
     }
   }
 
+  @Override
+  protected <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
+                                                                              ID<K, ?> indexId,
+                                                                              KeyDescriptor<K> keyDescriptor) {
+    return extension instanceof CustomInputsIndexFileBasedIndexExtension ?
+           ((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer() :
+           super.createInputsIndexExternalizer(extension, indexId, keyDescriptor);
+  }
+
+  @Override
   protected UpdateData<Key, Value> buildUpdateData(Map<Key, Value> data, NotNullComputable<Collection<Key>> oldKeysGetter, int savedInputId) {
     return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter);
   }
@@ -678,7 +527,8 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, savedInputId);
   }
 
-  private Collection<Key> readInputKeys(int inputId) throws IOException {
+  @Override
+  protected Collection<Key> readInputKeys(int inputId) throws IOException {
     if (myInMemoryMode.get()) {
       synchronized (myInMemoryKeys) {
         Collection<Key> keys = myInMemoryKeys.get(inputId);
@@ -710,10 +560,11 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
       }
       return keys;
     }
-    return myInputsIndex != null ? myInputsIndex.get(inputId) : null;
+    return super.readInputKeys(inputId);
   }
 
-  private void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
+  @Override
+  protected void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
     if (myInMemoryMode.get()) {
       synchronized (myInMemoryKeys) {
         myInMemoryKeys.put(inputId, newData.keySet());
@@ -722,14 +573,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
       if (myHasSnapshotMapping) {
         saveInputHashId(inputId, savedInputId);
       } else {
-        if (myInputsIndex != null) {
-          if (newData.size() > 0) {
-            myInputsIndex.put(inputId, newData.keySet());
-          }
-          else {
-            myInputsIndex.remove(inputId);
-          }
-        }
+        super.saveInputKeys(inputId, savedInputId, newData);
 
         if (SharedIndicesData.ourFileSharedIndicesEnabled) {
           Set<Key> newKeys = newData.keySet();
@@ -740,32 +584,6 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
     }
   }
 
-  private void checkValuesHaveProperEqualsAndHashCode(Map<Key, Value> data) {
-    for(Map.Entry<Key, Value> e: data.entrySet()) {
-      final Value value = e.getValue();
-      if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
-        LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter");
-      }
-
-      if (myValueExternalizer != null) {
-        try {
-          final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
-          DataOutputStream outputStream = new DataOutputStream(out);
-          myValueExternalizer.save(outputStream, value);
-          outputStream.close();
-          final Value deserializedValue =
-            myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
-
-          if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
-            LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
-          }
-        } catch (IOException ex) {
-          LOG.error(ex);
-        }
-      }
-    }
-  }
-
   private StringBuilder buildDiff(Map<Key, Value> data, Map<Key, Value> contentData) {
     StringBuilder moreInfo = new StringBuilder();
     if (contentData.size() != data.size()) {
@@ -906,84 +724,4 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
 
   private static final com.intellij.openapi.util.Key<Integer> ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id");
   private static final com.intellij.openapi.util.Key<Integer> ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id");
-
-  public IndexExtension<Key, Value, Input> getExtension() {
-    return myExtension;
-  }
-
-  public long getModificationStamp() {
-    return myModificationStamp.get();
-  }
-
-  public class SimpleUpdateData extends UpdateData<Key, Value> {
-    private final int savedInputId;
-    private final @NotNull Map<Key, Value> newData;
-    protected final @NotNull NotNullComputable<Collection<Key>> oldKeysGetter;
-
-    public SimpleUpdateData(ID<Key,Value> indexId, int id, @NotNull Map<Key, Value> data, @NotNull NotNullComputable<Collection<Key>> getter) {
-      super(indexId);
-      savedInputId = id;
-      newData = data;
-      oldKeysGetter = getter;
-    }
-
-    public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
-      MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer);
-    }
-
-    public void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException {
-      MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData);
-    }
-
-    @Override
-    public void save(int inputId) throws IOException {
-      saveInputKeys(inputId, savedInputId, newData);
-    }
-
-    public @NotNull Map<Key, Value> getNewData() {
-      return newData;
-    }
-  }
-
-  private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>
-    myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>() {
-    @Override
-    public void process(Key key, int inputId) throws StorageException {
-      myModificationStamp.incrementAndGet();
-      myStorage.removeAllValues(key, inputId);
-    }
-  };
-
-  private final MapDiffUpdateData.AddedKeyProcessor<Key, Value> myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor<Key, Value>() {
-    @Override
-    public void process(Key key, Value value, int inputId) throws StorageException {
-      myModificationStamp.incrementAndGet();
-      myStorage.addValue(key, inputId, value);
-    }
-  };
-
-  protected void updateWithMap(final int inputId,
-                               @NotNull UpdateData<Key, Value> updateData) throws StorageException {
-    getWriteLock().lock();
-    try {
-      try {
-        ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
-        updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation);
-        updateData.iterateAddedKeys(inputId, myAddedKeyProcessor);
-        updateData.save(inputId);
-      }
-      catch (ProcessCanceledException pce) {
-        throw pce; // extra care
-      }
-      catch (Throwable e) { // e.g. IOException, AssertionError
-        throw new StorageException(e);
-      }
-      finally {
-        ValueContainerImpl.ourDebugIndexInfo.set(null);
-      }
-    }
-    finally {
-      getWriteLock().unlock();
-    }
-  }
 }
index 858d93c09a4a7e3f5d47165659c9bf4a17d4c5fd..cf54797c31ba47c47bf0f05d305246343710dada 100644 (file)
@@ -18,199 +18,22 @@ package com.intellij.util.indexing;
 
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.Processor;
-import com.intellij.util.Processors;
-import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.IOException;
-import java.util.*;
-
 /**
  * This storage is needed for indexing yet unsaved data without saving those changes to 'main' backend storage
  *
  * @author Eugene Zhuravlev
  *         Date: Dec 10, 2007
  */
-public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value> {
-  private final Map<Key, ChangeTrackingValueContainer<Value>> myMap = new HashMap<>();
-  @NotNull
-  private final IndexStorage<Key, Value> myBackendStorage;
-  private final List<BufferingStateListener> myListeners = ContainerUtil.createLockFreeCopyOnWriteList();
-  private final ID<?, ?> myIndexId;
-  private boolean myBufferingEnabled;
-
-  public interface BufferingStateListener {
-    void bufferingStateChanged(boolean newState);
-
-    void memoryStorageCleared();
-  }
-
-  public MemoryIndexStorage(@NotNull IndexStorage<Key, Value> backend) {
-    this(backend, null);
-  }
-
-  public MemoryIndexStorage(@NotNull IndexStorage<Key, Value> backend, ID<?, ?> indexId) {
-    myBackendStorage = backend;
-    myIndexId = indexId;
-  }
-
-  @NotNull
-  public IndexStorage<Key, Value> getBackendStorage() {
-    return myBackendStorage;
-  }
-
-  public void addBufferingStateListener(@NotNull BufferingStateListener listener) {
-    myListeners.add(listener);
-  }
-
-  public void removeBufferingStateListener(@NotNull BufferingStateListener listener) {
-    myListeners.remove(listener);
-  }
-
-  public void setBufferingEnabled(boolean enabled) {
-    final boolean wasEnabled = myBufferingEnabled;
-    assert wasEnabled != enabled;
-
-    myBufferingEnabled = enabled;
-    for (BufferingStateListener listener : myListeners) {
-      listener.bufferingStateChanged(enabled);
-    }
-  }
-
-  public boolean isBufferingEnabled() {
-    return myBufferingEnabled;
-  }
-
-  public void clearMemoryMap() {
-    myMap.clear();
-  }
-
-  public void fireMemoryStorageCleared() {
-    for (BufferingStateListener listener : myListeners) {
-      listener.memoryStorageCleared();
-    }
-  }
-
-  void clearCaches() {
-    if (myMap.size() == 0) return;
-
-    if (DebugAssertions.DEBUG) {
-      String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size();
-      FileBasedIndexImpl.LOG.info(message);
-    }
-
-    for(ChangeTrackingValueContainer<Value> v:myMap.values()) {
-      v.dropMergedData();
-    }
-  }
-
-  @Override
-  public void close() throws StorageException {
-    myBackendStorage.close();
-  }
-
-  @Override
-  public void clear() throws StorageException {
-    clearMemoryMap();
-    myBackendStorage.clear();
-  }
-
-  @Override
-  public void flush() throws IOException {
-    myBackendStorage.flush();
-  }
-
-  @NotNull
-  @Override
-  public Collection<Key> getKeys() throws StorageException {
-    final Set<Key> keys = new HashSet<>();
-    processKeys(Processors.cancelableCollectProcessor(keys), null, null);
-    return keys;
+public class MemoryIndexStorage<Key, Value> extends MemoryIndexStorageBase<Key, Value, VfsAwareIndexStorage<Key, Value>> implements VfsAwareIndexStorage<Key, Value> {
+  public MemoryIndexStorage(@NotNull VfsAwareIndexStorage<Key, Value> backend, ID<?, ?> indexId) {
+    super(backend, indexId);
   }
 
   @Override
   public boolean processKeys(@NotNull final Processor<Key> processor, GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
-    final Set<Key> stopList = new HashSet<>();
-
-    Processor<Key> decoratingProcessor = key -> {
-      if (stopList.contains(key)) return true;
-
-      final UpdatableValueContainer<Value> container = myMap.get(key);
-      if (container != null && container.size() == 0) {
-        return true;
-      }
-      return processor.process(key);
-    };
-
-    for (Key key : myMap.keySet()) {
-      if (!decoratingProcessor.process(key)) {
-        return false;
-      }
-      stopList.add(key);
-    }
-    return myBackendStorage.processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter);
-  }
-
-  @Override
-  public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
-    if (myBufferingEnabled) {
-      getMemValueContainer(key).addValue(inputId, value);
-      return;
-    }
-    final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
-    if (valueContainer != null) {
-      valueContainer.dropMergedData();
-    }
-
-    myBackendStorage.addValue(key, inputId, value);
-  }
-
-  @Override
-  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
-    if (myBufferingEnabled) {
-      getMemValueContainer(key).removeAssociatedValue(inputId);
-      return;
-    }
-    final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
-    if (valueContainer != null) {
-      valueContainer.dropMergedData();
-    }
-
-    myBackendStorage.removeAllValues(key, inputId);
-  }
-
-  private UpdatableValueContainer<Value> getMemValueContainer(final Key key) {
-    ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
-    if (valueContainer == null) {
-      valueContainer = new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer<Value>() {
-        @Override
-        public Object getLock() {
-          return this;
-        }
-
-        @Override
-        public ValueContainer<Value> compute() {
-          try {
-            return myBackendStorage.read(key);
-          }
-          catch (StorageException e) {
-            throw new RuntimeException(e);
-          }
-        }
-      });
-      myMap.put(key, valueContainer);
-    }
-    return valueContainer;
-  }
-
-  @Override
-  @NotNull
-  public ValueContainer<Value> read(final Key key) throws StorageException {
-    final ValueContainer<Value> valueContainer = myMap.get(key);
-    if (valueContainer != null) {
-      return valueContainer;
-    }
-
-    return myBackendStorage.read(key);
+    final Processor<Key> uniqueResultProcessor = processCacheFirstProcessor(processor);
+    return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor, scope, idFilter);
   }
 }
index 3d8c5fb15d9b2f0a1bd0a11db3476cbe0674ac69..19facea95a81ad89bc51f6ac16bae152c5b8e301 100644 (file)
 
 package com.intellij.util.indexing;
 
-import com.intellij.openapi.util.Computable;
+import com.intellij.openapi.progress.ProcessCanceledException;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.search.GlobalSearchScope;
+import com.intellij.util.Processor;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.concurrent.locks.Lock;
-
 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 10, 2007
  */
-public interface UpdatableIndex<Key, Value, Input> extends AbstractIndex<Key,Value> {
-
-  void clear() throws StorageException;
-
-  void flush() throws StorageException;
-
-  /**
-   * @param inputId *positive* id of content.
-   */
-  @NotNull
-  Computable<Boolean> update(int inputId, @Nullable Input content);
-
-  @NotNull
-  Lock getReadLock();
-
-  @NotNull
-  Lock getWriteLock();
-  
-  void dispose();
+public interface UpdatableIndex<Key, Value, Input> extends AbstractUpdatableIndex<Key,Value, Input> {
+  boolean processAllKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
 
   void setIndexedStateForFile(int fileId, @NotNull VirtualFile file);
   void resetIndexedStateForFile(int fileId);
 
   boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file);
+
+  @Override
+  default boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException {
+    return processAllKeys(processor, null, null);
+  }
+
+  @NotNull
+  @Override
+  default Class<? extends RuntimeException> getProcessCanceledExceptionClass() {
+    return ProcessCanceledException.class;
+  }
 }
similarity index 65%
rename from platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java
rename to platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java
index 34b7937cb7a17a44a1b93c8fe5ebb776f9fbf50a..f6209fc03bb28b497920d5d766664a730b99694b 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package com.intellij.util.indexing;
 
+import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.Processor;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-/**
- * @author Eugene Zhuravlev
- *         Date: Dec 24, 2007
- */
-public interface AbstractIndex<Key, Value> {
-  @NotNull
-  ValueContainer<Value> getData(@NotNull Key key) throws StorageException;
+public interface VfsAwareIndexStorage<Key, Value> extends IndexStorage<Key, Value> {
+
+  boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
 
-  boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
+  @Override
+  default void checkCanceled() {
+    ProgressManager.checkCanceled();
+  }
 }
diff --git a/platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java b/platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java
new file mode 100644 (file)
index 0000000..91aec4b
--- /dev/null
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.util.Computable;
+import com.intellij.util.Processor;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.concurrent.locks.Lock;
+
+public interface AbstractUpdatableIndex<Key, Value, Input> {
+  void clear() throws StorageException;
+
+  void flush() throws StorageException;
+
+  /**
+   * @param inputId *positive* id of content.
+   */
+  @NotNull
+  Computable<Boolean> update(int inputId, @Nullable Input content);
+
+  @NotNull
+  ValueContainer<Value> getData(@NotNull Key key) throws StorageException;
+
+  boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException;
+
+  @NotNull
+  Lock getReadLock();
+
+  @NotNull
+  Lock getWriteLock();
+
+  void dispose();
+
+  void requestRebuild(@Nullable Exception ex);
+
+  @NotNull
+  Class<? extends RuntimeException> getProcessCanceledExceptionClass();
+}
diff --git a/platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java b/platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java
new file mode 100644 (file)
index 0000000..326cdd9
--- /dev/null
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.util.CommonProcessors;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Collection;
+
+class CancelableCollectProcessor<T> extends CommonProcessors.CollectProcessor<T> {
+  private final Runnable myCancelChecker;
+
+  public CancelableCollectProcessor(@NotNull Collection<T> collection, @NotNull Runnable cancelChecker) {
+    super(collection);
+    myCancelChecker = cancelChecker;
+  }
+
+  @Override
+  public boolean process(T o) {
+    myCancelChecker.run();
+    return super.process(o);
+  }
+}
similarity index 97%
rename from platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java
rename to platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java
index 467ba0cf3202c1b5a8c52d9aa9f2f603db660b20..2158a10f23eaabc41adbcac5996cf466389a0f1d 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -54,7 +54,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
       merged.addValue(inputId, value);
     }
 
-    if (myAdded == null) myAdded = new ValueContainerImpl<>();
+    if (myAdded == null) myAdded = new ValueContainerImpl<Value>();
     myAdded.addValue(inputId, value);
   }
 
@@ -131,7 +131,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
           (newMerged.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD ||
            (myAdded != null && myAdded.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD))) {
         // Calculate file ids that have Value mapped to avoid O(NumberOfValuesInMerged) during removal
-        fileId2ValueMapping = new FileId2ValueMapping<>(newMerged);
+        fileId2ValueMapping = new FileId2ValueMapping<Value>(newMerged);
       }
       final FileId2ValueMapping<Value> finalFileId2ValueMapping = fileId2ValueMapping;
       if (myInvalidated != null) {
similarity index 80%
rename from platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java
rename to platform/util/src/com/intellij/util/indexing/DebugAssertions.java
index e5b1248800327938416b990da27e85ce6805ae9d..428464fb8436afd7116975f97dba2ab40b0e8d6f 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
  */
 package com.intellij.util.indexing;
 
-import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.containers.hash.LinkedHashMap;
@@ -27,9 +26,10 @@ import java.util.Formatter;
 public class DebugAssertions {
   private static final Logger LOG = Logger.getInstance(DebugAssertions.class);
 
-  public static final boolean DEBUG = SystemProperties.getBooleanProperty(
+  @SuppressWarnings("StaticNonFinalField")
+  public static volatile boolean DEBUG = SystemProperties.getBooleanProperty(
     "intellij.idea.indices.debug",
-    ApplicationManager.getApplication().isInternal() || ApplicationManager.getApplication().isEAP()
+    false
   );
 
   public static final boolean EXTRA_SANITY_CHECKS = SystemProperties.getBooleanProperty(
@@ -56,9 +56,9 @@ public class DebugAssertions {
   static <Key> boolean equals(Collection<Key> keys, Collection<Key> keys2, KeyDescriptor<Key> keyDescriptor) {
     if (keys == null && keys2 == null) return true;
     if (keys == null || keys2 == null || keys.size() != keys2.size()) return false;
-    LinkedHashMap<Key, Boolean> map = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
+    LinkedHashMap<Key, Boolean> map = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
     for(Key key:keys) map.put(key, Boolean.TRUE);
-    LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
+    LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
     for(Key key:keys2) map2.put(key, Boolean.TRUE);
     return map.equals(map2);
   }
similarity index 95%
rename from platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java
rename to platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java
index 5a6b3357bc99329991d5077169fed5447865f4d5..3527503b464dc07f1dc8208e4a90cd540db1ebba 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ class FileId2ValueMapping<Value> {
   private boolean myOnePerFileValidationEnabled = true;
 
   FileId2ValueMapping(ValueContainerImpl<Value> _valueContainer) {
-    id2ValueMap = new TIntObjectHashMap<>();
+    id2ValueMap = new TIntObjectHashMap<Value>();
     valueContainer = _valueContainer;
 
     TIntArrayList removedFileIdList = null;
@@ -45,7 +45,7 @@ class FileId2ValueMapping<Value> {
         if (previousValue != null) {  // delay removal of duplicated id -> value mapping since it will affect valueIterator we are using
           if (removedFileIdList == null) {
             removedFileIdList = new TIntArrayList();
-            removedValueList = new SmartList<>();
+            removedValueList = new SmartList<Value>();
           }
           removedFileIdList.add(id);
           removedValueList.add(previousValue);
similarity index 83%
rename from platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java
rename to platform/util/src/com/intellij/util/indexing/IndexStorage.java
index 1020876ca2e46ea9eff4608586bf45060897fab6..74d1cb8451189557a83b27b34c3fa6f8f913d990 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
 
 package com.intellij.util.indexing;
 
-import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.Processor;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 import java.io.Flushable;
 import java.io.IOException;
@@ -40,7 +38,7 @@ public interface IndexStorage<Key, Value> extends Flushable {
   @NotNull
   ValueContainer<Value> read(Key key) throws StorageException;
 
-  boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
+  boolean processKeys(@NotNull Processor<Key> processor) throws StorageException;
 
   @NotNull
   Collection<Key> getKeys() throws StorageException;
@@ -49,4 +47,6 @@ public interface IndexStorage<Key, Value> extends Flushable {
 
   @Override
   void flush() throws IOException;
+
+  void checkCanceled();
 }
similarity index 95%
rename from platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java
rename to platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java
index b33074a2433ada55a80dd793dd3bf795597537a9..2f5686bd3852947a52584ccdb4ad07b1ef1f2559 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,7 +57,7 @@ public class InputIndexDataExternalizer<K> implements DataExternalizer<Collectio
   public Collection<K> read(@NotNull DataInput in) throws IOException {
     try {
       final int size = DataInputOutputUtil.readINT(in);
-      final List<K> list = new ArrayList<>(size);
+      final List<K> list = new ArrayList<K>(size);
       for (int idx = 0; idx < size; idx++) {
         list.add(myKeyDescriptor.read(in));
       }
similarity index 93%
rename from platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java
rename to platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java
index e4dcbfddfe0b9084f045643395ed46c69a3f4177..fd0f2f472daae91cac6fd8a1d8051946f03ea178 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2015 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
  */
 package com.intellij.util.indexing;
 
-import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.Comparing;
 import com.intellij.openapi.util.Ref;
@@ -44,7 +43,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
                                                            Map<Key, Value> data) throws StorageException {
     if (data instanceof THashMap) {
       // such map often (from IdIndex) contain 100x (avg ~240) of entries, also THashMap have no Entry inside so we optimize for gc too
-      final Ref<StorageException> exceptionRef = new Ref<>();
+      final Ref<StorageException> exceptionRef = new Ref<StorageException>();
       final boolean b = ((THashMap<Key, Value>)data).forEachEntry(new TObjectObjectProcedure<Key, Value>() {
         @Override
         public boolean execute(Key key, Value value) {
@@ -81,8 +80,6 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
     iterateRemovedKeys(removedOrChangedKeys.keySet(), inputId, consumer);
   }
 
-  private static final boolean DO_INFO_DUMP = ApplicationManager.getApplication().isInternal();
-
   private void calcDiff() throws StorageException {
     if (removedOrChangedKeys != null) return;
 
@@ -103,10 +100,10 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
           if (!Comparing.equal(newValueForKey, e.getValue()) ||
               newValueForKey == null && !newValue.containsKey(e.getKey())
             ) {
-            if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<>();
+            if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<Key, Value>();
             removedOrChangedKeys.put(e.getKey(), e.getValue());
             if (newValue.containsKey(e.getKey())) {
-              if (addedKeys == null) addedKeys = new THashMap<>();
+              if (addedKeys == null) addedKeys = new THashMap<Key, Value>();
               addedKeys.put(e.getKey(), newValueForKey);
             }
           }
@@ -130,7 +127,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
         }
         for (Map.Entry<Key, Value> e : newValue.entrySet()) {
           if (!currentValue.containsKey(e.getKey())) {
-            if (addedKeys == null) addedKeys = new THashMap<>();
+            if (addedKeys == null) addedKeys = new THashMap<Key, Value>();
             addedKeys.put(e.getKey(), e.getValue());
           }
         }
@@ -145,7 +142,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
       incrementalAdditions.addAndGet(addedKeys.size());
       incrementalRemovals.addAndGet(removedOrChangedKeys.size());
 
-      if ((totalRequests & 0xFFF) == 0 && DO_INFO_DUMP) {
+      if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) {
         Logger.getInstance(getClass()).info("Incremental index diff update:"+requests +
                                             ", removals:" + totalRemovals + "->" + incrementalRemovals +
                                             ", additions:" +totalAdditions + "->" +incrementalAdditions);
diff --git a/platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java b/platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java
new file mode 100644 (file)
index 0000000..3d61e06
--- /dev/null
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.util.Processor;
+import com.intellij.util.containers.SLRUCache;
+import com.intellij.util.io.*;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public abstract class MapIndexStorageBase<Key, Value> implements IndexStorage<Key, Value> {
+  private static final Logger LOG = Logger.getInstance(MapIndexStorageBase.class);
+  protected PersistentMap<Key, ValueContainer<Value>> myMap;
+  protected SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
+  protected final File myBaseStorageFile;
+  protected final KeyDescriptor<Key> myKeyDescriptor;
+  private final int myCacheSize;
+
+  protected final Lock l = new ReentrantLock();
+  private final DataExternalizer<Value> myDataExternalizer;
+  private final boolean myKeyIsUniqueForIndexedFile;
+
+  protected MapIndexStorageBase(@NotNull File storageFile,
+                             @NotNull KeyDescriptor<Key> keyDescriptor,
+                             @NotNull DataExternalizer<Value> valueExternalizer,
+                             final int cacheSize,
+                             boolean keyIsUniqueForIndexedFile) throws IOException {
+    myBaseStorageFile = storageFile;
+    myKeyDescriptor = keyDescriptor;
+    myCacheSize = cacheSize;
+    myDataExternalizer = valueExternalizer;
+    myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
+  }
+
+  protected void initMapAndCache() throws IOException {
+    final ValueContainerMap<Key, Value> map;
+    PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(
+      new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
+        @Override
+        public void checkCancellation() {
+          checkCanceled();
+        }
+      });
+    PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
+    try {
+      map = new ValueContainerMap<Key, Value>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
+    } finally {
+      PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
+      PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
+    }
+    myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
+      @Override
+      @NotNull
+      public ChangeTrackingValueContainer<Value> createValue(final Key key) {
+        return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
+          @NotNull
+          @Override
+          public Object getLock() {
+            return map.getDataAccessLock();
+          }
+
+          @Nullable
+          @Override
+          public ValueContainer<Value> compute() {
+            ValueContainer<Value> value;
+            try {
+              value = map.get(key);
+              if (value == null) {
+                value = new ValueContainerImpl<Value>();
+              }
+            }
+            catch (IOException e) {
+              throw new RuntimeException(e);
+            }
+            return value;
+          }
+        });
+      }
+
+      @Override
+      protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
+        if (valueContainer.isDirty()) {
+          try {
+            map.put(key, valueContainer);
+          }
+          catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        }
+      }
+    };
+
+    myMap = map;
+  }
+
+  @NotNull
+  private File getStorageFile() {
+    return new File(myBaseStorageFile.getPath() + ".storage");
+  }
+
+  @Override
+  public boolean processKeys(@NotNull Processor<Key> processor) throws StorageException {
+    l.lock();
+    try {
+      myCache.clear(); // this will ensure that all new keys are made into the map
+      return myMap.processKeys(processor);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      return unwrapCauseAndRethrow(e);
+    }
+    finally {
+      l.unlock();
+    }
+  }
+
+  @Override
+  public void flush() {
+    l.lock();
+    try {
+      if (!myMap.isClosed()) {
+        myCache.clear();
+        if (myMap.isDirty()) myMap.force();
+      }
+    }
+    finally {
+      l.unlock();
+    }
+  }
+
+  @Override
+  public void close() throws StorageException {
+    try {
+      flush();
+      myMap.close();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      unwrapCauseAndRethrow(e);
+    }
+  }
+
+  @Override
+  public void clear() throws StorageException{
+    try {
+      myMap.close();
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+    catch (RuntimeException e) {
+      LOG.error(e);
+    }
+    try {
+      IOUtil.deleteAllFilesStartingWith(getStorageFile());
+      initMapAndCache();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      unwrapCauseAndRethrow(e);
+    }
+  }
+
+  @NotNull
+  @Override
+  public Collection<Key> getKeys() throws StorageException {
+    List<Key> keys = new ArrayList<Key>();
+    processKeys(new CancelableCollectProcessor<Key>(keys, new Runnable() {
+      @Override
+      public void run() {checkCanceled();}
+    }));
+    return keys;
+  }
+
+  @Override
+  @NotNull
+  public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
+    l.lock();
+    try {
+      return myCache.get(key);
+    }
+    catch (RuntimeException e) {
+      return unwrapCauseAndRethrow(e);
+    }
+    finally {
+      l.unlock();
+    }
+  }
+
+  @Override
+  public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
+    try {
+      myMap.markDirty();
+      if (!myKeyIsUniqueForIndexedFile) {
+        read(key).addValue(inputId, value);
+        return;
+      }
+
+      ChangeTrackingValueContainer<Value> cached;
+      try {
+        l.lock();
+        cached = myCache.getIfCached(key);
+      } finally {
+        l.unlock();
+      }
+
+      if (cached != null) {
+        cached.addValue(inputId, value);
+        return;
+      }
+      // do not pollute the cache with keys unique to indexed file
+      ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<Value>(null);
+      valueContainer.addValue(inputId, value);
+      myMap.put(key, valueContainer);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+  }
+
+  @Override
+  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
+    try {
+      myMap.markDirty();
+      // important: assuming the key exists in the index
+      read(key).removeAssociatedValue(inputId);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+  }
+
+  protected static <T> T unwrapCauseAndRethrow(RuntimeException e) throws StorageException {
+    final Throwable cause = e.getCause();
+    if (cause instanceof IOException) {
+      throw new StorageException(cause);
+    }
+    if (cause instanceof StorageException) {
+      throw (StorageException)cause;
+    }
+    throw e;
+  }
+}
diff --git a/platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java b/platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java
new file mode 100644 (file)
index 0000000..8abc288
--- /dev/null
@@ -0,0 +1,441 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.Comparing;
+import com.intellij.openapi.util.Computable;
+import com.intellij.openapi.util.LowMemoryWatcher;
+import com.intellij.openapi.util.NotNullComputable;
+import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
+import com.intellij.util.Processor;
+import com.intellij.util.io.*;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public abstract class MapReduceIndexBase<Key,Value, Input> implements AbstractUpdatableIndex<Key, Value, Input> {
+  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
+  @Nullable protected final ID<Key, Value> myIndexId;
+  protected final DataIndexer<Key, Value, Input> myIndexer;
+  @NotNull protected final IndexStorage<Key, Value> myStorage;
+
+  protected final DataExternalizer<Value> myValueExternalizer;
+  protected final IndexExtension<Key, Value, Input> myExtension;
+  private final AtomicLong myModificationStamp = new AtomicLong();
+
+  @Nullable protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
+
+  private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
+
+  private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
+    @Override
+    public void run() {
+      try {
+        if (myStorage instanceof MemoryIndexStorageBase) {
+          Lock writeLock = getWriteLock();
+          if (writeLock.tryLock()) {
+            try {
+              ((MemoryIndexStorageBase<?, ?, ?>)myStorage).clearCaches();
+            } finally {
+              writeLock.unlock();
+            }
+          }
+        }
+        flush();
+      } catch (StorageException e) {
+        LOG.info(e);
+        requestRebuild(e);
+      }
+    }
+  });
+
+  protected MapReduceIndexBase(IndexExtension<Key, Value, Input> extension,
+                               @NotNull IndexStorage<Key, Value> storage,
+                               boolean createInputIndex) throws IOException {
+    myIndexId = extension.getName();
+    myExtension = extension;
+    myIndexer = extension.getIndexer();
+    myStorage = storage;
+    myValueExternalizer = extension.getValueExternalizer();
+
+    if (createInputIndex) {
+      myInputsIndex = createInputsIndex();
+    }
+
+  }
+
+  protected PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
+    return null;
+  }
+
+  protected <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
+                                                                                   ID<K, ?> indexId,
+                                                                                   KeyDescriptor<K> keyDescriptor) {
+    DataExternalizer<Collection<K>> externalizer;
+    externalizer = new InputIndexDataExternalizer<K>(keyDescriptor, indexId);
+    return externalizer;
+  }
+
+  @NotNull
+  public IndexStorage<Key, Value> getStorage() {
+    return myStorage;
+  }
+
+  @Override
+  public void clear() throws StorageException {
+    try {
+      getWriteLock().lock();
+      myStorage.clear();
+      if (myInputsIndex != null) {
+        cleanMapping(myInputsIndex);
+        myInputsIndex = createInputsIndex();
+      }
+      cleanMappings();
+    }
+    catch (StorageException e) {
+      LOG.error(e);
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+    finally {
+      getWriteLock().unlock();
+    }
+  }
+
+  protected void cleanMappings() throws IOException {
+
+  }
+
+  protected static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
+    final File baseFile = index.getBaseFile();
+    try {
+      index.close();
+    }
+    catch (Throwable ignored) {
+    }
+
+    IOUtil.deleteAllFilesStartingWith(baseFile);
+  }
+
+  @Override
+  public void flush() throws StorageException{
+    try {
+      getReadLock().lock();
+      doForce(myInputsIndex);
+      doForce();
+      myStorage.flush();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      final Throwable cause = e.getCause();
+      if (cause instanceof StorageException || cause instanceof IOException) {
+        throw new StorageException(cause);
+      }
+      else {
+        throw e;
+      }
+    }
+    finally {
+      getReadLock().unlock();
+    }
+  }
+
+  protected void doForce() {
+  }
+
+  protected static void doForce(PersistentHashMap<?, ?> inputsIndex) {
+    if (inputsIndex != null && inputsIndex.isDirty()) {
+      inputsIndex.force();
+    }
+  }
+
+  @Override
+  public void dispose() {
+    myLowMemoryFlusher.stop();
+    final Lock lock = getWriteLock();
+    try {
+      lock.lock();
+      try {
+        myStorage.close();
+      }
+      finally {
+        doClose(myInputsIndex);
+        doClose();
+      }
+    }
+    catch (StorageException e) {
+      LOG.error(e);
+    }
+    finally {
+      lock.unlock();
+    }
+  }
+
+  protected void doClose() {
+  }
+
+  protected static void doClose(PersistentHashMap<?, ?> index) {
+    if (index != null) {
+      try {
+        index.close();
+      }
+      catch (IOException e) {
+        LOG.error(e);
+      }
+    }
+  }
+
+  @NotNull
+  @Override
+  public final Lock getReadLock() {
+    return myLock.readLock();
+  }
+
+  @NotNull
+  @Override
+  public final Lock getWriteLock() {
+    return myLock.writeLock();
+  }
+
+  @Override
+  @NotNull
+  public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
+    final Lock lock = getReadLock();
+    try {
+      lock.lock();
+      ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
+      return myStorage.read(key);
+    }
+    finally {
+      ValueContainerImpl.ourDebugIndexInfo.set(null);
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException {
+    final Lock lock = getReadLock();
+    try {
+      lock.lock();
+      return myStorage.processKeys(processor);
+    }
+    finally {
+      lock.unlock();
+    }
+  }
+
+  @NotNull
+  @Override
+  public Computable<Boolean> update(final int inputId, @Nullable Input content) {
+    Map<Key, Value> data = content != null ? myIndexer.map(content) : Collections.<Key, Value>emptyMap();
+
+    checkValuesHaveProperEqualsAndHashCode(data);
+
+    myStorage.checkCanceled();
+
+    // do not depend on content!
+    final UpdateData<Key, Value> updateData = buildUpdateData(data,
+                                                              createOldKeysGetterByInputIndex(inputId),
+                                                              inputId);
+    return new Computable<Boolean>() {
+      @Override
+      public Boolean compute() {
+        try {
+          updateWithMap(inputId, updateData);
+        }
+        catch (StorageException ex) {
+          LOG.info("Exception during updateWithMap:" + ex);
+          requestRebuild(ex);
+          return Boolean.FALSE;
+        }
+        catch (RuntimeException ex) {
+          if (getProcessCanceledExceptionClass().isInstance(ex)) {
+            LOG.info("Exception during updateWithMap:" + ex);
+            requestRebuild(ex);
+            return Boolean.FALSE;
+          } else {
+            throw ex;
+          }
+        }
+
+        return Boolean.TRUE;
+      }
+    };
+  }
+
+  @NotNull
+  protected NotNullComputable<Collection<Key>> createOldKeysGetterByInputIndex(final int inputId) {
+    return new NotNullComputable<Collection<Key>>() {
+      @NotNull
+      @Override
+      public Collection<Key> compute() {
+        try {
+          Collection<Key> oldKeys = MapReduceIndexBase.this.readInputKeys(inputId);
+          return oldKeys == null ? Collections.<Key>emptyList() : oldKeys;
+        }
+        catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    };
+  }
+
+  protected UpdateData<Key, Value> buildUpdateData(Map<Key, Value> data, NotNullComputable<Collection<Key>> oldKeysGetter, int savedInputId) {
+    return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter);
+  }
+
+  protected Collection<Key> readInputKeys(int inputId) throws IOException {
+    return myInputsIndex != null ? myInputsIndex.get(inputId) : null;
+  }
+
+  protected void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
+    if (myInputsIndex != null) {
+      if (newData.size() > 0) {
+        myInputsIndex.put(inputId, newData.keySet());
+      }
+      else {
+        myInputsIndex.remove(inputId);
+      }
+    }
+  }
+
+  protected void checkValuesHaveProperEqualsAndHashCode(Map<Key, Value> data) {
+    if (DebugAssertions.DEBUG) {
+      for (Map.Entry<Key, Value> e : data.entrySet()) {
+        final Value value = e.getValue();
+        if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
+          LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter");
+        }
+
+        if (myValueExternalizer != null) {
+          try {
+            final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
+            DataOutputStream outputStream = new DataOutputStream(out);
+            myValueExternalizer.save(outputStream, value);
+            outputStream.close();
+            final Value deserializedValue =
+              myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
+
+            if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
+              LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
+            }
+          }
+          catch (IOException ex) {
+            LOG.error(ex);
+          }
+        }
+      }
+    }
+  }
+
+  public IndexExtension<Key, Value, Input> getExtension() {
+    return myExtension;
+  }
+
+  public long getModificationStamp() {
+    return myModificationStamp.get();
+  }
+
+  public class SimpleUpdateData extends UpdateData<Key, Value> {
+    private final int savedInputId;
+    private final @NotNull Map<Key, Value> newData;
+    protected final @NotNull NotNullComputable<Collection<Key>> oldKeysGetter;
+
+    public SimpleUpdateData(ID<Key,Value> indexId, int id, @NotNull Map<Key, Value> data, @NotNull NotNullComputable<Collection<Key>> getter) {
+      super(indexId);
+      savedInputId = id;
+      newData = data;
+      oldKeysGetter = getter;
+    }
+
+    public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
+      MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer);
+    }
+
+    public void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException {
+      MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData);
+    }
+
+    @Override
+    public void save(int inputId) throws IOException {
+      saveInputKeys(inputId, savedInputId, newData);
+    }
+
+    public @NotNull Map<Key, Value> getNewData() {
+      return newData;
+    }
+  }
+
+  private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>
+    myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>() {
+    @Override
+    public void process(Key key, int inputId) throws StorageException {
+      myModificationStamp.incrementAndGet();
+      myStorage.removeAllValues(key, inputId);
+    }
+  };
+
+  private final MapDiffUpdateData.AddedKeyProcessor<Key, Value> myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor<Key, Value>() {
+    @Override
+    public void process(Key key, Value value, int inputId) throws StorageException {
+      myModificationStamp.incrementAndGet();
+      myStorage.addValue(key, inputId, value);
+    }
+  };
+
+  protected void updateWithMap(final int inputId,
+                               @NotNull UpdateData<Key, Value> updateData) throws StorageException {
+    getWriteLock().lock();
+    try {
+      try {
+        ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
+        updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation);
+        updateData.iterateAddedKeys(inputId, myAddedKeyProcessor);
+        updateData.save(inputId);
+      }
+      catch (RuntimeException e) {
+        if (getProcessCanceledExceptionClass().isInstance(e)) {
+          throw e;
+        } else {
+          throw new StorageException(e);
+        }
+      }
+      catch (Throwable e) { // e.g. IOException, AssertionError
+        throw new StorageException(e);
+      }
+      finally {
+        ValueContainerImpl.ourDebugIndexInfo.set(null);
+      }
+    }
+    finally {
+      getWriteLock().unlock();
+    }
+  }
+}
+
diff --git a/platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java b/platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java
new file mode 100644 (file)
index 0000000..21a756e
--- /dev/null
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.util.Processor;
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.*;
+
+public abstract class MemoryIndexStorageBase<Key, Value, Storage extends IndexStorage<Key, Value>> implements IndexStorage<Key, Value> {
+  private final static Logger LOG = Logger.getInstance(MemoryIndexStorageBase.class);
+
+  private final Map<Key, ChangeTrackingValueContainer<Value>> myMap = new HashMap<Key, ChangeTrackingValueContainer<Value>>();
+  @NotNull
+  protected final Storage myBackendStorage;
+  private final List<BufferingStateListener> myListeners = ContainerUtil.createLockFreeCopyOnWriteList();
+  private final ID<?, ?> myIndexId;
+  private boolean myBufferingEnabled;
+
+  public interface BufferingStateListener {
+    void bufferingStateChanged(boolean newState);
+
+    void memoryStorageCleared();
+  }
+
+  public MemoryIndexStorageBase(@NotNull Storage backend, ID<?, ?> indexId) {
+    myBackendStorage = backend;
+    myIndexId = indexId;
+  }
+
+  @NotNull
+  public IndexStorage<Key, Value> getBackendStorage() {
+    return myBackendStorage;
+  }
+
+  public void addBufferingStateListener(@NotNull BufferingStateListener listener) {
+    myListeners.add(listener);
+  }
+
+  public void removeBufferingStateListener(@NotNull BufferingStateListener listener) {
+    myListeners.remove(listener);
+  }
+
+  public void setBufferingEnabled(boolean enabled) {
+    final boolean wasEnabled = myBufferingEnabled;
+    assert wasEnabled != enabled;
+
+    myBufferingEnabled = enabled;
+    for (BufferingStateListener listener : myListeners) {
+      listener.bufferingStateChanged(enabled);
+    }
+  }
+
+  public boolean isBufferingEnabled() {
+    return myBufferingEnabled;
+  }
+
+  public void clearMemoryMap() {
+    myMap.clear();
+  }
+
+  public void fireMemoryStorageCleared() {
+    for (BufferingStateListener listener : myListeners) {
+      listener.memoryStorageCleared();
+    }
+  }
+
+  void clearCaches() {
+    if (myMap.size() == 0) return;
+
+    if (DebugAssertions.DEBUG) {
+      String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size();
+      LOG.info(message);
+    }
+
+    for(ChangeTrackingValueContainer<Value> v:myMap.values()) {
+      v.dropMergedData();
+    }
+  }
+
+  @Override
+  public void close() throws StorageException {
+    myBackendStorage.close();
+  }
+
+  @Override
+  public void clear() throws StorageException {
+    clearMemoryMap();
+    myBackendStorage.clear();
+  }
+
+  @Override
+  public void flush() throws IOException {
+    myBackendStorage.flush();
+  }
+
+  @NotNull
+  @Override
+  public Collection<Key> getKeys() throws StorageException {
+    final Set<Key> keys = new HashSet<Key>();
+    processKeys(new CancelableCollectProcessor<Key>(keys, new Runnable() {
+      @Override
+      public void run() {
+        checkCanceled();
+      }
+    }));
+    return keys;
+  }
+
+  @Override
+  public boolean processKeys(@NotNull final Processor<Key> processor) throws StorageException {
+    final Processor<Key> uniqueResultProcessor = processCacheFirstProcessor(processor);
+    return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor);
+  }
+
+  @Nullable
+  protected Processor<Key> processCacheFirstProcessor(final Processor<Key> initialProcessor) {
+    if (myMap.isEmpty()) return initialProcessor;
+    final Set<Key> stopList = new HashSet<Key>();
+
+    Processor<Key> decoratingProcessor = new Processor<Key>() {
+      @Override
+      public boolean process(Key key) {
+        if (stopList.contains(key)) return true;
+
+        final UpdatableValueContainer<Value> container = myMap.get(key);
+        if (container != null && container.size() == 0) {
+          return true;
+        }
+        return initialProcessor.process(key);
+      }
+    };
+
+    for (Key key : myMap.keySet()) {
+      if (!decoratingProcessor.process(key)) {
+        return null;
+      }
+      stopList.add(key);
+    }
+
+    return decoratingProcessor;
+  }
+
+  @Override
+  public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
+    if (myBufferingEnabled) {
+      getMemValueContainer(key).addValue(inputId, value);
+      return;
+    }
+    final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
+    if (valueContainer != null) {
+      valueContainer.dropMergedData();
+    }
+
+    myBackendStorage.addValue(key, inputId, value);
+  }
+
+  @Override
+  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
+    if (myBufferingEnabled) {
+      getMemValueContainer(key).removeAssociatedValue(inputId);
+      return;
+    }
+    final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
+    if (valueContainer != null) {
+      valueContainer.dropMergedData();
+    }
+
+    myBackendStorage.removeAllValues(key, inputId);
+  }
+
+  private UpdatableValueContainer<Value> getMemValueContainer(final Key key) {
+    ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
+    if (valueContainer == null) {
+      valueContainer = new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
+        @Override
+        public Object getLock() {
+          return this;
+        }
+
+        @Override
+        public ValueContainer<Value> compute() {
+          try {
+            return myBackendStorage.read(key);
+          }
+          catch (StorageException e) {
+            throw new RuntimeException(e);
+          }
+        }
+      });
+      myMap.put(key, valueContainer);
+    }
+    return valueContainer;
+  }
+
+  @Override
+  @NotNull
+  public ValueContainer<Value> read(final Key key) throws StorageException {
+    final ValueContainer<Value> valueContainer = myMap.get(key);
+    if (valueContainer != null) {
+      return valueContainer;
+    }
+
+    return myBackendStorage.read(key);
+  }
+}
similarity index 96%
rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java
rename to platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java
index 591a4a0d5b6e7542897e35c02f6fe7b6bcf518fe..4f1d4fc4323991a26e43931efa04fcaecfbacf4f 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -79,20 +79,20 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
     return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0;
   }
 
-  static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<>();
+  static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<ID>();
 
   @Override
   public void removeAssociatedValue(int inputId) {
     if (myInputIdMapping == null) return;
     List<Object> fileSetObjects = null;
     List<Value> valueObjects = null;
-    for (final ValueIterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext();) {
+    for (final ValueContainer.ValueIterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext();) {
       final Value value = valueIterator.next();
 
       if (valueIterator.getValueAssociationPredicate().contains(inputId)) {
         if (fileSetObjects == null) {
-          fileSetObjects = new SmartList<>();
-          valueObjects = new SmartList<>();
+          fileSetObjects = new SmartList<Object>();
+          valueObjects = new SmartList<Value>();
         }
         else if (DebugAssertions.DEBUG) {
           LOG.error("Expected only one value per-inputId for " + ourDebugIndexInfo.get(), String.valueOf(fileSetObjects.get(0)), String.valueOf(value));
@@ -263,9 +263,9 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
     if (myInputIdMapping == null) {
       return Collections.emptyList();
     } else if (myInputIdMapping instanceof THashMap) {
-      return new ArrayList<>(((THashMap<Value, Object>)myInputIdMapping).keySet());
+      return new ArrayList<Value>(((THashMap<Value, Object>)myInputIdMapping).keySet());
     } else {
-      return new SmartList<>((Value)myInputIdMapping);
+      return new SmartList<Value>((Value)myInputIdMapping);
     }
   }
 
@@ -366,11 +366,11 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
   @NotNull
   public ValueContainerImpl<Value> copy() {
-    ValueContainerImpl<Value> container = new ValueContainerImpl<>();
+    ValueContainerImpl<Value> container = new ValueContainerImpl<Value>();
 
     if (myInputIdMapping instanceof THashMap) {
       final THashMap<Value, Object> mapping = (THashMap<Value, Object>)myInputIdMapping;
-      final THashMap<Value, Object> newMapping = new THashMap<>(mapping.size());
+      final THashMap<Value, Object> newMapping = new THashMap<Value, Object>(mapping.size());
       container.myInputIdMapping = newMapping;
 
       mapping.forEachEntry(new TObjectObjectProcedure<Value, Object>() {
@@ -485,7 +485,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
         final int inputId = -valueCount;
 
         if (mapping == null && size() > NUMBER_OF_VALUES_THRESHOLD) { // avoid O(NumberOfValues)
-          mapping = new FileId2ValueMapping<>(this);
+          mapping = new FileId2ValueMapping<Value>(this);
         }
 
         boolean doCompact;
similarity index 94%
rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java
rename to platform/util/src/com/intellij/util/indexing/ValueContainerMap.java
index e6789513ee355c8d08feae589c7ad0089b69c3cf..c50745debf58a6d931e7fe6a89cd3de8c890085b 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
                     @NotNull DataExternalizer<Value> valueExternalizer,
                     boolean keyIsUniqueForIndexedFile
                     ) throws IOException {
-    super(file, keyKeyDescriptor, new ValueContainerExternalizer<>(valueExternalizer));
+    super(file, keyKeyDescriptor, new ValueContainerExternalizer<Value>(valueExternalizer));
     myValueExternalizer = valueExternalizer;
     myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
   }
@@ -48,7 +48,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
   @Override
   protected void doPut(Key key, ValueContainer<Value> container) throws IOException {
     synchronized (myEnumerator) {
-      ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
+      final ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
 
       // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files
       // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular
@@ -83,7 +83,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
     @NotNull
     @Override
     public ValueContainerImpl<T> read(@NotNull final DataInput in) throws IOException {
-      final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<>();
+      final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
 
       valueContainer.readFrom((DataInputStream)in, myValueExternalizer);
       return valueContainer;
similarity index 99%
rename from platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java
rename to platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java
index 7e9076282b8e074498d5c636a14b1ceb8f6ad97d..7ca2968b20d217d82ba274387bbf8bafd8c6d474 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
similarity index 99%
rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java
rename to platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java
index 357c919f716293ca1a863e11c31e56c427f4b130..d82f0e223f1b9b0476bbe8d915cca35194359de0 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
similarity index 97%
rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java
rename to platform/util/src/com/intellij/util/indexing/containers/IdSet.java
index bbe7ff7891e7a321f147c11bf98ad1c2f75e18a0..9afcd7d6b48abaff56a4cf81999aa4bfb616a4f6 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
similarity index 98%
rename from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java
rename to platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java
index f42373c139c7e213af0548ee1eddd9dae4b0d898..af2a200cf559774e9a566b03a23fa497fa010d9c 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
index 389b2a0683306fddc0716268610b6395a1b9a073..f5d3d22966eaf515db7468dbc804b20fe6409b85 100644 (file)
@@ -144,7 +144,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
     }
 
     @Override
-    protected void requestRebuild(@Nullable Exception ex) {
+    public void requestRebuild(@Nullable Exception ex) {
       myFatalErrorHandler.consume(this, ex != null ? ex : new Exception("Index rebuild requested"));
     }
   }