[vcs-log] use enumerators instead of maps for keeping commit sets

For keeping commits without changed paths or commits without trigrams set-like structure is sufficient. So PersistenceSet interface is created to represent such a structure with an implementation based on PersistentBTreeEnumerator. This allows to reduce the amount of occupied disk space.

diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java
index b96dda768209e4cf0b830a479f5a56bfac9bff59..4228950f2ac232baf126cbd21960040841dc2c49 100644 (file)
--- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java
+++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java
@@ -23,8 +23,8 @@ import com.intellij.util.indexing.ScalarIndexExtension;
 import com.intellij.util.indexing.StorageException;
 import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.io.EnumeratorIntegerDescriptor;
-import com.intellij.util.io.PersistentHashMap;
 import com.intellij.vcs.log.VcsFullCommitDetails;
+import com.intellij.vcs.log.util.PersistentSet;
 import com.intellij.vcs.log.util.PersistentUtil;
 import gnu.trove.THashMap;
 import org.jetbrains.annotations.NotNull;
@@ -37,17 +37,16 @@ import java.util.Map;
 public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
   private static final Logger LOG = Logger.getInstance(VcsLogMessagesTrigramIndex.class);
   private static final String TRIGRAMS = "trigrams";
-  private static final int VALUE = 239;
 
-  @NotNull private final PersistentHashMap<Integer, Integer> myNoTrigramsCommits;
+  @NotNull private final PersistentSet<Integer> myNoTrigramsCommits;
 
   public VcsLogMessagesTrigramIndex(@NotNull String logId, @NotNull Disposable disposableParent) throws IOException {
     super(logId, TRIGRAMS, VcsLogPersistentIndex.getVersion(), new TrigramMessageIndexer(), ScalarIndexExtension.VOID_DATA_EXTERNALIZER,
           disposableParent);
 
     myNoTrigramsCommits =
-      PersistentUtil.createPersistentHashMap(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + TRIGRAMS, logId,
-                                             VcsLogPersistentIndex.getVersion());
+      PersistentUtil.createPersistentSet(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + TRIGRAMS, logId,
+                                         VcsLogPersistentIndex.getVersion());
   }
 
   @Nullable
@@ -63,7 +62,7 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
   @Override
   protected void onNotIndexableCommit(int commit) throws StorageException {
     try {
-      myNoTrigramsCommits.put(commit, VALUE);
+      myNoTrigramsCommits.put(commit);
     }
     catch (IOException e) {
       throw new StorageException(e);
@@ -72,13 +71,13 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
 
   @Override
   public boolean isIndexed(int commit) throws IOException {
-    return super.isIndexed(commit) || myNoTrigramsCommits.containsMapping(commit);
+    return super.isIndexed(commit) || myNoTrigramsCommits.contains(commit);
   }
 
   @Override
   public void flush() throws StorageException {
     super.flush();
-    myNoTrigramsCommits.force();
+    myNoTrigramsCommits.flush();
   }
 
   @Override
@@ -100,7 +99,7 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
 
   @NotNull
   public String getTrigramInfo(int commit) throws IOException {
-    if (myNoTrigramsCommits.containsMapping(commit)) {
+    if (myNoTrigramsCommits.contains(commit)) {
       return "No trigrams";
     }
 

diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java
index 4488b35de09442c920a7c9251a34df420b762656..7dabdfa0bded54d53ebf1958326151ccca7a9cbd 100644 (file)
--- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java
+++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPathsIndex.java
@@ -33,6 +33,7 @@ import com.intellij.util.text.CaseInsensitiveStringHashingStrategy;
 import com.intellij.vcs.log.VcsFullCommitDetails;
 import com.intellij.vcs.log.impl.FatalErrorHandler;
 import com.intellij.vcs.log.impl.VcsChangesLazilyParsedDetails;
+import com.intellij.vcs.log.util.PersistentSet;
 import com.intellij.vcs.log.util.PersistentUtil;
 import gnu.trove.THashMap;
 import gnu.trove.TIntHashSet;
@@ -51,9 +52,8 @@ import static com.intellij.util.containers.ContainerUtil.newTroveSet;
 public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
   private static final Logger LOG = Logger.getInstance(VcsLogPathsIndex.class);
   private static final String NAME = "paths";
-  private static final int VALUE = 239;
 
-  @NotNull private final PersistentHashMap<Integer, Integer> myEmptyCommits;
+  @NotNull private final PersistentSet<Integer> myEmptyCommits;
   @NotNull private final PathsIndexer myPathsIndexer;
 
   public VcsLogPathsIndex(@NotNull String logId,
@@ -63,8 +63,8 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
     super(logId, NAME, VcsLogPersistentIndex.getVersion(), new PathsIndexer(createPathsEnumerator(logId), roots),
           new NullableIntKeyDescriptor(), disposableParent);
 
-    myEmptyCommits = PersistentUtil.createPersistentHashMap(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + NAME, logId,
-                                                            VcsLogPersistentIndex.getVersion());
+    myEmptyCommits = PersistentUtil.createPersistentSet(EnumeratorIntegerDescriptor.INSTANCE, "index-no-" + NAME, logId,
+                                                        VcsLogPersistentIndex.getVersion());
     myPathsIndexer = (PathsIndexer)myIndexer;
     myPathsIndexer.setFatalErrorConsumer(e -> {
       fatalErrorHandler.consume(this, e);
@@ -93,7 +93,7 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
   @Override
   protected void onNotIndexableCommit(int commit) throws StorageException {
     try {
-      myEmptyCommits.put(commit, VALUE);
+      myEmptyCommits.put(commit);
     }
     catch (IOException e) {
       throw new StorageException(e);
@@ -102,13 +102,13 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
 
   @Override
   public boolean isIndexed(int commit) throws IOException {
-    return super.isIndexed(commit) || myEmptyCommits.containsMapping(commit);
+    return super.isIndexed(commit) || myEmptyCommits.contains(commit);
   }
 
   @Override
   public void flush() throws StorageException {
     super.flush();
-    myEmptyCommits.force();
+    myEmptyCommits.flush();
     myPathsIndexer.getPathsEnumerator().force();
   }
 
@@ -170,7 +170,7 @@ public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
 
   @NotNull
   public String getPathInfo(int commit) throws IOException {
-    if (myEmptyCommits.containsMapping(commit)) {
+    if (myEmptyCommits.contains(commit)) {
       return "No paths";
     }
     Collection<Integer> keys = getKeysForCommit(commit);

diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentSet.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentSet.java
new file mode 100644 (file)
index 0000000..4ca9a3c
--- /dev/null
+++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentSet.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.vcs.log.util;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+
+public interface PersistentSet<T> {
+
+  boolean contains(@NotNull T element) throws IOException;
+
+  void put(@NotNull T element) throws IOException;
+
+  void flush();
+
+  void markCorrupted();
+
+  void close() throws IOException;
+}

diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java
index b39928c3b663ec55ea95b30499b3228d3c8c3ddf..eb179679319ca2fdab8efc652e7b30d7ec849fc3 100644 (file)
--- a/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java
+++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/util/PersistentUtil.java
@@ -24,6 +24,7 @@ import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.io.*;
 import com.intellij.vcs.log.VcsLogProvider;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -73,6 +74,18 @@ public class PersistentUtil {
                                          storageFile);
   }
 
+  @NotNull
+  public static <T> PersistentSet<T> createPersistentSet(@NotNull KeyDescriptor<T> keyDescriptor,
+                                                         @NotNull String storageKind,
+                                                         @NotNull String logId,
+                                                         int version) throws IOException {
+    File storageFile = getStorageFile(storageKind, logId, version);
+
+    return IOUtil.openCleanOrResetBroken(() ->
+                                           new PersistentSetImpl<>(storageFile, keyDescriptor, Page.PAGE_SIZE, null, version),
+                                         storageFile);
+  }
+
   @NotNull
   public static <V> PersistentHashMap<Integer, V> createPersistentHashMap(@NotNull DataExternalizer<V> externalizer,
                                                                           @NotNull String storageKind,
@@ -89,4 +102,34 @@ public class PersistentUtil {
   public static File getCorruptionMarkerFile() {
     return new File(LOG_CACHE, CORRUPTION_MARKER);
   }
+
+  public static class PersistentSetImpl<T> extends PersistentBTreeEnumerator<T> implements PersistentSet<T> {
+
+    public PersistentSetImpl(@NotNull File file,
+                             @NotNull KeyDescriptor<T> dataDescriptor,
+                             int initialSize,
+                             @Nullable PagedFileStorage.StorageLockContext lockContext, int version) throws IOException {
+      super(file, dataDescriptor, initialSize, lockContext, version);
+    }
+
+    @Override
+    public boolean contains(@NotNull T element) throws IOException {
+      return tryEnumerate(element) != NULL_ID;
+    }
+
+    @Override
+    public void put(@NotNull T element) throws IOException {
+      enumerate(element);
+    }
+
+    @Override
+    public void flush() {
+      force();
+    }
+
+    @Override
+    public synchronized void markCorrupted() {
+      super.markCorrupted();
+    }
+  }
 }