Introduce new storage scheme: keep indexes in index/<project-log-id>.<index-id>.<version>
public static final int VERSION = 5;
private static final int REFS_VERSION = 1;
@NotNull private static final String ROOT_STORAGE_KIND = "roots";
- private static final int ROOTS_STORAGE_VERSION = 0;
public static final int NO_INDEX = -1;
// cleanup old root storages, to remove after 2016.3 release
PersistentUtil
- .cleanupOldStorageFile(ROOT_STORAGE_KIND, project.getName() + "." + project.getBaseDir().getPath().hashCode(), ROOTS_STORAGE_VERSION);
+ .cleanupOldStorageFile(ROOT_STORAGE_KIND, project.getName() + "." + project.getBaseDir().getPath().hashCode());
Disposer.register(parent, this);
}
import com.intellij.openapi.Disposable;
import com.intellij.openapi.util.Disposer;
import com.intellij.util.Consumer;
-import com.intellij.util.PathUtilRt;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import java.util.Set;
import java.util.function.ObjIntConsumer;
+import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
+
public class VcsLogFullDetailsIndex<T> implements Disposable {
- @NotNull protected static final String INDEX = "index-";
+ protected static final String INDEX = "index";
@NotNull protected final MyMapReduceIndex myMapReduceIndex;
@NotNull private final ID<Integer, T> myID;
@NotNull private final String myLogId;
}
@NotNull
- public File getStorageFile(int version) {
- return getStorageFile(INDEX + myName, myLogId, version);
- }
-
- @NotNull
- public static File getStorageFile(@NotNull String kind, @NotNull String id, int version) {
- File subdir = new File(PersistentUtil.LOG_CACHE, kind);
- String safeLogId = PathUtilRt.suggestFileName(id, true, true);
- return new File(subdir, safeLogId + "." + version);
+ public static File getStorageFile(@NotNull String kind, @NotNull String id) {
+ return PersistentUtil.getStorageFile(INDEX, kind, id, getVersion(), false);
}
private class MyMapReduceIndex extends MapReduceIndex<Integer, T, VcsFullCommitDetails> {
@NotNull DataExternalizer<T> externalizer,
int version) throws IOException {
super(new MyIndexExtension(indexer, externalizer, version),
- new MapIndexStorage<>(getStorageFile(version),
+ new MapIndexStorage<>(getStorageFile(myName, myLogId),
EnumeratorIntegerDescriptor.INSTANCE,
externalizer, 5000));
}
import static com.intellij.vcs.log.data.index.VcsLogPersistentIndex.getVersion;
public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
- private static final String TRIGRAMS = "trigrams";
+ public static final String TRIGRAMS = "trigrams";
public VcsLogMessagesTrigramIndex(@NotNull String logId,
@NotNull FatalErrorHandler fatalErrorHandler,
@NotNull
public static Collection<File> getStorageFiles(@NotNull String logId) {
- return Collections.singletonList(getStorageFile(TRIGRAMS, logId, getVersion()));
+ return Collections.singletonList(getStorageFile(TRIGRAMS, logId));
}
@Nullable
public class VcsLogPathsIndex extends VcsLogFullDetailsIndex<Integer> {
private static final Logger LOG = Logger.getInstance(VcsLogPathsIndex.class);
- private static final String PATHS = "paths";
- private static final String INDEX_PATHS_IDS = "index-paths-ids";
+ public static final String PATHS = "paths";
+ public static final String INDEX_PATHS_IDS = "paths-ids";
@NotNull private final PathsIndexer myPathsIndexer;
@NotNull
private static PersistentEnumeratorBase<String> createPathsEnumerator(@NotNull String logId) throws IOException {
- File storageFile = PersistentUtil.getStorageFile(INDEX_PATHS_IDS, logId, getVersion());
-
+ File storageFile = PersistentUtil.getStorageFile(INDEX, INDEX_PATHS_IDS, logId, getVersion(), true);
return new PersistentBTreeEnumerator<>(storageFile, SystemInfo.isFileSystemCaseSensitive ? EnumeratorStringDescriptor.INSTANCE
: new ToLowerCaseStringDescriptor(),
Page.PAGE_SIZE, null, getVersion());
@NotNull
public static Collection<File> getStorageFiles(@NotNull String logId) {
- return Arrays.asList(PersistentUtil.getStorageFile(INDEX_PATHS_IDS, logId, getVersion()),
- getStorageFile(PATHS, logId, getVersion()));
+ return Arrays.asList(getStorageFile(INDEX_PATHS_IDS, logId),
+ getStorageFile(PATHS, logId));
}
@Override
import com.intellij.vcs.log.impl.VcsLogUserFilterImpl;
import com.intellij.vcs.log.impl.VcsLogUtil;
import com.intellij.vcs.log.util.PersistentSet;
+import com.intellij.vcs.log.util.PersistentSetImpl;
import com.intellij.vcs.log.util.PersistentUtil;
import com.intellij.vcs.log.util.StopWatch;
import com.intellij.vcs.log.util.TroveUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
+import static com.intellij.vcs.log.data.index.VcsLogFullDetailsIndex.INDEX;
+import static com.intellij.vcs.log.util.PersistentUtil.getStorageFile;
+
public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
private static final Logger LOG = Logger.getInstance(VcsLogPersistentIndex.class);
private static final int BATCH_SIZE = 1000;
@NotNull private final VcsLogMessagesTrigramIndex trigrams;
@NotNull private final VcsLogUserIndex users;
@NotNull private final VcsLogPathsIndex paths;
+ private static final String INPUTS = "inputs";
public MyIndexStorage(@NotNull String logId,
@NotNull VcsUserRegistryImpl userRegistry,
Disposer.register(parentDisposable, disposable);
try {
- commits =
- PersistentUtil.createPersistentSetOrFailIfBroken(EnumeratorIntegerDescriptor.INSTANCE, COMMITS, logId, getVersion());
+ int version = getVersion();
+
+ File commitsStorage = getStorageFile(INDEX, COMMITS, logId, version, true);
+ commits = new PersistentSetImpl<>(commitsStorage, EnumeratorIntegerDescriptor.INSTANCE, Page.PAGE_SIZE, null, version);
Disposer.register(disposable, () -> catchAndWarn(commits::close));
- messages = new PersistentHashMap<>(PersistentUtil.getStorageFile(MESSAGES, logId, MESSAGES_VERSION), new IntInlineKeyDescriptor(),
- EnumeratorStringDescriptor.INSTANCE, Page.PAGE_SIZE);
+
+ File messagesStorage = getStorageFile(INDEX, MESSAGES, logId, MESSAGES_VERSION, true);
+ messages = new PersistentHashMap<>(messagesStorage, new IntInlineKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE,
+ Page.PAGE_SIZE);
Disposer.register(disposable, () -> catchAndWarn(messages::close));
trigrams = new VcsLogMessagesTrigramIndex(logId, fatalErrorHandler, disposable);
Disposer.dispose(disposable);
throw t;
}
+
+ // cleanup of old index storage files
+ // to remove after 2017.1 release
+ PersistentUtil.cleanupOldStorageFile(MESSAGES, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + VcsLogMessagesTrigramIndex.TRIGRAMS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-no-" + VcsLogMessagesTrigramIndex.TRIGRAMS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + INPUTS + "-" + VcsLogMessagesTrigramIndex.TRIGRAMS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + VcsLogPathsIndex.PATHS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-no-" + VcsLogPathsIndex.PATHS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + VcsLogPathsIndex.PATHS + "-ids", logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + INPUTS + "-" + VcsLogPathsIndex.PATHS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + VcsLogUserIndex.USERS, logId);
+ PersistentUtil.cleanupOldStorageFile(INDEX + "-" + INPUTS + "-" + VcsLogUserIndex.USERS, logId);
}
private static void catchAndWarn(@NotNull ThrowableRunnable<IOException> runnable) {
}
private static void cleanup(@NotNull String logId) {
- IOUtil.deleteAllFilesStartingWith(PersistentUtil.getStorageFile(COMMITS, logId, getVersion()));
- IOUtil.deleteAllFilesStartingWith(PersistentUtil.getStorageFile(MESSAGES, logId, MESSAGES_VERSION));
+ IOUtil.deleteAllFilesStartingWith(getStorageFile(COMMITS, logId, getVersion()));
+ IOUtil.deleteAllFilesStartingWith(getStorageFile(MESSAGES, logId, MESSAGES_VERSION));
VcsLogMessagesTrigramIndex.getStorageFiles(logId).forEach(IOUtil::deleteAllFilesStartingWith);
VcsLogUserIndex.getStorageFiles(logId).forEach(IOUtil::deleteAllFilesStartingWith);
import com.intellij.openapi.Disposable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.Consumer;
-import com.intellij.util.ObjectUtils;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.DataIndexer;
import com.intellij.util.indexing.ScalarIndexExtension;
public class VcsLogUserIndex extends VcsLogFullDetailsIndex<Void> {
private static final Logger LOG = Logger.getInstance(VcsLogUserIndex.class);
- private static final String USERS = "users";
+ public static final String USERS = "users";
@NotNull private final VcsUserRegistryImpl myUserRegistry;
public VcsLogUserIndex(@NotNull String logId,
@NotNull
public static Collection<File> getStorageFiles(@NotNull String logId) {
- return Collections.singletonList(getStorageFile(USERS, logId, getVersion()));
+ return Collections.singletonList(getStorageFile(USERS, logId));
}
public TIntHashSet getCommitsForUsers(@NotNull Set<VcsUser> users) throws IOException, StorageException {
--- /dev/null
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.vcs.log.util;
+
+import com.intellij.util.io.KeyDescriptor;
+import com.intellij.util.io.PagedFileStorage;
+import com.intellij.util.io.PersistentBTreeEnumerator;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+
+public class PersistentSetImpl<T> extends PersistentBTreeEnumerator<T> implements PersistentSet<T> {
+
+ public PersistentSetImpl(@NotNull File file,
+ @NotNull KeyDescriptor<T> dataDescriptor,
+ int initialSize,
+ @Nullable PagedFileStorage.StorageLockContext lockContext, int version) throws IOException {
+ super(file, dataDescriptor, initialSize, lockContext, version);
+ }
+
+ @Override
+ public boolean contains(@NotNull T element) throws IOException {
+ return tryEnumerate(element) != NULL_ID;
+ }
+
+ @Override
+ public void put(@NotNull T element) throws IOException {
+ enumerate(element);
+ }
+
+ @Override
+ public void flush() {
+ force();
+ }
+
+ @Override
+ public synchronized void markCorrupted() {
+ super.markCorrupted();
+ }
+}
import com.intellij.util.io.*;
import com.intellij.vcs.log.VcsLogProvider;
import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
return mapFile;
}
- public static void cleanupOldStorageFile(@NotNull String storageKind, @NotNull String logId, int version) {
- IOUtil.deleteAllFilesStartingWith(getStorageFile(storageKind, logId, version));
+ public static void cleanupOldStorageFile(@NotNull String storageKind, @NotNull String logId) {
+ File subdir = new File(LOG_CACHE, storageKind);
+ String safeLogId = PathUtilRt.suggestFileName(logId, true, true);
+ IOUtil.deleteAllFilesStartingWith(new File(subdir, safeLogId));
+
+ File[] files = subdir.listFiles();
+ if (files != null && files.length == 0) {
+ subdir.delete();
+ }
}
@NotNull
}
@NotNull
- public static <T> PersistentSet<T> createPersistentSetOrFailIfBroken(@NotNull KeyDescriptor<T> keyDescriptor,
- @NotNull String storageKind,
- @NotNull String logId,
- int version) throws IOException {
- File storageFile = getStorageFile(storageKind, logId, version);
- return new PersistentSetImpl<>(storageFile, keyDescriptor, Page.PAGE_SIZE, null, version);
+ public static File getStorageFile(@NotNull String subdirName,
+ @NotNull String kind,
+ @NotNull String id,
+ int version,
+ boolean cleanup) {
+ File subdir = new File(LOG_CACHE, subdirName);
+ String safeLogId = PathUtilRt.suggestFileName(id, true, true);
+ File file = new File(subdir, safeLogId + "." + kind + "." + version);
+ if (cleanup && !file.exists()) {
+ IOUtil.deleteAllFilesStartingWith(new File(subdir, safeLogId + "." + kind));
+ }
+ return file;
}
@NotNull
public static File getCorruptionMarkerFile() {
return new File(LOG_CACHE, CORRUPTION_MARKER);
}
-
- public static class PersistentSetImpl<T> extends PersistentBTreeEnumerator<T> implements PersistentSet<T> {
-
- public PersistentSetImpl(@NotNull File file,
- @NotNull KeyDescriptor<T> dataDescriptor,
- int initialSize,
- @Nullable PagedFileStorage.StorageLockContext lockContext, int version) throws IOException {
- super(file, dataDescriptor, initialSize, lockContext, version);
- }
-
- @Override
- public boolean contains(@NotNull T element) throws IOException {
- return tryEnumerate(element) != NULL_ID;
- }
-
- @Override
- public void put(@NotNull T element) throws IOException {
- enumerate(element);
- }
-
- @Override
- public void flush() {
- force();
- }
-
- @Override
- public synchronized void markCorrupted() {
- super.markCorrupted();
- }
- }
}
projects / idea / community.git / commitdiff