[git] do not loose commit which messages do not contain \n while reading details...
[idea/community.git] / plugins / git4idea / src / git4idea / history / GitHistoryUtils.java
1 /*
2  * Copyright 2000-2014 JetBrains s.r.o.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package git4idea.history;
17
18 import com.intellij.execution.process.ProcessOutputTypes;
19 import com.intellij.openapi.application.ApplicationManager;
20 import com.intellij.openapi.components.ServiceManager;
21 import com.intellij.openapi.diagnostic.Logger;
22 import com.intellij.openapi.progress.ProcessCanceledException;
23 import com.intellij.openapi.project.Project;
24 import com.intellij.openapi.util.*;
25 import com.intellij.openapi.util.registry.Registry;
26 import com.intellij.openapi.util.text.StringUtil;
27 import com.intellij.openapi.vcs.FilePath;
28 import com.intellij.openapi.vcs.FileStatus;
29 import com.intellij.openapi.vcs.VcsException;
30 import com.intellij.openapi.vcs.changes.Change;
31 import com.intellij.openapi.vcs.changes.ChangeListManager;
32 import com.intellij.openapi.vcs.diff.ItemLatestState;
33 import com.intellij.openapi.vcs.history.VcsFileRevision;
34 import com.intellij.openapi.vcs.history.VcsRevisionDescription;
35 import com.intellij.openapi.vcs.history.VcsRevisionDescriptionImpl;
36 import com.intellij.openapi.vcs.history.VcsRevisionNumber;
37 import com.intellij.openapi.vfs.VirtualFile;
38 import com.intellij.util.ArrayUtil;
39 import com.intellij.util.Consumer;
40 import com.intellij.util.NullableFunction;
41 import com.intellij.util.SmartList;
42 import com.intellij.util.concurrency.Semaphore;
43 import com.intellij.util.containers.ContainerUtil;
44 import com.intellij.util.containers.OpenTHashSet;
45 import com.intellij.vcs.log.*;
46 import com.intellij.vcs.log.impl.HashImpl;
47 import com.intellij.vcs.log.impl.LogDataImpl;
48 import com.intellij.vcs.log.util.StopWatch;
49 import git4idea.*;
50 import git4idea.branch.GitBranchUtil;
51 import git4idea.commands.*;
52 import git4idea.config.GitVersion;
53 import git4idea.config.GitVersionSpecialty;
54 import git4idea.history.browser.GitHeavyCommit;
55 import git4idea.history.browser.SHAHash;
56 import git4idea.history.browser.SymbolicRefs;
57 import git4idea.history.browser.SymbolicRefsI;
58 import git4idea.history.wholeTree.AbstractHash;
59 import git4idea.i18n.GitBundle;
60 import git4idea.log.GitLogProvider;
61 import git4idea.log.GitRefManager;
62 import org.jetbrains.annotations.NotNull;
63 import org.jetbrains.annotations.Nullable;
64
65 import java.util.*;
66 import java.util.concurrent.atomic.AtomicBoolean;
67 import java.util.concurrent.atomic.AtomicInteger;
68 import java.util.concurrent.atomic.AtomicReference;
69
70 import static com.intellij.util.ObjectUtils.notNull;
71 import static git4idea.history.GitLogParser.GitLogOption.*;
72
73 /**
74  * A collection of methods for retrieving history information from native Git.
75  */
76 public class GitHistoryUtils {
77
78   /**
79    * A parameter to {@code git log} which is equivalent to {@code --all}, but doesn't show the stuff from index or stash.
80    */
81   public static final List<String> LOG_ALL = Arrays.asList("HEAD", "--branches", "--remotes", "--tags");
82
83   private static final Logger LOG = Logger.getInstance("#git4idea.history.GitHistoryUtils");
84
85   private GitHistoryUtils() {
86   }
87
88   /**
89    * Get current revision for the file under git in the current or specified branch.
90    *
91    * @param project  a project
92    * @param filePath file path to the file which revision is to be retrieved.
93    * @param branch   name of branch or null if current branch wanted.
94    * @return revision number or null if the file is unversioned or new.
95    * @throws VcsException if there is a problem with running git.
96    */
97   @Nullable
98   public static VcsRevisionNumber getCurrentRevision(@NotNull Project project, @NotNull FilePath filePath,
99                                                      @Nullable String branch) throws VcsException {
100     filePath = getLastCommitName(project, filePath);
101     GitSimpleHandler h = new GitSimpleHandler(project, GitUtil.getGitRoot(filePath), GitCommand.LOG);
102     GitLogParser parser = new GitLogParser(project, HASH, COMMIT_TIME);
103     h.setSilent(true);
104     h.addParameters("-n1", parser.getPretty());
105     h.addParameters(!StringUtil.isEmpty(branch) ? branch : "--all");
106     h.endOptions();
107     h.addRelativePaths(filePath);
108     String result = h.run();
109     if (result.length() == 0) {
110       return null;
111     }
112     final GitLogRecord record = parser.parseOneRecord(result);
113     if (record == null) {
114       return null;
115     }
116     record.setUsedHandler(h);
117     return new GitRevisionNumber(record.getHash(), record.getDate());
118   }
119
120   @Nullable
121   public static VcsRevisionDescription getCurrentRevisionDescription(@NotNull Project project, @NotNull FilePath filePath)
122     throws VcsException {
123     filePath = getLastCommitName(project, filePath);
124     GitSimpleHandler h = new GitSimpleHandler(project, GitUtil.getGitRoot(filePath), GitCommand.LOG);
125     GitLogParser parser = new GitLogParser(project, HASH, COMMIT_TIME, AUTHOR_NAME, COMMITTER_NAME, SUBJECT, BODY, RAW_BODY);
126     h.setSilent(true);
127     h.addParameters("-n1", parser.getPretty());
128     h.addParameters("--all");
129     h.endOptions();
130     h.addRelativePaths(filePath);
131     String result = h.run();
132     if (result.length() == 0) {
133       return null;
134     }
135     final GitLogRecord record = parser.parseOneRecord(result);
136     if (record == null) {
137       return null;
138     }
139     record.setUsedHandler(h);
140
141     final String author = Comparing.equal(record.getAuthorName(), record.getCommitterName()) ? record.getAuthorName() :
142                           record.getAuthorName() + " (" + record.getCommitterName() + ")";
143     return new VcsRevisionDescriptionImpl(new GitRevisionNumber(record.getHash(), record.getDate()), record.getDate(), author,
144                                           record.getFullMessage());
145   }
146
147   /**
148    * Get current revision for the file under git
149    *
150    * @param project  a project
151    * @param filePath a file path
152    * @return a revision number or null if the file is unversioned or new
153    * @throws VcsException if there is problem with running git
154    */
155   @Nullable
156   public static ItemLatestState getLastRevision(@NotNull Project project, @NotNull FilePath filePath) throws VcsException {
157     VirtualFile root = GitUtil.getGitRoot(filePath);
158     GitBranch c = GitBranchUtil.getCurrentBranch(project, root);
159     GitBranch t = c == null ? null : GitBranchUtil.tracked(project, root, c.getName());
160     if (t == null) {
161       return new ItemLatestState(getCurrentRevision(project, filePath, null), true, false);
162     }
163     filePath = getLastCommitName(project, filePath);
164     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.LOG);
165     GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.STATUS, HASH, COMMIT_TIME, PARENTS);
166     h.setSilent(true);
167     h.addParameters("-n1", parser.getPretty(), "--name-status", t.getFullName());
168     h.endOptions();
169     h.addRelativePaths(filePath);
170     String result = h.run();
171     if (result.length() == 0) {
172       return null;
173     }
174     GitLogRecord record = parser.parseOneRecord(result);
175     if (record == null) {
176       return null;
177     }
178     final List<Change> changes = record.parseChanges(project, root);
179     boolean exists = changes.isEmpty() || !FileStatus.DELETED.equals(changes.get(0).getFileStatus());
180     record.setUsedHandler(h);
181     return new ItemLatestState(new GitRevisionNumber(record.getHash(), record.getDate()), exists, false);
182   }
183
184   /*
185    === Smart full log with renames ===
186    'git log --follow' does detect renames, but it has a bug - merge commits aren't handled properly: they just dissapear from the history.
187    See http://kerneltrap.org/mailarchive/git/2009/1/30/4861054 and the whole thread about that: --follow is buggy, but maybe it won't be fixed.
188    To get the whole history through renames we do the following:
189    1. 'git log <file>' - and we get the history since the first rename, if there was one.
190    2. 'git show -M --follow --name-status <first_commit_id> -- <file>'
191       where <first_commit_id> is the hash of the first commit in the history we got in #1.
192       With this command we get the rename-detection-friendly information about the first commit of the given file history.
193       (by specifying the <file> we filter out other changes in that commit; but in that case rename detection requires '--follow' to work,
194       that's safe for one commit though)
195       If the first commit was ADDING the file, then there were no renames with this file, we have the full history.
196       But if the first commit was RENAMING the file, we are going to query for the history before rename.
197       Now we have the previous name of the file:
198
199         ~/sandbox/git # git show --oneline --name-status -M 4185b97
200         4185b97 renamed a to b
201         R100    a       b
202
203    3. 'git log <rename_commit_id> -- <previous_file_name>' - get the history of a before the given commit.
204       We need to specify <rename_commit_id> here, because <previous_file_name> could have some new history, which has nothing common with our <file>.
205       Then we repeat 2 and 3 until the first commit is ADDING the file, not RENAMING it.
206
207     TODO: handle multiple repositories configuration: a file can be moved from one repo to another
208    */
209
210   /**
211    * Retrieves the history of the file, including renames.
212    *
213    * @param project
214    * @param path              FilePath which history is queried.
215    * @param root              Git root - optional: if this is null, then git root will be detected automatically.
216    * @param consumer          This consumer is notified ({@link Consumer#consume(Object)} when new history records are retrieved.
217    * @param exceptionConsumer This consumer is notified in case of error while executing git command.
218    * @param parameters        Optional parameters which will be added to the git log command just before the path.
219    */
220   public static void history(@NotNull Project project,
221                              @NotNull FilePath path,
222                              @Nullable VirtualFile root,
223                              @NotNull Consumer<GitFileRevision> consumer,
224                              @NotNull Consumer<VcsException> exceptionConsumer,
225                              String... parameters) {
226     history(project, path, root, GitRevisionNumber.HEAD, consumer, exceptionConsumer, parameters);
227   }
228
229   public static void history(@NotNull Project project,
230                              @NotNull FilePath path,
231                              @Nullable VirtualFile root,
232                              @NotNull VcsRevisionNumber startingRevision,
233                              @NotNull Consumer<GitFileRevision> consumer,
234                              @NotNull Consumer<VcsException> exceptionConsumer,
235                              String... parameters) {
236     // adjust path using change manager
237     final FilePath filePath = getLastCommitName(project, path);
238     final VirtualFile finalRoot;
239     try {
240       finalRoot = (root == null ? GitUtil.getGitRoot(filePath) : root);
241     }
242     catch (VcsException e) {
243       exceptionConsumer.consume(e);
244       return;
245     }
246     final GitLogParser logParser = new GitLogParser(project, GitLogParser.NameStatus.STATUS,
247                                                     HASH, COMMIT_TIME, AUTHOR_NAME, AUTHOR_EMAIL, COMMITTER_NAME, COMMITTER_EMAIL, PARENTS,
248                                                     SUBJECT, BODY, RAW_BODY, AUTHOR_TIME);
249
250     final AtomicReference<String> firstCommit = new AtomicReference<>(startingRevision.asString());
251     final AtomicReference<String> firstCommitParent = new AtomicReference<>(firstCommit.get());
252     final AtomicReference<FilePath> currentPath = new AtomicReference<>(filePath);
253     final AtomicReference<GitLineHandler> logHandler = new AtomicReference<>();
254     final AtomicBoolean skipFurtherOutput = new AtomicBoolean();
255
256     final Consumer<GitLogRecord> resultAdapter = record -> {
257       if (skipFurtherOutput.get()) {
258         return;
259       }
260       if (record == null) {
261         exceptionConsumer.consume(new VcsException("revision details are null."));
262         return;
263       }
264       record.setUsedHandler(logHandler.get());
265       final GitRevisionNumber revision = new GitRevisionNumber(record.getHash(), record.getDate());
266       firstCommit.set(record.getHash());
267       final String[] parentHashes = record.getParentsHashes();
268       if (parentHashes.length < 1) {
269         firstCommitParent.set(null);
270       }
271       else {
272         firstCommitParent.set(parentHashes[0]);
273       }
274       final String message = record.getFullMessage();
275
276       FilePath revisionPath;
277       try {
278         final List<FilePath> paths = record.getFilePaths(finalRoot);
279         if (paths.size() > 0) {
280           revisionPath = paths.get(0);
281         }
282         else {
283           // no paths are shown for merge commits, so we're using the saved path we're inspecting now
284           revisionPath = currentPath.get();
285         }
286
287         Couple<String> authorPair = Couple.of(record.getAuthorName(), record.getAuthorEmail());
288         Couple<String> committerPair = Couple.of(record.getCommitterName(), record.getCommitterEmail());
289         Collection<String> parents = Arrays.asList(parentHashes);
290         consumer.consume(new GitFileRevision(project, finalRoot, revisionPath, revision, Couple.of(authorPair, committerPair), message,
291                                              null, new Date(record.getAuthorTimeStamp()), parents));
292         List<GitLogStatusInfo> statusInfos = record.getStatusInfos();
293         if (statusInfos.isEmpty()) {
294           // can safely be empty, for example, for simple merge commits that don't change anything.
295           return;
296         }
297         if (statusInfos.get(0).getType() == GitChangeType.ADDED && !filePath.isDirectory()) {
298           skipFurtherOutput.set(true);
299         }
300       }
301       catch (VcsException e) {
302         exceptionConsumer.consume(e);
303       }
304     };
305
306     GitVcs vcs = GitVcs.getInstance(project);
307     GitVersion version = vcs != null ? vcs.getVersion() : GitVersion.NULL;
308     final AtomicBoolean criticalFailure = new AtomicBoolean();
309     while (currentPath.get() != null && firstCommitParent.get() != null) {
310       logHandler.set(getLogHandler(project, version, finalRoot, logParser, currentPath.get(), firstCommitParent.get(), parameters));
311       final MyTokenAccumulator accumulator = new MyTokenAccumulator(logParser);
312       final Semaphore semaphore = new Semaphore();
313
314       logHandler.get().addLineListener(new GitLineHandlerAdapter() {
315         @Override
316         public void onLineAvailable(String line, Key outputType) {
317           final GitLogRecord record = accumulator.acceptLine(line);
318           if (record != null) {
319             resultAdapter.consume(record);
320           }
321         }
322
323         @Override
324         public void startFailed(Throwable exception) {
325           //noinspection ThrowableInstanceNeverThrown
326           try {
327             exceptionConsumer.consume(new VcsException(exception));
328           }
329           finally {
330             criticalFailure.set(true);
331             semaphore.up();
332           }
333         }
334
335         @Override
336         public void processTerminated(int exitCode) {
337           try {
338             super.processTerminated(exitCode);
339             final GitLogRecord record = accumulator.processLast();
340             if (record != null) {
341               resultAdapter.consume(record);
342             }
343           }
344           catch (Throwable t) {
345             LOG.error(t);
346             exceptionConsumer.consume(new VcsException("Internal error " + t.getMessage(), t));
347             criticalFailure.set(true);
348           }
349           finally {
350             semaphore.up();
351           }
352         }
353       });
354       semaphore.down();
355       logHandler.get().start();
356       semaphore.waitFor();
357       if (criticalFailure.get()) {
358         return;
359       }
360
361       try {
362         Pair<String, FilePath> firstCommitParentAndPath = getFirstCommitParentAndPathIfRename(project, finalRoot, firstCommit.get(),
363                                                                                               currentPath.get(), version);
364         currentPath.set(firstCommitParentAndPath == null ? null : firstCommitParentAndPath.second);
365         firstCommitParent.set(firstCommitParentAndPath == null ? null : firstCommitParentAndPath.first);
366         skipFurtherOutput.set(false);
367       }
368       catch (VcsException e) {
369         LOG.warn("Tried to get first commit rename path", e);
370         exceptionConsumer.consume(e);
371         return;
372       }
373     }
374   }
375
376   @NotNull
377   private static GitLineHandler getLogHandler(@NotNull Project project,
378                                               @NotNull GitVersion version,
379                                               @NotNull VirtualFile root,
380                                               @NotNull GitLogParser parser,
381                                               @NotNull FilePath path,
382                                               @NotNull String lastCommit,
383                                               String... parameters) {
384     final GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
385     h.setStdoutSuppressed(true);
386     h.addParameters("--name-status", parser.getPretty(), "--encoding=UTF-8", lastCommit);
387     if (GitVersionSpecialty.FULL_HISTORY_SIMPLIFY_MERGES_WORKS_CORRECTLY.existsIn(version) && Registry.is("git.file.history.full")) {
388       h.addParameters("--full-history", "--simplify-merges");
389     }
390     if (parameters != null && parameters.length > 0) {
391       h.addParameters(parameters);
392     }
393     h.endOptions();
394     h.addRelativePaths(path);
395     return h;
396   }
397
398   /**
399    * Gets info of the given commit and checks if it was a RENAME.
400    * If yes, returns the older file path, which file was renamed from.
401    * If it's not a rename, returns null.
402    */
403   @Nullable
404   private static Pair<String, FilePath> getFirstCommitParentAndPathIfRename(@NotNull Project project,
405                                                                             @NotNull VirtualFile root,
406                                                                             @NotNull String commit,
407                                                                             @NotNull FilePath filePath,
408                                                                             @NotNull GitVersion version) throws VcsException {
409     // 'git show -M --name-status <commit hash>' returns the information about commit and detects renames.
410     // NB: we can't specify the filepath, because then rename detection will work only with the '--follow' option, which we don't wanna use.
411     final GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.SHOW);
412     final GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.STATUS, HASH, COMMIT_TIME, PARENTS);
413     h.setStdoutSuppressed(true);
414     h.addParameters("-M", "--name-status", parser.getPretty(), "--encoding=UTF-8", commit);
415     if (!GitVersionSpecialty.FOLLOW_IS_BUGGY_IN_THE_LOG.existsIn(version)) {
416       h.addParameters("--follow");
417       h.endOptions();
418       h.addRelativePaths(filePath);
419     }
420     else {
421       h.endOptions();
422     }
423     final String output = h.run();
424     final List<GitLogRecord> records = parser.parse(output);
425
426     if (records.isEmpty()) return null;
427     // we have information about all changed files of the commit. Extracting information about the file we need.
428     GitLogRecord record = records.get(0);
429     final List<Change> changes = record.parseChanges(project, root);
430     for (Change change : changes) {
431       if ((change.isMoved() || change.isRenamed()) && filePath.equals(notNull(change.getAfterRevision()).getFile())) {
432         final String[] parents = record.getParentsHashes();
433         String parent = parents.length > 0 ? parents[0] : null;
434         return Pair.create(parent, notNull(change.getBeforeRevision()).getFile());
435       }
436     }
437     return null;
438   }
439
440   @NotNull
441   public static List<? extends VcsShortCommitDetails> readMiniDetails(@NotNull Project project,
442                                                                       @NotNull VirtualFile root,
443                                                                       @NotNull List<String> hashes)
444     throws VcsException {
445     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
446     if (factory == null) {
447       return Collections.emptyList();
448     }
449
450     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.LOG);
451     GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.NONE, HASH, PARENTS, AUTHOR_NAME,
452                                            AUTHOR_EMAIL, COMMIT_TIME, SUBJECT, COMMITTER_NAME, COMMITTER_EMAIL, AUTHOR_TIME);
453     h.setSilent(true);
454     // git show can show either -p, or --name-status, or --name-only, but we need nothing, just details => using git log --no-walk
455     h.addParameters("--no-walk");
456     h.addParameters(parser.getPretty(), "--encoding=UTF-8");
457     h.addParameters(new ArrayList<>(hashes));
458     h.endOptions();
459
460     String output = h.run();
461     List<GitLogRecord> records = parser.parse(output);
462
463     return ContainerUtil.map(records, record -> {
464       List<Hash> parents = new SmartList<>();
465       for (String parent : record.getParentsHashes()) {
466         parents.add(HashImpl.build(parent));
467       }
468       return factory.createShortDetails(HashImpl.build(record.getHash()), parents, record.getCommitTime(), root,
469                                         record.getSubject(), record.getAuthorName(), record.getAuthorEmail(), record.getCommitterName(),
470                                         record.getCommitterEmail(),
471                                         record.getAuthorTimeStamp());
472     });
473   }
474
475   @Nullable
476   public static List<VcsCommitMetadata> readLastCommits(@NotNull Project project,
477                                                         @NotNull VirtualFile root,
478                                                         @NotNull String... refs)
479     throws VcsException {
480     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
481     if (factory == null) {
482       return null;
483     }
484
485     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.LOG);
486     GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.NONE, HASH, PARENTS, COMMIT_TIME, SUBJECT, AUTHOR_NAME,
487                                            AUTHOR_EMAIL, RAW_BODY, COMMITTER_NAME, COMMITTER_EMAIL, AUTHOR_TIME);
488
489     h.setSilent(true);
490     // git show can show either -p, or --name-status, or --name-only, but we need nothing, just details => using git log --no-walk
491     h.addParameters("--no-walk");
492     h.addParameters(parser.getPretty(), "--encoding=UTF-8");
493     h.addParameters(refs);
494     h.endOptions();
495
496     String output = h.run();
497     List<GitLogRecord> records = parser.parse(output);
498     if (records.size() != refs.length) return null;
499
500     return ContainerUtil.map(records,
501                              record -> factory.createCommitMetadata(factory.createHash(record.getHash()), getParentHashes(factory, record),
502                                                                     record.getCommitTime(),
503                                                                     root, record.getSubject(), record.getAuthorName(),
504                                                                     record.getAuthorEmail(),
505                                                                     record.getFullMessage(), record.getCommitterName(),
506                                                                     record.getCommitterEmail(),
507                                                                     record.getAuthorTimeStamp()));
508   }
509
510   private static void processHandlerOutputByLine(@NotNull GitLineHandler handler,
511                                                  @NotNull GitLogParser parser,
512                                                  @NotNull Consumer<GitLogRecord> recordConsumer) throws VcsException {
513     Ref<Throwable> parseError = new Ref<>();
514     processHandlerOutputByLine(handler, builder -> {
515       try {
516         GitLogRecord record = parser.parseOneRecord(builder.toString());
517         if (record != null) {
518           recordConsumer.consume(record);
519         }
520       }
521       catch (ProcessCanceledException pce) {
522         throw pce;
523       }
524       catch (Throwable t) {
525         if (parseError.isNull()) {
526           parseError.set(t);
527           LOG.error("Could not parse \" " + StringUtil.escapeStringCharacters(builder.toString()) + "\"\n" +
528                     "Command " + handler.printableCommandLine(), t);
529         }
530       }
531     }, 0);
532
533     if (!parseError.isNull()) {
534       throw new VcsException(parseError.get());
535     }
536   }
537
538   private static void processHandlerOutputByLine(@NotNull GitLineHandler handler,
539                                                  @NotNull Consumer<StringBuilder> recordConsumer,
540                                                  int bufferSize)
541     throws VcsException {
542     final StringBuilder output = new StringBuilder();
543     final StringBuilder errors = new StringBuilder();
544     final Ref<Boolean> foundRecordEnd = Ref.create(false);
545     final Ref<VcsException> ex = new Ref<>();
546     final AtomicInteger records = new AtomicInteger();
547     handler.addLineListener(new GitLineHandlerListener() {
548       @Override
549       public void onLineAvailable(String line, Key outputType) {
550         if (outputType == ProcessOutputTypes.STDERR) {
551           errors.append(line).append("\n");
552         }
553         else if (outputType == ProcessOutputTypes.STDOUT) {
554           try {
555             // format of the record is <RECORD_START>.*<RECORD_END>.*
556             // then next record goes
557             // (rather inconveniently, after RECORD_END there is a list of modified files)
558             // so here I'm trying to find text between two RECORD_START symbols
559             // that simultaneously contains a RECORD_END
560             // this helps to deal with commits like a929478f6720ac15d949117188cd6798b4a9c286 in linux repo that have RECORD_START symbols in the message
561             // wont help with RECORD_END symbols in the message however (have not seen those yet)
562
563             String tail = null;
564             if (!foundRecordEnd.get()) {
565               int recordEnd = line.indexOf(GitLogParser.RECORD_END);
566               if (recordEnd != -1) {
567                 foundRecordEnd.set(true);
568                 output.append(line.substring(0, recordEnd + 1));
569                 line = line.substring(recordEnd + 1);
570               }
571               else {
572                 output.append(line).append("\n");
573               }
574             }
575
576             if (foundRecordEnd.get()) {
577               int nextRecordStart = line.indexOf(GitLogParser.RECORD_START);
578               if (nextRecordStart == -1) {
579                 output.append(line).append("\n");
580               }
581               else if (nextRecordStart == 0) {
582                 tail = line + "\n";
583               }
584               else {
585                 output.append(line.substring(0, nextRecordStart));
586                 tail = line.substring(nextRecordStart) + "\n";
587               }
588             }
589
590             if (tail != null) {
591               if (records.incrementAndGet() > bufferSize) {
592                 recordConsumer.consume(output);
593                 output.setLength(0);
594               }
595               output.append(tail);
596               foundRecordEnd.set(tail.contains(GitLogParser.RECORD_END));
597             }
598           }
599           catch (Exception e) {
600             ex.set(new VcsException(e));
601           }
602         }
603       }
604
605       @Override
606       public void processTerminated(int exitCode) {
607         if (exitCode != 0) {
608           String errorMessage = errors.toString();
609           if (errorMessage.isEmpty()) {
610             errorMessage = GitBundle.message("git.error.exit", exitCode);
611           }
612           ex.set(new VcsException(errorMessage));
613         }
614         else {
615           try {
616             recordConsumer.consume(output);
617           }
618           catch (Exception e) {
619             ex.set(new VcsException(e));
620           }
621         }
622       }
623
624       @Override
625       public void startFailed(Throwable exception) {
626         ex.set(new VcsException(exception));
627       }
628     });
629     handler.runInCurrentThread(null);
630     if (!ex.isNull()) {
631       if (ex.get().getCause() instanceof ProcessCanceledException) {
632         throw (ProcessCanceledException)ex.get().getCause();
633       }
634       throw ex.get();
635     }
636   }
637
638   /*
639   Unlike loadDetails, which accepts list of hashes in parameters, loads details for all commits in the repository.
640   To optimize memory consumption, git log command output is parsed on-the-fly and resulting commits are immediately fed to the consumer
641   and not stored in memory.
642    */
643   public static void loadAllDetails(@NotNull Project project,
644                                     @NotNull VirtualFile root,
645                                     @NotNull Consumer<VcsFullCommitDetails> commitConsumer) throws VcsException {
646     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
647     if (factory == null) {
648       return;
649     }
650
651     GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
652     GitLogParser parser = createParserForDetails(h, project, false, true, ArrayUtil.toStringArray(LOG_ALL));
653
654     processHandlerOutputByLine(h, parser, record -> commitConsumer.consume(createCommit(project, root, record, factory)));
655   }
656
657   public static void readCommits(@NotNull Project project,
658                                  @NotNull VirtualFile root,
659                                  @NotNull List<String> parameters,
660                                  @NotNull Consumer<VcsUser> userConsumer,
661                                  @NotNull Consumer<VcsRef> refConsumer,
662                                  @NotNull Consumer<TimedVcsCommit> commitConsumer) throws VcsException {
663     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
664     if (factory == null) {
665       return;
666     }
667
668     GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
669     final GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.NONE, HASH, PARENTS, COMMIT_TIME,
670                                                  AUTHOR_NAME, AUTHOR_EMAIL, REF_NAMES);
671     h.setStdoutSuppressed(true);
672     h.addParameters(parser.getPretty(), "--encoding=UTF-8");
673     h.addParameters("--decorate=full");
674     h.addParameters(parameters);
675     h.endOptions();
676
677     final int COMMIT_BUFFER = 1000;
678     processHandlerOutputByLine(h, buffer -> {
679       List<TimedVcsCommit> commits = parseCommit(parser, buffer, userConsumer, refConsumer, factory, root);
680       for (TimedVcsCommit commit : commits) {
681         commitConsumer.consume(commit);
682       }
683     }, COMMIT_BUFFER);
684   }
685
686   @NotNull
687   private static List<TimedVcsCommit> parseCommit(@NotNull GitLogParser parser,
688                                                   @NotNull StringBuilder record,
689                                                   @NotNull Consumer<VcsUser> userRegistry,
690                                                   @NotNull Consumer<VcsRef> refConsumer,
691                                                   @NotNull VcsLogObjectsFactory factory,
692                                                   @NotNull VirtualFile root) {
693     List<GitLogRecord> gitLogRecords = parser.parse(record.toString());
694     return ContainerUtil.mapNotNull(gitLogRecords, gitLogRecord -> {
695       if (gitLogRecord == null) {
696         return null;
697       }
698       Pair<TimedVcsCommit, Collection<VcsRef>> pair = convert(gitLogRecord, factory, root);
699       TimedVcsCommit commit = pair.first;
700       for (VcsRef ref : pair.second) {
701         refConsumer.consume(ref);
702       }
703       userRegistry.consume(factory.createUser(gitLogRecord.getAuthorName(), gitLogRecord.getAuthorEmail()));
704       return commit;
705     });
706   }
707
708   @NotNull
709   private static Pair<TimedVcsCommit, Collection<VcsRef>> convert(@NotNull GitLogRecord rec,
710                                                                   @NotNull VcsLogObjectsFactory factory,
711                                                                   @NotNull VirtualFile root) {
712     Hash hash = HashImpl.build(rec.getHash());
713     List<Hash> parents = getParentHashes(factory, rec);
714     TimedVcsCommit commit = factory.createTimedCommit(hash, parents, rec.getCommitTime());
715     return Pair.create(commit, parseRefs(rec.getRefs(), hash, factory, root));
716   }
717
718   @NotNull
719   private static Collection<VcsRef> parseRefs(@NotNull Collection<String> refs,
720                                               @NotNull Hash hash,
721                                               @NotNull VcsLogObjectsFactory factory,
722                                               @NotNull VirtualFile root) {
723     return ContainerUtil.mapNotNull(refs, refName -> {
724       VcsRefType type = GitRefManager.getRefType(refName);
725       refName = GitBranchUtil.stripRefsPrefix(refName);
726       return refName.equals(GitUtil.ORIGIN_HEAD) ? null : factory.createRef(hash, refName, type, root);
727     });
728   }
729
730   @Nullable
731   private static VcsLogObjectsFactory getObjectsFactoryWithDisposeCheck(@NotNull Project project) {
732     return ApplicationManager.getApplication().runReadAction((Computable<VcsLogObjectsFactory>)() -> {
733       if (!project.isDisposed()) {
734         return ServiceManager.getService(project, VcsLogObjectsFactory.class);
735       }
736       return null;
737     });
738   }
739
740   private static class MyTokenAccumulator {
741     @NotNull private final StringBuilder myBuffer = new StringBuilder();
742     @NotNull private final GitLogParser myParser;
743
744     private boolean myNotStarted = true;
745
746     public MyTokenAccumulator(@NotNull GitLogParser parser) {
747       myParser = parser;
748     }
749
750     @Nullable
751     public GitLogRecord acceptLine(String s) {
752       final boolean recordStart = s.startsWith(GitLogParser.RECORD_START);
753       if (recordStart) {
754         s = s.substring(GitLogParser.RECORD_START.length());
755       }
756
757       if (myNotStarted) {
758         myBuffer.append(s);
759         myBuffer.append("\n");
760
761         myNotStarted = false;
762         return null;
763       }
764       else if (recordStart) {
765         final String line = myBuffer.toString();
766         myBuffer.setLength(0);
767
768         myBuffer.append(s);
769         myBuffer.append("\n");
770
771         return processResult(line);
772       }
773       else {
774         myBuffer.append(s);
775         myBuffer.append("\n");
776         return null;
777       }
778     }
779
780     @Nullable
781     public GitLogRecord processLast() {
782       return processResult(myBuffer.toString());
783     }
784
785     @Nullable
786     private GitLogRecord processResult(@NotNull String line) {
787       return myParser.parseOneRecord(line);
788     }
789   }
790
791   /**
792    * Get history for the file
793    *
794    * @param project the context project
795    * @param path    the file path
796    * @return the list of the revisions
797    * @throws VcsException if there is problem with running git
798    */
799   @NotNull
800   public static List<VcsFileRevision> history(@NotNull Project project, @NotNull FilePath path, String... parameters) throws VcsException {
801     final VirtualFile root = GitUtil.getGitRoot(path);
802     return history(project, path, root, parameters);
803   }
804
805   @NotNull
806   public static List<VcsFileRevision> history(@NotNull Project project,
807                                               @NotNull FilePath path,
808                                               @Nullable VirtualFile root,
809                                               String... parameters) throws VcsException {
810     return history(project, path, root, GitRevisionNumber.HEAD, parameters);
811   }
812
813   @NotNull
814   public static List<VcsFileRevision> history(@NotNull Project project,
815                                               @NotNull FilePath path,
816                                               @Nullable VirtualFile root,
817                                               @NotNull VcsRevisionNumber startingFrom,
818                                               String... parameters) throws VcsException {
819     final List<VcsFileRevision> rc = new ArrayList<>();
820     final List<VcsException> exceptions = new ArrayList<>();
821
822     history(project, path, root, startingFrom, rc::add, exceptions::add, parameters);
823     if (!exceptions.isEmpty()) {
824       throw exceptions.get(0);
825     }
826     return rc;
827   }
828
829   /**
830    * @deprecated To remove in IDEA 17
831    */
832   @Deprecated
833   @SuppressWarnings("unused")
834   @NotNull
835   public static List<Pair<SHAHash, Date>> onlyHashesHistory(@NotNull Project project, @NotNull FilePath path, String... parameters)
836     throws VcsException {
837     final VirtualFile root = GitUtil.getGitRoot(path);
838     return onlyHashesHistory(project, path, root, parameters);
839   }
840
841   /**
842    * @deprecated To remove in IDEA 17
843    */
844   @Deprecated
845   @NotNull
846   public static List<Pair<SHAHash, Date>> onlyHashesHistory(@NotNull Project project,
847                                                             @NotNull FilePath path,
848                                                             @NotNull VirtualFile root,
849                                                             String... parameters)
850     throws VcsException {
851     // adjust path using change manager
852     path = getLastCommitName(project, path);
853     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.LOG);
854     GitLogParser parser = new GitLogParser(project, HASH, COMMIT_TIME);
855     h.setStdoutSuppressed(true);
856     h.addParameters(parameters);
857     h.addParameters(parser.getPretty(), "--encoding=UTF-8");
858     h.endOptions();
859     h.addRelativePaths(path);
860     String output = h.run();
861
862     final List<Pair<SHAHash, Date>> rc = new ArrayList<>();
863     for (GitLogRecord record : parser.parse(output)) {
864       record.setUsedHandler(h);
865       rc.add(Pair.create(new SHAHash(record.getHash()), record.getDate()));
866     }
867     return rc;
868   }
869
870   @NotNull
871   public static VcsLogProvider.DetailedLogData loadMetadata(@NotNull final Project project,
872                                                             @NotNull final VirtualFile root,
873                                                             String... params) throws VcsException {
874     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
875     if (factory == null) {
876       return LogDataImpl.empty();
877     }
878     final Set<VcsRef> refs = new OpenTHashSet<>(GitLogProvider.DONT_CONSIDER_SHA);
879     final List<VcsCommitMetadata> commits =
880       loadDetails(project, root, true, false, record -> {
881         GitCommit commit = createCommit(project, root, record, factory);
882         Collection<VcsRef> refsInRecord = parseRefs(record.getRefs(), commit.getId(), factory, root);
883         for (VcsRef ref : refsInRecord) {
884           if (!refs.add(ref)) {
885             VcsRef otherRef = ContainerUtil.find(refs, r -> GitLogProvider.DONT_CONSIDER_SHA.equals(r, ref));
886             LOG.error("Adding duplicate element " + ref + " to the set containing " + otherRef);
887           }
888         }
889         return commit;
890       }, params);
891     return new LogDataImpl(refs, commits);
892   }
893
894   /**
895    * <p>Get & parse git log detailed output with commits, their parents and their changes.</p>
896    * <p>
897    * <p>Warning: this is method is efficient by speed, but don't query too much, because the whole log output is retrieved at once,
898    * and it can occupy too much memory. The estimate is ~600Kb for 1000 commits.</p>
899    */
900   @NotNull
901   public static List<GitCommit> history(@NotNull Project project, @NotNull VirtualFile root, String... parameters)
902     throws VcsException {
903     final VcsLogObjectsFactory factory = getObjectsFactoryWithDisposeCheck(project);
904     if (factory == null) {
905       return Collections.emptyList();
906     }
907     return loadDetails(project, root, false, true, record -> createCommit(project, root, record, factory), parameters);
908   }
909
910   @NotNull
911   private static GitLogParser createParserForDetails(@NotNull GitTextHandler h,
912                                                      @NotNull Project project,
913                                                      boolean withRefs,
914                                                      boolean withChanges,
915                                                      String... parameters) {
916     GitLogParser.NameStatus status = withChanges ? GitLogParser.NameStatus.STATUS : GitLogParser.NameStatus.NONE;
917     GitLogParser.GitLogOption[] options = {HASH, COMMIT_TIME, AUTHOR_NAME, AUTHOR_TIME, AUTHOR_EMAIL, COMMITTER_NAME, COMMITTER_EMAIL,
918       PARENTS, SUBJECT, BODY, RAW_BODY};
919     if (withRefs) {
920       options = ArrayUtil.append(options, REF_NAMES);
921     }
922     GitLogParser parser = new GitLogParser(project, status, options);
923     h.setStdoutSuppressed(true);
924     h.addParameters(parameters);
925     h.addParameters(parser.getPretty(), "--encoding=UTF-8");
926     if (withRefs) {
927       h.addParameters("--decorate=full");
928     }
929     if (withChanges) {
930       h.addParameters("-M", /*find and report renames*/
931                       "--name-status",
932                       "-c" /*single diff for merge commits, only showing files that were modified from both parents*/);
933     }
934     h.endOptions();
935
936     return parser;
937   }
938
939   @NotNull
940   public static <T> List<T> loadDetails(@NotNull final Project project,
941                                         @NotNull final VirtualFile root,
942                                         boolean withRefs,
943                                         boolean withChanges,
944                                         @NotNull NullableFunction<GitLogRecord, T> converter,
945                                         String... parameters)
946     throws VcsException {
947
948     GitLineHandler h = new GitLineHandler(project, root, GitCommand.LOG);
949     GitLogParser parser = createParserForDetails(h, project, withRefs, withChanges, parameters);
950
951     List<T> commits = ContainerUtil.newArrayList();
952
953     StopWatch sw = StopWatch.start("loading details");
954
955     processHandlerOutputByLine(h, parser, record -> commits.add(converter.fun(record)));
956
957     sw.report();
958
959     return commits;
960   }
961
962   @NotNull
963   private static GitCommit createCommit(@NotNull Project project, @NotNull VirtualFile root, @NotNull GitLogRecord record,
964                                         @NotNull VcsLogObjectsFactory factory) {
965     List<Hash> parents = getParentHashes(factory, record);
966     return new GitCommit(project, HashImpl.build(record.getHash()), parents, record.getCommitTime(), root, record.getSubject(),
967                          factory.createUser(record.getAuthorName(), record.getAuthorEmail()), record.getFullMessage(),
968                          factory.createUser(record.getCommitterName(), record.getCommitterEmail()), record.getAuthorTimeStamp(),
969                          record.getStatusInfos());
970   }
971
972   @NotNull
973   private static List<Hash> getParentHashes(@NotNull VcsLogObjectsFactory factory, @NotNull GitLogRecord record) {
974     return ContainerUtil.map(record.getParentsHashes(), factory::createHash);
975   }
976
977   @NotNull
978   private static GitHeavyCommit createCommit(@NotNull Project project, @Nullable SymbolicRefsI refs, @NotNull VirtualFile root,
979                                              @NotNull GitLogRecord record) throws VcsException {
980     final Collection<String> currentRefs = record.getRefs();
981     List<String> locals = new ArrayList<>();
982     List<String> remotes = new ArrayList<>();
983     List<String> tags = new ArrayList<>();
984     final String s = parseRefs(refs, currentRefs, locals, remotes, tags);
985
986     GitHeavyCommit
987       gitCommit = new GitHeavyCommit(root, AbstractHash.create(record.getHash()), new SHAHash(record.getHash()), record.getAuthorName(),
988                                      record.getCommitterName(),
989                                      record.getDate(), record.getSubject(), record.getFullMessage(),
990                                      new HashSet<>(Arrays.asList(record.getParentsHashes())), record.getFilePaths(root),
991                                      record.getAuthorEmail(),
992                                      record.getCommitterEmail(), tags, locals, remotes,
993                                      record.parseChanges(project, root), record.getAuthorTimeStamp());
994     gitCommit.setCurrentBranch(s);
995     return gitCommit;
996   }
997
998   @Nullable
999   private static String parseRefs(@Nullable SymbolicRefsI refs, @NotNull Collection<String> currentRefs, @NotNull List<String> locals,
1000                                   @NotNull List<String> remotes, @NotNull List<String> tags) {
1001     if (refs == null) {
1002       return null;
1003     }
1004     for (String ref : currentRefs) {
1005       final SymbolicRefs.Kind kind = refs.getKind(ref);
1006       if (SymbolicRefs.Kind.LOCAL.equals(kind)) {
1007         locals.add(ref);
1008       }
1009       else if (SymbolicRefs.Kind.REMOTE.equals(kind)) {
1010         remotes.add(ref);
1011       }
1012       else {
1013         tags.add(ref);
1014       }
1015     }
1016     if (refs.getCurrent() != null && currentRefs.contains(refs.getCurrent().getName())) {
1017       return refs.getCurrent().getName();
1018     }
1019     return null;
1020   }
1021
1022   @Deprecated
1023   @NotNull
1024   public static List<GitHeavyCommit> commitsDetails(@NotNull Project project, @NotNull FilePath path, @Nullable SymbolicRefsI refs,
1025                                                     @NotNull Collection<String> commitsIds) throws VcsException {
1026     path = getLastCommitName(project, path);     // adjust path using change manager
1027     VirtualFile root = GitUtil.getGitRoot(path);
1028     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.SHOW);
1029     GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.STATUS,
1030                                            HASH, HASH, COMMIT_TIME, AUTHOR_NAME, AUTHOR_TIME, AUTHOR_EMAIL, COMMITTER_NAME,
1031                                            COMMITTER_EMAIL, PARENTS, REF_NAMES, SUBJECT, BODY, RAW_BODY);
1032     h.setSilent(true);
1033     h.addParameters("--name-status", "-M", parser.getPretty(), "--encoding=UTF-8");
1034     h.addParameters(new ArrayList<>(commitsIds));
1035
1036     String output = h.run();
1037     final List<GitHeavyCommit> rc = new ArrayList<>();
1038     for (GitLogRecord record : parser.parse(output)) {
1039       final GitHeavyCommit gitCommit = createCommit(project, refs, root, record);
1040       rc.add(gitCommit);
1041     }
1042     return rc;
1043   }
1044
1045   public static long getAuthorTime(@NotNull Project project, @NotNull FilePath path, @NotNull String commitsId) throws VcsException {
1046     // adjust path using change manager
1047     path = getLastCommitName(project, path);
1048     final VirtualFile root = GitUtil.getGitRoot(path);
1049     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.SHOW);
1050     GitLogParser parser = new GitLogParser(project, GitLogParser.NameStatus.STATUS, AUTHOR_TIME);
1051     h.setSilent(true);
1052     h.addParameters("--name-status", parser.getPretty(), "--encoding=UTF-8");
1053     h.addParameters(commitsId);
1054
1055     String output = h.run();
1056     GitLogRecord logRecord = parser.parseOneRecord(output);
1057     if (logRecord == null) throw new VcsException("Can not parse log output \"" + output + "\"");
1058     return logRecord.getAuthorTimeStamp();
1059   }
1060
1061   /**
1062    * Get name of the file in the last commit. If file was renamed, returns the previous name.
1063    *
1064    * @param project the context project
1065    * @param path    the path to check
1066    * @return the name of file in the last commit or argument
1067    */
1068   public static FilePath getLastCommitName(@NotNull Project project, FilePath path) {
1069     if (project.isDefault()) return path;
1070     final ChangeListManager changeManager = ChangeListManager.getInstance(project);
1071     final Change change = changeManager.getChange(path);
1072     if (change != null && change.getType() == Change.Type.MOVED) {
1073       // GitContentRevision r = (GitContentRevision)change.getBeforeRevision();
1074       assert change.getBeforeRevision() != null : "Move change always have beforeRevision";
1075       path = change.getBeforeRevision().getFile();
1076     }
1077     return path;
1078   }
1079
1080   @Nullable
1081   public static GitRevisionNumber getMergeBase(@NotNull Project project, @NotNull VirtualFile root, @NotNull String first,
1082                                                @NotNull String second)
1083     throws VcsException {
1084     GitSimpleHandler h = new GitSimpleHandler(project, root, GitCommand.MERGE_BASE);
1085     h.setSilent(true);
1086     h.addParameters(first, second);
1087     String output = h.run().trim();
1088     if (output.length() == 0) {
1089       return null;
1090     }
1091     else {
1092       return GitRevisionNumber.resolve(project, root, output);
1093     }
1094   }
1095 }